From 8b33ee76e6573a5306e2813b1dd9390dbaee8df2 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Thu, 23 Apr 2026 14:16:49 -0400 Subject: [PATCH 1/6] Find tests that are skipped in every configuration --- .github/workflows/ci.yml | 96 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 292263f0c4..cd1a9fd78c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -416,6 +416,9 @@ jobs: name: Check job status if: always() runs-on: ubuntu-latest + permissions: + actions: read + contents: read needs: - should-skip - detect-changes @@ -426,6 +429,99 @@ jobs: - test-windows - doc steps: + - name: Report universally-skipped tests + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + + REPO="${GITHUB_REPOSITORY}" + SHA="${GITHUB_SHA}" + WORKDIR=$(mktemp -d) + trap 'rm -rf "${WORKDIR}"' EXIT + + # Fetch all workflow runs associated with this commit (up to 100) + all_runs=$(gh api \ + "repos/${REPO}/actions/runs?head_sha=${SHA}&per_page=100" \ + --jq '.workflow_runs[]') + + # Identify child runs by their workflow file path; test-wheel-linux.yml + # covers both linux-64 and linux-aarch64 (two separate runs). + TEST_WORKFLOW_PATHS=( + ".github/workflows/test-sdist-linux.yml" + ".github/workflows/test-sdist-windows.yml" + ".github/workflows/test-wheel-linux.yml" + ".github/workflows/test-wheel-windows.yml" + ) + + mkdir -p "${WORKDIR}/configs" + config_index=0 + + for wf_path in "${TEST_WORKFLOW_PATHS[@]}"; do + run_ids=$(echo "${all_runs}" | jq -r \ + --arg p "${wf_path}" \ + 'select(.path == $p) | .id') + for run_id in ${run_ids}; do + run_name=$(echo "${all_runs}" | jq -r \ + --argjson id "${run_id}" 'select(.id == $id) | .name' | head -1) + echo "Fetching logs for: ${run_name} (id=${run_id})" + config_dir="${WORKDIR}/configs/${config_index}" + mkdir -p "${config_dir}" + + if gh api "repos/${REPO}/actions/runs/${run_id}/logs" \ + > "${config_dir}/logs.zip" 2>/dev/null; then + unzip -q "${config_dir}/logs.zip" -d "${config_dir}/logs" 2>/dev/null || true + # Extract test IDs from lines matching the pytest SKIPPED output format + grep -rh ' SKIPPED' "${config_dir}/logs/" 2>/dev/null \ + | grep -oE '[^[:space:]]+\.py::[^[:space:]]+ SKIPPED' \ + | sed 's/ SKIPPED$//' \ + | sort -u > "${config_dir}/skipped.txt" || true + echo " -> $(wc -l < "${config_dir}/skipped.txt") skipped tests" + config_index=$((config_index + 1)) + else + echo " -> could not download logs (run may be skipped or unavailable)" >&2 + rm -rf "${config_dir}" + fi + done + done + + { + echo "## Universally-skipped tests" + echo "" + } >> "${GITHUB_STEP_SUMMARY}" + + if [[ ${config_index} -eq 0 ]]; then + echo "_No test run logs found._" >> "${GITHUB_STEP_SUMMARY}" + exit 0 + fi + + # Start the intersection from the first config and progressively + # narrow it down to only tests that appear in every config's list. + cp "${WORKDIR}/configs/0/skipped.txt" "${WORKDIR}/intersection.txt" + for i in $(seq 1 $((config_index - 1))); do + comm -12 \ + "${WORKDIR}/intersection.txt" \ + "${WORKDIR}/configs/${i}/skipped.txt" \ + > "${WORKDIR}/intersection_new.txt" + mv "${WORKDIR}/intersection_new.txt" "${WORKDIR}/intersection.txt" + done + + count=$(wc -l < "${WORKDIR}/intersection.txt") + { + echo "Tests skipped across all ${config_index} test configuration(s):" + echo "" + if [[ ${count} -eq 0 ]]; then + echo "_No tests were skipped in all configurations._" + else + echo "| Test |" + echo "| --- |" + while IFS= read -r test; do + [[ -z "${test}" ]] && continue + echo "| \`${test}\` |" + done < "${WORKDIR}/intersection.txt" + fi + } >> "${GITHUB_STEP_SUMMARY}" + - name: Exit run: | # if any dependencies were cancelled or failed, that's a failure From 8e4bd69f86b9540a0f4bee8862d98dc4f83e0dfc Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Thu, 23 Apr 2026 16:01:35 -0400 Subject: [PATCH 2/6] Second try --- .github/workflows/ci.yml | 121 +++++++++++++++++++++++---------------- 1 file changed, 72 insertions(+), 49 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd1a9fd78c..19fe7d259c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -436,53 +436,59 @@ jobs: set -euo pipefail REPO="${GITHUB_REPOSITORY}" - SHA="${GITHUB_SHA}" WORKDIR=$(mktemp -d) trap 'rm -rf "${WORKDIR}"' EXIT - # Fetch all workflow runs associated with this commit (up to 100) - all_runs=$(gh api \ - "repos/${REPO}/actions/runs?head_sha=${SHA}&per_page=100" \ - --jq '.workflow_runs[]') - - # Identify child runs by their workflow file path; test-wheel-linux.yml - # covers both linux-64 and linux-aarch64 (two separate runs). - TEST_WORKFLOW_PATHS=( - ".github/workflows/test-sdist-linux.yml" - ".github/workflows/test-sdist-windows.yml" - ".github/workflows/test-wheel-linux.yml" - ".github/workflows/test-wheel-windows.yml" + mkdir -p "${WORKDIR}/configs" + + # Collect all jobs for this run and group matrix jobs into the five + # top-level test configurations from ci.yml. + jobs_json=$(gh api --paginate \ + "repos/${REPO}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=100" \ + | jq -s '[.[].jobs[]]') + + declare -A CONFIG_PATTERNS=( + [test-sdist-linux]='^Test sdist linux-64 / ' + [test-sdist-windows]='^Test sdist win-64 / ' + [test-linux-64]='^Test linux-64 / ' + [test-linux-aarch64]='^Test linux-aarch64 / ' + [test-windows]='^Test (win-64|windows) / ' ) - mkdir -p "${WORKDIR}/configs" - config_index=0 - - for wf_path in "${TEST_WORKFLOW_PATHS[@]}"; do - run_ids=$(echo "${all_runs}" | jq -r \ - --arg p "${wf_path}" \ - 'select(.path == $p) | .id') - for run_id in ${run_ids}; do - run_name=$(echo "${all_runs}" | jq -r \ - --argjson id "${run_id}" 'select(.id == $id) | .name' | head -1) - echo "Fetching logs for: ${run_name} (id=${run_id})" - config_dir="${WORKDIR}/configs/${config_index}" - mkdir -p "${config_dir}" - - if gh api "repos/${REPO}/actions/runs/${run_id}/logs" \ - > "${config_dir}/logs.zip" 2>/dev/null; then - unzip -q "${config_dir}/logs.zip" -d "${config_dir}/logs" 2>/dev/null || true - # Extract test IDs from lines matching the pytest SKIPPED output format - grep -rh ' SKIPPED' "${config_dir}/logs/" 2>/dev/null \ - | grep -oE '[^[:space:]]+\.py::[^[:space:]]+ SKIPPED' \ - | sed 's/ SKIPPED$//' \ - | sort -u > "${config_dir}/skipped.txt" || true - echo " -> $(wc -l < "${config_dir}/skipped.txt") skipped tests" - config_index=$((config_index + 1)) - else - echo " -> could not download logs (run may be skipped or unavailable)" >&2 - rm -rf "${config_dir}" - fi + configs=( + test-sdist-linux + test-sdist-windows + test-linux-64 + test-linux-aarch64 + test-windows + ) + + for cfg in "${configs[@]}"; do + cfg_dir="${WORKDIR}/configs/${cfg}" + mkdir -p "${cfg_dir}/logs" + job_ids=$(echo "${jobs_json}" | jq -r \ + --arg pat "${CONFIG_PATTERNS[$cfg]}" \ + '.[] | select(.name | test($pat)) | .id') + + if [[ -z "${job_ids}" ]]; then + echo "No matrix jobs found for ${cfg}" >&2 + : > "${cfg_dir}/skipped.txt" + continue + fi + + for job_id in ${job_ids}; do + logfile="${cfg_dir}/logs/${job_id}.log" + # gh run view handles log retrieval for a job ID reliably. + gh run view "${GITHUB_RUN_ID}" --job "${job_id}" --log > "${logfile}" || true done + + # Extract test IDs from lines matching pytest SKIPPED output. + grep -h ' SKIPPED' "${cfg_dir}/logs/"*.log 2>/dev/null \ + | grep -oE '[^[:space:]]+\.py::[^[:space:]]+ SKIPPED' \ + | sed 's/ SKIPPED$//' \ + | sort -u > "${cfg_dir}/skipped.txt" || true + + echo "${cfg}: $(wc -l < "${cfg_dir}/skipped.txt") skipped tests" done { @@ -490,25 +496,42 @@ jobs: echo "" } >> "${GITHUB_STEP_SUMMARY}" - if [[ ${config_index} -eq 0 ]]; then - echo "_No test run logs found._" >> "${GITHUB_STEP_SUMMARY}" + available_configs=0 + missing_configs=() + for cfg in "${configs[@]}"; do + if [[ -s "${WORKDIR}/configs/${cfg}/skipped.txt" || -n "$(ls -A "${WORKDIR}/configs/${cfg}/logs" 2>/dev/null || true)" ]]; then + available_configs=$((available_configs + 1)) + else + missing_configs+=("${cfg}") + fi + done + + if [[ ${available_configs} -eq 0 ]]; then + echo "_No test job logs found in this run._" >> "${GITHUB_STEP_SUMMARY}" exit 0 fi - # Start the intersection from the first config and progressively - # narrow it down to only tests that appear in every config's list. - cp "${WORKDIR}/configs/0/skipped.txt" "${WORKDIR}/intersection.txt" - for i in $(seq 1 $((config_index - 1))); do + if [[ ${#missing_configs[@]} -gt 0 ]]; then + { + echo "_Warning: missing logs for configuration(s): ${missing_configs[*]}_" + echo "" + } >> "${GITHUB_STEP_SUMMARY}" + fi + + # Start intersection from first config, then narrow with each of the + # remaining four configurations. + cp "${WORKDIR}/configs/${configs[0]}/skipped.txt" "${WORKDIR}/intersection.txt" + for cfg in "${configs[@]:1}"; do comm -12 \ "${WORKDIR}/intersection.txt" \ - "${WORKDIR}/configs/${i}/skipped.txt" \ + "${WORKDIR}/configs/${cfg}/skipped.txt" \ > "${WORKDIR}/intersection_new.txt" mv "${WORKDIR}/intersection_new.txt" "${WORKDIR}/intersection.txt" done count=$(wc -l < "${WORKDIR}/intersection.txt") { - echo "Tests skipped across all ${config_index} test configuration(s):" + echo "Tests skipped across all five test configurations:" echo "" if [[ ${count} -eq 0 ]]; then echo "_No tests were skipped in all configurations._" From 942725cbec88f03c0aeaca390f9e5db5889e23ba Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Thu, 23 Apr 2026 17:02:59 -0400 Subject: [PATCH 3/6] Add a test that always skips to test this --- cuda_bindings/tests/test_cuda.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cuda_bindings/tests/test_cuda.py b/cuda_bindings/tests/test_cuda.py index e3eefb1fdd..0ea86a82c8 100644 --- a/cuda_bindings/tests/test_cuda.py +++ b/cuda_bindings/tests/test_cuda.py @@ -40,6 +40,11 @@ def callableBinary(name): return shutil.which(name) is not None +@pytest.mark.skipif(True, "Always skip!") +def test_always_skip(): + pass + + def test_cuda_memcpy(): # Get device From ebcbb8a1739c6d8f8532335a12750167baa87525 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 24 Apr 2026 10:34:31 -0400 Subject: [PATCH 4/6] Add reason --- cuda_bindings/tests/test_cuda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuda_bindings/tests/test_cuda.py b/cuda_bindings/tests/test_cuda.py index 0ea86a82c8..86b359ce04 100644 --- a/cuda_bindings/tests/test_cuda.py +++ b/cuda_bindings/tests/test_cuda.py @@ -40,7 +40,7 @@ def callableBinary(name): return shutil.which(name) is not None -@pytest.mark.skipif(True, "Always skip!") +@pytest.mark.skipif(True, reason="Always skip!") def test_always_skip(): pass From 63ca53a83f222f5e4d6b881f3f1997189b10d518 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 24 Apr 2026 15:04:57 -0400 Subject: [PATCH 5/6] Hopefully working now --- .github/workflows/ci.yml | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19fe7d259c..d5f8024097 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -441,23 +441,19 @@ jobs: mkdir -p "${WORKDIR}/configs" - # Collect all jobs for this run and group matrix jobs into the five - # top-level test configurations from ci.yml. + # Collect all jobs for this run and group matrix jobs into wheel + # test configurations from ci.yml. jobs_json=$(gh api --paginate \ "repos/${REPO}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=100" \ | jq -s '[.[].jobs[]]') declare -A CONFIG_PATTERNS=( - [test-sdist-linux]='^Test sdist linux-64 / ' - [test-sdist-windows]='^Test sdist win-64 / ' [test-linux-64]='^Test linux-64 / ' [test-linux-aarch64]='^Test linux-aarch64 / ' [test-windows]='^Test (win-64|windows) / ' ) configs=( - test-sdist-linux - test-sdist-windows test-linux-64 test-linux-aarch64 test-windows @@ -478,14 +474,18 @@ jobs: for job_id in ${job_ids}; do logfile="${cfg_dir}/logs/${job_id}.log" - # gh run view handles log retrieval for a job ID reliably. - gh run view "${GITHUB_RUN_ID}" --job "${job_id}" --log > "${logfile}" || true + # Prefer the job log API; fall back to gh run view if needed. + if ! gh api "repos/${REPO}/actions/jobs/${job_id}/logs" > "${logfile}" 2>/dev/null; then + gh run view "${GITHUB_RUN_ID}" --job "${job_id}" --log > "${logfile}" || true + fi done - # Extract test IDs from lines matching pytest SKIPPED output. - grep -h ' SKIPPED' "${cfg_dir}/logs/"*.log 2>/dev/null \ - | grep -oE '[^[:space:]]+\.py::[^[:space:]]+ SKIPPED' \ - | sed 's/ SKIPPED$//' \ + # Extract pytest node IDs from SKIPPED lines. This tolerates + # timestamped logs, ANSI escapes, and trailing skip reasons. + grep -h 'SKIPPED' "${cfg_dir}/logs/"*.log 2>/dev/null \ + | sed -E 's/\x1B\[[0-9;]*[[:alpha:]]//g' \ + | sed 's#\\#/#g' \ + | sed -nE 's#.*(tests/[[:graph:]]+\.py::[^[:space:]]+).*SKIPPED.*#\1#p' \ | sort -u > "${cfg_dir}/skipped.txt" || true echo "${cfg}: $(wc -l < "${cfg_dir}/skipped.txt") skipped tests" @@ -518,8 +518,7 @@ jobs: } >> "${GITHUB_STEP_SUMMARY}" fi - # Start intersection from first config, then narrow with each of the - # remaining four configurations. + # Start intersection from the first wheel configuration, then narrow. cp "${WORKDIR}/configs/${configs[0]}/skipped.txt" "${WORKDIR}/intersection.txt" for cfg in "${configs[@]:1}"; do comm -12 \ @@ -531,7 +530,7 @@ jobs: count=$(wc -l < "${WORKDIR}/intersection.txt") { - echo "Tests skipped across all five test configurations:" + echo "Tests skipped across wheel test configurations (${#configs[@]}):" echo "" if [[ ${count} -eq 0 ]]; then echo "_No tests were skipped in all configurations._" From 01ebb66c23db2009391f16f87f66ac128e8bc036 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Mon, 27 Apr 2026 18:06:15 -0400 Subject: [PATCH 6/6] Find universally skipped tests across wheel test configurations --- .github/workflows/ci.yml | 111 +------- ci/tools/report_universally_skipped_tests.py | 285 +++++++++++++++++++ 2 files changed, 286 insertions(+), 110 deletions(-) create mode 100644 ci/tools/report_universally_skipped_tests.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5f8024097..d209ce9e83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -433,116 +433,7 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | - set -euo pipefail - - REPO="${GITHUB_REPOSITORY}" - WORKDIR=$(mktemp -d) - trap 'rm -rf "${WORKDIR}"' EXIT - - mkdir -p "${WORKDIR}/configs" - - # Collect all jobs for this run and group matrix jobs into wheel - # test configurations from ci.yml. - jobs_json=$(gh api --paginate \ - "repos/${REPO}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=100" \ - | jq -s '[.[].jobs[]]') - - declare -A CONFIG_PATTERNS=( - [test-linux-64]='^Test linux-64 / ' - [test-linux-aarch64]='^Test linux-aarch64 / ' - [test-windows]='^Test (win-64|windows) / ' - ) - - configs=( - test-linux-64 - test-linux-aarch64 - test-windows - ) - - for cfg in "${configs[@]}"; do - cfg_dir="${WORKDIR}/configs/${cfg}" - mkdir -p "${cfg_dir}/logs" - job_ids=$(echo "${jobs_json}" | jq -r \ - --arg pat "${CONFIG_PATTERNS[$cfg]}" \ - '.[] | select(.name | test($pat)) | .id') - - if [[ -z "${job_ids}" ]]; then - echo "No matrix jobs found for ${cfg}" >&2 - : > "${cfg_dir}/skipped.txt" - continue - fi - - for job_id in ${job_ids}; do - logfile="${cfg_dir}/logs/${job_id}.log" - # Prefer the job log API; fall back to gh run view if needed. - if ! gh api "repos/${REPO}/actions/jobs/${job_id}/logs" > "${logfile}" 2>/dev/null; then - gh run view "${GITHUB_RUN_ID}" --job "${job_id}" --log > "${logfile}" || true - fi - done - - # Extract pytest node IDs from SKIPPED lines. This tolerates - # timestamped logs, ANSI escapes, and trailing skip reasons. - grep -h 'SKIPPED' "${cfg_dir}/logs/"*.log 2>/dev/null \ - | sed -E 's/\x1B\[[0-9;]*[[:alpha:]]//g' \ - | sed 's#\\#/#g' \ - | sed -nE 's#.*(tests/[[:graph:]]+\.py::[^[:space:]]+).*SKIPPED.*#\1#p' \ - | sort -u > "${cfg_dir}/skipped.txt" || true - - echo "${cfg}: $(wc -l < "${cfg_dir}/skipped.txt") skipped tests" - done - - { - echo "## Universally-skipped tests" - echo "" - } >> "${GITHUB_STEP_SUMMARY}" - - available_configs=0 - missing_configs=() - for cfg in "${configs[@]}"; do - if [[ -s "${WORKDIR}/configs/${cfg}/skipped.txt" || -n "$(ls -A "${WORKDIR}/configs/${cfg}/logs" 2>/dev/null || true)" ]]; then - available_configs=$((available_configs + 1)) - else - missing_configs+=("${cfg}") - fi - done - - if [[ ${available_configs} -eq 0 ]]; then - echo "_No test job logs found in this run._" >> "${GITHUB_STEP_SUMMARY}" - exit 0 - fi - - if [[ ${#missing_configs[@]} -gt 0 ]]; then - { - echo "_Warning: missing logs for configuration(s): ${missing_configs[*]}_" - echo "" - } >> "${GITHUB_STEP_SUMMARY}" - fi - - # Start intersection from the first wheel configuration, then narrow. - cp "${WORKDIR}/configs/${configs[0]}/skipped.txt" "${WORKDIR}/intersection.txt" - for cfg in "${configs[@]:1}"; do - comm -12 \ - "${WORKDIR}/intersection.txt" \ - "${WORKDIR}/configs/${cfg}/skipped.txt" \ - > "${WORKDIR}/intersection_new.txt" - mv "${WORKDIR}/intersection_new.txt" "${WORKDIR}/intersection.txt" - done - - count=$(wc -l < "${WORKDIR}/intersection.txt") - { - echo "Tests skipped across wheel test configurations (${#configs[@]}):" - echo "" - if [[ ${count} -eq 0 ]]; then - echo "_No tests were skipped in all configurations._" - else - echo "| Test |" - echo "| --- |" - while IFS= read -r test; do - [[ -z "${test}" ]] && continue - echo "| \`${test}\` |" - done < "${WORKDIR}/intersection.txt" - fi - } >> "${GITHUB_STEP_SUMMARY}" + python ci/tools/report_universally_skipped_tests.py - name: Exit run: | diff --git a/ci/tools/report_universally_skipped_tests.py b/ci/tools/report_universally_skipped_tests.py new file mode 100644 index 0000000000..5163fd6d94 --- /dev/null +++ b/ci/tools/report_universally_skipped_tests.py @@ -0,0 +1,285 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Report tests skipped across all wheel test configurations. + +The script can run in GitHub Actions (using GITHUB_REPOSITORY/GITHUB_RUN_ID +and GITHUB_STEP_SUMMARY) or locally by passing --repo and --run-id. +""" + +from __future__ import annotations + +import argparse +import contextlib +import dataclasses +import json +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Iterable + +CONFIG_PATTERNS = { + "test-linux-64": r"^Test linux-64 / ", + "test-linux-aarch64": r"^Test linux-aarch64 / ", + "test-windows": r"^Test (win-64|windows) / ", +} + +ANSI_ESCAPE = re.compile(r"\x1B\[[0-9;]*[A-Za-z]") +PYTEST_NODE_ID = re.compile(r"tests/\S+\.py::\S+") +PYTEST_TEST_OUTCOME = re.compile(r"(tests/\S+\.py::\S+)\s+(PASSED|FAILED|ERROR|SKIPPED|XFAIL|XPASS)\b") + + +@dataclasses.dataclass(frozen=True) +class ConfigResult: + name: str + job_ids: list[int] + skipped: set[str] + has_logs: bool + + +@dataclasses.dataclass(frozen=True) +class ConfigLogs: + name: str + job_ids: list[int] + log_paths: list[Path] + + +def run_gh(*args: str, check: bool = True) -> subprocess.CompletedProcess[str]: + gh_exe = shutil.which("gh") + if not gh_exe: + raise RuntimeError("Could not find 'gh' executable in PATH") + + return subprocess.run( # noqa: S603 + [gh_exe, *args], + capture_output=True, + text=True, + check=check, + ) + + +def load_run_jobs(repo: str, run_id: str) -> list[dict]: + # --jq emits one compact JSON object per line across all pages. + result = run_gh( + "api", + "--paginate", + f"repos/{repo}/actions/runs/{run_id}/jobs?per_page=100", + "--jq", + ".jobs[] | @json", + ) + jobs = [] + for line in result.stdout.splitlines(): + text = line.strip() + if not text: + continue + jobs.append(json.loads(text)) + return jobs + + +def download_job_log(repo: str, run_id: str, job_id: int, out_path: Path) -> bool: + out_path.parent.mkdir(parents=True, exist_ok=True) + + api_result = run_gh("api", f"repos/{repo}/actions/jobs/{job_id}/logs", check=False) + if api_result.returncode == 0: + out_path.write_text(api_result.stdout, encoding="utf-8", errors="replace") + return True + + view_result = run_gh("run", "view", run_id, "--job", str(job_id), "--log", check=False) + if view_result.returncode == 0: + out_path.write_text(view_result.stdout, encoding="utf-8", errors="replace") + return True + + return False + + +def extract_test_status_sets(text: str) -> tuple[set[str], set[str]]: + skipped: set[str] = set() + non_skipped: set[str] = set() + + for raw_line in text.splitlines(): + line = ANSI_ESCAPE.sub("", raw_line).replace("\\", "/") + + # Parse per-test outcomes first so PASS/FAIL lines disqualify tests. + for test_id, outcome in PYTEST_TEST_OUTCOME.findall(line): + if outcome == "SKIPPED": + skipped.add(test_id) + else: + non_skipped.add(test_id) + + if "SKIPPED" not in line: + continue + + # Keep compatibility with summary-style SKIPPED lines that may still + # include a node id but don't match the strict outcome pattern above. + for test_id in PYTEST_NODE_ID.findall(line): + skipped.add(test_id) + + return skipped, non_skipped + + +def match_job_ids(jobs: Iterable[dict], pattern: str) -> list[int]: + regex = re.compile(pattern) + return [int(job["id"]) for job in jobs if regex.search(str(job.get("name", "")))] + + +def discover_config_logs(logs_root: Path) -> list[ConfigLogs]: + configs: list[ConfigLogs] = [] + + for config in CONFIG_PATTERNS: + config_dir = logs_root / config + log_paths = sorted(config_dir.glob("*.log")) if config_dir.exists() else [] + job_ids: list[int] = [] + for log_path in log_paths: + with contextlib.suppress(ValueError): + job_ids.append(int(log_path.stem)) + configs.append(ConfigLogs(name=config, job_ids=job_ids, log_paths=log_paths)) + + return configs + + +def download_config_logs(jobs: list[dict], repo: str, run_id: str, logs_root: Path) -> list[ConfigLogs]: + configs: list[ConfigLogs] = [] + + for config, pattern in CONFIG_PATTERNS.items(): + config_dir = logs_root / config + job_ids = match_job_ids(jobs, pattern) + log_paths: list[Path] = [] + + for job_id in job_ids: + log_path = config_dir / f"{job_id}.log" + if not log_path.exists() and not download_job_log(repo, run_id, job_id, log_path): + continue + log_paths.append(log_path) + + configs.append(ConfigLogs(name=config, job_ids=job_ids, log_paths=log_paths)) + + return configs + + +def analyze_config_logs(config_logs: list[ConfigLogs]) -> list[ConfigResult]: + results: list[ConfigResult] = [] + + for config in config_logs: + skipped_any: set[str] = set() + non_skipped_any: set[str] = set() + for log_path in config.log_paths: + text = log_path.read_text(encoding="utf-8", errors="replace") + + skipped_in_log, non_skipped_in_log = extract_test_status_sets(text) + skipped_any.update(skipped_in_log) + non_skipped_any.update(non_skipped_in_log) + + # For sharded matrices, a test may only appear in one log. Treat it as + # config-skipped if it is skipped at least once and never non-skipped + # (passed/failed/error/xpass/xfail) in that config. + skipped_for_config = skipped_any - non_skipped_any + + results.append( + ConfigResult( + name=config.name, + job_ids=config.job_ids, + skipped=skipped_for_config, + has_logs=bool(config.log_paths), + ) + ) + + return results + + +def build_summary(results: list[ConfigResult]) -> str: + lines = ["## Universally-skipped tests", ""] + + available = [r for r in results if r.job_ids or r.has_logs] + missing = [r.name for r in results if not (r.job_ids or r.has_logs)] + + if not available: + lines.append("_No test job logs found in this run._") + return "\n".join(lines) + "\n" + + if missing: + lines.append(f"_Warning: missing logs for configuration(s): {' '.join(missing)}_") + lines.append("") + + intersection: set[str] | None = None + for result in results: + if intersection is None: + intersection = set(result.skipped) + continue + intersection &= result.skipped + + if intersection is None or "tests/test_cuda.py::test_always_skip" not in intersection: + lines.append( + "_Note: the test `tests/test_cuda.py::test_always_skip` is expected to be skipped in all configurations, but is missing._" + ) + + universal = sorted(intersection or set()) + lines.append(f"Tests skipped across wheel test configurations ({len(results)}):") + lines.append("") + if not universal: + lines.append("_No tests were skipped in all configurations._") + else: + lines.append("| Test |") + lines.append("| --- |") + for test in universal: + lines.append(f"| `{test}` |") + + return "\n".join(lines) + "\n" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", default=os.environ.get("GITHUB_REPOSITORY"), help="owner/repo") + parser.add_argument("--run-id", default=os.environ.get("GITHUB_RUN_ID"), help="GitHub Actions run id") + parser.add_argument( + "--summary-path", + default=os.environ.get("GITHUB_STEP_SUMMARY"), + help="Path to write markdown summary (stdout if omitted)", + ) + parser.add_argument( + "--logs-dir", + default=None, + help="Directory to store downloaded logs (defaults to temporary CI-style dir)", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + logs_root = Path(args.logs_dir) if args.logs_dir else Path(".tmp-universally-skipped-logs") + + if args.logs_dir and logs_root.exists(): + if not logs_root.is_dir(): + print(f"--logs-dir path exists but is not a directory: {logs_root}", file=sys.stderr) + return 2 + print(f"Using existing logs in {logs_root}; skipping log downloads") + config_logs = discover_config_logs(logs_root) + else: + if not args.repo or not args.run_id: + print("--repo and --run-id are required (or set GITHUB_REPOSITORY/GITHUB_RUN_ID)", file=sys.stderr) + return 2 + + logs_root.mkdir(parents=True, exist_ok=True) + jobs = load_run_jobs(args.repo, str(args.run_id)) + config_logs = download_config_logs(jobs=jobs, repo=args.repo, run_id=str(args.run_id), logs_root=logs_root) + + results = analyze_config_logs(config_logs) + + for result in results: + print(f"{result.name}: {len(result.skipped)} skipped tests") + + summary = build_summary(results) + if args.summary_path: + Path(args.summary_path).write_text(summary, encoding="utf-8") + else: + print(summary) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())