From de393f9a7d5300b9b29e9b4355a7f388d9130b22 Mon Sep 17 00:00:00 2001 From: runpod-Henrik Date: Tue, 25 Aug 2026 11:05:47 -0700 Subject: [PATCH 1/4] chore: upload coverage.xml artifact from CI The test job already runs `make test-coverage` on every push, but the result only ever went to the terminal. Emit Cobertura XML and upload it so the weekly coverage report can read the number from the newest successful run on main instead of it being collected by hand. `if-no-files-found: error` so a silently-missing report fails the step rather than publishing an empty artifact. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 11 +++++++++++ Makefile | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 30975f8..e314da7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,17 @@ jobs: - name: Run all tests with coverage run: make test-coverage + # Consumed by the weekly coverage report, which reads the artifact from + # the newest successful run on main. Name is per-matrix-version because + # upload-artifact@v4 requires unique names within a run. + - name: Upload coverage report + uses: actions/upload-artifact@v4 + if: always() + with: + name: coverage-${{ matrix.python-version }} + path: coverage.xml + if-no-files-found: error + lint: runs-on: ubuntu-latest if: github.event_name != 'pull_request' || github.head_ref != 'release-please--branches--main' diff --git a/Makefile b/Makefile index 250536c..ab693e8 100644 --- a/Makefile +++ b/Makefile @@ -291,7 +291,7 @@ test-integration: # Run integration tests only uv run pytest tests/integration/ -v -m integration test-coverage: # Run tests with coverage report (parallel) - uv run pytest tests/ -v -n auto --dist loadscope --cov=handler --cov=remote_execution --cov-report=term-missing + uv run pytest tests/ -v -n auto --dist loadscope --cov=handler --cov=remote_execution --cov-report=term-missing --cov-report=xml test-fast: # Run tests with fast-fail mode uv run pytest tests/ -v -x --tb=short From 7336f1e5026832bf485de00fb46d106534819adb Mon Sep 17 00:00:00 2001 From: runpod-Henrik Date: Tue, 25 Aug 2026 15:56:34 -0700 Subject: [PATCH 2/4] chore: render coverage numbers into the job summary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The coverage report was only readable by downloading the artifact. Write a Test Results + Coverage Summary table to $GITHUB_STEP_SUMMARY so the numbers show up on the run page, matching what the ai-api component workflow does. Adds --junitxml so the run's test counts can be reported alongside coverage; pytest-results.xml is gitignored. Stdlib only, so there is no extra install step, and `if: always()` means the summary still renders when tests fail — which is when it is most useful. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 58 ++++++++++++++++++++++++++++++++++++++++ .gitignore | 1 + Makefile | 2 +- 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e314da7..1c4cdc0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,64 @@ jobs: - name: Run all tests with coverage run: make test-coverage + # Renders the numbers into the run's summary page so they are readable + # without downloading the artifact. Stdlib only — no extra install step. + - name: Coverage summary + if: always() + run: | + python - coverage.xml pytest-results.xml <<'PY' >> "$GITHUB_STEP_SUMMARY" + import glob, sys, xml.etree.ElementTree as ET + from pathlib import Path + + cov, junit = sys.argv[1], sys.argv[2:] + o = ["## Coverage Report", ""] + + t = f = s = 0 + seen = False + for pat in junit: + for path in sorted(glob.glob(pat)): + try: + r = ET.parse(path).getroot() + except ET.ParseError: + continue + seen = True + for su in (r.iter("testsuite") if r.tag == "testsuites" else [r]): + t += int(su.get("tests") or 0) + f += int(su.get("failures") or 0) + int(su.get("errors") or 0) + s += int(su.get("skipped") or 0) + if seen: + o += ["### Test Results", "", + "| Status | Passed | Failed | Skipped | Total |", + "|---|---:|---:|---:|---:|", + f"| {'✅ Passed' if not f else '❌ Failed'} | {t - f - s} | {f} | {s} | {t} |", ""] + + p = Path(cov) + if not p.exists(): + o += [f"> No coverage report at `{cov}`."] + else: + r = ET.parse(p).getroot() + c, v = int(r.get("lines-covered") or 0), int(r.get("lines-valid") or 0) + o += ["### Coverage Summary", "", "| Metric | Value |", "|---|---:|", + f"| Line coverage | **{(100.0 * c / v if v else 0):.1f}%** |", + f"| Lines covered | {c:,} / {v:,} |", + f"| Lines missing | {v - c:,} |", ""] + rows = [] + for cl in r.iter("class"): + ls = list(cl.iter("line")) + if ls: + h = sum(1 for x in ls if int(x.get("hits") or 0) > 0) + rows.append((100.0 * h / len(ls), + cl.get("filename") or cl.get("name") or "?", h, len(ls))) + rows.sort() + if rows: + o += ["
", + f"Per-file coverage ({len(rows)} files, least covered first)", + "", "| File | Coverage | Covered / Total |", "|---|---:|---:|"] + o += [f"| `{n}` | {q:.1f}% | {h} / {tt} |" for q, n, h, tt in rows] + o += ["", "
", ""] + print("\n".join(o)) + PY + # Consumed by the weekly coverage report, which reads the artifact from # the newest successful run on main. Name is per-matrix-version because # upload-artifact@v4 requires unique names within a run. diff --git a/.gitignore b/.gitignore index 684bef0..cea9d68 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ htmlcov/ .cache nosetests.xml coverage.xml +pytest-results.xml *.cover *.py,cover .hypothesis/ diff --git a/Makefile b/Makefile index ab693e8..2ac0f8c 100644 --- a/Makefile +++ b/Makefile @@ -291,7 +291,7 @@ test-integration: # Run integration tests only uv run pytest tests/integration/ -v -m integration test-coverage: # Run tests with coverage report (parallel) - uv run pytest tests/ -v -n auto --dist loadscope --cov=handler --cov=remote_execution --cov-report=term-missing --cov-report=xml + uv run pytest tests/ -v -n auto --dist loadscope --cov=handler --cov=remote_execution --cov-report=term-missing --cov-report=xml --junitxml=pytest-results.xml test-fast: # Run tests with fast-fail mode uv run pytest tests/ -v -x --tb=short From 37944e484dfae87e4064906819eda3efa92b35c1 Mon Sep 17 00:00:00 2001 From: runpod-Henrik Date: Wed, 26 Aug 2026 08:28:25 -0700 Subject: [PATCH 3/4] fix: harden the coverage summary step against bad input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback from #373. Three defects, all reachable because this step runs under `if: always()`: * the coverage-XML parse was unguarded while the junit parse beside it was. A report truncated by a timeout, OOM or crashed xdist worker made ET.parse raise, so the summary step exited non-zero and stacked a spurious failure on top of the real one. Verified: the old script exits 1 on a truncated report, the new one exits 0 and says so in the summary. * hits/lines attributes were parsed with bare int(), so a malformed value raised rather than degrading. * the status cell treated `0 failures` as passing even when no tests ran at all. A suite dying at import reports errors>0 with tests possibly 0, so the check is now `no failures AND at least one test`. Also switch the artifact upload to `if-no-files-found: warn`. With `error` a run that never produced coverage.xml — pytest erroring at collection, before pytest-cov writes anything — failed the upload step too, red-flagging the job and masking the root cause. This upload hangs off the PR-gating test job, so it should not be able to fail a PR on its own. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 60 +++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c4cdc0..b3c6680 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,51 +56,79 @@ jobs: import glob, sys, xml.etree.ElementTree as ET from pathlib import Path + MAX_ROWS = 300 cov, junit = sys.argv[1], sys.argv[2:] o = ["## Coverage Report", ""] + + def num(v, default=0): + """Attribute values are text, and a malformed one must degrade rather than + raise: this step runs under `if: always()`, so an exception here would + stack a spurious failure on top of whatever actually went wrong.""" + try: + return int(v) + except (TypeError, ValueError): + return default + + + def parse(path): + try: + return ET.parse(path).getroot(), None + except (ET.ParseError, OSError) as e: + return None, e + + t = f = s = 0 seen = False for pat in junit: for path in sorted(glob.glob(pat)): - try: - r = ET.parse(path).getroot() - except ET.ParseError: + r, _ = parse(path) + if r is None: continue seen = True for su in (r.iter("testsuite") if r.tag == "testsuites" else [r]): - t += int(su.get("tests") or 0) - f += int(su.get("failures") or 0) + int(su.get("errors") or 0) - s += int(su.get("skipped") or 0) + t += num(su.get("tests")) + f += num(su.get("failures")) + num(su.get("errors")) + s += num(su.get("skipped")) if seen: + # A suite dying at import reports errors>0 with tests possibly 0, so + # "no failures AND something actually ran" is what keeps a crashed run + # from rendering as passed. + ok = f == 0 and t > 0 o += ["### Test Results", "", "| Status | Passed | Failed | Skipped | Total |", "|---|---:|---:|---:|---:|", - f"| {'✅ Passed' if not f else '❌ Failed'} | {t - f - s} | {f} | {s} | {t} |", ""] + f"| {'✅ Passed' if ok else '❌ Failed'} | {t - f - s} | {f} | {s} | {t} |", ""] p = Path(cov) - if not p.exists(): - o += [f"> No coverage report at `{cov}`."] + root, err = (None, None) if not p.exists() else parse(p) + if root is None: + o += [f"> No coverage report at `{cov}`." if err is None + else f"> Coverage report at `{cov}` could not be parsed: {err}"] else: - r = ET.parse(p).getroot() - c, v = int(r.get("lines-covered") or 0), int(r.get("lines-valid") or 0) + c, v = num(root.get("lines-covered")), num(root.get("lines-valid")) o += ["### Coverage Summary", "", "| Metric | Value |", "|---|---:|", f"| Line coverage | **{(100.0 * c / v if v else 0):.1f}%** |", f"| Lines covered | {c:,} / {v:,} |", f"| Lines missing | {v - c:,} |", ""] rows = [] - for cl in r.iter("class"): + for cl in root.iter("class"): ls = list(cl.iter("line")) if ls: - h = sum(1 for x in ls if int(x.get("hits") or 0) > 0) + h = sum(1 for x in ls if num(x.get("hits")) > 0) rows.append((100.0 * h / len(ls), cl.get("filename") or cl.get("name") or "?", h, len(ls))) - rows.sort() + rows.sort(key=lambda r: (r[0], r[1])) if rows: + shown = rows[:MAX_ROWS] o += ["
", f"Per-file coverage ({len(rows)} files, least covered first)", "", "| File | Coverage | Covered / Total |", "|---|---:|---:|"] - o += [f"| `{n}` | {q:.1f}% | {h} / {tt} |" for q, n, h, tt in rows] + o += [f"| `{n}` | {q:.1f}% | {h} / {tt} |" for q, n, h, tt in shown] + # Capped so a growing repo cannot push the summary past GitHub's 1 MiB + # limit, which would truncate it silently. + if len(rows) > MAX_ROWS: + o += [f"| _…and {len(rows) - MAX_ROWS} more files_ | | |"] o += ["", "
", ""] print("\n".join(o)) PY @@ -114,7 +142,7 @@ jobs: with: name: coverage-${{ matrix.python-version }} path: coverage.xml - if-no-files-found: error + if-no-files-found: warn lint: runs-on: ubuntu-latest From fb230788c8c6a70a184656201443ac3baec841e2 Mon Sep 17 00:00:00 2001 From: runpod-Henrik Date: Wed, 26 Aug 2026 12:56:25 -0700 Subject: [PATCH 4/4] feat: collect branch coverage Branch coverage was never collected, so the job summary could only ever show line coverage. Adds --cov-branch. Purely additive: line coverage is unchanged (verified per repo), so the weekly trend, which reads line coverage, is unaffected. The Cobertura report now carries branches-valid/covered, which the summary renders as its own row. Note --cov-fail-under gates on coverage.py total, which now blends lines and branches, so that number drops even though line coverage does not. Measured before committing: this repo stays comfortably above its gate. Co-Authored-By: Claude Opus 5 (1M context) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2ac0f8c..23d645b 100644 --- a/Makefile +++ b/Makefile @@ -291,7 +291,7 @@ test-integration: # Run integration tests only uv run pytest tests/integration/ -v -m integration test-coverage: # Run tests with coverage report (parallel) - uv run pytest tests/ -v -n auto --dist loadscope --cov=handler --cov=remote_execution --cov-report=term-missing --cov-report=xml --junitxml=pytest-results.xml + uv run pytest tests/ -v -n auto --dist loadscope --cov=handler --cov=remote_execution --cov-branch --cov-report=term-missing --cov-report=xml --junitxml=pytest-results.xml test-fast: # Run tests with fast-fail mode uv run pytest tests/ -v -x --tb=short