diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..ffffcbc --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,83 @@ +name: Hands Tests + +# PyAutoHands's own unit suite — the executor's self-test. Until this existed, +# PyAutoHands PRs carried ZERO check runs: `python_matrix.yml` is a weekly cron +# over the five *libraries'* suites, `navigator_check.yml` is `workflow_call` +# only (invoked by the workspaces), and `release.yml`'s pytest step runs inside +# `${{ matrix.project.path }}` — a matrix of the five libraries, with +# PyAutoHands checked out beside them only as a helper. So the ~300 tests +# covering build_util (script/notebook execution, per-script timeouts, +# clean-skip exits), env_config (profile discovery, per-script env, JAX +# marking, workspace precedence), result_collector, check_navigator, +# clone_seed, the release/Slack notes and the workflow parsers ran in no CI at +# all, and a Hands PR's only gate was whatever the authoring session ran +# locally. +# +# That gap had already bitten: b038fdc promoted Python 3.14 to a required +# matrix leg and retired the experimental_python_314 job, but left +# test_python_matrix_workflow.py asserting the pre-promotion contract. The +# guard test sat failing with nothing to report it. +# +# Deliberately ONLY pytest. It must not invoke `autohands generate` / `run_all` +# / `pre_build` against live workspaces, and must not reach the network — those +# need sibling workspace checkouts and belong to release.yml and the scheduled +# drivers, not to a PR gate. What runs here is stdlib plus four small packages, +# so it stays fast (~6s) and flake-free. +# +# Dependencies are named explicitly rather than `-r requirements.txt`, which +# would drag in jupyterlab + ipykernel and turn a 6s gate into a slow one: +# pytest the runner +# PyYAML env_config / validate_env_profiles / the workflow parsers +# ipynb-py-convert a CLI binary build_util shells out to (not an import) +# Pillow generate_markdown's PNG optimisation path +# FOUR tests self-skip here, and they are two different kinds: +# - 3 on absent nbformat/jupyter (test_run_notebook_cwd, +# test_notebook_skip_exit) — intentional, that is the notebook-execution +# path release.yml covers. +# - 1 because this checkout has no sibling workspaces: +# test_workspace_config_precedence.test_actual_workspace_files_exist walks +# `repo_root.parent / ` asserting each of the six workspaces owns +# its config/build/{no_run,profile_smoke,visualise_notebooks}.yaml, and +# skips at the first one absent. In a full local workspace it asserts all +# six; here it asserts nothing. +# That second one is a real hole in this gate, accepted deliberately: closing it +# means six extra checkouts for a repo-layout invariant, which would couple a 6s +# gate to six other repos. It is a local/developer check, not a PR check. Local +# baseline is therefore 301 passed / 3 skipped; CI is 300 passed / 4 skipped. + +# One run per commit: PR events carry the CI; pushes only build main. +# Superseded runs are cancelled on PR refs only — a cancelled main run would +# read as red CI (cancelled is in Heart's FAILURE_CONCLUSIONS). +on: + push: + branches: [main] + pull_request: + +concurrency: + group: hands-tests-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +permissions: + contents: read + +jobs: + pytest: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # Tracks the required set python_matrix.yml declares supported (3.14 + # promoted in b038fdc). test_python_matrix_workflow.py asserts the two + # stay equal, so this list cannot silently fall behind that policy. + python-version: ["3.12", "3.13", "3.14"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + - name: Install (the whole dependency set — see header) + run: pip install pytest PyYAML ipynb-py-convert Pillow + - name: Run tests + run: pytest tests/ -q diff --git a/tests/test_python_matrix_workflow.py b/tests/test_python_matrix_workflow.py index 8b02e47..9025f7a 100644 --- a/tests/test_python_matrix_workflow.py +++ b/tests/test_python_matrix_workflow.py @@ -6,6 +6,9 @@ WORKFLOW = ( Path(__file__).resolve().parents[1] / ".github" / "workflows" / "python_matrix.yml" ) +SELF_TEST_WORKFLOW = ( + Path(__file__).resolve().parents[1] / ".github" / "workflows" / "tests.yml" +) AUTOHANDS = Path(__file__).resolve().parents[1] / "bin" / "autohands" @@ -13,37 +16,55 @@ def load_workflow(): return yaml.safe_load(WORKFLOW.read_text()) +def test_self_test_gate_tracks_the_supported_python_set(): + """Hands's own gate (tests.yml) must run the same Python set python_matrix.yml + declares required. + + Drift between a version policy and the file guarding it is exactly how this + module went stale: b038fdc promoted 3.14 in python_matrix.yml and nothing + reported that the guard still asserted the old shape. Tying the two lists + together means promoting or dropping a version has to touch both. + """ + required = load_workflow()["jobs"]["unit_tests"]["strategy"]["matrix"][ + "python-version" + ] + gate = yaml.safe_load(SELF_TEST_WORKFLOW.read_text())["jobs"]["pytest"][ + "strategy" + ]["matrix"]["python-version"] + + assert gate == required + + def test_required_matrices_cover_only_supported_python_versions(): jobs = load_workflow()["jobs"] assert jobs["unit_tests"]["strategy"]["matrix"]["python-version"] == [ "3.12", "3.13", + "3.14", ] assert jobs["smoke_tests"]["strategy"]["matrix"]["python-version"] == [ "3.12", "3.13", + "3.14", ] -def test_python_314_is_isolated_and_non_required(): +def test_python_314_is_a_required_leg_not_an_isolated_experiment(): + """3.14 was promoted to a required leg of both matrices (b038fdc, following + the PyAutoFit#1439 forkserver fix), which retired the soft + `experimental_python_314` job. Guard the promoted shape: 3.14 must sit in + the required matrices above, and must not quietly regrow a + `continue-on-error` home where its failures stop counting.""" jobs = load_workflow()["jobs"] - experimental = jobs["experimental_python_314"] - - assert experimental["continue-on-error"] is True - assert experimental["strategy"]["matrix"]["python-version"] == ["3.14"] - assert len(experimental["strategy"]["matrix"]["project"]) == 5 - assert "experimental_python_314" in jobs["summary"]["needs"] - assert "continue-on-error" not in jobs["unit_tests"] - assert "continue-on-error" not in jobs["smoke_tests"] - - record_step = next( - step for step in experimental["steps"] - if step.get("name") == "Record experimental cell result" - ) - assert record_step["if"] == "always()" - assert "job.status" in record_step["run"] - assert "does not cover workspace scripts" in record_step["run"] + + assert "experimental_python_314" not in jobs + + for name in ("unit_tests", "smoke_tests"): + assert "3.14" in jobs[name]["strategy"]["matrix"]["python-version"] + assert "continue-on-error" not in jobs[name] + + assert sorted(jobs["summary"]["needs"]) == ["smoke_tests", "unit_tests"] def test_no_below_floor_success_or_banner_contract_remains():