diff --git a/.github/workflows/publish-nuget.yml b/.github/workflows/publish-nuget.yml index bbf07b4..fc51250 100644 --- a/.github/workflows/publish-nuget.yml +++ b/.github/workflows/publish-nuget.yml @@ -66,8 +66,12 @@ jobs: - name: Recheck source identity, allowlist, contents and hashes before authentication env: PACKAGE_VERSION: ${{ needs.candidate.outputs.version }} + ARTIFACT_NAME: ${{ needs.candidate.outputs.artifact_name }} run: | - python eng/packaging/release_channels.py --expected-version "$PACKAGE_VERSION" + # Authorize the candidate allocated for this run (retained by a failed-jobs-only retry) and bind + # the downloaded manifest and artifact name to it; never re-derive a version from this attempt. + python eng/packaging/release_channels.py --expected-version "$PACKAGE_VERSION" \ + --manifest artifacts/packages/manifest.json --artifact-name "$ARTIFACT_NAME" python eng/packaging/packages.py verify --version "$PACKAGE_VERSION" --commit "$GITHUB_SHA" - name: Exchange GitHub OIDC identity for a short-lived NuGet API key id: login diff --git a/docs/dependency-policy.md b/docs/dependency-policy.md index d9a714d..f9261f2 100644 --- a/docs/dependency-policy.md +++ b/docs/dependency-policy.md @@ -19,7 +19,8 @@ Relevant runtime diagnostics remain local and conditional on existing tools; missing coverage is recorded honestly. No toolchain installation is implied. Main publishes exact `1.0.0-ci.RUN.ATTEMPT` candidates to nuget.org through the -existing trusted publisher. Deliberate canonical `vX.Y.Z` tags may publish `X.Y.Z` +existing trusted publisher; ATTEMPT is the attempt that allocated the candidate, which a +failed-jobs-only retry keeps and publication verifies instead of re-deriving. Deliberate canonical `vX.Y.Z` tags may publish `X.Y.Z` only when the tagged commit belongs to main history. The `nuget` environment must permit main and `v*` tags; the portable guard rejects noncanonical tags, other repositories/events and stable closures containing prerelease packages. diff --git a/eng/packaging/README.md b/eng/packaging/README.md index 2bcf5da..4951b9f 100644 --- a/eng/packaging/README.md +++ b/eng/packaging/README.md @@ -77,7 +77,12 @@ The workflow allocates `1.0.0-ci..` once befor `1.0.0-ci.3.1`, then `1.0.0-ci.4.1`. GitHub owns the counter; no version commit or tag is written back to the repository. The counter continues from earlier runs of this workflow, and failed runs can leave gaps. Re-running all jobs uses the new attempt suffix. Retrying only failed downstream jobs retains -the already allocated version and producer artifact. Main remains a prerelease stream. Stable tags +the already allocated version and producer artifact. Publication therefore authorizes that retained +candidate instead of deriving a version from its own attempt: the version must carry this run number +and an allocation attempt no later than the current attempt, and the downloaded manifest and artifact +name must name this commit, this run and a producing attempt between the allocation and the current +attempt. An artifact from another run, attempt or candidate is rejected; there is no latest-artifact +selection. Main remains a prerelease stream. Stable tags select the exact stable version before building and reject prerelease dependency closures. Never create a tag solely for verification. See [dependency admission](../../docs/dependency-policy.md). @@ -93,7 +98,9 @@ and must run through all gates. Local builds and PR candidates never upload to t Duplicate versions fail; there is deliberately no `--skip-duplicate`. NuGet cannot atomically publish ten packages. If upload partially succeeds, inspect the registry and retained manifest and re-run all jobs to allocate a new complete version; do not promote a partial release set or retry it blindly. -Retrying a diagnosed failed publication uses the retained candidate. A bad published version is superseded by a new +Retrying a diagnosed failed publication (re-run failed jobs) uses the retained candidate and its version; +re-running all jobs deliberately builds and publishes a new candidate. A re-run always executes the +workflow of the original commit, so a fix to publication tooling applies only to runs of later commits. A bad published version is superseded by a new version; consumers retain their prior exact version/lock until the upgrade is approved. ## Consume diff --git a/eng/packaging/release_channels.py b/eng/packaging/release_channels.py index 4287a26..ecb813d 100644 --- a/eng/packaging/release_channels.py +++ b/eng/packaging/release_channels.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only """Select and authorize the original main or deliberate stable-tag candidate.""" import argparse +import json import os from pathlib import Path import re @@ -12,9 +13,12 @@ ROOT = Path(__file__).resolve().parents[2] STABLE = r'(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)' +MAIN_CANDIDATE = re.compile(r'1\.0\.0-ci\.([1-9][0-9]*)\.([1-9][0-9]*)') def selected(env, ancestor): + """Allocate the candidate version once, before build (preflight): the current run and attempt on + main, or the canonical stable tag. Publication authorizes it with `authorized`, never re-selects.""" if env.get('GITHUB_REPOSITORY') != 'ArcForges/DesktopPlatform' or env.get('GITHUB_EVENT_NAME') != 'push': raise ValueError('Wrong publisher repository or event') commit = env.get('GITHUB_SHA', '') @@ -38,22 +42,76 @@ def verified_tag(ref, commit, resolve, ancestor): ancestor(commit) +def authorized(env, expected, ancestor): + """Authorize publishing the candidate that preflight allocated for this run. + + The version is allocated once, before build. A failed-jobs-only retry keeps that allocation and + its producer artifact while the publisher runs in a later attempt, so the candidate is validated + against this run rather than re-derived from the publisher's own attempt: a main candidate must + carry this run number and an allocation attempt no later than the current attempt. Re-running + all jobs allocates and builds a new candidate. Stable tags do not depend on the attempt.""" + current = selected(env, ancestor) + if env['GITHUB_REF'] != 'refs/heads/main': + if expected != current: + raise ValueError('Candidate version does not match publisher identity') + return expected, None + match = MAIN_CANDIDATE.fullmatch(expected or '') + if not match or match[1] != env['GITHUB_RUN_NUMBER'] or int(match[2]) > int(env['GITHUB_RUN_ATTEMPT']): + raise ValueError('Candidate version does not match publisher identity') + return expected, int(match[2]) + + +def bound_candidate(env, version, allocated, manifest, artifact_name): + """The downloaded artifact is the authorized candidate: this source, version and workflow run, + packed at or after the allocation attempt and no later than the current attempt, under the artifact + name of that producing attempt. An artifact from another run, attempt or candidate is rejected.""" + build = manifest.get('build') if isinstance(manifest.get('build'), dict) else {} + if manifest.get('version') != version or manifest.get('sourceCommit') != env['GITHUB_SHA']: + raise ValueError('Candidate manifest does not match the authorized version and source') + run_id, attempt = build.get('runId'), str(build.get('runAttempt') or '') + if run_id != env.get('GITHUB_RUN_ID') or not re.fullmatch('[1-9][0-9]*', attempt): + raise ValueError('Candidate was not produced by this workflow run') + produced = int(attempt) + if produced > int(env['GITHUB_RUN_ATTEMPT']) or (allocated is not None and produced < allocated): + raise ValueError('Candidate producer attempt is outside its allocation') + if artifact_name != f'nuget-candidate-{run_id}-{produced}': + raise ValueError('Candidate artifact name does not match its producer') + return produced + + +def publication(env, expected, manifest, artifact_name, ancestor): + version, allocated = authorized(env, expected, ancestor) + produced = bound_candidate(env, version, allocated, manifest, artifact_name) + return version, allocated, produced + + def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--expected-version') + parser.add_argument('--expected-version', help='authorize this allocated candidate for publication') + parser.add_argument('--manifest', type=Path, help='manifest.json of the downloaded candidate artifact') + parser.add_argument('--artifact-name', help='name of the downloaded candidate artifact') args = parser.parse_args() def ancestor(commit): verified_tag(os.environ['GITHUB_REF'], commit, lambda ref: subprocess.check_output(['git', 'rev-parse', '--verify', ref], cwd=ROOT, text=True).strip(), lambda sha: subprocess.run(['git', 'merge-base', '--is-ancestor', sha, 'origin/main'], cwd=ROOT, check=True)) - version = selected(os.environ, ancestor) - if args.expected_version and args.expected_version != version: - raise ValueError('Candidate version does not match publisher identity') + if args.expected_version is None: + version = selected(os.environ, ancestor) + message = f'Allocated immutable candidate {version}' + else: + if not (args.manifest and args.artifact_name): + raise ValueError('Publication requires the candidate --manifest and --artifact-name') + manifest = json.loads(args.manifest.read_text(encoding='utf-8')) + version, allocated, produced = publication(os.environ, args.expected_version, manifest, + args.artifact_name, ancestor) + message = (f'Authorized immutable candidate {version}' + + (f' (allocated in attempt {allocated}, packed in attempt {produced}, ' + f'publishing in attempt {os.environ["GITHUB_RUN_ATTEMPT"]})' if allocated else '')) audit(stable='-' not in version) if 'GITHUB_OUTPUT' in os.environ: with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as output: output.write(f'version={version}\n') - print(f'Authorized immutable candidate {version}') + print(message) if __name__ == '__main__': diff --git a/eng/test_dependency_policy.py b/eng/test_dependency_policy.py index f728a6b..253dc11 100644 --- a/eng/test_dependency_policy.py +++ b/eng/test_dependency_policy.py @@ -8,7 +8,7 @@ from dependency_policy import ROOT, POLICY, audit, check_admission, check_history, check_python, closure, exact, framework_upgrade, python_closure sys.path.insert(0, str(Path(__file__).resolve().parent / 'packaging')) -from release_channels import selected, verified_tag +from release_channels import authorized, bound_candidate, publication, selected, verified_tag class AdmissionTests(unittest.TestCase): @@ -106,6 +106,68 @@ def resolve(ref): with self.assertRaisesRegex(ValueError, 'tag target'): verified_tag('refs/tags/v1.2.3', 'b' * 40, resolve, seen.append) + PUBLISHER = {'GITHUB_REPOSITORY': 'ArcForges/DesktopPlatform', 'GITHUB_EVENT_NAME': 'push', + 'GITHUB_SHA': 'a' * 40, 'GITHUB_REF': 'refs/heads/main', 'GITHUB_RUN_ID': '9001', + 'GITHUB_RUN_NUMBER': '24', 'GITHUB_RUN_ATTEMPT': '1'} + + @staticmethod + def candidate(version, producer_attempt, run_id='9001'): + return {'version': version, 'sourceCommit': 'a' * 40, + 'build': {'runId': run_id, 'runAttempt': str(producer_attempt)}} + + def test_publication_keeps_the_allocated_candidate_across_supported_retries(self): + seen = [] + # Initial execution: preflight allocates in attempt 1 and the same attempt publishes it. + env = dict(self.PUBLISHER) + version = selected(env, seen.append) + self.assertEqual(version, '1.0.0-ci.24.1') + self.assertEqual(publication(env, version, self.candidate(version, 1), 'nuget-candidate-9001-1', seen.append), + ('1.0.0-ci.24.1', 1, 1)) + # Failed-jobs-only retry: preflight is not re-run, so attempt 2 publishes the retained allocation, + # whether the producer artifact is retained from attempt 1 or packing was re-run in attempt 2. + retry = {**env, 'GITHUB_RUN_ATTEMPT': '2'} + self.assertEqual(publication(retry, version, self.candidate(version, 1), 'nuget-candidate-9001-1', seen.append), + ('1.0.0-ci.24.1', 1, 1)) + self.assertEqual(publication(retry, version, self.candidate(version, 2), 'nuget-candidate-9001-2', seen.append), + ('1.0.0-ci.24.1', 1, 2)) + # Re-running all jobs is an intentional new candidate: preflight allocates the new attempt suffix. + rerun = {**env, 'GITHUB_RUN_ATTEMPT': '3'} + fresh = selected(rerun, seen.append) + self.assertEqual(fresh, '1.0.0-ci.24.3') + self.assertEqual(publication(rerun, fresh, self.candidate(fresh, 3), 'nuget-candidate-9001-3', seen.append), + ('1.0.0-ci.24.3', 3, 3)) + # Stable tags select the tag version in every attempt. + tag = {**retry, 'GITHUB_REF': 'refs/tags/v1.2.3'} + self.assertEqual(publication(tag, '1.2.3', self.candidate('1.2.3', 1), 'nuget-candidate-9001-1', seen.append), + ('1.2.3', None, 1)) + self.assertEqual(seen, ['a' * 40]) + + def test_publication_rejects_mismatched_candidate_identities(self): + seen = [] + env = {**self.PUBLISHER, 'GITHUB_RUN_ATTEMPT': '2'} + for expected in ('1.0.0-ci.23.1', '1.0.0-ci.24.3', '1.0.0-ci.24', '1.0.0-ci.24.0', '1.0.0-ci.024.1', + '1.0.0-ci.24.1.1', '1.0.0-ci.24.1-rc', '1.2.3', '', None): + with self.subTest(expected=expected), self.assertRaisesRegex(ValueError, 'does not match publisher identity'): + authorized(env, expected, seen.append) + with self.assertRaisesRegex(ValueError, 'does not match publisher identity'): + authorized({**env, 'GITHUB_REF': 'refs/tags/v1.2.3'}, '1.2.4', seen.append) + version, good = '1.0.0-ci.24.1', self.candidate('1.0.0-ci.24.1', 2) + cases = [ + ('other version', dict(good, version='1.0.0-ci.24.2'), 'nuget-candidate-9001-2', 'version and source'), + ('other source', dict(good, sourceCommit='b' * 40), 'nuget-candidate-9001-2', 'version and source'), + ('other run', self.candidate(version, 2, run_id='9000'), 'nuget-candidate-9000-2', 'this workflow run'), + ('no producer', dict(good, build={}), 'nuget-candidate-9001-2', 'this workflow run'), + ('future producer', self.candidate(version, 3), 'nuget-candidate-9001-3', 'outside its allocation'), + ('artifact of another attempt', good, 'nuget-candidate-9001-1', 'artifact name'), + ('artifact of another run', good, 'nuget-candidate-9000-2', 'artifact name'), + ('unbound latest artifact', good, 'nuget-candidate-latest', 'artifact name'), + ] + for label, manifest, artifact, message in cases: + with self.subTest(label), self.assertRaisesRegex(ValueError, message): + bound_candidate(env, version, 1, manifest, artifact) + with self.assertRaisesRegex(ValueError, 'outside its allocation'): + bound_candidate(env, '1.0.0-ci.24.2', 2, self.candidate('1.0.0-ci.24.2', 1), 'nuget-candidate-9001-1') + if __name__ == '__main__': unittest.main()