Publish the retained candidate on failed-jobs publication retries

.github/workflows/publish-nuget.yml

@@ -66,8 +66,12 @@ jobs:
       - name: Recheck source identity, allowlist, contents and hashes before authentication
         env:
           PACKAGE_VERSION: ${{ needs.candidate.outputs.version }}
+          ARTIFACT_NAME: ${{ needs.candidate.outputs.artifact_name }}
         run: |
-          python eng/packaging/release_channels.py --expected-version "$PACKAGE_VERSION"
+          # Authorize the candidate allocated for this run (retained by a failed-jobs-only retry) and bind
+          # the downloaded manifest and artifact name to it; never re-derive a version from this attempt.
+          python eng/packaging/release_channels.py --expected-version "$PACKAGE_VERSION" \
+            --manifest artifacts/packages/manifest.json --artifact-name "$ARTIFACT_NAME"
           python eng/packaging/packages.py verify --version "$PACKAGE_VERSION" --commit "$GITHUB_SHA"
       - name: Exchange GitHub OIDC identity for a short-lived NuGet API key
         id: login

docs/dependency-policy.md

@@ -19,7 +19,8 @@ Relevant runtime diagnostics remain local and conditional on existing tools;
 missing coverage is recorded honestly. No toolchain installation is implied.
 
 Main publishes exact `1.0.0-ci.RUN.ATTEMPT` candidates to nuget.org through the
-existing trusted publisher. Deliberate canonical `vX.Y.Z` tags may publish `X.Y.Z`
+existing trusted publisher; ATTEMPT is the attempt that allocated the candidate, which a
+failed-jobs-only retry keeps and publication verifies instead of re-deriving. Deliberate canonical `vX.Y.Z` tags may publish `X.Y.Z`
 only when the tagged commit belongs to main history. The `nuget` environment
 must permit main and `v*` tags; the portable guard rejects noncanonical tags,
 other repositories/events and stable closures containing prerelease packages.

eng/packaging/README.md

@@ -77,7 +77,12 @@ The workflow allocates `1.0.0-ci.<workflow-run-number>.<run-attempt>` once befor
 `1.0.0-ci.3.1`, then `1.0.0-ci.4.1`. GitHub owns the counter; no version commit or tag is written back
 to the repository. The counter continues from earlier runs of this workflow, and failed runs can leave
 gaps. Re-running all jobs uses the new attempt suffix. Retrying only failed downstream jobs retains
-the already allocated version and producer artifact. Main remains a prerelease stream. Stable tags
+the already allocated version and producer artifact. Publication therefore authorizes that retained
+candidate instead of deriving a version from its own attempt: the version must carry this run number
+and an allocation attempt no later than the current attempt, and the downloaded manifest and artifact
+name must name this commit, this run and a producing attempt between the allocation and the current
+attempt. An artifact from another run, attempt or candidate is rejected; there is no latest-artifact
+selection. Main remains a prerelease stream. Stable tags
 select the exact stable version before building and reject prerelease dependency closures. Never create
 a tag solely for verification. See [dependency admission](../../docs/dependency-policy.md).
 
@@ -93,7 +98,9 @@ and must run through all gates. Local builds and PR candidates never upload to t
 Duplicate versions fail; there is deliberately no `--skip-duplicate`. NuGet cannot atomically publish
 ten packages. If upload partially succeeds, inspect the registry and retained manifest and re-run all
 jobs to allocate a new complete version; do not promote a partial release set or retry it blindly.
-Retrying a diagnosed failed publication uses the retained candidate. A bad published version is superseded by a new
+Retrying a diagnosed failed publication (re-run failed jobs) uses the retained candidate and its version;
+re-running all jobs deliberately builds and publishes a new candidate. A re-run always executes the
+workflow of the original commit, so a fix to publication tooling applies only to runs of later commits. A bad published version is superseded by a new
 version; consumers retain their prior exact version/lock until the upgrade is approved.
 
 ## Consume

eng/packaging/release_channels.py

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 """Select and authorize the original main or deliberate stable-tag candidate."""
 import argparse
+import json
 import os
 from pathlib import Path
 import re
@@ -12,9 +13,12 @@
 
 ROOT = Path(__file__).resolve().parents[2]
 STABLE = r'(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)'
+MAIN_CANDIDATE = re.compile(r'1\.0\.0-ci\.([1-9][0-9]*)\.([1-9][0-9]*)')
 
 
 def selected(env, ancestor):
+    """Allocate the candidate version once, before build (preflight): the current run and attempt on
+    main, or the canonical stable tag. Publication authorizes it with `authorized`, never re-selects."""
     if env.get('GITHUB_REPOSITORY') != 'ArcForges/DesktopPlatform' or env.get('GITHUB_EVENT_NAME') != 'push':
         raise ValueError('Wrong publisher repository or event')
     commit = env.get('GITHUB_SHA', '')
@@ -38,22 +42,76 @@ def verified_tag(ref, commit, resolve, ancestor):
     ancestor(commit)
 
 
+def authorized(env, expected, ancestor):
+    """Authorize publishing the candidate that preflight allocated for this run.
+
+    The version is allocated once, before build. A failed-jobs-only retry keeps that allocation and
+    its producer artifact while the publisher runs in a later attempt, so the candidate is validated
+    against this run rather than re-derived from the publisher's own attempt: a main candidate must
+    carry this run number and an allocation attempt no later than the current attempt. Re-running
+    all jobs allocates and builds a new candidate. Stable tags do not depend on the attempt."""
+    current = selected(env, ancestor)
+    if env['GITHUB_REF'] != 'refs/heads/main':
+        if expected != current:
+            raise ValueError('Candidate version does not match publisher identity')
+        return expected, None
+    match = MAIN_CANDIDATE.fullmatch(expected or '')
+    if not match or match[1] != env['GITHUB_RUN_NUMBER'] or int(match[2]) > int(env['GITHUB_RUN_ATTEMPT']):
+        raise ValueError('Candidate version does not match publisher identity')
+    return expected, int(match[2])
+
+
+def bound_candidate(env, version, allocated, manifest, artifact_name):
+    """The downloaded artifact is the authorized candidate: this source, version and workflow run,
+    packed at or after the allocation attempt and no later than the current attempt, under the artifact
+    name of that producing attempt. An artifact from another run, attempt or candidate is rejected."""
+    build = manifest.get('build') if isinstance(manifest.get('build'), dict) else {}
+    if manifest.get('version') != version or manifest.get('sourceCommit') != env['GITHUB_SHA']:
+        raise ValueError('Candidate manifest does not match the authorized version and source')
+    run_id, attempt = build.get('runId'), str(build.get('runAttempt') or '')
+    if run_id != env.get('GITHUB_RUN_ID') or not re.fullmatch('[1-9][0-9]*', attempt):
+        raise ValueError('Candidate was not produced by this workflow run')
+    produced = int(attempt)
+    if produced > int(env['GITHUB_RUN_ATTEMPT']) or (allocated is not None and produced < allocated):
+        raise ValueError('Candidate producer attempt is outside its allocation')
+    if artifact_name != f'nuget-candidate-{run_id}-{produced}':
+        raise ValueError('Candidate artifact name does not match its producer')
+    return produced
+
+
+def publication(env, expected, manifest, artifact_name, ancestor):
+    version, allocated = authorized(env, expected, ancestor)
+    produced = bound_candidate(env, version, allocated, manifest, artifact_name)
+    return version, allocated, produced
+
+
 def main():
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument('--expected-version')
+    parser.add_argument('--expected-version', help='authorize this allocated candidate for publication')
+    parser.add_argument('--manifest', type=Path, help='manifest.json of the downloaded candidate artifact')
+    parser.add_argument('--artifact-name', help='name of the downloaded candidate artifact')
     args = parser.parse_args()
     def ancestor(commit):
         verified_tag(os.environ['GITHUB_REF'], commit,
             lambda ref: subprocess.check_output(['git', 'rev-parse', '--verify', ref], cwd=ROOT, text=True).strip(),
             lambda sha: subprocess.run(['git', 'merge-base', '--is-ancestor', sha, 'origin/main'], cwd=ROOT, check=True))
-    version = selected(os.environ, ancestor)
-    if args.expected_version and args.expected_version != version:
-        raise ValueError('Candidate version does not match publisher identity')
+    if args.expected_version is None:
+        version = selected(os.environ, ancestor)
+        message = f'Allocated immutable candidate {version}'
+    else:
+        if not (args.manifest and args.artifact_name):
+            raise ValueError('Publication requires the candidate --manifest and --artifact-name')
+        manifest = json.loads(args.manifest.read_text(encoding='utf-8'))
+        version, allocated, produced = publication(os.environ, args.expected_version, manifest,
+                                                   args.artifact_name, ancestor)
+        message = (f'Authorized immutable candidate {version}'
+                   + (f' (allocated in attempt {allocated}, packed in attempt {produced}, '
+                      f'publishing in attempt {os.environ["GITHUB_RUN_ATTEMPT"]})' if allocated else ''))
     audit(stable='-' not in version)
     if 'GITHUB_OUTPUT' in os.environ:
         with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as output:
             output.write(f'version={version}\n')
-    print(f'Authorized immutable candidate {version}')
+    print(message)
 
 
 if __name__ == '__main__':

eng/test_dependency_policy.py

@@ -8,7 +8,7 @@
 
 from dependency_policy import ROOT, POLICY, audit, check_admission, check_history, check_python, closure, exact, framework_upgrade, python_closure
 sys.path.insert(0, str(Path(__file__).resolve().parent / 'packaging'))
-from release_channels import selected, verified_tag
+from release_channels import authorized, bound_candidate, publication, selected, verified_tag
 
 
 class AdmissionTests(unittest.TestCase):
@@ -106,6 +106,68 @@ def resolve(ref):
         with self.assertRaisesRegex(ValueError, 'tag target'):
             verified_tag('refs/tags/v1.2.3', 'b' * 40, resolve, seen.append)
 
+    PUBLISHER = {'GITHUB_REPOSITORY': 'ArcForges/DesktopPlatform', 'GITHUB_EVENT_NAME': 'push',
+                 'GITHUB_SHA': 'a' * 40, 'GITHUB_REF': 'refs/heads/main', 'GITHUB_RUN_ID': '9001',
+                 'GITHUB_RUN_NUMBER': '24', 'GITHUB_RUN_ATTEMPT': '1'}
+
+    @staticmethod
+    def candidate(version, producer_attempt, run_id='9001'):
+        return {'version': version, 'sourceCommit': 'a' * 40,
+                'build': {'runId': run_id, 'runAttempt': str(producer_attempt)}}
+
+    def test_publication_keeps_the_allocated_candidate_across_supported_retries(self):
+        seen = []
+        # Initial execution: preflight allocates in attempt 1 and the same attempt publishes it.
+        env = dict(self.PUBLISHER)
+        version = selected(env, seen.append)
+        self.assertEqual(version, '1.0.0-ci.24.1')
+        self.assertEqual(publication(env, version, self.candidate(version, 1), 'nuget-candidate-9001-1', seen.append),
+                         ('1.0.0-ci.24.1', 1, 1))
+        # Failed-jobs-only retry: preflight is not re-run, so attempt 2 publishes the retained allocation,
+        # whether the producer artifact is retained from attempt 1 or packing was re-run in attempt 2.
+        retry = {**env, 'GITHUB_RUN_ATTEMPT': '2'}
+        self.assertEqual(publication(retry, version, self.candidate(version, 1), 'nuget-candidate-9001-1', seen.append),
+                         ('1.0.0-ci.24.1', 1, 1))
+        self.assertEqual(publication(retry, version, self.candidate(version, 2), 'nuget-candidate-9001-2', seen.append),
+                         ('1.0.0-ci.24.1', 1, 2))
+        # Re-running all jobs is an intentional new candidate: preflight allocates the new attempt suffix.
+        rerun = {**env, 'GITHUB_RUN_ATTEMPT': '3'}
+        fresh = selected(rerun, seen.append)
+        self.assertEqual(fresh, '1.0.0-ci.24.3')
+        self.assertEqual(publication(rerun, fresh, self.candidate(fresh, 3), 'nuget-candidate-9001-3', seen.append),
+                         ('1.0.0-ci.24.3', 3, 3))
+        # Stable tags select the tag version in every attempt.
+        tag = {**retry, 'GITHUB_REF': 'refs/tags/v1.2.3'}
+        self.assertEqual(publication(tag, '1.2.3', self.candidate('1.2.3', 1), 'nuget-candidate-9001-1', seen.append),
+                         ('1.2.3', None, 1))
+        self.assertEqual(seen, ['a' * 40])
+
+    def test_publication_rejects_mismatched_candidate_identities(self):
+        seen = []
+        env = {**self.PUBLISHER, 'GITHUB_RUN_ATTEMPT': '2'}
+        for expected in ('1.0.0-ci.23.1', '1.0.0-ci.24.3', '1.0.0-ci.24', '1.0.0-ci.24.0', '1.0.0-ci.024.1',
+                         '1.0.0-ci.24.1.1', '1.0.0-ci.24.1-rc', '1.2.3', '', None):
+            with self.subTest(expected=expected), self.assertRaisesRegex(ValueError, 'does not match publisher identity'):
+                authorized(env, expected, seen.append)
+        with self.assertRaisesRegex(ValueError, 'does not match publisher identity'):
+            authorized({**env, 'GITHUB_REF': 'refs/tags/v1.2.3'}, '1.2.4', seen.append)
+        version, good = '1.0.0-ci.24.1', self.candidate('1.0.0-ci.24.1', 2)
+        cases = [
+            ('other version', dict(good, version='1.0.0-ci.24.2'), 'nuget-candidate-9001-2', 'version and source'),
+            ('other source', dict(good, sourceCommit='b' * 40), 'nuget-candidate-9001-2', 'version and source'),
+            ('other run', self.candidate(version, 2, run_id='9000'), 'nuget-candidate-9000-2', 'this workflow run'),
+            ('no producer', dict(good, build={}), 'nuget-candidate-9001-2', 'this workflow run'),
+            ('future producer', self.candidate(version, 3), 'nuget-candidate-9001-3', 'outside its allocation'),
+            ('artifact of another attempt', good, 'nuget-candidate-9001-1', 'artifact name'),
+            ('artifact of another run', good, 'nuget-candidate-9000-2', 'artifact name'),
+            ('unbound latest artifact', good, 'nuget-candidate-latest', 'artifact name'),
+        ]
+        for label, manifest, artifact, message in cases:
+            with self.subTest(label), self.assertRaisesRegex(ValueError, message):
+                bound_candidate(env, version, 1, manifest, artifact)
+        with self.assertRaisesRegex(ValueError, 'outside its allocation'):
+            bound_candidate(env, '1.0.0-ci.24.2', 2, self.candidate('1.0.0-ci.24.2', 1), 'nuget-candidate-9001-1')
+
 
 if __name__ == '__main__':
     unittest.main()