diff --git a/docs/cli/changelog/cmd-backfill.md b/docs/cli/changelog/cmd-backfill.md index d750e7a51..b32d8554d 100644 --- a/docs/cli/changelog/cmd-backfill.md +++ b/docs/cli/changelog/cmd-backfill.md @@ -8,7 +8,7 @@ For each product in scope the command: 2. Parses each `## {version}` section into typed entries. Unrecognized `### …` subsections whose body is bullets become `Other`-typed entries; prose subsections are preserved in the bundle description. 3. Writes bundle YAML to `{output}/{product}/changelog/bundles/{version}.yaml`. 4. Writes per-entry YAML files: `{pr}.yaml` for entries that reference a pull request, `note-{slug}.yaml` for entries with no PR reference. -5. Writes `notes-{target}.json` per version listing all `note-*.yaml` files for that target (the notes registry). This is a **local** backfill artifact, not the CDN dual-write layout (`notes-{product}-{version}.json` plus legacy `notes-{version}.json`). +5. Writes `notes-{target}.json` per version listing all `note-*.yaml` files for that target (the notes registry). This is a **local** backfill artifact, not the CDN key `notes-{product}-{version}.json`. 6. Prints a per-product report including entry counts and no-PR rates. The primary use case is measuring how much published release-notes content cannot be traced back to a PR — determining how much historical content needs the PR-less "note" format from [docs-eng-team#789](https://github.com/elastic/docs-eng-team/issues/789). diff --git a/docs/cli/changelog/cmd-note.md b/docs/cli/changelog/cmd-note.md index 23f21a888..2349b537b 100644 --- a/docs/cli/changelog/cmd-note.md +++ b/docs/cli/changelog/cmd-note.md @@ -59,12 +59,10 @@ products: ## After creation Upload is the same as for other changelog YAML files. -The scrubber writes two indexes per note: - -- `changelog/{org}/{repo}/notes-{product}-{version}.json` — notes for that product and version. `changelog bundle` reads this key first. -- `changelog/{org}/{repo}/notes-{version}.json` — the union of notes for that version across products, kept for older CLI pins. - -`{changelog}` does not read these indexes. It loads published bundle YAML (and amend sidecars listed in `bundle/{product}/registry.json`). +The scrubber writes `changelog/{org}/{repo}/notes-{product}-{version}.json` listing every note +file for that product and version. `changelog bundle` reads that key. `{changelog}` does not — +it loads published bundle YAML (and amend sidecars listed in `bundle/{product}/registry.json`), +so removing a notes-index object from S3 does not change `:cdn:` pages. If the release bundle for that product and version or date has already shipped when you upload, the scrubber generates an amend sidecar for **that product**, then rebuilds that product's `bundle/{product}/registry.json` so `{changelog}` `:cdn:` pages pick it up. Other products that share the version are left alone. diff --git a/docs/development/changelog-bundle-registry.md b/docs/development/changelog-bundle-registry.md index 17cfc3972..4623c594d 100644 --- a/docs/development/changelog-bundle-registry.md +++ b/docs/development/changelog-bundle-registry.md @@ -80,10 +80,10 @@ narrowed reconciliation to the bundle tree): same way as numbered `.amend-{N}` files, after those numbered amends. The `.amend-notes` suffix is **reserved** — do not create files with that suffix manually; see [](/cli/changelog/bundle-amend.md). -- **Notes index** — `changelog/{org}/{repo}/notes-{product}-{version}.json` (product-scoped) plus a dual-written - legacy `changelog/{org}/{repo}/notes-{version}.json` (version union), **public bucket only**, produced by - the scrubber Lambda's `NotesIndexReconciler`. See - [Notes-index format](#notes-index-format) below. +- **Notes index** — `changelog/{org}/{repo}/notes-{product}-{version}.json`, **public bucket + only**, produced by the scrubber Lambda's `NotesIndexReconciler`. Refer to + [Notes-index format](#notes-index-format). `{changelog}` `:cdn:` does not read + these keys; it loads `bundle/{product}/registry.json` and the YAML files that manifest lists. - **Changelog-entry index** — `changelog/{org}/{repo}/{branch}/registry.json`, a **legacy client-authored pass-through**: the current `changelog upload` never writes one, but manifests written by older CLI versions are still mirrored verbatim from the private bucket, because @@ -129,15 +129,12 @@ zero-bundle state. The reconciler deliberately restores the former. ## Notes-index format [notes-index-format] For each product and release version that has at least one note, the scrubber Lambda writes a -product-scoped notes index at `changelog/{org}/{repo}/notes-{product}-{version}.json`. It also -dual-writes a legacy version-union index at `changelog/{org}/{repo}/notes-{version}.json` so -older `changelog bundle` clients that still GET the version-only key keep working. The product -and version segments are concatenated into the new filename; consumers must not parse that slug -apart. +product-scoped notes index at `changelog/{org}/{repo}/notes-{product}-{version}.json`. The product +and version segments are concatenated into the filename; consumers must not parse that slug apart. -`changelog bundle` GETs the product-scoped key first and falls back to the version-only key -only on HTTP 404. An empty product-scoped index does not fall back. `{changelog}` does not read -these indexes; it loads bundle YAML from the product registry. +`changelog bundle` GETs this key. A 404 means no notes for that product and version. +`{changelog}` does not read these indexes. It loads bundle YAML from the product registry +(`bundle/{product}/registry.json` and the listed parent plus amend files). Product-scoped schema (`schema_version: 1`): @@ -153,13 +150,10 @@ Product-scoped schema (`schema_version: 1`): } ``` -The legacy `notes-{version}.json` body omits `product` and `version` and lists the union of note -paths for that version across products. - | Field | Meaning | |---|---| | `schema_version` | Schema version. Currently `1`. | -| `product` / `version` | Set on product-scoped indexes; omitted from the legacy version-union body. | +| `product` / `version` | Product id and release version this index belongs to. | | `notes[].path` | Pool-relative path of the note within `changelog/{org}/{repo}/`. The leading segment before the first `/` is the branch. | | `notes[].bundle_seq` | Derived reporting field: `0` = no bundle published for this version yet, `1` = note shipped in the original bundle, `2` = note carried by the Lambda-generated `.amend-notes` sidecar. | @@ -167,11 +161,10 @@ paths for that version across products. reconcile by comparing the notes index against the set of entries in the published bundle and its amend sidecars. -A 404 on both the product-scoped index and the legacy version-union index means "no notes -published for this product and version". An empty `notes` array never appears on a successfully -reconciled index — the index is deleted rather than emptied, following the same -[absent ≠ empty](#absent-empty) rule as the bundle registry. Until older clients stop reading -`notes-{version}.json`, the Lambda keeps that key while any note still declares the version. +A 404 on the product-scoped index means "no notes published for this product and version". An +empty `notes` array never appears on a successfully reconciled index — the index is deleted +rather than emptied, following the same [absent ≠ empty](#absent-empty) rule as the bundle +registry. ## Shallow per-tree change maps [shallow-maps] @@ -322,10 +315,8 @@ ship silently). `CdnChangelogEntryFetcher` reuses a shared `HttpClient` in produ owned client only when a test injects a handler, mirroring `CdnChangelogFetcher`. When CDN sourcing is on, the same fetcher also merges changelog notes for each concrete -`--output-products` pair: it GETs `changelog/{org}/{repo}/notes-{product}-{version}.json` first -and falls back to `notes-{version}.json` only on HTTP 404. See -[Notes-index format](#notes-index-format). Checksum dedup keeps a single copy if two products at -the same version still share the legacy index. +`--output-products` pair: it GETs `changelog/{org}/{repo}/notes-{product}-{version}.json`. See +[Notes-index format](#notes-index-format). `{changelog}` `:cdn:` does not use this path. ## Consumer: `{changelog}` directive `cdn:` mode (implemented) @@ -382,6 +373,7 @@ notes use `NoopReleaseNotesResolver`. 2. Parse it; for each `bundles[].file`, `GET {cdnBase}/bundle/{product}/{file}`. 3. Feed the downloaded YAML into the existing `BundleLoader` → `MergeBundlesByTarget` → render pipeline. **Rendering is unchanged**; only the source of the bundle bytes differs. + Notes-index objects under `changelog/{org}/{repo}/notes-*.json` are never fetched here. Implemented by `CdnChangelogFetcher` (a stateless async fetch engine in `Elastic.Documentation.Configuration`) and `BundleLoader.LoadBundlesFromContent`. Because public diff --git a/src/Elastic.Documentation.Configuration/ReleaseNotes/CdnChangelogEntryFetcher.cs b/src/Elastic.Documentation.Configuration/ReleaseNotes/CdnChangelogEntryFetcher.cs index cfb6ad933..364a24bdc 100644 --- a/src/Elastic.Documentation.Configuration/ReleaseNotes/CdnChangelogEntryFetcher.cs +++ b/src/Elastic.Documentation.Configuration/ReleaseNotes/CdnChangelogEntryFetcher.cs @@ -489,10 +489,9 @@ private static Uri CombineSegments(Uri baseUri, IReadOnlyList segments) /// /// Fetches all note-*.yml entries for / and /// at from the CDN. Reads - /// notes-{product}-{version}.json first; on HTTP 404 only, falls back to the legacy - /// notes-{version}.json. A missing index (both 404) means no notes (not an error). - /// An empty product-scoped index does not fall back. A listed note that cannot be fetched - /// is a hard error — the index is an authoritative promise that the note exists. + /// notes-{product}-{version}.json. A missing index (404) means no notes (not an error). + /// A listed note that cannot be fetched is a hard error — the index is an authoritative + /// promise that the note exists. /// public async Task> FetchNotesAsync( Uri baseUri, @@ -516,44 +515,26 @@ Cancel ctx return []; } - var productIndexUri = CombineSegments(baseUri, ["changelog", org, repo, $"notes-{product}-{version}.json"]); - var (productMissing, productIndex) = await TryLoadNotesIndexAsync( - productIndexUri, - org, - repo, - version, - emitError, - ctx - ).ConfigureAwait(false); - if (productIndex is not null) - return await FetchListedNotesAsync(baseUri, org, repo, version, productIndex, emitError, ctx).ConfigureAwait(false); - if (!productMissing) - return []; - - var legacyIndexUri = CombineSegments(baseUri, ["changelog", org, repo, $"notes-{version}.json"]); - var (legacyMissing, legacyIndex) = await TryLoadNotesIndexAsync(legacyIndexUri, org, repo, version, emitError, ctx).ConfigureAwait( - false - ); - if (legacyIndex is not null) - return await FetchListedNotesAsync(baseUri, org, repo, version, legacyIndex, emitError, ctx).ConfigureAwait(false); - if (legacyMissing) + var indexUri = CombineSegments(baseUri, ["changelog", org, repo, $"notes-{product}-{version}.json"]); + var (missing, index) = await TryLoadNotesIndexAsync(indexUri, org, repo, version, emitError, ctx).ConfigureAwait(false); + if (index is not null) + return await FetchListedNotesAsync(baseUri, org, repo, version, index, emitError, ctx).ConfigureAwait(false); + if (missing) { _logger.LogDebug( - "Notes index for {Org}/{Repo}/{Product}@{Version} not found at {ProductUri} or {LegacyUri}; no notes to bundle", + "Notes index for {Org}/{Repo}/{Product}@{Version} not found at {Uri}; no notes to bundle", org, repo, product, version, - productIndexUri, - legacyIndexUri + indexUri ); } return []; } /// - /// Loads a notes index. Returns missing: true only on HTTP 404. A present but empty - /// or unparseable body is missing: false with a null index so callers do not fall back. + /// Loads a notes index. Returns missing: true only on HTTP 404. /// private async Task<(bool Missing, NotesIndex? Index)> TryLoadNotesIndexAsync( Uri indexUri, diff --git a/src/Elastic.Documentation.Configuration/ReleaseNotes/CdnChangelogFetcher.cs b/src/Elastic.Documentation.Configuration/ReleaseNotes/CdnChangelogFetcher.cs index feb3d9067..ebc43cde6 100644 --- a/src/Elastic.Documentation.Configuration/ReleaseNotes/CdnChangelogFetcher.cs +++ b/src/Elastic.Documentation.Configuration/ReleaseNotes/CdnChangelogFetcher.cs @@ -21,7 +21,8 @@ namespace Elastic.Documentation.Configuration.ReleaseNotes; /// Fetches changelog bundles for a single product from the public CDN. It reads /// {base}/bundle/{product}/registry.json to enumerate bundles, downloads each /// {base}/bundle/{product}/{file}, and parses them via -/// . +/// . It does not read notes indexes +/// (changelog/{org}/{repo}/notes-*.json); those are used only by changelog bundle. /// /// /// diff --git a/src/Elastic.Documentation.Configuration/ReleaseNotes/ChangelogKeys.cs b/src/Elastic.Documentation.Configuration/ReleaseNotes/ChangelogKeys.cs index dbffdc39d..27691259c 100644 --- a/src/Elastic.Documentation.Configuration/ReleaseNotes/ChangelogKeys.cs +++ b/src/Elastic.Documentation.Configuration/ReleaseNotes/ChangelogKeys.cs @@ -93,17 +93,9 @@ public static string ChangelogFileKey(string org, string repo, string branch, st public static string ChangelogRegistryKey(string poolGroup) => $"{ChangelogPrefix}{poolGroup}/{RegistryFileName}"; /// - /// The legacy notes-index key for one release version within a repo: - /// changelog/{org}/{repo}/notes-{version}.json. - /// Repo-level and branch-agnostic — the union of notes for that version across every product. + /// The leftover version-only key changelog/{org}/{repo}/notes-{version}.json. + /// No longer written. The notes reconciler deletes remaining objects at this key. /// - /// - /// The slug in the key is the release version (e.g. 9.3.0 or 2026-05-15). - /// Previously this parameter was named target to match the obsolete target: YAML field; - /// it was renamed to version when that field was replaced by versions: on notes. - /// Older changelog bundle clients still GET this key; the scrubber dual-writes it - /// alongside . - /// public static string NotesIndexKey(string org, string repo, string version) => $"{ChangelogPrefix}{org}/{repo}/notes-{version}.json"; /// @@ -123,8 +115,9 @@ public static string NotesIndexKey(string org, string repo, string product, stri /// /// Returns true when is a notes-index key of the form /// changelog/{org}/{repo}/notes-{slug}.json (exactly two group segments, then - /// a notes--prefixed JSON file with a non-empty slug). Covers both the legacy - /// notes-{version}.json shape and the product-scoped notes-{product}-{version}.json shape. + /// a notes--prefixed JSON file with a non-empty slug). Covers the product-scoped + /// notes-{product}-{version}.json shape and leftover version-only + /// notes-{version}.json objects so they can still be listed and deleted. /// public static bool IsNotesIndex(string key) { diff --git a/src/Elastic.Documentation.Configuration/ReleaseNotes/NotesIndex.cs b/src/Elastic.Documentation.Configuration/ReleaseNotes/NotesIndex.cs index f4f68da42..1fbedc329 100644 --- a/src/Elastic.Documentation.Configuration/ReleaseNotes/NotesIndex.cs +++ b/src/Elastic.Documentation.Configuration/ReleaseNotes/NotesIndex.cs @@ -41,8 +41,7 @@ public sealed record NoteIndexEntry } /// -/// Notes index published at changelog/{org}/{repo}/notes-{product}-{version}.json -/// (and dual-written to the legacy notes-{version}.json key). +/// Notes index published at changelog/{org}/{repo}/notes-{product}-{version}.json. /// Lists all note-*.yml fragments for one product and release version, /// across every branch of the repo. /// @@ -50,8 +49,7 @@ public sealed record NoteIndexEntry /// Contents are paths, not bodies — the note files remain the single source of truth. /// A stale index can only omit or over-list, never serve stale prose. Bundling a version /// is therefore 1 GET for the index + one GET per listed note. -/// and are set on product-scoped writes and -/// omitted from legacy version-union bodies so older deserializers keep working. +/// and are set on every write. /// public sealed record NotesIndex { @@ -62,14 +60,12 @@ public sealed record NotesIndex public const int CurrentSchemaVersion = 1; /// - /// Bundle product id this index belongs to. Present on product-scoped keys; omitted on - /// the legacy version-union body. + /// Bundle product id this index belongs to. /// public string? Product { get; init; } /// - /// Release version this index belongs to. Present on product-scoped keys; omitted on - /// the legacy version-union body. + /// Release version this index belongs to. /// public string? Version { get; init; } diff --git a/src/services/Elastic.Changelog/Bundling/ChangelogBundlingService.cs b/src/services/Elastic.Changelog/Bundling/ChangelogBundlingService.cs index a8f7cadb2..c2b2e7abc 100644 --- a/src/services/Elastic.Changelog/Bundling/ChangelogBundlingService.cs +++ b/src/services/Elastic.Changelog/Bundling/ChangelogBundlingService.cs @@ -1604,8 +1604,6 @@ Cancel ctx return entries; // Dedup by checksum: a note body identical to a PR entry (edge case) should appear once. - // Two output products at the same version may both fall back to notes-{version}.json - // until product-scoped indexes exist; checksum skip keeps a single copy. var seen = new HashSet(entries.Select(e => e.Checksum), StringComparer.OrdinalIgnoreCase); var combined = new List(entries); diff --git a/src/services/Elastic.Changelog/Reconciliation/NoteAmendReconciler.cs b/src/services/Elastic.Changelog/Reconciliation/NoteAmendReconciler.cs index 3211bea69..81152d0bc 100644 --- a/src/services/Elastic.Changelog/Reconciliation/NoteAmendReconciler.cs +++ b/src/services/Elastic.Changelog/Reconciliation/NoteAmendReconciler.cs @@ -19,7 +19,6 @@ namespace Elastic.Changelog.Reconciliation; /// ({parent}.amend-notes.yaml) that carries notes that arrived after the release shipped. /// Also updates each bundle_seq on the product-scoped notes index: 0 = no bundle yet, /// 1 = shipped in the original bundle or a human amend, 2 = carried by the reconciler amend sidecar. -/// The dual-written legacy notes-{version}.json path list is refreshed in the same pass. /// /// /// @@ -52,8 +51,8 @@ public sealed class NoteAmendReconciler( /// /// For the given repository scope, scans each product that appears in /// (not every bundle/{product}/ prefix), writes or - /// deletes that product's reconciler-owned amend sidecars, and re-writes product-scoped and - /// legacy notes indexes with correct bundle_seq values. Empty version lists mean that + /// deletes that product's reconciler-owned amend sidecars, and re-writes product-scoped + /// notes indexes with correct bundle_seq values. Empty version lists mean that /// product×version just vanished from the notes index; amend still runs so sidecars can drop. /// /// @@ -128,18 +127,6 @@ Cancel ctx var updatedEntries = WithSeqs(write.Notes, seqs); await notesIndexReconciler.WriteIndexAsync(indexKey, updatedEntries, ct, new NotesIndexMetadata(write.Product, write.Version)); }); - - foreach (var (version, unionNotes) in UnionByVersion(notesByProduct)) - { - ctx.ThrowIfCancellationRequested(); - if (unionNotes.Count == 0) - continue; - - var seqs = MaxSeqsForVersion(notesByProduct.Keys, version, seqMap); - var updatedEntries = WithSeqs(unionNotes, seqs); - var indexKey = ChangelogKeys.NotesIndexKey(org, repo, version); - await notesIndexReconciler.WriteIndexAsync(indexKey, updatedEntries, ctx); - } } private static List WithSeqs(IReadOnlyList notes, IReadOnlyDictionary seqs) => @@ -148,47 +135,6 @@ private static List WithSeqs(IReadOnlyList notes .OrderBy(n => n.Path, StringComparer.Ordinal) .ToList(); - private static Dictionary> UnionByVersion( - IReadOnlyDictionary>> notesByProduct - ) - { - var byVersion = new Dictionary>(StringComparer.Ordinal); - foreach (var byProductVersion in notesByProduct.Values) - { - foreach (var (version, notes) in byProductVersion) - { - if (!byVersion.TryGetValue(version, out var union)) - byVersion[version] = union = []; - foreach (var note in notes) - { - if (!union.Any(e => e.Path == note.Path)) - union.Add(note); - } - } - } - return byVersion; - } - - private static Dictionary MaxSeqsForVersion( - IEnumerable products, - string version, - IReadOnlyDictionary> seqMap - ) - { - var max = new Dictionary(StringComparer.Ordinal); - foreach (var product in products) - { - if (!seqMap.TryGetValue(ProductVersionKey(product, version), out var seqs)) - continue; - foreach (var (path, seq) in seqs) - { - if (!max.TryGetValue(path, out var current) || seq > current) - max[path] = seq; - } - } - return max; - } - private static string ProductVersionKey(string product, string version) => $"{product}/{version}"; private async Task ProcessProductAsync( diff --git a/src/services/Elastic.Changelog/Reconciliation/NotesIndexReconciler.cs b/src/services/Elastic.Changelog/Reconciliation/NotesIndexReconciler.cs index 60aa4cb32..7c3143b33 100644 --- a/src/services/Elastic.Changelog/Reconciliation/NotesIndexReconciler.cs +++ b/src/services/Elastic.Changelog/Reconciliation/NotesIndexReconciler.cs @@ -12,11 +12,10 @@ namespace Elastic.Changelog.Reconciliation; /// -/// Rebuilds notes indexes for one repository by listing all note-*.yml objects under -/// changelog/{org}/{repo}/, reading each to extract (product, version) pairs -/// from products[] (versions, then legacy target), and writing both -/// product-scoped notes-{product}-{version}.json keys and the legacy -/// notes-{version}.json union keys with conditional S3 writes. +/// Rebuilds product-scoped notes indexes for one repository by listing all note-*.yml +/// objects under changelog/{org}/{repo}/, reading each to extract (product, version) +/// pairs from products[], and writing notes-{product}-{version}.json with +/// conditional S3 writes. Leftover version-only notes-{version}.json keys are deleted. /// /// /// A note may declare multiple versions, so one note can appear in several indexes. The index @@ -43,16 +42,16 @@ public sealed class NotesIndexReconciler( private readonly string _sourceBucketName = sourceBucketName ?? publicBucketName; /// - /// Rebuilds product-scoped and legacy version-union notes indexes for the given repository scope. + /// Rebuilds product-scoped notes indexes for the given repository scope. /// All currently published note-*.yml files across every branch are listed and /// read to derive the grouping; every affected index is then (re)written. + /// Leftover version-only notes-{version}.json objects are deleted as stale. /// /// - /// Notes grouped by product, then version. Legacy version-union indexes are still written - /// in this pass but are not part of the return value; uses - /// the product map only. A product×version whose product-scoped index is stale is included - /// with an empty note list so amend can drop that product's sidecar. Those keys stay in S3 - /// until amend succeeds; leftover version-union indexes are deleted in this pass. + /// Notes grouped by product, then version. Consumed by . + /// A product×version whose product-scoped index is stale is included with an empty note list + /// so amend can drop that product's sidecar. Those keys stay in S3 until amend succeeds; + /// leftover version-only indexes are deleted in this pass. /// public async Task>>> ReconcileRepoAsync( ChangelogScope notesScope, @@ -68,7 +67,6 @@ Cancel ctx _logger.LogDebug("Found {Count} note file(s) for {Repo}", noteObjects.Count, notesScope.Group); var byProductVersion = new Dictionary>(StringComparer.Ordinal); - var byVersion = new Dictionary>(StringComparer.Ordinal); foreach (var obj in noteObjects) { ctx.ThrowIfCancellationRequested(); @@ -76,48 +74,38 @@ Cancel ctx var pairs = await ExtractProductVersionsAsync(obj.Key, ctx); foreach (var (product, version) in pairs) - { AddIndexEntry(byProductVersion, ProductVersionGroupKey(product, version), poolRelativePath); - AddIndexEntry(byVersion, version, poolRelativePath); - } } var groupParts = notesScope.Group.Split('/'); var (org, repo) = (groupParts[0], groupParts[1]); var existingIndexKeys = await ListExistingNotesIndexes(notesScope, ctx); - var intendedKeys = IntendedIndexKeys(org, repo, byProductVersion.Keys, byVersion.Keys); + var intendedKeys = IntendedIndexKeys(org, repo, byProductVersion.Keys); var writtenProducts = new Dictionary>>(StringComparer.Ordinal); IReadOnlyList<(string Product, string Version)> vanished = []; try { - if (byVersion.Count == 0) + if (byProductVersion.Count == 0) { _logger.LogDebug("No versions found for repo {Repo}; removing any stale indexes", notesScope.Group); } else { - var writes = BuildIndexWrites(org, repo, byProductVersion, byVersion); + var writes = BuildIndexWrites(org, repo, byProductVersion); await Parallel.ForEachAsync(writes, new ParallelOptions { MaxDegreeOfParallelism = MaxParallelReads, CancellationToken = ctx }, async (write, ct) => { - await WriteIndexAsync( - write.Key, - write.Entries, - ct, - write.Product is null ? null : new NotesIndexMetadata(write.Product, write.Version!) - ); - if (write.Product is null) - return; + await WriteIndexAsync(write.Key, write.Entries, ct, new NotesIndexMetadata(write.Product, write.Version)); lock (writtenProducts) { if (!writtenProducts.TryGetValue(write.Product, out var byVer)) writtenProducts[write.Product] = byVer = [with(StringComparer.Ordinal)]; - byVer[write.Version!] = write.Entries; + byVer[write.Version] = write.Entries; } }); } @@ -160,12 +148,7 @@ private static void AddIndexEntry(Dictionary> map, private static string ProductVersionGroupKey(string product, string version) => $"{product}/{version}"; - private static HashSet IntendedIndexKeys( - string org, - string repo, - IEnumerable productVersionKeys, - IEnumerable versions - ) + private static HashSet IntendedIndexKeys(string org, string repo, IEnumerable productVersionKeys) { var intended = new HashSet(StringComparer.Ordinal); foreach (var groupKey in productVersionKeys) @@ -173,19 +156,16 @@ IEnumerable versions var slash = groupKey.IndexOf('/', StringComparison.Ordinal); _ = intended.Add(ChangelogKeys.NotesIndexKey(org, repo, groupKey[..slash], groupKey[(slash + 1)..])); } - foreach (var version in versions) - _ = intended.Add(ChangelogKeys.NotesIndexKey(org, repo, version)); return intended; } private static List BuildIndexWrites( string org, string repo, - Dictionary> byProductVersion, - Dictionary> byVersion + Dictionary> byProductVersion ) { - var writes = new List(byProductVersion.Count + byVersion.Count); + var writes = new List(byProductVersion.Count); foreach (var (groupKey, entries) in byProductVersion) { var slash = groupKey.IndexOf('/', StringComparison.Ordinal); @@ -195,10 +175,6 @@ Dictionary> byVersion new NotesIndexWrite(ChangelogKeys.NotesIndexKey(org, repo, product, version), SortEntries(entries), product, version) ); } - foreach (var (version, entries) in byVersion) - { - writes.Add(new NotesIndexWrite(ChangelogKeys.NotesIndexKey(org, repo, version), SortEntries(entries), Product: null, version)); - } return writes; } @@ -546,7 +522,7 @@ public async Task WriteIndexAsync(string key, IReadOnlyList entr } } - private readonly record struct NotesIndexWrite(string Key, IReadOnlyList Entries, string? Product, string? Version); + private readonly record struct NotesIndexWrite(string Key, IReadOnlyList Entries, string Product, string Version); private enum StaleIndexReadKind { diff --git a/tests/Elastic.Changelog.Tests/Changelogs/BundleCdnSourcingTests.cs b/tests/Elastic.Changelog.Tests/Changelogs/BundleCdnSourcingTests.cs index 9eeed8fdf..7c2fc7a8d 100644 --- a/tests/Elastic.Changelog.Tests/Changelogs/BundleCdnSourcingTests.cs +++ b/tests/Elastic.Changelog.Tests/Changelogs/BundleCdnSourcingTests.cs @@ -632,7 +632,7 @@ public async Task ProfileGitHubRelease_ScopesByOutputProductsAndFiltersByRelease [Fact] public async Task BackportCollision_MainBranchWins_WarnAndKeepMain() { - // notes-9.3.0.json lists both main/note-known-issue.yml and 9.4/note-known-issue.yml. + // notes-elasticsearch-9.3.0.json lists both main/note-known-issue.yml and 9.4/note-known-issue.yml. // The fetcher requests note-known-issue.yml twice (same leaf URL), returning main content first // and 9.4-branch content second. The backport rule must keep the main copy and warn about the 9.4 copy. var callCount = new Dictionary(StringComparer.Ordinal); @@ -641,7 +641,7 @@ public async Task BackportCollision_MainBranchWins_WarnAndKeepMain() var path = req.RequestUri!.AbsolutePath; if (path.EndsWith("/registry.json", StringComparison.Ordinal)) return Json(RegistryJson); - if (path.EndsWith("notes-9.3.0.json", StringComparison.Ordinal)) + if (path.EndsWith("notes-elasticsearch-9.3.0.json", StringComparison.Ordinal)) return Json(/*lang=json,strict*/ """{"schema_version":1,"notes":[{"path":"main/note-known-issue.yml","bundle_seq":0},{"path":"9.4/note-known-issue.yml","bundle_seq":0}]}""" ); @@ -694,7 +694,7 @@ public async Task BackportCollision_NoMainOrMaster_KeepsOrdinalFirst() var path = req.RequestUri!.AbsolutePath; if (path.EndsWith("/registry.json", StringComparison.Ordinal)) return Json(RegistryJson); - if (path.EndsWith("notes-9.3.0.json", StringComparison.Ordinal)) + if (path.EndsWith("notes-elasticsearch-9.3.0.json", StringComparison.Ordinal)) return Json(/*lang=json,strict*/ """{"schema_version":1,"notes":[{"path":"9.4/note-known-issue.yml","bundle_seq":0},{"path":"feature/note-known-issue.yml","bundle_seq":0}]}""" ); @@ -746,7 +746,7 @@ public async Task BackportCollision_IdenticalContent_ChecksumDedupHandlesIt() var path = req.RequestUri!.AbsolutePath; if (path.EndsWith("/registry.json", StringComparison.Ordinal)) return Json(RegistryJson); - if (path.EndsWith("notes-9.3.0.json", StringComparison.Ordinal)) + if (path.EndsWith("notes-elasticsearch-9.3.0.json", StringComparison.Ordinal)) return Json(/*lang=json,strict*/ """{"schema_version":1,"notes":[{"path":"main/note-known-issue.yml","bundle_seq":0},{"path":"9.4/note-known-issue.yml","bundle_seq":0}]}""" ); diff --git a/tests/Elastic.Changelog.Tests/Reconciliation/NoteAmendReconcilerTests.cs b/tests/Elastic.Changelog.Tests/Reconciliation/NoteAmendReconcilerTests.cs index 0de2c2e96..d94cd05b3 100644 --- a/tests/Elastic.Changelog.Tests/Reconciliation/NoteAmendReconcilerTests.cs +++ b/tests/Elastic.Changelog.Tests/Reconciliation/NoteAmendReconcilerTests.cs @@ -138,7 +138,7 @@ public async Task LateNote_NoBundleAmendYet_WritesAmendNotesSidecar() _s3.Exists(PublicBucket, AmendNotesKey(parent)).Should().BeTrue("late note must produce an amend sidecar"); // Notes index must be re-written with bundle_seq = 2. - var indexKey = ChangelogKeys.NotesIndexKey(Org, Repo, Version); + var indexKey = ChangelogKeys.NotesIndexKey(Org, Repo, Product, Version); _s3.Exists(PublicBucket, indexKey).Should().BeTrue("notes index must be re-written with bundle_seq values"); var index = ReadNotesIndex(_s3.ContentOf(PublicBucket, indexKey)); index.Notes.Should().ContainSingle().Which.BundleSeq.Should().Be(2); @@ -165,7 +165,7 @@ public async Task NoteShippedInParent_NoAmendWritten_SeqIsOne() _s3.Exists(PublicBucket, AmendNotesKey(parent)).Should().BeFalse("note already in parent → no amend needed"); // bundle_seq must be 1 (shipped in original bundle). - var indexKey = ChangelogKeys.NotesIndexKey(Org, Repo, Version); + var indexKey = ChangelogKeys.NotesIndexKey(Org, Repo, Product, Version); var index = ReadNotesIndex(_s3.ContentOf(PublicBucket, indexKey)); index.Notes.Should().ContainSingle().Which.BundleSeq.Should().Be(1); } @@ -204,7 +204,7 @@ public async Task NoteShippedInHumanAmend_NoAmendNotesWritten_SeqIsOne() .Should() .BeFalse("note already in human amend → reconciler amend-notes must not be created"); - var indexKey = ChangelogKeys.NotesIndexKey(Org, Repo, Version); + var indexKey = ChangelogKeys.NotesIndexKey(Org, Repo, Product, Version); var index = ReadNotesIndex(_s3.ContentOf(PublicBucket, indexKey)); index.Notes.Should().ContainSingle().Which.BundleSeq.Should().Be(1); } @@ -231,7 +231,7 @@ public async Task ParentBundleHasNoFileAnnotations_Skipped_SeqRemainsZero() _s3.Exists(PublicBucket, AmendNotesKey(parent)).Should().BeFalse("unknown shipped state → skip, no amend"); // Notes index is still re-written, but bundle_seq stays 0. - var indexKey = ChangelogKeys.NotesIndexKey(Org, Repo, Version); + var indexKey = ChangelogKeys.NotesIndexKey(Org, Repo, Product, Version); var index = ReadNotesIndex(_s3.ContentOf(PublicBucket, indexKey)); index.Notes.Should().ContainSingle().Which.BundleSeq.Should().Be(0); } @@ -422,7 +422,7 @@ public async Task NoBundleForVersion_NoAmend_SeqRemainsZero() _s3.Puts.Should().NotContain(p => p.Key.Contains("amend-notes"), "no matching bundle → no amend possible"); // Notes index re-written with bundle_seq = 0. - var indexKey = ChangelogKeys.NotesIndexKey(Org, Repo, Version); + var indexKey = ChangelogKeys.NotesIndexKey(Org, Repo, Product, Version); var index = ReadNotesIndex(_s3.ContentOf(PublicBucket, indexKey)); index.Notes.Should().ContainSingle().Which.BundleSeq.Should().Be(0); } diff --git a/tests/Elastic.Changelog.Tests/Reconciliation/NotesIndexReconcilerTests.cs b/tests/Elastic.Changelog.Tests/Reconciliation/NotesIndexReconcilerTests.cs index 06d2f68e3..949867233 100644 --- a/tests/Elastic.Changelog.Tests/Reconciliation/NotesIndexReconcilerTests.cs +++ b/tests/Elastic.Changelog.Tests/Reconciliation/NotesIndexReconcilerTests.cs @@ -57,11 +57,7 @@ private static ChangelogScope NotesScope(string org = "elastic", string repo = " private void SeedNote(string branch, string fileName, string yaml) => _s3.Seed(PublicBucket, $"changelog/elastic/elasticsearch/{branch}/{fileName}", yaml); - private NotesIndex ReadIndex(string version) => - JsonSerializer.Deserialize( - _s3.ContentOf(PublicBucket, ChangelogKeys.NotesIndexKey("elastic", "elasticsearch", version)), - NotesIndexJsonContext.Default.NotesIndex - )!; + private NotesIndex ReadIndex(string version) => ReadProductIndex("elasticsearch", version); private NotesIndex ReadProductIndex(string product, string version) => JsonSerializer.Deserialize( @@ -112,15 +108,11 @@ public async Task ReconcileRepo_SingleNote_WritesIndex() .Count .Should() .BeGreaterThan(0, $"reconciler should have written the index; ListCalls={_s3.ListCalls} Gets={_s3.Gets.Count}"); - var index = ReadIndex("9.0.0"); - Paths(index).Should().BeEquivalentTo(["main/note-slow-rollover.yml"]); - index.Product.Should().BeNull(); - index.Version.Should().BeNull(); - var productIndex = ReadProductIndex("elasticsearch", "9.0.0"); Paths(productIndex).Should().BeEquivalentTo(["main/note-slow-rollover.yml"]); productIndex.Product.Should().Be("elasticsearch"); productIndex.Version.Should().Be("9.0.0"); + _s3.Exists(PublicBucket, ChangelogKeys.NotesIndexKey("elastic", "elasticsearch", "9.0.0")).Should().BeFalse(); } [Fact] @@ -282,7 +274,7 @@ public async Task ReconcileRepo_TwoProductsSameVersion_ProductIndexesAreIsolated Paths(ReadProductIndex("elasticsearch", "9.0.0")).Should().BeEquivalentTo(["main/note-slow-rollover.yml"]); Paths(ReadProductIndex("kibana", "9.0.0")).Should().BeEquivalentTo(["main/note-kibana.yml"]); - Paths(ReadIndex("9.0.0")).Should().BeEquivalentTo(["main/note-kibana.yml", "main/note-slow-rollover.yml"]); + _s3.Exists(PublicBucket, ChangelogKeys.NotesIndexKey("elastic", "elasticsearch", "9.0.0")).Should().BeFalse(); } [Fact] @@ -295,14 +287,21 @@ public async Task ReconcileRepo_ProductScopedStale_ReturnedAsEmptyBucketWithoutD /*lang=json,strict*/ """{"schema_version":1,"product":"kibana","version":"9.0.0","notes":[]}""" ); + _s3.Seed( + PublicBucket, + ChangelogKeys.NotesIndexKey("elastic", "elasticsearch", "9.0.0"), + /*lang=json,strict*/ + """{"schema_version":1,"notes":[]}""" + ); SeedNote("main", "note-slow-rollover.yml", NoteYaml); var map = await _reconciler.ReconcileRepoAsync(NotesScope(), TestContext.Current.CancellationToken); - _s3.Exists(PublicBucket, ChangelogKeys.NotesIndexKey("elastic", "elasticsearch", "9.0.0")).Should().BeTrue(); _s3.Exists(PublicBucket, ChangelogKeys.NotesIndexKey("elastic", "elasticsearch", "elasticsearch", "9.0.0")).Should().BeTrue(); _s3.Exists(PublicBucket, kibanaKey).Should().BeTrue(); + _s3.Exists(PublicBucket, ChangelogKeys.NotesIndexKey("elastic", "elasticsearch", "9.0.0")).Should().BeFalse(); _s3.Deletes.Should().NotContain(d => d.Key == kibanaKey); + _s3.Deletes.Select(d => d.Key).Should().Contain(ChangelogKeys.NotesIndexKey("elastic", "elasticsearch", "9.0.0")); map.Should().ContainKey("kibana"); map["kibana"].Should().ContainKey("9.0.0"); map["kibana"]["9.0.0"].Should().BeEmpty(); diff --git a/tests/Elastic.Changelog.Tests/Scrubbing/ScrubberProcessorTests.cs b/tests/Elastic.Changelog.Tests/Scrubbing/ScrubberProcessorTests.cs index 915c3d931..7aeb9aeb9 100644 --- a/tests/Elastic.Changelog.Tests/Scrubbing/ScrubberProcessorTests.cs +++ b/tests/Elastic.Changelog.Tests/Scrubbing/ScrubberProcessorTests.cs @@ -512,8 +512,9 @@ public async Task Process_NoteFile_ScrubbedAndNotesReconcileTriggered() failed.Should().BeEmpty(); // The note was scrubbed and copied to the public bucket _s3.ContentOf(PublicBucket, "changelog/elastic/elasticsearch/main/note-rollover.yml").Should().StartWith("scrubbed:"); - // The notes index was written (reconciler dual-writes notes-9.0.0.json and notes-elasticsearch-9.0.0.json) - _s3.Exists(PublicBucket, "changelog/elastic/elasticsearch/notes-9.0.0.json").Should().BeTrue(); + // The notes index was written (product-scoped notes-elasticsearch-9.0.0.json). + // Version-only notes-9.0.0.json is no longer written. + _s3.Exists(PublicBucket, "changelog/elastic/elasticsearch/notes-9.0.0.json").Should().BeFalse(); _s3.Exists(PublicBucket, "changelog/elastic/elasticsearch/notes-elasticsearch-9.0.0.json").Should().BeTrue(); } diff --git a/tests/Elastic.Documentation.Configuration.Tests/ReleaseNotes/CdnChangelogEntryFetcherTests.cs b/tests/Elastic.Documentation.Configuration.Tests/ReleaseNotes/CdnChangelogEntryFetcherTests.cs index d1ed7999b..8dd5ad4de 100644 --- a/tests/Elastic.Documentation.Configuration.Tests/ReleaseNotes/CdnChangelogEntryFetcherTests.cs +++ b/tests/Elastic.Documentation.Configuration.Tests/ReleaseNotes/CdnChangelogEntryFetcherTests.cs @@ -294,10 +294,7 @@ public async Task FetchNotesAsync_IndexAbsent_ReturnsEmptyWithNoError() entries.Should().BeEmpty(); errors.Should().BeEmpty("a missing index is expected for targets with no notes"); - handler - .RequestedPaths - .Should() - .Equal("/changelog/elastic/elasticsearch/notes-elasticsearch-9.0.0.json", "/changelog/elastic/elasticsearch/notes-9.0.0.json"); + handler.RequestedPaths.Should().Equal("/changelog/elastic/elasticsearch/notes-elasticsearch-9.0.0.json"); } [Fact] @@ -385,18 +382,11 @@ public async Task FetchNotesAsync_EmptyIndex_ReturnsEmpty() } [Fact] - public async Task FetchNotesAsync_ProductIndex404_FallsBackToLegacy() + public async Task FetchNotesAsync_ProductIndex404_DoesNotFetchLegacy() { var handler = new StubHandler(req => { var path = req.RequestUri!.AbsolutePath; - if ( - path.EndsWith("/notes-9.0.0.json", StringComparison.Ordinal) - && !path.Contains("notes-elasticsearch-", StringComparison.Ordinal) - ) - return Json(/*lang=json,strict*/ - """{"schema_version":1,"notes":[{"path":"main/note-slow-rollover.yml","bundle_seq":0}]}""" - ); if (path.EndsWith("note-slow-rollover.yml", StringComparison.Ordinal)) return Yaml(SampleEntry); return new HttpResponseMessage(HttpStatusCode.NotFound); @@ -414,10 +404,9 @@ public async Task FetchNotesAsync_ProductIndex404_FallsBackToLegacy() TestContext.Current.CancellationToken ); + entries.Should().BeEmpty(); errors.Should().BeEmpty(); - entries.Select(e => e.FileName).Should().BeEquivalentTo("main/note-slow-rollover.yml"); - handler.RequestedPaths.Should().Contain(p => p.EndsWith("/notes-elasticsearch-9.0.0.json", StringComparison.Ordinal)); - handler.RequestedPaths.Should().Contain(p => p.EndsWith("/notes-9.0.0.json", StringComparison.Ordinal)); + handler.RequestedPaths.Should().ContainSingle().Which.Should().EndWith("/notes-elasticsearch-9.0.0.json"); } [Fact]