From 051054eeba90203012bf7a45d0c47b4aa1c3b79c Mon Sep 17 00:00:00 2001 From: Hawkynt Date: Sun, 6 Sep 2026 17:11:18 +0200 Subject: [PATCH] * edit GOB v2 by rewriting its trailing directory --- Compression.Tests/Gob/GobInPlaceEditTests.cs | 240 ++++++++++++++ .../FileFormat.Gob/GobFormatDescriptor.cs | 201 +++++++----- .../FileFormat.Gob/GobInPlaceModifier.cs | 297 ++++++++++++++++++ 3 files changed, 663 insertions(+), 75 deletions(-) create mode 100644 Compression.Tests/Gob/GobInPlaceEditTests.cs create mode 100644 FileFormats/FileFormat.Gob/GobInPlaceModifier.cs diff --git a/Compression.Tests/Gob/GobInPlaceEditTests.cs b/Compression.Tests/Gob/GobInPlaceEditTests.cs new file mode 100644 index 000000000..948b2b23b --- /dev/null +++ b/Compression.Tests/Gob/GobInPlaceEditTests.cs @@ -0,0 +1,240 @@ +#pragma warning disable CS1591 +using System.Buffers.Binary; +using System.Text; +using Compression.Registry; +using FileFormat.Gob; + +namespace Compression.Tests.Gob; + +[TestFixture] +public sealed class GobInPlaceEditTests { + private const long IoBudget = 128 * 1024; + private const int DirectoryEntrySize = 136; + private const int NameFieldSize = 128; + + [Test, Category("ByteIdentity")] + public void Add_WritesNewPayloadAtOldDirectoryOffset_AndKeepsSurvivorOffset() { + var keep = Pattern(8192, 7); + var original = Build(("data\\keep.bin", keep)); + var oldDirectoryOffset = DirectoryOffset(original); + var oldKeepOffset = Entries(original)["data\\keep.bin"].Offset; + + using var stream = Load(original); + GobInPlaceModifier.AddFile(stream, "data\\new.bin", "tiny"u8.ToArray()); + var result = stream.ToArray(); + var entries = Entries(result); + + Assert.Multiple(() => { + Assert.That(entries["data\\new.bin"].Offset, Is.EqualTo(oldDirectoryOffset)); + Assert.That(entries["data\\keep.bin"].Offset, Is.EqualTo(oldKeepOffset)); + Assert.That(Read(result, "data\\keep.bin"), Is.EqualTo(keep)); + Assert.That(Read(result, "data\\new.bin"), Is.EqualTo("tiny"u8.ToArray())); + }); + } + + [Test, Category("Performance")] + public void DescriptorAdd_DoesNotReadOrRewriteFourMiBSibling() { + var keep = Pattern(4 * 1024 * 1024, 13); + var original = Build(("large\\keep.bin", keep), ("small.txt", "seed"u8.ToArray())); + var oldOffset = Entries(original)["large\\keep.bin"].Offset; + + using var inner = Load(original); + using var counted = new CountingStream(inner); + new GobFormatDescriptor().Add(counted, [ArchiveInputInfo.InMemory("added.txt", "small"u8.ToArray())]); + var reads = counted.BytesRead; + var writes = counted.BytesWritten; + var result = inner.ToArray(); + + Assert.Multiple(() => { + Assert.That(reads, Is.LessThan(IoBudget), $"add read {reads} archive bytes"); + Assert.That(writes, Is.LessThan(IoBudget), $"add wrote {writes} archive bytes"); + Assert.That(Entries(result)["large\\keep.bin"].Offset, Is.EqualTo(oldOffset)); + Assert.That(Read(result, "large\\keep.bin"), Is.EqualTo(keep)); + Assert.That(Read(result, "added.txt"), Is.EqualTo("small"u8.ToArray())); + }); + } + + [Test, Category("Performance")] + public void DescriptorReplace_WipesOldPayloadWithoutTouchingLargeSibling() { + var keep = Pattern(4 * 1024 * 1024, 29); + var victim = Pattern(4096, 31); + var original = Build(("keep.bin", keep), ("victim.bin", victim)); + var before = Entries(original); + var keepOffset = before["keep.bin"].Offset; + var victimEntry = before["victim.bin"]; + + using var inner = Load(original); + using var counted = new CountingStream(inner); + new GobFormatDescriptor().Add(counted, [ArchiveInputInfo.InMemory("victim.bin", "replacement"u8.ToArray())]); + var reads = counted.BytesRead; + var writes = counted.BytesWritten; + var result = inner.ToArray(); + + Assert.Multiple(() => { + Assert.That(reads, Is.LessThan(IoBudget)); + Assert.That(writes, Is.LessThan(IoBudget)); + Assert.That(Entries(result)["keep.bin"].Offset, Is.EqualTo(keepOffset)); + Assert.That(Read(result, "keep.bin"), Is.EqualTo(keep)); + Assert.That(Read(result, "victim.bin"), Is.EqualTo("replacement"u8.ToArray())); + Assert.That(result.AsSpan((int)victimEntry.Offset, (int)victimEntry.Size).ToArray(), Is.All.EqualTo((byte)0)); + }); + } + + [Test, Category("Performance")] + public void DescriptorRemove_RewritesDirectoryAndWipesOnlyVictim() { + var keep = Pattern(4 * 1024 * 1024, 37); + var victim = Pattern(4096, 41); + var original = Build(("keep.bin", keep), ("victim.bin", victim)); + var before = Entries(original); + var keepOffset = before["keep.bin"].Offset; + var victimEntry = before["victim.bin"]; + + using var inner = Load(original); + using var counted = new CountingStream(inner); + new GobFormatDescriptor().Remove(counted, ["victim.bin"]); + var reads = counted.BytesRead; + var writes = counted.BytesWritten; + var result = inner.ToArray(); + + Assert.Multiple(() => { + Assert.That(reads, Is.LessThan(IoBudget)); + Assert.That(writes, Is.LessThan(IoBudget)); + Assert.That(Entries(result).ContainsKey("victim.bin"), Is.False); + Assert.That(Entries(result)["keep.bin"].Offset, Is.EqualTo(keepOffset)); + Assert.That(Read(result, "keep.bin"), Is.EqualTo(keep)); + Assert.That(result.AsSpan((int)victimEntry.Offset, (int)victimEntry.Size).ToArray(), Is.All.EqualTo((byte)0)); + }); + } + + [Test, Category("EdgeCase")] + public void RemoveMissingName_PerformsZeroWrites() { + var original = Build(("keep.bin", Pattern(1024, 3))); + using var inner = Load(original); + using var counted = new CountingStream(inner); + + Assert.That(GobInPlaceModifier.RemoveFile(counted, "ghost.bin"), Is.False); + Assert.That(counted.BytesWritten, Is.Zero); + Assert.That(inner.ToArray(), Is.EqualTo(original)); + } + + [Test, Category("EdgeCase")] + public void RemoveAlias_DoesNotWipeSharedPayload() { + var original = BuildAliased(); + var before = Entries(original); + Assert.That(before["a.bin"].Offset, Is.EqualTo(before["b.bin"].Offset)); + + using var stream = Load(original); + Assert.That(GobInPlaceModifier.RemoveFile(stream, "a.bin"), Is.True); + var result = stream.ToArray(); + + Assert.Multiple(() => { + Assert.That(Entries(result).ContainsKey("a.bin"), Is.False); + Assert.That(Read(result, "b.bin"), Is.EqualTo("shared-data"u8.ToArray())); + }); + } + + [Test, Category("Layout")] + public void DescriptorLayout_IncludesHeaderPayloadAndDirectory() { + var original = Build(("x.bin", Pattern(32, 1))); + using var stream = new MemoryStream(original, writable: false); + var blocks = new GobFormatDescriptor().EnumerateLayout(stream).ToArray(); + + Assert.Multiple(() => { + Assert.That(blocks.Any(block => block.Offset == 0 && block.Length == 12 && block.Kind == DefragBlockKind.MetadataReserved), Is.True); + Assert.That(blocks.Any(block => block.FileName == "x.bin" && block.Kind == DefragBlockKind.Used), Is.True); + Assert.That(blocks.Any(block => block.Offset == DirectoryOffset(original) && block.Kind == DefragBlockKind.MetadataReserved), Is.True); + }); + } + + private static byte[] Build(params (string Name, byte[] Data)[] entries) { + using var stream = new MemoryStream(); + using (var writer = new GobWriter(stream, leaveOpen: true)) + foreach (var (name, data) in entries) + writer.AddEntry(name, data); + return stream.ToArray(); + } + + private static byte[] BuildAliased() { + var bytes = Build(("a.bin", "shared-data"u8.ToArray())); + var directoryOffset = DirectoryOffset(bytes); + var oldLength = bytes.Length; + Array.Resize(ref bytes, oldLength + DirectoryEntrySize); + bytes.AsSpan(directoryOffset + 4, DirectoryEntrySize) + .CopyTo(bytes.AsSpan(directoryOffset + 4 + DirectoryEntrySize, DirectoryEntrySize)); + bytes.AsSpan(directoryOffset + 4 + DirectoryEntrySize + 8, NameFieldSize).Clear(); + Encoding.ASCII.GetBytes("b.bin").CopyTo(bytes, directoryOffset + 4 + DirectoryEntrySize + 8); + BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(directoryOffset, 4), 2); + return bytes; + } + + private static int DirectoryOffset(byte[] archive) + => checked((int)BinaryPrimitives.ReadUInt32LittleEndian(archive.AsSpan(8, 4))); + + private static Dictionary Entries(byte[] archive) { + using var stream = new MemoryStream(archive, writable: false); + using var reader = new GobReader(stream); + return reader.Entries.ToDictionary(entry => entry.Name, StringComparer.OrdinalIgnoreCase); + } + + private static byte[] Read(byte[] archive, string name) { + using var stream = new MemoryStream(archive, writable: false); + using var reader = new GobReader(stream); + var entry = reader.Entries.Single(entry => string.Equals(entry.Name, name, StringComparison.OrdinalIgnoreCase)); + return reader.Extract(entry); + } + + private static byte[] Pattern(int size, int seed) { + var result = new byte[size]; + for (var i = 0; i < result.Length; ++i) + result[i] = (byte)((i * 137 + seed) & 0xFF); + return result; + } + + private static MemoryStream Load(byte[] bytes) { + var stream = new MemoryStream(); + stream.Write(bytes); + stream.Position = 0; + return stream; + } + + private sealed class CountingStream(Stream inner) : Stream { + public long BytesRead { get; private set; } + public long BytesWritten { get; private set; } + public override bool CanRead => inner.CanRead; + public override bool CanSeek => inner.CanSeek; + public override bool CanWrite => inner.CanWrite; + public override long Length => inner.Length; + public override long Position { get => inner.Position; set => inner.Position = value; } + public override void Flush() => inner.Flush(); + public override long Seek(long offset, SeekOrigin origin) => inner.Seek(offset, origin); + public override void SetLength(long value) => inner.SetLength(value); + public override int Read(byte[] buffer, int offset, int count) { + var read = inner.Read(buffer, offset, count); + this.BytesRead += read; + return read; + } + public override int Read(Span buffer) { + var read = inner.Read(buffer); + this.BytesRead += read; + return read; + } + public override int ReadByte() { + var value = inner.ReadByte(); + if (value >= 0) ++this.BytesRead; + return value; + } + public override void Write(byte[] buffer, int offset, int count) { + inner.Write(buffer, offset, count); + this.BytesWritten += count; + } + public override void Write(ReadOnlySpan buffer) { + inner.Write(buffer); + this.BytesWritten += buffer.Length; + } + public override void WriteByte(byte value) { + inner.WriteByte(value); + ++this.BytesWritten; + } + protected override void Dispose(bool disposing) { } + } +} diff --git a/FileFormats/FileFormat.Gob/GobFormatDescriptor.cs b/FileFormats/FileFormat.Gob/GobFormatDescriptor.cs index 07b0721ad..c36751b8c 100644 --- a/FileFormats/FileFormat.Gob/GobFormatDescriptor.cs +++ b/FileFormats/FileFormat.Gob/GobFormatDescriptor.cs @@ -1,4 +1,5 @@ #pragma warning disable CS1591 +using System.Buffers.Binary; using Compression.Registry; using static Compression.Registry.FormatHelpers; @@ -17,123 +18,173 @@ public sealed class GobFormatDescriptor : IFormatDescriptor, IArchiveFormatOpera /// public IEnumerable EnumerateLayout(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); archive.Position = 0; - var r = new GobReader(archive); - foreach (var e in r.Entries) { - if (e.Size > 0) - yield return new DefragBlockInfo(e.Offset, e.Size, DefragBlockKind.Used, FileName: e.Name); - } + using var reader = new GobReader(archive, leaveOpen: true); + + yield return new DefragBlockInfo(0, GobConstants.HeaderSize, DefragBlockKind.MetadataReserved, FileName: "GOB header"); + foreach (var entry in reader.Entries) + if (entry.Size > 0) + yield return new DefragBlockInfo(entry.Offset, entry.Size, DefragBlockKind.Used, FileName: entry.Name); + + var directoryOffsetBytes = new byte[4]; + archive.Position = 8; + archive.ReadExactly(directoryOffsetBytes); + var directoryOffset = BinaryPrimitives.ReadUInt32LittleEndian(directoryOffsetBytes); + var directoryLength = checked(4L + (long)reader.Entries.Count * GobConstants.DirectoryEntrySize); + yield return new DefragBlockInfo(directoryOffset, directoryLength, DefragBlockKind.MetadataReserved, FileName: "GOB directory"); } - /// - /// Gets the id. - /// + /// Gets the id. public string Id => "Gob"; - /// - /// Gets the display name. - /// + + /// Gets the display name. public string DisplayName => "Lucasarts GOB"; - /// - /// Gets the category. - /// + + /// Gets the category. public FormatCategory Category => FormatCategory.Archive; - // R/W: a mutable archive. Add/Replace/Remove go through the verified extract -> - // edit -> re-create rebuild (default IArchiveModifiable); relayouting the container - // on edit is honest R/W. See FormatCapabilities.cs (WORM vs R/W). - /// - /// Gets the capabilities. - /// + + /// Gets the capabilities. public FormatCapabilities Capabilities => FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate | FormatCapabilities.CanModify | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries; - /// - /// Gets the default extension. - /// + + /// Gets the default extension. public string DefaultExtension => ".gob"; - /// - /// Gets the extensions. - /// + + /// Gets the extensions. public IReadOnlyList Extensions => [".gob", ".goo"]; - /// - /// Gets the compound extensions. - /// + + /// Gets the compound extensions. public IReadOnlyList CompoundExtensions => []; + // Trailing space is part of the GOB v2 magic — without it we would collide with // GOB v1 (Dark Forces) which is structurally different and out of scope here. - /// - /// Gets the magic signatures. - /// - public IReadOnlyList MagicSignatures => [ - new("GOB "u8.ToArray(), Confidence: 0.95) - ]; - /// - /// Gets the methods. - /// + /// Gets the magic signatures. + public IReadOnlyList MagicSignatures => [new("GOB "u8.ToArray(), Confidence: 0.95)]; + + /// Gets the methods. public IReadOnlyList Methods => [new("gob2", "GOB v2")]; - /// - /// Gets the tar compression format id. - /// + + /// Gets the tar compression format id. public string? TarCompressionFormatId => null; - /// - /// Gets the family. - /// + + /// Gets the family. public AlgorithmFamily Family => AlgorithmFamily.Archive; - /// - /// Gets the description. - /// + + /// Gets the description. public string Description => "Lucasarts archive (Jedi Knight, Outlaws)"; - /// - /// Lists the entries in the supplied container. - /// + /// Lists the entries in the supplied container. public List List(Stream stream, string? password) { - var r = new GobReader(stream); - return r.Entries.Select((e, i) => new ArchiveEntryInfo(i, e.Name, e.Size, e.Size, + using var reader = new GobReader(stream, leaveOpen: true); + return reader.Entries.Select((entry, index) => new ArchiveEntryInfo(index, entry.Name, entry.Size, entry.Size, "Stored", false, false, null)).ToList(); } - /// - /// Decodes the supplied input. - /// + /// Extracts matching entries. public void Extract(Stream stream, string outputDir, string? password, string[]? files) { - var r = new GobReader(stream); - foreach (var e in r.Entries) { - if (files != null && !MatchesFilter(e.Name, files)) continue; - WriteFile(outputDir, e.Name, r.Extract(e)); + using var reader = new GobReader(stream, leaveOpen: true); + foreach (var entry in reader.Entries) { + if (files != null && !MatchesFilter(entry.Name, files)) + continue; + WriteFile(outputDir, entry.Name, reader.Extract(entry)); } } - /// - /// Performs the create operation. - /// + /// Creates a canonical GOB v2 archive. public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { - using var w = new GobWriter(output, leaveOpen: true); + using var writer = new GobWriter(output, leaveOpen: true); foreach (var (name, data) in FlatFiles(inputs)) - w.AddEntry(name, data); + writer.AddEntry(name, data); } /// - /// Performs the defragment operation. + /// Adds or same-name replaces entries. Canonical archives with a trailing + /// directory use : changed payload bytes replace + /// the old directory and a regenerated directory follows them. Unsupported + /// layouts retain the verified extract/re-create fallback. + /// + public void Add(Stream archive, IReadOnlyList inputs) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(inputs); + var files = FilesOnly(inputs).ToList(); + if (files.Count == 0) + return; + + try { + archive.Position = 0; + GobInPlaceModifier.AddFiles(archive, files); + return; + } catch (NotSupportedException) { + if (archive.CanSeek) + archive.Position = 0; + } + + RebuildVerb.EditViaRebuild(archive, this, this, temporaryDirectory => { + foreach (var (name, data) in files) { + var relative = name.Replace('\\', Path.DirectorySeparatorChar).Replace('/', Path.DirectorySeparatorChar); + var destination = Path.Combine(temporaryDirectory, relative); + var directory = Path.GetDirectoryName(destination); + if (!string.IsNullOrEmpty(directory)) + Directory.CreateDirectory(directory); + File.WriteAllBytes(destination, data); + } + }); + } + + /// + /// Removes entries by rewriting only the trailing directory and wiping + /// unreferenced removed payload ranges. Unsupported layouts rebuild. /// + public void Remove(Stream archive, string[] entryNames) { + ArgumentNullException.ThrowIfNull(archive); + ArgumentNullException.ThrowIfNull(entryNames); + if (entryNames.Length == 0) + return; + + try { + archive.Position = 0; + GobInPlaceModifier.RemoveFiles(archive, entryNames, wipeData: true); + return; + } catch (NotSupportedException) { + if (archive.CanSeek) + archive.Position = 0; + } + + var remove = new HashSet(entryNames.Select(NormalizeMatchName), StringComparer.OrdinalIgnoreCase); + RebuildVerb.EditViaRebuild(archive, this, this, temporaryDirectory => { + foreach (var file in Directory.GetFiles(temporaryDirectory, "*", SearchOption.AllDirectories)) { + var relative = NormalizeMatchName(Path.GetRelativePath(temporaryDirectory, file)); + var separator = relative.LastIndexOf('\\'); + var leaf = separator >= 0 ? relative[(separator + 1)..] : relative; + if (remove.Contains(relative) || remove.Contains(leaf)) + File.Delete(file); + } + }); + } + + /// Rebuild-based defrag. public void Defragment(Stream archive) => this.Defragment(archive, new DefragOptions { Mode = DefragMode.ConsolidateAtStart }); - /// - /// Performs the defragment operation. - /// + /// Rebuild-based defrag per the requested mode. public void Defragment(Stream archive, DefragOptions options) { DefragRebuilder.Rebuild(archive, options, readEntries: stream => { - var r = new GobReader(stream); - return r.Entries.Select(e => (e.Name, r.Extract(e))); + using var reader = new GobReader(stream, leaveOpen: true); + return reader.Entries.Select(entry => (entry.Name, reader.Extract(entry))).ToList(); }, buildImage: files => { - using var ms = new MemoryStream(); - using (var w = new GobWriter(ms, leaveOpen: true)) { - foreach (var (n, d) in files) w.AddEntry(n, d); - } - return ms.ToArray(); + using var memory = new MemoryStream(); + using (var writer = new GobWriter(memory, leaveOpen: true)) + foreach (var (name, data) in files) + writer.AddEntry(name, data); + return memory.ToArray(); }); } + + private static string NormalizeMatchName(string name) + => name.Replace('/', '\\').TrimStart('\\'); } diff --git a/FileFormats/FileFormat.Gob/GobInPlaceModifier.cs b/FileFormats/FileFormat.Gob/GobInPlaceModifier.cs new file mode 100644 index 000000000..8e278daa4 --- /dev/null +++ b/FileFormats/FileFormat.Gob/GobInPlaceModifier.cs @@ -0,0 +1,297 @@ +using System.Buffers.Binary; +using System.Text; + +namespace FileFormat.Gob; + +/// +/// Changed-byte editor for canonical GOB v2 archives whose directory is the +/// exact physical trailer. Added/replacement payloads reuse the old directory +/// position; removal rewrites only the directory and leaves survivor payloads +/// at their original offsets. Removed bytes are wiped only when no surviving +/// directory record overlaps them. +/// +public static class GobInPlaceModifier { + private sealed record Entry(byte[] NameField, string Name, uint Offset, uint Size); + private sealed record State(uint Version, uint DirectoryOffset, List Entries); + private readonly record struct Range(long Offset, long Length); + + /// Adds or same-name replaces one stored entry. + public static void AddFile(Stream archive, string name, byte[] data, bool wipeReplacedData = true) + => AddFiles(archive, [(name, data)], wipeReplacedData); + + /// + /// Adds or same-name replaces multiple entries with one directory rewrite. + /// Structural rejection and replacement-directory serialization happen before + /// the first archive write. + /// + public static void AddFiles( + Stream archive, + IReadOnlyList<(string Name, byte[] Data)> files, + bool wipeReplacedData = true) { + ValidateWritable(archive); + ArgumentNullException.ThrowIfNull(files); + if (files.Count == 0) + return; + + var requests = new List<(string Name, byte[] NameField, byte[] Data)>(files.Count); + var requestNames = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var (name, data) in files) { + ArgumentNullException.ThrowIfNull(data); + var nameField = EncodeName(name, out var normalized); + if (!requestNames.Add(NormalizeForMatch(normalized))) + throw new ArgumentException($"Duplicate GOB mutation name '{normalized}'.", nameof(files)); + requests.Add((normalized, nameField, data)); + } + + var state = ReadCanonicalState(archive); + var planned = new List(state.Entries); + var targetIndex = BuildUniqueTargetIndex(planned, requests.Select(request => request.Name)); + var appendOffset = (long)state.DirectoryOffset; + var wipes = new List(); + + foreach (var request in requests) { + if (appendOffset > uint.MaxValue || request.Data.LongLength > uint.MaxValue || appendOffset + request.Data.LongLength > uint.MaxValue) + throw new NotSupportedException("GOB v2 uses 32-bit payload offsets and sizes."); + + var key = NormalizeForMatch(request.Name); + var replacement = new Entry(request.NameField, request.Name, checked((uint)appendOffset), checked((uint)request.Data.Length)); + if (targetIndex.TryGetValue(key, out var index)) { + var old = planned[index]; + if (wipeReplacedData && old.Size > 0) + wipes.Add(new Range(old.Offset, old.Size)); + planned[index] = replacement; + } else { + targetIndex.Add(key, planned.Count); + planned.Add(replacement); + } + appendOffset = checked(appendOffset + request.Data.LongLength); + } + + if (appendOffset > uint.MaxValue) + throw new NotSupportedException("GOB v2 directory offset exceeds UInt32."); + var directoryBytes = SerializeDirectory(planned); + var newLength = checked(appendOffset + directoryBytes.LongLength); + var safeWipes = PlanSafeWipes(wipes, planned); + var directoryOffsetPatch = new byte[4]; + BinaryPrimitives.WriteUInt32LittleEndian(directoryOffsetPatch, checked((uint)appendOffset)); + + archive.Position = state.DirectoryOffset; + foreach (var request in requests) + if (request.Data.Length > 0) + archive.Write(request.Data); + archive.Write(directoryBytes); + archive.Position = 8; + archive.Write(directoryOffsetPatch); + archive.SetLength(newLength); + ZeroRanges(archive, safeWipes); + archive.Flush(); + } + + /// Removes one entry. Returns false without writing when it is absent. + public static bool RemoveFile(Stream archive, string name, bool wipeData = true) + => RemoveFiles(archive, [name], wipeData) > 0; + + /// + /// Removes matching full paths or leaf names. Survivor payloads are not moved; + /// only the directory and unreferenced removed payload ranges are written. + /// + public static int RemoveFiles(Stream archive, IReadOnlyCollection names, bool wipeData = true) { + ValidateWritable(archive); + ArgumentNullException.ThrowIfNull(names); + if (names.Count == 0) + return 0; + + var requested = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var name in names) + if (!string.IsNullOrWhiteSpace(name)) + requested.Add(NormalizeForMatch(name)); + if (requested.Count == 0) + return 0; + + var state = ReadCanonicalState(archive); + var kept = new List(state.Entries.Count); + var wipes = new List(); + var removed = 0; + foreach (var entry in state.Entries) { + if (!Matches(entry.Name, requested)) { + kept.Add(entry); + continue; + } + ++removed; + if (wipeData && entry.Size > 0) + wipes.Add(new Range(entry.Offset, entry.Size)); + } + if (removed == 0) + return 0; + + var directoryBytes = SerializeDirectory(kept); + var newLength = checked((long)state.DirectoryOffset + directoryBytes.LongLength); + var safeWipes = PlanSafeWipes(wipes, kept); + + archive.Position = state.DirectoryOffset; + archive.Write(directoryBytes); + archive.SetLength(newLength); + ZeroRanges(archive, safeWipes); + archive.Flush(); + return removed; + } + + private static State ReadCanonicalState(Stream archive) { + if (archive.Length < GobConstants.HeaderSize) + throw new InvalidDataException("GOB stream is shorter than its fixed header."); + + Span header = stackalloc byte[GobConstants.HeaderSize]; + archive.Position = 0; + archive.ReadExactly(header); + if (!header[..4].SequenceEqual(GobConstants.Magic)) + throw new InvalidDataException("Invalid GOB v2 magic."); + + var version = BinaryPrimitives.ReadUInt32LittleEndian(header[4..8]); + var directoryOffset = BinaryPrimitives.ReadUInt32LittleEndian(header[8..12]); + if (directoryOffset < GobConstants.HeaderSize || directoryOffset > archive.Length - 4) + throw new InvalidDataException("GOB directory offset is out of range."); + + Span countBytes = stackalloc byte[4]; + archive.Position = directoryOffset; + archive.ReadExactly(countBytes); + var count = BinaryPrimitives.ReadUInt32LittleEndian(countBytes); + if (count > int.MaxValue) + throw new NotSupportedException("GOB directory entry count exceeds the managed limit."); + var directoryLength = checked(4L + (long)count * GobConstants.DirectoryEntrySize); + if ((long)directoryOffset + directoryLength != archive.Length) + throw new NotSupportedException("Changed-byte GOB editing requires the directory to be the exact physical trailer."); + + var entries = new List((int)count); + var record = new byte[GobConstants.DirectoryEntrySize]; + for (var i = 0; i < (int)count; ++i) { + archive.ReadExactly(record); + var offset = BinaryPrimitives.ReadUInt32LittleEndian(record.AsSpan(0, 4)); + var size = BinaryPrimitives.ReadUInt32LittleEndian(record.AsSpan(4, 4)); + var rawName = record.AsSpan(8, GobConstants.NameFieldSize).ToArray(); + var name = DecodeName(rawName); + if (offset < GobConstants.HeaderSize || (long)offset + size > directoryOffset) + throw new NotSupportedException( + $"Changed-byte GOB editing requires payload '{name}' to lie wholly before the trailing directory."); + entries.Add(new Entry(rawName, name, offset, size)); + } + + return new State(version, directoryOffset, entries); + } + + private static Dictionary BuildUniqueTargetIndex( + IReadOnlyList entries, + IEnumerable targetNames) { + var targets = new HashSet(targetNames.Select(NormalizeForMatch), StringComparer.OrdinalIgnoreCase); + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + for (var i = 0; i < entries.Count; ++i) { + var key = NormalizeForMatch(entries[i].Name); + if (!targets.Contains(key)) + continue; + if (!result.TryAdd(key, i)) + throw new NotSupportedException( + $"GOB contains duplicate entries named '{entries[i].Name}'; replacement semantics are ambiguous."); + } + return result; + } + + private static byte[] SerializeDirectory(IReadOnlyList entries) { + var result = new byte[checked(4 + entries.Count * GobConstants.DirectoryEntrySize)]; + BinaryPrimitives.WriteUInt32LittleEndian(result.AsSpan(0, 4), checked((uint)entries.Count)); + for (var i = 0; i < entries.Count; ++i) { + var position = 4 + i * GobConstants.DirectoryEntrySize; + BinaryPrimitives.WriteUInt32LittleEndian(result.AsSpan(position, 4), entries[i].Offset); + BinaryPrimitives.WriteUInt32LittleEndian(result.AsSpan(position + 4, 4), entries[i].Size); + entries[i].NameField.CopyTo(result, position + 8); + } + return result; + } + + private static byte[] EncodeName(string name, out string normalized) { + ArgumentException.ThrowIfNullOrWhiteSpace(name); + normalized = name.Replace('/', '\\').TrimStart('\\'); + if (normalized.Length == 0 || normalized.EndsWith('\\')) + throw new ArgumentException("GOB entry name must identify a file.", nameof(name)); + foreach (var part in normalized.Split('\\')) + if (part.Length == 0 || part is "." or "..") + throw new ArgumentException("Unsafe GOB entry path.", nameof(name)); + if (normalized.Any(c => c == '\0' || c > '\x7F')) + throw new ArgumentException("GOB v2 entry names are 7-bit archive paths.", nameof(name)); + + var bytes = Encoding.ASCII.GetBytes(normalized); + if (bytes.Length > GobConstants.MaxNameLength) + throw new ArgumentException($"GOB entry names are limited to {GobConstants.MaxNameLength} bytes.", nameof(name)); + var field = new byte[GobConstants.NameFieldSize]; + bytes.CopyTo(field, 0); + return field; + } + + private static string DecodeName(byte[] field) { + var terminator = Array.IndexOf(field, (byte)0); + var length = terminator >= 0 ? terminator : field.Length; + return Encoding.ASCII.GetString(field, 0, length); + } + + private static string NormalizeForMatch(string name) + => name.Replace('/', '\\').TrimStart('\\'); + + private static bool Matches(string path, HashSet requested) { + var normalized = NormalizeForMatch(path); + if (requested.Contains(normalized)) + return true; + var separator = normalized.LastIndexOf('\\'); + return requested.Contains(separator >= 0 ? normalized[(separator + 1)..] : normalized); + } + + private static List PlanSafeWipes(IEnumerable candidates, IReadOnlyList survivors) { + var safe = new List(); + foreach (var candidate in candidates) { + var overlapsLive = survivors.Any(entry => entry.Size > 0 && Overlaps(candidate, new Range(entry.Offset, entry.Size))); + if (!overlapsLive) + safe.Add(candidate); + } + if (safe.Count < 2) + return safe; + + safe.Sort((left, right) => left.Offset.CompareTo(right.Offset)); + var merged = new List(); + var current = safe[0]; + for (var i = 1; i < safe.Count; ++i) { + var next = safe[i]; + var currentEnd = checked(current.Offset + current.Length); + if (next.Offset <= currentEnd) { + var end = Math.Max(currentEnd, checked(next.Offset + next.Length)); + current = new Range(current.Offset, checked(end - current.Offset)); + } else { + merged.Add(current); + current = next; + } + } + merged.Add(current); + return merged; + } + + private static bool Overlaps(Range left, Range right) + => left.Length > 0 && right.Length > 0 && + left.Offset < right.Offset + right.Length && right.Offset < left.Offset + left.Length; + + private static void ZeroRanges(Stream archive, IReadOnlyList ranges) { + if (ranges.Count == 0) + return; + var zeroes = new byte[64 * 1024]; + foreach (var range in ranges) { + archive.Position = range.Offset; + var remaining = range.Length; + while (remaining > 0) { + var count = (int)Math.Min(remaining, zeroes.Length); + archive.Write(zeroes, 0, count); + remaining -= count; + } + } + } + + private static void ValidateWritable(Stream archive) { + ArgumentNullException.ThrowIfNull(archive); + if (!archive.CanRead || !archive.CanWrite || !archive.CanSeek) + throw new NotSupportedException("Changed-byte GOB editing requires a seekable, readable, writable stream."); + } +}