diff --git a/Compression.Tests/Tux2/Tux2PlannedDefragTests.cs b/Compression.Tests/Tux2/Tux2PlannedDefragTests.cs deleted file mode 100644 index 6eed6b6ae..000000000 --- a/Compression.Tests/Tux2/Tux2PlannedDefragTests.cs +++ /dev/null @@ -1,109 +0,0 @@ -#pragma warning disable CS1591 -using Compression.Registry; -using FileSystem.Tux2; - -namespace Compression.Tests.Tux2; - -/// -/// Tux2 lays a container out again by moving whole records, and on one this -/// writer produced the answer is usually that nothing is out of place. -/// -/// -/// A record's data sits behind the header naming it at an offset nothing -/// records — the reader finds the next record by adding this one's length to a -/// cursor — so nothing has to be repointed, and the only layout the walk -/// reaches is one with the records in some order and nothing between them. -/// Removing a file writes the container out packed, so a pass over ours finds -/// no gap; what it is for is a container that arrived from somewhere else. -/// -[TestFixture] -public class Tux2PlannedDefragTests { - - private static byte[] Payload(int seed, int length) { - var data = new byte[length]; - for (var i = 0; i < length; ++i) data[i] = (byte)((i * 13 + seed * 29) % 251); - return data; - } - - private static MemoryStream Volume(out Dictionary files) { - var work = Path.Combine(Path.GetTempPath(), "cwb_Tux2_" + Guid.NewGuid().ToString("N")[..8]); - Directory.CreateDirectory(work); - files = new Dictionary(StringComparer.Ordinal); - try { - var inputs = new List(); - for (var k = 0; k < 5; ++k) { - var data = Payload(k, 3000 + k * 1500); - var path = Path.Combine(work, $"F{k}.BIN"); - File.WriteAllBytes(path, data); - inputs.Add(new ArchiveInputInfo(path, $"F{k}.BIN", false)); - files[$"F{k}.BIN"] = data; - } - - var image = new MemoryStream(); - new Tux2FormatDescriptor().Create(image, inputs, new FormatCreateOptions()); - return image; - } finally { - try { Directory.Delete(work, true); } catch { /* scratch is gone already */ } - } - } - - private static Dictionary ReadBack(MemoryStream image) { - image.Position = 0; - using var reader = new Tux2Reader(image); - return reader.Entries - .Where(e => e.Size > 0 && !e.Name.StartsWith("FULL.", StringComparison.Ordinal) - && e.Name != "metadata.ini") - .ToDictionary(e => e.Name, reader.Extract, StringComparer.Ordinal); - } - - [Test, Category("RoundTrip")] - [TestCase(DefragMode.ConsolidateAtStart)] - [TestCase(DefragMode.ConsolidateAtEnd)] - public void Defragment_KeepsEveryPayloadAndTheContainersSize(DefragMode mode) { - using var image = Volume(out var files); - var size = image.Length; - - image.Position = 0; - new Tux2FormatDescriptor().Defragment(image, new DefragOptions { Mode = mode }); - Assert.That(image.Length, Is.EqualTo(size), "a container keeps its size"); - - var read = ReadBack(image); - foreach (var (name, data) in files) { - Assert.That(read.Keys, Does.Contain(name), $"{name} must still be in the container"); - Assert.That(read[name], Is.EqualTo(data), $"{name} must read back byte for byte"); - } - } - - [Test] - public void Defragment_FindsNothingToMoveOnOneOfOurOwn() { - using var image = Volume(out _); - var before = image.ToArray(); - - image.Position = 0; - new Tux2FormatDescriptor().Defragment(image, - new DefragOptions { Mode = DefragMode.ConsolidateAtStart }); - - // The writer packs its records, so front-packing has nothing to do — and - // doing nothing must mean writing nothing. - Assert.That(image.ToArray(), Is.EqualTo(before), - "a container already packed must come back byte for byte"); - } - - [Test] - public void Defragment_LeavesTheRecordsWithNothingBetweenThem() { - using var image = Volume(out _); - image.Position = 0; - new Tux2FormatDescriptor().Defragment(image, new DefragOptions { Mode = DefragMode.ConsolidateAtEnd }); - - image.Position = 0; - var records = Tux2RecordMap.Enumerate(image) - .Where(e => e.Kind == DefragBlockKind.Used) - .OrderBy(e => e.Offset) - .ToList(); - Assert.That(records, Is.Not.Empty, "the container must still describe its records"); - - for (var i = 1; i < records.Count; ++i) - Assert.That(records[i].Offset, Is.EqualTo(records[i - 1].Offset + records[i - 1].Length), - "a gap between records is what the walk cannot get past"); - } -} diff --git a/Compression.Tests/Tux2/Tux2SchemaTests.cs b/Compression.Tests/Tux2/Tux2SchemaTests.cs deleted file mode 100644 index aafcd8634..000000000 --- a/Compression.Tests/Tux2/Tux2SchemaTests.cs +++ /dev/null @@ -1,47 +0,0 @@ -using Compression.Registry; -using FileSystem.Tux2; - -namespace Compression.Tests.Tux2; - -/// -/// Schema-knob contract tests for : proves the -/// published Version option is a real knob the writer honours and the -/// reader reads back. -/// -[TestFixture] -public class Tux2SchemaTests { - - [Test, Category("Spec")] - public void Descriptor_ExposesVersionSchema() { - var d = new Tux2FormatDescriptor(); - Assert.That(d, Is.InstanceOf()); - Assert.That(d, Is.InstanceOf()); - var schema = ((IFormatOptionsSchema)d).OptionsSchema; - Assert.That(schema.Any(o => o.Key == "Version"), Is.True); - } - - [Test, Category("HappyPath")] - public void Create_Version_TakesEffectAndFilesRoundTrip() { - var d = new Tux2FormatDescriptor(); - var payload = "tux2 version knob"u8.ToArray(); - using var ms = new MemoryStream(); - d.Create(ms, - [ArchiveInputInfo.InMemory("note.txt", payload)], - new FormatCreateOptions { FormatSpecific = new Dictionary { ["Version"] = "7" } }); - - ms.Position = 0; - var r = new Tux2Reader(ms); - Assert.That(r.Version, Is.EqualTo(7u), "Version knob must land in the header."); - var entry = r.Entries.Single(e => e.Name == "note.txt"); - Assert.That(r.Extract(entry), Is.EqualTo(payload), "File must round-trip."); - } - - [Test, Category("Equivalence")] - public void Create_DefaultVersion_IsOne() { - var d = new Tux2FormatDescriptor(); - using var ms = new MemoryStream(); - d.Create(ms, [ArchiveInputInfo.InMemory("a.txt", "x"u8.ToArray())], new FormatCreateOptions()); - ms.Position = 0; - Assert.That(new Tux2Reader(ms).Version, Is.EqualTo(1u)); - } -} diff --git a/Compression.Tests/Tux2/Tux2Tests.cs b/Compression.Tests/Tux2/Tux2Tests.cs index 64dc3825d..911f17f16 100644 --- a/Compression.Tests/Tux2/Tux2Tests.cs +++ b/Compression.Tests/Tux2/Tux2Tests.cs @@ -1,142 +1,90 @@ using System.Buffers.Binary; using System.Text; using Compression.Registry; +using FileSystem.Tux2; namespace Compression.Tests.Tux2; [TestFixture] public class Tux2Tests { - - // Build a minimal synthetic TUX2 image with two embedded files. - private static byte[] BuildSyntheticImage((string name, byte[] data)[] files) { - using var ms = new MemoryStream(); - ms.Write(FileSystem.Tux2.Tux2Reader.Magic); - var hdr = new byte[8]; - BinaryPrimitives.WriteUInt32LittleEndian(hdr.AsSpan(0, 4), 1u); // version - BinaryPrimitives.WriteUInt32LittleEndian(hdr.AsSpan(4, 4), (uint)files.Length); - ms.Write(hdr); - foreach (var (name, data) in files) { - var nameBytes = Encoding.UTF8.GetBytes(name); - var rec = new byte[2]; - BinaryPrimitives.WriteUInt16LittleEndian(rec, (ushort)nameBytes.Length); - ms.Write(rec); - ms.Write(nameBytes); - var sizeRec = new byte[4]; - BinaryPrimitives.WriteUInt32LittleEndian(sizeRec, (uint)data.Length); - ms.Write(sizeRec); - ms.Write(data); - } - return ms.ToArray(); - } - - [Test, Category("HappyPath")] - public void Descriptor_Properties() { - var d = new FileSystem.Tux2.Tux2FormatDescriptor(); - Assert.That(d.Id, Is.EqualTo("Tux2")); - Assert.That(d.DisplayName, Is.EqualTo("TUX2")); - Assert.That(d.Extensions, Does.Contain(".tux2")); - Assert.That(d.Category, Is.EqualTo(FormatCategory.Archive)); - Assert.That(d.MagicSignatures, Has.Count.EqualTo(1)); - Assert.That(d.MagicSignatures[0].Offset, Is.EqualTo(0)); - } - - [Test, Category("HappyPath")] - public void Read_SyntheticImage() { - var img = BuildSyntheticImage([ - ("hello.txt", "Hello TUX2!"u8.ToArray()), - ("data.bin", new byte[] { 1, 2, 3, 4, 5 }), - ]); - using var ms = new MemoryStream(img); - var r = new FileSystem.Tux2.Tux2Reader(ms); - Assert.That(r.ValidHeader, Is.True); - Assert.That(r.Version, Is.EqualTo(1u)); - Assert.That(r.FileCount, Is.EqualTo(2u)); - - // Entries: FULL.tux2 + metadata.ini + 2 files - Assert.That(r.Entries, Has.Count.EqualTo(4)); - var byName = r.Entries.ToDictionary(e => e.Name); - Assert.That(byName.ContainsKey("FULL.tux2"), Is.True); - Assert.That(byName.ContainsKey("metadata.ini"), Is.True); - Assert.That(byName.ContainsKey("hello.txt"), Is.True); - Assert.That(byName.ContainsKey("data.bin"), Is.True); - - Assert.That(Encoding.UTF8.GetString(r.Extract(byName["hello.txt"])), Is.EqualTo("Hello TUX2!")); - Assert.That(r.Extract(byName["data.bin"]), Is.EqualTo(new byte[] { 1, 2, 3, 4, 5 })); - } - - [Test, Category("HappyPath")] - public void Descriptor_List_Extract() { - var img = BuildSyntheticImage([("one.txt", "ONE"u8.ToArray())]); - using var ms = new MemoryStream(img); - var d = new FileSystem.Tux2.Tux2FormatDescriptor(); - var entries = d.List(ms, null); - Assert.That(entries.Count, Is.GreaterThanOrEqualTo(3)); - - var tmp = Path.Combine(Path.GetTempPath(), $"tux2-{Guid.NewGuid():N}"); - Directory.CreateDirectory(tmp); - try { - ms.Position = 0; - d.Extract(ms, tmp, null, null); - Assert.That(File.Exists(Path.Combine(tmp, "one.txt")), Is.True); - Assert.That(File.ReadAllText(Path.Combine(tmp, "one.txt")), Is.EqualTo("ONE")); - } finally { - Directory.Delete(tmp, recursive: true); - } + [Test, Category("Spec")] + public void Descriptor_DoesNotInventAStandaloneDiskFormatOrWriteSupport() { + var descriptor = new Tux2FormatDescriptor(); + + Assert.Multiple(() => { + Assert.That(descriptor.Id, Is.EqualTo("Tux2")); + Assert.That(descriptor.MagicSignatures, Is.Empty, + "TUX2 has no stable independent magic; TUX2FS was a workbench-private invention."); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.False); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.SupportsMultipleEntries), Is.False); + Assert.That(descriptor is IArchiveCreatable, Is.False); + Assert.That(descriptor is IArchiveModifiable, Is.False); + Assert.That(descriptor.Description, Does.Contain("no stable standalone").IgnoreCase); + }); } - [Test, Category("Sad")] - public void InvalidMagic_Throws() { - using var ms = new MemoryStream(new byte[32]); - Assert.Throws(() => _ = new FileSystem.Tux2.Tux2Reader(ms)); + [Test, Category("Spec")] + public void Reader_SurfacesImageOpaque_AndTreatsExt2MagicOnlyAsACompatibilityHint() { + var image = new byte[4096]; + for (var i = 0; i < image.Length; ++i) + image[i] = (byte)(i * 17 + i / 31); + BinaryPrimitives.WriteUInt16LittleEndian(image.AsSpan(1024 + 56, 2), 0xEF53); + + using var stream = new MemoryStream(image, writable: false); + using var reader = new Tux2Reader(stream); + + Assert.That(reader.LooksLikeExt2, Is.True); + Assert.That(reader.Entries.Select(entry => entry.Name), + Is.EquivalentTo(new[] { "FULL.tux2", "metadata.ini" })); + + var full = reader.Entries.Single(entry => entry.Name == "FULL.tux2"); + Assert.That(reader.Extract(full), Is.EqualTo(image)); + + var metadata = Encoding.UTF8.GetString(reader.Extract( + reader.Entries.Single(entry => entry.Name == "metadata.ini"))); + Assert.Multiple(() => { + Assert.That(metadata, Does.Contain("parse_status=opaque")); + Assert.That(metadata, Does.Contain("self_identifying=false")); + Assert.That(metadata, Does.Contain("ext2_superblock_magic=present")); + }); } - /// - /// Carving a hole used to be refused outright, because the rebuild always - /// packed from the front. The pass moves whole records now, so a container - /// with no records in it has nothing to refuse: there is nothing to move and - /// nothing to carve around. - /// - [Test] - public void Defragment_CarveHole_OnAnEmptyContainer_DoesNothing() { - var d = new FileSystem.Tux2.Tux2FormatDescriptor(); - var empty = BuildSyntheticImage([]); - using var ms = new MemoryStream(empty.Length); - ms.Write(empty, 0, empty.Length); - - ms.Position = 0; - Assert.DoesNotThrow(() => d.Defragment(ms, new DefragOptions { Mode = DefragMode.CarveHole })); - Assert.That(ms.ToArray(), Is.EqualTo(empty), "an empty container comes back byte for byte"); + [Test, Category("Regression")] + public void FormerPrivateTux2FsMagic_DoesNotCreateSyntheticFiles() { + var image = new byte[128]; + "TUX2FS\0\0"u8.CopyTo(image); + BinaryPrimitives.WriteUInt32LittleEndian(image.AsSpan(8, 4), 1); + BinaryPrimitives.WriteUInt32LittleEndian(image.AsSpan(12, 4), 1); + + using var stream = new MemoryStream(image, writable: false); + using var reader = new Tux2Reader(stream); + + Assert.Multiple(() => { + Assert.That(reader.LooksLikeExt2, Is.False); + Assert.That(reader.Entries, Has.Count.EqualTo(2)); + Assert.That(reader.Entries.Select(entry => entry.Name), + Is.EquivalentTo(new[] { "FULL.tux2", "metadata.ini" })); + }); } [Test, Category("HappyPath")] - public void Defragment_PreservesFiles() { - var descriptor = new FileSystem.Tux2.Tux2FormatDescriptor(); - var payload = new byte[5000]; - for (var i = 0; i < payload.Length; ++i) payload[i] = (byte)(i * 11); + public void Descriptor_ExtractsOnlyOpaqueImageAndMetadata() { + var image = Enumerable.Range(0, 257).Select(i => (byte)i).ToArray(); + using var stream = new MemoryStream(image, writable: false); + var descriptor = new Tux2FormatDescriptor(); + var output = Path.Combine(Path.GetTempPath(), $"tux2-opaque-{Guid.NewGuid():N}"); + Directory.CreateDirectory(output); - var path = Path.Combine(Path.GetTempPath(), "tux2_defrag_" + Guid.NewGuid().ToString("N")); - var outDir = path + "_out"; try { - using (var create = File.Create(path)) - descriptor.Create(create, [ArchiveInputInfo.InMemory("data.bin", payload)], new FormatCreateOptions()); - using (var archive = File.Open(path, FileMode.Open, FileAccess.ReadWrite)) - descriptor.Defragment(archive); - - Directory.CreateDirectory(outDir); - using (var read = File.OpenRead(path)) - descriptor.Extract(read, outDir, null, ["data.bin"]); - Assert.That(File.ReadAllBytes(Path.Combine(outDir, "data.bin")), Is.EqualTo(payload)); + descriptor.Extract(stream, output, null, null); + Assert.Multiple(() => { + Assert.That(File.ReadAllBytes(Path.Combine(output, "FULL.tux2")), Is.EqualTo(image)); + Assert.That(File.ReadAllText(Path.Combine(output, "metadata.ini")), + Does.Contain("no stable standalone TUX2 disk signature/layout")); + }); } finally { - try { File.Delete(path); } catch { /* scratch file already gone */ } - try { Directory.Delete(outDir, recursive: true); } catch { /* ignore */ } + Directory.Delete(output, recursive: true); } } - - [Test, Category("HappyPath")] - public void Implements_IArchiveCreatable() { - var d = new FileSystem.Tux2.Tux2FormatDescriptor(); - Assert.That(d, Is.InstanceOf()); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.True); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.SupportsMultipleEntries), Is.True); - } } diff --git a/Compression.Tests/Tux2/Tux2WriterTests.cs b/Compression.Tests/Tux2/Tux2WriterTests.cs deleted file mode 100644 index 045983d1b..000000000 --- a/Compression.Tests/Tux2/Tux2WriterTests.cs +++ /dev/null @@ -1,141 +0,0 @@ -using System.Buffers.Binary; -using System.Text; -using Compression.Registry; -using FileSystem.Tux2; - -namespace Compression.Tests.Tux2; - -[TestFixture] -public class Tux2WriterTests { - - [Test, Category("HappyPath"), Category("RoundTrip")] - public void Writer_EmitsValidHeader() { - var w = new Tux2Writer(); - w.AddFile("hello.txt", "Hello TUX2!"u8.ToArray()); - var image = w.Build(); - - Assert.That(image.AsSpan(0, 8).SequenceEqual(Tux2Reader.Magic), Is.True); - Assert.That(BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan(8, 4)), Is.EqualTo(1u)); - Assert.That(BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan(12, 4)), Is.EqualTo(1u)); - } - - [Test, Category("HappyPath"), Category("RoundTrip")] - public void Writer_RoundTripsSingleFile() { - var body = Encoding.UTF8.GetBytes("Hello TUX2 World!"); - var w = new Tux2Writer(); - w.AddFile("hello.txt", body); - using var ms = new MemoryStream(w.Build()); - - var r = new Tux2Reader(ms); - Assert.That(r.ValidHeader, Is.True); - Assert.That(r.FileCount, Is.EqualTo(1u)); - - var byName = r.Entries.ToDictionary(e => e.Name); - Assert.That(byName.ContainsKey("hello.txt"), Is.True); - Assert.That(r.Extract(byName["hello.txt"]), Is.EqualTo(body)); - } - - [Test, Category("HappyPath"), Category("RoundTrip")] - public void Writer_RoundTripsMultipleFiles() { - var w = new Tux2Writer(); - w.AddFile("a.txt", "alpha"u8.ToArray()); - w.AddFile("b.bin", new byte[] { 1, 2, 3, 4, 5 }); - w.AddFile("c.dat", new byte[1024]); // larger payload - using var ms = new MemoryStream(w.Build()); - - var r = new Tux2Reader(ms); - Assert.That(r.FileCount, Is.EqualTo(3u)); - var byName = r.Entries.ToDictionary(e => e.Name); - Assert.That(byName.ContainsKey("a.txt"), Is.True); - Assert.That(byName.ContainsKey("b.bin"), Is.True); - Assert.That(byName.ContainsKey("c.dat"), Is.True); - Assert.That(Encoding.UTF8.GetString(r.Extract(byName["a.txt"])), Is.EqualTo("alpha")); - Assert.That(r.Extract(byName["b.bin"]), Is.EqualTo(new byte[] { 1, 2, 3, 4, 5 })); - Assert.That(r.Extract(byName["c.dat"]).Length, Is.EqualTo(1024)); - } - - [Test, Category("HappyPath"), Category("RoundTrip")] - public void Writer_RoundTripsEmptyFile() { - var w = new Tux2Writer(); - w.AddFile("empty.txt", []); - using var ms = new MemoryStream(w.Build()); - - var r = new Tux2Reader(ms); - Assert.That(r.FileCount, Is.EqualTo(1u)); - var entry = r.Entries.First(e => e.Name == "empty.txt"); - Assert.That(entry.Data.Length, Is.EqualTo(0)); - } - - [Test, Category("HappyPath"), Category("RoundTrip")] - public void Writer_HandlesUtf8Names() { - var w = new Tux2Writer(); - w.AddFile("héllo-世界.txt", "unicode"u8.ToArray()); - using var ms = new MemoryStream(w.Build()); - - var r = new Tux2Reader(ms); - var entry = r.Entries.First(e => e.Name == "héllo-世界.txt"); - // The reader leaves a file's bytes in the image and records where they are, - // so the content comes back through Extract rather than off the entry. - Assert.That(Encoding.UTF8.GetString(r.Extract(entry)), Is.EqualTo("unicode")); - } - - [Test, Category("HappyPath"), Category("RoundTrip")] - public void Descriptor_Create_List_Roundtrip() { - var tmp = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".bin"); - File.WriteAllBytes(tmp, Encoding.ASCII.GetBytes("file contents")); - try { - var desc = new Tux2FormatDescriptor(); - using var ms = new MemoryStream(); - desc.Create(ms, [new ArchiveInputInfo(tmp, "myfile.txt", false)], new FormatCreateOptions()); - ms.Position = 0; - var listed = desc.List(ms, null); - Assert.That(listed.Select(e => e.Name), Does.Contain("myfile.txt")); - } finally { - File.Delete(tmp); - } - } - - [Test, Category("HappyPath"), Category("RoundTrip")] - public void Descriptor_Create_Extract_Roundtrip() { - var tmp = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".bin"); - var body = "round-trip contents"u8.ToArray(); - File.WriteAllBytes(tmp, body); - var outDir = Path.Combine(Path.GetTempPath(), $"tux2-out-{Guid.NewGuid():N}"); - Directory.CreateDirectory(outDir); - try { - var desc = new Tux2FormatDescriptor(); - using var ms = new MemoryStream(); - desc.Create(ms, [new ArchiveInputInfo(tmp, "out.bin", false)], new FormatCreateOptions()); - ms.Position = 0; - desc.Extract(ms, outDir, null, null); - var extracted = File.ReadAllBytes(Path.Combine(outDir, "out.bin")); - Assert.That(extracted, Is.EqualTo(body)); - } finally { - File.Delete(tmp); - if (Directory.Exists(outDir)) Directory.Delete(outDir, recursive: true); - } - } - - [Test, Category("EdgeCase")] - public void Writer_AddFile_EmptyName_Throws() { - var w = new Tux2Writer(); - Assert.That(() => w.AddFile("", [1, 2, 3]), Throws.InstanceOf()); - } - - [Test, Category("EdgeCase")] - public void Writer_AddFile_NullData_Throws() { - var w = new Tux2Writer(); - Assert.That(() => w.AddFile("x.txt", null!), Throws.InstanceOf()); - } - - [Test, Category("HappyPath")] - public void Descriptor_NoInputs_EmitsValidEmptyImage() { - var desc = new Tux2FormatDescriptor(); - using var ms = new MemoryStream(); - desc.Create(ms, [], new FormatCreateOptions()); - ms.Position = 0; - var r = new Tux2Reader(ms); - Assert.That(r.FileCount, Is.EqualTo(0u)); - Assert.That(r.ValidHeader, Is.True); - } -} diff --git a/Compression.Tests/Tux3/Tux3PlannedDefragTests.cs b/Compression.Tests/Tux3/Tux3PlannedDefragTests.cs deleted file mode 100644 index 3a24d6977..000000000 --- a/Compression.Tests/Tux3/Tux3PlannedDefragTests.cs +++ /dev/null @@ -1,109 +0,0 @@ -#pragma warning disable CS1591 -using Compression.Registry; -using FileSystem.Tux3; - -namespace Compression.Tests.Tux3; - -/// -/// Tux3 lays a container out again by moving whole records, and on one this -/// writer produced the answer is usually that nothing is out of place. -/// -/// -/// A record's data sits behind the header naming it at an offset nothing -/// records — the reader finds the next record by adding this one's length to a -/// cursor — so nothing has to be repointed, and the only layout the walk -/// reaches is one with the records in some order and nothing between them. -/// Removing a file writes the container out packed, so a pass over ours finds -/// no gap; what it is for is a container that arrived from somewhere else. -/// -[TestFixture] -public class Tux3PlannedDefragTests { - - private static byte[] Payload(int seed, int length) { - var data = new byte[length]; - for (var i = 0; i < length; ++i) data[i] = (byte)((i * 13 + seed * 29) % 251); - return data; - } - - private static MemoryStream Volume(out Dictionary files) { - var work = Path.Combine(Path.GetTempPath(), "cwb_Tux3_" + Guid.NewGuid().ToString("N")[..8]); - Directory.CreateDirectory(work); - files = new Dictionary(StringComparer.Ordinal); - try { - var inputs = new List(); - for (var k = 0; k < 5; ++k) { - var data = Payload(k, 3000 + k * 1500); - var path = Path.Combine(work, $"F{k}.BIN"); - File.WriteAllBytes(path, data); - inputs.Add(new ArchiveInputInfo(path, $"F{k}.BIN", false)); - files[$"F{k}.BIN"] = data; - } - - var image = new MemoryStream(); - new Tux3FormatDescriptor().Create(image, inputs, new FormatCreateOptions()); - return image; - } finally { - try { Directory.Delete(work, true); } catch { /* scratch is gone already */ } - } - } - - private static Dictionary ReadBack(MemoryStream image) { - image.Position = 0; - using var reader = new Tux3Reader(image); - return reader.Entries - .Where(e => e.Size > 0 && !e.Name.StartsWith("FULL.", StringComparison.Ordinal) - && e.Name != "metadata.ini") - .ToDictionary(e => e.Name, reader.Extract, StringComparer.Ordinal); - } - - [Test, Category("RoundTrip")] - [TestCase(DefragMode.ConsolidateAtStart)] - [TestCase(DefragMode.ConsolidateAtEnd)] - public void Defragment_KeepsEveryPayloadAndTheContainersSize(DefragMode mode) { - using var image = Volume(out var files); - var size = image.Length; - - image.Position = 0; - new Tux3FormatDescriptor().Defragment(image, new DefragOptions { Mode = mode }); - Assert.That(image.Length, Is.EqualTo(size), "a container keeps its size"); - - var read = ReadBack(image); - foreach (var (name, data) in files) { - Assert.That(read.Keys, Does.Contain(name), $"{name} must still be in the container"); - Assert.That(read[name], Is.EqualTo(data), $"{name} must read back byte for byte"); - } - } - - [Test] - public void Defragment_FindsNothingToMoveOnOneOfOurOwn() { - using var image = Volume(out _); - var before = image.ToArray(); - - image.Position = 0; - new Tux3FormatDescriptor().Defragment(image, - new DefragOptions { Mode = DefragMode.ConsolidateAtStart }); - - // The writer packs its records, so front-packing has nothing to do — and - // doing nothing must mean writing nothing. - Assert.That(image.ToArray(), Is.EqualTo(before), - "a container already packed must come back byte for byte"); - } - - [Test] - public void Defragment_LeavesTheRecordsWithNothingBetweenThem() { - using var image = Volume(out _); - image.Position = 0; - new Tux3FormatDescriptor().Defragment(image, new DefragOptions { Mode = DefragMode.ConsolidateAtEnd }); - - image.Position = 0; - var records = Tux3RecordMap.Enumerate(image) - .Where(e => e.Kind == DefragBlockKind.Used) - .OrderBy(e => e.Offset) - .ToList(); - Assert.That(records, Is.Not.Empty, "the container must still describe its records"); - - for (var i = 1; i < records.Count; ++i) - Assert.That(records[i].Offset, Is.EqualTo(records[i - 1].Offset + records[i - 1].Length), - "a gap between records is what the walk cannot get past"); - } -} diff --git a/Compression.Tests/Tux3/Tux3SchemaTests.cs b/Compression.Tests/Tux3/Tux3SchemaTests.cs deleted file mode 100644 index e1e2c6967..000000000 --- a/Compression.Tests/Tux3/Tux3SchemaTests.cs +++ /dev/null @@ -1,51 +0,0 @@ -using Compression.Registry; -using FileSystem.Tux3; - -namespace Compression.Tests.Tux3; - -/// -/// Schema-knob contract tests for : proves the -/// published Birthday option is a real knob the writer stamps into the -/// superblock and the reader reads back. -/// -[TestFixture] -public class Tux3SchemaTests { - - [Test, Category("Spec")] - public void Descriptor_ExposesBirthdaySchema() { - var d = new Tux3FormatDescriptor(); - Assert.That(d, Is.InstanceOf()); - Assert.That(d, Is.InstanceOf()); - var schema = ((IFormatOptionsSchema)d).OptionsSchema; - Assert.That(schema.Any(o => o.Key == "Birthday"), Is.True); - } - - [Test, Category("HappyPath")] - public void Create_Birthday_TakesEffectAndFilesRoundTrip() { - var d = new Tux3FormatDescriptor(); - var payload = "tux3 birthday knob"u8.ToArray(); - using var ms = new MemoryStream(); - d.Create(ms, - [ArchiveInputInfo.InMemory("note.txt", payload)], - new FormatCreateOptions { FormatSpecific = new Dictionary { ["Birthday"] = "0xCAFEF00DBAADBEEF" } }); - - ms.Position = 0; - var r = new Tux3Reader(ms); - Assert.That(r.Birthday, Is.EqualTo(0xCAFEF00DBAADBEEFUL), "Birthday knob must land in the superblock."); - var entry = r.Entries.Single(e => e.Name == "note.txt"); - Assert.That(r.Extract(entry), Is.EqualTo(payload), "File must round-trip."); - } - - [Test, Category("Equivalence")] - public void Create_WithoutBirthday_StampsTheMomentOfCreation() { - var before = (ulong)DateTimeOffset.UtcNow.ToUnixTimeSeconds(); - var d = new Tux3FormatDescriptor(); - using var ms = new MemoryStream(); - d.Create(ms, [ArchiveInputInfo.InMemory("a.txt", "x"u8.ToArray())], new FormatCreateOptions()); - ms.Position = 0; - var birthday = new Tux3Reader(ms).Birthday; - var after = (ulong)DateTimeOffset.UtcNow.ToUnixTimeSeconds(); - Assert.That(birthday, Is.InRange(before, after), - "Asked for no birthday, the volume takes one from the clock rather than a constant that would name its writer."); - } -} diff --git a/Compression.Tests/Tux3/Tux3Tests.cs b/Compression.Tests/Tux3/Tux3Tests.cs index 136c35953..9fa402519 100644 --- a/Compression.Tests/Tux3/Tux3Tests.cs +++ b/Compression.Tests/Tux3/Tux3Tests.cs @@ -1,131 +1,117 @@ using System.Buffers.Binary; +using System.Text; using Compression.Registry; +using FileSystem.Tux3; namespace Compression.Tests.Tux3; [TestFixture] public class Tux3Tests { + private static byte[] BuildNativeImage(bool legacy2012 = false) { + var image = new byte[16 * 1024]; + var super = image.AsSpan(Tux3Reader.SuperblockOffset, Tux3Reader.DiskSuperSize); + (legacy2012 ? Tux3Reader.Legacy2012Magic : Tux3Reader.Magic).CopyTo(super); - // Build a minimal TUX3 image with valid superblock at offset 4096. - private static byte[] BuildMinimalImage() { - var image = new byte[8 * 1024]; - var sb = 4096; - FileSystem.Tux3.Tux3Reader.Magic.CopyTo(image.AsSpan(sb)); - BinaryPrimitives.WriteUInt64LittleEndian(image.AsSpan(sb + 0x08, 8), 0x1234_5678_9ABC_DEF0UL); // birthday - BinaryPrimitives.WriteUInt64LittleEndian(image.AsSpan(sb + 0x10, 8), 0x0000_0000_0000_0001UL); // flags - BinaryPrimitives.WriteUInt64LittleEndian(image.AsSpan(sb + 0x18, 8), 100UL); // iroot - BinaryPrimitives.WriteUInt64LittleEndian(image.AsSpan(sb + 0x20, 8), 200UL); // oroot - BinaryPrimitives.WriteUInt64LittleEndian(image.AsSpan(sb + 0x28, 8), 300UL); // aroot - BinaryPrimitives.WriteUInt64LittleEndian(image.AsSpan(sb + 0x30, 8), 12UL); // blockbits => 4096 - BinaryPrimitives.WriteUInt64LittleEndian(image.AsSpan(sb + 0x38, 8), 1024UL); // volblocks - BinaryPrimitives.WriteUInt64LittleEndian(image.AsSpan(sb + 0x40, 8), 512UL); // freeblocks + BinaryPrimitives.WriteUInt64BigEndian(super.Slice(0x08, 8), 0x0123_4567_89AB_CDEFUL); + BinaryPrimitives.WriteUInt64BigEndian(super.Slice(0x10, 8), 0x1020_3040_5060_7080UL); + BinaryPrimitives.WriteUInt16BigEndian(super.Slice(0x18, 2), 12); + BinaryPrimitives.WriteUInt64BigEndian(super.Slice(0x20, 8), 0x0000_0000_0000_1234UL); + BinaryPrimitives.WriteUInt64BigEndian(super.Slice(0x28, 8), 0x0001_0000_0000_0042UL); + BinaryPrimitives.WriteUInt64BigEndian(super.Slice(0x30, 8), 0x8002_0000_0000_0043UL); + BinaryPrimitives.WriteUInt64BigEndian(super.Slice(0x38, 8), 0x0000_0000_0000_0040UL); + BinaryPrimitives.WriteUInt64BigEndian(super.Slice(0x40, 8), 0x0000_0000_0000_0080UL); + BinaryPrimitives.WriteUInt64BigEndian(super.Slice(0x48, 8), 0x0000_0000_0000_0100UL); + BinaryPrimitives.WriteUInt32BigEndian(super.Slice(0x50, 4), 0x1020_3040U); + BinaryPrimitives.WriteUInt32BigEndian(super.Slice(0x54, 4), 0x5060_7080U); + BinaryPrimitives.WriteUInt64BigEndian(super.Slice(0x58, 8), 0x0000_0000_0000_2222UL); + BinaryPrimitives.WriteUInt32BigEndian(super.Slice(0x60, 4), 0x0000_0003U); return image; } - [Test, Category("HappyPath")] - public void Descriptor_Properties() { - var d = new FileSystem.Tux3.Tux3FormatDescriptor(); - Assert.That(d.Id, Is.EqualTo("Tux3")); - Assert.That(d.DisplayName, Is.EqualTo("TUX3")); - Assert.That(d.Extensions, Does.Contain(".tux3")); - Assert.That(d.Category, Is.EqualTo(FormatCategory.Archive)); - Assert.That(d.MagicSignatures, Has.Count.EqualTo(1)); - Assert.That(d.MagicSignatures[0].Offset, Is.EqualTo(4096)); - } + [Test, Category("Spec")] + public void Reader_ParsesCanonicalPackedBigEndianDiskSuper() { + var image = BuildNativeImage(); + using var stream = new MemoryStream(image, writable: false); + using var reader = new Tux3Reader(stream); - [Test, Category("HappyPath")] - public void Read_MinimalSyntheticImage() { - var img = BuildMinimalImage(); - using var ms = new MemoryStream(img); - var r = new FileSystem.Tux3.Tux3Reader(ms); - Assert.That(r.ValidSuperblock, Is.True); - Assert.That(r.Birthday, Is.EqualTo(0x1234_5678_9ABC_DEF0UL)); - Assert.That(r.IRoot, Is.EqualTo(100UL)); - Assert.That(r.BlockBits, Is.EqualTo(12UL)); - Assert.That(r.VolBlocks, Is.EqualTo(1024UL)); - Assert.That(r.FreeBlocks, Is.EqualTo(512UL)); + Assert.Multiple(() => { + Assert.That(reader.ValidSuperblock, Is.True); + Assert.That(reader.Revision, Is.EqualTo("2014-05-06")); + Assert.That(reader.Birthday, Is.EqualTo(0x0123_4567_89AB_CDEFUL)); + Assert.That(reader.Flags, Is.EqualTo(0x1020_3040_5060_7080UL)); + Assert.That(reader.BlockBits, Is.EqualTo(12)); + Assert.That(reader.VolBlocks, Is.EqualTo(0x1234UL)); + Assert.That(reader.IRoot, Is.EqualTo(0x0001_0000_0000_0042UL)); + Assert.That(reader.ORoot, Is.EqualTo(0x8002_0000_0000_0043UL)); + Assert.That(reader.UsedInodes, Is.EqualTo(0x40UL)); + Assert.That(reader.NextBlock, Is.EqualTo(0x80UL)); + Assert.That(reader.AtomDictionarySize, Is.EqualTo(0x100UL)); + Assert.That(reader.FreeAtom, Is.EqualTo(0x1020_3040U)); + Assert.That(reader.AtomGeneration, Is.EqualTo(0x5060_7080U)); + Assert.That(reader.LogChain, Is.EqualTo(0x2222UL)); + Assert.That(reader.LogCount, Is.EqualTo(3U)); + }); - var names = r.Entries.Select(e => e.Name).ToHashSet(); - Assert.That(names, Does.Contain("FULL.tux3")); - Assert.That(names, Does.Contain("metadata.ini")); - Assert.That(names, Does.Contain("superblock.bin")); + Assert.That(reader.Entries.Select(entry => entry.Name), + Is.EquivalentTo(new[] { "FULL.tux3", "metadata.ini", "superblock.bin" })); + var superblock = reader.Extract(reader.Entries.Single(entry => entry.Name == "superblock.bin")); + Assert.That(superblock, Is.EqualTo(image.AsSpan(Tux3Reader.SuperblockOffset, Tux3Reader.DiskSuperSize).ToArray())); } - [Test, Category("HappyPath")] - public void Descriptor_List_Extract() { - var img = BuildMinimalImage(); - using var ms = new MemoryStream(img); - var d = new FileSystem.Tux3.Tux3FormatDescriptor(); - var entries = d.List(ms, null); - Assert.That(entries, Has.Count.EqualTo(3)); - - var tmp = Path.Combine(Path.GetTempPath(), $"tux3-{Guid.NewGuid():N}"); - Directory.CreateDirectory(tmp); - try { - ms.Position = 0; - d.Extract(ms, tmp, null, null); - Assert.That(File.Exists(Path.Combine(tmp, "metadata.ini")), Is.True); - var meta = File.ReadAllText(Path.Combine(tmp, "metadata.ini")); - Assert.That(meta, Does.Contain("format=TUX3")); - Assert.That(meta, Does.Contain("blockbits=12")); - } finally { - Directory.Delete(tmp, recursive: true); - } + [Test, Category("Spec")] + public void Reader_AcceptsKnown2012DiskRevision() { + using var stream = new MemoryStream(BuildNativeImage(legacy2012: true), writable: false); + using var reader = new Tux3Reader(stream); + Assert.That(reader.Revision, Is.EqualTo("2012-12-20")); } - [Test, Category("Sad")] - public void InvalidMagic_Throws() { - var img = new byte[8 * 1024]; - using var ms = new MemoryStream(img); - Assert.Throws(() => _ = new FileSystem.Tux3.Tux3Reader(ms)); + [Test, Category("Regression")] + public void FormerPrivateTux3SuprMagic_IsRejected() { + var image = new byte[16 * 1024]; + "TUX3SUPR"u8.CopyTo(image.AsSpan(Tux3Reader.SuperblockOffset)); + using var stream = new MemoryStream(image, writable: false); + + Assert.Throws(() => _ = new Tux3Reader(stream)); } - /// - /// Carving a hole used to be refused outright, because the rebuild always - /// packed from the front. The pass moves whole records now, so a container - /// with no records in it has nothing to refuse. - /// - [Test] - public void Defragment_CarveHole_OnAMinimalContainer_DoesNothing() { - var d = new FileSystem.Tux3.Tux3FormatDescriptor(); - var minimal = BuildMinimalImage(); - using var ms = new MemoryStream(minimal.Length); - ms.Write(minimal, 0, minimal.Length); + [Test, Category("Spec")] + public void Descriptor_AdvertisesOnlyTheNativeMetadataSurface() { + var descriptor = new Tux3FormatDescriptor(); - ms.Position = 0; - Assert.DoesNotThrow(() => d.Defragment(ms, new DefragOptions { Mode = DefragMode.CarveHole })); - Assert.That(ms.ToArray(), Is.EqualTo(minimal), "a container with no records comes back unchanged"); + Assert.Multiple(() => { + Assert.That(descriptor.MagicSignatures, Has.Count.EqualTo(2)); + Assert.That(descriptor.MagicSignatures.All(signature => signature.Offset == Tux3Reader.SuperblockOffset), Is.True); + Assert.That(descriptor.MagicSignatures[0].Bytes, Is.EqualTo(Tux3Reader.Magic)); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.False); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanModify), Is.False); + Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.SupportsMultipleEntries), Is.False); + Assert.That(descriptor is IArchiveCreatable, Is.False); + Assert.That(descriptor is IArchiveModifiable, Is.False); + Assert.That(descriptor.Description, Does.Contain("native big-endian superblock")); + }); } [Test, Category("HappyPath")] - public void Defragment_PreservesFiles() { - var descriptor = new FileSystem.Tux3.Tux3FormatDescriptor(); - var payload = new byte[5000]; - for (var i = 0; i < payload.Length; ++i) payload[i] = (byte)(i * 11); + public void Descriptor_ExtractsNativeMetadataWithoutInventedFileTable() { + var image = BuildNativeImage(); + using var stream = new MemoryStream(image, writable: false); + var descriptor = new Tux3FormatDescriptor(); + var output = Path.Combine(Path.GetTempPath(), $"tux3-native-{Guid.NewGuid():N}"); + Directory.CreateDirectory(output); - var path = Path.Combine(Path.GetTempPath(), "tux3_defrag_" + Guid.NewGuid().ToString("N")); - var outDir = path + "_out"; try { - using (var create = File.Create(path)) - descriptor.Create(create, [ArchiveInputInfo.InMemory("data.bin", payload)], new FormatCreateOptions()); - using (var archive = File.Open(path, FileMode.Open, FileAccess.ReadWrite)) - descriptor.Defragment(archive); - - Directory.CreateDirectory(outDir); - using (var read = File.OpenRead(path)) - descriptor.Extract(read, outDir, null, ["data.bin"]); - Assert.That(File.ReadAllBytes(Path.Combine(outDir, "data.bin")), Is.EqualTo(payload)); + descriptor.Extract(stream, output, null, null); + var metadata = File.ReadAllText(Path.Combine(output, "metadata.ini")); + Assert.Multiple(() => { + Assert.That(metadata, Does.Contain("parse_status=superblock-only")); + Assert.That(metadata, Does.Contain("revision=2014-05-06")); + Assert.That(metadata, Does.Contain("blockbits=12")); + Assert.That(metadata, Does.Not.Contain("TUX3WORM")); + Assert.That(Directory.EnumerateFiles(output).Select(Path.GetFileName), + Is.EquivalentTo(new[] { "FULL.tux3", "metadata.ini", "superblock.bin" })); + }); } finally { - try { File.Delete(path); } catch { /* scratch file already gone */ } - try { Directory.Delete(outDir, recursive: true); } catch { /* ignore */ } + Directory.Delete(output, recursive: true); } } - - [Test, Category("HappyPath")] - public void Implements_IArchiveCreatable() { - var d = new FileSystem.Tux3.Tux3FormatDescriptor(); - Assert.That(d, Is.InstanceOf()); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.CanCreate), Is.True); - Assert.That(d.Capabilities.HasFlag(FormatCapabilities.SupportsMultipleEntries), Is.True); - } } diff --git a/Compression.Tests/Tux3/Tux3WriterTests.cs b/Compression.Tests/Tux3/Tux3WriterTests.cs deleted file mode 100644 index 381f86a73..000000000 --- a/Compression.Tests/Tux3/Tux3WriterTests.cs +++ /dev/null @@ -1,162 +0,0 @@ -using System.Buffers.Binary; -using System.Text; -using Compression.Registry; -using FileSystem.Tux3; - -namespace Compression.Tests.Tux3; - -[TestFixture] -public class Tux3WriterTests { - - [Test, Category("HappyPath"), Category("RoundTrip")] - public void Writer_EmitsValidSuperblock() { - var w = new Tux3Writer(); - w.AddFile("a.txt", "alpha"u8.ToArray()); - var image = w.Build(); - - Assert.That(image.AsSpan(4096, 8).SequenceEqual(Tux3Reader.Magic), Is.True); - var blockBits = BinaryPrimitives.ReadUInt64LittleEndian(image.AsSpan(4096 + 0x30, 8)); - Assert.That(blockBits, Is.EqualTo(12UL)); - var volBlocks = BinaryPrimitives.ReadUInt64LittleEndian(image.AsSpan(4096 + 0x38, 8)); - Assert.That(volBlocks, Is.GreaterThan(0UL)); - } - - [Test, Category("HappyPath"), Category("RoundTrip")] - public void Writer_EmitsWormTableAtBlock2() { - var w = new Tux3Writer(); - w.AddFile("a.txt", "alpha"u8.ToArray()); - var image = w.Build(); - Assert.That(image.AsSpan(8192, 8).SequenceEqual(Tux3Reader.WormTableMagic), Is.True); - Assert.That(BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan(8192 + 8, 4)), Is.EqualTo(1u)); - } - - [Test, Category("HappyPath"), Category("RoundTrip")] - public void Writer_RoundTripsSingleFile() { - var body = Encoding.UTF8.GetBytes("Hello TUX3!"); - var w = new Tux3Writer(); - w.AddFile("hello.txt", body); - using var ms = new MemoryStream(w.Build()); - - var r = new Tux3Reader(ms); - Assert.That(r.ValidSuperblock, Is.True); - Assert.That(r.HasWormTable, Is.True); - Assert.That(r.WormFileCount, Is.EqualTo(1u)); - - var byName = r.Entries.ToDictionary(e => e.Name); - Assert.That(byName.ContainsKey("hello.txt"), Is.True); - Assert.That(r.Extract(byName["hello.txt"]), Is.EqualTo(body)); - } - - [Test, Category("HappyPath"), Category("RoundTrip")] - public void Writer_RoundTripsMultipleFiles() { - var w = new Tux3Writer(); - w.AddFile("a.txt", "alpha"u8.ToArray()); - w.AddFile("b.bin", new byte[] { 1, 2, 3, 4, 5 }); - w.AddFile("c.dat", new byte[1024]); - using var ms = new MemoryStream(w.Build()); - - var r = new Tux3Reader(ms); - Assert.That(r.WormFileCount, Is.EqualTo(3u)); - var byName = r.Entries.ToDictionary(e => e.Name); - Assert.That(byName.ContainsKey("a.txt"), Is.True); - Assert.That(byName.ContainsKey("b.bin"), Is.True); - Assert.That(byName.ContainsKey("c.dat"), Is.True); - Assert.That(Encoding.UTF8.GetString(r.Extract(byName["a.txt"])), Is.EqualTo("alpha")); - Assert.That(r.Extract(byName["b.bin"]), Is.EqualTo(new byte[] { 1, 2, 3, 4, 5 })); - Assert.That(r.Extract(byName["c.dat"]).Length, Is.EqualTo(1024)); - } - - [Test, Category("HappyPath"), Category("RoundTrip")] - public void Writer_RoundTripsEmptyFile() { - var w = new Tux3Writer(); - w.AddFile("empty.txt", []); - using var ms = new MemoryStream(w.Build()); - - var r = new Tux3Reader(ms); - Assert.That(r.WormFileCount, Is.EqualTo(1u)); - var entry = r.Entries.First(e => e.Name == "empty.txt"); - Assert.That(entry.Data.Length, Is.EqualTo(0)); - } - - [Test, Category("HappyPath"), Category("RoundTrip")] - public void Writer_ImageSizeAlignedToBlock() { - var w = new Tux3Writer(); - w.AddFile("a.txt", "abc"u8.ToArray()); - var image = w.Build(); - Assert.That(image.Length % 4096, Is.EqualTo(0)); - Assert.That(image.Length, Is.GreaterThanOrEqualTo(3 * 4096)); // boot + superblock + worm-table blocks - } - - [Test, Category("HappyPath"), Category("RoundTrip")] - public void Writer_NoFiles_StillProducesValidImage() { - var w = new Tux3Writer(); - var image = w.Build(); - using var ms = new MemoryStream(image); - var r = new Tux3Reader(ms); - Assert.That(r.ValidSuperblock, Is.True); - Assert.That(r.HasWormTable, Is.True); - Assert.That(r.WormFileCount, Is.EqualTo(0u)); - } - - [Test, Category("HappyPath"), Category("RoundTrip")] - public void Descriptor_Create_List_Roundtrip() { - var tmp = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".bin"); - File.WriteAllBytes(tmp, Encoding.ASCII.GetBytes("file contents")); - try { - var desc = new Tux3FormatDescriptor(); - using var ms = new MemoryStream(); - desc.Create(ms, [new ArchiveInputInfo(tmp, "myfile.txt", false)], new FormatCreateOptions()); - ms.Position = 0; - var listed = desc.List(ms, null); - Assert.That(listed.Select(e => e.Name), Does.Contain("myfile.txt")); - } finally { - File.Delete(tmp); - } - } - - [Test, Category("HappyPath"), Category("RoundTrip")] - public void Descriptor_Create_Extract_Roundtrip() { - var tmp = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".bin"); - var body = "round-trip contents"u8.ToArray(); - File.WriteAllBytes(tmp, body); - var outDir = Path.Combine(Path.GetTempPath(), $"tux3-out-{Guid.NewGuid():N}"); - Directory.CreateDirectory(outDir); - try { - var desc = new Tux3FormatDescriptor(); - using var ms = new MemoryStream(); - desc.Create(ms, [new ArchiveInputInfo(tmp, "out.bin", false)], new FormatCreateOptions()); - ms.Position = 0; - desc.Extract(ms, outDir, null, null); - var extracted = File.ReadAllBytes(Path.Combine(outDir, "out.bin")); - Assert.That(extracted, Is.EqualTo(body)); - } finally { - File.Delete(tmp); - if (Directory.Exists(outDir)) Directory.Delete(outDir, recursive: true); - } - } - - [Test, Category("EdgeCase")] - public void Reader_NoWormTable_StillReadsLegacyImages() { - // Build an image with only the documented superblock (no WORM sentinel) — - // simulates a real linux-tux3 prototype dump (HasWormTable=false). - var image = new byte[8 * 1024]; - var sb = 4096; - Tux3Reader.Magic.CopyTo(image.AsSpan(sb)); - BinaryPrimitives.WriteUInt64LittleEndian(image.AsSpan(sb + 0x30, 8), 12UL); - BinaryPrimitives.WriteUInt64LittleEndian(image.AsSpan(sb + 0x38, 8), 1024UL); - - using var ms = new MemoryStream(image); - var r = new Tux3Reader(ms); - Assert.That(r.ValidSuperblock, Is.True); - Assert.That(r.HasWormTable, Is.False); - Assert.That(r.WormFileCount, Is.EqualTo(0u)); - Assert.That(r.Entries.Select(e => e.Name), Does.Contain("FULL.tux3")); - Assert.That(r.Entries.Select(e => e.Name), Does.Contain("superblock.bin")); - } - - [Test, Category("EdgeCase")] - public void Writer_AddFile_EmptyName_Throws() { - var w = new Tux3Writer(); - Assert.That(() => w.AddFile("", [1, 2, 3]), Throws.InstanceOf()); - } -} diff --git a/FileSystems/FileSystem.Tux2/Tux2BlockMover.cs b/FileSystems/FileSystem.Tux2/Tux2BlockMover.cs deleted file mode 100644 index fe487df9c..000000000 --- a/FileSystems/FileSystem.Tux2/Tux2BlockMover.cs +++ /dev/null @@ -1,58 +0,0 @@ -#pragma warning disable CS1591 -using Compression.Registry; - -namespace FileSystem.Tux2; - -/// -/// Moves a whole record inside the container, which needs nothing else -/// rewritten. -/// -/// -/// A record's data sits behind the header naming it, and the reader finds the -/// next record by adding this one's length to a cursor. So nothing records a -/// position and nothing has to be repointed — but the walk only reaches a -/// record that is still in order with nothing before it, which is what the -/// guard checks by reading every payload back afterwards. -/// -public sealed class Tux2BlockMover : IFilesystemBlockMover { - - /// A byte: records are packed to the byte, not to any larger unit. - public int BlockSize => 1; - - /// First byte a record may occupy: past the container's header. - public long FirstDataByte => Tux2RecordMap.HeaderSize; - - /// Each call moves the record it is given and nothing else. - public bool RepointsRunsIndependently => true; - - /// - /// A record may be held outside the container while the rest of the layout - /// moves, which is what lets a full one be rearranged at all. - /// - public bool SupportsHeldRuns => true; - - /// - public void MoveExtent(Stream image, long srcOffset, long dstOffset, long length, bool zeroSource = false) { - if (length <= 0 || srcOffset == dstOffset) return; - - // Overlap-safe: a run shifted forward by less than its own length - // overwrites its own tail, and copying that front to back reads bytes - // the copy has already replaced. - Compression.Core.DiskImage.ExtentCopy.Move(image, srcOffset, dstOffset, length); - if (zeroSource) - Compression.Core.DiskImage.ExtentCopy.Zero(image, srcOffset, length); - } - - /// - /// - /// Nothing to do: a record carries its own name and lengths, and where it - /// sits is recorded nowhere. - /// - /// - /// Performs the update allocation after move operation. - /// - public void UpdateAllocationAfterMove(Stream image, string fileName, long oldOffset, long newOffset, long length) { - ArgumentNullException.ThrowIfNull(image); - ArgumentNullException.ThrowIfNull(fileName); - } -} diff --git a/FileSystems/FileSystem.Tux2/Tux2FormatDescriptor.cs b/FileSystems/FileSystem.Tux2/Tux2FormatDescriptor.cs index f25338504..f7a77afb2 100644 --- a/FileSystems/FileSystem.Tux2/Tux2FormatDescriptor.cs +++ b/FileSystems/FileSystem.Tux2/Tux2FormatDescriptor.cs @@ -5,383 +5,78 @@ namespace FileSystem.Tux2; /// -/// Read+WORM descriptor for TUX2 — Daniel Phillips's 2002 phase-tree -/// filesystem proposal (OLS 2002 paper, never-stabilised research format). -/// Recognises a deterministic header pattern (magic "TUX2FS\0\0" at offset 0) -/// so research images we generate round-trip through the reader. Writer emits -/// a single-phase image only (no alpha/beta phases, no version chain) — real -/// legacy prototype images would need a custom parser matching the specific -/// snapshot of the in-progress code that produced them. -/// -/// References: -/// -/// Daniel Phillips, "The Tux2 Filesystem" (Ottawa Linux Symposium 2002 proceedings) — the defining paper -/// https://en.wikipedia.org/wiki/Tux3 — Wikipedia article covering the phase-tree lineage -/// -/// -/// -/// Why there is nothing here to lay out again. +/// Opaque/manual descriptor for Daniel Phillips's TUX2 phase-tree research filesystem. /// /// -/// The records run end to end from the header: a name length, the name, a -/// data length, the data, then the next one. The reader walks that by adding -/// each record's length to a cursor, so a gap anywhere makes everything after -/// it unreadable — the only layout this format can express is packed from the -/// front. -/// -/// Which is the layout it is always already in. Removing a file writes -/// the container out compacted rather than leaving a hole, so there is never -/// space between records to close up. A pass over one of these would find -/// nothing to move on every volume it was ever handed. +/// TUX2 was announced as an Ext2 variation and explicitly aimed to mount existing Ext2 +/// partitions. No stable, independently identifying TUX2 disk format or magic was published. +/// Consequently this descriptor does not manufacture a private signature, writer, modifier or +/// defragmenter and does not claim that an Ext2 image is uniquely TUX2. Manual selection surfaces +/// the image plus compatibility metadata for forensic work. /// -public sealed class Tux2FormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, IArchiveShrinkable, IArchiveModifiable, IArchiveDefragmentable, IFormatOptionsSchema, ILayoutOptimizable, IFilesystemExtentMap, IWipeEmpty, ISyntheticEntryNames { - - /// - public IReadOnlySet SyntheticEntryNames => SyntheticNames; - - - // ── Synthetic, non-file entries the reader always surfaces ────────────── +public sealed class Tux2FormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, ISyntheticEntryNames { private static readonly HashSet SyntheticNames = new(StringComparer.OrdinalIgnoreCase) { "FULL.tux2", "metadata.ini" }; - // ── IFormatOptionsSchema ──────────────────────────────────────────────── - - /// - /// The single tunable the single-phase WORM writer honours: the on-disk - /// format version stamped into the header at offset 0x08. - /// is written verbatim and reads it back, so - /// the knob round-trips. Defaults to 1 (the version the reader documents). - /// - public IReadOnlyList OptionsSchema { get; } = [ - new FormatOptionDescriptor( - Key: "Version", DisplayName: "Image version", Kind: FormatOptionKind.Integer, Default: "1", - Description: "Format version stamped into the TUX2 header at offset 0x08."), - ]; + /// + public IReadOnlySet SyntheticEntryNames => SyntheticNames; - /// - /// Gets the id. - /// + /// public string Id => "Tux2"; - /// - /// Gets the display name. - /// + + /// public string DisplayName => "TUX2"; - /// - /// Gets the category. - /// + + /// public FormatCategory Category => FormatCategory.Archive; - /// - /// Gets the capabilities. - /// + + /// public FormatCapabilities Capabilities => - FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest | - FormatCapabilities.CanCreate | FormatCapabilities.CanModify | - FormatCapabilities.SupportsMultipleEntries; - /// - /// Gets the default extension. - /// + FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest; + + /// public string DefaultExtension => ".tux2"; - /// - /// Gets the extensions. - /// + + /// public IReadOnlyList Extensions => [".tux2"]; - /// - /// Gets the compound extensions. - /// + + /// public IReadOnlyList CompoundExtensions => []; - /// - /// Gets the magic signatures. - /// - public IReadOnlyList MagicSignatures => [ - new("TUX2FS\0\0"u8.ToArray(), Offset: 0, Confidence: 0.90), - ]; - /// - /// Gets the methods. - /// - public IReadOnlyList Methods => [new("stored", "Stored")]; - /// - /// Gets the tar compression format id. - /// - public string? TarCompressionFormatId => null; - /// - /// Gets the family. - /// - public AlgorithmFamily Family => AlgorithmFamily.Archive; - /// - /// Gets the description. - /// - public string Description => "TUX2 phase-tree research filesystem (Daniel Phillips, OLS 2002) — single-phase synthetic image."; - /// - /// Lists the entries in the supplied container. - /// - public List List(Stream stream, string? password) { - var r = new Tux2Reader(stream); - return r.Entries.Select((e, i) => new ArchiveEntryInfo( - i, e.Name, e.Size, e.Size, "Stored", e.IsDirectory, false, null)).ToList(); - } + /// + public IReadOnlyList MagicSignatures => []; - /// - /// Decodes the supplied input. - /// - public void Extract(Stream stream, string outputDir, string? password, string[]? files) { - using var r = new Tux2Reader(stream); - foreach (var e in r.Entries) { - if (e.IsDirectory) continue; - if (files != null && files.Length > 0 && !MatchesFilter(e.Name, files)) continue; - var target = Path.Combine(outputDir, e.Name.Replace('/', Path.DirectorySeparatorChar)); - Directory.CreateDirectory(Path.GetDirectoryName(target) ?? outputDir); - using var output = File.Create(target); - r.ExtractTo(e, output); - } - } + /// + public IReadOnlyList Methods => [new("stored", "Stored")]; - /// - /// Emits a fresh single-phase TUX2 image: 16-byte header (magic + version + - /// file count) followed by per-file records (u16 name length, UTF-8 name, - /// u32 data length, raw bytes). Round-trips through . - /// - public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { - var version = (uint)Math.Max(0, options.GetOptionInt("Version", 1)); - var w = new Tux2Writer { Version = version }; - foreach (var (name, data) in FilesOnly(inputs)) - w.AddFile(name, data); - w.WriteTo(output); - } + /// + public string? TarCompressionFormatId => null; - /// - /// Performs the defragment operation. - /// - public void Defragment(Stream archive) - => this.Defragment(archive, new DefragOptions { Mode = DefragMode.ConsolidateAtStart }); + /// + public AlgorithmFamily Family => AlgorithmFamily.Archive; - /// - /// Rewrites the image with every file laid out contiguously from the start. - /// Records are copied straight from the old image into the new one, so a - /// multi-gigabyte volume never has to fit in memory — the previous refusal - /// was for want of wiring it up, not for want of a writer. - /// - /// - /// Largest container the in-place pass is offered for. Its guard holds a copy - /// of the image to compare payloads across the pass. - /// - private const long PlannerImageCap = 256L * 1024 * 1024; + /// + public string Description => + "TUX2 phase-tree research filesystem — no stable standalone on-disk signature; manual opaque image surface only."; - /// Every file's bytes, as the guard compares them before and after. - private static IReadOnlyList ReadPayloadsForGuard(Stream stream) { - stream.Position = 0; + /// + public List List(Stream stream, string? password) { using var reader = new Tux2Reader(stream); - return reader.Entries - .Where(e => !SyntheticNames.Contains(e.Name)) - .Select(reader.Extract) - .ToList(); - } - - /// Plans a record-level layout and moves the records into it. - private static void DefragmentWithPlanner(Stream archive, DefragOptions options) { - archive.Position = 0; - var mover = new Tux2BlockMover(); - - archive.Position = 0; - var extents = Tux2RecordMap.Enumerate(archive).ToList(); - if (extents.Count == 0) return; - - options.OnProgress?.Invoke(new DefragProgressEvent( - "scanning", 0, 0, -1, archive.Length, extents, "Analysing layout")); - - var moves = Compression.Core.Layout.DefragPlanner.Plan( - extents, mover.FirstDataByte, archive.Length, mover.BlockSize, - options.Profile, options.Mode, holeSize: options.HoleSize, holeAt: options.HoleAt, - metadataZone: options.MetadataZonePlacement); - if (moves.Count == 0) { - options.OnProgress?.Invoke(new DefragProgressEvent( - "complete", 1, -1, -1, archive.Length, extents, "Already defragmented")); - return; - } - - Compression.Core.Layout.DefragPlannerExecutor.Execute(archive, options, mover, moves, - archive.Length, reinitAfterMove: null); - - archive.Position = 0; - var postExtents = Tux2RecordMap.Enumerate(archive).ToList(); - options.OnProgress?.Invoke(new DefragProgressEvent( - "complete", 1, -1, -1, archive.Length, postExtents, "Defragmentation complete")); - } - - /// - /// Performs the defragment operation. - /// - public void Defragment(Stream archive, DefragOptions options) { - ArgumentNullException.ThrowIfNull(archive); - ArgumentNullException.ThrowIfNull(options); - - // Moving what is out of place beats writing the container out again, and - // on one of ours the answer is usually that nothing is: removing a file - // writes the records out packed, so there is no gap to close. What this is - // for is a container that arrived from somewhere else. - // - // The unit that moves is the whole record — a file's bytes sit behind the - // header naming them at an offset nothing records — and the walk only - // reaches a record still in order with nothing before it, which is what - // reading every payload back afterwards checks. - if (archive.CanSeek && archive.Length <= PlannerImageCap) { - var planned = false; - // The in-place pass is kept only if every payload still reads back: it - // can refuse partway, and a rebuild is the honest answer when it does. - DefragContentGuard.RunOrRebuild(archive, - readContents: ReadPayloadsForGuard, - inPlace: () => { DefragmentWithPlanner(archive, options); planned = true; }, - rebuild: () => planned = false); - if (planned) return; - archive.Position = 0; - } - // Every consolidate mode lands on the same layout here: the writer emits a - // fresh volume packed from the first data block, and has no way to place - // files against the tail. Carving a hole is the one request it cannot meet. - if (options.Mode is DefragMode.CarveHole) - throw new NotSupportedException( - "Tux2 defragmentation cannot carve a hole: the rebuild always start-packs the volume."); - - var tempPath = Path.GetTempFileName(); - try { - using (var temp = File.Open(tempPath, FileMode.Open, FileAccess.ReadWrite)) { - using (var reader = new Tux2Reader(archive)) { - var w = new Tux2Writer(); - foreach (var entry in reader.Entries) { - if (entry.IsDirectory || SyntheticNames.Contains(entry.Name)) continue; - var e = entry; - w.AddStreamingFile(e.Name, e.Size, s => reader.ExtractTo(e, s)); - } - w.WriteTo(temp); - } - - options.OnProgress?.Invoke(new DefragProgressEvent( - Phase: "commit", Fraction: 1.0, CurrentReadOffset: archive.Length, - CurrentWriteOffset: temp.Length, ImageSize: temp.Length, BlockMap: null)); - - temp.Position = 0; - archive.Position = 0; - temp.CopyTo(archive); - archive.SetLength(temp.Length); - archive.Flush(); - } - } finally { - File.Delete(tempPath); - } - } - - // ── IArchiveModifiable (genuine in-place R/W) ─────────────────────────── - // - // Tux2InPlaceModifier appends/overwrites inline records, leaving the header - // and all preceding records byte-identical at their original offsets. New - // entries are append-only; same-size replaces overwrite data in place; - // resize/delete tail-rewrite from the changed record onward (still O(tail)). - // The rebuild fallback only fires on a malformed image. - - /// - /// Adds the supplied entry to the target container. - /// - public void Add(Stream archive, IReadOnlyList inputs) { - // The in-place modifier walks the volume in memory, which a volume past two - // gigabytes does not fit in. Above that the edit unpacks and relays it out. - if (ModifyRebuilder.NeedsLargeVolumePath(archive)) { - ModifyRebuilder.AddLargeVolume(archive, inputs, this, this, SyntheticNames); - return; - } - - Tux2InPlaceModifier.Add(archive, inputs, - (a, i) => ModifyRebuilder.Add(a, i, ReadEntries, BuildImage, largeVolumeCreator: this)); - } - - /// - /// Removes the specified entry from the target container. - /// - public void Remove(Stream archive, string[] entryNames) { - // See Add: past two gigabytes the volume cannot be walked in memory. - if (ModifyRebuilder.NeedsLargeVolumePath(archive)) { - ModifyRebuilder.RemoveLargeVolume(archive, entryNames, this, this, SyntheticNames); - return; - } - - Tux2InPlaceModifier.Remove(archive, entryNames, - (a, n) => ModifyRebuilder.Remove(a, n, ReadEntries, BuildImage, largeVolumeCreator: this)); - } - - // ── Shared rebuild delegates (exclude the reader's synthetic entries) ──── - - private static IEnumerable<(string Name, byte[] Data)> ReadEntries(Stream stream) { - stream.Position = 0; - using var ms = new MemoryStream(); - stream.CopyTo(ms); - return Tux2InPlaceModifier.ReadRealEntries(ms.ToArray()).ToList(); - } - - private static byte[] BuildImage(IReadOnlyList<(string Name, byte[] Data)> files) { - var w = new Tux2Writer(); - foreach (var (n, d) in files) - if (!SyntheticNames.Contains(n)) - w.AddFile(n, d); - return w.Build(); - } - - // ── IFilesystemExtentMap / IWipeEmpty ────────────────────────────────── - - /// Header, per-record prefixes and any tail slack are metadata; each record body is the file that owns it. - public IEnumerable EnumerateExtents(Stream image) { - ArgumentNullException.ThrowIfNull(image); - var result = new List(); - try { - if (image.CanSeek) image.Position = 0; - using var reader = new Tux2Reader(image); - var cursor = 0L; - foreach (var e in reader.Entries - .Where(x => x.Offset >= 0 && x.Size > 0 && !SyntheticNames.Contains(x.Name)) - .OrderBy(x => x.Offset)) { - if (e.Offset > cursor) - result.Add(new DefragBlockInfo(cursor, e.Offset - cursor, DefragBlockKind.MetadataReserved)); - result.Add(new DefragBlockInfo(e.Offset, e.Size, DefragBlockKind.Used, e.Name)); - cursor = Math.Max(cursor, e.Offset + e.Size); - } - if (cursor == 0 && image.Length > 0) - result.Add(new DefragBlockInfo(0, Math.Min(4096, image.Length), DefragBlockKind.MetadataReserved)); - } catch { - // An image we cannot parse claims nothing, and a wipe of it would zero - // every byte — so say it has no known extents and let the caller decide. - return []; - } - return result; + return reader.Entries.Select((entry, index) => new ArchiveEntryInfo( + index, entry.Name, entry.Size, entry.Size, "Stored", entry.IsDirectory, false, null)).ToList(); } /// - public long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true) { - ArgumentNullException.ThrowIfNull(image); - var extents = this.EnumerateExtents(image).ToList(); - if (extents.Count == 0) return 0; - // Records are packed to the byte, so there are no cluster tips to trim — - // only the slack a removal or a shorter replacement left behind. - return UnusedSpaceWiper.Wipe(image, extents, image.Length, - wipeClusterTips: false, fileSizeLookup: null); - } - - - /// - /// Re-lays the volume out with the requested geometry. The generic default - /// would feed this reader's synthetic entries — the raw image and the - /// metadata sheet — back in as files; they are excluded so the rebuilt - /// volume holds the same files the original did. - /// - public void RebuildStreaming(Stream source, Stream target, LayoutRebuildOptions options) { - ArgumentNullException.ThrowIfNull(source); - ArgumentNullException.ThrowIfNull(target); - ArgumentNullException.ThrowIfNull(options); - - var parameters = new Dictionary(StringComparer.Ordinal); - if (options.Parameters != null) - foreach (var kv in options.Parameters) - parameters[kv.Key] = kv.Value; + public void Extract(Stream stream, string outputDir, string? password, string[]? files) { + using var reader = new Tux2Reader(stream); + foreach (var entry in reader.Entries) { + if (entry.IsDirectory) continue; + if (files is { Length: > 0 } && !MatchesFilter(entry.Name, files)) continue; - RebuildVerb.RebuildToStream(source, target, this, this, - parameters.Count > 0 ? parameters : null, SyntheticNames); + var target = Path.Combine(outputDir, entry.Name.Replace('/', Path.DirectorySeparatorChar)); + Directory.CreateDirectory(Path.GetDirectoryName(target) ?? outputDir); + using var output = File.Create(target); + reader.ExtractTo(entry, output); + } } - } diff --git a/FileSystems/FileSystem.Tux2/Tux2InPlaceModifier.cs b/FileSystems/FileSystem.Tux2/Tux2InPlaceModifier.cs deleted file mode 100644 index 86ff6a370..000000000 --- a/FileSystems/FileSystem.Tux2/Tux2InPlaceModifier.cs +++ /dev/null @@ -1,278 +0,0 @@ -#pragma warning disable CS1591 -using System.Buffers.Binary; -using System.Text; -using Compression.Registry; - -namespace FileSystem.Tux2; - -/// -/// Genuine in-place R/W mutation for TUX2 synthetic images. The image is a -/// 16-byte header ("TUX2FS\0\0" magic, u32 version, u32 -/// file_count) followed by back-to-back per-file records -/// (u16 nameLen, name, u32 dataLen, data). -/// -/// -/// Byte-preservation guarantees per operation: -/// -/// Add (new name) — appends a fresh record at the end of the -/// image and bumps file_count. The header's first 12 bytes and -/// every prior record stay byte-identical at their original offsets. -/// Genuine append-only in-place. -/// Replace, same encoded size — overwrites the matched record's -/// data bytes in place. Every other byte in the image (header, preceding -/// and following records) stays byte-identical. Genuine in-place. -/// Replace, different size and Remove — the inline -/// variable-length layout means resizing/dropping one record shifts every -/// following record. We rewrite the tail starting at the changed record's -/// offset; the header and all preceding records stay byte-identical -/// at their original offsets. This is a localized O(tail) relayout, not a -/// full re-encode. -/// -/// -internal static class Tux2InPlaceModifier { - - private const int HeaderSize = 16; - private const int CountOffset = 12; - - // Reader emits these synthetic non-file entries; they must never be treated - // as real records when planning a mutation. - private static readonly HashSet Synthetic = - new(StringComparer.OrdinalIgnoreCase) { "FULL.tux2", "metadata.ini" }; - - // ── Public entry points ──────────────────────────────────────────── - - public static void Add( - Stream archive, - IReadOnlyList inputs, - Action> rebuild) { - ArgumentNullException.ThrowIfNull(archive); - ArgumentNullException.ThrowIfNull(inputs); - ArgumentNullException.ThrowIfNull(rebuild); - - var payloads = new List<(string Name, byte[] Data)>(); - foreach (var (name, data) in FormatHelpers.FilesOnly(inputs)) - payloads.Add((name, data)); - if (payloads.Count == 0) return; - - archive.Position = 0; - using var ms = new MemoryStream(); - archive.CopyTo(ms); - var image = ms.ToArray(); - - if (!TryAddInPlace(image, payloads, out var result)) { - rebuild(archive, inputs); - return; - } - - archive.Position = 0; - archive.Write(result); - archive.SetLength(result.Length); - } - - public static void Remove( - Stream archive, - string[] entryNames, - Action rebuild) { - ArgumentNullException.ThrowIfNull(archive); - ArgumentNullException.ThrowIfNull(entryNames); - ArgumentNullException.ThrowIfNull(rebuild); - if (entryNames.Length == 0) return; - - archive.Position = 0; - using var ms = new MemoryStream(); - archive.CopyTo(ms); - var image = ms.ToArray(); - - if (!TryRemoveInPlace(image, entryNames, out var result)) { - rebuild(archive, entryNames); - return; - } - - archive.Position = 0; - archive.Write(result); - archive.SetLength(result.Length); - } - - // ── Core ─────────────────────────────────────────────────────────── - - /// - /// Parsed view of every real record: name, encoded byte offset, total - /// encoded length, and the data span. - /// - private readonly record struct Record(string Name, int Offset, int Length, int DataOffset, int DataLength); - - private static bool TryParse(byte[] image, out uint version, out List records) { - version = 0; - records = []; - if (image.Length < HeaderSize) return false; - if (!image.AsSpan(0, 8).SequenceEqual(Tux2Reader.Magic)) return false; - - version = BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan(8)); - var declared = BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan(CountOffset)); - - var pos = HeaderSize; - var count = 0u; - while (count < declared && pos + 2 <= image.Length) { - var start = pos; - var nameLen = BinaryPrimitives.ReadUInt16LittleEndian(image.AsSpan(pos)); - pos += 2; - if (pos + nameLen + 4 > image.Length) return false; - var name = Encoding.UTF8.GetString(image, pos, nameLen); - pos += nameLen; - var dataLen = BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan(pos)); - pos += 4; - if (dataLen > int.MaxValue || pos + (long)dataLen > image.Length) return false; - var dataOff = pos; - pos += (int)dataLen; - records.Add(new Record(name, start, pos - start, dataOff, (int)dataLen)); - count++; - } - // file_count must be accurate for a clean in-place edit. - return count == declared; - } - - private static bool TryAddInPlace(byte[] image, List<(string Name, byte[] Data)> payloads, out byte[] result) { - result = image; - if (!TryParse(image, out var version, out var records)) return false; - - // No nested directories in TUX2. - foreach (var (name, _) in payloads) - if (name.Contains('/') || name.Contains('\\') || Synthetic.Contains(name)) - return false; - - var working = new MemoryStream(); - working.Write(image, 0, image.Length); - - var liveCount = records.Count; - - foreach (var (name, data) in payloads) { - var idx = records.FindIndex(r => string.Equals(r.Name, name, StringComparison.OrdinalIgnoreCase)); - if (idx < 0) { - // New record — append at end, bump count. Genuine in-place. - AppendRecord(working, name, data); - liveCount++; - // Refresh the record table view (append doesn't shift existing records). - var arr0 = working.ToArray(); - if (!TryParse(arr0, out version, out records)) return false; - continue; - } - - var rec = records[idx]; - var newDataLen = data.Length; - if (newDataLen == rec.DataLength) { - // Same size — overwrite data bytes in place. Fully byte-preserving. - var buf = working.GetBuffer(); - data.CopyTo(buf.AsSpan(rec.DataOffset)); - // record table unchanged - continue; - } - - // Different size — tail-rewrite from this record's offset onward. - var arr = working.ToArray(); - var rebuilt = RewriteTail(arr, version, records, idx, name, data, liveCount); - working.Dispose(); - working = new MemoryStream(); - working.Write(rebuilt, 0, rebuilt.Length); - if (!TryParse(rebuilt, out version, out records)) return false; - } - - result = working.ToArray(); - working.Dispose(); - return true; - } - - private static bool TryRemoveInPlace(byte[] image, string[] entryNames, out byte[] result) { - result = image; - if (!TryParse(image, out var version, out var records)) return false; - - var toRemove = new HashSet( - entryNames.Select(n => n.Replace('\\', '/').TrimStart('/')), - StringComparer.OrdinalIgnoreCase); - - var firstHit = records.FindIndex(r => - toRemove.Contains(r.Name.Replace('\\', '/').TrimStart('/'))); - if (firstHit < 0) { - // Nothing matched among real records — nothing to do (clean success). - result = image; - return true; - } - - // Tail-rewrite from the first removed record onward, dropping every match. - var surviving = new List<(string Name, byte[] Data)>(); - for (var i = firstHit; i < records.Count; i++) { - var r = records[i]; - if (toRemove.Contains(r.Name.Replace('\\', '/').TrimStart('/'))) continue; - surviving.Add((r.Name, image.AsSpan(r.DataOffset, r.DataLength).ToArray())); - } - - var newCount = firstHit + surviving.Count; - - using var ms = new MemoryStream(); - ms.Write(image, 0, records[firstHit].Offset); // header + preceding records, byte-identical - foreach (var (name, data) in surviving) - AppendRecordBytes(ms, name, data); - PatchCount(ms, (uint)newCount); - result = ms.ToArray(); - return true; - } - - // ── Helpers ──────────────────────────────────────────────────────── - - /// Rewrites the tail of the image from record - /// onward, replacing that record's data with . - private static byte[] RewriteTail(byte[] image, uint version, List records, - int idx, string name, byte[] newData, int liveCount) { - using var ms = new MemoryStream(); - ms.Write(image, 0, records[idx].Offset); // header + preceding records - AppendRecordBytes(ms, name, newData); // replacement record - for (var i = idx + 1; i < records.Count; i++) { - var r = records[i]; - ms.Write(image, r.Offset, r.Length); // trailing records, verbatim - } - PatchCount(ms, (uint)liveCount); - return ms.ToArray(); - } - - private static void AppendRecord(MemoryStream working, string name, byte[] data) { - AppendRecordBytes(working, name, data); - PatchCount(working, ReadCount(working) + 1); - } - - private static void AppendRecordBytes(MemoryStream ms, string name, byte[] data) { - var nameBytes = Encoding.UTF8.GetBytes(name); - Span u16 = stackalloc byte[2]; - Span u32 = stackalloc byte[4]; - var save = ms.Position; - ms.Position = ms.Length; - BinaryPrimitives.WriteUInt16LittleEndian(u16, (ushort)nameBytes.Length); - ms.Write(u16); - ms.Write(nameBytes); - BinaryPrimitives.WriteUInt32LittleEndian(u32, (uint)data.Length); - ms.Write(u32); - if (data.Length > 0) ms.Write(data); - ms.Position = save; - } - - private static uint ReadCount(MemoryStream ms) { - var buf = ms.GetBuffer(); - return BinaryPrimitives.ReadUInt32LittleEndian(buf.AsSpan(CountOffset)); - } - - private static void PatchCount(MemoryStream ms, uint count) { - var save = ms.Position; - ms.Position = CountOffset; - Span u32 = stackalloc byte[4]; - BinaryPrimitives.WriteUInt32LittleEndian(u32, count); - ms.Write(u32); - ms.Position = save; - } - - /// Lists the real (non-synthetic) file records — used for the rebuild fallback. - public static IEnumerable<(string Name, byte[] Data)> ReadRealEntries(byte[] image) { - if (!TryParse(image, out _, out var records)) - yield break; - foreach (var r in records) - if (!Synthetic.Contains(r.Name)) - yield return (r.Name, image.AsSpan(r.DataOffset, r.DataLength).ToArray()); - } -} diff --git a/FileSystems/FileSystem.Tux2/Tux2Reader.cs b/FileSystems/FileSystem.Tux2/Tux2Reader.cs index 423269277..7a1e906a4 100644 --- a/FileSystems/FileSystem.Tux2/Tux2Reader.cs +++ b/FileSystems/FileSystem.Tux2/Tux2Reader.cs @@ -1,160 +1,106 @@ #pragma warning disable CS1591 -using System.Buffers.Binary; -using Compression.Core.DiskImage; -using System.Globalization; using System.Text; +using Compression.Core.DiskImage; namespace FileSystem.Tux2; /// -/// Detection-only / synthetic-image reader for TUX2 — Daniel Phillips's -/// 2000-era "phase tree" filesystem proposal. TUX2 was a research design -/// (atomic phase-tree commits, copy-on-write metadata) that never reached -/// a stable on-disk layout shipped to end users. No public spec for the -/// in-progress prototype's on-disk format ever stabilised; the project -/// was eventually superseded by TUX3. -/// -/// Because no canonical TUX2 images exist in the wild, this reader -/// recognises a deterministic synthetic header — a chosen 8-byte ASCII -/// magic "TUX2FS\0\0" at offset 0 followed by a small JSON-ish payload — -/// so that the descriptor at least round-trips its own synthetic images -/// for testing. Real TUX2 prototype dumps (if any survive) would need a -/// custom parser matching the specific cvs-era code path that produced -/// them. -/// -/// Synthetic header layout (little-endian): -/// 0x00 8 bytes Magic = "TUX2FS\0\0" -/// 0x08 u32 version (1) -/// 0x0C u32 file_count -/// 0x10 ... per-file records: -/// u16 name_len -/// name (UTF-8, name_len bytes) -/// u32 data_len -/// data (data_len bytes) +/// Opaque reader for the historical TUX2 research filesystem. /// +/// +/// TUX2 deliberately has no invented private container format here. Daniel Phillips's +/// original announcement described TUX2 as an Ext2 variation whose goals included mounting an +/// existing Ext2 partition as TUX2. No stable, independently identifying TUX2 on-disk signature +/// was published, so an image cannot be authenticated as TUX2 from a made-up magic value. +/// The reader therefore surfaces the selected image verbatim plus diagnostic metadata. It +/// reports the Ext2 superblock magic only as a compatibility clue; that magic is not evidence that +/// the image was ever written by TUX2. +/// public sealed class Tux2Reader : IDisposable { - private readonly ImageAccessor _img; - private readonly long _len; + private const long Ext2SuperblockMagicOffset = 1024 + 56; + private const ushort Ext2SuperblockMagic = 0xEF53; + + private readonly ImageAccessor _image; + private readonly long _length; private readonly List _entries = []; - /// - /// Gets the entries. - /// - public IReadOnlyList Entries => _entries; + /// Gets the entries exposed by this opaque reader. + public IReadOnlyList Entries => this._entries; - /// - /// Gets or sets the version. - /// - public uint Version { get; private set; } - /// - /// Gets or sets the file count. - /// - public uint FileCount { get; private set; } - /// - /// Gets a value indicating whether valid header. - /// - public bool ValidHeader { get; private set; } + /// Gets the total size of the selected image. + public long Length => this._length; /// - /// Provides the magic value. + /// Gets whether the image carries the Ext2 family superblock magic at the canonical offset. + /// This is only a compatibility hint, not a TUX2 identity test. /// - public static readonly byte[] Magic = "TUX2FS\0\0"u8.ToArray(); + public bool LooksLikeExt2 { get; private set; } - /// - /// Initializes a new instance of . - /// + /// Initializes a reader over the selected image. public Tux2Reader(Stream stream) { ArgumentNullException.ThrowIfNull(stream); if (stream.CanSeek) stream.Position = 0; - // Records are located on demand: copying the image in, and then every file's - // bytes out of it, held the payload twice over. - _img = new ImageAccessor(stream); - _len = _img.Length; - Parse(); + this._image = new ImageAccessor(stream); + this._length = this._image.Length; + this.Parse(); } - /// Total size of the backing image in bytes. - public long Length => this._len; - private void Parse() { - if (_len < 16) - throw new InvalidDataException("Tux2: image too small for header."); - - if (!_img.Read(0, 8).AsSpan().SequenceEqual(Magic)) - throw new InvalidDataException("Tux2: missing TUX2FS magic at offset 0."); - - this.Version = _img.ReadUInt32(8); - this.FileCount = _img.ReadUInt32(12); - this.ValidHeader = true; - - // Always emit metadata + raw image so the descriptor is useful even on - // images we can't fully parse (research-grade). - _entries.Add(new Tux2Entry { Name = "FULL.tux2", Size = _len, Offset = 0 }); - _entries.Add(new Tux2Entry { Name = "metadata.ini", Data = BuildMetadata() }); - - // Walk synthetic per-file records if the file_count looks sane. - var pos = 16L; - var count = 0u; - while (count < this.FileCount && pos + 2 <= _len) { - var nameLen = _img.ReadUInt16(pos); - pos += 2; - if (pos + nameLen + 4 > _len) break; - var name = Encoding.UTF8.GetString(_img.Read(pos, nameLen)); - pos += nameLen; - var dataLen = _img.ReadUInt32(pos); - pos += 4; - if (pos + dataLen > _len) break; - - // The bytes stay where they are; the entry records where to find them. - _entries.Add(new Tux2Entry { Name = name, Size = dataLen, Offset = pos }); - pos += dataLen; - count++; - } - - // Finalise metadata after walking so file_walk_status is set. - _entries[1] = new Tux2Entry { Name = "metadata.ini", Data = BuildMetadata(count) }; + this.LooksLikeExt2 = this._length >= Ext2SuperblockMagicOffset + sizeof(ushort) + && this._image.ReadUInt16(Ext2SuperblockMagicOffset) == Ext2SuperblockMagic; + + this._entries.Add(new Tux2Entry { + Name = "FULL.tux2", + Size = this._length, + Offset = 0, + }); + + var metadata = this.BuildMetadata(); + this._entries.Add(new Tux2Entry { + Name = "metadata.ini", + Size = metadata.LongLength, + Data = metadata, + }); } - private byte[] BuildMetadata(uint? walked = null) { - var bldr = new StringBuilder(); - bldr.Append("parse_status=ok\n"); - bldr.Append("format=TUX2 (synthetic research format)\n"); - bldr.Append(CultureInfo.InvariantCulture, $"version={this.Version}\n"); - bldr.Append(CultureInfo.InvariantCulture, $"file_count={this.FileCount}\n"); - if (walked.HasValue) - bldr.Append(CultureInfo.InvariantCulture, $"files_walked={walked.Value}\n"); - bldr.Append("note=TUX2 was Daniel Phillips's 2000 phase-tree proposal; no canonical on-disk format ever shipped.\n"); - return Encoding.UTF8.GetBytes(bldr.ToString()); + private byte[] BuildMetadata() { + var builder = new StringBuilder(); + builder.Append("parse_status=opaque\n"); + builder.Append("format=TUX2 research prototype\n"); + builder.Append("self_identifying=false\n"); + builder.Append(this.LooksLikeExt2 + ? "ext2_superblock_magic=present\n" + : "ext2_superblock_magic=absent\n"); + builder.Append("note=TUX2 targeted Ext2 compatibility and no stable standalone TUX2 disk signature/layout was published; this reader does not guess one.\n"); + return Encoding.UTF8.GetBytes(builder.ToString()); } - /// - /// Decodes the supplied input. - /// + /// Returns an entry as a byte array when it fits the CLR array limit. public byte[] Extract(Tux2Entry entry) { ArgumentNullException.ThrowIfNull(entry); if (entry.Offset < 0) return entry.Data; if (entry.Size > Array.MaxLength) - throw new IOException( - $"Tux2: '{entry.Name}' is {entry.Size:N0} bytes, past the array limit; use ExtractTo."); - return _img.Read(entry.Offset, (int)entry.Size); + throw new IOException($"Tux2: '{entry.Name}' is {entry.Size:N0} bytes, past the array limit; use ExtractTo."); + if (entry.Size == 0) return []; + return this._image.Read(entry.Offset, checked((int)entry.Size)); } - /// Writes 's bytes into . + /// Streams an entry to . public long ExtractTo(Tux2Entry entry, Stream destination) { ArgumentNullException.ThrowIfNull(entry); ArgumentNullException.ThrowIfNull(destination); + if (entry.Offset < 0) { destination.Write(entry.Data); - return entry.Data.Length; + return entry.Data.LongLength; } - var take = Math.Min(entry.Size, _len - entry.Offset); - if (take <= 0) return 0; - _img.CopyTo(entry.Offset, destination, take); - return take; + + var count = Math.Min(entry.Size, this._length - entry.Offset); + if (count <= 0) return 0; + this._image.CopyTo(entry.Offset, destination, count); + return count; } - /// - /// Releases resources held by this instance. - /// - public void Dispose() => this._img.Dispose(); + /// + public void Dispose() => this._image.Dispose(); } diff --git a/FileSystems/FileSystem.Tux2/Tux2RecordMap.cs b/FileSystems/FileSystem.Tux2/Tux2RecordMap.cs deleted file mode 100644 index d47f56c11..000000000 --- a/FileSystems/FileSystem.Tux2/Tux2RecordMap.cs +++ /dev/null @@ -1,57 +0,0 @@ -#pragma warning disable CS1591 -using System.Buffers.Binary; -using System.Text; -using Compression.Core.DiskImage; -using Compression.Registry; - -namespace FileSystem.Tux2; - -/// -/// Describes the container one whole record at a time — the name, the lengths -/// and the bytes that follow them. -/// -/// -/// This is not what the descriptor's own map describes, and the -/// difference is the point. A file's bytes sit immediately behind the header -/// naming them, at an offset nothing records: the reader finds them by adding -/// each record's length to a cursor. So the unit that can move is the record, -/// and it can only move somewhere the walk still reaches — which means the -/// records must stay in order with nothing between them. -/// -/// Which is how they always are, because removing a file writes the -/// container out packed. A pass over one of these finds nothing to move; what -/// it is for is an image that arrived from somewhere else. -/// -public static class Tux2RecordMap { - - private static readonly byte[] Magic = "TUX2FS\0\0"u8.ToArray(); - - /// Bytes of header before the first record. - internal const int HeaderSize = 16; - - /// The layout a pass plans against: the header, then one run per record. - public static IEnumerable Enumerate(Stream image) { - ArgumentNullException.ThrowIfNull(image); - - var data = new ImageAccessor(image, leaveOpen: true); - if (data.Length < HeaderSize) yield break; - if (!data.Read(0, Magic.Length).AsSpan().SequenceEqual(Magic)) yield break; - - yield return new DefragBlockInfo(0, HeaderSize, DefragBlockKind.MetadataReserved, "TUX2 header"); - - var count = BinaryPrimitives.ReadUInt32LittleEndian(data.Read(12, 4)); - var at = (long)HeaderSize; - for (var i = 0u; i < count && at + 2 <= data.Length; ++i) { - var nameLength = BinaryPrimitives.ReadUInt16LittleEndian(data.Read(at, 2)); - if (at + 2 + nameLength + 4 > data.Length) yield break; - - var name = Encoding.UTF8.GetString(data.Read(at + 2, nameLength)); - var dataLength = BinaryPrimitives.ReadUInt32LittleEndian(data.Read(at + 2 + nameLength, 4)); - var length = 2L + nameLength + 4 + dataLength; - if (at + length > data.Length) yield break; - - yield return new DefragBlockInfo(at, length, DefragBlockKind.Used, name); - at += length; - } - } -} diff --git a/FileSystems/FileSystem.Tux2/Tux2Writer.cs b/FileSystems/FileSystem.Tux2/Tux2Writer.cs deleted file mode 100644 index f14cdfea3..000000000 --- a/FileSystems/FileSystem.Tux2/Tux2Writer.cs +++ /dev/null @@ -1,117 +0,0 @@ -#pragma warning disable CS1591 -using System.Buffers.Binary; -using System.Text; - -namespace FileSystem.Tux2; - -/// -/// WORM writer for the TUX2 synthetic image layout that -/// parses. TUX2 was a 2002-era phase-tree research filesystem (Daniel Phillips, -/// kernel.org/doc/ols/2002/) whose on-disk format never stabilised — no -/// canonical real-world images exist. The reader documents (and round-trips) -/// a deterministic synthetic header that we emit here: -/// -/// -/// 0x00 8 bytes Magic = "TUX2FS\0\0" -/// 0x08 u32 version (1) -/// 0x0C u32 file_count -/// 0x10 ... per-file records: -/// u16 name_len -/// name (UTF-8, name_len bytes) -/// u32 data_len -/// data (data_len bytes) -/// -/// -/// Single-phase only (no alpha/beta phases, no version chain) — matches the -/// goal of "WORM emit single-phase image with N files (no research-level -/// snapshots)". Round-trips through . -/// -public sealed class Tux2Writer { - private readonly List _files = []; - - /// - /// Gets or sets the version. - /// - public uint Version { get; init; } = 1; - - /// One file to emit: either its bytes, or a copier that streams them. - private readonly record struct Item(string Name, long Size, byte[]? Data, Action? Copy); - - /// - /// Performs the add file operation. - /// - public void AddFile(string name, byte[] data) { - ArgumentNullException.ThrowIfNull(data); - this._files.Add(new Item(CheckName(name), data.LongLength, data, null)); - if (data.LongLength > uint.MaxValue) - throw new ArgumentException("File data length exceeds 4 GiB.", nameof(data)); - } - - /// - /// Adds a file whose bytes are written straight into the output by - /// . Nothing is buffered, so a record may be as large - /// as the record header's u32 length field allows. - /// - public void AddStreamingFile(string name, long size, Action copy) { - ArgumentNullException.ThrowIfNull(copy); - ArgumentOutOfRangeException.ThrowIfNegative(size); - if (size > uint.MaxValue) - throw new ArgumentException("File data length exceeds 4 GiB.", nameof(size)); - this._files.Add(new Item(CheckName(name), size, null, copy)); - } - - private static string CheckName(string name) { - ArgumentNullException.ThrowIfNull(name); - if (name.Length == 0) throw new ArgumentException("Name cannot be empty.", nameof(name)); - if (Encoding.UTF8.GetByteCount(name) > ushort.MaxValue) - throw new ArgumentException("Name UTF-8 length exceeds 65535 bytes.", nameof(name)); - return name; - } - - /// - /// Writes the to to the supplied output. - /// - public void WriteTo(Stream output) { - ArgumentNullException.ThrowIfNull(output); - - Span hdr = stackalloc byte[16]; - Tux2Reader.Magic.CopyTo(hdr); - BinaryPrimitives.WriteUInt32LittleEndian(hdr.Slice(8, 4), this.Version); - BinaryPrimitives.WriteUInt32LittleEndian(hdr.Slice(12, 4), (uint)this._files.Count); - output.Write(hdr); - - Span u16 = stackalloc byte[2]; - Span u32 = stackalloc byte[4]; - - foreach (var file in this._files) { - var nameBytes = Encoding.UTF8.GetBytes(file.Name); - BinaryPrimitives.WriteUInt16LittleEndian(u16, (ushort)nameBytes.Length); - output.Write(u16); - output.Write(nameBytes); - BinaryPrimitives.WriteUInt32LittleEndian(u32, (uint)file.Size); - output.Write(u32); - if (file.Size <= 0) continue; - - var before = output.Position; - if (file.Data != null) - output.Write(file.Data); - else - file.Copy!(output); - - var written = output.Position - before; - if (written != file.Size) - throw new InvalidOperationException( - $"'{file.Name}' was announced as {file.Size:N0} bytes but {written:N0} were written; " + - "the record length and the record body would disagree."); - } - } - - /// - /// Performs the build operation. - /// - public byte[] Build() { - using var ms = new MemoryStream(); - this.WriteTo(ms); - return ms.ToArray(); - } -} diff --git a/FileSystems/FileSystem.Tux3/Tux3BlockMover.cs b/FileSystems/FileSystem.Tux3/Tux3BlockMover.cs deleted file mode 100644 index e6c58f7e3..000000000 --- a/FileSystems/FileSystem.Tux3/Tux3BlockMover.cs +++ /dev/null @@ -1,58 +0,0 @@ -#pragma warning disable CS1591 -using Compression.Registry; - -namespace FileSystem.Tux3; - -/// -/// Moves a whole record inside the container, which needs nothing else -/// rewritten. -/// -/// -/// A record's data sits behind the header naming it, and the reader finds the -/// next record by adding this one's length to a cursor. So nothing records a -/// position and nothing has to be repointed — but the walk only reaches a -/// record that is still in order with nothing before it, which is what the -/// guard checks by reading every payload back afterwards. -/// -public sealed class Tux3BlockMover : IFilesystemBlockMover { - - /// A byte: records are packed to the byte, not to any larger unit. - public int BlockSize => 1; - - /// First byte a record may occupy: past the container's header. - public long FirstDataByte => Tux3RecordMap.FirstRecord; - - /// Each call moves the record it is given and nothing else. - public bool RepointsRunsIndependently => true; - - /// - /// A record may be held outside the container while the rest of the layout - /// moves, which is what lets a full one be rearranged at all. - /// - public bool SupportsHeldRuns => true; - - /// - public void MoveExtent(Stream image, long srcOffset, long dstOffset, long length, bool zeroSource = false) { - if (length <= 0 || srcOffset == dstOffset) return; - - // Overlap-safe: a run shifted forward by less than its own length - // overwrites its own tail, and copying that front to back reads bytes - // the copy has already replaced. - Compression.Core.DiskImage.ExtentCopy.Move(image, srcOffset, dstOffset, length); - if (zeroSource) - Compression.Core.DiskImage.ExtentCopy.Zero(image, srcOffset, length); - } - - /// - /// - /// Nothing to do: a record carries its own name and lengths, and where it - /// sits is recorded nowhere. - /// - /// - /// Performs the update allocation after move operation. - /// - public void UpdateAllocationAfterMove(Stream image, string fileName, long oldOffset, long newOffset, long length) { - ArgumentNullException.ThrowIfNull(image); - ArgumentNullException.ThrowIfNull(fileName); - } -} diff --git a/FileSystems/FileSystem.Tux3/Tux3FormatDescriptor.cs b/FileSystems/FileSystem.Tux3/Tux3FormatDescriptor.cs index cb19be5c3..63aaff81d 100644 --- a/FileSystems/FileSystem.Tux3/Tux3FormatDescriptor.cs +++ b/FileSystems/FileSystem.Tux3/Tux3FormatDescriptor.cs @@ -5,391 +5,80 @@ namespace FileSystem.Tux3; /// -/// Read+WORM descriptor for TUX3 — Daniel Phillips's version-tree -/// successor to TUX2 (linux-tux3 prototype). Magic "TUX3SUPR" sits -/// at file offset 4096 (the start of the superblock block). The WORM -/// writer emits a single-version image (no version chain, no atomic-commit -/// log) — the documented superblock prefix plus a sentinel "TUX3WORM" file -/// table at block 2 that walks. Full -/// itable/otable/atable B-tree traversal of real linux-tux3 prototype dumps -/// is out of scope. -/// -/// References: -/// -/// https://github.com/OGAWAHirofumi/linux-tux3 — the linux-tux3 prototype tree — canonical source -/// https://en.wikipedia.org/wiki/Tux3 — Wikipedia article -/// Daniel Phillips's Tux3 design postings (LKML / tux3 mailing list) -/// -/// -/// -/// Why there is nothing here to lay out again. +/// Read-only native-superblock descriptor for the linux-tux3 research filesystem. /// /// -/// The records run end to end and the reader walks them by adding each one's -/// length to a cursor, so a gap makes everything after it unreadable: packed -/// from the front is the only layout expressible. And it is the one the -/// container is always in, because removing a file writes it out compacted -/// rather than leaving a hole. See , which -/// has the same shape. -public sealed class Tux3FormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, IArchiveCreatable, IArchiveShrinkable, IArchiveModifiable, IArchiveDefragmentable, IFormatOptionsSchema, ILayoutOptimizable, IFilesystemExtentMap, IWipeEmpty, ISyntheticEntryNames { - - /// - public IReadOnlySet SyntheticEntryNames => SyntheticNames; - - - // ── Synthetic, non-file entries the reader always surfaces ────────────── +/// The descriptor recognises real linux-tux3 disk-format revisions and parses the packed, +/// big-endian struct disksuper at byte 4096. Native tree traversal and mutation are not +/// implemented, so Create/Modify/Defragment capabilities are intentionally withheld rather than +/// routing files through a private side-table that no TUX3 implementation understands. +/// +public sealed class Tux3FormatDescriptor : IFormatDescriptor, IArchiveFormatOperations, ISyntheticEntryNames { private static readonly HashSet SyntheticNames = new(StringComparer.OrdinalIgnoreCase) { "FULL.tux3", "metadata.ini", "superblock.bin" }; - // ── IFormatOptionsSchema ──────────────────────────────────────────────── - - /// - /// The single tunable the single-version WORM writer honours: the 64-bit - /// birthday field stamped into the superblock at offset 0x08. - /// is written verbatim and - /// reads it back, so the knob round-trips. - /// Supplied as a hexadecimal string (with or without a leading 0x); - /// left blank the writer takes the moment of creation from the clock. - /// - public IReadOnlyList OptionsSchema { get; } = [ - new FormatOptionDescriptor( - Key: "Birthday", DisplayName: "Birthday (hex)", Kind: FormatOptionKind.String, - Default: "", - Description: "64-bit creation stamp written to the superblock at offset 0x08 (hexadecimal)."), - ]; + /// + public IReadOnlySet SyntheticEntryNames => SyntheticNames; - /// - /// Gets the id. - /// + /// public string Id => "Tux3"; - /// - /// Gets the display name. - /// + + /// public string DisplayName => "TUX3"; - /// - /// Gets the category. - /// + + /// public FormatCategory Category => FormatCategory.Archive; - /// - /// Gets the capabilities. - /// + + /// public FormatCapabilities Capabilities => - FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest | - FormatCapabilities.CanCreate | FormatCapabilities.CanModify | - FormatCapabilities.SupportsMultipleEntries; - /// - /// Gets the default extension. - /// + FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanTest; + + /// public string DefaultExtension => ".tux3"; - /// - /// Gets the extensions. - /// + + /// public IReadOnlyList Extensions => [".tux3"]; - /// - /// Gets the compound extensions. - /// + + /// public IReadOnlyList CompoundExtensions => []; - /// - /// Gets the magic signatures. - /// + + /// public IReadOnlyList MagicSignatures => [ - new("TUX3SUPR"u8.ToArray(), Offset: 4096, Confidence: 0.90), + new(Tux3Reader.Magic, Offset: Tux3Reader.SuperblockOffset, Confidence: 0.99), + new(Tux3Reader.Legacy2012Magic, Offset: Tux3Reader.SuperblockOffset, Confidence: 0.95), ]; - /// - /// Gets the methods. - /// + + /// public IReadOnlyList Methods => [new("stored", "Stored")]; - /// - /// Gets the tar compression format id. - /// + + /// public string? TarCompressionFormatId => null; - /// - /// Gets the family. - /// + + /// public AlgorithmFamily Family => AlgorithmFamily.Archive; - /// - /// Gets the description. - /// - public string Description => "TUX3 version-tree research filesystem (linux-tux3) — single-version WORM image."; - /// - /// Lists the entries in the supplied container. - /// + /// + public string Description => + "TUX3 version-tree research filesystem — native big-endian superblock detection/metadata; tree traversal and writing not yet implemented."; + + /// public List List(Stream stream, string? password) { - var r = new Tux3Reader(stream); - return r.Entries.Select((e, i) => new ArchiveEntryInfo( - i, e.Name, e.Size, e.Size, "Stored", e.IsDirectory, false, null)).ToList(); + using var reader = new Tux3Reader(stream); + return reader.Entries.Select((entry, index) => new ArchiveEntryInfo( + index, entry.Name, entry.Size, entry.Size, "Stored", entry.IsDirectory, false, null)).ToList(); } - /// - /// Decodes the supplied input. - /// + /// public void Extract(Stream stream, string outputDir, string? password, string[]? files) { - using var r = new Tux3Reader(stream); - foreach (var e in r.Entries) { - if (e.IsDirectory) continue; - if (files != null && files.Length > 0 && !MatchesFilter(e.Name, files)) continue; - var target = Path.Combine(outputDir, e.Name.Replace('/', Path.DirectorySeparatorChar)); - Directory.CreateDirectory(Path.GetDirectoryName(target) ?? outputDir); - using var output = File.Create(target); - r.ExtractTo(e, output); - } - } - - /// - /// Emits a fresh single-version TUX3 image: zeroed boot region (block 0), - /// documented superblock prefix (block 1, "TUX3SUPR" magic at offset 4096), - /// and a sentinel WORM file table at block 2 carrying the per-file - /// records. Round-trips through . - /// - public void Create(Stream output, IReadOnlyList inputs, FormatCreateOptions options) { - var birthday = ParseBirthday(options.GetOption("Birthday", "")); - var w = birthday is { } stamp ? new Tux3Writer { Birthday = stamp } : new Tux3Writer(); - foreach (var (name, data) in FilesOnly(inputs)) - w.AddFile(name, data); - w.WriteTo(output); - } - - /// Parses the hex Birthday knob; blank or unreadable leaves it to the writer. - private static ulong? ParseBirthday(string value) { - var s = value.Trim(); - if (s.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) - s = s[2..]; - return ulong.TryParse(s, System.Globalization.NumberStyles.HexNumber, - System.Globalization.CultureInfo.InvariantCulture, out var n) ? n : null; - } - - /// - /// Performs the defragment operation. - /// - public void Defragment(Stream archive) - => this.Defragment(archive, new DefragOptions { Mode = DefragMode.ConsolidateAtStart }); - - /// - /// Rewrites the image with every file laid out contiguously from the start. - /// Records are copied straight from the old image into the new one, so a - /// multi-gigabyte volume never has to fit in memory — the previous refusal - /// was for want of wiring it up, not for want of a writer. - /// - /// - /// Largest container the in-place pass is offered for. Its guard holds a copy - /// of the image to compare payloads across the pass. - /// - private const long PlannerImageCap = 256L * 1024 * 1024; - - /// Every file's bytes, as the guard compares them before and after. - private static IReadOnlyList ReadPayloadsForGuard(Stream stream) { - stream.Position = 0; using var reader = new Tux3Reader(stream); - return reader.Entries - .Where(e => !SyntheticNames.Contains(e.Name)) - .Select(reader.Extract) - .ToList(); - } - - /// Plans a record-level layout and moves the records into it. - private static void DefragmentWithPlanner(Stream archive, DefragOptions options) { - archive.Position = 0; - var mover = new Tux3BlockMover(); - - archive.Position = 0; - var extents = Tux3RecordMap.Enumerate(archive).ToList(); - if (extents.Count == 0) return; - - options.OnProgress?.Invoke(new DefragProgressEvent( - "scanning", 0, 0, -1, archive.Length, extents, "Analysing layout")); - - var moves = Compression.Core.Layout.DefragPlanner.Plan( - extents, mover.FirstDataByte, archive.Length, mover.BlockSize, - options.Profile, options.Mode, holeSize: options.HoleSize, holeAt: options.HoleAt, - metadataZone: options.MetadataZonePlacement); - if (moves.Count == 0) { - options.OnProgress?.Invoke(new DefragProgressEvent( - "complete", 1, -1, -1, archive.Length, extents, "Already defragmented")); - return; - } - - Compression.Core.Layout.DefragPlannerExecutor.Execute(archive, options, mover, moves, - archive.Length, reinitAfterMove: null); - - archive.Position = 0; - var postExtents = Tux3RecordMap.Enumerate(archive).ToList(); - options.OnProgress?.Invoke(new DefragProgressEvent( - "complete", 1, -1, -1, archive.Length, postExtents, "Defragmentation complete")); - } - - /// - /// Performs the defragment operation. - /// - public void Defragment(Stream archive, DefragOptions options) { - ArgumentNullException.ThrowIfNull(archive); - ArgumentNullException.ThrowIfNull(options); + foreach (var entry in reader.Entries) { + if (entry.IsDirectory) continue; + if (files is { Length: > 0 } && !MatchesFilter(entry.Name, files)) continue; - // Moving what is out of place beats writing the container out again, and - // on one of ours the answer is usually that nothing is: removing a file - // writes the records out packed, so there is no gap to close. What this is - // for is a container that arrived from somewhere else. - // - // The unit that moves is the whole record — a file's bytes sit behind the - // header naming them at an offset nothing records — and the walk only - // reaches a record still in order with nothing before it, which is what - // reading every payload back afterwards checks. - if (archive.CanSeek && archive.Length <= PlannerImageCap) { - var planned = false; - // The in-place pass is kept only if every payload still reads back: it - // can refuse partway, and a rebuild is the honest answer when it does. - DefragContentGuard.RunOrRebuild(archive, - readContents: ReadPayloadsForGuard, - inPlace: () => { DefragmentWithPlanner(archive, options); planned = true; }, - rebuild: () => planned = false); - if (planned) return; - archive.Position = 0; - } - // Every consolidate mode lands on the same layout here: the writer emits a - // fresh volume packed from the first data block, and has no way to place - // files against the tail. Carving a hole is the one request it cannot meet. - if (options.Mode is DefragMode.CarveHole) - throw new NotSupportedException( - "Tux3 defragmentation cannot carve a hole: the rebuild always start-packs the volume."); - - var tempPath = Path.GetTempFileName(); - try { - using (var temp = File.Open(tempPath, FileMode.Open, FileAccess.ReadWrite)) { - using (var reader = new Tux3Reader(archive)) { - var w = new Tux3Writer(); - foreach (var entry in reader.Entries) { - if (entry.IsDirectory || SyntheticNames.Contains(entry.Name)) continue; - var e = entry; - w.AddStreamingFile(e.Name, e.Size, s => reader.ExtractTo(e, s)); - } - w.WriteTo(temp); - } - - options.OnProgress?.Invoke(new DefragProgressEvent( - Phase: "commit", Fraction: 1.0, CurrentReadOffset: archive.Length, - CurrentWriteOffset: temp.Length, ImageSize: temp.Length, BlockMap: null)); - - temp.Position = 0; - archive.Position = 0; - temp.CopyTo(archive); - archive.SetLength(temp.Length); - archive.Flush(); - } - } finally { - File.Delete(tempPath); - } - } - - // ── IArchiveModifiable (genuine in-place R/W) ─────────────────────────── - // - // Tux3InPlaceModifier appends/overwrites inline WORM-table records, keeping - // the boot block, superblock, table header and all preceding records - // byte-identical, then re-pads to a 4096 boundary and refreshes vol_blocks. - // New entries are append-only; same-size replaces overwrite in place; - // resize/delete tail-rewrite from the changed record onward. - - /// - /// Adds the supplied entry to the target container. - /// - public void Add(Stream archive, IReadOnlyList inputs) { - // The in-place modifier walks the volume in memory, which a volume past two - // gigabytes does not fit in. Above that the edit unpacks and relays it out. - if (ModifyRebuilder.NeedsLargeVolumePath(archive)) { - ModifyRebuilder.AddLargeVolume(archive, inputs, this, this, SyntheticNames); - return; - } - - Tux3InPlaceModifier.Add(archive, inputs, - (a, i) => ModifyRebuilder.Add(a, i, ReadEntries, BuildImage, largeVolumeCreator: this)); - } - - /// - /// Removes the specified entry from the target container. - /// - public void Remove(Stream archive, string[] entryNames) { - // See Add: past two gigabytes the volume cannot be walked in memory. - if (ModifyRebuilder.NeedsLargeVolumePath(archive)) { - ModifyRebuilder.RemoveLargeVolume(archive, entryNames, this, this, SyntheticNames); - return; - } - - Tux3InPlaceModifier.Remove(archive, entryNames, - (a, n) => ModifyRebuilder.Remove(a, n, ReadEntries, BuildImage, largeVolumeCreator: this)); - } - - // ── Shared rebuild delegates (real WORM-table records only) ───────────── - - private static IEnumerable<(string Name, byte[] Data)> ReadEntries(Stream stream) { - stream.Position = 0; - using var ms = new MemoryStream(); - stream.CopyTo(ms); - return Tux3InPlaceModifier.ReadRealEntries(ms.ToArray()).ToList(); - } - - private static byte[] BuildImage(IReadOnlyList<(string Name, byte[] Data)> files) { - var w = new Tux3Writer(); - foreach (var (n, d) in files) w.AddFile(n, d); - return w.Build(); - } - - // ── IFilesystemExtentMap / IWipeEmpty ────────────────────────────────── - - /// Boot block, superblock, WORM-table prefixes and the tail padding are metadata; each record body is the file that owns it. - public IEnumerable EnumerateExtents(Stream image) { - ArgumentNullException.ThrowIfNull(image); - var result = new List(); - try { - if (image.CanSeek) image.Position = 0; - using var reader = new Tux3Reader(image); - var cursor = 0L; - foreach (var e in reader.Entries - .Where(x => x.Offset >= 0 && x.Size > 0 && !SyntheticNames.Contains(x.Name)) - .OrderBy(x => x.Offset)) { - if (e.Offset > cursor) - result.Add(new DefragBlockInfo(cursor, e.Offset - cursor, DefragBlockKind.MetadataReserved)); - result.Add(new DefragBlockInfo(e.Offset, e.Size, DefragBlockKind.Used, e.Name)); - cursor = Math.Max(cursor, e.Offset + e.Size); - } - if (cursor == 0 && image.Length > 0) - result.Add(new DefragBlockInfo(0, Math.Min(4096, image.Length), DefragBlockKind.MetadataReserved)); - } catch { - // An image we cannot parse claims nothing, and a wipe of it would zero - // every byte — so say it has no known extents and let the caller decide. - return []; + var target = Path.Combine(outputDir, entry.Name.Replace('/', Path.DirectorySeparatorChar)); + Directory.CreateDirectory(Path.GetDirectoryName(target) ?? outputDir); + using var output = File.Create(target); + reader.ExtractTo(entry, output); } - return result; - } - - /// - public long WipeUnusedSpace(Stream image, bool wipeClusterTips = true, bool wipeDeletedEntries = true) { - ArgumentNullException.ThrowIfNull(image); - var extents = this.EnumerateExtents(image).ToList(); - if (extents.Count == 0) return 0; - // Records are packed to the byte, so there are no cluster tips to trim — - // only the slack a removal or a shorter replacement left behind. - return UnusedSpaceWiper.Wipe(image, extents, image.Length, - wipeClusterTips: false, fileSizeLookup: null); } - - - /// - /// Re-lays the volume out with the requested geometry. The generic default - /// would feed this reader's synthetic entries — the raw image and the - /// metadata sheet — back in as files; they are excluded so the rebuilt - /// volume holds the same files the original did. - /// - public void RebuildStreaming(Stream source, Stream target, LayoutRebuildOptions options) { - ArgumentNullException.ThrowIfNull(source); - ArgumentNullException.ThrowIfNull(target); - ArgumentNullException.ThrowIfNull(options); - - var parameters = new Dictionary(StringComparer.Ordinal); - if (options.Parameters != null) - foreach (var kv in options.Parameters) - parameters[kv.Key] = kv.Value; - - RebuildVerb.RebuildToStream(source, target, this, this, - parameters.Count > 0 ? parameters : null, SyntheticNames); - } - } diff --git a/FileSystems/FileSystem.Tux3/Tux3InPlaceModifier.cs b/FileSystems/FileSystem.Tux3/Tux3InPlaceModifier.cs deleted file mode 100644 index 7d1e93de8..000000000 --- a/FileSystems/FileSystem.Tux3/Tux3InPlaceModifier.cs +++ /dev/null @@ -1,278 +0,0 @@ -#pragma warning disable CS1591 -using System.Buffers.Binary; -using System.Text; -using Compression.Registry; - -namespace FileSystem.Tux3; - -/// -/// Genuine in-place R/W mutation for TUX3 single-version WORM images. The -/// on-disk surface is block 0 boot (zeroed 4096), block 1 superblock -/// ("TUX3SUPR" magic at offset 4096), block 2 the WORM file table -/// ("TUX3WORM" magic at offset 8192, u32 count at 8200, then -/// back-to-back records: u16 nameLen, name, u32 dataLen, data). -/// -/// -/// Same inline-record strategy as TUX2, with two extra invariants kept -/// intact: the image stays padded to a whole 4096-byte block, and the -/// superblock's vol_blocks field (offset 4096 + 0x38) is refreshed to -/// imageLen / 4096 whenever the image length changes. -/// Byte-preservation guarantees: -/// -/// Add (new name) — appends a record after the last existing one -/// and bumps the table count; the boot block, superblock block, table -/// header, and every prior record stay byte-identical. (When the image was -/// block-padded, the new record overwrites trailing zero padding, then we -/// re-pad and refresh vol_blocks.) -/// Replace, same size — overwrites the matched record's data in -/// place; every other byte stays identical. -/// Replace, different size and Remove — tail-rewrite from -/// the changed record's offset onward; boot+superblock+table-header+all -/// preceding records stay byte-identical. -/// -/// -internal static class Tux3InPlaceModifier { - - private const int BlockSize = 4096; - private const int SuperblockOffset = 4096; - private const int VolBlocksOffset = SuperblockOffset + 0x38; - private const int FreeBlocksOffset = SuperblockOffset + 0x40; - private const int TableOffset = 8192; - private const int CountOffset = TableOffset + 8; - private const int FirstRecordOffset = TableOffset + 12; - - // ── Public entry points ──────────────────────────────────────────── - - public static void Add( - Stream archive, - IReadOnlyList inputs, - Action> rebuild) { - ArgumentNullException.ThrowIfNull(archive); - ArgumentNullException.ThrowIfNull(inputs); - ArgumentNullException.ThrowIfNull(rebuild); - - var payloads = new List<(string Name, byte[] Data)>(); - foreach (var (name, data) in FormatHelpers.FilesOnly(inputs)) - payloads.Add((name, data)); - if (payloads.Count == 0) return; - - archive.Position = 0; - using var ms = new MemoryStream(); - archive.CopyTo(ms); - var image = ms.ToArray(); - - if (!TryAddInPlace(image, payloads, out var result)) { - rebuild(archive, inputs); - return; - } - - archive.Position = 0; - archive.Write(result); - archive.SetLength(result.Length); - } - - public static void Remove( - Stream archive, - string[] entryNames, - Action rebuild) { - ArgumentNullException.ThrowIfNull(archive); - ArgumentNullException.ThrowIfNull(entryNames); - ArgumentNullException.ThrowIfNull(rebuild); - if (entryNames.Length == 0) return; - - archive.Position = 0; - using var ms = new MemoryStream(); - archive.CopyTo(ms); - var image = ms.ToArray(); - - if (!TryRemoveInPlace(image, entryNames, out var result)) { - rebuild(archive, entryNames); - return; - } - - archive.Position = 0; - archive.Write(result); - archive.SetLength(result.Length); - } - - // ── Core ─────────────────────────────────────────────────────────── - - private readonly record struct Record(string Name, int Offset, int Length, int DataOffset, int DataLength); - - private static bool TryParse(byte[] image, out List records, out int tableEnd) { - records = []; - tableEnd = FirstRecordOffset; - if (image.Length < FirstRecordOffset) return false; - if (!image.AsSpan(SuperblockOffset, 8).SequenceEqual(Tux3Reader.Magic)) return false; - if (!image.AsSpan(TableOffset, 8).SequenceEqual(Tux3Reader.WormTableMagic)) return false; - - var declared = BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan(CountOffset)); - var pos = FirstRecordOffset; - var count = 0u; - while (count < declared && pos + 2 <= image.Length) { - var start = pos; - var nameLen = BinaryPrimitives.ReadUInt16LittleEndian(image.AsSpan(pos)); - pos += 2; - if (pos + nameLen + 4 > image.Length) return false; - var name = Encoding.UTF8.GetString(image, pos, nameLen); - pos += nameLen; - var dataLen = BinaryPrimitives.ReadUInt32LittleEndian(image.AsSpan(pos)); - pos += 4; - if (dataLen > int.MaxValue || pos + (long)dataLen > image.Length) return false; - var dataOff = pos; - pos += (int)dataLen; - records.Add(new Record(name, start, pos - start, dataOff, (int)dataLen)); - count++; - } - tableEnd = pos; - return count == declared; - } - - private static bool TryAddInPlace(byte[] image, List<(string Name, byte[] Data)> payloads, out byte[] result) { - result = image; - if (!TryParse(image, out var records, out var tableEnd)) return false; - - foreach (var (name, _) in payloads) - if (name.Contains('/') || name.Contains('\\')) - return false; - - // Build the working buffer trimmed to the live table end (drops block - // padding so appends land contiguously); re-pad + refresh vol_blocks at the - // end. Boot/superblock/table-header and preceding records are preserved. - var head = image.AsSpan(0, tableEnd).ToArray(); - using var ms = new MemoryStream(); - ms.Write(head, 0, head.Length); - - var count = (uint)records.Count; - - foreach (var (name, data) in payloads) { - var idx = records.FindIndex(r => string.Equals(r.Name, name, StringComparison.OrdinalIgnoreCase)); - if (idx < 0) { - AppendRecordBytes(ms, name, data); - count++; - continue; - } - - var rec = records[idx]; - if (data.Length == rec.DataLength) { - // Same size — patch data in place inside the already-written head copy. - var buf = ms.GetBuffer(); - data.CopyTo(buf.AsSpan(rec.DataOffset)); - continue; - } - - // Different size — rebuild the whole table region from scratch with the - // current records, applying the replacement. Simpler and still O(tail). - var rebuilt = RebuildTable(image, records, replaceIdx: idx, replaceName: name, replaceData: data, - removeIdx: -1); - ms.SetLength(0); - ms.Write(rebuilt, 0, rebuilt.Length); - // Re-parse so subsequent payloads see updated offsets/count. - var snap = ms.ToArray(); - if (!TryParse(snap, out records, out _)) return false; - count = (uint)records.Count; - } - - PatchCount(ms, count); - result = Finalize(ms.ToArray()); - return true; - } - - private static bool TryRemoveInPlace(byte[] image, string[] entryNames, out byte[] result) { - result = image; - if (!TryParse(image, out var records, out _)) return false; - - var toRemove = new HashSet( - entryNames.Select(n => n.Replace('\\', '/').TrimStart('/')), - StringComparer.OrdinalIgnoreCase); - - var anyHit = records.Any(r => toRemove.Contains(r.Name.Replace('\\', '/').TrimStart('/'))); - if (!anyHit) { result = image; return true; } - - var surviving = new List<(string Name, byte[] Data)>(); - foreach (var r in records) { - if (toRemove.Contains(r.Name.Replace('\\', '/').TrimStart('/'))) continue; - surviving.Add((r.Name, image.AsSpan(r.DataOffset, r.DataLength).ToArray())); - } - - using var ms = new MemoryStream(); - ms.Write(image, 0, FirstRecordOffset); // boot + superblock + table header, byte-identical - foreach (var (name, data) in surviving) - AppendRecordBytes(ms, name, data); - PatchCount(ms, (uint)surviving.Count); - result = Finalize(ms.ToArray()); - return true; - } - - // ── Helpers ──────────────────────────────────────────────────────── - - /// Rebuilds the whole table region preserving boot+superblock+header, - /// applying an optional replace (by index) and/or skipping a removed index. - private static byte[] RebuildTable(byte[] image, List records, - int replaceIdx, string? replaceName, byte[]? replaceData, int removeIdx) { - using var ms = new MemoryStream(); - ms.Write(image, 0, FirstRecordOffset); - var count = 0; - for (var i = 0; i < records.Count; i++) { - if (i == removeIdx) continue; - if (i == replaceIdx) { - AppendRecordBytes(ms, replaceName!, replaceData!); - } else { - var r = records[i]; - AppendRecordBytes(ms, r.Name, image.AsSpan(r.DataOffset, r.DataLength).ToArray()); - } - count++; - } - PatchCount(ms, (uint)count); - return ms.ToArray(); - } - - private static void AppendRecordBytes(MemoryStream ms, string name, byte[] data) { - var nameBytes = Encoding.UTF8.GetBytes(name); - Span u16 = stackalloc byte[2]; - Span u32 = stackalloc byte[4]; - var save = ms.Position; - ms.Position = ms.Length; - BinaryPrimitives.WriteUInt16LittleEndian(u16, (ushort)nameBytes.Length); - ms.Write(u16); - ms.Write(nameBytes); - BinaryPrimitives.WriteUInt32LittleEndian(u32, (uint)data.Length); - ms.Write(u32); - if (data.Length > 0) ms.Write(data); - ms.Position = save; - } - - private static void PatchCount(MemoryStream ms, uint count) { - var save = ms.Position; - ms.Position = CountOffset; - Span u32 = stackalloc byte[4]; - BinaryPrimitives.WriteUInt32LittleEndian(u32, count); - ms.Write(u32); - ms.Position = save; - } - - /// Pads the image up to a whole 4096-byte block and refreshes the - /// superblock's vol_blocks field. free_blocks is left at 0 - /// (matching the writer's WORM accounting). - private static byte[] Finalize(byte[] image) { - var len = image.Length; - var pad = (int)(((long)BlockSize - (len % BlockSize)) % BlockSize); - if (pad > 0) { - var grown = new byte[len + pad]; - Array.Copy(image, grown, len); - image = grown; - } - var volBlocks = (ulong)(image.Length / BlockSize); - BinaryPrimitives.WriteUInt64LittleEndian(image.AsSpan(VolBlocksOffset), volBlocks); - BinaryPrimitives.WriteUInt64LittleEndian(image.AsSpan(FreeBlocksOffset), 0UL); - return image; - } - - /// Lists the real file records — used for the rebuild fallback. - public static IEnumerable<(string Name, byte[] Data)> ReadRealEntries(byte[] image) { - if (!TryParse(image, out var records, out _)) - yield break; - foreach (var r in records) - yield return (r.Name, image.AsSpan(r.DataOffset, r.DataLength).ToArray()); - } -} diff --git a/FileSystems/FileSystem.Tux3/Tux3Reader.cs b/FileSystems/FileSystem.Tux3/Tux3Reader.cs index 932eea5c1..93bd5e12f 100644 --- a/FileSystems/FileSystem.Tux3/Tux3Reader.cs +++ b/FileSystems/FileSystem.Tux3/Tux3Reader.cs @@ -1,262 +1,161 @@ #pragma warning disable CS1591 using System.Buffers.Binary; -using Compression.Core.DiskImage; using System.Globalization; using System.Text; +using Compression.Core.DiskImage; namespace FileSystem.Tux3; /// -/// Detection / metadata-surface reader for TUX3 — Daniel Phillips's -/// successor to TUX2, a version-tree based filesystem with copy-on-write -/// metadata and atomic commit semantics. The Tux3 prototype lives in -/// the linux-tux3 tree on kernel.org and uses a superblock magic of -/// "TUX3SUPR" (8 ASCII bytes). Full B-tree traversal of itable / atable -/// is multi-week work; this reader surfaces the parsed superblock as -/// structured metadata plus the raw image. -/// -/// Superblock layout (the documented prefix; little-endian; sits at -/// file offset 4096 == one 4KiB block): -/// 0x00 8 bytes Magic = "TUX3SUPR" -/// 0x08 u64 birthday -/// 0x10 u64 flags -/// 0x18 u64 iroot (root of itable B-tree) -/// 0x20 u64 oroot (root of otable B-tree) -/// 0x28 u64 aroot (root of atable B-tree) -/// 0x30 u64 blockbits -/// 0x38 u64 volblocks -/// 0x40 u64 freeblocks -/// 0x48 u64 nextalloc -/// 0x50 u32 atomgen -/// 0x54 u32 freeatom -/// ... -/// -/// -/// On top of the documented superblock surface, this reader also recognises -/// an optional WORM file table emitted by : -/// a sentinel header "TUX3WORM" placed at offset 8192 (block 2, -/// immediately after the superblock block) followed by a u32 file count and -/// per-file records (u16 name length, UTF-8 name, u32 data length, raw -/// bytes). Single-version WORM images created by -/// round-trip through this reader; B-tree-formatted prototype images -/// continue to surface only as FULL.tux3 + metadata.ini + -/// superblock.bin. -/// +/// Native-superblock reader for the linux-tux3 research filesystem. /// +/// +/// The on-disk structure is taken from the canonical linux-tux3 struct disksuper: +/// it starts at byte 4096, is packed, and all integer fields are big-endian. +/// This reader intentionally stops at the superblock. It does not interpret the inode, +/// orphan, allocation, atom or directory trees and therefore exposes no invented file table. +/// The old private TUX3SUPR/TUX3WORM dialect is not accepted. +/// public sealed class Tux3Reader : IDisposable { - private readonly ImageAccessor _img; - private readonly long _len; + /// Current linux-tux3 disk-format magic (2014-05-06 revision). + public static readonly byte[] Magic = [0x74, 0x75, 0x78, 0x33, 0x20, 0x14, 0x05, 0x06]; + + /// Older 2012-12-20 userspace-tree disk-format magic. + public static readonly byte[] Legacy2012Magic = [0x74, 0x75, 0x78, 0x33, 0x20, 0x12, 0x12, 0x20]; + + /// Fixed byte offset of struct disksuper. + public const int SuperblockOffset = 1 << 12; + + /// Size in bytes of the packed current struct disksuper. + public const int DiskSuperSize = 0x64; + + private readonly ImageAccessor _image; + private readonly long _length; private readonly List _entries = []; - /// - /// Gets the entries. - /// - public IReadOnlyList Entries => _entries; + /// Gets the entries exposed by this metadata-only reader. + public IReadOnlyList Entries => this._entries; + + /// Gets the total image size. + public long Length => this._length; + + /// Gets whether a supported native superblock was parsed. + public bool ValidSuperblock { get; private set; } + + /// Gets the disk-format revision identified by the eight-byte magic. + public string Revision { get; private set; } = ""; - /// - /// Gets or sets the birthday. - /// public ulong Birthday { get; private set; } - /// - /// Gets or sets the flags. - /// public ulong Flags { get; private set; } - /// - /// Gets or sets the i root. - /// + public ushort BlockBits { get; private set; } + public ulong VolBlocks { get; private set; } public ulong IRoot { get; private set; } - /// - /// Gets or sets the o root. - /// public ulong ORoot { get; private set; } - /// - /// Gets or sets the a root. - /// - public ulong ARoot { get; private set; } - /// - /// Gets or sets the block bits. - /// - public ulong BlockBits { get; private set; } - /// - /// Gets or sets the vol blocks. - /// - public ulong VolBlocks { get; private set; } - /// - /// Gets or sets the free blocks. - /// - public ulong FreeBlocks { get; private set; } - /// - /// Gets a value indicating whether valid superblock. - /// - public bool ValidSuperblock { get; private set; } - /// - /// Gets a value indicating whether has worm table. - /// - public bool HasWormTable { get; private set; } - /// - /// Gets or sets the worm file count. - /// - public uint WormFileCount { get; private set; } - - /// - /// Provides the magic value. - /// - public static readonly byte[] Magic = "TUX3SUPR"u8.ToArray(); - - /// - /// Sentinel marker for the optional WORM file table appended after the - /// superblock at . - /// - public static readonly byte[] WormTableMagic = "TUX3WORM"u8.ToArray(); - - /// - /// Defines the superblock offset constant value. - /// - public const int SuperblockOffset = 4096; - /// - /// Defines the worm table offset constant value. - /// - public const int WormTableOffset = 8192; - - /// - /// Initializes a new instance of . - /// + public ulong UsedInodes { get; private set; } + public ulong NextBlock { get; private set; } + public ulong AtomDictionarySize { get; private set; } + public uint FreeAtom { get; private set; } + public uint AtomGeneration { get; private set; } + public ulong LogChain { get; private set; } + public uint LogCount { get; private set; } + + /// Initializes a reader over a TUX3 image. public Tux3Reader(Stream stream) { ArgumentNullException.ThrowIfNull(stream); if (stream.CanSeek) stream.Position = 0; - // Records are located on demand: copying the image in, and then every file's - // bytes out of it, held the payload twice over. - _img = new ImageAccessor(stream); - _len = _img.Length; - Parse(); + this._image = new ImageAccessor(stream); + this._length = this._image.Length; + this.Parse(); } - /// Total size of the backing image in bytes. - public long Length => this._len; - private void Parse() { - if (_len < SuperblockOffset + 0x60) - throw new InvalidDataException("Tux3: image too small for superblock."); - - var sb = _img.Read(SuperblockOffset, Math.Min(512, (int)Math.Min(int.MaxValue, _len - SuperblockOffset))).AsSpan(); - if (!sb.Slice(0, 8).SequenceEqual(Magic)) - throw new InvalidDataException("Tux3: missing TUX3SUPR magic at superblock offset 4096."); - + if (this._length < SuperblockOffset + DiskSuperSize) + throw new InvalidDataException("Tux3: image too small for the native disksuper at byte 4096."); + + var super = this._image.Read(SuperblockOffset, DiskSuperSize); + var span = super.AsSpan(); + var magic = span[..8]; + + if (magic.SequenceEqual(Magic)) + this.Revision = "2014-05-06"; + else if (magic.SequenceEqual(Legacy2012Magic)) + this.Revision = "2012-12-20"; + else + throw new InvalidDataException("Tux3: unsupported or missing native disk-format magic at byte 4096."); + + this.Birthday = BinaryPrimitives.ReadUInt64BigEndian(span.Slice(0x08, 8)); + this.Flags = BinaryPrimitives.ReadUInt64BigEndian(span.Slice(0x10, 8)); + this.BlockBits = BinaryPrimitives.ReadUInt16BigEndian(span.Slice(0x18, 2)); + this.VolBlocks = BinaryPrimitives.ReadUInt64BigEndian(span.Slice(0x20, 8)); + this.IRoot = BinaryPrimitives.ReadUInt64BigEndian(span.Slice(0x28, 8)); + this.ORoot = BinaryPrimitives.ReadUInt64BigEndian(span.Slice(0x30, 8)); + this.UsedInodes = BinaryPrimitives.ReadUInt64BigEndian(span.Slice(0x38, 8)); + this.NextBlock = BinaryPrimitives.ReadUInt64BigEndian(span.Slice(0x40, 8)); + this.AtomDictionarySize = BinaryPrimitives.ReadUInt64BigEndian(span.Slice(0x48, 8)); + this.FreeAtom = BinaryPrimitives.ReadUInt32BigEndian(span.Slice(0x50, 4)); + this.AtomGeneration = BinaryPrimitives.ReadUInt32BigEndian(span.Slice(0x54, 4)); + this.LogChain = BinaryPrimitives.ReadUInt64BigEndian(span.Slice(0x58, 8)); + this.LogCount = BinaryPrimitives.ReadUInt32BigEndian(span.Slice(0x60, 4)); this.ValidSuperblock = true; - this.Birthday = BinaryPrimitives.ReadUInt64LittleEndian(sb.Slice(0x08)); - this.Flags = BinaryPrimitives.ReadUInt64LittleEndian(sb.Slice(0x10)); - this.IRoot = BinaryPrimitives.ReadUInt64LittleEndian(sb.Slice(0x18)); - this.ORoot = BinaryPrimitives.ReadUInt64LittleEndian(sb.Slice(0x20)); - this.ARoot = BinaryPrimitives.ReadUInt64LittleEndian(sb.Slice(0x28)); - this.BlockBits = BinaryPrimitives.ReadUInt64LittleEndian(sb.Slice(0x30)); - this.VolBlocks = BinaryPrimitives.ReadUInt64LittleEndian(sb.Slice(0x38)); - this.FreeBlocks = BinaryPrimitives.ReadUInt64LittleEndian(sb.Slice(0x40)); - - var sbBytes = sb.ToArray(); - - _entries.Add(new Tux3Entry { Name = "FULL.tux3", Size = _len, Offset = 0 }); - // Probe for the optional WORM table. We add the placeholder for metadata - // first so it stays at index 1, then walk WORM entries, then finalise - // metadata once we know the walk count. - _entries.Add(new Tux3Entry { Name = "metadata.ini", Data = [] }); - _entries.Add(new Tux3Entry { Name = "superblock.bin", Size = sbBytes.Length, Data = sbBytes }); + this._entries.Add(new Tux3Entry { Name = "FULL.tux3", Size = this._length, Offset = 0 }); - var walked = TryWalkWormTable(); - - var meta = BuildMetadata(walked); - _entries[1] = new Tux3Entry { Name = "metadata.ini", Size = meta.Length, Data = meta }; + var metadata = this.BuildMetadata(); + this._entries.Add(new Tux3Entry { Name = "metadata.ini", Size = metadata.LongLength, Data = metadata }); + this._entries.Add(new Tux3Entry { Name = "superblock.bin", Size = super.LongLength, Data = super }); } - /// - /// Probes for and walks the optional WORM file table at offset 8192. - /// Returns the number of records successfully decoded, or null if no WORM - /// table is present. - /// - private uint? TryWalkWormTable() { - // Need at least sentinel (8) + u32 count. - if (_len < WormTableOffset + 12) return null; - var tbl = _img.Read(WormTableOffset, 12).AsSpan(); - if (!tbl[..8].SequenceEqual(WormTableMagic)) return null; - - this.HasWormTable = true; - var declared = BinaryPrimitives.ReadUInt32LittleEndian(tbl.Slice(8, 4)); - this.WormFileCount = declared; - - var pos = (long)WormTableOffset + 12; - var count = 0u; - while (count < declared && pos + 2 <= _len) { - var nameLen = _img.ReadUInt16(pos); - pos += 2; - if (pos + nameLen + 4 > _len) break; - var name = Encoding.UTF8.GetString(_img.Read(pos, nameLen)); - pos += nameLen; - var dataLen = _img.ReadUInt32(pos); - pos += 4; - if (pos + dataLen > _len) break; - - // The bytes stay where they are; the entry records where to find them. - _entries.Add(new Tux3Entry { Name = name, Size = dataLen, Offset = pos }); - pos += dataLen; - count++; - } - return count; - } - - private byte[] BuildMetadata(uint? walked) { - var bldr = new StringBuilder(); - bldr.Append("parse_status=ok\n"); - bldr.Append("format=TUX3 (linux-tux3 prototype)\n"); - bldr.Append(CultureInfo.InvariantCulture, $"superblock_offset={SuperblockOffset}\n"); - bldr.Append(CultureInfo.InvariantCulture, $"birthday=0x{this.Birthday:X16}\n"); - bldr.Append(CultureInfo.InvariantCulture, $"flags=0x{this.Flags:X16}\n"); - bldr.Append(CultureInfo.InvariantCulture, $"iroot=0x{this.IRoot:X16}\n"); - bldr.Append(CultureInfo.InvariantCulture, $"oroot=0x{this.ORoot:X16}\n"); - bldr.Append(CultureInfo.InvariantCulture, $"aroot=0x{this.ARoot:X16}\n"); - bldr.Append(CultureInfo.InvariantCulture, $"blockbits={this.BlockBits}\n"); - bldr.Append(CultureInfo.InvariantCulture, $"block_size={(this.BlockBits == 0 ? 0 : 1ul << (int)this.BlockBits)}\n"); - bldr.Append(CultureInfo.InvariantCulture, $"vol_blocks={this.VolBlocks}\n"); - bldr.Append(CultureInfo.InvariantCulture, $"free_blocks={this.FreeBlocks}\n"); - if (this.HasWormTable) { - bldr.Append(CultureInfo.InvariantCulture, $"worm_table=present (offset={WormTableOffset})\n"); - bldr.Append(CultureInfo.InvariantCulture, $"worm_file_count={this.WormFileCount}\n"); - if (walked.HasValue) - bldr.Append(CultureInfo.InvariantCulture, $"worm_files_walked={walked.Value}\n"); - } else { - bldr.Append("worm_table=absent\n"); - bldr.Append("note=itable/otable/atable B-tree traversal not implemented (research read-only).\n"); - } - return Encoding.UTF8.GetBytes(bldr.ToString()); + private byte[] BuildMetadata() { + var builder = new StringBuilder(); + builder.Append("parse_status=superblock-only\n"); + builder.Append("format=TUX3 (linux-tux3 prototype)\n"); + builder.Append(CultureInfo.InvariantCulture, $"revision={this.Revision}\n"); + builder.Append(CultureInfo.InvariantCulture, $"superblock_offset={SuperblockOffset}\n"); + builder.Append(CultureInfo.InvariantCulture, $"birthday=0x{this.Birthday:X16}\n"); + builder.Append(CultureInfo.InvariantCulture, $"flags=0x{this.Flags:X16}\n"); + builder.Append(CultureInfo.InvariantCulture, $"blockbits={this.BlockBits}\n"); + if (this.BlockBits < 63) + builder.Append(CultureInfo.InvariantCulture, $"block_size={1UL << this.BlockBits}\n"); + builder.Append(CultureInfo.InvariantCulture, $"volblocks={this.VolBlocks}\n"); + builder.Append(CultureInfo.InvariantCulture, $"iroot=0x{this.IRoot:X16}\n"); + builder.Append(CultureInfo.InvariantCulture, $"oroot=0x{this.ORoot:X16}\n"); + builder.Append(CultureInfo.InvariantCulture, $"usedinodes={this.UsedInodes}\n"); + builder.Append(CultureInfo.InvariantCulture, $"nextblock={this.NextBlock}\n"); + builder.Append(CultureInfo.InvariantCulture, $"atomdictsize={this.AtomDictionarySize}\n"); + builder.Append(CultureInfo.InvariantCulture, $"freeatom={this.FreeAtom}\n"); + builder.Append(CultureInfo.InvariantCulture, $"atomgen={this.AtomGeneration}\n"); + builder.Append(CultureInfo.InvariantCulture, $"logchain=0x{this.LogChain:X16}\n"); + builder.Append(CultureInfo.InvariantCulture, $"logcount={this.LogCount}\n"); + builder.Append("note=Native itable/otable/allocation/directory traversal is not implemented; no private file table is assumed.\n"); + return Encoding.UTF8.GetBytes(builder.ToString()); } - /// - /// Decodes the supplied input. - /// + /// Returns an entry as a byte array when it fits the CLR array limit. public byte[] Extract(Tux3Entry entry) { ArgumentNullException.ThrowIfNull(entry); if (entry.Offset < 0) return entry.Data; if (entry.Size > Array.MaxLength) - throw new IOException( - $"Tux3: '{entry.Name}' is {entry.Size:N0} bytes, past the array limit; use ExtractTo."); - return _img.Read(entry.Offset, (int)entry.Size); + throw new IOException($"Tux3: '{entry.Name}' is {entry.Size:N0} bytes, past the array limit; use ExtractTo."); + if (entry.Size == 0) return []; + return this._image.Read(entry.Offset, checked((int)entry.Size)); } - /// Writes 's bytes into . + /// Streams an entry to . public long ExtractTo(Tux3Entry entry, Stream destination) { ArgumentNullException.ThrowIfNull(entry); ArgumentNullException.ThrowIfNull(destination); + if (entry.Offset < 0) { destination.Write(entry.Data); - return entry.Data.Length; + return entry.Data.LongLength; } - var take = Math.Min(entry.Size, _len - entry.Offset); - if (take <= 0) return 0; - _img.CopyTo(entry.Offset, destination, take); - return take; + + var count = Math.Min(entry.Size, this._length - entry.Offset); + if (count <= 0) return 0; + this._image.CopyTo(entry.Offset, destination, count); + return count; } - /// - /// Releases resources held by this instance. - /// - public void Dispose() => this._img.Dispose(); + /// + public void Dispose() => this._image.Dispose(); } diff --git a/FileSystems/FileSystem.Tux3/Tux3RecordMap.cs b/FileSystems/FileSystem.Tux3/Tux3RecordMap.cs deleted file mode 100644 index 27c7eaf1f..000000000 --- a/FileSystems/FileSystem.Tux3/Tux3RecordMap.cs +++ /dev/null @@ -1,62 +0,0 @@ -#pragma warning disable CS1591 -using System.Buffers.Binary; -using System.Text; -using Compression.Core.DiskImage; -using Compression.Registry; - -namespace FileSystem.Tux3; - -/// -/// Describes the WORM table one whole record at a time — the name, the length -/// and the bytes that follow them. -/// -/// -/// A file's bytes sit immediately behind the header naming them, at an -/// offset nothing records: the reader finds the next record by adding this -/// one's length to a cursor. So the unit that can move is the record, and it -/// can only move somewhere the walk still reaches — the records must stay in -/// some order with nothing between them. -/// -/// Which is how they always are on one of ours, because removing a file -/// writes the container out packed. A pass finds nothing to move; what it is -/// for is a container that arrived from somewhere else. -/// -public static class Tux3RecordMap { - - /// Where the table of records begins. - internal const long TableOffset = Tux3Reader.WormTableOffset; - - /// Bytes of table header before the first record. - internal const int HeaderSize = 12; - - /// First byte a record may occupy. - internal const long FirstRecord = TableOffset + HeaderSize; - - /// The layout a pass plans against: the head, then one run per record. - public static IEnumerable Enumerate(Stream image) { - ArgumentNullException.ThrowIfNull(image); - - var data = new ImageAccessor(image, leaveOpen: true); - var magic = Tux3Reader.WormTableMagic; - if (data.Length < FirstRecord) yield break; - if (!data.Read(TableOffset, magic.Length).AsSpan().SequenceEqual(magic)) yield break; - - yield return new DefragBlockInfo(0, FirstRecord, DefragBlockKind.MetadataReserved, - "TUX3 superblock and table header"); - - var count = BinaryPrimitives.ReadUInt32LittleEndian(data.Read(TableOffset + 8, 4)); - var at = FirstRecord; - for (var i = 0u; i < count && at + 2 <= data.Length; ++i) { - var nameLength = BinaryPrimitives.ReadUInt16LittleEndian(data.Read(at, 2)); - if (at + 2 + nameLength + 4 > data.Length) yield break; - - var name = Encoding.UTF8.GetString(data.Read(at + 2, nameLength)); - var dataLength = BinaryPrimitives.ReadUInt32LittleEndian(data.Read(at + 2 + nameLength, 4)); - var length = 2L + nameLength + 4 + dataLength; - if (at + length > data.Length) yield break; - - yield return new DefragBlockInfo(at, length, DefragBlockKind.Used, name); - at += length; - } - } -} diff --git a/FileSystems/FileSystem.Tux3/Tux3Writer.cs b/FileSystems/FileSystem.Tux3/Tux3Writer.cs deleted file mode 100644 index 27c383d4d..000000000 --- a/FileSystems/FileSystem.Tux3/Tux3Writer.cs +++ /dev/null @@ -1,200 +0,0 @@ -#pragma warning disable CS1591 -using System.Buffers.Binary; -using System.Text; - -namespace FileSystem.Tux3; - -/// -/// WORM writer for the TUX3 prototype on-disk surface that -/// parses. TUX3 was Daniel Phillips's version-tree successor to TUX2; the -/// linux-tux3 prototype was never declared stable, so this writer emits the -/// documented superblock prefix (magic "TUX3SUPR" at block offset 4096 plus -/// the documented 0x60-byte field set) followed by a sentinel WORM file -/// table at block 2 (offset 8192). The version-tree itself is collapsed to a -/// single version — no version chain, no atomic-commit log — matching the -/// goal "WORM emit single-version image with N files". -/// -/// -/// Layout produced (little-endian): -/// -/// -/// 0x0000 zeroed boot region (4096 bytes, block 0) -/// 0x1000 TUX3 superblock: -/// +0x00 8 bytes Magic = "TUX3SUPR" -/// +0x08 u64 birthday -/// +0x10 u64 flags (0) -/// +0x18 u64 iroot (0 — no B-tree) -/// +0x20 u64 oroot (0) -/// +0x28 u64 aroot (0) -/// +0x30 u64 blockbits (12 — 4096-byte blocks) -/// +0x38 u64 volblocks (image size / 4096) -/// +0x40 u64 freeblocks (volblocks − reserved) -/// +0x48 u64 nextalloc -/// +0x50 u32 atomgen -/// +0x54 u32 freeatom -/// ...zero-padded to end of block... -/// 0x2000 WORM file table (block 2): -/// +0x00 8 bytes Sentinel "TUX3WORM" -/// +0x08 u32 file_count -/// +0x0C ... per-file records: -/// u16 name_len -/// name (UTF-8, name_len bytes) -/// u32 data_len -/// data (data_len bytes) -/// -/// -/// -/// Round-trips through . Real linux-tux3 prototype -/// dumps that use the itable/otable/atable B-trees are not emitted -/// by this writer (the B-tree code paths in the prototype were never -/// stabilised); a real-world dump would need a full B-tree writer. -/// -/// -public sealed class Tux3Writer { - private readonly List _files = []; - - /// - /// When the volume claims it was made. Taken from the clock unless set, because - /// a birthday that reads the same on every volume is a maker's mark. - /// - public ulong Birthday { get; init; } = (ulong)DateTimeOffset.UtcNow.ToUnixTimeSeconds(); - /// - /// Gets or sets the flags. - /// - public ulong Flags { get; init; } - /// - /// Gets or sets the block bits. - /// - public ulong BlockBits { get; init; } = 12; // 4 KiB blocks (matches Tux3 prototype default) - - /// One file to emit: either its bytes, or a copier that streams them. - private readonly record struct Item(string Name, long Size, byte[]? Data, Action? Copy); - - /// - /// Performs the add file operation. - /// - public void AddFile(string name, byte[] data) { - ArgumentNullException.ThrowIfNull(data); - this._files.Add(new Item(CheckName(name), data.LongLength, data, null)); - if (data.LongLength > uint.MaxValue) - throw new ArgumentException("File data length exceeds 4 GiB.", nameof(data)); - } - - /// - /// Adds a file whose bytes are written straight into the output by - /// . Nothing is buffered, so a record may be as large - /// as the record header's u32 length field allows. - /// - public void AddStreamingFile(string name, long size, Action copy) { - ArgumentNullException.ThrowIfNull(copy); - ArgumentOutOfRangeException.ThrowIfNegative(size); - if (size > uint.MaxValue) - throw new ArgumentException("File data length exceeds 4 GiB.", nameof(size)); - this._files.Add(new Item(CheckName(name), size, null, copy)); - } - - private static string CheckName(string name) { - ArgumentNullException.ThrowIfNull(name); - if (name.Length == 0) throw new ArgumentException("Name cannot be empty.", nameof(name)); - if (Encoding.UTF8.GetByteCount(name) > ushort.MaxValue) - throw new ArgumentException("Name UTF-8 length exceeds 65535 bytes.", nameof(name)); - return name; - } - - /// - /// Writes the to to the supplied output. - /// - public void WriteTo(Stream output) { - ArgumentNullException.ThrowIfNull(output); - - var blockSize = 1 << (int)this.BlockBits; - if (blockSize < 4096) throw new InvalidOperationException("BlockBits must give a block size >= 4096."); - - // Block 0: reserved (zeroed). Block 1: superblock. Block 2: WORM table. - var bootRegion = new byte[blockSize]; - output.Write(bootRegion); - - // Superblock block — documented 0x60-byte prefix + zero pad. - var sb = new byte[blockSize]; - Tux3Reader.Magic.CopyTo(sb.AsSpan(0)); - BinaryPrimitives.WriteUInt64LittleEndian(sb.AsSpan(0x08), this.Birthday); - BinaryPrimitives.WriteUInt64LittleEndian(sb.AsSpan(0x10), this.Flags); - // iroot/oroot/aroot all 0 — no B-tree in single-version WORM mode. - BinaryPrimitives.WriteUInt64LittleEndian(sb.AsSpan(0x18), 0); - BinaryPrimitives.WriteUInt64LittleEndian(sb.AsSpan(0x20), 0); - BinaryPrimitives.WriteUInt64LittleEndian(sb.AsSpan(0x28), 0); - BinaryPrimitives.WriteUInt64LittleEndian(sb.AsSpan(0x30), this.BlockBits); - - // Compute vol_blocks / free_blocks after we know the on-disk size — write - // pass 1 placeholder here, patch after we've serialised everything. - BinaryPrimitives.WriteUInt64LittleEndian(sb.AsSpan(0x38), 0); // vol_blocks (patched) - BinaryPrimitives.WriteUInt64LittleEndian(sb.AsSpan(0x40), 0); // free_blocks (patched) - BinaryPrimitives.WriteUInt64LittleEndian(sb.AsSpan(0x48), 3); // nextalloc — past block 2 (WORM table) - BinaryPrimitives.WriteUInt32LittleEndian(sb.AsSpan(0x50), 0); // atomgen - BinaryPrimitives.WriteUInt32LittleEndian(sb.AsSpan(0x54), 0); // freeatom - - var sbStartPos = output.Position; - output.Write(sb); - - // Block 2: WORM file table. - Span wormHdr = stackalloc byte[12]; - Tux3Reader.WormTableMagic.CopyTo(wormHdr); - BinaryPrimitives.WriteUInt32LittleEndian(wormHdr.Slice(8, 4), (uint)this._files.Count); - output.Write(wormHdr); - - Span u16 = stackalloc byte[2]; - Span u32 = stackalloc byte[4]; - - foreach (var file in this._files) { - var nameBytes = Encoding.UTF8.GetBytes(file.Name); - BinaryPrimitives.WriteUInt16LittleEndian(u16, (ushort)nameBytes.Length); - output.Write(u16); - output.Write(nameBytes); - BinaryPrimitives.WriteUInt32LittleEndian(u32, (uint)file.Size); - output.Write(u32); - if (file.Size <= 0) continue; - - var before = output.Position; - if (file.Data != null) - output.Write(file.Data); - else - file.Copy!(output); - - var written = output.Position - before; - if (written != file.Size) - throw new InvalidOperationException( - $"'{file.Name}' was announced as {file.Size:N0} bytes but {written:N0} were written; " + - "the record length and the record body would disagree."); - } - - // Pad to a whole block so vol_blocks accurately reflects the image. - var endPos = output.Position; - var blockPad = (int)((blockSize - (endPos % blockSize)) % blockSize); - if (blockPad > 0) output.Write(new byte[blockPad]); - - // Patch vol_blocks / free_blocks now we know the final image size. - var finalLen = output.Position; - var volBlocks = (ulong)(finalLen / blockSize); - // Reserved: block 0 (boot), block 1 (superblock), block 2 (WORM table) = - // 3 blocks. Treat all data-bearing blocks past that as "used"; we don't - // track per-file allocation in WORM mode, so free_blocks is the tail - // padding region (0 by default since we pad up to the next block). - var reserved = 3UL; - var freeBlocks = volBlocks > reserved ? 0UL : 0UL; - output.Position = sbStartPos + 0x38; - Span patch = stackalloc byte[16]; - BinaryPrimitives.WriteUInt64LittleEndian(patch.Slice(0, 8), volBlocks); - BinaryPrimitives.WriteUInt64LittleEndian(patch.Slice(8, 8), freeBlocks); - output.Write(patch); - output.Position = finalLen; - } - - /// - /// Performs the build operation. - /// - public byte[] Build() { - using var ms = new MemoryStream(); - this.WriteTo(ms); - return ms.ToArray(); - } -}