diff --git a/CHANGELOG.md b/CHANGELOG.md index c0aa34813..9f1403511 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The following emojis are used to highlight certain changes: ### Added +- ✨ `ipld/unixfs`: reads of both `PBNode` field orders are now covered by tests, and a documented low-level opt-in (`UnixFSProfile.PBNodeFieldOrder`, applied via `merkledag.DefaultPBNodeFieldOrder`) lets writers that need streaming-friendly blocks encode the `Data` field before `Links` per [IPIP-550](https://github.com/ipfs/specs/pull/550). Off by default and selected by no named profile: `UnixFS_v0_2015` and `UnixFS_v1_2025` pin the canonical links-first order explicitly, so defaults and existing CIDs are unchanged. Enabling data-first changes the CID of every dag-pb node that has both fields (directories, HAMT shards, multi-chunk file roots), is process-wide (`ApplyGlobals` affects every `merkledag.ProtoNode` encoded in the process, not only UnixFS nodes), and re-encodes links-first directories in the new order the next time they are opened through the directory API and stored again (for example MFS directories on their next access). [#1212](https://github.com/ipfs/boxo/pull/1212) - ✨ `gateway`: responses now include the `Ipfs-Uri` header with a canonical `ipfs://` or `ipns://` URI for the requested content path, and expose it via the default `Access-Control-Expose-Headers`. The header carries the content root in canonical form (base32 CIDv1 for `/ipfs/`, base36 CIDv1 for cryptographic `/ipns/` names, lowercase FQDN for DNSLink) with percent-encoded path segments, so clients get a value that is safe in HTTP field context regardless of bytes in the underlying path. [IPIP-548](https://github.com/ipfs/specs/pull/548) [#1209](https://github.com/ipfs/boxo/pull/1209) ### Changed diff --git a/ipld/merkledag/coding.go b/ipld/merkledag/coding.go index 035e964ef..228a004ca 100644 --- a/ipld/merkledag/coding.go +++ b/ipld/merkledag/coding.go @@ -104,6 +104,15 @@ func (n *ProtoNode) marshalImmutable() (*immutableProtoNode, error) { if err != nil { return nil, err } + switch order := DefaultPBNodeFieldOrder; order { + case PBNodeLinksFirst: + case PBNodeDataFirst: + if n.data != nil { + enc = moveDataFirst(enc, len(n.data)) + } + default: + return nil, fmt.Errorf("unknown PBNodeFieldOrder %d", order) + } return &immutableProtoNode{enc, nd.(dagpb.PBNode)}, nil } diff --git a/ipld/merkledag/fieldorder.go b/ipld/merkledag/fieldorder.go new file mode 100644 index 000000000..61068e2a0 --- /dev/null +++ b/ipld/merkledag/fieldorder.go @@ -0,0 +1,71 @@ +package merkledag + +import ( + "slices" + + "google.golang.org/protobuf/encoding/protowire" +) + +// PBNodeFieldOrder selects the order of the top-level PBNode fields in the +// serialized dag-pb form. Both orders decode to the same logical node, but +// produce different bytes and therefore different CIDs. +type PBNodeFieldOrder int + +const ( + // PBNodeLinksFirst writes the repeated Links field (field number 2) + // before the Data field (field number 1). This is the order the DAG-PB + // spec requires encoders to produce [1], used by all UnixFS profiles + // through unixfs-v1-2025. + // + // [1]: https://ipld.io/specs/codecs/dag-pb/spec/#protobuf-strictness + PBNodeLinksFirst PBNodeFieldOrder = iota + + // PBNodeDataFirst writes the Data field (field number 1) before the + // repeated Links field (field number 2), so streaming readers can + // process Data (e.g. HAMT parameters) before reading links. The DAG-PB + // spec says decoders should accept either order [1]; IPIP-550 + // (https://github.com/ipfs/specs/pull/550) defines this one as a + // low-level opt-in for writers that need it. No named profile selects + // it, and enabling it changes CIDs. + // + // [1]: https://ipld.io/specs/codecs/dag-pb/spec/#protobuf-strictness + PBNodeDataFirst +) + +// DefaultPBNodeFieldOrder is the field order used when encoding a ProtoNode. +// PBNodeDataFirst changes the bytes, and so the CID, of every encoded node +// that has both Data and Links: directories, HAMT shards, and the root and +// intermediate nodes of files larger than one chunk. A node with only one of +// the two fields encodes the same under both orders. +// +// Like the other UnixFS import globals that io.UnixFSProfile.ApplyGlobals +// writes, this is a process-wide setting, not a per-node option. +// Per-node plumbing would touch every producer and consumer of ProtoNode, so +// the global is the accepted compromise. What follows from it: +// +// - Set it once at startup, before the first encode, and never change it +// while the process runs. It is read on every encode without +// synchronization, and a node that was already encoded keeps its cached +// bytes and CID until it is mutated or re-encoded with +// EncodeProtobuf(true). +// - It applies to every ProtoNode, not only UnixFS ones. +// - A node decoded from storage keeps its wire bytes as its encode cache, +// so storing it back unchanged keeps its CID. Copy and every mutation +// drop that cache, and the next encode uses the current order. Switching +// the order on an existing repository therefore changes the CIDs of +// nodes whose content did not change, for example MFS directories, +// which are copied when loaded (io.NewDirectoryFromNode). +var DefaultPBNodeFieldOrder = PBNodeLinksFirst + +// moveDataFirst rewrites a links-first dag-pb encoding produced by +// dagpb.AppendEncode into the PBNodeDataFirst order. AppendEncode writes the +// Data field last, so the field occupies the trailing tag+length+bytes span +// of enc and moving that span to the front is a rotation. Reusing the +// reference encoder keeps one source of truth for link sorting and field +// presence. dataLen is the length of the Data field that was encoded; the +// field must be present. +func moveDataFirst(enc []byte, dataLen int) []byte { + span := protowire.SizeTag(1) + protowire.SizeBytes(dataLen) + split := len(enc) - span + return slices.Concat(enc[split:], enc[:split]) +} diff --git a/ipld/merkledag/fieldorder_test.go b/ipld/merkledag/fieldorder_test.go new file mode 100644 index 000000000..7da46438d --- /dev/null +++ b/ipld/merkledag/fieldorder_test.go @@ -0,0 +1,318 @@ +package merkledag_test + +import ( + "bytes" + "encoding/hex" + "math/rand/v2" + "slices" + "strings" + "testing" + + "github.com/ipfs/boxo/ipld/merkledag" + cid "github.com/ipfs/go-cid" + format "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/encoding/protowire" +) + +// Test fixtures from IPIP-550 (https://github.com/ipfs/specs/pull/550): +// the same UnixFS directory and HAMT shard, each serialized with the +// canonical links-first order and with the opt-in data-first order. +const ( + dirLinksFirstHex = "12330a24015512205891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03120968656c6c6f2e74787418060a020801" + dirDataFirstHex = "0a02080112330a24015512205891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03120968656c6c6f2e7478741806" + + hamtLinksFirstHex = "12350a24015512205891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03120b444668656c6c6f2e74787418060a250805121c800000000000000000000000000000000000000000000000000000002822308002" + hamtDataFirstHex = "0a250805121c80000000000000000000000000000000000000000000000000000000282230800212350a24015512205891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03120b444668656c6c6f2e7478741806" + + dirLinksFirstCid = "bafybeigdcg7pksx2zk5336vrfsktjodlr4rbfz37qr3koc5xboxe5ekv24" + dirDataFirstCid = "bafybeigqvyloizmfcdy6scaxnyltftzptaruqa3hnnplfzsbf4sqteiwlm" + hamtLinksFirstCid = "bafybeicjwkfslu7gwyywffvqgse5kiibojtktxcdqhgv7ldj5fjdacuceq" + hamtDataFirstCid = "bafybeicwgy2rlqmqqu3yy2tqvm2wbgdvy3snu4sbbv4wqpvpnoplpzxz74" +) + +func saveFieldOrder(t *testing.T) { + old := merkledag.DefaultPBNodeFieldOrder + t.Cleanup(func() { merkledag.DefaultPBNodeFieldOrder = old }) +} + +func mustDecodeHex(t *testing.T, s string) []byte { + t.Helper() + b, err := hex.DecodeString(s) + if err != nil { + t.Fatal(err) + } + return b +} + +// reencode decodes a raw dag-pb block and re-encodes it under the given +// field order, returning the bytes and the CIDv1 of the result. +func reencode(t *testing.T, raw []byte, order merkledag.PBNodeFieldOrder) ([]byte, cid.Cid) { + t.Helper() + node, err := merkledag.DecodeProtobuf(raw) + if err != nil { + t.Fatalf("decode: %v", err) + } + if err := node.SetCidBuilder(cid.Prefix{ + Version: 1, + Codec: cid.DagProtobuf, + MhType: mh.SHA2_256, + MhLength: -1, + }); err != nil { + t.Fatal(err) + } + merkledag.DefaultPBNodeFieldOrder = order + enc, err := node.EncodeProtobuf(true) + if err != nil { + t.Fatalf("encode: %v", err) + } + return enc, node.Cid() +} + +func TestPBNodeFieldOrder(t *testing.T) { + cases := []struct { + name string + linksFirstHex, dataFirstHex string + linksFirstCid, dataFirstCid string + }{ + {"directory", dirLinksFirstHex, dirDataFirstHex, dirLinksFirstCid, dirDataFirstCid}, + {"hamt shard", hamtLinksFirstHex, hamtDataFirstHex, hamtLinksFirstCid, hamtDataFirstCid}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + saveFieldOrder(t) + linksFirst := mustDecodeHex(t, tc.linksFirstHex) + dataFirst := mustDecodeHex(t, tc.dataFirstHex) + + // Decoding must accept both orders and yield the same logical node. + a, err := merkledag.DecodeProtobuf(linksFirst) + if err != nil { + t.Fatalf("decode links-first: %v", err) + } + b, err := merkledag.DecodeProtobuf(dataFirst) + if err != nil { + t.Fatalf("decode data-first: %v", err) + } + if !bytes.Equal(a.Data(), b.Data()) { + t.Error("Data differs between orders") + } + if len(a.Links()) != len(b.Links()) { + t.Fatal("link count differs between orders") + } + for i, la := range a.Links() { + lb := b.Links()[i] + if la.Name != lb.Name || la.Size != lb.Size || !la.Cid.Equals(lb.Cid) { + t.Errorf("link %d differs between orders", i) + } + } + + // Re-encoding either input under each order must reproduce the + // fixture bytes and CIDs exactly. + for _, input := range [][]byte{linksFirst, dataFirst} { + enc, c := reencode(t, input, merkledag.PBNodeLinksFirst) + if !bytes.Equal(enc, linksFirst) { + t.Errorf("links-first re-encode: got %x, want %s", enc, tc.linksFirstHex) + } + if c.String() != tc.linksFirstCid { + t.Errorf("links-first CID: got %s, want %s", c, tc.linksFirstCid) + } + + enc, c = reencode(t, input, merkledag.PBNodeDataFirst) + if !bytes.Equal(enc, dataFirst) { + t.Errorf("data-first re-encode: got %x, want %s", enc, tc.dataFirstHex) + } + if c.String() != tc.dataFirstCid { + t.Errorf("data-first CID: got %s, want %s", c, tc.dataFirstCid) + } + } + }) + } +} + +func TestPBNodeFieldOrderUnknown(t *testing.T) { + saveFieldOrder(t) + merkledag.DefaultPBNodeFieldOrder = merkledag.PBNodeFieldOrder(7) + + node := merkledag.NodeWithData([]byte("x")) + _, err := node.EncodeProtobuf(true) + require.ErrorContains(t, err, "unknown PBNodeFieldOrder 7") +} + +// encodeLinksFirst is a test-only dag-pb encoder built directly on +// protowire. Unlike the reference encoder it keeps links in the given order, +// so it can produce blocks with unsorted links, which is what a node decoded +// from another implementation may look like. +func encodeLinksFirst(data []byte, links []*format.Link) []byte { + var enc []byte + for _, l := range links { + var pl []byte + pl = protowire.AppendTag(pl, 1, protowire.BytesType) + pl = protowire.AppendBytes(pl, l.Cid.Bytes()) + pl = protowire.AppendTag(pl, 2, protowire.BytesType) + pl = protowire.AppendString(pl, l.Name) + pl = protowire.AppendTag(pl, 3, protowire.VarintType) + pl = protowire.AppendVarint(pl, l.Size) + enc = protowire.AppendTag(enc, 2, protowire.BytesType) + enc = protowire.AppendBytes(enc, pl) + } + if data != nil { + enc = protowire.AppendTag(enc, 1, protowire.BytesType) + enc = protowire.AppendBytes(enc, data) + } + return enc +} + +func randomBytes(r *rand.Rand, n int) []byte { + b := make([]byte, n) + for i := range b { + b[i] = byte(r.IntN(256)) + } + return b +} + +func randomName(r *rand.Rand, n int) string { + const alphabet = "abcdefghijklmnopqrstuvwxyz0123456789-_." + var sb strings.Builder + sb.Grow(n) + for range n { + sb.WriteByte(alphabet[r.IntN(len(alphabet))]) + } + return sb.String() +} + +func randomCid(t *testing.T, r *rand.Rand) cid.Cid { + payload := randomBytes(r, r.IntN(64)+1) + switch r.IntN(3) { + case 0: + h, err := mh.Sum(payload, mh.SHA2_256, -1) + require.NoError(t, err) + return cid.NewCidV0(h) + case 1: + h, err := mh.Sum(payload, mh.SHA2_256, -1) + require.NoError(t, err) + return cid.NewCidV1(cid.Raw, h) + default: + h, err := mh.Sum(payload, mh.IDENTITY, -1) + require.NoError(t, err) + return cid.NewCidV1(cid.DagProtobuf, h) + } +} + +// pickLen returns a length that crosses the 1, 2 and 3 byte varint +// boundaries with useful frequency; the largest class is rare because it +// dominates block size. +func pickLen(r *rand.Rand) int { + switch r.IntN(20) { + case 0: + return 0 + case 1, 2, 3: + return 128 + r.IntN(2000) + case 4: + return 16384 + r.IntN(600) + default: + return 1 + r.IntN(127) + } +} + +type randomNode struct { + data []byte + links []*format.Link +} + +func newRandomNode(t *testing.T, r *rand.Rand) randomNode { + var n randomNode + switch r.IntN(4) { + case 0: + n.data = nil + case 1: + n.data = []byte{} + default: + n.data = randomBytes(r, pickLen(r)) + } + var count int + switch r.IntN(6) { + case 0: + count = 0 + case 1: + count = 128 + r.IntN(200) + default: + count = 1 + r.IntN(16) + } + n.links = make([]*format.Link, 0, count) + for range count { + nameLen := pickLen(r) + if count > 16 && nameLen > 2000 { + nameLen = 2000 + } + n.links = append(n.links, &format.Link{ + Name: randomName(r, nameLen), + Size: r.Uint64N(1 << 63), + Cid: randomCid(t, r), + }) + } + return n +} + +func requireSameLinks(t *testing.T, want, got []*format.Link) { + t.Helper() + require.Len(t, got, len(want)) + for i := range want { + require.Equal(t, want[i].Name, got[i].Name, "link %d name", i) + require.Equal(t, want[i].Size, got[i].Size, "link %d size", i) + require.True(t, want[i].Cid.Equals(got[i].Cid), "link %d cid", i) + } +} + +// TestPBNodeFieldOrderRandomNodes decodes randomly shaped links-first blocks +// (including ones with unsorted links) and checks that re-encoding them under +// each order yields a block that decodes to the same node with links sorted +// by name, that the Data field leads the data-first form, and that the two +// forms differ only by where the Data field sits. +func TestPBNodeFieldOrderRandomNodes(t *testing.T) { + saveFieldOrder(t) + r := rand.New(rand.NewPCG(550, 2026)) + + for i := range 300 { + n := newRandomNode(t, r) + r.Shuffle(len(n.links), func(a, b int) { n.links[a], n.links[b] = n.links[b], n.links[a] }) + sorted := slices.Clone(n.links) + slices.SortStableFunc(sorted, func(a, b *format.Link) int { return strings.Compare(a.Name, b.Name) }) + + raw := encodeLinksFirst(n.data, n.links) + node, err := merkledag.DecodeProtobuf(raw) + require.NoError(t, err, "node %d", i) + + var encoded [2][]byte + for _, order := range []merkledag.PBNodeFieldOrder{merkledag.PBNodeLinksFirst, merkledag.PBNodeDataFirst} { + merkledag.DefaultPBNodeFieldOrder = order + enc, err := node.EncodeProtobuf(true) + require.NoError(t, err, "node %d order %d", i, order) + encoded[order] = enc + + back, err := merkledag.DecodeProtobuf(enc) + require.NoError(t, err, "node %d order %d", i, order) + if len(n.data) == 0 { + require.Empty(t, back.Data(), "node %d order %d", i, order) + } else { + require.Equal(t, n.data, back.Data(), "node %d order %d", i, order) + } + requireSameLinks(t, sorted, back.Links()) + } + + linksFirst, dataFirst := encoded[merkledag.PBNodeLinksFirst], encoded[merkledag.PBNodeDataFirst] + require.Len(t, dataFirst, len(linksFirst), "node %d", i) + if n.data == nil { + require.Equal(t, linksFirst, dataFirst, "node %d: no Data field, orders must agree", i) + continue + } + num, typ, tagLen := protowire.ConsumeTag(dataFirst) + require.Equal(t, protowire.Number(1), num, "node %d: first field", i) + require.Equal(t, protowire.BytesType, typ, "node %d: first field type", i) + field, fieldLen := protowire.ConsumeBytes(dataFirst[tagLen:]) + require.Equal(t, n.data, field, "node %d: leading Data field", i) + span := tagLen + fieldLen + require.Equal(t, linksFirst, slices.Concat(dataFirst[span:], dataFirst[:span]), "node %d: forms differ beyond Data placement", i) + } +} diff --git a/ipld/unixfs/io/doc.go b/ipld/unixfs/io/doc.go index b6d524fce..75d60ea65 100644 --- a/ipld/unixfs/io/doc.go +++ b/ipld/unixfs/io/doc.go @@ -37,5 +37,20 @@ // - [UnixFS_v0_2015]: Legacy CIDv0 settings (256 KiB chunks, dag-pb leaves) // - [UnixFS_v1_2025]: Modern CIDv1 settings (1 MiB chunks, raw leaves) // -// See https://specs.ipfs.tech/ipips/ipip-0499/ for specification details. +// Both pin the canonical links-first dag-pb field order. Writers that need +// the streaming-friendly Data-first order can opt in via the low-level +// [UnixFSProfile.PBNodeFieldOrder] knob (IPIP-550); no named profile +// selects it, and enabling it changes CIDs. +// +// See https://specs.ipfs.tech/ipips/ipip-0499/ and +// https://github.com/ipfs/specs/pull/550 for specification details. +// +// # Global Settings +// +// [UnixFSProfile.ApplyGlobals] writes a profile into package-level variables +// ([HAMTShardingSize], [HAMTSizeEstimation], [DefaultShardWidth], +// chunk.DefaultBlockSize, helpers.DefaultLinksPerBlock and +// merkledag.DefaultPBNodeFieldOrder). They are read on every import without +// synchronization, so apply a profile once at startup, before the first +// import, and do not change these variables while the process runs. package io diff --git a/ipld/unixfs/io/profile.go b/ipld/unixfs/io/profile.go index 9db6e9466..a3a94ec91 100644 --- a/ipld/unixfs/io/profile.go +++ b/ipld/unixfs/io/profile.go @@ -2,6 +2,7 @@ package io import ( chunk "github.com/ipfs/boxo/chunker" + mdag "github.com/ipfs/boxo/ipld/merkledag" "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" "github.com/ipfs/go-cid" mh "github.com/multiformats/go-multihash" @@ -56,6 +57,21 @@ type UnixFSProfile struct { // HAMTShardWidth is the fanout for HAMT directory nodes. // Must be a power of 2 and multiple of 8. HAMTShardWidth int + + // PBNodeFieldOrder controls the order of the top-level PBNode fields in + // serialized dag-pb blocks. merkledag.PBNodeLinksFirst (the zero value) + // is the canonical DAG-PB order, pinned explicitly by every named + // profile. merkledag.PBNodeDataFirst is a low-level opt-in knob from + // IPIP-550 (https://github.com/ipfs/specs/pull/550) for writers that + // need streaming-friendly blocks; no named profile selects it. Enabling + // it changes the CID of every dag-pb node that has both Data and Links: + // directories, HAMT shards, and the root and intermediate nodes of + // files larger than one chunk. Single-chunk raw-leaf files keep their + // CIDs. Links-first data touched through the directory API is + // re-encoded in the new order when it is stored again (for example MFS + // directories on their next change), so CIDs change without a content + // change. + PBNodeFieldOrder mdag.PBNodeFieldOrder } // Predefined profiles matching IPIP-499 specifications. @@ -82,6 +98,7 @@ var ( HAMTShardingSize: int(256 * unitKiB), HAMTSizeEstimation: SizeEstimationLinks, HAMTShardWidth: 256, + PBNodeFieldOrder: mdag.PBNodeLinksFirst, // canonical order, pinned } // UnixFS_v1_2025 matches the unixfs-v1-2025 profile from IPIP-499. @@ -96,6 +113,7 @@ var ( HAMTShardingSize: int(256 * unitKiB), HAMTSizeEstimation: SizeEstimationBlock, HAMTShardWidth: 256, + PBNodeFieldOrder: mdag.PBNodeLinksFirst, // canonical order, pinned } ) @@ -103,9 +121,16 @@ var ( // This affects all subsequent file and directory import operations. // Note: RawLeaves and CidBuilder are not globals; pass them to DAG builder options. // -// Thread safety: this function modifies global variables and is not safe -// for concurrent use. Call it once during program initialization, before -// starting any imports. Do not call from multiple goroutines. +// The settings written here (chunk.DefaultBlockSize, +// helpers.DefaultLinksPerBlock, HAMTShardingSize, HAMTSizeEstimation, +// DefaultShardWidth and merkledag.DefaultPBNodeFieldOrder) are process-wide +// globals rather than per-call options: threading them through every +// producer and consumer of the import pipeline would amount to a rewrite. +// Call ApplyGlobals once at startup, before the first import, and do not +// call it again while the process runs. The globals are read on every +// import without synchronization, and a ProtoNode that was already encoded +// keeps its cached bytes and CID until it is mutated (see +// merkledag.DefaultPBNodeFieldOrder). func (p UnixFSProfile) ApplyGlobals() { // File settings chunk.DefaultBlockSize = p.ChunkSize @@ -115,6 +140,9 @@ func (p UnixFSProfile) ApplyGlobals() { HAMTShardingSize = p.HAMTShardingSize HAMTSizeEstimation = p.HAMTSizeEstimation DefaultShardWidth = p.HAMTShardWidth + + // dag-pb encoding settings + mdag.DefaultPBNodeFieldOrder = p.PBNodeFieldOrder } // CidBuilder returns a cid.Builder configured for this profile. diff --git a/ipld/unixfs/io/profile_test.go b/ipld/unixfs/io/profile_test.go index 688c1e100..f1c844f04 100644 --- a/ipld/unixfs/io/profile_test.go +++ b/ipld/unixfs/io/profile_test.go @@ -1,15 +1,21 @@ package io import ( + "bytes" "context" + "encoding/hex" "fmt" "os" + "slices" "testing" "time" + chunk "github.com/ipfs/boxo/chunker" mdag "github.com/ipfs/boxo/ipld/merkledag" mdtest "github.com/ipfs/boxo/ipld/merkledag/test" ft "github.com/ipfs/boxo/ipld/unixfs" + "github.com/ipfs/boxo/ipld/unixfs/importer/balanced" + "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" "github.com/ipfs/boxo/ipld/unixfs/private/linksize" cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" @@ -53,6 +59,11 @@ func TestUnixFSProfiles(t *testing.T) { assert.Equal(t, 256, UnixFS_v1_2025.HAMTShardWidth, "HAMTShardWidth should be 256") }) + t.Run("profiles pin canonical links-first order", func(t *testing.T) { + assert.Equal(t, mdag.PBNodeLinksFirst, UnixFS_v1_2025.PBNodeFieldOrder, "UnixFS_v1_2025 must keep links-first") + assert.Equal(t, mdag.PBNodeLinksFirst, UnixFS_v0_2015.PBNodeFieldOrder, "UnixFS_v0_2015 must keep links-first") + }) + t.Run("CidBuilder returns correct prefix", func(t *testing.T) { t.Run("UnixFS_v0_2015", func(t *testing.T) { builder := UnixFS_v0_2015.CidBuilder() @@ -72,48 +83,221 @@ func TestUnixFSProfiles(t *testing.T) { }) t.Run("ApplyGlobals sets global variables", func(t *testing.T) { - // Save original values - oldShardingSize := HAMTShardingSize - oldEstimation := HAMTSizeEstimation - oldShardWidth := DefaultShardWidth - t.Cleanup(func() { - HAMTShardingSize = oldShardingSize - HAMTSizeEstimation = oldEstimation - DefaultShardWidth = oldShardWidth - }) - - // Apply UnixFS_v1_2025 - UnixFS_v1_2025.ApplyGlobals() - - assert.Equal(t, UnixFS_v1_2025.HAMTShardingSize, HAMTShardingSize) - assert.Equal(t, UnixFS_v1_2025.HAMTSizeEstimation, HAMTSizeEstimation) - assert.Equal(t, UnixFS_v1_2025.HAMTShardWidth, DefaultShardWidth) - - // Apply UnixFS_v0_2015 - UnixFS_v0_2015.ApplyGlobals() + saveAndRestoreGlobals(t) - assert.Equal(t, UnixFS_v0_2015.HAMTShardingSize, HAMTShardingSize) - assert.Equal(t, UnixFS_v0_2015.HAMTSizeEstimation, HAMTSizeEstimation) - assert.Equal(t, UnixFS_v0_2015.HAMTShardWidth, DefaultShardWidth) + dataFirst := UnixFS_v1_2025 + dataFirst.PBNodeFieldOrder = mdag.PBNodeDataFirst + for _, p := range []UnixFSProfile{UnixFS_v1_2025, dataFirst, UnixFS_v0_2015} { + p.ApplyGlobals() + + assert.Equal(t, p.ChunkSize, chunk.DefaultBlockSize) + assert.Equal(t, p.FileDAGWidth, helpers.DefaultLinksPerBlock) + assert.Equal(t, p.HAMTShardingSize, HAMTShardingSize) + assert.Equal(t, p.HAMTSizeEstimation, HAMTSizeEstimation) + assert.Equal(t, p.HAMTShardWidth, DefaultShardWidth) + assert.Equal(t, p.PBNodeFieldOrder, mdag.DefaultPBNodeFieldOrder) + } }) } // saveAndRestoreGlobals saves the current global settings and restores them -// after the test completes. Use this in tests that modify HAMTShardingSize, -// HAMTSizeEstimation, DefaultShardWidth, or linksize.LinkSizeFunction. +// after the test completes. Use this in tests that call ApplyGlobals or +// modify any of the globals it writes, or linksize.LinkSizeFunction. func saveAndRestoreGlobals(t *testing.T) { + oldBlockSize := chunk.DefaultBlockSize + oldLinksPerBlock := helpers.DefaultLinksPerBlock oldShardingSize := HAMTShardingSize oldEstimation := HAMTSizeEstimation oldShardWidth := DefaultShardWidth oldLinkSize := linksize.LinkSizeFunction + oldFieldOrder := mdag.DefaultPBNodeFieldOrder t.Cleanup(func() { + chunk.DefaultBlockSize = oldBlockSize + helpers.DefaultLinksPerBlock = oldLinksPerBlock HAMTShardingSize = oldShardingSize HAMTSizeEstimation = oldEstimation DefaultShardWidth = oldShardWidth linksize.LinkSizeFunction = oldLinkSize + mdag.DefaultPBNodeFieldOrder = oldFieldOrder }) } +// TestProfilePBNodeFieldOrderFixtures verifies the directory fixtures from +// IPIP-550 (https://github.com/ipfs/specs/pull/550): the same directory +// containing hello.txt yields the canonical links-first encoding under +// UnixFS_v1_2025 and the data-first encoding when the PBNodeFieldOrder +// knob is set on top of it. +func TestProfilePBNodeFieldOrderFixtures(t *testing.T) { + const ( + leafCid = "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am" + linksFirstHex = "12330a24015512205891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03120968656c6c6f2e74787418060a020801" + linksFirstCid = "bafybeigdcg7pksx2zk5336vrfsktjodlr4rbfz37qr3koc5xboxe5ekv24" + dataFirstHex = "0a02080112330a24015512205891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03120968656c6c6f2e7478741806" + dataFirstCid = "bafybeigqvyloizmfcdy6scaxnyltftzptaruqa3hnnplfzsbf4sqteiwlm" + ) + + dataFirst := UnixFS_v1_2025 + dataFirst.PBNodeFieldOrder = mdag.PBNodeDataFirst + + cases := []struct { + name string + profile UnixFSProfile + expectedHex string + expectedCid string + }{ + {"UnixFS_v1_2025 writes links first", UnixFS_v1_2025, linksFirstHex, linksFirstCid}, + {"data-first knob writes data first", dataFirst, dataFirstHex, dataFirstCid}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + saveAndRestoreGlobals(t) + tc.profile.ApplyGlobals() + + ds := mdtest.Mock() + ctx := t.Context() + + leaf := mdag.NewRawNode([]byte("hello\n")) + require.Equal(t, leafCid, leaf.Cid().String()) + require.NoError(t, ds.Add(ctx, leaf)) + + dir, err := NewDirectory(ds) + require.NoError(t, err) + require.NoError(t, dir.AddChild(ctx, "hello.txt", leaf)) + + node, err := dir.GetNode() + require.NoError(t, err) + pn, ok := node.(*mdag.ProtoNode) + require.True(t, ok) + require.NoError(t, pn.SetCidBuilder(tc.profile.CidBuilder())) + + enc, err := pn.EncodeProtobuf(true) + require.NoError(t, err) + assert.Equal(t, tc.expectedHex, hex.EncodeToString(enc)) + assert.Equal(t, tc.expectedCid, pn.Cid().String()) + }) + } +} + +// TestProfilePBNodeFieldOrderEndToEnd builds a multi-chunk file and a sharded +// directory from fixed inputs under UnixFS_v1_2025 and under the same profile +// with the data-first knob set, and pins the resulting root CIDs. The file +// inputs and expected CIDs match what `ipfs add --chunker=size-1000` produces +// under each configuration. Every dag-pb block of both DAGs must lead with +// the configured first field, and the two DAGs must decode to the same nodes. +func TestProfilePBNodeFieldOrderEndToEnd(t *testing.T) { + const ( + fileSize = 3000 + chunkSize = 1000 + hamtEntries = 64 + ) + + dataFirst := UnixFS_v1_2025 + dataFirst.PBNodeFieldOrder = mdag.PBNodeDataFirst + + cases := []struct { + name string + profile UnixFSProfile + firstByte byte + fileCid string + hamtCid string + }{ + { + "UnixFS_v1_2025", UnixFS_v1_2025, 0x12, + "bafybeiapp6tzng2hpxzmopnplvg6hoxldufypompu6uzmhc6rdyw4m2qx4", + "bafybeiahnfucbarnualj2uzakbqnj3b3nvrszwu4jrl2x7ekbfqshpd524", + }, + { + "UnixFS_v1_2025+data-first", dataFirst, 0x0a, + "bafybeigwq5lxlau4ced4hlvjxuz4xtdbs4zz32dgndqpcc2ggcbiei6k2y", + "bafybeihfyecdexcpn3g3xqgzz23pexyespuldzfoh3tvjris56cxkropau", + }, + } + + // fingerprints of every dag-pb node reachable from the roots, per profile; + // link CIDs are left out because dag-pb children differ between profiles + fingerprints := map[string][]string{} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + saveAndRestoreGlobals(t) + tc.profile.ApplyGlobals() + HAMTShardingSize = 1024 // shard at a size a unit test can reach + + ds := mdtest.Mock() + ctx := t.Context() + + params := helpers.DagBuilderParams{ + Maxlinks: tc.profile.FileDAGWidth, + RawLeaves: tc.profile.RawLeaves, + CidBuilder: tc.profile.CidBuilder(), + Dagserv: ds, + } + content := bytes.Repeat([]byte("a"), fileSize) + db, err := params.New(chunk.NewSizeSplitter(bytes.NewReader(content), chunkSize)) + require.NoError(t, err) + fileRoot, err := balanced.Layout(db) + require.NoError(t, err) + assert.Equal(t, tc.fileCid, fileRoot.Cid().String(), "file root CID") + + dir, err := NewDirectory(ds) + require.NoError(t, err) + dir.SetCidBuilder(tc.profile.CidBuilder()) + for i := range hamtEntries { + leaf := mdag.NewRawNode(fmt.Appendf(nil, "file %d\n", i)) + require.NoError(t, ds.Add(ctx, leaf)) + require.NoError(t, dir.AddChild(ctx, fmt.Sprintf("file-%d.txt", i), leaf)) + } + _, isHAMT := dir.(*DynamicDirectory).Directory.(*HAMTDirectory) + require.True(t, isHAMT, "directory should have switched to HAMT") + dirRoot, err := dir.GetNode() + require.NoError(t, err) + require.NoError(t, ds.Add(ctx, dirRoot)) + assert.Equal(t, tc.hamtCid, dirRoot.Cid().String(), "HAMT root CID") + + var dagpbNodes, subShards int + for _, root := range []cid.Cid{fileRoot.Cid(), dirRoot.Cid()} { + err := mdag.Walk(ctx, mdag.GetLinksDirect(ds), root, func(c cid.Cid) bool { + if c.Type() != cid.DagProtobuf { + return true + } + node, err := ds.Get(ctx, c) + require.NoError(t, err) + pn, ok := node.(*mdag.ProtoNode) + require.True(t, ok) + require.NotEmpty(t, pn.Data(), "%s: every dag-pb node here carries UnixFS Data", c) + require.NotEmpty(t, pn.Links(), "%s: every dag-pb node here carries links", c) + assert.Equal(t, tc.firstByte, pn.RawData()[0], "%s: first field", c) + dagpbNodes++ + + names := make([]string, 0, len(pn.Links())) + sizes := make([]uint64, 0, len(pn.Links())) + for _, l := range pn.Links() { + names = append(names, l.Name) + sizes = append(sizes, l.Size) + if c.Equals(dirRoot.Cid()) && len(l.Name) == 2 { + subShards++ + } + } + fingerprints[tc.name] = append(fingerprints[tc.name], + fmt.Sprintf("%x|%q|%v", pn.Data(), names, sizes)) + return true + }) + require.NoError(t, err) + } + assert.Greater(t, subShards, 0, "HAMT root should link to child shards") + assert.Greater(t, dagpbNodes, 2, "file root, HAMT root and child shards expected") + t.Logf("%s: %d dag-pb nodes, %d child shards", tc.name, dagpbNodes, subShards) + }) + } + + for _, fp := range fingerprints { + slices.Sort(fp) + } + assert.Equal(t, fingerprints["UnixFS_v1_2025"], fingerprints["UnixFS_v1_2025+data-first"], + "both orders must produce the same logical nodes") +} + func TestProfileHAMTThresholdBehavior(t *testing.T) { // Use fixed link size for predictable testing const fixedLinkSize = 100