From b83d891519394c7faf397042731d02b990adf9e0 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 27 Aug 2026 20:58:57 +0200 Subject: [PATCH 1/8] feat(unixfs): unixfs-v1-2026 profile (IPIP-550) Opt-in Data-first PBNode field ordering behind the new io.UnixFS_v1_2026 profile, per IPIP-550. Default output is unchanged: all existing profiles keep the legacy Links-first order and their CIDs. - ipld/merkledag: PBNodeFieldOrder global and a Data-first encoder used when a profile opts in (candidate for upstreaming to go-codec-dagpb) - ipld/unixfs/io: PBNodeFieldOrder profile parameter and UnixFS_v1_2026, wired through ApplyGlobals - tests assert byte-exact fixtures from the IPIP table Refs ipfs/specs#550 --- CHANGELOG.md | 1 + ipld/merkledag/coding.go | 10 ++- ipld/merkledag/fieldorder.go | 93 ++++++++++++++++++++++ ipld/merkledag/fieldorder_test.go | 126 ++++++++++++++++++++++++++++++ ipld/unixfs/io/profile.go | 24 ++++++ ipld/unixfs/io/profile_test.go | 72 +++++++++++++++++ 6 files changed, 323 insertions(+), 3 deletions(-) create mode 100644 ipld/merkledag/fieldorder.go create mode 100644 ipld/merkledag/fieldorder_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index c0aa34813..40a98d9b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The following emojis are used to highlight certain changes: ### Added +- `ipld/unixfs`: opt-in `io.UnixFS_v1_2026` profile (and `UnixFSProfile.PBNodeFieldOrder` parameter) writes the `PBNode` `Data` field before `Links`, so streaming readers can read HAMT parameters before links. All other profiles and the default (`merkledag.DefaultPBNodeFieldOrder`) keep the canonical links-first bytes and CIDs. [IPIP-550](https://github.com/ipfs/specs/pull/550) - ✨ `gateway`: responses now include the `Ipfs-Uri` header with a canonical `ipfs://` or `ipns://` URI for the requested content path, and expose it via the default `Access-Control-Expose-Headers`. The header carries the content root in canonical form (base32 CIDv1 for `/ipfs/`, base36 CIDv1 for cryptographic `/ipns/` names, lowercase FQDN for DNSLink) with percent-encoded path segments, so clients get a value that is safe in HTTP field context regardless of bytes in the underlying path. [IPIP-548](https://github.com/ipfs/specs/pull/548) [#1209](https://github.com/ipfs/boxo/pull/1209) ### Changed diff --git a/ipld/merkledag/coding.go b/ipld/merkledag/coding.go index 035e964ef..088fc8f2a 100644 --- a/ipld/merkledag/coding.go +++ b/ipld/merkledag/coding.go @@ -100,9 +100,13 @@ func (n *ProtoNode) marshalImmutable() (*immutableProtoNode, error) { // without having to grow the buffer and cause allocations. enc := make([]byte, 0, 1024) - enc, err = dagpb.AppendEncode(enc, nd) - if err != nil { - return nil, err + if DefaultPBNodeFieldOrder == PBNodeDataFirst { + enc = appendEncodeDataFirst(enc, n.data, links) + } else { + enc, err = dagpb.AppendEncode(enc, nd) + if err != nil { + return nil, err + } } return &immutableProtoNode{enc, nd.(dagpb.PBNode)}, nil } diff --git a/ipld/merkledag/fieldorder.go b/ipld/merkledag/fieldorder.go new file mode 100644 index 000000000..684d92ad9 --- /dev/null +++ b/ipld/merkledag/fieldorder.go @@ -0,0 +1,93 @@ +package merkledag + +import ( + "encoding/binary" + + format "github.com/ipfs/go-ipld-format" +) + +// PBNodeFieldOrder selects the order of the top-level PBNode fields in the +// serialized dag-pb form. Both orders decode to the same logical node, but +// produce different bytes and therefore different CIDs. +type PBNodeFieldOrder int + +const ( + // PBNodeLinksFirst writes the repeated Links field (field number 2) + // before the Data field (field number 1). This is the canonical DAG-PB + // order, produced by all UnixFS profiles through unixfs-v1-2025. + PBNodeLinksFirst PBNodeFieldOrder = iota + + // PBNodeDataFirst writes the Data field (field number 1) before the + // repeated Links field (field number 2), so streaming readers can + // process Data (e.g. HAMT parameters) before reading links. Proposed + // by IPIP-550 (https://github.com/ipfs/specs/pull/550) for the + // unixfs-v1-2026 profile. + PBNodeDataFirst +) + +// DefaultPBNodeFieldOrder is the field order used when encoding a ProtoNode. +// The default, PBNodeLinksFirst, keeps the bytes and CIDs boxo has always +// produced; PBNodeDataFirst is opt-in and changes the CID of every encoded +// node that has both fields. +// +// Thread safety: this variable is read on every encode and is not safe for +// concurrent modification. Set it once during program initialization, before +// starting any imports, e.g. via io.UnixFSProfile.ApplyGlobals. +var DefaultPBNodeFieldOrder = PBNodeLinksFirst + +// appendEncodeDataFirst encodes a PBNode with the Data field before the +// repeated Links field. go-codec-dagpb only writes the canonical links-first +// order, hence this local encoder. Field presence mirrors the go-codec-dagpb +// path in marshalImmutable: Data is written when non-nil (even if empty), +// links with an undefined CID are dropped, and every written link carries +// Hash, Name, and Tsize in that order. +// +// TODO: this could be upstreamed to github.com/ipld/go-codec-dagpb as an +// encode option if IPIP-550 is ratified. +func appendEncodeDataFirst(enc []byte, data []byte, links []*format.Link) []byte { + const ( + tagPBNodeData = 0x0a // field 1, wire type 2 (bytes) + tagPBNodeLinks = 0x12 // field 2, wire type 2 (embedded message) + tagPBLinkHash = 0x0a // field 1, wire type 2 (bytes) + tagPBLinkName = 0x12 // field 2, wire type 2 (string) + tagPBLinkTsize = 0x18 // field 3, wire type 0 (varint) + ) + + if data != nil { + enc = append(enc, tagPBNodeData) + enc = binary.AppendUvarint(enc, uint64(len(data))) + enc = append(enc, data...) + } + for _, link := range links { + if !link.Cid.Defined() { + continue + } + hash := link.Cid.Bytes() + // overflow, >MaxInt64 is almost certainly an error + tsize := uint64(max(int64(link.Size), 0)) + linkLen := 1 + uvarintLen(uint64(len(hash))) + len(hash) + + 1 + uvarintLen(uint64(len(link.Name))) + len(link.Name) + + 1 + uvarintLen(tsize) + enc = append(enc, tagPBNodeLinks) + enc = binary.AppendUvarint(enc, uint64(linkLen)) + enc = append(enc, tagPBLinkHash) + enc = binary.AppendUvarint(enc, uint64(len(hash))) + enc = append(enc, hash...) + enc = append(enc, tagPBLinkName) + enc = binary.AppendUvarint(enc, uint64(len(link.Name))) + enc = append(enc, link.Name...) + enc = append(enc, tagPBLinkTsize) + enc = binary.AppendUvarint(enc, tsize) + } + return enc +} + +// uvarintLen returns the number of bytes binary.AppendUvarint writes for v. +func uvarintLen(v uint64) int { + n := 1 + for v >= 0x80 { + v >>= 7 + n++ + } + return n +} diff --git a/ipld/merkledag/fieldorder_test.go b/ipld/merkledag/fieldorder_test.go new file mode 100644 index 000000000..bfbbd039a --- /dev/null +++ b/ipld/merkledag/fieldorder_test.go @@ -0,0 +1,126 @@ +package merkledag_test + +import ( + "bytes" + "encoding/hex" + "testing" + + "github.com/ipfs/boxo/ipld/merkledag" + cid "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" +) + +// Test fixtures from IPIP-550 (https://github.com/ipfs/specs/pull/550): +// the same UnixFS directory and HAMT shard, each serialized with the +// canonical links-first order and with the opt-in data-first order. +const ( + dirLinksFirstHex = "12330a24015512205891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03120968656c6c6f2e74787418060a020801" + dirDataFirstHex = "0a02080112330a24015512205891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03120968656c6c6f2e7478741806" + + hamtLinksFirstHex = "12350a24015512205891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03120b444668656c6c6f2e74787418060a250805121c800000000000000000000000000000000000000000000000000000002822308002" + hamtDataFirstHex = "0a250805121c80000000000000000000000000000000000000000000000000000000282230800212350a24015512205891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03120b444668656c6c6f2e7478741806" + + dirLinksFirstCid = "bafybeigdcg7pksx2zk5336vrfsktjodlr4rbfz37qr3koc5xboxe5ekv24" + dirDataFirstCid = "bafybeigqvyloizmfcdy6scaxnyltftzptaruqa3hnnplfzsbf4sqteiwlm" + hamtLinksFirstCid = "bafybeicjwkfslu7gwyywffvqgse5kiibojtktxcdqhgv7ldj5fjdacuceq" + hamtDataFirstCid = "bafybeicwgy2rlqmqqu3yy2tqvm2wbgdvy3snu4sbbv4wqpvpnoplpzxz74" +) + +func saveFieldOrder(t *testing.T) { + old := merkledag.DefaultPBNodeFieldOrder + t.Cleanup(func() { merkledag.DefaultPBNodeFieldOrder = old }) +} + +func mustDecodeHex(t *testing.T, s string) []byte { + t.Helper() + b, err := hex.DecodeString(s) + if err != nil { + t.Fatal(err) + } + return b +} + +// reencode decodes a raw dag-pb block and re-encodes it under the given +// field order, returning the bytes and the CIDv1 of the result. +func reencode(t *testing.T, raw []byte, order merkledag.PBNodeFieldOrder) ([]byte, cid.Cid) { + t.Helper() + node, err := merkledag.DecodeProtobuf(raw) + if err != nil { + t.Fatalf("decode: %v", err) + } + if err := node.SetCidBuilder(cid.Prefix{ + Version: 1, + Codec: cid.DagProtobuf, + MhType: mh.SHA2_256, + MhLength: -1, + }); err != nil { + t.Fatal(err) + } + merkledag.DefaultPBNodeFieldOrder = order + enc, err := node.EncodeProtobuf(true) + if err != nil { + t.Fatalf("encode: %v", err) + } + return enc, node.Cid() +} + +func TestPBNodeFieldOrder(t *testing.T) { + cases := []struct { + name string + linksFirstHex, dataFirstHex string + linksFirstCid, dataFirstCid string + }{ + {"directory", dirLinksFirstHex, dirDataFirstHex, dirLinksFirstCid, dirDataFirstCid}, + {"hamt shard", hamtLinksFirstHex, hamtDataFirstHex, hamtLinksFirstCid, hamtDataFirstCid}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + saveFieldOrder(t) + linksFirst := mustDecodeHex(t, tc.linksFirstHex) + dataFirst := mustDecodeHex(t, tc.dataFirstHex) + + // Decoding must accept both orders and yield the same logical node. + a, err := merkledag.DecodeProtobuf(linksFirst) + if err != nil { + t.Fatalf("decode links-first: %v", err) + } + b, err := merkledag.DecodeProtobuf(dataFirst) + if err != nil { + t.Fatalf("decode data-first: %v", err) + } + if !bytes.Equal(a.Data(), b.Data()) { + t.Error("Data differs between orders") + } + if len(a.Links()) != len(b.Links()) { + t.Fatal("link count differs between orders") + } + for i, la := range a.Links() { + lb := b.Links()[i] + if la.Name != lb.Name || la.Size != lb.Size || !la.Cid.Equals(lb.Cid) { + t.Errorf("link %d differs between orders", i) + } + } + + // Re-encoding either input under each order must reproduce the + // fixture bytes and CIDs exactly. + for _, input := range [][]byte{linksFirst, dataFirst} { + enc, c := reencode(t, input, merkledag.PBNodeLinksFirst) + if !bytes.Equal(enc, linksFirst) { + t.Errorf("links-first re-encode: got %x, want %s", enc, tc.linksFirstHex) + } + if c.String() != tc.linksFirstCid { + t.Errorf("links-first CID: got %s, want %s", c, tc.linksFirstCid) + } + + enc, c = reencode(t, input, merkledag.PBNodeDataFirst) + if !bytes.Equal(enc, dataFirst) { + t.Errorf("data-first re-encode: got %x, want %s", enc, tc.dataFirstHex) + } + if c.String() != tc.dataFirstCid { + t.Errorf("data-first CID: got %s, want %s", c, tc.dataFirstCid) + } + } + }) + } +} diff --git a/ipld/unixfs/io/profile.go b/ipld/unixfs/io/profile.go index 9db6e9466..34e233eed 100644 --- a/ipld/unixfs/io/profile.go +++ b/ipld/unixfs/io/profile.go @@ -2,6 +2,7 @@ package io import ( chunk "github.com/ipfs/boxo/chunker" + mdag "github.com/ipfs/boxo/ipld/merkledag" "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" "github.com/ipfs/go-cid" mh "github.com/multiformats/go-multihash" @@ -56,6 +57,14 @@ type UnixFSProfile struct { // HAMTShardWidth is the fanout for HAMT directory nodes. // Must be a power of 2 and multiple of 8. HAMTShardWidth int + + // PBNodeFieldOrder controls the order of the top-level PBNode fields in + // serialized dag-pb blocks. The zero value (merkledag.PBNodeLinksFirst) + // is the canonical DAG-PB order used by all profiles through + // unixfs-v1-2025. merkledag.PBNodeDataFirst is the opt-in order proposed + // by IPIP-550 (https://github.com/ipfs/specs/pull/550) and changes the + // CIDs of directories and HAMT shards. + PBNodeFieldOrder mdag.PBNodeFieldOrder } // Predefined profiles matching IPIP-499 specifications. @@ -97,6 +106,18 @@ var ( HAMTSizeEstimation: SizeEstimationBlock, HAMTShardWidth: 256, } + + // UnixFS_v1_2026 matches the unixfs-v1-2026 profile proposed in IPIP-550 + // (https://github.com/ipfs/specs/pull/550). It inherits all settings from + // UnixFS_v1_2025 and additionally writes the PBNode Data field before + // Links, so streaming readers can process HAMT parameters before reading + // links. Opt-in: directories and HAMT shards get different CIDs than + // under UnixFS_v1_2025. + UnixFS_v1_2026 = func() UnixFSProfile { + p := UnixFS_v1_2025 + p.PBNodeFieldOrder = mdag.PBNodeDataFirst + return p + }() ) // ApplyGlobals sets the global variables to match this profile's settings. @@ -115,6 +136,9 @@ func (p UnixFSProfile) ApplyGlobals() { HAMTShardingSize = p.HAMTShardingSize HAMTSizeEstimation = p.HAMTSizeEstimation DefaultShardWidth = p.HAMTShardWidth + + // dag-pb encoding settings + mdag.DefaultPBNodeFieldOrder = p.PBNodeFieldOrder } // CidBuilder returns a cid.Builder configured for this profile. diff --git a/ipld/unixfs/io/profile_test.go b/ipld/unixfs/io/profile_test.go index 688c1e100..83c6df9d2 100644 --- a/ipld/unixfs/io/profile_test.go +++ b/ipld/unixfs/io/profile_test.go @@ -2,6 +2,7 @@ package io import ( "context" + "encoding/hex" "fmt" "os" "testing" @@ -53,6 +54,13 @@ func TestUnixFSProfiles(t *testing.T) { assert.Equal(t, 256, UnixFS_v1_2025.HAMTShardWidth, "HAMTShardWidth should be 256") }) + t.Run("UnixFS_v1_2026 has correct values", func(t *testing.T) { + expected := UnixFS_v1_2025 + expected.PBNodeFieldOrder = mdag.PBNodeDataFirst + assert.Equal(t, expected, UnixFS_v1_2026, + "UnixFS_v1_2026 should equal UnixFS_v1_2025 plus data-first PBNode field order") + }) + t.Run("CidBuilder returns correct prefix", func(t *testing.T) { t.Run("UnixFS_v0_2015", func(t *testing.T) { builder := UnixFS_v0_2015.CidBuilder() @@ -76,10 +84,12 @@ func TestUnixFSProfiles(t *testing.T) { oldShardingSize := HAMTShardingSize oldEstimation := HAMTSizeEstimation oldShardWidth := DefaultShardWidth + oldFieldOrder := mdag.DefaultPBNodeFieldOrder t.Cleanup(func() { HAMTShardingSize = oldShardingSize HAMTSizeEstimation = oldEstimation DefaultShardWidth = oldShardWidth + mdag.DefaultPBNodeFieldOrder = oldFieldOrder }) // Apply UnixFS_v1_2025 @@ -88,6 +98,12 @@ func TestUnixFSProfiles(t *testing.T) { assert.Equal(t, UnixFS_v1_2025.HAMTShardingSize, HAMTShardingSize) assert.Equal(t, UnixFS_v1_2025.HAMTSizeEstimation, HAMTSizeEstimation) assert.Equal(t, UnixFS_v1_2025.HAMTShardWidth, DefaultShardWidth) + assert.Equal(t, mdag.PBNodeLinksFirst, mdag.DefaultPBNodeFieldOrder) + + // Apply UnixFS_v1_2026 + UnixFS_v1_2026.ApplyGlobals() + + assert.Equal(t, mdag.PBNodeDataFirst, mdag.DefaultPBNodeFieldOrder) // Apply UnixFS_v0_2015 UnixFS_v0_2015.ApplyGlobals() @@ -95,6 +111,7 @@ func TestUnixFSProfiles(t *testing.T) { assert.Equal(t, UnixFS_v0_2015.HAMTShardingSize, HAMTShardingSize) assert.Equal(t, UnixFS_v0_2015.HAMTSizeEstimation, HAMTSizeEstimation) assert.Equal(t, UnixFS_v0_2015.HAMTShardWidth, DefaultShardWidth) + assert.Equal(t, mdag.PBNodeLinksFirst, mdag.DefaultPBNodeFieldOrder) }) } @@ -106,14 +123,69 @@ func saveAndRestoreGlobals(t *testing.T) { oldEstimation := HAMTSizeEstimation oldShardWidth := DefaultShardWidth oldLinkSize := linksize.LinkSizeFunction + oldFieldOrder := mdag.DefaultPBNodeFieldOrder t.Cleanup(func() { HAMTShardingSize = oldShardingSize HAMTSizeEstimation = oldEstimation DefaultShardWidth = oldShardWidth linksize.LinkSizeFunction = oldLinkSize + mdag.DefaultPBNodeFieldOrder = oldFieldOrder }) } +// TestProfilePBNodeFieldOrderFixtures verifies the directory fixtures from +// IPIP-550 (https://github.com/ipfs/specs/pull/550): the same directory +// containing hello.txt yields the legacy links-first encoding under +// UnixFS_v1_2025 and the data-first encoding under UnixFS_v1_2026. +func TestProfilePBNodeFieldOrderFixtures(t *testing.T) { + const ( + leafCid = "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am" + linksFirstHex = "12330a24015512205891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03120968656c6c6f2e74787418060a020801" + linksFirstCid = "bafybeigdcg7pksx2zk5336vrfsktjodlr4rbfz37qr3koc5xboxe5ekv24" + dataFirstHex = "0a02080112330a24015512205891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03120968656c6c6f2e7478741806" + dataFirstCid = "bafybeigqvyloizmfcdy6scaxnyltftzptaruqa3hnnplfzsbf4sqteiwlm" + ) + + cases := []struct { + name string + profile UnixFSProfile + expectedHex string + expectedCid string + }{ + {"UnixFS_v1_2025 writes links first", UnixFS_v1_2025, linksFirstHex, linksFirstCid}, + {"UnixFS_v1_2026 writes data first", UnixFS_v1_2026, dataFirstHex, dataFirstCid}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + saveAndRestoreGlobals(t) + tc.profile.ApplyGlobals() + + ds := mdtest.Mock() + ctx := t.Context() + + leaf := mdag.NewRawNode([]byte("hello\n")) + require.Equal(t, leafCid, leaf.Cid().String()) + require.NoError(t, ds.Add(ctx, leaf)) + + dir, err := NewDirectory(ds) + require.NoError(t, err) + require.NoError(t, dir.AddChild(ctx, "hello.txt", leaf)) + + node, err := dir.GetNode() + require.NoError(t, err) + pn, ok := node.(*mdag.ProtoNode) + require.True(t, ok) + require.NoError(t, pn.SetCidBuilder(tc.profile.CidBuilder())) + + enc, err := pn.EncodeProtobuf(true) + require.NoError(t, err) + assert.Equal(t, tc.expectedHex, hex.EncodeToString(enc)) + assert.Equal(t, tc.expectedCid, pn.Cid().String()) + }) + } +} + func TestProfileHAMTThresholdBehavior(t *testing.T) { // Use fixed link size for predictable testing const fixedLinkSize = 100 From b1c6abb68bb2cdd0b3548ab5a1c722e993558687 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 27 Aug 2026 20:58:57 +0200 Subject: [PATCH 2/8] ci: run gateway-conformance from ipip-550 branch Temporary pin so the PBNode field ordering tests from ipfs/gateway-conformance#304 run against boxo gateway backends. Switch back to a tagged release once one ships. --- .github/workflows/gateway-conformance.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index b32d2b633..0696dd3ba 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -22,7 +22,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.14 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@8244a307fbefeff34085e03ee98ecad99216f176 with: output: fixtures merged: true @@ -47,7 +47,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@v0.14 + uses: ipfs/gateway-conformance/.github/actions/test@8244a307fbefeff34085e03ee98ecad99216f176 with: gateway-url: http://127.0.0.1:8040 subdomain-url: http://example.net:8040 @@ -84,7 +84,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.14 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@8244a307fbefeff34085e03ee98ecad99216f176 with: output: fixtures merged: true @@ -114,7 +114,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@v0.14 + uses: ipfs/gateway-conformance/.github/actions/test@8244a307fbefeff34085e03ee98ecad99216f176 with: gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote block gateway subdomain-url: http://example.net:8040 @@ -152,7 +152,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.14 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@8244a307fbefeff34085e03ee98ecad99216f176 with: output: fixtures merged: true @@ -182,7 +182,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@v0.14 + uses: ipfs/gateway-conformance/.github/actions/test@8244a307fbefeff34085e03ee98ecad99216f176 with: gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote car gateway subdomain-url: http://example.net:8040 From 64633479eba0e8c37693ec791793c6ee878fc9bd Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 27 Aug 2026 21:42:39 +0200 Subject: [PATCH 3/8] ci: gateway-conformance back to v0.14 v0.14.1 shipped the ipfs/gateway-conformance#304 tests, so the moving v0.14 tag covers them again. --- .github/workflows/gateway-conformance.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 0696dd3ba..b32d2b633 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -22,7 +22,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@8244a307fbefeff34085e03ee98ecad99216f176 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.14 with: output: fixtures merged: true @@ -47,7 +47,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@8244a307fbefeff34085e03ee98ecad99216f176 + uses: ipfs/gateway-conformance/.github/actions/test@v0.14 with: gateway-url: http://127.0.0.1:8040 subdomain-url: http://example.net:8040 @@ -84,7 +84,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@8244a307fbefeff34085e03ee98ecad99216f176 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.14 with: output: fixtures merged: true @@ -114,7 +114,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@8244a307fbefeff34085e03ee98ecad99216f176 + uses: ipfs/gateway-conformance/.github/actions/test@v0.14 with: gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote block gateway subdomain-url: http://example.net:8040 @@ -152,7 +152,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@8244a307fbefeff34085e03ee98ecad99216f176 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.14 with: output: fixtures merged: true @@ -182,7 +182,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@8244a307fbefeff34085e03ee98ecad99216f176 + uses: ipfs/gateway-conformance/.github/actions/test@v0.14 with: gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote car gateway subdomain-url: http://example.net:8040 From b780a24de017b0404544458898faeec0bdfbd5ad Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sun, 30 Aug 2026 21:38:52 +0200 Subject: [PATCH 4/8] fix(merkledag): rotate dagpb output for data-first Derive PBNodeDataFirst bytes from dagpb.AppendEncode by moving the trailing Data field to the front, instead of a second hand-written encoder. One encoder owns link sorting and field presence, so a decoded block with unsorted links now re-encodes sorted under both orders, and byte parity with links-first holds by construction. Unknown PBNodeFieldOrder values return an error instead of silently encoding links-first. - fieldorder.go: moveDataFirst; godoc spells out the process-wide nature of the setting, the set-once-at-startup constraint, and cites the DAG-PB strictness section - coding.go: switch on the order after AppendEncode, error on unknown - fieldorder_test.go: unknown-order test; property test over random nodes against a protowire-based links-first oracle, covering unsorted links, nil and empty Data, multi-byte length prefixes, and CIDv0, identity, and CIDv1 link hashes --- ipld/merkledag/coding.go | 17 ++- ipld/merkledag/fieldorder.go | 111 +++++++---------- ipld/merkledag/fieldorder_test.go | 192 ++++++++++++++++++++++++++++++ 3 files changed, 247 insertions(+), 73 deletions(-) diff --git a/ipld/merkledag/coding.go b/ipld/merkledag/coding.go index 088fc8f2a..228a004ca 100644 --- a/ipld/merkledag/coding.go +++ b/ipld/merkledag/coding.go @@ -100,13 +100,18 @@ func (n *ProtoNode) marshalImmutable() (*immutableProtoNode, error) { // without having to grow the buffer and cause allocations. enc := make([]byte, 0, 1024) - if DefaultPBNodeFieldOrder == PBNodeDataFirst { - enc = appendEncodeDataFirst(enc, n.data, links) - } else { - enc, err = dagpb.AppendEncode(enc, nd) - if err != nil { - return nil, err + enc, err = dagpb.AppendEncode(enc, nd) + if err != nil { + return nil, err + } + switch order := DefaultPBNodeFieldOrder; order { + case PBNodeLinksFirst: + case PBNodeDataFirst: + if n.data != nil { + enc = moveDataFirst(enc, len(n.data)) } + default: + return nil, fmt.Errorf("unknown PBNodeFieldOrder %d", order) } return &immutableProtoNode{enc, nd.(dagpb.PBNode)}, nil } diff --git a/ipld/merkledag/fieldorder.go b/ipld/merkledag/fieldorder.go index 684d92ad9..9d0be4692 100644 --- a/ipld/merkledag/fieldorder.go +++ b/ipld/merkledag/fieldorder.go @@ -1,9 +1,9 @@ package merkledag import ( - "encoding/binary" + "slices" - format "github.com/ipfs/go-ipld-format" + "google.golang.org/protobuf/encoding/protowire" ) // PBNodeFieldOrder selects the order of the top-level PBNode fields in the @@ -13,81 +13,58 @@ type PBNodeFieldOrder int const ( // PBNodeLinksFirst writes the repeated Links field (field number 2) - // before the Data field (field number 1). This is the canonical DAG-PB - // order, produced by all UnixFS profiles through unixfs-v1-2025. + // before the Data field (field number 1). This is the order the DAG-PB + // spec requires encoders to produce [1], used by all UnixFS profiles + // through unixfs-v1-2025. + // + // [1]: https://ipld.io/specs/codecs/dag-pb/spec/#protobuf-strictness PBNodeLinksFirst PBNodeFieldOrder = iota // PBNodeDataFirst writes the Data field (field number 1) before the // repeated Links field (field number 2), so streaming readers can - // process Data (e.g. HAMT parameters) before reading links. Proposed - // by IPIP-550 (https://github.com/ipfs/specs/pull/550) for the + // process Data (e.g. HAMT parameters) before reading links. The DAG-PB + // spec says decoders should accept either order [1]; IPIP-550 + // (https://github.com/ipfs/specs/pull/550) proposes this one for the // unixfs-v1-2026 profile. + // + // [1]: https://ipld.io/specs/codecs/dag-pb/spec/#protobuf-strictness PBNodeDataFirst ) // DefaultPBNodeFieldOrder is the field order used when encoding a ProtoNode. -// The default, PBNodeLinksFirst, keeps the bytes and CIDs boxo has always -// produced; PBNodeDataFirst is opt-in and changes the CID of every encoded -// node that has both fields. +// PBNodeDataFirst changes the bytes, and so the CID, of every encoded node +// that has both Data and Links: directories, HAMT shards, and the root and +// intermediate nodes of files larger than one chunk. A node with only one of +// the two fields encodes the same under both orders. // -// Thread safety: this variable is read on every encode and is not safe for -// concurrent modification. Set it once during program initialization, before -// starting any imports, e.g. via io.UnixFSProfile.ApplyGlobals. -var DefaultPBNodeFieldOrder = PBNodeLinksFirst - -// appendEncodeDataFirst encodes a PBNode with the Data field before the -// repeated Links field. go-codec-dagpb only writes the canonical links-first -// order, hence this local encoder. Field presence mirrors the go-codec-dagpb -// path in marshalImmutable: Data is written when non-nil (even if empty), -// links with an undefined CID are dropped, and every written link carries -// Hash, Name, and Tsize in that order. +// Like the other UnixFS import globals that io.UnixFSProfile.ApplyGlobals +// writes, this is a process-wide setting, not a per-node option. +// Per-node plumbing would touch every producer and consumer of ProtoNode, so +// the global is the accepted compromise. What follows from it: // -// TODO: this could be upstreamed to github.com/ipld/go-codec-dagpb as an -// encode option if IPIP-550 is ratified. -func appendEncodeDataFirst(enc []byte, data []byte, links []*format.Link) []byte { - const ( - tagPBNodeData = 0x0a // field 1, wire type 2 (bytes) - tagPBNodeLinks = 0x12 // field 2, wire type 2 (embedded message) - tagPBLinkHash = 0x0a // field 1, wire type 2 (bytes) - tagPBLinkName = 0x12 // field 2, wire type 2 (string) - tagPBLinkTsize = 0x18 // field 3, wire type 0 (varint) - ) - - if data != nil { - enc = append(enc, tagPBNodeData) - enc = binary.AppendUvarint(enc, uint64(len(data))) - enc = append(enc, data...) - } - for _, link := range links { - if !link.Cid.Defined() { - continue - } - hash := link.Cid.Bytes() - // overflow, >MaxInt64 is almost certainly an error - tsize := uint64(max(int64(link.Size), 0)) - linkLen := 1 + uvarintLen(uint64(len(hash))) + len(hash) + - 1 + uvarintLen(uint64(len(link.Name))) + len(link.Name) + - 1 + uvarintLen(tsize) - enc = append(enc, tagPBNodeLinks) - enc = binary.AppendUvarint(enc, uint64(linkLen)) - enc = append(enc, tagPBLinkHash) - enc = binary.AppendUvarint(enc, uint64(len(hash))) - enc = append(enc, hash...) - enc = append(enc, tagPBLinkName) - enc = binary.AppendUvarint(enc, uint64(len(link.Name))) - enc = append(enc, link.Name...) - enc = append(enc, tagPBLinkTsize) - enc = binary.AppendUvarint(enc, tsize) - } - return enc -} +// - Set it once at startup, before the first encode, and never change it +// while the process runs. It is read on every encode without +// synchronization, and a node that was already encoded keeps its cached +// bytes and CID until it is mutated or re-encoded with +// EncodeProtobuf(true). +// - It applies to every ProtoNode, not only UnixFS ones. +// - A node decoded from storage keeps its wire bytes as its encode cache, +// so storing it back unchanged keeps its CID. Copy and every mutation +// drop that cache, and the next encode uses the current order. Switching +// the order on an existing repository therefore changes the CIDs of +// nodes whose content did not change, for example MFS directories, +// which are copied when loaded (io.NewDirectoryFromNode). +var DefaultPBNodeFieldOrder = PBNodeLinksFirst -// uvarintLen returns the number of bytes binary.AppendUvarint writes for v. -func uvarintLen(v uint64) int { - n := 1 - for v >= 0x80 { - v >>= 7 - n++ - } - return n +// moveDataFirst rewrites a links-first dag-pb encoding produced by +// dagpb.AppendEncode into the PBNodeDataFirst order. AppendEncode writes the +// Data field last, so the field occupies the trailing tag+length+bytes span +// of enc and moving that span to the front is a rotation. Reusing the +// reference encoder keeps one source of truth for link sorting and field +// presence. dataLen is the length of the Data field that was encoded; the +// field must be present. +func moveDataFirst(enc []byte, dataLen int) []byte { + span := protowire.SizeTag(1) + protowire.SizeBytes(dataLen) + split := len(enc) - span + return slices.Concat(enc[split:], enc[:split]) } diff --git a/ipld/merkledag/fieldorder_test.go b/ipld/merkledag/fieldorder_test.go index bfbbd039a..7da46438d 100644 --- a/ipld/merkledag/fieldorder_test.go +++ b/ipld/merkledag/fieldorder_test.go @@ -3,11 +3,17 @@ package merkledag_test import ( "bytes" "encoding/hex" + "math/rand/v2" + "slices" + "strings" "testing" "github.com/ipfs/boxo/ipld/merkledag" cid "github.com/ipfs/go-cid" + format "github.com/ipfs/go-ipld-format" mh "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/encoding/protowire" ) // Test fixtures from IPIP-550 (https://github.com/ipfs/specs/pull/550): @@ -124,3 +130,189 @@ func TestPBNodeFieldOrder(t *testing.T) { }) } } + +func TestPBNodeFieldOrderUnknown(t *testing.T) { + saveFieldOrder(t) + merkledag.DefaultPBNodeFieldOrder = merkledag.PBNodeFieldOrder(7) + + node := merkledag.NodeWithData([]byte("x")) + _, err := node.EncodeProtobuf(true) + require.ErrorContains(t, err, "unknown PBNodeFieldOrder 7") +} + +// encodeLinksFirst is a test-only dag-pb encoder built directly on +// protowire. Unlike the reference encoder it keeps links in the given order, +// so it can produce blocks with unsorted links, which is what a node decoded +// from another implementation may look like. +func encodeLinksFirst(data []byte, links []*format.Link) []byte { + var enc []byte + for _, l := range links { + var pl []byte + pl = protowire.AppendTag(pl, 1, protowire.BytesType) + pl = protowire.AppendBytes(pl, l.Cid.Bytes()) + pl = protowire.AppendTag(pl, 2, protowire.BytesType) + pl = protowire.AppendString(pl, l.Name) + pl = protowire.AppendTag(pl, 3, protowire.VarintType) + pl = protowire.AppendVarint(pl, l.Size) + enc = protowire.AppendTag(enc, 2, protowire.BytesType) + enc = protowire.AppendBytes(enc, pl) + } + if data != nil { + enc = protowire.AppendTag(enc, 1, protowire.BytesType) + enc = protowire.AppendBytes(enc, data) + } + return enc +} + +func randomBytes(r *rand.Rand, n int) []byte { + b := make([]byte, n) + for i := range b { + b[i] = byte(r.IntN(256)) + } + return b +} + +func randomName(r *rand.Rand, n int) string { + const alphabet = "abcdefghijklmnopqrstuvwxyz0123456789-_." + var sb strings.Builder + sb.Grow(n) + for range n { + sb.WriteByte(alphabet[r.IntN(len(alphabet))]) + } + return sb.String() +} + +func randomCid(t *testing.T, r *rand.Rand) cid.Cid { + payload := randomBytes(r, r.IntN(64)+1) + switch r.IntN(3) { + case 0: + h, err := mh.Sum(payload, mh.SHA2_256, -1) + require.NoError(t, err) + return cid.NewCidV0(h) + case 1: + h, err := mh.Sum(payload, mh.SHA2_256, -1) + require.NoError(t, err) + return cid.NewCidV1(cid.Raw, h) + default: + h, err := mh.Sum(payload, mh.IDENTITY, -1) + require.NoError(t, err) + return cid.NewCidV1(cid.DagProtobuf, h) + } +} + +// pickLen returns a length that crosses the 1, 2 and 3 byte varint +// boundaries with useful frequency; the largest class is rare because it +// dominates block size. +func pickLen(r *rand.Rand) int { + switch r.IntN(20) { + case 0: + return 0 + case 1, 2, 3: + return 128 + r.IntN(2000) + case 4: + return 16384 + r.IntN(600) + default: + return 1 + r.IntN(127) + } +} + +type randomNode struct { + data []byte + links []*format.Link +} + +func newRandomNode(t *testing.T, r *rand.Rand) randomNode { + var n randomNode + switch r.IntN(4) { + case 0: + n.data = nil + case 1: + n.data = []byte{} + default: + n.data = randomBytes(r, pickLen(r)) + } + var count int + switch r.IntN(6) { + case 0: + count = 0 + case 1: + count = 128 + r.IntN(200) + default: + count = 1 + r.IntN(16) + } + n.links = make([]*format.Link, 0, count) + for range count { + nameLen := pickLen(r) + if count > 16 && nameLen > 2000 { + nameLen = 2000 + } + n.links = append(n.links, &format.Link{ + Name: randomName(r, nameLen), + Size: r.Uint64N(1 << 63), + Cid: randomCid(t, r), + }) + } + return n +} + +func requireSameLinks(t *testing.T, want, got []*format.Link) { + t.Helper() + require.Len(t, got, len(want)) + for i := range want { + require.Equal(t, want[i].Name, got[i].Name, "link %d name", i) + require.Equal(t, want[i].Size, got[i].Size, "link %d size", i) + require.True(t, want[i].Cid.Equals(got[i].Cid), "link %d cid", i) + } +} + +// TestPBNodeFieldOrderRandomNodes decodes randomly shaped links-first blocks +// (including ones with unsorted links) and checks that re-encoding them under +// each order yields a block that decodes to the same node with links sorted +// by name, that the Data field leads the data-first form, and that the two +// forms differ only by where the Data field sits. +func TestPBNodeFieldOrderRandomNodes(t *testing.T) { + saveFieldOrder(t) + r := rand.New(rand.NewPCG(550, 2026)) + + for i := range 300 { + n := newRandomNode(t, r) + r.Shuffle(len(n.links), func(a, b int) { n.links[a], n.links[b] = n.links[b], n.links[a] }) + sorted := slices.Clone(n.links) + slices.SortStableFunc(sorted, func(a, b *format.Link) int { return strings.Compare(a.Name, b.Name) }) + + raw := encodeLinksFirst(n.data, n.links) + node, err := merkledag.DecodeProtobuf(raw) + require.NoError(t, err, "node %d", i) + + var encoded [2][]byte + for _, order := range []merkledag.PBNodeFieldOrder{merkledag.PBNodeLinksFirst, merkledag.PBNodeDataFirst} { + merkledag.DefaultPBNodeFieldOrder = order + enc, err := node.EncodeProtobuf(true) + require.NoError(t, err, "node %d order %d", i, order) + encoded[order] = enc + + back, err := merkledag.DecodeProtobuf(enc) + require.NoError(t, err, "node %d order %d", i, order) + if len(n.data) == 0 { + require.Empty(t, back.Data(), "node %d order %d", i, order) + } else { + require.Equal(t, n.data, back.Data(), "node %d order %d", i, order) + } + requireSameLinks(t, sorted, back.Links()) + } + + linksFirst, dataFirst := encoded[merkledag.PBNodeLinksFirst], encoded[merkledag.PBNodeDataFirst] + require.Len(t, dataFirst, len(linksFirst), "node %d", i) + if n.data == nil { + require.Equal(t, linksFirst, dataFirst, "node %d: no Data field, orders must agree", i) + continue + } + num, typ, tagLen := protowire.ConsumeTag(dataFirst) + require.Equal(t, protowire.Number(1), num, "node %d: first field", i) + require.Equal(t, protowire.BytesType, typ, "node %d: first field type", i) + field, fieldLen := protowire.ConsumeBytes(dataFirst[tagLen:]) + require.Equal(t, n.data, field, "node %d: leading Data field", i) + span := tagLen + fieldLen + require.Equal(t, linksFirst, slices.Concat(dataFirst[span:], dataFirst[:span]), "node %d: forms differ beyond Data placement", i) + } +} From d3be3254ae5e35ffa0bfd3fa8f7239e28ece883e Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sun, 30 Aug 2026 21:38:52 +0200 Subject: [PATCH 5/8] test(unixfs): pin unixfs-v1-2026 CIDs end to end Build a three-chunk file and a sharded directory from fixed inputs under UnixFS_v1_2025 and UnixFS_v1_2026 and pin the root CIDs; the file CIDs match `ipfs add --chunker=size-1000` output under each profile. Every dag-pb block in both DAGs must lead with the profile's first field, and the two DAGs must decode to the same nodes. - profile.go: UnixFS_v1_2026 written as a full literal so the test asserts each parameter instead of reconstructing the copy - profile_test.go: per-field asserts for all three profiles; ApplyGlobals subtest checks all six globals; saveAndRestoreGlobals also restores chunk.DefaultBlockSize and helpers.DefaultLinksPerBlock so applied profiles no longer leak into later tests --- ipld/unixfs/io/profile.go | 16 ++- ipld/unixfs/io/profile_test.go | 180 ++++++++++++++++++++++++++------- 2 files changed, 155 insertions(+), 41 deletions(-) diff --git a/ipld/unixfs/io/profile.go b/ipld/unixfs/io/profile.go index 34e233eed..e282e0007 100644 --- a/ipld/unixfs/io/profile.go +++ b/ipld/unixfs/io/profile.go @@ -113,11 +113,17 @@ var ( // Links, so streaming readers can process HAMT parameters before reading // links. Opt-in: directories and HAMT shards get different CIDs than // under UnixFS_v1_2025. - UnixFS_v1_2026 = func() UnixFSProfile { - p := UnixFS_v1_2025 - p.PBNodeFieldOrder = mdag.PBNodeDataFirst - return p - }() + UnixFS_v1_2026 = UnixFSProfile{ + CIDVersion: 1, + MhType: mh.SHA2_256, + ChunkSize: int64(1 * unitMiB), + FileDAGWidth: 1024, + RawLeaves: true, // raw leaves for CIDv1 + HAMTShardingSize: int(256 * unitKiB), + HAMTSizeEstimation: SizeEstimationBlock, + HAMTShardWidth: 256, + PBNodeFieldOrder: mdag.PBNodeDataFirst, + } ) // ApplyGlobals sets the global variables to match this profile's settings. diff --git a/ipld/unixfs/io/profile_test.go b/ipld/unixfs/io/profile_test.go index 83c6df9d2..e827af725 100644 --- a/ipld/unixfs/io/profile_test.go +++ b/ipld/unixfs/io/profile_test.go @@ -1,16 +1,21 @@ package io import ( + "bytes" "context" "encoding/hex" "fmt" "os" + "slices" "testing" "time" + chunk "github.com/ipfs/boxo/chunker" mdag "github.com/ipfs/boxo/ipld/merkledag" mdtest "github.com/ipfs/boxo/ipld/merkledag/test" ft "github.com/ipfs/boxo/ipld/unixfs" + "github.com/ipfs/boxo/ipld/unixfs/importer/balanced" + "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" "github.com/ipfs/boxo/ipld/unixfs/private/linksize" cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" @@ -55,10 +60,17 @@ func TestUnixFSProfiles(t *testing.T) { }) t.Run("UnixFS_v1_2026 has correct values", func(t *testing.T) { - expected := UnixFS_v1_2025 - expected.PBNodeFieldOrder = mdag.PBNodeDataFirst - assert.Equal(t, expected, UnixFS_v1_2026, - "UnixFS_v1_2026 should equal UnixFS_v1_2025 plus data-first PBNode field order") + assert.Equal(t, 1, UnixFS_v1_2026.CIDVersion, "CIDVersion should be 1") + assert.Equal(t, uint64(mh.SHA2_256), UnixFS_v1_2026.MhType, "MhType should be SHA2_256") + assert.Equal(t, int64(1024*1024), UnixFS_v1_2026.ChunkSize, "ChunkSize should be 1 MiB") + assert.Equal(t, 1024, UnixFS_v1_2026.FileDAGWidth, "FileDAGWidth should be 1024") + assert.True(t, UnixFS_v1_2026.RawLeaves, "RawLeaves should be true for CIDv1") + assert.Equal(t, 256*1024, UnixFS_v1_2026.HAMTShardingSize, "HAMTShardingSize should be 256 KiB") + assert.Equal(t, SizeEstimationBlock, UnixFS_v1_2026.HAMTSizeEstimation, "should use block-based estimation") + assert.Equal(t, 256, UnixFS_v1_2026.HAMTShardWidth, "HAMTShardWidth should be 256") + assert.Equal(t, mdag.PBNodeDataFirst, UnixFS_v1_2026.PBNodeFieldOrder, "PBNodeFieldOrder should be data-first") + assert.Equal(t, mdag.PBNodeLinksFirst, UnixFS_v1_2025.PBNodeFieldOrder, "UnixFS_v1_2025 should keep links-first") + assert.Equal(t, mdag.PBNodeLinksFirst, UnixFS_v0_2015.PBNodeFieldOrder, "UnixFS_v0_2015 should keep links-first") }) t.Run("CidBuilder returns correct prefix", func(t *testing.T) { @@ -80,51 +92,35 @@ func TestUnixFSProfiles(t *testing.T) { }) t.Run("ApplyGlobals sets global variables", func(t *testing.T) { - // Save original values - oldShardingSize := HAMTShardingSize - oldEstimation := HAMTSizeEstimation - oldShardWidth := DefaultShardWidth - oldFieldOrder := mdag.DefaultPBNodeFieldOrder - t.Cleanup(func() { - HAMTShardingSize = oldShardingSize - HAMTSizeEstimation = oldEstimation - DefaultShardWidth = oldShardWidth - mdag.DefaultPBNodeFieldOrder = oldFieldOrder - }) - - // Apply UnixFS_v1_2025 - UnixFS_v1_2025.ApplyGlobals() - - assert.Equal(t, UnixFS_v1_2025.HAMTShardingSize, HAMTShardingSize) - assert.Equal(t, UnixFS_v1_2025.HAMTSizeEstimation, HAMTSizeEstimation) - assert.Equal(t, UnixFS_v1_2025.HAMTShardWidth, DefaultShardWidth) - assert.Equal(t, mdag.PBNodeLinksFirst, mdag.DefaultPBNodeFieldOrder) - - // Apply UnixFS_v1_2026 - UnixFS_v1_2026.ApplyGlobals() - - assert.Equal(t, mdag.PBNodeDataFirst, mdag.DefaultPBNodeFieldOrder) + saveAndRestoreGlobals(t) - // Apply UnixFS_v0_2015 - UnixFS_v0_2015.ApplyGlobals() + for _, p := range []UnixFSProfile{UnixFS_v1_2025, UnixFS_v1_2026, UnixFS_v0_2015} { + p.ApplyGlobals() - assert.Equal(t, UnixFS_v0_2015.HAMTShardingSize, HAMTShardingSize) - assert.Equal(t, UnixFS_v0_2015.HAMTSizeEstimation, HAMTSizeEstimation) - assert.Equal(t, UnixFS_v0_2015.HAMTShardWidth, DefaultShardWidth) - assert.Equal(t, mdag.PBNodeLinksFirst, mdag.DefaultPBNodeFieldOrder) + assert.Equal(t, p.ChunkSize, chunk.DefaultBlockSize) + assert.Equal(t, p.FileDAGWidth, helpers.DefaultLinksPerBlock) + assert.Equal(t, p.HAMTShardingSize, HAMTShardingSize) + assert.Equal(t, p.HAMTSizeEstimation, HAMTSizeEstimation) + assert.Equal(t, p.HAMTShardWidth, DefaultShardWidth) + assert.Equal(t, p.PBNodeFieldOrder, mdag.DefaultPBNodeFieldOrder) + } }) } // saveAndRestoreGlobals saves the current global settings and restores them -// after the test completes. Use this in tests that modify HAMTShardingSize, -// HAMTSizeEstimation, DefaultShardWidth, or linksize.LinkSizeFunction. +// after the test completes. Use this in tests that call ApplyGlobals or +// modify any of the globals it writes, or linksize.LinkSizeFunction. func saveAndRestoreGlobals(t *testing.T) { + oldBlockSize := chunk.DefaultBlockSize + oldLinksPerBlock := helpers.DefaultLinksPerBlock oldShardingSize := HAMTShardingSize oldEstimation := HAMTSizeEstimation oldShardWidth := DefaultShardWidth oldLinkSize := linksize.LinkSizeFunction oldFieldOrder := mdag.DefaultPBNodeFieldOrder t.Cleanup(func() { + chunk.DefaultBlockSize = oldBlockSize + helpers.DefaultLinksPerBlock = oldLinksPerBlock HAMTShardingSize = oldShardingSize HAMTSizeEstimation = oldEstimation DefaultShardWidth = oldShardWidth @@ -186,6 +182,118 @@ func TestProfilePBNodeFieldOrderFixtures(t *testing.T) { } } +// TestProfilePBNodeFieldOrderEndToEnd builds a multi-chunk file and a sharded +// directory under UnixFS_v1_2025 and UnixFS_v1_2026 from fixed inputs and +// pins the resulting root CIDs. The file inputs and expected CIDs match what +// `ipfs add --chunker=size-1000` produces under each profile. Every dag-pb +// block of both DAGs must lead with the profile's first field, and the two +// DAGs must decode to the same nodes. +func TestProfilePBNodeFieldOrderEndToEnd(t *testing.T) { + const ( + fileSize = 3000 + chunkSize = 1000 + hamtEntries = 64 + ) + + cases := []struct { + name string + profile UnixFSProfile + firstByte byte + fileCid string + hamtCid string + }{ + {"UnixFS_v1_2025", UnixFS_v1_2025, 0x12, + "bafybeiapp6tzng2hpxzmopnplvg6hoxldufypompu6uzmhc6rdyw4m2qx4", + "bafybeiahnfucbarnualj2uzakbqnj3b3nvrszwu4jrl2x7ekbfqshpd524"}, + {"UnixFS_v1_2026", UnixFS_v1_2026, 0x0a, + "bafybeigwq5lxlau4ced4hlvjxuz4xtdbs4zz32dgndqpcc2ggcbiei6k2y", + "bafybeihfyecdexcpn3g3xqgzz23pexyespuldzfoh3tvjris56cxkropau"}, + } + + // fingerprints of every dag-pb node reachable from the roots, per profile; + // link CIDs are left out because dag-pb children differ between profiles + fingerprints := map[string][]string{} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + saveAndRestoreGlobals(t) + tc.profile.ApplyGlobals() + HAMTShardingSize = 1024 // shard at a size a unit test can reach + + ds := mdtest.Mock() + ctx := t.Context() + + params := helpers.DagBuilderParams{ + Maxlinks: tc.profile.FileDAGWidth, + RawLeaves: tc.profile.RawLeaves, + CidBuilder: tc.profile.CidBuilder(), + Dagserv: ds, + } + content := bytes.Repeat([]byte("a"), fileSize) + db, err := params.New(chunk.NewSizeSplitter(bytes.NewReader(content), chunkSize)) + require.NoError(t, err) + fileRoot, err := balanced.Layout(db) + require.NoError(t, err) + assert.Equal(t, tc.fileCid, fileRoot.Cid().String(), "file root CID") + + dir, err := NewDirectory(ds) + require.NoError(t, err) + dir.SetCidBuilder(tc.profile.CidBuilder()) + for i := range hamtEntries { + leaf := mdag.NewRawNode(fmt.Appendf(nil, "file %d\n", i)) + require.NoError(t, ds.Add(ctx, leaf)) + require.NoError(t, dir.AddChild(ctx, fmt.Sprintf("file-%d.txt", i), leaf)) + } + _, isHAMT := dir.(*DynamicDirectory).Directory.(*HAMTDirectory) + require.True(t, isHAMT, "directory should have switched to HAMT") + dirRoot, err := dir.GetNode() + require.NoError(t, err) + require.NoError(t, ds.Add(ctx, dirRoot)) + assert.Equal(t, tc.hamtCid, dirRoot.Cid().String(), "HAMT root CID") + + var dagpbNodes, subShards int + for _, root := range []cid.Cid{fileRoot.Cid(), dirRoot.Cid()} { + err := mdag.Walk(ctx, mdag.GetLinksDirect(ds), root, func(c cid.Cid) bool { + if c.Type() != cid.DagProtobuf { + return true + } + node, err := ds.Get(ctx, c) + require.NoError(t, err) + pn, ok := node.(*mdag.ProtoNode) + require.True(t, ok) + require.NotEmpty(t, pn.Data(), "%s: every dag-pb node here carries UnixFS Data", c) + require.NotEmpty(t, pn.Links(), "%s: every dag-pb node here carries links", c) + assert.Equal(t, tc.firstByte, pn.RawData()[0], "%s: first field", c) + dagpbNodes++ + + names := make([]string, 0, len(pn.Links())) + sizes := make([]uint64, 0, len(pn.Links())) + for _, l := range pn.Links() { + names = append(names, l.Name) + sizes = append(sizes, l.Size) + if c.Equals(dirRoot.Cid()) && len(l.Name) == 2 { + subShards++ + } + } + fingerprints[tc.name] = append(fingerprints[tc.name], + fmt.Sprintf("%x|%q|%v", pn.Data(), names, sizes)) + return true + }) + require.NoError(t, err) + } + assert.Greater(t, subShards, 0, "HAMT root should link to child shards") + assert.Greater(t, dagpbNodes, 2, "file root, HAMT root and child shards expected") + t.Logf("%s: %d dag-pb nodes, %d child shards", tc.name, dagpbNodes, subShards) + }) + } + + for _, fp := range fingerprints { + slices.Sort(fp) + } + assert.Equal(t, fingerprints["UnixFS_v1_2025"], fingerprints["UnixFS_v1_2026"], + "both profiles must produce the same logical nodes") +} + func TestProfileHAMTThresholdBehavior(t *testing.T) { // Use fixed link size for predictable testing const fixedLinkSize = 100 From 04a079ec27b1605112e49ebc0ebd6a1afbecc7b7 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sun, 30 Aug 2026 21:38:52 +0200 Subject: [PATCH 6/8] docs(unixfs): which CIDs unixfs-v1-2026 changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every dag-pb node with both Data and Links gets a new CID, files larger than one chunk included; single-chunk raw-leaf files keep theirs. Existing links-first directories are re-encoded when reopened through the directory API (MFS directories on their next access), a sharded root first and each child shard as it is loaded. The godocs also state why the setting is a process-wide global and that it must be applied once at startup. - profile.go: field, profile and ApplyGlobals godoc - doc.go: UnixFS_v1_2026 in the profile list, Global Settings section - CHANGELOG.md: scope, MFS re-encode, ✨ marker, PR link --- CHANGELOG.md | 2 +- ipld/unixfs/io/doc.go | 14 +++++++++++++- ipld/unixfs/io/profile.go | 37 +++++++++++++++++++++++++++---------- 3 files changed, 41 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40a98d9b4..d2766e1c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ The following emojis are used to highlight certain changes: ### Added -- `ipld/unixfs`: opt-in `io.UnixFS_v1_2026` profile (and `UnixFSProfile.PBNodeFieldOrder` parameter) writes the `PBNode` `Data` field before `Links`, so streaming readers can read HAMT parameters before links. All other profiles and the default (`merkledag.DefaultPBNodeFieldOrder`) keep the canonical links-first bytes and CIDs. [IPIP-550](https://github.com/ipfs/specs/pull/550) +- ✨ `ipld/unixfs`: opt-in `io.UnixFS_v1_2026` profile (and `UnixFSProfile.PBNodeFieldOrder` parameter) writes the `PBNode` `Data` field before `Links`, so streaming readers can read HAMT parameters before links. It changes the CID of every dag-pb node that has both fields: directories, HAMT shards, and the root and intermediate nodes of files larger than one chunk. Existing links-first directories get new CIDs the next time they are opened through the directory API and stored again (for example MFS directories on their next access); a sharded directory's root is re-encoded first and each child shard once it is loaded. The order is a process-wide setting (`merkledag.DefaultPBNodeFieldOrder`) applied once at startup with `ApplyGlobals`; it affects every `merkledag.ProtoNode` encoded in the process, not only UnixFS nodes. All other profiles and the default keep the canonical links-first bytes and CIDs. [IPIP-550](https://github.com/ipfs/specs/pull/550) [#1212](https://github.com/ipfs/boxo/pull/1212) - ✨ `gateway`: responses now include the `Ipfs-Uri` header with a canonical `ipfs://` or `ipns://` URI for the requested content path, and expose it via the default `Access-Control-Expose-Headers`. The header carries the content root in canonical form (base32 CIDv1 for `/ipfs/`, base36 CIDv1 for cryptographic `/ipns/` names, lowercase FQDN for DNSLink) with percent-encoded path segments, so clients get a value that is safe in HTTP field context regardless of bytes in the underlying path. [IPIP-548](https://github.com/ipfs/specs/pull/548) [#1209](https://github.com/ipfs/boxo/pull/1209) ### Changed diff --git a/ipld/unixfs/io/doc.go b/ipld/unixfs/io/doc.go index b6d524fce..b948c5b90 100644 --- a/ipld/unixfs/io/doc.go +++ b/ipld/unixfs/io/doc.go @@ -36,6 +36,18 @@ // // - [UnixFS_v0_2015]: Legacy CIDv0 settings (256 KiB chunks, dag-pb leaves) // - [UnixFS_v1_2025]: Modern CIDv1 settings (1 MiB chunks, raw leaves) +// - [UnixFS_v1_2026]: UnixFS_v1_2025 with the dag-pb Data field written +// before Links (IPIP-550) // -// See https://specs.ipfs.tech/ipips/ipip-0499/ for specification details. +// See https://specs.ipfs.tech/ipips/ipip-0499/ and +// https://github.com/ipfs/specs/pull/550 for specification details. +// +// # Global Settings +// +// [UnixFSProfile.ApplyGlobals] writes a profile into package-level variables +// ([HAMTShardingSize], [HAMTSizeEstimation], [DefaultShardWidth], +// chunk.DefaultBlockSize, helpers.DefaultLinksPerBlock and +// merkledag.DefaultPBNodeFieldOrder). They are read on every import without +// synchronization, so apply a profile once at startup, before the first +// import, and do not change these variables while the process runs. package io diff --git a/ipld/unixfs/io/profile.go b/ipld/unixfs/io/profile.go index e282e0007..e7d5648c3 100644 --- a/ipld/unixfs/io/profile.go +++ b/ipld/unixfs/io/profile.go @@ -62,8 +62,10 @@ type UnixFSProfile struct { // serialized dag-pb blocks. The zero value (merkledag.PBNodeLinksFirst) // is the canonical DAG-PB order used by all profiles through // unixfs-v1-2025. merkledag.PBNodeDataFirst is the opt-in order proposed - // by IPIP-550 (https://github.com/ipfs/specs/pull/550) and changes the - // CIDs of directories and HAMT shards. + // by IPIP-550 (https://github.com/ipfs/specs/pull/550). It changes the + // CID of every dag-pb node that has both Data and Links: directories, + // HAMT shards, and the root and intermediate nodes of files larger than + // one chunk. Single-chunk raw-leaf files keep their CIDs. PBNodeFieldOrder mdag.PBNodeFieldOrder } @@ -108,11 +110,19 @@ var ( } // UnixFS_v1_2026 matches the unixfs-v1-2026 profile proposed in IPIP-550 - // (https://github.com/ipfs/specs/pull/550). It inherits all settings from - // UnixFS_v1_2025 and additionally writes the PBNode Data field before - // Links, so streaming readers can process HAMT parameters before reading - // links. Opt-in: directories and HAMT shards get different CIDs than - // under UnixFS_v1_2025. + // (https://github.com/ipfs/specs/pull/550): the UnixFS_v1_2025 settings + // with the PBNode Data field written before Links, so streaming readers + // can process HAMT parameters before reading links. Opt-in: every dag-pb + // node with both Data and Links (directories, HAMT shards, files larger + // than one chunk) gets a different CID than under UnixFS_v1_2025. + // + // Applying this profile to a repository that holds links-first data + // re-encodes existing directories in the new order when they are loaded + // and stored again (for example MFS directories on their next access), + // so their CIDs change without a content change. A sharded directory's + // root is re-encoded first; each child shard follows once a lookup, a + // listing, or a change loads it (a listing loads all of them). Both + // orders decode identically. UnixFS_v1_2026 = UnixFSProfile{ CIDVersion: 1, MhType: mh.SHA2_256, @@ -130,9 +140,16 @@ var ( // This affects all subsequent file and directory import operations. // Note: RawLeaves and CidBuilder are not globals; pass them to DAG builder options. // -// Thread safety: this function modifies global variables and is not safe -// for concurrent use. Call it once during program initialization, before -// starting any imports. Do not call from multiple goroutines. +// The settings written here (chunk.DefaultBlockSize, +// helpers.DefaultLinksPerBlock, HAMTShardingSize, HAMTSizeEstimation, +// DefaultShardWidth and merkledag.DefaultPBNodeFieldOrder) are process-wide +// globals rather than per-call options: threading them through every +// producer and consumer of the import pipeline would amount to a rewrite. +// Call ApplyGlobals once at startup, before the first import, and do not +// call it again while the process runs. The globals are read on every +// import without synchronization, and a ProtoNode that was already encoded +// keeps its cached bytes and CID until it is mutated (see +// merkledag.DefaultPBNodeFieldOrder). func (p UnixFSProfile) ApplyGlobals() { // File settings chunk.DefaultBlockSize = p.ChunkSize From adefcafc26766f6d263dac843e5dcf415f855078 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sun, 30 Aug 2026 21:53:22 +0200 Subject: [PATCH 7/8] style(unixfs): gofumpt profile_test.go --- ipld/unixfs/io/profile_test.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ipld/unixfs/io/profile_test.go b/ipld/unixfs/io/profile_test.go index e827af725..e209a64f6 100644 --- a/ipld/unixfs/io/profile_test.go +++ b/ipld/unixfs/io/profile_test.go @@ -202,12 +202,16 @@ func TestProfilePBNodeFieldOrderEndToEnd(t *testing.T) { fileCid string hamtCid string }{ - {"UnixFS_v1_2025", UnixFS_v1_2025, 0x12, + { + "UnixFS_v1_2025", UnixFS_v1_2025, 0x12, "bafybeiapp6tzng2hpxzmopnplvg6hoxldufypompu6uzmhc6rdyw4m2qx4", - "bafybeiahnfucbarnualj2uzakbqnj3b3nvrszwu4jrl2x7ekbfqshpd524"}, - {"UnixFS_v1_2026", UnixFS_v1_2026, 0x0a, + "bafybeiahnfucbarnualj2uzakbqnj3b3nvrszwu4jrl2x7ekbfqshpd524", + }, + { + "UnixFS_v1_2026", UnixFS_v1_2026, 0x0a, "bafybeigwq5lxlau4ced4hlvjxuz4xtdbs4zz32dgndqpcc2ggcbiei6k2y", - "bafybeihfyecdexcpn3g3xqgzz23pexyespuldzfoh3tvjris56cxkropau"}, + "bafybeihfyecdexcpn3g3xqgzz23pexyespuldzfoh3tvjris56cxkropau", + }, } // fingerprints of every dag-pb node reachable from the roots, per profile; From 5b06f4ba66090abd9ca1ce70c918355cc00d9997 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 2 Sep 2026 14:43:41 +0200 Subject: [PATCH 8/8] refactor: drop UnixFS_v1_2026, keep opt-in knob A dated successor profile invites unintentional adoption and a de facto new CIDv1 default. PBNodeFieldOrder stays as a documented low-level opt-in; UnixFS_v0_2015 and UnixFS_v1_2025 now pin PBNodeLinksFirst explicitly. Tests derive data-first from UnixFS_v1_2025 plus the knob and keep asserting the same IPIP-550 fixture bytes and CIDs. Refs ipfs/specs#550 --- CHANGELOG.md | 2 +- ipld/merkledag/fieldorder.go | 5 ++-- ipld/unixfs/io/doc.go | 7 +++-- ipld/unixfs/io/profile.go | 47 ++++++++++----------------------- ipld/unixfs/io/profile_test.go | 48 +++++++++++++++++----------------- 5 files changed, 47 insertions(+), 62 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2766e1c5..9f1403511 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ The following emojis are used to highlight certain changes: ### Added -- ✨ `ipld/unixfs`: opt-in `io.UnixFS_v1_2026` profile (and `UnixFSProfile.PBNodeFieldOrder` parameter) writes the `PBNode` `Data` field before `Links`, so streaming readers can read HAMT parameters before links. It changes the CID of every dag-pb node that has both fields: directories, HAMT shards, and the root and intermediate nodes of files larger than one chunk. Existing links-first directories get new CIDs the next time they are opened through the directory API and stored again (for example MFS directories on their next access); a sharded directory's root is re-encoded first and each child shard once it is loaded. The order is a process-wide setting (`merkledag.DefaultPBNodeFieldOrder`) applied once at startup with `ApplyGlobals`; it affects every `merkledag.ProtoNode` encoded in the process, not only UnixFS nodes. All other profiles and the default keep the canonical links-first bytes and CIDs. [IPIP-550](https://github.com/ipfs/specs/pull/550) [#1212](https://github.com/ipfs/boxo/pull/1212) +- ✨ `ipld/unixfs`: reads of both `PBNode` field orders are now covered by tests, and a documented low-level opt-in (`UnixFSProfile.PBNodeFieldOrder`, applied via `merkledag.DefaultPBNodeFieldOrder`) lets writers that need streaming-friendly blocks encode the `Data` field before `Links` per [IPIP-550](https://github.com/ipfs/specs/pull/550). Off by default and selected by no named profile: `UnixFS_v0_2015` and `UnixFS_v1_2025` pin the canonical links-first order explicitly, so defaults and existing CIDs are unchanged. Enabling data-first changes the CID of every dag-pb node that has both fields (directories, HAMT shards, multi-chunk file roots), is process-wide (`ApplyGlobals` affects every `merkledag.ProtoNode` encoded in the process, not only UnixFS nodes), and re-encodes links-first directories in the new order the next time they are opened through the directory API and stored again (for example MFS directories on their next access). [#1212](https://github.com/ipfs/boxo/pull/1212) - ✨ `gateway`: responses now include the `Ipfs-Uri` header with a canonical `ipfs://` or `ipns://` URI for the requested content path, and expose it via the default `Access-Control-Expose-Headers`. The header carries the content root in canonical form (base32 CIDv1 for `/ipfs/`, base36 CIDv1 for cryptographic `/ipns/` names, lowercase FQDN for DNSLink) with percent-encoded path segments, so clients get a value that is safe in HTTP field context regardless of bytes in the underlying path. [IPIP-548](https://github.com/ipfs/specs/pull/548) [#1209](https://github.com/ipfs/boxo/pull/1209) ### Changed diff --git a/ipld/merkledag/fieldorder.go b/ipld/merkledag/fieldorder.go index 9d0be4692..61068e2a0 100644 --- a/ipld/merkledag/fieldorder.go +++ b/ipld/merkledag/fieldorder.go @@ -24,8 +24,9 @@ const ( // repeated Links field (field number 2), so streaming readers can // process Data (e.g. HAMT parameters) before reading links. The DAG-PB // spec says decoders should accept either order [1]; IPIP-550 - // (https://github.com/ipfs/specs/pull/550) proposes this one for the - // unixfs-v1-2026 profile. + // (https://github.com/ipfs/specs/pull/550) defines this one as a + // low-level opt-in for writers that need it. No named profile selects + // it, and enabling it changes CIDs. // // [1]: https://ipld.io/specs/codecs/dag-pb/spec/#protobuf-strictness PBNodeDataFirst diff --git a/ipld/unixfs/io/doc.go b/ipld/unixfs/io/doc.go index b948c5b90..75d60ea65 100644 --- a/ipld/unixfs/io/doc.go +++ b/ipld/unixfs/io/doc.go @@ -36,8 +36,11 @@ // // - [UnixFS_v0_2015]: Legacy CIDv0 settings (256 KiB chunks, dag-pb leaves) // - [UnixFS_v1_2025]: Modern CIDv1 settings (1 MiB chunks, raw leaves) -// - [UnixFS_v1_2026]: UnixFS_v1_2025 with the dag-pb Data field written -// before Links (IPIP-550) +// +// Both pin the canonical links-first dag-pb field order. Writers that need +// the streaming-friendly Data-first order can opt in via the low-level +// [UnixFSProfile.PBNodeFieldOrder] knob (IPIP-550); no named profile +// selects it, and enabling it changes CIDs. // // See https://specs.ipfs.tech/ipips/ipip-0499/ and // https://github.com/ipfs/specs/pull/550 for specification details. diff --git a/ipld/unixfs/io/profile.go b/ipld/unixfs/io/profile.go index e7d5648c3..a3a94ec91 100644 --- a/ipld/unixfs/io/profile.go +++ b/ipld/unixfs/io/profile.go @@ -59,13 +59,18 @@ type UnixFSProfile struct { HAMTShardWidth int // PBNodeFieldOrder controls the order of the top-level PBNode fields in - // serialized dag-pb blocks. The zero value (merkledag.PBNodeLinksFirst) - // is the canonical DAG-PB order used by all profiles through - // unixfs-v1-2025. merkledag.PBNodeDataFirst is the opt-in order proposed - // by IPIP-550 (https://github.com/ipfs/specs/pull/550). It changes the - // CID of every dag-pb node that has both Data and Links: directories, - // HAMT shards, and the root and intermediate nodes of files larger than - // one chunk. Single-chunk raw-leaf files keep their CIDs. + // serialized dag-pb blocks. merkledag.PBNodeLinksFirst (the zero value) + // is the canonical DAG-PB order, pinned explicitly by every named + // profile. merkledag.PBNodeDataFirst is a low-level opt-in knob from + // IPIP-550 (https://github.com/ipfs/specs/pull/550) for writers that + // need streaming-friendly blocks; no named profile selects it. Enabling + // it changes the CID of every dag-pb node that has both Data and Links: + // directories, HAMT shards, and the root and intermediate nodes of + // files larger than one chunk. Single-chunk raw-leaf files keep their + // CIDs. Links-first data touched through the directory API is + // re-encoded in the new order when it is stored again (for example MFS + // directories on their next change), so CIDs change without a content + // change. PBNodeFieldOrder mdag.PBNodeFieldOrder } @@ -93,6 +98,7 @@ var ( HAMTShardingSize: int(256 * unitKiB), HAMTSizeEstimation: SizeEstimationLinks, HAMTShardWidth: 256, + PBNodeFieldOrder: mdag.PBNodeLinksFirst, // canonical order, pinned } // UnixFS_v1_2025 matches the unixfs-v1-2025 profile from IPIP-499. @@ -107,32 +113,7 @@ var ( HAMTShardingSize: int(256 * unitKiB), HAMTSizeEstimation: SizeEstimationBlock, HAMTShardWidth: 256, - } - - // UnixFS_v1_2026 matches the unixfs-v1-2026 profile proposed in IPIP-550 - // (https://github.com/ipfs/specs/pull/550): the UnixFS_v1_2025 settings - // with the PBNode Data field written before Links, so streaming readers - // can process HAMT parameters before reading links. Opt-in: every dag-pb - // node with both Data and Links (directories, HAMT shards, files larger - // than one chunk) gets a different CID than under UnixFS_v1_2025. - // - // Applying this profile to a repository that holds links-first data - // re-encodes existing directories in the new order when they are loaded - // and stored again (for example MFS directories on their next access), - // so their CIDs change without a content change. A sharded directory's - // root is re-encoded first; each child shard follows once a lookup, a - // listing, or a change loads it (a listing loads all of them). Both - // orders decode identically. - UnixFS_v1_2026 = UnixFSProfile{ - CIDVersion: 1, - MhType: mh.SHA2_256, - ChunkSize: int64(1 * unitMiB), - FileDAGWidth: 1024, - RawLeaves: true, // raw leaves for CIDv1 - HAMTShardingSize: int(256 * unitKiB), - HAMTSizeEstimation: SizeEstimationBlock, - HAMTShardWidth: 256, - PBNodeFieldOrder: mdag.PBNodeDataFirst, + PBNodeFieldOrder: mdag.PBNodeLinksFirst, // canonical order, pinned } ) diff --git a/ipld/unixfs/io/profile_test.go b/ipld/unixfs/io/profile_test.go index e209a64f6..f1c844f04 100644 --- a/ipld/unixfs/io/profile_test.go +++ b/ipld/unixfs/io/profile_test.go @@ -59,18 +59,9 @@ func TestUnixFSProfiles(t *testing.T) { assert.Equal(t, 256, UnixFS_v1_2025.HAMTShardWidth, "HAMTShardWidth should be 256") }) - t.Run("UnixFS_v1_2026 has correct values", func(t *testing.T) { - assert.Equal(t, 1, UnixFS_v1_2026.CIDVersion, "CIDVersion should be 1") - assert.Equal(t, uint64(mh.SHA2_256), UnixFS_v1_2026.MhType, "MhType should be SHA2_256") - assert.Equal(t, int64(1024*1024), UnixFS_v1_2026.ChunkSize, "ChunkSize should be 1 MiB") - assert.Equal(t, 1024, UnixFS_v1_2026.FileDAGWidth, "FileDAGWidth should be 1024") - assert.True(t, UnixFS_v1_2026.RawLeaves, "RawLeaves should be true for CIDv1") - assert.Equal(t, 256*1024, UnixFS_v1_2026.HAMTShardingSize, "HAMTShardingSize should be 256 KiB") - assert.Equal(t, SizeEstimationBlock, UnixFS_v1_2026.HAMTSizeEstimation, "should use block-based estimation") - assert.Equal(t, 256, UnixFS_v1_2026.HAMTShardWidth, "HAMTShardWidth should be 256") - assert.Equal(t, mdag.PBNodeDataFirst, UnixFS_v1_2026.PBNodeFieldOrder, "PBNodeFieldOrder should be data-first") - assert.Equal(t, mdag.PBNodeLinksFirst, UnixFS_v1_2025.PBNodeFieldOrder, "UnixFS_v1_2025 should keep links-first") - assert.Equal(t, mdag.PBNodeLinksFirst, UnixFS_v0_2015.PBNodeFieldOrder, "UnixFS_v0_2015 should keep links-first") + t.Run("profiles pin canonical links-first order", func(t *testing.T) { + assert.Equal(t, mdag.PBNodeLinksFirst, UnixFS_v1_2025.PBNodeFieldOrder, "UnixFS_v1_2025 must keep links-first") + assert.Equal(t, mdag.PBNodeLinksFirst, UnixFS_v0_2015.PBNodeFieldOrder, "UnixFS_v0_2015 must keep links-first") }) t.Run("CidBuilder returns correct prefix", func(t *testing.T) { @@ -94,7 +85,9 @@ func TestUnixFSProfiles(t *testing.T) { t.Run("ApplyGlobals sets global variables", func(t *testing.T) { saveAndRestoreGlobals(t) - for _, p := range []UnixFSProfile{UnixFS_v1_2025, UnixFS_v1_2026, UnixFS_v0_2015} { + dataFirst := UnixFS_v1_2025 + dataFirst.PBNodeFieldOrder = mdag.PBNodeDataFirst + for _, p := range []UnixFSProfile{UnixFS_v1_2025, dataFirst, UnixFS_v0_2015} { p.ApplyGlobals() assert.Equal(t, p.ChunkSize, chunk.DefaultBlockSize) @@ -131,8 +124,9 @@ func saveAndRestoreGlobals(t *testing.T) { // TestProfilePBNodeFieldOrderFixtures verifies the directory fixtures from // IPIP-550 (https://github.com/ipfs/specs/pull/550): the same directory -// containing hello.txt yields the legacy links-first encoding under -// UnixFS_v1_2025 and the data-first encoding under UnixFS_v1_2026. +// containing hello.txt yields the canonical links-first encoding under +// UnixFS_v1_2025 and the data-first encoding when the PBNodeFieldOrder +// knob is set on top of it. func TestProfilePBNodeFieldOrderFixtures(t *testing.T) { const ( leafCid = "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am" @@ -142,6 +136,9 @@ func TestProfilePBNodeFieldOrderFixtures(t *testing.T) { dataFirstCid = "bafybeigqvyloizmfcdy6scaxnyltftzptaruqa3hnnplfzsbf4sqteiwlm" ) + dataFirst := UnixFS_v1_2025 + dataFirst.PBNodeFieldOrder = mdag.PBNodeDataFirst + cases := []struct { name string profile UnixFSProfile @@ -149,7 +146,7 @@ func TestProfilePBNodeFieldOrderFixtures(t *testing.T) { expectedCid string }{ {"UnixFS_v1_2025 writes links first", UnixFS_v1_2025, linksFirstHex, linksFirstCid}, - {"UnixFS_v1_2026 writes data first", UnixFS_v1_2026, dataFirstHex, dataFirstCid}, + {"data-first knob writes data first", dataFirst, dataFirstHex, dataFirstCid}, } for _, tc := range cases { @@ -183,11 +180,11 @@ func TestProfilePBNodeFieldOrderFixtures(t *testing.T) { } // TestProfilePBNodeFieldOrderEndToEnd builds a multi-chunk file and a sharded -// directory under UnixFS_v1_2025 and UnixFS_v1_2026 from fixed inputs and -// pins the resulting root CIDs. The file inputs and expected CIDs match what -// `ipfs add --chunker=size-1000` produces under each profile. Every dag-pb -// block of both DAGs must lead with the profile's first field, and the two -// DAGs must decode to the same nodes. +// directory from fixed inputs under UnixFS_v1_2025 and under the same profile +// with the data-first knob set, and pins the resulting root CIDs. The file +// inputs and expected CIDs match what `ipfs add --chunker=size-1000` produces +// under each configuration. Every dag-pb block of both DAGs must lead with +// the configured first field, and the two DAGs must decode to the same nodes. func TestProfilePBNodeFieldOrderEndToEnd(t *testing.T) { const ( fileSize = 3000 @@ -195,6 +192,9 @@ func TestProfilePBNodeFieldOrderEndToEnd(t *testing.T) { hamtEntries = 64 ) + dataFirst := UnixFS_v1_2025 + dataFirst.PBNodeFieldOrder = mdag.PBNodeDataFirst + cases := []struct { name string profile UnixFSProfile @@ -208,7 +208,7 @@ func TestProfilePBNodeFieldOrderEndToEnd(t *testing.T) { "bafybeiahnfucbarnualj2uzakbqnj3b3nvrszwu4jrl2x7ekbfqshpd524", }, { - "UnixFS_v1_2026", UnixFS_v1_2026, 0x0a, + "UnixFS_v1_2025+data-first", dataFirst, 0x0a, "bafybeigwq5lxlau4ced4hlvjxuz4xtdbs4zz32dgndqpcc2ggcbiei6k2y", "bafybeihfyecdexcpn3g3xqgzz23pexyespuldzfoh3tvjris56cxkropau", }, @@ -294,8 +294,8 @@ func TestProfilePBNodeFieldOrderEndToEnd(t *testing.T) { for _, fp := range fingerprints { slices.Sort(fp) } - assert.Equal(t, fingerprints["UnixFS_v1_2025"], fingerprints["UnixFS_v1_2026"], - "both profiles must produce the same logical nodes") + assert.Equal(t, fingerprints["UnixFS_v1_2025"], fingerprints["UnixFS_v1_2025+data-first"], + "both orders must produce the same logical nodes") } func TestProfileHAMTThresholdBehavior(t *testing.T) {