From a38febd34c9754a967f76f215e48f30a78875a15 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 22 May 2026 14:01:53 +0100 Subject: [PATCH] feat: port piece library --- go.mod | 5 +- go.sum | 8 ++- piece/piece.go | 118 ++++++++++++++++++++++++++++++++++++++++++++ piece/piece_test.go | 111 +++++++++++++++++++++++++++++++++++++++++ piece/size.go | 58 ++++++++++++++++++++++ 5 files changed, 297 insertions(+), 3 deletions(-) create mode 100644 piece/piece.go create mode 100644 piece/piece_test.go create mode 100644 piece/size.go diff --git a/go.mod b/go.mod index 44741d9..e849221 100644 --- a/go.mod +++ b/go.mod @@ -7,11 +7,15 @@ require ( github.com/fil-forge/automobile v0.0.1 github.com/fil-forge/ucantone v0.0.0-20260521210642-84d8c533075b github.com/filecoin-project/go-data-segment v0.0.1 + github.com/filecoin-project/go-fil-commcid v0.3.1 + github.com/filecoin-project/go-fil-commp-hashhash v0.2.0 github.com/gobwas/glob v0.2.3 github.com/ipfs/go-cid v0.6.1 github.com/ipfs/go-log/v2 v2.9.1 github.com/multiformats/go-multibase v0.3.0 + github.com/multiformats/go-multicodec v0.10.0 github.com/multiformats/go-multihash v0.2.3 + github.com/multiformats/go-varint v0.1.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/stretchr/testify v1.11.1 github.com/whyrusleeping/cbor-gen v0.3.1 @@ -27,7 +31,6 @@ require ( github.com/mr-tron/base58 v1.3.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/multiformats/go-varint v0.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b // indirect diff --git a/go.sum b/go.sum index 2d6326c..e740baf 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,10 @@ github.com/fil-forge/ucantone v0.0.0-20260521210642-84d8c533075b h1:ILG7dtSWiOO/ github.com/fil-forge/ucantone v0.0.0-20260521210642-84d8c533075b/go.mod h1:XAVqsZwYoZ9vncjZoRUAJ+mL/ApLMFn9HHX7ipohVdY= github.com/filecoin-project/go-data-segment v0.0.1 h1:1wmDxOG4ubWQm3ZC1XI5nCon5qgSq7Ra3Rb6Dbu10Gs= github.com/filecoin-project/go-data-segment v0.0.1/go.mod h1:H0/NKbsRxmRFBcLibmABv+yFNHdmtl5AyplYLnb0Zv4= -github.com/filecoin-project/go-fil-commcid v0.2.0 h1:B+5UX8XGgdg/XsdUpST4pEBviKkFOw+Fvl2bLhSKGpI= -github.com/filecoin-project/go-fil-commcid v0.2.0/go.mod h1:8yigf3JDIil+/WpqR5zoKyP0jBPCOGtEqq/K1CcMy9Q= +github.com/filecoin-project/go-fil-commcid v0.3.1 h1:4EfxpHSlvtkOqa9weG2Yt5kxFmPib2xU7Uc9Lbqk7fs= +github.com/filecoin-project/go-fil-commcid v0.3.1/go.mod h1:z7Ssf8d7kspF9QRAVHDbZ+43JK4mkhbGH5lyph1TnKY= +github.com/filecoin-project/go-fil-commp-hashhash v0.2.0 h1:HYIUugzjq78YvV3vC6rL95+SfC/aSTVSnZSZiDV5pCk= +github.com/filecoin-project/go-fil-commp-hashhash v0.2.0/go.mod h1:VH3fAFOru4yyWar4626IoS5+VGE8SfZiBODJLUigEo4= github.com/filecoin-project/go-state-types v0.18.0 h1:oDcjihXRlf2cM176atZzllp79Zc+kcbiuQM9DPL/1a4= github.com/filecoin-project/go-state-types v0.18.0/go.mod h1:CcyG4ZQRDWW+QUY2WDf1KtVDRN7W4twjsfgnGbQfJVI= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= @@ -38,6 +40,8 @@ github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9 github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= github.com/multiformats/go-multibase v0.3.0 h1:8helZD2+4Db7NNWFiktk2NePbF0boolBe6bDQvM4r68= github.com/multiformats/go-multibase v0.3.0/go.mod h1:MoBLQPCkRTOL3eveIPO81860j2AQY8JwcnNlRkGRUfI= +github.com/multiformats/go-multicodec v0.10.0 h1:UpP223cig/Cx8J76jWt91njpK3GTAO1w02sdcjZDSuc= +github.com/multiformats/go-multicodec v0.10.0/go.mod h1:wg88pM+s2kZJEQfRCKBNU+g32F5aWBEjyFHXvZLTcLI= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-varint v0.1.0 h1:i2wqFp4sdl3IcIxfAonHQV9qU5OsZ4Ts9IOoETFs5dI= diff --git a/piece/piece.go b/piece/piece.go new file mode 100644 index 0000000..fed7e06 --- /dev/null +++ b/piece/piece.go @@ -0,0 +1,118 @@ +package piece + +import ( + "fmt" + + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multicodec" + "github.com/multiformats/go-multihash" + "github.com/multiformats/go-varint" +) + +// fr32Code is the multihash code for FR32_SHA256_TRUNC254_PADDED_BINARY_TREE +// (FRC-0069). Aliased for use in this package. +const fr32Code = uint64(multicodec.Fr32Sha256Trunc254Padbintree) + +// Piece is a Filecoin commP piece reference. Its on-wire form is a +// CIDv1 (Raw codec) wrapping an FR32 multihash whose payload is: +// +// varint(padding) || height (1 byte) || commitment (32 bytes) +type Piece struct { + padding uint64 + height uint8 + commitment [32]byte +} + +func (p Piece) Padding() uint64 { return p.padding } +func (p Piece) Height() uint8 { return p.height } + +func (p Piece) DataCommitment() []byte { + out := make([]byte, 32) + copy(out, p.commitment[:]) + return out +} + +// PaddedSize is the FR32-padded tree size in bytes (32 << height). +func (p Piece) PaddedSize() uint64 { return HeightToPaddedSize(p.height) } + +// Multihash builds the FR32 multihash bytes for this piece. +func (p Piece) Multihash() multihash.Multihash { + paddingSize := varint.UvarintSize(p.padding) + digestSize := 32 + 1 + paddingSize + + buf := make( + []byte, + varint.UvarintSize(fr32Code)+varint.UvarintSize(uint64(digestSize))+digestSize, + ) + pos := varint.PutUvarint(buf, fr32Code) + pos += varint.PutUvarint(buf[pos:], uint64(digestSize)) + pos += varint.PutUvarint(buf[pos:], p.padding) + buf[pos] = p.height + pos++ + copy(buf[pos:], p.commitment[:]) + return buf +} + +// CID returns the v2 piece CID: CIDv1 with Raw codec wrapping the FR32 +// multihash. Bit-identical to libstoracha's PieceLink.Link(). +func (p Piece) CID() cid.Cid { return MultihashToCommpCID(p.Multihash()) } + +// FromCID parses a v2 piece CID into a Piece. The CID must use the Raw +// codec and an FR32 multihash. +func FromCID(c cid.Cid) (Piece, error) { + if c.Prefix().Codec != uint64(multicodec.Raw) { + return Piece{}, fmt.Errorf("piece CID must use raw codec, got 0x%x", c.Prefix().Codec) + } + return FromMultihash(c.Hash()) +} + +// FromMultihash parses an FR32 multihash into a Piece. +func FromMultihash(mh multihash.Multihash) (Piece, error) { + decoded, err := multihash.Decode(mh) + if err != nil { + return Piece{}, fmt.Errorf("decoding multihash: %w", err) + } + if decoded.Code != fr32Code { + return Piece{}, fmt.Errorf("multihash code must be 0x%x, got 0x%x", fr32Code, decoded.Code) + } + + padding, n, err := varint.FromUvarint(decoded.Digest) + if err != nil { + return Piece{}, fmt.Errorf("reading padding varint: %w", err) + } + if len(decoded.Digest) < n+1+32 { + return Piece{}, fmt.Errorf("digest too short: %d bytes", len(decoded.Digest)) + } + height := decoded.Digest[n] + var commitment [32]byte + copy(commitment[:], decoded.Digest[n+1:n+1+32]) + + return Piece{padding: padding, height: height, commitment: commitment}, nil +} + +// FromCommitmentAndSize builds a Piece from a raw 32-byte data +// commitment and an unpadded input size in bytes. +func FromCommitmentAndSize(commD []byte, unpaddedDataSize uint64) (Piece, error) { + if len(commD) != 32 { + return Piece{}, fmt.Errorf("commitments must be 32 bytes long, got %d", len(commD)) + } + if unpaddedDataSize < 127 { + return Piece{}, fmt.Errorf("unpadded data size must be at least 127, got %d", unpaddedDataSize) + } + + height, padding, err := UnpaddedSizeToV1TreeHeightAndPadding(unpaddedDataSize) + if err != nil { + return Piece{}, err + } + if padding > varint.MaxValueUvarint63 { + return Piece{}, fmt.Errorf("padding must be less than 2^63-1, got %d", padding) + } + + var commitment [32]byte + copy(commitment[:], commD) + return Piece{padding: padding, height: height, commitment: commitment}, nil +} + +func MultihashToCommpCID(mh multihash.Multihash) cid.Cid { + return cid.NewCidV1(cid.Raw, mh) +} diff --git a/piece/piece_test.go b/piece/piece_test.go new file mode 100644 index 0000000..9447541 --- /dev/null +++ b/piece/piece_test.go @@ -0,0 +1,111 @@ +package piece_test + +import ( + "bytes" + "io" + "math/rand" + "testing" + + "github.com/fil-forge/libforge/piece" + commcid "github.com/filecoin-project/go-fil-commcid" + commp "github.com/filecoin-project/go-fil-commp-hashhash" + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multicodec" + "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/require" +) + +// fixedCommP builds a deterministic 10KiB commP for use as a stable +// fixture. Returns the data commitment + the unpadded input size. +func fixedCommP(t *testing.T, seed int64, size int) (commD []byte, unpadded uint64) { + t.Helper() + r := rand.New(rand.NewSource(seed)) + data := make([]byte, size) + _, err := io.ReadFull(r, data) + require.NoError(t, err) + + c := &commp.Calc{} + _, err = io.Copy(c, bytes.NewReader(data)) + require.NoError(t, err) + commD, _, err = c.Digest() + require.NoError(t, err) + return commD, uint64(size) +} + +func TestPieceRoundTrip(t *testing.T) { + commD, unpadded := fixedCommP(t, 1, 10*1024) + + p, err := piece.FromCommitmentAndSize(commD, unpadded) + require.NoError(t, err) + require.Equal(t, commD, p.DataCommitment()) + + c := p.CID() + require.Equal(t, uint64(multicodec.Raw), c.Prefix().Codec) + require.Equal(t, uint64(multicodec.Fr32Sha256Trunc254Padbintree), c.Prefix().MhType) + + got, err := piece.FromCID(c) + require.NoError(t, err) + require.Equal(t, p.Padding(), got.Padding()) + require.Equal(t, p.Height(), got.Height()) + require.Equal(t, p.PaddedSize(), got.PaddedSize()) + require.Equal(t, p.DataCommitment(), got.DataCommitment()) +} + +// TestPieceWireCompat verifies the piri-internal Piece produces a CID +// byte-identical to the canonical go-fil-commcid implementation. This +// is the wire-compat guarantee: any existing v2 piece CID written by +// libstoracha (or by another commcid-based actor) decodes the same +// way through our type. +func TestPieceWireCompat(t *testing.T) { + cases := []struct { + name string + seed int64 + size int + }{ + {"10KiB", 1, 10 * 1024}, + {"127B", 2, 127}, + {"1MiB", 3, 1 << 20}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + commD, unpadded := fixedCommP(t, tc.seed, tc.size) + + oracleCID, err := commcid.DataCommitmentToPieceCidv2(commD, unpadded) + require.NoError(t, err) + + p, err := piece.FromCommitmentAndSize(commD, unpadded) + require.NoError(t, err) + require.Equal(t, oracleCID.String(), p.CID().String(), "wire format diverges from go-fil-commcid") + require.Equal(t, oracleCID.Bytes(), p.CID().Bytes()) + require.Equal(t, []byte(oracleCID.Hash()), []byte(p.Multihash())) + }) + } +} + +func TestFromCIDRejectsWrongCodec(t *testing.T) { + commD, unpadded := fixedCommP(t, 4, 10*1024) + p, err := piece.FromCommitmentAndSize(commD, unpadded) + require.NoError(t, err) + + bad := cid.NewCidV1(uint64(multicodec.DagCbor), p.Multihash()) + _, err = piece.FromCID(bad) + require.Error(t, err) +} + +func TestFromMultihashRejectsWrongCode(t *testing.T) { + mh, err := multihash.Sum([]byte("not a piece"), multihash.SHA2_256, -1) + require.NoError(t, err) + _, err = piece.FromMultihash(mh) + require.Error(t, err) +} + +func TestSizeHelpers(t *testing.T) { + // Fr32PaddedSizeToV1TreeHeight: sanity boundaries. + require.Equal(t, uint8(0), piece.Fr32PaddedSizeToV1TreeHeight(32)) + require.Equal(t, uint8(1), piece.Fr32PaddedSizeToV1TreeHeight(64)) + require.Equal(t, uint8(2), piece.Fr32PaddedSizeToV1TreeHeight(128)) + require.Equal(t, uint8(2), piece.Fr32PaddedSizeToV1TreeHeight(65)) // rounds up + + require.Equal(t, uint64(32)<<10, piece.HeightToPaddedSize(10)) + require.Equal(t, uint64(127), piece.MaxDataSize(128)) +} diff --git a/piece/size.go b/piece/size.go new file mode 100644 index 0000000..c2e9ed2 --- /dev/null +++ b/piece/size.go @@ -0,0 +1,58 @@ +package piece + +import ( + "fmt" + "math/bits" +) + +// Fr32PaddedSizeToV1TreeHeight returns the binary-tree height for an +// FR32-padded piece of the given size. Sizes that aren't a power of two +// round up under the assumption the tree is padded with zeros. +func Fr32PaddedSizeToV1TreeHeight(size uint64) uint8 { + if size <= 32 { + return 0 + } + + b := 63 - bits.LeadingZeros64(size) + b -= 5 // 2^5 == 32-byte leaves + + if 32<