Migrate go-unixfsnode

go.mod

@@ -6,48 +6,84 @@ require (
 	github.com/benbjohnson/clock v1.3.0
 	github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3
 	github.com/gorilla/mux v1.8.0
+	github.com/ipfs/go-bitfield v1.0.0
 	github.com/ipfs/go-cid v0.3.2
+	github.com/ipfs/go-ipfs-chunker v0.0.5
+	github.com/ipfs/go-ipfs-util v0.0.2
+	github.com/ipfs/go-ipld-format v0.4.0
 	github.com/ipfs/go-ipns v0.3.0
 	github.com/ipfs/go-log/v2 v2.5.1
+	github.com/ipfs/go-merkledag v0.9.0
+	github.com/ipfs/go-unixfs v0.4.1
+	github.com/ipld/go-car/v2 v2.5.1
+	github.com/ipld/go-codec-dagpb v1.5.0
+	github.com/ipld/go-ipld-prime v0.19.0
 	github.com/libp2p/go-libp2p v0.23.4
 	github.com/libp2p/go-libp2p-record v0.2.0
 	github.com/multiformats/go-multiaddr v0.8.0
 	github.com/multiformats/go-multibase v0.1.1
+	github.com/multiformats/go-multicodec v0.6.0
 	github.com/multiformats/go-multihash v0.2.1
 	github.com/samber/lo v1.36.0
+	github.com/spaolacci/murmur3 v1.1.0
 	github.com/stretchr/testify v1.8.1
 	go.opencensus.io v0.23.0
 	golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab
+	google.golang.org/protobuf v1.28.1
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/ipfs/go-ipfs-util v0.0.2 // indirect
-	github.com/ipld/go-ipld-prime v0.9.0 // indirect
-	github.com/klauspost/cpuid/v2 v2.1.1 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/ipfs/bbloom v0.0.4 // indirect
+	github.com/ipfs/go-block-format v0.0.3 // indirect
+	github.com/ipfs/go-blockservice v0.5.0 // indirect
+	github.com/ipfs/go-datastore v0.6.0 // indirect
+	github.com/ipfs/go-ipfs-blockstore v1.2.0 // indirect
+	github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
+	github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect
+	github.com/ipfs/go-ipfs-exchange-offline v0.3.0 // indirect
+	github.com/ipfs/go-ipld-cbor v0.0.6 // indirect
+	github.com/ipfs/go-ipld-legacy v0.1.1 // indirect
+	github.com/ipfs/go-log v1.0.5 // indirect
+	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
+	github.com/ipfs/go-unixfsnode v1.5.1 // indirect
+	github.com/ipfs/go-verifcid v0.0.2 // indirect
+	github.com/jbenet/goprocess v0.1.4 // indirect
+	github.com/klauspost/cpuid/v2 v2.1.2 // indirect
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
 	github.com/libp2p/go-openssl v0.1.0 // indirect
-	github.com/mattn/go-isatty v0.0.16 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/mattn/go-pointer v0.0.1 // indirect
 	github.com/minio/sha256-simd v1.0.0 // indirect
 	github.com/mr-tron/base58 v1.2.0 // indirect
 	github.com/multiformats/go-base32 v0.1.0 // indirect
 	github.com/multiformats/go-base36 v0.1.0 // indirect
-	github.com/multiformats/go-multicodec v0.6.0 // indirect
-	github.com/multiformats/go-varint v0.0.6 // indirect
+	github.com/multiformats/go-varint v0.0.7 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1 // indirect
+	github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect
 	github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
-	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
+	github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
+	github.com/whyrusleeping/cbor-gen v0.0.0-20221220214510-0333c149dec0 // indirect
+	github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
+	go.opentelemetry.io/otel v1.7.0 // indirect
+	go.opentelemetry.io/otel/trace v1.7.0 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
 	go.uber.org/zap v1.23.0 // indirect
 	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
 	golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b // indirect
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	lukechampine.com/blake3 v1.1.7 // indirect
 )

unixfs/node/README.md

@@ -0,0 +1,20 @@
+# go-unixfsnode
+
+This is an IPLD ADL that provides string based pathing for protobuf nodes. The top level node behaves like a map where LookupByString returns the Hash property on the Link in the protobufs list of Links whos Name property matches the key. This should enable selector traversals that work based of paths.
+
+Note that while it works internally with go-codec-dagpb, the Reify method (used to get a UnixFSNode from a DagPB node should actually work successfully with go-ipld-prime-proto nodes)
+
+## Usage
+
+The primary interaction with this package is to register an ADL on a link system. This is done with via a helper method.
+
+```go
+AddUnixFSReificationToLinkSystem(lsys *ipld.LinkSystem)
+```
+
+For link systems which have UnixFS reification registered, two ADLs will be available to the [`InterpretAs`](https://ipld.io/specs/selectors/) selector: 'unixfs' and 'unixfs-preload'. The different between these two ADLs is that the preload variant will access all blocks within a UnixFS Object (file or directory) when that object is accessed by a selector traversal. The non-preload variant in contrast will only access the subset of blocks strictly needed for the traversal. In practice, this means the subset of a sharded directory needed to access a specific file, or the sub-range of a file directly accessed by a range selector.
+
+
+## License
+
+Apache-2.0/MIT © Protocol Labs

unixfs/node/data/builder/builder.go

@@ -0,0 +1,142 @@
+package builder
+
+import (
+	"errors"
+	"strconv"
+	"time"
+
+	"github.com/ipfs/go-libipfs/unixfs/node/data"
+	"github.com/ipld/go-ipld-prime"
+	"github.com/ipld/go-ipld-prime/fluent/qp"
+)
+
+// BuildUnixFS provides a clean, validated interface to building data structures
+// that match the UnixFS protobuf encoded in the Data member of a ProtoNode
+// with sensible defaults
+//
+//	  smallFileData, err := BuildUnixFS(func(b *Builder) {
+//	     Data(b, []byte{"hello world"})
+//	     Mtime(b, func(tb TimeBuilder) {
+//					Time(tb, time.Now())
+//				})
+//	  })
+func BuildUnixFS(fn func(*Builder)) (data.UnixFSData, error) {
+	nd, err := qp.BuildMap(data.Type.UnixFSData, -1, func(ma ipld.MapAssembler) {
+		b := &Builder{MapAssembler: ma}
+		fn(b)
+		if !b.hasBlockSizes {
+			qp.MapEntry(ma, data.Field__BlockSizes, qp.List(0, func(ipld.ListAssembler) {}))
+		}
+		if !b.hasDataType {
+			qp.MapEntry(ma, data.Field__DataType, qp.Int(data.Data_File))
+		}
+	})
+	if err != nil {
+		return nil, err
+	}
+	return nd.(data.UnixFSData), nil
+}
+
+// Builder is an interface for making UnixFS data nodes
+type Builder struct {
+	ipld.MapAssembler
+	hasDataType   bool
+	hasBlockSizes bool
+}
+
+// DataType sets the default on a builder for a UnixFS node - default is File
+func DataType(b *Builder, dataType int64) {
+	_, ok := data.DataTypeNames[dataType]
+	if !ok {
+		panic(data.ErrInvalidDataType{DataType: dataType})
+	}
+	qp.MapEntry(b.MapAssembler, data.Field__DataType, qp.Int(dataType))
+	b.hasDataType = true
+}
+
+// Data sets the data member inside the UnixFS data
+func Data(b *Builder, dataBytes []byte) {
+	qp.MapEntry(b.MapAssembler, data.Field__Data, qp.Bytes(dataBytes))
+}
+
+// FileSize sets the file size which should be the size of actual bytes underneath
+// this node for large files, w/o additional bytes to encode intermediate nodes
+func FileSize(b *Builder, fileSize uint64) {
+	qp.MapEntry(b.MapAssembler, data.Field__FileSize, qp.Int(int64(fileSize)))
+}
+
+// BlockSizes encodes block sizes for each child node
+func BlockSizes(b *Builder, blockSizes []uint64) {
+	qp.MapEntry(b.MapAssembler, data.Field__BlockSizes, qp.List(int64(len(blockSizes)), func(la ipld.ListAssembler) {
+		for _, bs := range blockSizes {
+			qp.ListEntry(la, qp.Int(int64(bs)))
+		}
+	}))
+	b.hasBlockSizes = true
+}
+
+// HashType sets the hash function for this node -- only applicable to HAMT
+func HashType(b *Builder, hashType uint64) {
+	qp.MapEntry(b.MapAssembler, data.Field__HashType, qp.Int(int64(hashType)))
+}
+
+// Fanout sets the fanout in a HAMT tree
+func Fanout(b *Builder, fanout uint64) {
+	qp.MapEntry(b.MapAssembler, data.Field__Fanout, qp.Int(int64(fanout)))
+}
+
+// Permissions sets file permissions for the Mode member of the UnixFS node
+func Permissions(b *Builder, mode int) {
+	mode = mode & 0xFFF
+	qp.MapEntry(b.MapAssembler, data.Field__Mode, qp.Int(int64(mode)))
+}
+
+func parseModeString(modeString string) (uint64, error) {
+	if len(modeString) > 0 && modeString[0] == '0' {
+		return strconv.ParseUint(modeString, 8, 32)
+	}
+	return strconv.ParseUint(modeString, 10, 32)
+}
+
+// PermissionsString sets file permissions for the Mode member of the UnixFS node,
+// parsed from a typical octect encoded permission string (eg '0755')
+func PermissionsString(b *Builder, modeString string) {
+	mode64, err := parseModeString(modeString)
+	if err != nil {
+		panic(err)
+	}
+	mode64 = mode64 & 0xFFF
+	qp.MapEntry(b.MapAssembler, data.Field__Mode, qp.Int(int64(mode64)))
+}
+
+// Mtime sets the modification time for this node using the time builder interface
+// and associated methods
+func Mtime(b *Builder, fn func(tb TimeBuilder)) {
+	qp.MapEntry(b.MapAssembler, data.Field__Mtime, qp.Map(-1, func(ma ipld.MapAssembler) {
+		fn(ma)
+	}))
+}
+
+// TimeBuilder is a simple interface for constructing the time member of UnixFS data
+type TimeBuilder ipld.MapAssembler
+
+// Time sets the modification time from a golang time value
+func Time(ma TimeBuilder, t time.Time) {
+	Seconds(ma, t.Unix())
+	FractionalNanoseconds(ma, int32(t.Nanosecond()))
+}
+
+// Seconds sets the seconds for a modification time
+func Seconds(ma TimeBuilder, seconds int64) {
+	qp.MapEntry(ma, data.Field__Seconds, qp.Int(seconds))
+
+}
+
+// FractionalNanoseconds sets the nanoseconds for a modification time (must
+// be between 0 & a billion)
+func FractionalNanoseconds(ma TimeBuilder, nanoseconds int32) {
+	if nanoseconds < 0 || nanoseconds > 999999999 {
+		panic(errors.New("mtime-nsecs must be within the range [0,999999999]"))
+	}
+	qp.MapEntry(ma, data.Field__Nanoseconds, qp.Int(int64(nanoseconds)))
+}