From eccd3d54e2fa19c770020538621306ad11db05c7 Mon Sep 17 00:00:00 2001 From: Andrew Gillis Date: Thu, 31 Aug 2023 14:00:06 -0700 Subject: [PATCH] Implement ipni-sync http over libp2p (#113) * Implement ipni-sync http over libp2p Use the new libp2phttp functionality for serving and requesting ipnisync over libp2p. * Move libp2phttp functionality inside ipnisync * Subscriber does not use existing http client * Add HttpTimeout option for subscriber * Add option to use retryable http client * Change option name from `WithServer` to `WithStartServer` * Sync gets addrs from peerstore if none supplied * If server is not libp2phttp, then use address to choose plain HTTP or dtsync. * If publisher HTTP is not available at IPNI path, then retry without IPNI path. This supports legacy HTTP served without IPNI path. * ignore emtpy http listen addrs * Update log messages * Log peer.ID consistently as peer * Convert most tests to use ipnisync publisher * Change AsyncErr to Err in SyncFinished * Move old p2p head client/server (legs protocol ID) into dtsync, since that is the only place it is used. * Add tests for libp2phttp * gostream package relocated * update graphsync * Use IPNIPath for libp2p protocol ID --- announce/httpsender/sender.go | 8 +- dagsync/announce_test.go | 55 ++- dagsync/{p2p/protocol => dtsync}/head/head.go | 6 +- .../protocol => dtsync}/head/head_test.go | 2 +- dagsync/dtsync/publisher.go | 2 +- dagsync/dtsync/syncer.go | 2 +- dagsync/example_test.go | 13 +- dagsync/http_test.go | 2 +- dagsync/ipnisync/ipnipath.go | 16 +- dagsync/ipnisync/option.go | 116 +++++- dagsync/ipnisync/publisher.go | 205 ++++++---- dagsync/ipnisync/publisher_test.go | 365 +++++++++++++++++- dagsync/ipnisync/sync.go | 231 ++++++++--- dagsync/ipnisync/sync_test.go | 28 +- dagsync/option.go | 52 ++- dagsync/subscriber.go | 57 ++- dagsync/subscriber_test.go | 42 +- dagsync/sync_test.go | 136 +++++-- dagsync/test/util.go | 12 +- go.mod | 58 +-- go.sum | 143 ++++--- 21 files changed, 1159 insertions(+), 392 deletions(-) rename dagsync/{p2p/protocol => dtsync}/head/head.go (96%) rename dagsync/{p2p/protocol => dtsync}/head/head_test.go (98%) diff --git a/announce/httpsender/sender.go b/announce/httpsender/sender.go index 14663ed..0834006 100644 --- a/announce/httpsender/sender.go +++ b/announce/httpsender/sender.go @@ -27,10 +27,10 @@ type Sender struct { userAgent string } -// New creates a new Sender that sends announce messages over HTTP. Announce -// messages are sent to the specified URLs. The addresses in announce messages -// are modified to include the specified peerID, which is necessary to -// communicate the publisher ID over HTTP. +// New creates a new Sender that sends advertisement announcement messages over +// HTTP. Announcements are sent directly to the specified URLs. The specified +// peerID is added to the multiaddrs contained in the announcements, which is +// how the publisher ID is communicated over HTTP. func New(announceURLs []*url.URL, peerID peer.ID, options ...Option) (*Sender, error) { if len(announceURLs) == 0 { return nil, errors.New("no announce urls") diff --git a/dagsync/announce_test.go b/dagsync/announce_test.go index e3b6de3..b262be5 100644 --- a/dagsync/announce_test.go +++ b/dagsync/announce_test.go @@ -27,36 +27,29 @@ import ( func TestAnnounceReplace(t *testing.T) { t.Parallel() - srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) dstStore := dssync.MutexWrap(datastore.NewMapDatastore()) - srcHost := test.MkTestHost(t) - srcHostInfo := peer.AddrInfo{ - ID: srcHost.ID(), - Addrs: srcHost.Addrs(), - } - srcLnkS := test.MkLinkSystem(srcStore) dstHost := test.MkTestHost(t) - - srcHost.Peerstore().AddAddrs(dstHost.ID(), dstHost.Addrs(), time.Hour) - dstHost.Peerstore().AddAddrs(srcHost.ID(), srcHost.Addrs(), time.Hour) - //dstLnkS := test.MkLinkSystem(dstStore) - dstLnkS, blocked := test.MkBlockedLinkSystem(dstStore) blocksSeenByHook := make(map[cid.Cid]struct{}) blockHook := func(p peer.ID, c cid.Cid, _ dagsync.SegmentSyncActions) { blocksSeenByHook[c] = struct{}{} } - pub, err := dtsync.NewPublisher(srcHost, srcStore, srcLnkS, testTopic) - require.NoError(t, err) - defer pub.Close() - sub, err := dagsync.NewSubscriber(dstHost, dstStore, dstLnkS, testTopic, dagsync.RecvAnnounce(), dagsync.BlockHook(blockHook)) require.NoError(t, err) defer sub.Close() - require.NoError(t, test.WaitForP2PPublisher(pub, dstHost, testTopic)) + srcHost, srcPrivKey := test.MkTestHostPK(t) + srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) + srcLnkS := test.MkLinkSystem(srcStore) + + pub, err := ipnisync.NewPublisher(srcLnkS, srcPrivKey, ipnisync.WithStreamHost(srcHost), ipnisync.WithHeadTopic(testTopic)) + require.NoError(t, err) + defer pub.Close() + + srcHost.Peerstore().AddAddrs(dstHost.ID(), dstHost.Addrs(), time.Hour) + dstHost.Peerstore().AddAddrs(srcHost.ID(), srcHost.Addrs(), time.Hour) watcher, cncl := sub.OnSyncFinished() defer cncl() @@ -67,6 +60,11 @@ func TestAnnounceReplace(t *testing.T) { firstCid := chainLnks[2].(cidlink.Link).Cid pub.SetRoot(firstCid) + srcHostInfo := peer.AddrInfo{ + ID: srcHost.ID(), + Addrs: srcHost.Addrs(), + } + // Have the subscriber receive an announce. This is the same as if it was // published by the publisher without having to wait for it to arrive. err = sub.Announce(context.Background(), firstCid, srcHostInfo) @@ -154,7 +152,7 @@ func TestAnnounce_LearnsHttpPublisherAddr(t *testing.T) { pubh := test.MkTestHost(t) pubds := dssync.MutexWrap(datastore.NewMapDatastore()) publs := test.MkLinkSystem(pubds) - pub, err := ipnisync.NewPublisher("0.0.0.0:0", publs, pubh.Peerstore().PrivKey(pubh.ID())) + pub, err := ipnisync.NewPublisher(publs, pubh.Peerstore().PrivKey(pubh.ID()), ipnisync.WithHTTPListenAddrs("0.0.0.0:0")) require.NoError(t, err) defer pub.Close() @@ -214,11 +212,7 @@ func TestAnnounce_LearnsHttpPublisherAddr(t *testing.T) { func TestAnnounceRepublish(t *testing.T) { srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) dstStore := dssync.MutexWrap(datastore.NewMapDatastore()) - srcHost := test.MkTestHost(t) - srcHostInfo := peer.AddrInfo{ - ID: srcHost.ID(), - Addrs: srcHost.Addrs(), - } + srcHost, srcPrivKey := test.MkTestHostPK(t) srcLnkS := test.MkLinkSystem(srcStore) dstHost := test.MkTestHost(t) @@ -243,10 +237,9 @@ func TestAnnounceRepublish(t *testing.T) { require.NoError(t, err) defer sub1.Close() - pub, err := dtsync.NewPublisher(srcHost, srcStore, srcLnkS, testTopic) + pub, err := ipnisync.NewPublisher(srcLnkS, srcPrivKey, ipnisync.WithStreamHost(srcHost), ipnisync.WithHeadTopic(testTopic)) require.NoError(t, err) defer pub.Close() - require.NoError(t, test.WaitForP2PPublisher(pub, dstHost, testTopic)) watcher2, cncl := sub2.OnSyncFinished() defer cncl() @@ -258,7 +251,11 @@ func TestAnnounceRepublish(t *testing.T) { pub.SetRoot(firstCid) // Announce one CID to subscriber1. - err = sub1.Announce(context.Background(), firstCid, srcHostInfo) + pubInfo := peer.AddrInfo{ + ID: pub.ID(), + Addrs: pub.Addrs(), + } + err = sub1.Announce(context.Background(), firstCid, pubInfo) require.NoError(t, err) t.Log("Sent announce for first CID", firstCid) @@ -444,7 +441,7 @@ func mkLnk(t *testing.T, srcStore datastore.Batching) cid.Cid { } func initPubSub(t *testing.T, srcStore, dstStore datastore.Batching, allowPeer func(peer.ID) bool) (host.Host, host.Host, dagsync.Publisher, *dagsync.Subscriber, announce.Sender) { - srcHost := test.MkTestHost(t) + srcHost, srcPrivKey := test.MkTestHostPK(t) dstHost := test.MkTestHost(t) topics := test.WaitForMeshWithMessage(t, testTopic, srcHost, dstHost) @@ -453,7 +450,7 @@ func initPubSub(t *testing.T, srcStore, dstStore datastore.Batching, allowPeer f p2pSender, err := p2psender.New(nil, "", p2psender.WithTopic(topics[0]), p2psender.WithExtraData([]byte("t01000"))) require.NoError(t, err) - pub, err := dtsync.NewPublisher(srcHost, srcStore, srcLnkS, testTopic) + pub, err := ipnisync.NewPublisher(srcLnkS, srcPrivKey, ipnisync.WithStreamHost(srcHost), ipnisync.WithHeadTopic(testTopic)) require.NoError(t, err) srcHost.Peerstore().AddAddrs(dstHost.ID(), dstHost.Addrs(), time.Hour) @@ -467,7 +464,5 @@ func initPubSub(t *testing.T, srcStore, dstStore datastore.Batching, allowPeer f err = srcHost.Connect(context.Background(), dstHost.Peerstore().PeerInfo(dstHost.ID())) require.NoError(t, err) - require.NoError(t, test.WaitForP2PPublisher(pub, dstHost, testTopic)) - return srcHost, dstHost, pub, sub, p2pSender } diff --git a/dagsync/p2p/protocol/head/head.go b/dagsync/dtsync/head/head.go similarity index 96% rename from dagsync/p2p/protocol/head/head.go rename to dagsync/dtsync/head/head.go index d39690a..f6ec73e 100644 --- a/dagsync/p2p/protocol/head/head.go +++ b/dagsync/dtsync/head/head.go @@ -13,11 +13,11 @@ import ( "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" - gostream "github.com/libp2p/go-libp2p-gostream" "github.com/libp2p/go-libp2p/core/host" - peer "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" - multistream "github.com/multiformats/go-multistream" + "github.com/libp2p/go-libp2p/p2p/net/gostream" + "github.com/multiformats/go-multistream" ) const closeTimeout = 30 * time.Second diff --git a/dagsync/p2p/protocol/head/head_test.go b/dagsync/dtsync/head/head_test.go similarity index 98% rename from dagsync/p2p/protocol/head/head_test.go rename to dagsync/dtsync/head/head_test.go index d136309..6c7acc5 100644 --- a/dagsync/p2p/protocol/head/head_test.go +++ b/dagsync/dtsync/head/head_test.go @@ -12,7 +12,7 @@ import ( _ "github.com/ipld/go-ipld-prime/codec/dagjson" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" - "github.com/ipni/go-libipni/dagsync/p2p/protocol/head" + "github.com/ipni/go-libipni/dagsync/dtsync/head" "github.com/ipni/go-libipni/dagsync/test" "github.com/libp2p/go-libp2p/core/protocol" "github.com/multiformats/go-multiaddr" diff --git a/dagsync/dtsync/publisher.go b/dagsync/dtsync/publisher.go index 4546099..692778a 100644 --- a/dagsync/dtsync/publisher.go +++ b/dagsync/dtsync/publisher.go @@ -11,7 +11,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" "github.com/ipld/go-ipld-prime" - "github.com/ipni/go-libipni/dagsync/p2p/protocol/head" + "github.com/ipni/go-libipni/dagsync/dtsync/head" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" diff --git a/dagsync/dtsync/syncer.go b/dagsync/dtsync/syncer.go index 0b1c1d7..e8a8141 100644 --- a/dagsync/dtsync/syncer.go +++ b/dagsync/dtsync/syncer.go @@ -12,7 +12,7 @@ import ( "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/traversal" "github.com/ipld/go-ipld-prime/traversal/selector" - "github.com/ipni/go-libipni/dagsync/p2p/protocol/head" + "github.com/ipni/go-libipni/dagsync/dtsync/head" "github.com/libp2p/go-libp2p/core/peer" ) diff --git a/dagsync/example_test.go b/dagsync/example_test.go index ced5850..1dfb0db 100644 --- a/dagsync/example_test.go +++ b/dagsync/example_test.go @@ -3,6 +3,7 @@ package dagsync_test import ( "bytes" "context" + "crypto/rand" "fmt" "io" "log" @@ -15,8 +16,9 @@ import ( cidlink "github.com/ipld/go-ipld-prime/linking/cid" basicnode "github.com/ipld/go-ipld-prime/node/basic" "github.com/ipni/go-libipni/dagsync" - "github.com/ipni/go-libipni/dagsync/dtsync" + "github.com/ipni/go-libipni/dagsync/ipnisync" "github.com/libp2p/go-libp2p" + "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/host" "github.com/multiformats/go-multicodec" ) @@ -25,12 +27,13 @@ var srcHost host.Host func ExamplePublisher() { // Init dagsync publisher and subscriber. - srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) - srcHost, _ = libp2p.New() + srcPrivKey, _, _ := crypto.GenerateEd25519Key(rand.Reader) + srcHost, _ = libp2p.New(libp2p.Identity(srcPrivKey)) defer srcHost.Close() + srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) srcLnkS := makeLinkSystem(srcStore) - pub, err := dtsync.NewPublisher(srcHost, srcStore, srcLnkS, testTopic) + pub, err := ipnisync.NewPublisher(srcLnkS, srcPrivKey, ipnisync.WithStreamHost(srcHost), ipnisync.WithHeadTopic("/indexer/ingest/testnet")) if err != nil { panic(err) } @@ -65,7 +68,7 @@ func ExampleSubscriber() { srcHost.Peerstore().AddAddrs(dstHost.ID(), dstHost.Addrs(), time.Hour) dstHost.Peerstore().AddAddrs(srcHost.ID(), srcHost.Addrs(), time.Hour) - sub, err := dagsync.NewSubscriber(dstHost, dstStore, dstLnkSys, "/indexer/ingest/testnet", nil) + sub, err := dagsync.NewSubscriber(dstHost, dstStore, dstLnkSys, "/indexer/ingest/testnet") if err != nil { panic(err) } diff --git a/dagsync/http_test.go b/dagsync/http_test.go index cdf08ba..84a561d 100644 --- a/dagsync/http_test.go +++ b/dagsync/http_test.go @@ -40,7 +40,7 @@ func setupPublisherSubscriber(t *testing.T, subscriberOptions []dagsync.Option) srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) srcLinkSys := test.MkLinkSystem(srcStore) - pub, err := ipnisync.NewPublisher("127.0.0.1:0", srcLinkSys, srcPrivKey, ipnisync.WithServer(true)) + pub, err := ipnisync.NewPublisher(srcLinkSys, srcPrivKey, ipnisync.WithHTTPListenAddrs("127.0.0.1:0")) require.NoError(t, err) t.Cleanup(func() { pub.Close() diff --git a/dagsync/ipnisync/ipnipath.go b/dagsync/ipnisync/ipnipath.go index ac2e7d4..650fcf4 100644 --- a/dagsync/ipnisync/ipnipath.go +++ b/dagsync/ipnisync/ipnipath.go @@ -1,7 +1,15 @@ package ipnisync -import "path" +import "github.com/libp2p/go-libp2p/core/protocol" -const protoVersion = "v1" - -var IpniPath = path.Join("/ipni", protoVersion, "ad") +const ( + // IPNIPath is the path that the Publisher expects as the last port of the + // HTTP request URL path. The sync client automatically adds this to the + // request path. + IPNIPath = "/ipni/v1/ad" + // ProtocolID is the libp2p protocol ID used to identify the ipni-sync + // protocol. With libp2phttp this protocol ID maps directly to a single + // HTTP path, so the value of the protocol ID is the same as the IPNI path + // for the ipni-sync protocol. + ProtocolID = protocol.ID(IPNIPath) +) diff --git a/dagsync/ipnisync/option.go b/dagsync/ipnisync/option.go index 9274ca3..6ba59f2 100644 --- a/dagsync/ipnisync/option.go +++ b/dagsync/ipnisync/option.go @@ -2,13 +2,20 @@ package ipnisync import ( "fmt" + "time" + + "github.com/libp2p/go-libp2p/core/host" ) -// pubConfig contains all options for configuring Publisher. +// config contains all options for configuring Publisher. type config struct { handlerPath string startServer bool topic string + + streamHost host.Host + requireTLS bool + httpAddrs []string } // Option is a function that sets a value in a config. @@ -27,8 +34,28 @@ func getOpts(opts []Option) (config, error) { return cfg, nil } +// WithHTTPListenAddrs sets the HTTP addresses to listen on. These are in +// addresses:port format and may be prefixed with "https://" or "http://" or to +// specify whether or not TLS is required. If there is no prefix, then one is +// assumed based on the value specified by WithRequireTLS. +// +// Setting HTTP listen addresses is optional when a stream host is provided by +// the WithStreamHost option. +func WithHTTPListenAddrs(addrs ...string) Option { + return func(c *config) error { + for _, addr := range addrs { + if addr != "" { + c.httpAddrs = append(c.httpAddrs, addr) + } + } + return nil + } +} + // WithHandlerPath sets the path used to handle requests to this publisher. -// This should only include the path before the /ipni/v1/ad/ part of the path. +// This specifies the portion of the path before the implicit /ipni/v1/ad/ part +// of the path. Calling WithHandlerPath("/foo/bar") configures the publisher to +// handle HTTP requests on the path "/foo/bar/ipni/v1/ad/". func WithHandlerPath(urlPath string) Option { return func(c *config) error { c.handlerPath = urlPath @@ -45,12 +72,87 @@ func WithHeadTopic(topic string) Option { } } -// WithServer, if true, starts an http server listening on the given address. -// an HTTP server. If this option is not specified, then no server is started -// and this will need to be done by the caller. -func WithServer(serve bool) Option { +// WithStartServer, if true, starts an http server listening on the given +// address. an HTTP server. If this option is not specified, then no server is +// started and this will need to be done by the caller. +func WithStartServer(start bool) Option { return func(c *config) error { - c.startServer = serve + c.startServer = start return nil } } + +// WithRequireTLS tells whether to allow the publisher to require https (true) +// or to serve non-secure http (false). Default is false, allowing non-secure +// HTTP. +func WithRequireTLS(require bool) Option { + return func(c *config) error { + c.requireTLS = require + return nil + } +} + +// WithStreamHost specifies an optional stream based libp2p host used to do +// HTTP over libp2p streams. +func WithStreamHost(h host.Host) Option { + return func(c *config) error { + c.streamHost = h + return nil + } +} + +type clientConfig struct { + authPeerID bool + streamHost host.Host + + httpTimeout time.Duration + httpRetryMax int + httpRetryWaitMin time.Duration + httpRetryWaitMax time.Duration +} + +// Option is a function that sets a value in a config. +type ClientOption func(*clientConfig) + +// getClientOpts creates a pubConfig and applies Options to it. +func getClientOpts(opts []ClientOption) clientConfig { + var cfg clientConfig + for _, opt := range opts { + opt(&cfg) + } + return cfg +} + +// ClientAuthServerPeerID tells the sync client that it must authenticate the +// server's peer ID. +func ClientAuthServerPeerID(require bool) ClientOption { + return func(c *clientConfig) { + c.authPeerID = require + } +} + +// ClientHTTPTimeout specifies a time limit for HTTP requests made by the sync +// client. A value of zero means no timeout. +func ClientHTTPTimeout(to time.Duration) ClientOption { + return func(c *clientConfig) { + c.httpTimeout = to + } +} + +// ClientStreamHost specifies an optional stream based libp2p host used by the +// sync client to do HTTP over libp2p streams. +func ClientStreamHost(h host.Host) ClientOption { + return func(c *clientConfig) { + c.streamHost = h + } +} + +// ClientHTTPRetry configures a retriable HTTP client. Setting retryMax to +// zero, the default, disables the retriable client. +func ClientHTTPRetry(retryMax int, waitMin, waitMax time.Duration) ClientOption { + return func(c *clientConfig) { + c.httpRetryMax = retryMax + c.httpRetryWaitMin = waitMin + c.httpRetryWaitMax = waitMax + } +} diff --git a/dagsync/ipnisync/publisher.go b/dagsync/ipnisync/publisher.go index 92c060b..601751f 100644 --- a/dagsync/ipnisync/publisher.go +++ b/dagsync/ipnisync/publisher.go @@ -3,8 +3,6 @@ package ipnisync import ( "errors" "fmt" - "io" - "net" "net/http" "net/url" "path" @@ -17,16 +15,15 @@ import ( cidlink "github.com/ipld/go-ipld-prime/linking/cid" basicnode "github.com/ipld/go-ipld-prime/node/basic" headschema "github.com/ipni/go-libipni/dagsync/ipnisync/head" + "github.com/ipni/go-libipni/maurl" ic "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" + libp2phttp "github.com/libp2p/go-libp2p/p2p/http" "github.com/multiformats/go-multiaddr" - manet "github.com/multiformats/go-multiaddr/net" ) // Publisher serves an advertisement chain over HTTP. type Publisher struct { - addr multiaddr.Multiaddr - closer io.Closer lsys ipld.LinkSystem handlerPath string peerID peer.ID @@ -34,13 +31,17 @@ type Publisher struct { lock sync.Mutex root cid.Cid topic string + + pubHost *libp2phttp.Host + // httpAddrs is returned by Addrs when not starting the server. + httpAddrs []multiaddr.Multiaddr } var _ http.Handler = (*Publisher)(nil) -// NewPublisher creates a new http publisher. It is optional with start a server, listening on the specified -// address, using the WithStartServer option. -func NewPublisher(address string, lsys ipld.LinkSystem, privKey ic.PrivKey, options ...Option) (*Publisher, error) { +// NewPublisher creates a new ipni-sync publisher. Optionally, a libp2p stream +// host can be provided to serve HTTP over libp2p. +func NewPublisher(lsys ipld.LinkSystem, privKey ic.PrivKey, options ...Option) (*Publisher, error) { opts, err := getOpts(options) if err != nil { return nil, err @@ -52,84 +53,104 @@ func NewPublisher(address string, lsys ipld.LinkSystem, privKey ic.PrivKey, opti if err != nil { return nil, fmt.Errorf("could not get peer id from private key: %w", err) } - proto, _ := multiaddr.NewMultiaddr("/http") - handlerPath := strings.TrimPrefix(opts.handlerPath, "/") - if handlerPath != "" { - httpath, err := multiaddr.NewComponent("httpath", url.PathEscape(handlerPath)) - if err != nil { - return nil, err - } - proto = multiaddr.Join(proto, httpath) - handlerPath = "/" + handlerPath + if opts.streamHost != nil && opts.streamHost.ID() != peerID { + return nil, errors.New("stream host ID must match private key ID") } pub := &Publisher{ - lsys: lsys, - handlerPath: path.Join(handlerPath, IpniPath), - peerID: peerID, - privKey: privKey, - topic: opts.topic, + lsys: lsys, + peerID: peerID, + privKey: privKey, + topic: opts.topic, } - var addr net.Addr - if opts.startServer { - l, err := net.Listen("tcp", address) - if err != nil { - return nil, err - } - pub.closer = l - addr = l.Addr() - - // Run service on configured port. - server := &http.Server{ - Handler: pub, - Addr: l.Addr().String(), - } - go server.Serve(l) + // Construct expected request path prefix. If server is started this will + // get stripped off. If using an external server, look for this path when + // handling requests. + var handlerPath string + opts.handlerPath = strings.TrimPrefix(opts.handlerPath, "/") + if opts.handlerPath != "" { + handlerPath = path.Join(opts.handlerPath, IPNIPath) } else { - addr, err = net.ResolveTCPAddr("tcp", address) + handlerPath = strings.TrimPrefix(IPNIPath, "/") + } + + if !opts.startServer { + httpListenAddrs, err := httpAddrsToMultiaddrs(opts.httpAddrs, opts.requireTLS, opts.handlerPath) if err != nil { return nil, err } + if opts.streamHost != nil { + return nil, errors.New("server must be started to serve http over stream host") + } + // If the server is not started, the handlerPath does not get stripped + // from the HTTP request, so leave it as part of the prefix to match in + // the ServeHTTP handler. + pub.handlerPath = handlerPath + pub.httpAddrs = httpListenAddrs + return pub, nil } - maddr, err := manet.FromNetAddr(addr) + httpListenAddrs, err := httpAddrsToMultiaddrs(opts.httpAddrs, opts.requireTLS, "") if err != nil { - if pub.closer != nil { - pub.closer.Close() - } return nil, err } - pub.addr = multiaddr.Join(maddr, proto) + if len(httpListenAddrs) == 0 && opts.streamHost == nil { + return nil, errors.New("at least one http listen address or libp2p stream host is needed") + } - return pub, nil -} + // This is the "HTTP Host". It's like the libp2p "stream host" (aka core + // host.Host), but it uses HTTP semantics instead of stream semantics. + publisherHost := &libp2phttp.Host{ + StreamHost: opts.streamHost, + ListenAddrs: httpListenAddrs, + InsecureAllowHTTP: !opts.requireTLS, + } + pub.pubHost = publisherHost -// NewPublisherForListener creates a new http publisher for an existing -// listener. When providing an existing listener, running the HTTP server -// is the caller's responsibility. ServeHTTP on the returned Publisher -// can be used to handle requests. handlerPath is the path to handle -// requests on, e.g. "ipni" for `/ipni/...` requests. -// -// DEPRECATED: use NewPublisherWithoutServer(listener.Addr(), ...) -func NewPublisherForListener(listener net.Listener, handlerPath string, lsys ipld.LinkSystem, privKey ic.PrivKey) (*Publisher, error) { - return NewPublisherWithoutServer(listener.Addr().String(), handlerPath, lsys, privKey) + // Here is where this Publisher is attached as a request handler. This + // mounts the "/ipnisync/v1" protocol at "/opt_handler_path/ipni/v1/ad/", + // where opt_handler_path is the optional user specified handler path. If + // opts.handlerPath is "/foo/", this mounts it at "/foo/ipni/v1/ad/". This + // Publisher will only receive requests whose path begins with the + // handlerPath. libp2phttp manages this mapping and clients can learn about + // the mapping at .well-known/libp2p. + // + // In this case we also want the HTTP handler to not even know about the + // prefix, so we use the stdlib http.StripPrefix. + publisherHost.SetHTTPHandlerAtPath(ProtocolID, "/"+handlerPath, pub) + + go publisherHost.Serve() + + // Calling publisherHost.Addrs() waits until listeners are ready. + log.Infow("Publisher ready", "listenOn", publisherHost.Addrs()) + + return pub, nil } // NewPublisherWithoutServer creates a new http publisher for an existing -// network address. When providing an existing network address, running -// the HTTP server is the caller's responsibility. ServeHTTP on the -// returned Publisher can be used to handle requests. +// network address. When providing an existing network address, running the +// HTTP server is the caller's responsibility. ServeHTTP on the returned +// Publisher can be used to handle requests. handlerPath is the path to handle +// requests on before the /ipni/v1/ad/ portion of the path. See +// WithHandlerPath. // -// DEPRECATED: use NewPublisher(address, lsys, privKey, WithHandlerPath(handlerPath)) +// DEPRECATED: use NewPublisher(lsys, privKey, WithHTTPListenAddrs(address), WithHandlerPath(handlerPath), WithStartServer(false)) func NewPublisherWithoutServer(address, handlerPath string, lsys ipld.LinkSystem, privKey ic.PrivKey, options ...Option) (*Publisher, error) { - return NewPublisher(address, lsys, privKey, WithHandlerPath(handlerPath), WithServer(false)) + return NewPublisher(lsys, privKey, WithHTTPListenAddrs(address), WithHandlerPath(handlerPath), WithStartServer(false)) } -// Addrs returns the addresses, as []multiaddress, that the Publisher is +// Addrs returns the slice of multiaddr addresses that the Publisher is // listening on. +// +// If the server is not started, WithStartServer(false), then this returns the +// multiaddr versions of the list of addresses given by WithHTTPListenAddrs and +// may not actually be a listening address. func (p *Publisher) Addrs() []multiaddr.Multiaddr { - return []multiaddr.Multiaddr{p.addr} + if p.pubHost == nil { + return p.httpAddrs + } + return p.pubHost.Addrs() } // ID returns the p2p peer ID of the Publisher. @@ -137,7 +158,7 @@ func (p *Publisher) ID() peer.ID { return p.peerID } -// Protocol returns the multihash protocol ID of the transport used by the +// Protocol returns the multiaddr protocol ID of the transport used by the // publisher. func (p *Publisher) Protocol() int { return multiaddr.P_HTTP @@ -152,19 +173,32 @@ func (p *Publisher) SetRoot(c cid.Cid) { // Close closes the Publisher. func (p *Publisher) Close() error { - return p.closer.Close() + if p.pubHost != nil { + return p.pubHost.Close() + } + return nil } // ServeHTTP implements the http.Handler interface. func (p *Publisher) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if p.handlerPath != "" && !strings.HasPrefix(r.URL.Path, p.handlerPath) { + // If we expect publisher requests to have a prefix in the request path, + // then check for the expected prefix.. This happens when using an external + // server with this Publisher as the request handler. + urlPath := strings.TrimPrefix(r.URL.Path, "/") + if p.handlerPath != "" { + // A URL path from http will have a leading "/". A URL from libp2phttp will not. + if !strings.HasPrefix(urlPath, p.handlerPath) { + http.Error(w, "invalid request path: "+r.URL.Path, http.StatusBadRequest) + return + } + } else if path.Dir(urlPath) != "." { http.Error(w, "invalid request path: "+r.URL.Path, http.StatusBadRequest) return } ask := path.Base(r.URL.Path) if ask == "head" { - // serve the head + // Serve the head message. p.lock.Lock() rootCid := p.root p.lock.Unlock() @@ -184,7 +218,7 @@ func (p *Publisher) ServeHTTP(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(marshalledMsg) return } - // interpret `ask` as a CID to serve. + // Interpret `ask` as a CID to serve. c, err := cid.Parse(ask) if err != nil { http.Error(w, "invalid request: not a cid", http.StatusBadRequest) @@ -213,3 +247,42 @@ func newEncodedSignedHead(rootCid cid.Cid, topic string, privKey ic.PrivKey) ([] } return signedHead.Encode() } + +func httpAddrsToMultiaddrs(httpAddrs []string, requireTLS bool, handlerPath string) ([]multiaddr.Multiaddr, error) { + if len(httpAddrs) == 0 { + return nil, nil + } + + var defaultScheme string + if requireTLS { + defaultScheme = "https://" + } else { + defaultScheme = "http://" + } + maddrs := make([]multiaddr.Multiaddr, len(httpAddrs)) + for i, addr := range httpAddrs { + if !strings.HasPrefix(addr, "https://") && !strings.HasPrefix(addr, "http://") { + addr = defaultScheme + addr + } + u, err := url.Parse(addr) + if err != nil { + return nil, err + } + if handlerPath != "" { + u = u.JoinPath(handlerPath) + } + if requireTLS && u.Scheme == "http" { + log.Warnf("Ignored non-secure HTTP address: %s", addr) + continue + } + maddrs[i], err = maurl.FromURL(u) + if err != nil { + return nil, err + } + } + if len(maddrs) == 0 && requireTLS { + return nil, errors.New("no usable http listen addresses: https required") + } + + return maddrs, nil +} diff --git a/dagsync/ipnisync/publisher_test.go b/dagsync/ipnisync/publisher_test.go index e83f9d1..d77e355 100644 --- a/dagsync/ipnisync/publisher_test.go +++ b/dagsync/ipnisync/publisher_test.go @@ -16,19 +16,234 @@ import ( "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/fluent" "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/storage/memstore" "github.com/ipld/go-ipld-prime/traversal" + selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" "github.com/ipni/go-libipni/announce" "github.com/ipni/go-libipni/announce/message" "github.com/ipni/go-libipni/dagsync/ipnisync" + "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" + "github.com/multiformats/go-multicodec" "github.com/stretchr/testify/require" ) +func TestPublisherWithLibp2pHTTP(t *testing.T) { + ctx := context.Background() + req := require.New(t) + + publisherStore := &correctedMemStore{&memstore.Store{ + Bag: make(map[string][]byte), + }} + publisherLsys := cidlink.DefaultLinkSystem() + publisherLsys.TrustedStorage = true + publisherLsys.SetReadStorage(publisherStore) + publisherLsys.SetWriteStorage(publisherStore) + + privKey, _, err := crypto.GenerateKeyPairWithReader(crypto.Ed25519, 256, rand.Reader) + req.NoError(err) + + // Use same identity as publisher. This is necessary so that same ID that + // the publisher uses to sign head/ query responses is the same as the ID + // used to identify the publisherStreamHost. Otherwise, it would be + // necessary for the sync client to know both IDs: one for the stream host + // to connect to, and one for the publisher to validate the dignatuse with. + publisherStreamHost, err := libp2p.New(libp2p.Identity(privKey), libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0")) + req.NoError(err) + + publisher, err := ipnisync.NewPublisher(publisherLsys, privKey, + ipnisync.WithHTTPListenAddrs("http://127.0.0.1:0"), + ipnisync.WithStreamHost(publisherStreamHost), + ipnisync.WithRequireTLS(false), + ) + req.NoError(err) + + req.Equal(2, len(publisher.Addrs())) + serverStreamMa := publisher.Addrs()[0] + serverHTTPMa := publisher.Addrs()[1] + req.Contains(serverHTTPMa.String(), "/http") + t.Log("libp2p stream server address:", serverStreamMa.String()) + t.Log("libp2p http server address:", serverHTTPMa.String()) + + link, err := publisherLsys.Store( + ipld.LinkContext{Ctx: ctx}, + cidlink.LinkPrototype{ + Prefix: cid.Prefix{ + Version: 1, + Codec: uint64(multicodec.DagJson), + MhType: uint64(multicodec.Sha2_256), + MhLength: -1, + }, + }, + fluent.MustBuildMap(basicnode.Prototype.Map, 4, func(na fluent.MapAssembler) { + na.AssembleEntry("fish").AssignString("lobster") + na.AssembleEntry("fish1").AssignString("lobster1") + na.AssembleEntry("fish2").AssignString("lobster2") + na.AssembleEntry("fish0").AssignString("lobster0") + })) + req.NoError(err) + publisher.SetRoot(link.(cidlink.Link).Cid) + + testCases := []struct { + name string + publisher peer.AddrInfo + streamHost func(t *testing.T) host.Host + }{ + { + "HTTP transport", + peer.AddrInfo{Addrs: []multiaddr.Multiaddr{serverHTTPMa}}, + func(t *testing.T) host.Host { + return nil + }, + }, + { + "libp2p stream transport", + peer.AddrInfo{ID: publisherStreamHost.ID(), Addrs: []multiaddr.Multiaddr{serverStreamMa}}, + func(t *testing.T) host.Host { + streamHost, err := libp2p.New(libp2p.NoListenAddrs) + req.NoError(err) + return streamHost + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Plumbing to set up the test. + clientStore := &correctedMemStore{&memstore.Store{ + Bag: make(map[string][]byte), + }} + clientLsys := cidlink.DefaultLinkSystem() + clientLsys.TrustedStorage = true + clientLsys.SetReadStorage(clientStore) + clientLsys.SetWriteStorage(clientStore) + clientSync := ipnisync.NewSync(clientLsys, nil, ipnisync.ClientStreamHost(tc.streamHost(t))) + + // In a dagsync Subscriber, the clientSync is created once and + // lives for the lifetime of the Subscriber (lifetime of indexer), + // The clientSyncer is created for each sync operation and only + // lives for the duration of the sync. The publisher's address may + // change from one sync to the next, and we do not know the + // addresses ahead of time. + t.Log("Syncing to publisher at:", tc.publisher.Addrs) + clientSyncer, err := clientSync.NewSyncer(tc.publisher) + req.NoError(err) + + headCid, err := clientSyncer.GetHead(ctx) + req.NoError(err) + + req.Equal(link.(cidlink.Link).Cid, headCid) + + clientSyncer.Sync(ctx, headCid, selectorparse.CommonSelector_MatchPoint) + require.NoError(t, err) + + // Assert that data is loadable from the link system. + wantLink := cidlink.Link{Cid: headCid} + node, err := clientLsys.Load(ipld.LinkContext{Ctx: ctx}, wantLink, basicnode.Prototype.Any) + require.NoError(t, err) + + // Assert synced node link matches the computed link, i.e. is spec-compliant. + gotLink, err := clientLsys.ComputeLink(wantLink.Prototype(), node) + require.NoError(t, err) + require.Equal(t, gotLink, wantLink, "computed %s but got %s", gotLink.String(), wantLink.String()) + }) + } +} + +func TestExistingServerWithPublisher(t *testing.T) { + ctx := context.Background() + req := require.New(t) + + publisherStore := &correctedMemStore{&memstore.Store{ + Bag: make(map[string][]byte), + }} + publisherLsys := cidlink.DefaultLinkSystem() + publisherLsys.TrustedStorage = true + publisherLsys.SetReadStorage(publisherStore) + publisherLsys.SetWriteStorage(publisherStore) + + privKey, _, err := crypto.GenerateKeyPairWithReader(crypto.Ed25519, 256, rand.Reader) + req.NoError(err) + + // Start a server without using libp2phttp to: + // 1. Demonstrate this works without using libp2phttp + // 2. Give example of how an existing listener can be used to server the publisher. + const listenAddress = "127.0.0.1:0" + l, err := net.Listen("tcp", listenAddress) + req.NoError(err) + defer l.Close() + addr := "http://" + l.Addr().String() + publisher, err := ipnisync.NewPublisher(publisherLsys, privKey, ipnisync.WithHTTPListenAddrs(addr), ipnisync.WithStartServer(false)) + req.NoError(err) + go http.Serve(l, publisher) + + serverHTTPMa := publisher.Addrs()[0] + req.Contains(serverHTTPMa.String(), "/http") + t.Log("libp2p http server address:", serverHTTPMa.String()) + + link, err := publisherLsys.Store( + ipld.LinkContext{Ctx: ctx}, + cidlink.LinkPrototype{ + Prefix: cid.Prefix{ + Version: 1, + Codec: uint64(multicodec.DagJson), + MhType: uint64(multicodec.Sha2_256), + MhLength: -1, + }, + }, + fluent.MustBuildMap(basicnode.Prototype.Map, 4, func(na fluent.MapAssembler) { + na.AssembleEntry("fish").AssignString("lobster") + na.AssembleEntry("fish1").AssignString("lobster1") + na.AssembleEntry("fish2").AssignString("lobster2") + na.AssembleEntry("fish0").AssignString("lobster0") + })) + req.NoError(err) + publisher.SetRoot(link.(cidlink.Link).Cid) + + pubInfo := peer.AddrInfo{ + ID: publisher.ID(), + Addrs: []multiaddr.Multiaddr{serverHTTPMa}, + } + + // Plumbing to set up the test. + clientStore := &correctedMemStore{&memstore.Store{ + Bag: make(map[string][]byte), + }} + clientLsys := cidlink.DefaultLinkSystem() + clientLsys.TrustedStorage = true + clientLsys.SetReadStorage(clientStore) + clientLsys.SetWriteStorage(clientStore) + clientSync := ipnisync.NewSync(clientLsys, nil) + + t.Log("Syncing to publisher at:", pubInfo.Addrs) + clientSyncer, err := clientSync.NewSyncer(pubInfo) + req.NoError(err) + + headCid, err := clientSyncer.GetHead(ctx) + req.NoError(err) + req.Equal(link.(cidlink.Link).Cid, headCid) + + clientSyncer.Sync(ctx, headCid, selectorparse.CommonSelector_MatchPoint) + require.NoError(t, err) + + // Assert that data is loadable from the link system. + wantLink := cidlink.Link{Cid: headCid} + node, err := clientLsys.Load(ipld.LinkContext{Ctx: ctx}, wantLink, basicnode.Prototype.Any) + require.NoError(t, err) + + // Assert synced node link matches the computed link, i.e. is spec-compliant. + gotLink, err := clientLsys.ComputeLink(wantLink.Prototype(), node) + require.NoError(t, err) + require.Equal(t, gotLink, wantLink, "computed %s but got %s", gotLink.String(), wantLink.String()) +} + func TestNewPublisherForListener(t *testing.T) { req := require.New(t) ctx := context.Background() @@ -51,7 +266,7 @@ func TestNewPublisherForListener(t *testing.T) { privKey, _, err := crypto.GenerateKeyPairWithReader(crypto.RSA, 2048, rand.Reader) req.NoError(err) sender := &fakeSender{} - subject, err := ipnisync.NewPublisher(l.Addr().String(), lsys, privKey, ipnisync.WithServer(false), ipnisync.WithHandlerPath(handlerPath)) + subject, err := ipnisync.NewPublisher(lsys, privKey, ipnisync.WithHTTPListenAddrs(l.Addr().String()), ipnisync.WithHandlerPath(handlerPath), ipnisync.WithStartServer(false)) req.NoError(err) rootCid := rootLnk.(cidlink.Link).Cid @@ -71,10 +286,9 @@ func TestNewPublisherForListener(t *testing.T) { resp := &mockResponseWriter{} u := &url.URL{ - Path: path.Join("/", handlerPath, ipnisync.IpniPath, "/head"), + Path: path.Join("/", handlerPath, ipnisync.IPNIPath, "/head"), } - //u = u.JoinPath(handlerPath, ipnisync.IpniPath, "/head") - // + subject.ServeHTTP(resp, &http.Request{URL: u}) req.Equal(0, resp.status) // not explicitly set req.Nil(resp.header) @@ -96,6 +310,149 @@ func TestNewPublisherForListener(t *testing.T) { } } +func TestHandlerPath(t *testing.T) { + //t.Skip("needs work") + req := require.New(t) + ctx := context.Background() + + publisherStore := &correctedMemStore{&memstore.Store{ + Bag: make(map[string][]byte), + }} + publisherLsys := cidlink.DefaultLinkSystem() + publisherLsys.TrustedStorage = true + publisherLsys.SetReadStorage(publisherStore) + publisherLsys.SetWriteStorage(publisherStore) + + privKey, _, err := crypto.GenerateKeyPairWithReader(crypto.Ed25519, 256, rand.Reader) + req.NoError(err) + + publisher, err := ipnisync.NewPublisher(publisherLsys, privKey, + ipnisync.WithHTTPListenAddrs("http://127.0.0.1:0"), + ipnisync.WithHandlerPath("/boop/bop/beep"), + ) + req.NoError(err) + + req.Equal(1, len(publisher.Addrs())) + serverHTTPMa := publisher.Addrs()[0] + req.Contains(serverHTTPMa.String(), "/http") + t.Log("libp2p http server address:", serverHTTPMa.String()) + + link, err := publisherLsys.Store( + ipld.LinkContext{Ctx: ctx}, + cidlink.LinkPrototype{ + Prefix: cid.Prefix{ + Version: 1, + Codec: uint64(multicodec.DagJson), + MhType: uint64(multicodec.Sha2_256), + MhLength: -1, + }, + }, + fluent.MustBuildMap(basicnode.Prototype.Map, 4, func(na fluent.MapAssembler) { + na.AssembleEntry("fish").AssignString("lobster") + na.AssembleEntry("fish1").AssignString("lobster1") + na.AssembleEntry("fish2").AssignString("lobster2") + na.AssembleEntry("fish0").AssignString("lobster0") + })) + req.NoError(err) + publisher.SetRoot(link.(cidlink.Link).Cid) + + testCases := []struct { + name string + httpPath string + expectErr bool + }{ + { + "badPath1", + "", + true, + }, + { + "badPath2", + "/", + true, + }, + { + "badPath3", + "boop/bop", + true, + }, + { + "badPath4", + "/boop/bop", + true, + }, + { + "goodPath1 no leading slash", + "boop/bop/beep", + false, + }, + { + "goodPath2 leading slash", + "/boop/bop/beep", + false, + }, + { + "goodPath3 trailing slash", + "boop/bop/beep/", + false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Plumbing to set up the test. + clientStore := &correctedMemStore{&memstore.Store{ + Bag: make(map[string][]byte), + }} + clientLsys := cidlink.DefaultLinkSystem() + clientLsys.TrustedStorage = true + clientLsys.SetReadStorage(clientStore) + clientLsys.SetWriteStorage(clientStore) + clientSync := ipnisync.NewSync(clientLsys, nil) + + httpPath := strings.TrimPrefix(tc.httpPath, "/") + var maddr multiaddr.Multiaddr + if httpPath != "" { + httpath, err := multiaddr.NewComponent("httpath", url.PathEscape(httpPath)) + req.NoError(err) + maddr = multiaddr.Join(serverHTTPMa, httpath) + } else { + maddr = serverHTTPMa + } + pubInfo := peer.AddrInfo{ + //ID: publisher.ID(), // optional + Addrs: []multiaddr.Multiaddr{maddr}, + } + t.Log("Syncing to publisher at:", pubInfo.Addrs) + clientSyncer, err := clientSync.NewSyncer(pubInfo) + req.NoError(err) + + headCid, err := clientSyncer.GetHead(ctx) + + if tc.expectErr { + req.Error(err) + return + } + + req.NoError(err) + req.Equal(link.(cidlink.Link).Cid, headCid) + + clientSyncer.Sync(ctx, headCid, selectorparse.CommonSelector_MatchPoint) + require.NoError(t, err) + + // Assert that data is loadable from the link system. + wantLink := cidlink.Link{Cid: headCid} + node, err := clientLsys.Load(ipld.LinkContext{Ctx: ctx}, wantLink, basicnode.Prototype.Any) + require.NoError(t, err) + + // Assert synced node link matches the computed link, i.e. is spec-compliant. + gotLink, err := clientLsys.ComputeLink(wantLink.Prototype(), node) + require.NoError(t, err) + require.Equal(t, gotLink, wantLink, "computed %s but got %s", gotLink.String(), wantLink.String()) + }) + } +} + func mapKeys(t *testing.T, n ipld.Node) []string { var keys []string require.Equal(t, n.Kind(), datamodel.Kind_Map) diff --git a/dagsync/ipnisync/sync.go b/dagsync/ipnisync/sync.go index e984606..b400944 100644 --- a/dagsync/ipnisync/sync.go +++ b/dagsync/ipnisync/sync.go @@ -8,8 +8,11 @@ import ( "io" "net/http" "net/url" + "strings" + "sync" "time" + "github.com/hashicorp/go-retryablehttp" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" "github.com/ipld/go-ipld-prime" @@ -20,67 +23,152 @@ import ( "github.com/ipld/go-ipld-prime/traversal/selector" headschema "github.com/ipni/go-libipni/dagsync/ipnisync/head" "github.com/ipni/go-libipni/maurl" + "github.com/ipni/go-libipni/mautil" "github.com/libp2p/go-libp2p/core/peer" - "github.com/multiformats/go-multiaddr" + libp2phttp "github.com/libp2p/go-libp2p/p2p/http" "github.com/multiformats/go-multihash" ) -const defaultHttpTimeout = 10 * time.Second - var log = logging.Logger("dagsync/ipnisync") +var ErrNoHTTPServer = errors.New("publisher has libp2p server without HTTP") + // Sync provides sync functionality for use with all http syncs. type Sync struct { - blockHook func(peer.ID, cid.Cid) - client *http.Client - lsys ipld.LinkSystem + blockHook func(peer.ID, cid.Cid) + client http.Client + lsys ipld.LinkSystem + httpTimeout time.Duration + + // libp2phttp + clientHost *libp2phttp.Host + clientHostMutex sync.Mutex + authPeerID bool + rclient *retryablehttp.Client +} + +// Syncer provides sync functionality for a single sync with a peer. +type Syncer struct { + client *http.Client + peerID peer.ID + rootURL url.URL + urls []*url.URL + sync *Sync + + // For legacy HTTP and external server support without IPNI path. + noPath bool + plainHTTP bool } // NewSync creates a new Sync. -func NewSync(lsys ipld.LinkSystem, client *http.Client, blockHook func(peer.ID, cid.Cid)) *Sync { - if client == nil { - client = &http.Client{ - Timeout: defaultHttpTimeout, - } - } - return &Sync{ +func NewSync(lsys ipld.LinkSystem, blockHook func(peer.ID, cid.Cid), options ...ClientOption) *Sync { + opts := getClientOpts(options) + + s := &Sync{ blockHook: blockHook, - client: client, - lsys: lsys, + client: http.Client{ + Timeout: opts.httpTimeout, + }, + clientHost: &libp2phttp.Host{ + StreamHost: opts.streamHost, + }, + lsys: lsys, + authPeerID: opts.authPeerID, + httpTimeout: opts.httpTimeout, + } + + if opts.httpRetryMax != 0 { + // Configure retryable HTTP client created by calls to NewSyncer. + s.rclient = &retryablehttp.Client{ + RetryWaitMin: opts.httpRetryWaitMin, + RetryWaitMax: opts.httpRetryWaitMax, + RetryMax: opts.httpRetryMax, + } } + + return s } -// NewSyncer creates a new Syncer to use for a single sync operation against a peer. -func (s *Sync) NewSyncer(peerID peer.ID, peerAddrs []multiaddr.Multiaddr) (*Syncer, error) { - urls := make([]*url.URL, len(peerAddrs)) - for i := range peerAddrs { - u, err := maurl.ToURL(peerAddrs[i]) +// NewSyncer creates a new Syncer to use for a single sync operation against a +// peer. A value for peerInfo.ID is optional for the HTTP transport. +func (s *Sync) NewSyncer(peerInfo peer.AddrInfo) (*Syncer, error) { + var cli http.Client + var httpClient *http.Client + var err error + var rtOpts []libp2phttp.RoundTripperOption + if s.authPeerID { + rtOpts = append(rtOpts, libp2phttp.ServerMustAuthenticatePeerID) + } + s.clientHostMutex.Lock() + cli, err = s.clientHost.NamespacedClient(ProtocolID, peerInfo, rtOpts...) + s.clientHostMutex.Unlock() + var plainHTTP bool + if err != nil { + httpAddrs := mautil.FindHTTPAddrs(peerInfo.Addrs) + if len(httpAddrs) == 0 { + return nil, ErrNoHTTPServer + } + log.Infow("Publisher is not a libp2phttp server. Using plain http", "err", err, "peer", peerInfo.ID) + httpClient = &s.client + plainHTTP = true + } else { + log.Infow("Publisher supports libp2phttp", "peer", peerInfo.ID) + httpClient = &cli + } + httpClient.Timeout = s.httpTimeout + + if s.rclient != nil { + // Instantiate retryable HTTP client. + rclient := &retryablehttp.Client{ + HTTPClient: httpClient, + RetryWaitMin: s.rclient.RetryWaitMin, + RetryWaitMax: s.rclient.RetryWaitMax, + RetryMax: s.rclient.RetryMax, + CheckRetry: retryablehttp.DefaultRetryPolicy, + Backoff: retryablehttp.DefaultBackoff, + } + httpClient = rclient.StandardClient() + } + + if len(peerInfo.Addrs) == 0 { + if s.clientHost.StreamHost == nil { + return nil, errors.New("no peer addrs and no stream host") + } + peerStore := s.clientHost.StreamHost.Peerstore() + if peerStore == nil { + return nil, errors.New("no peer addrs and no stream host peerstore") + } + peerInfo.Addrs = peerStore.Addrs(peerInfo.ID) + if len(peerInfo.Addrs) == 0 { + return nil, errors.New("no peer addrs and none found in peertore") + } + } + + urls := make([]*url.URL, len(peerInfo.Addrs)) + for i, addr := range peerInfo.Addrs { + u, err := maurl.ToURL(addr) if err != nil { return nil, err } - urls[i] = u.JoinPath(IpniPath) + urls[i] = u.JoinPath(IPNIPath) } return &Syncer{ - peerID: peerID, + client: httpClient, + peerID: peerInfo.ID, rootURL: *urls[0], urls: urls[1:], sync: s, + + plainHTTP: plainHTTP, }, nil } func (s *Sync) Close() { s.client.CloseIdleConnections() -} - -var errHeadFromUnexpectedPeer = errors.New("found head signed from an unexpected peer") - -// Syncer provides sync functionality for a single sync with a peer. -type Syncer struct { - peerID peer.ID - rootURL url.URL - urls []*url.URL - sync *Sync + s.clientHostMutex.Lock() + s.clientHost.Close() + s.clientHostMutex.Unlock() } // GetHead fetches the head of the peer's advertisement chain. @@ -99,14 +187,18 @@ func (s *Syncer) GetHead(ctx context.Context) (cid.Cid, error) { if err != nil { return cid.Undef, err } - if signerID != s.peerID { - return cid.Undef, errHeadFromUnexpectedPeer + if s.peerID == "" { + log.Warn("Cannot verify publisher signature without peer ID") + } else if signerID != s.peerID { + return cid.Undef, errors.New("found head signed by an unexpected peer") } - // TODO: Do something with signedHead.Topic. - // - // Should it be returned (and for what purpose)? - // Is it needed to construct the advertisement fetch URL? + // TODO: Check that the returned topic, if any, matches the expected topic. + //if signedHead.Topic != nil && *signedHead.Topic != "" && expectedTopic != "" { + // if *signedHead.Topic != expectedTopic { + // return nil, ErrTopicMismatch + // } + //} return signedHead.Head.(cidlink.Link).Cid, nil } @@ -123,38 +215,34 @@ func (s *Syncer) Sync(ctx context.Context, nextCid cid.Cid, sel ipld.Node) error return fmt.Errorf("failed to traverse requested dag: %w", err) } - // We run the block hook to emulate the behavior of graphsync's - // `OnIncomingBlockHook` callback (gets called even if block is already stored - // locally). + // The blockHook callback gets called for every synced block, even if block + // is already stored locally. // // We are purposefully not doing this in the StorageReadOpener because the - // hook can do anything, including deleting the block from the block store. If - // it did that then we would not be able to continue our traversal. So instead - // we remember the blocks seen during traversal and then call the hook at the - // end when we no longer care what it does with the blocks. + // hook can do anything, including deleting the block from the block store. + // If it did that then we would not be able to continue our traversal. So + // instead we remember the blocks seen during traversal and then call the + // hook at the end when we no longer care what it does with the blocks. if s.sync.blockHook != nil { for _, c := range cids { s.sync.blockHook(s.peerID, c) } } - s.sync.client.CloseIdleConnections() + s.client.CloseIdleConnections() return nil } -// walkFetch is run by a traversal of the selector. For each block that the +// walkFetch is run by a traversal of the selector. For each block that the // selector walks over, walkFetch will look to see if it can find it in the -// local data store. If it cannot, it will then go and get it over HTTP. This -// emulates way libp2p/graphsync fetches data, but the actual fetch of data is -// done over HTTP. +// local data store. If it cannot, it will then go and get it over HTTP. func (s *Syncer) walkFetch(ctx context.Context, rootCid cid.Cid, sel selector.Selector) ([]cid.Cid, error) { - // Track the order of cids we've seen during our traversal so we can call the - // block hook function in the same order. We emulate the behavior of - // graphsync's `OnIncomingBlockHook`, this means we call the blockhook even if - // we have the block locally. + // Track the order of cids seen during traversal so that the block hook + // function gets called in the same order. var traversalOrder []cid.Cid getMissingLs := cidlink.DefaultLinkSystem() - // trusted because it'll be hashed/verified on the way into the link system when fetched. + // Trusted because it will be hashed/verified on the way into the link + // system when fetched. getMissingLs.TrustedStorage = true getMissingLs.StorageReadOpener = func(lc ipld.LinkContext, l ipld.Link) (io.Reader, error) { c := l.(cidlink.Link).Cid @@ -184,14 +272,14 @@ func (s *Syncer) walkFetch(ctx context.Context, rootCid cid.Cid, sel selector.Se }, Path: datamodel.NewPath([]datamodel.PathSegment{}), } + // get the direct node. rootNode, err := getMissingLs.Load(ipld.LinkContext{Ctx: ctx}, cidlink.Link{Cid: rootCid}, basicnode.Prototype.Any) if err != nil { return nil, fmt.Errorf("failed to load node for root cid %s: %w", rootCid, err) } - err = progress.WalkMatching(rootNode, sel, func(p traversal.Progress, n datamodel.Node) error { - return nil - }) + + err = progress.WalkMatching(rootNode, sel, func(_ traversal.Progress, _ datamodel.Node) error { return nil }) if err != nil { return nil, err } @@ -201,18 +289,20 @@ func (s *Syncer) walkFetch(ctx context.Context, rootCid cid.Cid, sel selector.Se func (s *Syncer) fetch(ctx context.Context, rsrc string, cb func(io.Reader) error) error { nextURL: fetchURL := s.rootURL.JoinPath(rsrc) - req, err := http.NewRequestWithContext(ctx, "GET", fetchURL.String(), nil) if err != nil { return err } - resp, err := s.sync.client.Do(req) + resp, err := s.client.Do(req) if err != nil { if len(s.urls) != 0 { log.Errorw("Fetch request failed, will retry with next address", "err", err) s.rootURL = *s.urls[0] s.urls = s.urls[1:] + if s.noPath { + s.rootURL.Path = strings.TrimSuffix(s.rootURL.Path, strings.Trim(IPNIPath, "/")) + } goto nextURL } return fmt.Errorf("fetch request failed: %w", err) @@ -220,15 +310,30 @@ nextURL: defer resp.Body.Close() switch resp.StatusCode { + case http.StatusOK: + return cb(resp.Body) case http.StatusNotFound: + if s.plainHTTP && !s.noPath { + // Try again with no path for legacy http. + log.Warnw("Plain HTTP got not found response, retrying without IPNI path for legacy HTTP") + s.rootURL.Path = strings.TrimSuffix(s.rootURL.Path, strings.Trim(IPNIPath, "/")) + s.noPath = true + goto nextURL + } log.Errorw("Block not found from HTTP publisher", "resource", rsrc) // Include the string "content not found" so that indexers that have not // upgraded gracefully handle the error case. Because, this string is // being checked already. return fmt.Errorf("content not found: %w", ipld.ErrNotExists{}) - case http.StatusOK: - log.Debugw("Found block from HTTP publisher", "resource", rsrc) - return cb(resp.Body) + case http.StatusForbidden: + if s.plainHTTP && !s.noPath { + // Try again with no path for legacy http. + log.Warnw("Plain HTTP got forbidden response, retrying without IPNI path for legacy HTTP") + s.rootURL.Path = strings.TrimSuffix(s.rootURL.Path, strings.Trim(IPNIPath, "/")) + s.noPath = true + goto nextURL + } + fallthrough default: return fmt.Errorf("non success http fetch response at %s: %d", fetchURL.String(), resp.StatusCode) } diff --git a/dagsync/ipnisync/sync_test.go b/dagsync/ipnisync/sync_test.go index 976afb4..c748b27 100644 --- a/dagsync/ipnisync/sync_test.go +++ b/dagsync/ipnisync/sync_test.go @@ -86,8 +86,12 @@ func TestIPNISync_NFTStorage_DigestCheck(t *testing.T) { pubmaddr, err := maurl.FromURL(puburl) require.NoError(t, err) - sync := ipnisync.NewSync(ls, http.DefaultClient, nil) - syncer, err := sync.NewSyncer(pubid, []multiaddr.Multiaddr{pubmaddr}) + sync := ipnisync.NewSync(ls, nil) + pubInfo := peer.AddrInfo{ + ID: pubid, + Addrs: []multiaddr.Multiaddr{pubmaddr}, + } + syncer, err := sync.NewSyncer(pubInfo) require.NoError(t, err) head, err := syncer.GetHead(ctx) @@ -133,7 +137,7 @@ func TestIPNIsync_AcceptsSpecCompliantDagJson(t *testing.T) { publs.SetWriteStorage(pubstore) publs.SetReadStorage(pubstore) - pub, err := ipnisync.NewPublisher("0.0.0.0:0", publs, pubPrK, ipnisync.WithHeadTopic(testTopic), ipnisync.WithServer(true)) + pub, err := ipnisync.NewPublisher(publs, pubPrK, ipnisync.WithHeadTopic(testTopic), ipnisync.WithHTTPListenAddrs("0.0.0.0:0")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, pub.Close()) }) @@ -161,8 +165,12 @@ func TestIPNIsync_AcceptsSpecCompliantDagJson(t *testing.T) { ls.SetWriteStorage(store) ls.SetReadStorage(store) - sync := ipnisync.NewSync(ls, http.DefaultClient, nil) - syncer, err := sync.NewSyncer(pubID, pub.Addrs()) + sync := ipnisync.NewSync(ls, nil) + pubInfo := peer.AddrInfo{ + ID: pubID, + Addrs: pub.Addrs(), + } + syncer, err := sync.NewSyncer(pubInfo) require.NoError(t, err) head, err := syncer.GetHead(ctx) @@ -197,7 +205,7 @@ func TestIPNIsync_NotFoundReturnsContentNotFoundErr(t *testing.T) { return nil, ipld.ErrNotExists{} } - pub, err := ipnisync.NewPublisher("0.0.0.0:0", publs, pubPrK, ipnisync.WithServer(true)) + pub, err := ipnisync.NewPublisher(publs, pubPrK, ipnisync.WithHTTPListenAddrs("0.0.0.0:0")) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, pub.Close()) }) @@ -206,8 +214,12 @@ func TestIPNIsync_NotFoundReturnsContentNotFoundErr(t *testing.T) { ls.SetWriteStorage(store) ls.SetReadStorage(store) - sync := ipnisync.NewSync(ls, http.DefaultClient, nil) - syncer, err := sync.NewSyncer(pubID, pub.Addrs()) + sync := ipnisync.NewSync(ls, nil) + pubInfo := peer.AddrInfo{ + ID: pubID, + Addrs: pub.Addrs(), + } + syncer, err := sync.NewSyncer(pubInfo) require.NoError(t, err) mh, err := multihash.Sum([]byte("fish"), multihash.SHA2_256, -1) diff --git a/dagsync/option.go b/dagsync/option.go index eae5d2a..3c2d49e 100644 --- a/dagsync/option.go +++ b/dagsync/option.go @@ -1,8 +1,8 @@ package dagsync import ( + "errors" "fmt" - "net/http" "time" "github.com/ipfs/go-cid" @@ -21,9 +21,11 @@ const ( defaultIdleHandlerTTL = time.Hour // defaultSegDepthLimit disables (-1) segmented sync by default. defaultSegDepthLimit = -1 - // Maximum number of in-prgress graphsync requests. + // Maximum number of in-progress graphsync requests. defaultGsMaxInRequests = 1024 defaultGsMaxOutRequests = 1024 + // defaultHttpTimeout is time limit for requests made by the HTTP client. + defaultHttpTimeout = 10 * time.Second ) type LastKnownSyncFunc func(peer.ID) (cid.Cid, bool) @@ -34,8 +36,7 @@ type config struct { topic *pubsub.Topic - blockHook BlockHookFunc - httpClient *http.Client + blockHook BlockHookFunc idleHandlerTTL time.Duration lastKnownSync LastKnownSyncFunc @@ -52,6 +53,11 @@ type config struct { gsMaxOutRequests uint64 strictAdsSelSeq bool + + httpTimeout time.Duration + httpRetryMax int + httpRetryWaitMin time.Duration + httpRetryWaitMax time.Duration } // Option is a function that sets a value in a config. @@ -61,6 +67,7 @@ type Option func(*config) error func getOpts(opts []Option) (config, error) { cfg := config{ addrTTL: defaultAddrTTL, + httpTimeout: defaultHttpTimeout, idleHandlerTTL: defaultIdleHandlerTTL, segDepthLimit: defaultSegDepthLimit, gsMaxInRequests: defaultGsMaxInRequests, @@ -93,18 +100,37 @@ func Topic(topic *pubsub.Topic) Option { } } -// HttpClient provides Subscriber with an existing http client. -func HttpClient(client *http.Client) Option { +// HttpTimeout specifies a time limit for HTTP requests made by the sync +// HTTP client. A value of zero means no timeout. +func HttpTimeout(to time.Duration) Option { + return func(c *config) error { + c.httpTimeout = to + return nil + } +} + +// RetryableHTTPClient configures a retriable HTTP client. Setting retryMax to +// zero, the default, disables the retriable client. +func RetryableHTTPClient(retryMax int, waitMin, waitMax time.Duration) Option { return func(c *config) error { - c.httpClient = client + if waitMin > waitMax { + return errors.New("minimum retry wait time cannot be greater than maximum") + } + if retryMax < 0 { + retryMax = 0 + } + c.httpRetryMax = retryMax + c.httpRetryWaitMin = waitMin + c.httpRetryWaitMax = waitMax return nil } } -// BlockHook adds a hook that is run when a block is received via Subscriber.Sync along with a -// SegmentSyncActions to control the sync flow if segmented sync is enabled. -// Note that if segmented sync is disabled, calls on SegmentSyncActions will have no effect. -// See: SegmentSyncActions, SegmentDepthLimit, ScopedBlockHook. +// BlockHook adds a hook that is run when a block is received via +// Subscriber.Sync along with a SegmentSyncActions to control the sync flow if +// segmented sync is enabled. Note that if segmented sync is disabled, calls on +// SegmentSyncActions will have no effect. See: SegmentSyncActions, +// SegmentDepthLimit, ScopedBlockHook. func BlockHook(blockHook BlockHookFunc) Option { return func(c *config) error { c.blockHook = blockHook @@ -171,7 +197,7 @@ func RecvAnnounce(opts ...announce.Option) Option { // MaxAsyncConcurrency sets the maximum number of concurrent asynchrouous syncs // (started by announce messages). This only takes effect if there is an -// announcement reveiver configured by the RecvAnnounce option. +// announcement receiver configured by the RecvAnnounce option. func MaxAsyncConcurrency(n int) Option { return func(c *config) error { if n != 0 { @@ -241,7 +267,7 @@ func WithStopAdCid(stopAd cid.Cid) SyncOption { } } -// WithResyncAds causes the current sync to ignore anvertisements that have been +// WithResyncAds causes the current sync to ignore advertisements that have been // previously synced. When true, sync does not record the latest synced CID or // send sync finished notification. func WithAdsResync(resync bool) SyncOption { diff --git a/dagsync/subscriber.go b/dagsync/subscriber.go index 28b1f6d..ec514be 100644 --- a/dagsync/subscriber.go +++ b/dagsync/subscriber.go @@ -133,9 +133,9 @@ type SyncFinished struct { PeerID peer.ID // Count is the number of CID synced. Count int - // AsyncErr is used to return a failure to asynchronous sync in response to - // an announcement. - AsyncErr error + // Err is used to return a failure to complete an asynchronous sync in + // response to an announcement. + Err error } // handler holds state that is specific to a peer @@ -196,6 +196,11 @@ func NewSubscriber(host host.Host, ds datastore.Batching, lsys ipld.LinkSystem, ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) all := ssb.ExploreAll(ssb.ExploreRecursiveEdge()) + ipniSync := ipnisync.NewSync(lsys, blockHook, + ipnisync.ClientStreamHost(host), + ipnisync.ClientHTTPTimeout(opts.httpTimeout), + ipnisync.ClientHTTPRetry(opts.httpRetryMax, opts.httpRetryWaitMin, opts.httpRetryWaitMax)) + s := &Subscriber{ host: host, @@ -209,7 +214,7 @@ func NewSubscriber(host host.Host, ds datastore.Batching, lsys ipld.LinkSystem, rmEventChan: make(chan chan<- SyncFinished), dtSync: dtSync, - ipniSync: ipnisync.NewSync(lsys, opts.httpClient, blockHook), + ipniSync: ipniSync, httpPeerstore: httpPeerstore, @@ -372,7 +377,7 @@ func (s *Subscriber) RemoveHandler(peerID peer.ID) bool { return false } - log.Infow("Removing handler for publisher", "peer", peerID) + log.Infow("Removing sync handler", "peer", peerID) delete(s.handlers, peerID) return true @@ -549,7 +554,7 @@ func (s *Subscriber) SyncHAMTEntries(ctx context.Context, peerInfo peer.AddrInfo func (s *Subscriber) syncEntries(ctx context.Context, peerInfo peer.AddrInfo, entCid cid.Cid, sel ipld.Node, bh BlockHookFunc, segdl int64) error { if entCid == cid.Undef { - log.Info("No entries to sync") + log.Info("No entries to sync", "peer", peerInfo.ID) return nil } @@ -572,8 +577,7 @@ func (s *Subscriber) syncEntries(ctx context.Context, peerInfo peer.AddrInfo, en return err } - log := log.With("peer", peerInfo.ID, "cid", entCid) - log.Info("Start entries sync sync") + log.Info("Start entries sync", "peer", peerInfo.ID, "cid", entCid) // Check for an existing handler for the specified peer (publisher). If // none, create one if allowed. @@ -676,7 +680,7 @@ func (s *Subscriber) idleHandlerCleaner() { for pid, hnd := range s.handlers { if now.After(hnd.expires) { delete(s.handlers, pid) - log.Debugw("Removed idle handler", "publisherID", pid) + log.Debugw("Removed idle handler", "peer", pid) } } s.handlersMutex.Unlock() @@ -711,7 +715,7 @@ func (s *Subscriber) watch() { // If rhw previous pending message was not nil, then there is an // existing request to sync the ad chain. if oldMsg != nil { - log.Infow("Pending announce replaced by new", "previous_cid", oldMsg.Cid, "new_cid", amsg.Cid, "publisher", hnd.peerID) + log.Infow("Pending announce replaced by new", "previous_cid", oldMsg.Cid, "new_cid", amsg.Cid, "peer", hnd.peerID) continue } @@ -766,7 +770,11 @@ func (s *Subscriber) makeSyncer(peerInfo peer.AddrInfo, doUpdate bool) (Syncer, // Store this http address so that future calls to sync will work without a // peerAddr (given that it happens within the TTL) s.httpPeerstore.AddAddrs(peerInfo.ID, httpAddrs, tempAddrTTL) - syncer, err := s.ipniSync.NewSyncer(peerInfo.ID, httpAddrs) + httpPeerInfo := peer.AddrInfo{ + ID: peerInfo.ID, + Addrs: httpAddrs, + } + syncer, err := s.ipniSync.NewSyncer(httpPeerInfo) if err != nil { return nil, nil, fmt.Errorf("cannot create ipni-sync handler: %w", err) } @@ -794,8 +802,17 @@ func (s *Subscriber) makeSyncer(peerInfo peer.AddrInfo, doUpdate bool) (Syncer, update = func() {} } } - // Not an httpPeerAddr, so use the dtSync. - return s.dtSync.NewSyncer(peerInfo.ID, s.topicName), update, nil + + syncer, err := s.ipniSync.NewSyncer(peerInfo) + if err != nil { + if errors.Is(err, ipnisync.ErrNoHTTPServer) { + log.Warnw("Using data-transfer sync", "peer", peerInfo.ID, "reason", err.Error()) + // Publisher is libp2p without HTTP, so use the dtSync. + return s.dtSync.NewSyncer(peerInfo.ID, s.topicName), update, nil + } + return nil, nil, fmt.Errorf("cannot create ipni-sync handler: %w", err) + } + return syncer, update, nil } // asyncSyncAdChain processes the latest announce message received over pubsub @@ -803,7 +820,7 @@ func (s *Subscriber) makeSyncer(peerInfo peer.AddrInfo, doUpdate bool) (Syncer, // one goroutine per advertisement publisher. func (h *handler) asyncSyncAdChain(ctx context.Context) { if ctx.Err() != nil { - log.Warnw("Abandoned pending sync", "err", ctx.Err(), "publisher", h.peerID) + log.Warnw("Abandoned pending sync", "err", ctx.Err(), "peer", h.peerID) return } @@ -815,7 +832,7 @@ func (h *handler) asyncSyncAdChain(ctx context.Context) { } syncer, updatePeerstore, err := h.subscriber.makeSyncer(peerInfo, true) if err != nil { - log.Errorw("Cannot make syncer for announce", "err", err) + log.Errorw("Cannot make syncer for announce", "err", err, "peer", h.peerID) return } @@ -825,7 +842,7 @@ func (h *handler) asyncSyncAdChain(ctx context.Context) { if latestSyncLink != nil { stopAtCid = latestSyncLink.(cidlink.Link).Cid if stopAtCid == nextCid { - log.Infow("cid to sync to is the stop node. Nothing to do") + log.Infow("CID to sync to is the stop node. Nothing to do.", "peer", h.peerID) return } } @@ -837,7 +854,7 @@ func (h *handler) asyncSyncAdChain(ctx context.Context) { if h.subscriber.receiver != nil { h.subscriber.receiver.UncacheCid(nextCid) } - log.Errorw("Cannot process message", "err", err, "publisher", h.peerID) + log.Errorw("Cannot process message", "err", err, "peer", h.peerID) if strings.Contains(err.Error(), "response rejected") { // A "response rejected" error happens when the indexer does not // allow a provider. This is not an error with provider, so do not @@ -845,9 +862,9 @@ func (h *handler) asyncSyncAdChain(ctx context.Context) { return } h.subscriber.inEvents <- SyncFinished{ - Cid: nextCid, - PeerID: h.peerID, - AsyncErr: err, + Cid: nextCid, + PeerID: h.peerID, + Err: err, } return } diff --git a/dagsync/subscriber_test.go b/dagsync/subscriber_test.go index c2f4cc3..31d6a46 100644 --- a/dagsync/subscriber_test.go +++ b/dagsync/subscriber_test.go @@ -46,10 +46,10 @@ func TestScopedBlockHook(t *testing.T) { err := quick.Check(func(ll llBuilder) bool { return t.Run("Quickcheck", func(t *testing.T) { ds := dssync.MutexWrap(datastore.NewMapDatastore()) - pubHost := test.MkTestHost(t) + pubHost, privKey := test.MkTestHostPK(t) lsys := test.MkLinkSystem(ds) - pub, err := dtsync.NewPublisher(pubHost, ds, lsys, testTopic) + pub, err := ipnisync.NewPublisher(lsys, privKey, ipnisync.WithStreamHost(pubHost)) require.NoError(t, err) head := ll.Build(t, lsys) @@ -64,8 +64,6 @@ func TestScopedBlockHook(t *testing.T) { subDS := dssync.MutexWrap(datastore.NewMapDatastore()) subLsys := test.MkLinkSystem(subDS) - require.NoError(t, test.WaitForP2PPublisher(pub, subHost, testTopic)) - var calledGeneralBlockHookTimes int64 sub, err := dagsync.NewSubscriber(subHost, subDS, subLsys, testTopic, dagsync.BlockHook(func(i peer.ID, c cid.Cid, _ dagsync.SegmentSyncActions) { @@ -114,9 +112,9 @@ func TestSyncedCidsReturned(t *testing.T) { err := quick.Check(func(ll llBuilder) bool { return t.Run("Quickcheck", func(t *testing.T) { ds := dssync.MutexWrap(datastore.NewMapDatastore()) - pubHost := test.MkTestHost(t) + pubHost, privKey := test.MkTestHostPK(t) lsys := test.MkLinkSystem(ds) - pub, err := dtsync.NewPublisher(pubHost, ds, lsys, testTopic) + pub, err := ipnisync.NewPublisher(lsys, privKey, ipnisync.WithStreamHost(pubHost)) require.NoError(t, err) head := ll.Build(t, lsys) @@ -131,8 +129,6 @@ func TestSyncedCidsReturned(t *testing.T) { subDS := dssync.MutexWrap(datastore.NewMapDatastore()) subLsys := test.MkLinkSystem(subDS) - require.NoError(t, test.WaitForP2PPublisher(pub, subHost, testTopic)) - sub, err := dagsync.NewSubscriber(subHost, subDS, subLsys, testTopic, dagsync.StrictAdsSelector(false)) require.NoError(t, err) @@ -175,12 +171,11 @@ func TestConcurrentSync(t *testing.T) { for i := 0; i < publisherCount; i++ { ds := dssync.MutexWrap(datastore.NewMapDatastore()) - pubHost := test.MkTestHost(t) + pubHost, privKey := test.MkTestHostPK(t) lsys := test.MkLinkSystem(ds) - pub, err := dtsync.NewPublisher(pubHost, ds, lsys, testTopic) + pub, err := ipnisync.NewPublisher(lsys, privKey, ipnisync.WithStreamHost(pubHost)) require.NoError(t, err) - require.NoError(t, test.WaitForP2PPublisher(pub, subHost, testTopic)) publishers = append(publishers, pubMeta{pub, pubHost}) @@ -391,11 +386,11 @@ func TestRoundTripSimple(t *testing.T) { func TestRoundTrip(t *testing.T) { // Init dagsync publisher and subscriber srcStore1 := dssync.MutexWrap(datastore.NewMapDatastore()) - srcHost1 := test.MkTestHost(t) + srcHost1, privKey1 := test.MkTestHostPK(t) srcLnkS1 := test.MkLinkSystem(srcStore1) srcStore2 := dssync.MutexWrap(datastore.NewMapDatastore()) - srcHost2 := test.MkTestHost(t) + srcHost2, privKey2 := test.MkTestHostPK(t) srcLnkS2 := test.MkLinkSystem(srcStore2) dstStore := dssync.MutexWrap(datastore.NewMapDatastore()) @@ -407,14 +402,14 @@ func TestRoundTrip(t *testing.T) { p2pSender1, err := p2psender.New(nil, "", p2psender.WithTopic(topics[0])) require.NoError(t, err) - pub1, err := dtsync.NewPublisher(srcHost1, srcStore1, srcLnkS1, "") + pub1, err := ipnisync.NewPublisher(srcLnkS1, privKey1, ipnisync.WithStreamHost(srcHost1)) require.NoError(t, err) defer pub1.Close() p2pSender2, err := p2psender.New(nil, "", p2psender.WithTopic(topics[1])) require.NoError(t, err) - pub2, err := dtsync.NewPublisher(srcHost2, srcStore2, srcLnkS2, "") + pub2, err := ipnisync.NewPublisher(srcLnkS2, privKey2, ipnisync.WithStreamHost(srcHost2)) require.NoError(t, err) defer pub2.Close() @@ -605,16 +600,16 @@ func TestSyncFinishedAlwaysDelivered(t *testing.T) { func TestMaxAsyncSyncs(t *testing.T) { // Create two publishers srcStore1 := dssync.MutexWrap(datastore.NewMapDatastore()) - srcHost1 := test.MkTestHost(t) + srcHost1, privKey1 := test.MkTestHostPK(t) srcLnkS1 := test.MkLinkSystem(srcStore1) - pub1, err := dtsync.NewPublisher(srcHost1, srcStore1, srcLnkS1, "") + pub1, err := ipnisync.NewPublisher(srcLnkS1, privKey1, ipnisync.WithStreamHost(srcHost1)) require.NoError(t, err) defer pub1.Close() srcStore2 := dssync.MutexWrap(datastore.NewMapDatastore()) - srcHost2 := test.MkTestHost(t) + srcHost2, privKey2 := test.MkTestHostPK(t) srcLnkS2 := test.MkLinkSystem(srcStore2) - pub2, err := dtsync.NewPublisher(srcHost2, srcStore2, srcLnkS2, "") + pub2, err := ipnisync.NewPublisher(srcLnkS2, privKey2, ipnisync.WithStreamHost(srcHost2)) require.NoError(t, err) defer pub2.Close() @@ -851,6 +846,7 @@ func TestIdleHandlerCleaner(t *testing.T) { type dagsyncPubSubBuilder struct { IsHttp bool + IsDtSync bool P2PAnnounce bool } @@ -884,13 +880,15 @@ func (b dagsyncPubSubBuilder) Build(t *testing.T, topicName string, pubSys hostS var pub dagsync.Publisher var err error if b.IsHttp { - pub, err = ipnisync.NewPublisher("127.0.0.1:0", pubSys.lsys, pubSys.privKey, ipnisync.WithHeadTopic(topicName)) + pub, err = ipnisync.NewPublisher(pubSys.lsys, pubSys.privKey, ipnisync.WithHeadTopic(topicName), ipnisync.WithHTTPListenAddrs("127.0.0.1:0")) require.NoError(t, err) - require.NoError(t, test.WaitForHttpPublisher(pub)) - } else { + } else if b.IsDtSync { pub, err = dtsync.NewPublisher(pubSys.host, pubSys.ds, pubSys.lsys, topicName) require.NoError(t, err) require.NoError(t, test.WaitForP2PPublisher(pub, subSys.host, topicName)) + } else { + pub, err = ipnisync.NewPublisher(pubSys.lsys, pubSys.privKey, ipnisync.WithStreamHost(pubSys.host), ipnisync.WithHeadTopic(topicName)) + require.NoError(t, err) } subOpts = append(subOpts, dagsync.StrictAdsSelector(false)) diff --git a/dagsync/sync_test.go b/dagsync/sync_test.go index 6666bd3..00c7ffa 100644 --- a/dagsync/sync_test.go +++ b/dagsync/sync_test.go @@ -16,6 +16,7 @@ import ( "github.com/ipni/go-libipni/announce/p2psender" "github.com/ipni/go-libipni/dagsync" "github.com/ipni/go-libipni/dagsync/dtsync" + "github.com/ipni/go-libipni/dagsync/ipnisync" "github.com/ipni/go-libipni/dagsync/test" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/require" @@ -24,7 +25,7 @@ import ( func TestLatestSyncSuccess(t *testing.T) { srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) dstStore := dssync.MutexWrap(datastore.NewMapDatastore()) - srcHost := test.MkTestHost(t) + srcHost, srcPrivKey := test.MkTestHostPK(t) srcLnkS := test.MkLinkSystem(srcStore) dstHost := test.MkTestHost(t) @@ -37,7 +38,7 @@ func TestLatestSyncSuccess(t *testing.T) { p2pSender, err := p2psender.New(nil, "", p2psender.WithTopic(topics[0])) require.NoError(t, err) - pub, err := dtsync.NewPublisher(srcHost, srcStore, srcLnkS, testTopic) + pub, err := ipnisync.NewPublisher(srcLnkS, srcPrivKey, ipnisync.WithStreamHost(srcHost)) require.NoError(t, err) defer pub.Close() @@ -47,8 +48,6 @@ func TestLatestSyncSuccess(t *testing.T) { require.NoError(t, err) defer sub.Close() - require.NoError(t, test.WaitForP2PPublisher(pub, dstHost, topics[0].String())) - watcher, cncl := sub.OnSyncFinished() defer cncl() @@ -67,7 +66,7 @@ func TestSyncFn(t *testing.T) { t.Parallel() srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) dstStore := dssync.MutexWrap(datastore.NewMapDatastore()) - srcHost := test.MkTestHost(t) + srcHost, srcPrivKey := test.MkTestHostPK(t) srcLnkS := test.MkLinkSystem(srcStore) dstHost := test.MkTestHost(t) @@ -80,7 +79,7 @@ func TestSyncFn(t *testing.T) { p2pSender, err := p2psender.New(nil, "", p2psender.WithTopic(topics[0])) require.NoError(t, err) - pub, err := dtsync.NewPublisher(srcHost, srcStore, srcLnkS, testTopic) + pub, err := ipnisync.NewPublisher(srcLnkS, srcPrivKey, ipnisync.WithStreamHost(srcHost)) require.NoError(t, err) defer pub.Close() @@ -104,8 +103,6 @@ func TestSyncFn(t *testing.T) { // Store the whole chain in source node chainLnks := test.MkChain(srcLnkS, true) - require.NoError(t, test.WaitForP2PPublisher(pub, dstHost, topics[0].String())) - watcher, cancelWatcher := sub.OnSyncFinished() defer cancelWatcher() @@ -184,7 +181,7 @@ func TestPartialSync(t *testing.T) { srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) testStore := dssync.MutexWrap(datastore.NewMapDatastore()) dstStore := dssync.MutexWrap(datastore.NewMapDatastore()) - srcHost := test.MkTestHost(t) + srcHost, srcPrivKey := test.MkTestHostPK(t) srcLnkS := test.MkLinkSystem(srcStore) testLnkS := test.MkLinkSystem(testStore) @@ -200,7 +197,7 @@ func TestPartialSync(t *testing.T) { p2pSender, err := p2psender.New(nil, "", p2psender.WithTopic(topics[0])) require.NoError(t, err) - pub, err := dtsync.NewPublisher(srcHost, srcStore, srcLnkS, testTopic) + pub, err := ipnisync.NewPublisher(srcLnkS, srcPrivKey, ipnisync.WithStreamHost(srcHost)) require.NoError(t, err) defer pub.Close() test.MkChain(srcLnkS, true) @@ -217,8 +214,6 @@ func TestPartialSync(t *testing.T) { err = srcHost.Connect(context.Background(), dstHost.Peerstore().PeerInfo(dstHost.ID())) require.NoError(t, err) - require.NoError(t, test.WaitForP2PPublisher(pub, dstHost, topics[0].String())) - watcher, cncl := sub.OnSyncFinished() defer cncl() @@ -250,7 +245,7 @@ func TestStepByStepSync(t *testing.T) { dstStore := dssync.MutexWrap(datastore.NewMapDatastore()) srcLnkS := test.MkLinkSystem(srcStore) - srcHost := test.MkTestHost(t) + srcHost, srcPrivKey := test.MkTestHostPK(t) dstHost := test.MkTestHost(t) topics := test.WaitForMeshWithMessage(t, testTopic, srcHost, dstHost) @@ -260,7 +255,7 @@ func TestStepByStepSync(t *testing.T) { p2pSender, err := p2psender.New(nil, "", p2psender.WithTopic(topics[0])) require.NoError(t, err) - pub, err := dtsync.NewPublisher(srcHost, srcStore, srcLnkS, testTopic) + pub, err := ipnisync.NewPublisher(srcLnkS, srcPrivKey, ipnisync.WithStreamHost(srcHost)) require.NoError(t, err) defer pub.Close() @@ -270,8 +265,6 @@ func TestStepByStepSync(t *testing.T) { require.NoError(t, err) defer sub.Close() - require.NoError(t, test.WaitForP2PPublisher(pub, dstHost, topics[0].String())) - watcher, cncl := sub.OnSyncFinished() defer cncl() @@ -293,9 +286,9 @@ func TestLatestSyncFailure(t *testing.T) { t.Parallel() srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) dstStore := dssync.MutexWrap(datastore.NewMapDatastore()) - srcHost := test.MkTestHost(t) + srcHost, srcPrivKey := test.MkTestHostPK(t) srcLnkS := test.MkLinkSystem(srcStore) - pub, err := dtsync.NewPublisher(srcHost, srcStore, srcLnkS, testTopic) + pub, err := ipnisync.NewPublisher(srcLnkS, srcPrivKey, ipnisync.WithStreamHost(srcHost)) require.NoError(t, err) defer pub.Close() @@ -320,8 +313,6 @@ func TestLatestSyncFailure(t *testing.T) { err = sub.SetLatestSync(srcHost.ID(), chainLnks[3].(cidlink.Link).Cid) require.NoError(t, err) - require.NoError(t, test.WaitForP2PPublisher(pub, dstHost, testTopic)) - watcher, cncl := sub.OnSyncFinished() t.Log("Testing sync fail when the other end does not have the data") @@ -345,50 +336,125 @@ func TestLatestSyncFailure(t *testing.T) { require.NoError(t, err) } -func TestAnnounce(t *testing.T) { - srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) +func TestSyncOnAnnounceDataTransfer(t *testing.T) { dstStore := dssync.MutexWrap(datastore.NewMapDatastore()) + dstHost := test.MkTestHost(t) + dstLnkS := test.MkLinkSystem(dstStore) + + sub, err := dagsync.NewSubscriber(dstHost, dstStore, dstLnkS, testTopic, + dagsync.RecvAnnounce(), dagsync.StrictAdsSelector(false)) + require.NoError(t, err) + defer sub.Close() + + watcher, cncl := sub.OnSyncFinished() + defer cncl() + srcHost := test.MkTestHost(t) + srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) srcLnkS := test.MkLinkSystem(srcStore) - dstHost := test.MkTestHost(t) + pub, err := dtsync.NewPublisher(srcHost, srcStore, srcLnkS, testTopic) + require.NoError(t, err) + defer pub.Close() + require.NoError(t, test.WaitForP2PPublisher(pub, dstHost, testTopic)) srcHost.Peerstore().AddAddrs(dstHost.ID(), dstHost.Addrs(), time.Hour) dstHost.Peerstore().AddAddrs(srcHost.ID(), srcHost.Addrs(), time.Hour) + + // Store the whole chain in source node + chainLnks := test.MkChain(srcLnkS, true) + + t.Log("Testing announce-sync with dtsync publisher at:", pub.Addrs()) + pubInfo := peer.AddrInfo{ + ID: pub.ID(), + Addrs: pub.Addrs(), + } + err = newAnnounceTest(pub, sub, dstStore, watcher, pubInfo, chainLnks[2], chainLnks[2].(cidlink.Link).Cid) + require.NoError(t, err) + err = newAnnounceTest(pub, sub, dstStore, watcher, pubInfo, chainLnks[1], chainLnks[1].(cidlink.Link).Cid) + require.NoError(t, err) + err = newAnnounceTest(pub, sub, dstStore, watcher, pubInfo, chainLnks[0], chainLnks[0].(cidlink.Link).Cid) + require.NoError(t, err) +} + +func TestSyncOnAnnounceIPNI(t *testing.T) { + dstStore := dssync.MutexWrap(datastore.NewMapDatastore()) + dstHost := test.MkTestHost(t) dstLnkS := test.MkLinkSystem(dstStore) - pub, err := dtsync.NewPublisher(srcHost, srcStore, srcLnkS, testTopic) + sub, err := dagsync.NewSubscriber(dstHost, dstStore, dstLnkS, testTopic, + dagsync.RecvAnnounce(), dagsync.StrictAdsSelector(false)) + require.NoError(t, err) + defer sub.Close() + + watcher, cncl := sub.OnSyncFinished() + defer cncl() + + srcHost, srcPrivKey := test.MkTestHostPK(t) + srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) + srcLnkS := test.MkLinkSystem(srcStore) + pub, err := ipnisync.NewPublisher(srcLnkS, srcPrivKey, ipnisync.WithStreamHost(srcHost), ipnisync.WithHeadTopic(testTopic)) require.NoError(t, err) defer pub.Close() + srcHost.Peerstore().AddAddrs(dstHost.ID(), dstHost.Addrs(), time.Hour) + dstHost.Peerstore().AddAddrs(srcHost.ID(), srcHost.Addrs(), time.Hour) + + // Store the whole chain in source node + chainLnks := test.MkChain(srcLnkS, true) + + t.Log("Testing announce-sync with libp2phttp publisher at:", pub.Addrs()) + pubInfo := peer.AddrInfo{ + ID: pub.ID(), + Addrs: pub.Addrs(), + } + err = newAnnounceTest(pub, sub, dstStore, watcher, pubInfo, chainLnks[2], chainLnks[2].(cidlink.Link).Cid) + require.NoError(t, err) + err = newAnnounceTest(pub, sub, dstStore, watcher, pubInfo, chainLnks[1], chainLnks[1].(cidlink.Link).Cid) + require.NoError(t, err) + err = newAnnounceTest(pub, sub, dstStore, watcher, pubInfo, chainLnks[0], chainLnks[0].(cidlink.Link).Cid) + require.NoError(t, err) +} + +func TestSyncOnAnnounceHTTP(t *testing.T) { + dstStore := dssync.MutexWrap(datastore.NewMapDatastore()) + dstHost := test.MkTestHost(t) + dstLnkS := test.MkLinkSystem(dstStore) + sub, err := dagsync.NewSubscriber(dstHost, dstStore, dstLnkS, testTopic, dagsync.RecvAnnounce(), dagsync.StrictAdsSelector(false)) require.NoError(t, err) defer sub.Close() - require.NoError(t, test.WaitForP2PPublisher(pub, dstHost, testTopic)) - watcher, cncl := sub.OnSyncFinished() defer cncl() + _, srcPrivKey := test.MkTestHostPK(t) + srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) + srcLnkS := test.MkLinkSystem(srcStore) + pub, err := ipnisync.NewPublisher(srcLnkS, srcPrivKey, ipnisync.WithHTTPListenAddrs("http://127.0.0.1:0"), ipnisync.WithHeadTopic(testTopic)) + require.NoError(t, err) + defer pub.Close() + // Store the whole chain in source node chainLnks := test.MkChain(srcLnkS, true) - srcHostInfo := peer.AddrInfo{ - ID: srcHost.ID(), - Addrs: srcHost.Addrs(), + t.Log("Testing announce-sync with HTTP publisher at:", pub.Addrs()) + pubInfo := peer.AddrInfo{ + ID: pub.ID(), + Addrs: pub.Addrs(), } - err = newAnnounceTest(pub, sub, dstStore, watcher, srcHostInfo, chainLnks[2], chainLnks[2].(cidlink.Link).Cid) + err = newAnnounceTest(pub, sub, dstStore, watcher, pubInfo, chainLnks[2], chainLnks[2].(cidlink.Link).Cid) require.NoError(t, err) - err = newAnnounceTest(pub, sub, dstStore, watcher, srcHostInfo, chainLnks[1], chainLnks[1].(cidlink.Link).Cid) + err = newAnnounceTest(pub, sub, dstStore, watcher, pubInfo, chainLnks[1], chainLnks[1].(cidlink.Link).Cid) require.NoError(t, err) - err = newAnnounceTest(pub, sub, dstStore, watcher, srcHostInfo, chainLnks[0], chainLnks[0].(cidlink.Link).Cid) + err = newAnnounceTest(pub, sub, dstStore, watcher, pubInfo, chainLnks[0], chainLnks[0].(cidlink.Link).Cid) require.NoError(t, err) } func TestCancelDeadlock(t *testing.T) { srcStore := dssync.MutexWrap(datastore.NewMapDatastore()) dstStore := dssync.MutexWrap(datastore.NewMapDatastore()) - srcHost := test.MkTestHost(t) + srcHost, srcPrivKey := test.MkTestHostPK(t) srcLnkS := test.MkLinkSystem(srcStore) dstHost := test.MkTestHost(t) @@ -396,7 +462,7 @@ func TestCancelDeadlock(t *testing.T) { dstHost.Peerstore().AddAddrs(srcHost.ID(), srcHost.Addrs(), time.Hour) dstLnkS := test.MkLinkSystem(dstStore) - pub, err := dtsync.NewPublisher(srcHost, srcStore, srcLnkS, testTopic) + pub, err := ipnisync.NewPublisher(srcLnkS, srcPrivKey, ipnisync.WithStreamHost(srcHost)) require.NoError(t, err) defer pub.Close() @@ -404,8 +470,6 @@ func TestCancelDeadlock(t *testing.T) { require.NoError(t, err) defer sub.Close() - require.NoError(t, test.WaitForP2PPublisher(pub, dstHost, testTopic)) - watcher, cncl := sub.OnSyncFinished() // Store the whole chain in source node diff --git a/dagsync/test/util.go b/dagsync/test/util.go index da66c96..154f972 100644 --- a/dagsync/test/util.go +++ b/dagsync/test/util.go @@ -3,6 +3,7 @@ package test import ( "bytes" "context" + "crypto/rand" "errors" "fmt" "io" @@ -18,13 +19,14 @@ import ( "github.com/ipld/go-ipld-prime/fluent" cidlink "github.com/ipld/go-ipld-prime/linking/cid" basicnode "github.com/ipld/go-ipld-prime/node/basic" + "github.com/ipni/go-libipni/dagsync/dtsync/head" "github.com/ipni/go-libipni/dagsync/ipnisync" - "github.com/ipni/go-libipni/dagsync/p2p/protocol/head" "github.com/ipni/go-libipni/ingest/schema" "github.com/ipni/go-libipni/maurl" "github.com/ipni/go-libipni/test" "github.com/libp2p/go-libp2p" pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" @@ -237,6 +239,12 @@ func MkTestHost(t *testing.T, options ...libp2p.Option) host.Host { return h } +func MkTestHostPK(t *testing.T) (host.Host, crypto.PrivKey) { + privKey, _, err := crypto.GenerateEd25519Key(rand.Reader) + require.NoError(t, err) + return MkTestHost(t, libp2p.Identity(privKey)), privKey +} + // Return the chain with all nodes or just half of it for testing func MkChain(lsys ipld.LinkSystem, full bool) []ipld.Link { out := make([]ipld.Link, 4) @@ -319,7 +327,7 @@ func WaitForHttpPublisher(publisher TestPublisher) error { if err != nil { return err } - headURL.Path = path.Join(headURL.Path, ipnisync.IpniPath, "head") + headURL.Path = path.Join(headURL.Path, ipnisync.IPNIPath, "head") ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() diff --git a/go.mod b/go.mod index f094f84..e11f16c 100644 --- a/go.mod +++ b/go.mod @@ -1,30 +1,30 @@ module github.com/ipni/go-libipni -go 1.19 +go 1.20 require ( github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc7 github.com/gammazero/channelqueue v0.2.1 github.com/hashicorp/go-multierror v1.1.1 + github.com/hashicorp/go-retryablehttp v0.7.4 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 - github.com/ipfs/go-graphsync v0.14.7 - github.com/ipfs/go-ipld-format v0.3.0 + github.com/ipfs/go-graphsync v0.14.8 + github.com/ipfs/go-ipld-format v0.4.0 github.com/ipfs/go-log/v2 v2.5.1 - github.com/ipld/go-ipld-prime v0.20.0 - github.com/libp2p/go-libp2p v0.29.2 - github.com/libp2p/go-libp2p-gostream v0.6.0 + github.com/ipld/go-ipld-prime v0.21.0 + github.com/libp2p/go-libp2p v0.31.0 github.com/libp2p/go-libp2p-pubsub v0.9.3 github.com/libp2p/go-msgio v0.3.0 github.com/mr-tron/base58 v1.2.0 - github.com/multiformats/go-multiaddr v0.10.1 + github.com/multiformats/go-multiaddr v0.11.0 github.com/multiformats/go-multicodec v0.9.0 github.com/multiformats/go-multihash v0.2.3 github.com/multiformats/go-multistream v0.4.1 github.com/multiformats/go-varint v0.0.7 github.com/stretchr/testify v1.8.4 github.com/whyrusleeping/cbor-gen v0.0.0-20230418232409-daab9ece03a0 - golang.org/x/crypto v0.11.0 + golang.org/x/crypto v0.12.0 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 google.golang.org/protobuf v1.30.0 ) @@ -50,28 +50,29 @@ require ( github.com/francoispqt/gojay v1.2.13 // indirect github.com/gammazero/deque v0.2.1 // indirect github.com/go-logr/logr v1.2.4 // indirect - github.com/go-logr/stdr v1.2.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/gopacket v1.1.19 // indirect - github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 // indirect + github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/hannahhoward/cbor-gen-for v0.0.0-20230214144701-5d17c9d5243c // indirect github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // indirect github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect github.com/huin/goupnp v1.2.0 // indirect - github.com/ipfs/go-block-format v0.0.3 // indirect - github.com/ipfs/go-ipfs-pq v0.0.2 // indirect + github.com/ipfs/go-block-format v0.1.2 // indirect + github.com/ipfs/go-ipfs-pq v0.0.3 // indirect github.com/ipfs/go-ipfs-util v0.0.2 // indirect - github.com/ipfs/go-ipld-cbor v0.0.5 // indirect + github.com/ipfs/go-ipld-cbor v0.0.6 // indirect github.com/ipfs/go-log v1.0.5 // indirect - github.com/ipfs/go-peertaskqueue v0.8.0 // indirect - github.com/ipld/go-codec-dagpb v1.5.0 // indirect + github.com/ipfs/go-peertaskqueue v0.8.1 // indirect + github.com/ipld/go-codec-dagpb v1.6.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect @@ -85,7 +86,7 @@ require ( github.com/libp2p/go-libp2p-asn-util v0.3.0 // indirect github.com/libp2p/go-nat v0.2.0 // indirect github.com/libp2p/go-netroute v0.2.1 // indirect - github.com/libp2p/go-reuseport v0.3.0 // indirect + github.com/libp2p/go-reuseport v0.4.0 // indirect github.com/libp2p/go-yamux/v4 v4.0.1 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-isatty v0.0.19 // indirect @@ -100,7 +101,7 @@ require ( github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect github.com/onsi/ginkgo/v2 v2.11.0 // indirect - github.com/opencontainers/runtime-spec v1.0.2 // indirect + github.com/opencontainers/runtime-spec v1.1.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pkg/errors v0.9.1 // indirect @@ -111,9 +112,8 @@ require ( github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/quic-go/qpack v0.4.0 // indirect - github.com/quic-go/qtls-go1-19 v0.3.3 // indirect - github.com/quic-go/qtls-go1-20 v0.2.3 // indirect - github.com/quic-go/quic-go v0.36.4 // indirect + github.com/quic-go/qtls-go1-20 v0.3.3 // indirect + github.com/quic-go/quic-go v0.38.1 // indirect github.com/quic-go/webtransport-go v0.5.3 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect github.com/russross/blackfriday/v2 v2.0.1 // indirect @@ -121,20 +121,20 @@ require ( github.com/smartystreets/assertions v1.13.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/urfave/cli/v2 v2.0.0 // indirect - go.opentelemetry.io/otel v1.3.0 // indirect - go.opentelemetry.io/otel/trace v1.3.0 // indirect + go.opentelemetry.io/otel v1.13.0 // indirect + go.opentelemetry.io/otel/trace v1.13.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/dig v1.17.0 // indirect go.uber.org/fx v1.20.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.24.0 // indirect - golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect + go.uber.org/zap v1.25.0 // indirect + golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.12.0 // indirect + golang.org/x/net v0.14.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.10.0 // indirect - golang.org/x/text v0.11.0 // indirect - golang.org/x/tools v0.11.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect + golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect ) diff --git a/go.sum b/go.sum index db430d3..1ea0ce7 100644 --- a/go.sum +++ b/go.sum @@ -90,7 +90,7 @@ github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= -github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gammazero/channelqueue v0.2.1 h1:AcK6wnLrj8koTTn3RxjRCyfmS677TjhIZb1FSMi14qc= github.com/gammazero/channelqueue v0.2.1/go.mod h1:824o5HHE+yO1xokh36BIuSv8YWwXW0364ku91eRMFS4= @@ -102,12 +102,11 @@ github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclK github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/stdr v1.2.0 h1:j4LrlVXgrbIWO83mmQUnK0Hi+YnbD+vzrE1z/EphbFE= -github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= @@ -146,7 +145,6 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -156,8 +154,8 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 h1:n6vlPhxsA+BW/XsS5+uqi7GyzaLa5MH7qlSLBZtRdiA= -github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= +github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b h1:h9U78+dx9a4BKdQkBBos92HalKpaGKHrp+3Uo6yTodo= +github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -182,28 +180,33 @@ github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e/go.mod h1:I github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= +github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= -github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU= -github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4= +github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= -github.com/ipfs/go-block-format v0.0.3 h1:r8t66QstRp/pd/or4dpnbVfXT5Gt7lOqRvC+/dDTpMc= -github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= -github.com/ipfs/go-blockservice v0.3.0 h1:cDgcZ+0P0Ih3sl8+qjFr2sVaMdysg/YZpLj5WJ8kiiw= +github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+DyPUaJo= +github.com/ipfs/go-block-format v0.1.2/go.mod h1:mACVcrxarQKstUU3Yf/RdwbC4DzPV6++rO2a3d+a/KE= +github.com/ipfs/go-blockservice v0.5.0 h1:B2mwhhhVQl2ntW2EIpaWPwSCxSuqr5fFA93Ms4bYLEY= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.4-0.20191112011718-79e75dffeb10/go.mod h1:/BYOuUoxkE+0f6tGzlzMvycuN+5l35VOR4Bpg2sCmds= github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= -github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= @@ -213,30 +216,30 @@ github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0M github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= -github.com/ipfs/go-graphsync v0.14.7 h1:V90NORSdCpUHAgqQhApU/bmPSLOnwtSHM2v7R90k9Do= -github.com/ipfs/go-graphsync v0.14.7/go.mod h1:yT0AfjFgicOoWdAlUJ96tQ5AkuGI4r1taIQX/aHbBQo= +github.com/ipfs/go-graphsync v0.14.8 h1:NFFHquTNnwPi05tJhdpPj4CJMnqRBLxpZd+IfPRauf4= +github.com/ipfs/go-graphsync v0.14.8/go.mod h1:qyHjUvHey6EfKUDMQPwCuVkMOurRG3hcjRm+FaVP6bE= github.com/ipfs/go-ipfs-blockstore v1.2.0 h1:n3WTeJ4LdICWs/0VSfjHrlqpPpl6MZ+ySd3j8qz0ykw= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-ds-help v1.1.0 h1:yLE2w9RAsl31LtfMt91tRZcrx+e61O5mDxFRR994w4Q= -github.com/ipfs/go-ipfs-exchange-interface v0.1.0 h1:TiMekCrOGQuWYtZO3mf4YJXDIdNgnKWZ9IE3fGlnWfo= -github.com/ipfs/go-ipfs-exchange-offline v0.2.0 h1:2PF4o4A7W656rC0RxuhUace997FTcDTcIQ6NoEtyjAI= +github.com/ipfs/go-ipfs-exchange-interface v0.2.0 h1:8lMSJmKogZYNo2jjhUs0izT+dck05pqUw4mWNW9Pw6Y= +github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= -github.com/ipfs/go-ipfs-pq v0.0.2 h1:e1vOOW6MuOwG2lqxcLA+wEn93i/9laCY8sXAw76jFOY= -github.com/ipfs/go-ipfs-pq v0.0.2/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= +github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= +github.com/ipfs/go-ipfs-pq v0.0.3/go.mod h1:btNw5hsHBpRcSSgZtiNm/SLj5gYIZ18AKtv3kERkRb4= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= -github.com/ipfs/go-ipld-cbor v0.0.5 h1:ovz4CHKogtG2KB/h1zUp5U0c/IzZrL435rCh5+K/5G8= -github.com/ipfs/go-ipld-cbor v0.0.5/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4= +github.com/ipfs/go-ipld-cbor v0.0.6 h1:pYuWHyvSpIsOOLw4Jy7NbBkCyzLDcl64Bf/LZW7eBQ0= +github.com/ipfs/go-ipld-cbor v0.0.6/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA= github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= -github.com/ipfs/go-ipld-format v0.3.0 h1:Mwm2oRLzIuUwEPewWAWyMuuBQUsn3awfFEYVb8akMOQ= -github.com/ipfs/go-ipld-format v0.3.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= -github.com/ipfs/go-ipld-legacy v0.1.0 h1:wxkkc4k8cnvIGIjPO0waJCe7SHEyFgl+yQdafdjGrpA= -github.com/ipfs/go-libipfs v0.1.0 h1:I6CrHHp4cIiqsWJPVU3QBH4BZrRWSljS2aAbA3Eg9AY= +github.com/ipfs/go-ipld-format v0.4.0 h1:yqJSaJftjmjc9jEOFYlpkwOLVKv68OD27jFLlSghBlQ= +github.com/ipfs/go-ipld-format v0.4.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= +github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2cdcc= +github.com/ipfs/go-libipfs v0.6.0 h1:3FuckAJEm+zdHbHbf6lAyk0QUzc45LsFcGw102oBCZM= github.com/ipfs/go-log v1.0.0/go.mod h1:JO7RzlMK6rA+CIxFMLOuB6Wf5b81GDiKElL7UPSIKjA= github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I= github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8= @@ -245,17 +248,17 @@ github.com/ipfs/go-log/v2 v2.0.1/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBW github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= -github.com/ipfs/go-merkledag v0.8.1 h1:N3yrqSre/ffvdwtHL4MXy0n7XH+VzN8DlzDrJySPa94= +github.com/ipfs/go-merkledag v0.10.0 h1:IUQhj/kzTZfam4e+LnaEpoiZ9vZF6ldimVlby+6OXL4= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= -github.com/ipfs/go-peertaskqueue v0.8.0 h1:JyNO144tfu9bx6Hpo119zvbEL9iQ760FHOiJYsUjqaU= -github.com/ipfs/go-peertaskqueue v0.8.0/go.mod h1:cz8hEnnARq4Du5TGqiWKgMr/BOSQ5XOgMOh1K5YYKKM= -github.com/ipfs/go-unixfs v0.4.3 h1:EdDc1sNZNFDUlo4UrVAvvAofVI5EwTnKu8Nv8mgXkWQ= -github.com/ipfs/go-unixfsnode v1.5.2 h1:CvsiTt58W2uR5dD8bqQv+aAY0c1qolmXmSyNbPHYiew= -github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= -github.com/ipld/go-codec-dagpb v1.5.0 h1:RspDRdsJpLfgCI0ONhTAnbHdySGD4t+LHSPK4X1+R0k= -github.com/ipld/go-codec-dagpb v1.5.0/go.mod h1:0yRIutEFD8o1DGVqw4RSHh+BUTlJA9XWldxaaWR/o4g= -github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g= -github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M= +github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= +github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= +github.com/ipfs/go-unixfs v0.4.4 h1:D/dLBOJgny5ZLIur2vIXVQVW0EyDHdOMBDEhgHrt6rY= +github.com/ipfs/go-unixfsnode v1.7.4 h1:iLvKyAVKUYOIAW2t4kDYqsT7VLGj31eXJE2aeqGfbwA= +github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= +github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc= +github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s= +github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E= +github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= @@ -299,12 +302,10 @@ github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38y github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.29.2 h1:uPw/c8hOxoLP/KhFnzlc5Ejqf+OmAL1dwIsqE31WBtY= -github.com/libp2p/go-libp2p v0.29.2/go.mod h1:OU7nSq0aEZMsV2wY8nXn1+XNNt9q2UiR8LjW3Kmp2UE= +github.com/libp2p/go-libp2p v0.31.0 h1:LFShhP8F6xthWiBBq3euxbKjZsoRajVEyBS9snfHxYg= +github.com/libp2p/go-libp2p v0.31.0/go.mod h1:W/FEK1c/t04PbRH3fA9i5oucu5YcgrG0JVoBWT1B7Eg= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= -github.com/libp2p/go-libp2p-gostream v0.6.0 h1:QfAiWeQRce6pqnYfmIVWJFXNdDyfiR/qkCnjyaZUPYU= -github.com/libp2p/go-libp2p-gostream v0.6.0/go.mod h1:Nywu0gYZwfj7Jc91PQvbGU8dIpqbQQkjWgDuOrFaRdA= github.com/libp2p/go-libp2p-pubsub v0.9.3 h1:ihcz9oIBMaCK9kcx+yHWm3mLAFBMAUsM4ux42aikDxo= github.com/libp2p/go-libp2p-pubsub v0.9.3/go.mod h1:RYA7aM9jIic5VV47WXu4GkcRxRhrdElWf8xtyli+Dzc= github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= @@ -314,8 +315,8 @@ github.com/libp2p/go-nat v0.2.0 h1:Tyz+bUFAYqGyJ/ppPPymMGbIgNRH+WqC5QrT5fKrrGk= github.com/libp2p/go-nat v0.2.0/go.mod h1:3MJr+GRpRkyT65EpVPBstXLvOlAPzUVlG6Pwg9ohLJk= github.com/libp2p/go-netroute v0.2.1 h1:V8kVrpD8GK0Riv15/7VN6RbUQ3URNZVosw7H2v9tksU= github.com/libp2p/go-netroute v0.2.1/go.mod h1:hraioZr0fhBjG0ZRXJJ6Zj2IVEVNx6tDTFQfSmcq7mQ= -github.com/libp2p/go-reuseport v0.3.0 h1:iiZslO5byUYZEg9iCwJGf5h+sf1Agmqx2V2FDjPyvUw= -github.com/libp2p/go-reuseport v0.3.0/go.mod h1:laea40AimhtfEqysZ71UpYj4S+R9VpH8PgqLo7L+SwI= +github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s= +github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU= github.com/libp2p/go-yamux/v4 v4.0.1 h1:FfDR4S1wj6Bw2Pqbc8Uz7pCxeRBPbwsBbEdfwiCypkQ= github.com/libp2p/go-yamux/v4 v4.0.1/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= @@ -359,8 +360,8 @@ github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9 github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= -github.com/multiformats/go-multiaddr v0.10.1 h1:HghtFrWyZEPrpTvgAMFJi6gFdgHfs2cb0pyfDsk+lqU= -github.com/multiformats/go-multiaddr v0.10.1/go.mod h1:jLEZsA61rwWNZQTHHnqq2HNa+4os/Hz54eqiRnsRqYQ= +github.com/multiformats/go-multiaddr v0.11.0 h1:XqGyJ8ufbCE0HmTDwx2kPdsrQ36AGPZNZX6s6xfJH10= +github.com/multiformats/go-multiaddr v0.11.0/go.mod h1:gWUm0QLR4thQ6+ZF6SXUw8YjtwQSPapICM+NmCkxHSM= github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= @@ -376,7 +377,6 @@ github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= -github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-multistream v0.4.1 h1:rFy0Iiyn3YT0asivDUIR05leAdwZq3de4741sbiSdfo= @@ -390,8 +390,9 @@ github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= -github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= +github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= @@ -422,12 +423,10 @@ github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJf github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= -github.com/quic-go/qtls-go1-19 v0.3.3 h1:wznEHvJwd+2X3PqftRha0SUKmGsnb6dfArMhy9PeJVE= -github.com/quic-go/qtls-go1-19 v0.3.3/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI= -github.com/quic-go/qtls-go1-20 v0.2.3 h1:m575dovXn1y2ATOb1XrRFcrv0F+EQmlowTkoraNkDPI= -github.com/quic-go/qtls-go1-20 v0.2.3/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM= -github.com/quic-go/quic-go v0.36.4 h1:CXn/ZLN5Vntlk53fjR+kUMC8Jt7flfQe+I5Ty5A+k0o= -github.com/quic-go/quic-go v0.36.4/go.mod h1:qxQumdeKw5GmWs1OsTZZnOxzSI+RJWuhf1O8FN35L2o= +github.com/quic-go/qtls-go1-20 v0.3.3 h1:17/glZSLI9P9fDAeyCHBFSWSqJcwx1byhLwP5eUIDCM= +github.com/quic-go/qtls-go1-20 v0.3.3/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= +github.com/quic-go/quic-go v0.38.1 h1:M36YWA5dEhEeT+slOu/SwMEucbYd0YFidxG3KlGPZaE= +github.com/quic-go/quic-go v0.38.1/go.mod h1:ijnZM7JsFIkp4cRyjxJNIzdSfCLmUMg9wdyhGmg+SN4= github.com/quic-go/webtransport-go v0.5.3 h1:5XMlzemqB4qmOlgIus5zB45AcZ2kCgCy2EptUrfOPWU= github.com/quic-go/webtransport-go v0.5.3/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= @@ -493,7 +492,7 @@ github.com/urfave/cli/v2 v2.0.0 h1:+HU9SCbu8GnEUFtIBfuUNXN39ofWViIEJIp6SURMpCg= github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= -github.com/warpfork/go-testmark v0.11.0 h1:J6LnV8KpceDvo7spaNU4+DauH2n1x+6RaO2rJrmpQ9U= +github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= @@ -513,11 +512,11 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/otel v1.3.0 h1:APxLf0eiBwLl+SOXiJJCVYzA1OOJNyAoV8C5RNRyy7Y= -go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= +go.opentelemetry.io/otel v1.13.0 h1:1ZAKnNQKwBBxFtww/GwxNUyTf0AxkZzrukO8MeXqe4Y= +go.opentelemetry.io/otel v1.13.0/go.mod h1:FH3RtdZCzRkJYFTCsAKDy9l/XYjMdNv6QrkFFB8DvVg= go.opentelemetry.io/otel/sdk v1.2.0 h1:wKN260u4DesJYhyjxDa7LRFkuhH7ncEVKU37LWcyNIo= -go.opentelemetry.io/otel/trace v1.3.0 h1:doy8Hzb1RJ+I3yFhtDmwNc7tIyw1tNMOIsyPzp1NOGY= -go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk= +go.opentelemetry.io/otel/trace v1.13.0 h1:CBgRZ6ntv+Amuj1jDsMhZtlAPT6gbyIRdaIzFhfBSdY= +go.opentelemetry.io/otel/trace v1.13.0/go.mod h1:muCvmmO9KKpvuXSf3KKAXXB2ygNYHQ+ZfI5X08d3tds= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -529,7 +528,7 @@ go.uber.org/dig v1.17.0/go.mod h1:rTxpf7l5I0eBTlE6/9RL+lDybC7WFwY2QH55ZSjy1mU= go.uber.org/fx v1.20.0 h1:ZMC/pnRvhsthOZh9MZjMq5U8Or3mA9zBSPaLnzs3ihQ= go.uber.org/fx v1.20.0/go.mod h1:qCUj0btiR3/JnanEr1TYEePfSw6o/4qYJscgvzQ5Ub0= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -540,8 +539,8 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9E go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= +go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= go4.org v0.0.0-20200411211856-f5505b9728dd h1:BNJlw5kRTzdmyfh5U8F93HA2OwkP7ZGwA51eJ/0wKOU= go4.org v0.0.0-20200411211856-f5505b9728dd/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= @@ -557,8 +556,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -567,8 +566,8 @@ golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= -golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ= +golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -615,8 +614,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -665,16 +664,16 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -713,8 +712,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= -golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= +golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 h1:Vve/L0v7CXXuxUmaMGIEK/dEeq7uiqb5qBgQrZzIE7E= +golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=