feat: handle retrieval queries for unindexed identity payload CIDs (#747

)

* feat: handle retrieval queries for unindexed identity payload CIDs

There are valid cases where a CAR may have an identity CID as its root that is
not represented as a 'block' within the CAR body and therefore isn't indexed
by the dagstore. In this case, we inspect the identity CID content and treat
the query as a query for the intersection of all of the links within the block.

Ref: filecoin-project/boost#715

* fix: refactor out multiple calls to dagStore.GetPiecesContainingBlock

1. to support identity PayloadCID without having to duplicate decode & lookup
   logic
2. because it's not cheap, especially for identity PayloadCIDs with lots of
   links

The tradeoff is that in some cases we end up calling the PieceStore more than
we otherwise would.

* feat: impose limits on identity PayloadCIDs

* Byte limit (2048)
* Link limit (32)

* feat: handle retrievals for nested identity CIDs

* chore: expand testing to cover dag-pb identity CIDs

go.mod

@@ -40,6 +40,7 @@ require (
 	github.com/ipfs/go-unixfs v0.3.1
 	github.com/ipld/go-car v0.4.0
 	github.com/ipld/go-car/v2 v2.4.1
+	github.com/ipld/go-codec-dagpb v1.3.1
 	github.com/ipld/go-ipld-prime v0.17.0
 	github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
 	github.com/jpillora/backoff v1.0.0
@@ -98,7 +99,6 @@ require (
 	github.com/ipfs/go-peertaskqueue v0.7.1 // indirect
 	github.com/ipfs/go-unixfsnode v1.4.0 // indirect
 	github.com/ipfs/go-verifcid v0.0.1 // indirect
-	github.com/ipld/go-codec-dagpb v1.3.1 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect
 	github.com/klauspost/cpuid/v2 v2.0.14 // indirect

piecestore/types.go

@@ -53,6 +53,10 @@ type PieceInfo struct {
 // PieceInfoUndefined is piece info with no information
 var PieceInfoUndefined = PieceInfo{}
 
+func (pi PieceInfo) Defined() bool {
+	return pi.PieceCID.Defined() || len(pi.Deals) > 0
+}
+
 // PieceStore is a saved database of piece info that can be modified and queried
 type PieceStore interface {
 	Start(ctx context.Context) error

retrievalmarket/impl/integration_test.go

@@ -3,6 +3,7 @@ package retrievalimpl_test
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
@@ -22,7 +23,6 @@ import (
 	selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"
 	dtimpl "github.com/filecoin-project/go-data-transfer/impl"
@@ -71,11 +71,11 @@ func TestClientCanMakeQueryToProvider(t *testing.T) {
 	})
 
 	t.Run("when there is some other error, returns error", func(t *testing.T) {
-		pieceStore.ReturnErrorFromGetPieceInfo(xerrors.Errorf("someerr"))
+		pieceStore.ReturnErrorFromGetPieceInfo(fmt.Errorf("someerr"))
 		expectedQR.Status = retrievalmarket.QueryResponseError
 		expectedQR.PieceCIDFound = retrievalmarket.QueryItemUnavailable
 		expectedQR.Size = 0
-		expectedQR.Message = "failed to fetch piece to retrieve from: could not locate piece: someerr"
+		expectedQR.Message = "failed to fetch piece to retrieve from: someerr"
 		actualQR, err := client.Query(bgCtx, retrievalPeer, expectedCIDs[0], retrievalmarket.QueryParams{})
 		assert.NoError(t, err)
 		actualQR.MaxPaymentInterval = expectedQR.MaxPaymentInterval