Merge branch 'main' into release/booster-bitswap

cmd/boost/deal_cmd.go

@@ -78,14 +78,19 @@ var dealFlags = []cli.Flag{
 		Value: true,
 	},
 	&cli.BoolFlag{
-		Name:  "fast-retrieval",
-		Usage: "indicates that data should be available for fast retrieval",
-		Value: true,
+		Name:  "remove-unsealed-copy",
+		Usage: "indicates that an unsealed copy of the sector in not required for fast retrieval",
+		Value: false,
 	},
 	&cli.StringFlag{
 		Name:  "wallet",
 		Usage: "wallet address to be used to initiate the deal",
 	},
+	&cli.BoolFlag{
+		Name:  "skip-ipni-announce",
+		Usage: "indicates that deal index should not be announced to the IPNI(Network Indexer)",
+		Value: false,
+	},
 }
 
 var dealCmd = &cli.Command{
@@ -256,7 +261,8 @@ func dealCmdAction(cctx *cli.Context, isOnline bool) error {
 		DealDataRoot:       rootCid,
 		IsOffline:          !isOnline,
 		Transfer:           transfer,
-		FastRetrieval:      cctx.Bool("fast-retrieval"),
+		RemoveUnsealedCopy: cctx.Bool("remove-unsealed-copy"),
+		SkipIPNIAnnounce:   cctx.Bool("skip-ipni-announce"),
 	}
 
 	log.Debugw("about to submit deal proposal", "uuid", dealUuid.String())

db/deals.go

@@ -79,6 +79,7 @@ func newDealAccessor(db *sql.DB, deal *types.ProviderDealState) *dealAccessor {
 			"Error":                 &fielddef.FieldDef{F: &deal.Err},
 			"Retry":                 &fielddef.FieldDef{F: &deal.Retry},
 			"FastRetrieval":         &fielddef.FieldDef{F: &deal.FastRetrieval},
+			"AnnounceToIPNI":        &fielddef.FieldDef{F: &deal.AnnounceToIPNI},
 
 			// Needed so the deal can be looked up by signed proposal cid
 			"SignedProposalCID": &fielddef.SignedPropFieldDef{Prop: deal.ClientDealProposal},

db/fixtures.go

@@ -84,15 +84,16 @@ func GenerateNDeals(count int) ([]types.ProviderDealState, error) {
 					Params: []byte(fmt.Sprintf(`{"url":"http://files.org/file%d.car"}`, rand.Intn(1000))),
 					Size:   uint64(rand.Intn(10000)),
 				},
-				ChainDealID:   abi.DealID(rand.Intn(10000)),
-				PublishCID:    &publishCid,
-				SectorID:      abi.SectorNumber(rand.Intn(10000)),
-				Offset:        abi.PaddedPieceSize(rand.Intn(1000000)),
-				Length:        abi.PaddedPieceSize(rand.Intn(1000000)),
-				Checkpoint:    dealcheckpoints.Accepted,
-				Retry:         types.DealRetryAuto,
-				Err:           dealErr,
-				FastRetrieval: false,
+				ChainDealID:    abi.DealID(rand.Intn(10000)),
+				PublishCID:     &publishCid,
+				SectorID:       abi.SectorNumber(rand.Intn(10000)),
+				Offset:         abi.PaddedPieceSize(rand.Intn(1000000)),
+				Length:         abi.PaddedPieceSize(rand.Intn(1000000)),
+				Checkpoint:     dealcheckpoints.Accepted,
+				Retry:          types.DealRetryAuto,
+				Err:            dealErr,
+				FastRetrieval:  false,
+				AnnounceToIPNI: false,
 			}
 
 			deals = append(deals, deal)

db/migrations/20230104230242_deal_announce_to_ipni.sql

@@ -0,0 +1,10 @@
+-- +goose Up
+-- +goose StatementBegin
+ALTER TABLE Deals
+    ADD AnnounceToIPNI BOOL;
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+SELECT 'down SQL query';
+-- +goose StatementEnd

db/migrations/20230104230411_deal_announce_to_ipni.go

@@ -0,0 +1,24 @@
+package migrations
+
+import (
+	"database/sql"
+
+	"github.com/pressly/goose/v3"
+)
+
+func init() {
+	goose.AddMigration(upSetdealsAnnounceToIPNI, downSetdealsAnnounceToIPNI)
+}
+
+func upSetdealsAnnounceToIPNI(tx *sql.Tx) error {
+	_, err := tx.Exec("UPDATE Deals SET AnnounceToIPNI=?;", true)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func downSetdealsAnnounceToIPNI(tx *sql.Tx) error {
+	// This code is executed when the migration is rolled back.
+	return nil
+}

db/migrations_tests/deals_announce_to_ipni_test.go

@@ -0,0 +1,46 @@
+package migrations_tests
+
+import (
+	"context"
+	"testing"
+
+	"github.com/filecoin-project/boost/db"
+	"github.com/filecoin-project/boost/db/migrations"
+	"github.com/pressly/goose/v3"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDealAnnounceToIPNI(t *testing.T) {
+	req := require.New(t)
+	ctx := context.Background()
+
+	sqldb := db.CreateTestTmpDB(t)
+	req.NoError(db.CreateAllBoostTables(ctx, sqldb, sqldb))
+
+	// Run migrations up to the one that adds the AnnounceToIPNI field to Deals
+	goose.SetBaseFS(migrations.EmbedMigrations)
+	req.NoError(goose.SetDialect("sqlite3"))
+	req.NoError(goose.UpTo(sqldb, ".", 20230104230242))
+
+	// Generate 1 deal
+	dealsDB := db.NewDealsDB(sqldb)
+	deals, err := db.GenerateNDeals(1)
+	req.NoError(err)
+
+	// Insert the deals in DB
+	err = dealsDB.Insert(ctx, &deals[0])
+	require.NoError(t, err)
+
+	// Get deal state
+	dealState, err := dealsDB.ByID(ctx, deals[0].DealUuid)
+	require.NoError(t, err)
+	require.False(t, dealState.AnnounceToIPNI)
+
+	//Run migration
+	req.NoError(goose.UpByOne(sqldb, "."))
+
+	// Check the deal state again
+	dealState, err = dealsDB.ByID(ctx, deals[0].DealUuid)
+	require.NoError(t, err)
+	require.True(t, dealState.AnnounceToIPNI)
+}

db/migrations_tests/deals_fast_retrieval_test.go

@@ -22,25 +22,41 @@ func TestDealFastRetrieval(t *testing.T) {
 	req.NoError(goose.SetDialect("sqlite3"))
 	req.NoError(goose.UpTo(sqldb, ".", 20221124191002))
 
-	// Generate 2 deals
-	dealsDB := db.NewDealsDB(sqldb)
+	// Generate 1 deal
 	deals, err := db.GenerateNDeals(1)
 	req.NoError(err)
 
-	// Insert the deals in DB
-	err = dealsDB.Insert(ctx, &deals[0])
-	require.NoError(t, err)
+	deal := deals[0]
+
+	_, err = sqldb.Exec(`INSERT INTO Deals ("ID", "CreatedAt", "DealProposalSignature", "PieceCID", "PieceSize",
+                   "VerifiedDeal", "IsOffline", "ClientAddress", "ProviderAddress","Label", "StartEpoch", "EndEpoch",
+                   "StoragePricePerEpoch", "ProviderCollateral", "ClientCollateral", "ClientPeerID", "DealDataRoot",
+                   "InboundFilePath", "TransferType", "TransferParams", "TransferSize", "ChainDealID", "PublishCID",
+                   "SectorID", "Offset", "Length", "Checkpoint", "CheckpointAt", "Error", "Retry", "SignedProposalCID") 
+                   VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
+		deal.DealUuid, deal.CreatedAt, []byte("test"), deal.ClientDealProposal.Proposal.PieceCID.String(),
+		deal.ClientDealProposal.Proposal.PieceSize, deal.ClientDealProposal.Proposal.VerifiedDeal, deal.IsOffline,
+		deal.ClientDealProposal.Proposal.Client.String(), deal.ClientDealProposal.Proposal.Provider.String(), "test",
+		deal.ClientDealProposal.Proposal.StartEpoch, deal.ClientDealProposal.Proposal.EndEpoch, deal.ClientDealProposal.Proposal.StoragePricePerEpoch.Uint64(),
+		deal.ClientDealProposal.Proposal.ProviderCollateral.Int64(), deal.ClientDealProposal.Proposal.ClientCollateral.Uint64(), deal.ClientPeerID.String(),
+		deal.DealDataRoot.String(), deal.InboundFilePath, deal.Transfer.Type, deal.Transfer.Params, deal.Transfer.Size, deal.ChainDealID,
+		deal.PublishCID.String(), deal.SectorID, deal.Offset, deal.Length, deal.Checkpoint, deal.CheckpointAt, deal.Err, deal.Retry, []byte("test"))
 
-	// Get deal state
-	dealState, err := dealsDB.ByID(ctx, deals[0].DealUuid)
 	require.NoError(t, err)
-	require.False(t, dealState.FastRetrieval)
 
 	//Run migration
 	req.NoError(goose.UpByOne(sqldb, "."))
 
-	// Check the deal state again
-	dealState, err = dealsDB.ByID(ctx, deals[0].DealUuid)
+	rows, err := sqldb.Query("SELECT FastRetrieval FROM Deals WHERE ID =?", deal.DealUuid)
 	require.NoError(t, err)
-	require.True(t, dealState.FastRetrieval)
+	defer rows.Close()
+
+	for rows.Next() {
+
+		var ft bool
+
+		err = rows.Scan(&ft)
+		require.NoError(t, err)
+		require.True(t, ft)
+	}
 }

db/migrations_tests/storagetagged_set_host_test.go

@@ -24,13 +24,6 @@ func TestStorageTaggedSetHost(t *testing.T) {
 	req.NoError(goose.SetDialect("sqlite3"))
 	req.NoError(goose.UpTo(sqldb, ".", 20220908122510))
 
-	// Generate 2 deals
-	dealsDB := db.NewDealsDB(sqldb)
-
-	// Add FastRetrieval to allow tests to works
-	_, err := sqldb.Exec(`ALTER TABLE Deals ADD FastRetrieval BOOL`)
-	require.NoError(t, err)
-
 	deals, err := db.GenerateNDeals(2)
 	req.NoError(err)
 
@@ -44,8 +37,21 @@ func TestStorageTaggedSetHost(t *testing.T) {
 			Params: []byte(fmt.Sprintf(`{"url":"http://%s/file.car"}`, getHost(i))),
 			Size:   uint64(1024),
 		}
-		err = dealsDB.Insert(ctx, &deal)
-		req.NoError(err)
+		_, err = sqldb.Exec(`INSERT INTO Deals ("ID", "CreatedAt", "DealProposalSignature", "PieceCID", "PieceSize",
+                   "VerifiedDeal", "IsOffline", "ClientAddress", "ProviderAddress","Label", "StartEpoch", "EndEpoch",
+                   "StoragePricePerEpoch", "ProviderCollateral", "ClientCollateral", "ClientPeerID", "DealDataRoot",
+                   "InboundFilePath", "TransferType", "TransferParams", "TransferSize", "ChainDealID", "PublishCID",
+                   "SectorID", "Offset", "Length", "Checkpoint", "CheckpointAt", "Error", "Retry", "SignedProposalCID") 
+                   VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
+			deal.DealUuid, deal.CreatedAt, []byte("test"), deal.ClientDealProposal.Proposal.PieceCID.String(),
+			deal.ClientDealProposal.Proposal.PieceSize, deal.ClientDealProposal.Proposal.VerifiedDeal, deal.IsOffline,
+			deal.ClientDealProposal.Proposal.Client.String(), deal.ClientDealProposal.Proposal.Provider.String(), "test",
+			deal.ClientDealProposal.Proposal.StartEpoch, deal.ClientDealProposal.Proposal.EndEpoch, deal.ClientDealProposal.Proposal.StoragePricePerEpoch.Uint64(),
+			deal.ClientDealProposal.Proposal.ProviderCollateral.Int64(), deal.ClientDealProposal.Proposal.ClientCollateral.Uint64(), deal.ClientPeerID.String(),
+			deal.DealDataRoot.String(), deal.InboundFilePath, deal.Transfer.Type, deal.Transfer.Params, deal.Transfer.Size, deal.ChainDealID,
+			deal.PublishCID.String(), deal.SectorID, deal.Offset, deal.Length, deal.Checkpoint, deal.CheckpointAt, deal.Err, deal.Retry, []byte("test"))
+
+		require.NoError(t, err)
 	}
 
 	// Simulate tagging a deal

documentation/en/api-v1-methods.md

@@ -398,7 +398,8 @@ Response:
   "Err": "string value",
   "Retry": "auto",
   "NBytesReceived": 9,
-  "FastRetrieval": true
+  "FastRetrieval": true,
+  "AnnounceToIPNI": true
 }
 ```
 
@@ -464,7 +465,8 @@ Response:
   "Err": "string value",
   "Retry": "auto",
   "NBytesReceived": 9,
-  "FastRetrieval": true
+  "FastRetrieval": true,
+  "AnnounceToIPNI": true
 }
 ```
 
@@ -509,7 +511,8 @@ Inputs:
       "Params": "Ynl0ZSBhcnJheQ==",
       "Size": 42
     },
-    "FastRetrieval": true
+    "RemoveUnsealedCopy": true,
+    "SkipIPNIAnnounce": true
   }
 ]
 ```

go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/filecoin-project/go-bitfield v0.2.4
 	github.com/filecoin-project/go-cbor-util v0.0.1
 	github.com/filecoin-project/go-commp-utils v0.1.3
-	github.com/filecoin-project/go-data-transfer v1.15.2
+	github.com/filecoin-project/go-data-transfer v1.15.3
 	github.com/filecoin-project/go-fil-commcid v0.1.0
 	github.com/filecoin-project/go-fil-commp-hashhash v0.1.0
 	github.com/filecoin-project/go-fil-markets v1.25.2

go.sum

@@ -350,8 +350,8 @@ github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20220905160352-62059082
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
 github.com/filecoin-project/go-crypto v0.0.1 h1:AcvpSGGCgjaY8y1az6AMfKQWreF/pWO2JJGLl6gCq6o=
 github.com/filecoin-project/go-crypto v0.0.1/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
-github.com/filecoin-project/go-data-transfer v1.15.2 h1:PzqsFr2Q/onMGKrGh7TtRT0dKsJcVJrioJJnjnKmxlk=
-github.com/filecoin-project/go-data-transfer v1.15.2/go.mod h1:qXOJ3IF5dEJQHykXXTwcaRxu17bXAxr+LglXzkL6bZQ=
+github.com/filecoin-project/go-data-transfer v1.15.3 h1:zA0XrH9EQ9TtKRFt3joseShvuSNSDYOF01uuZtYghOo=
+github.com/filecoin-project/go-data-transfer v1.15.3/go.mod h1:qXOJ3IF5dEJQHykXXTwcaRxu17bXAxr+LglXzkL6bZQ=
 github.com/filecoin-project/go-ds-versioning v0.1.2 h1:to4pTadv3IeV1wvgbCbN6Vqd+fu+7tveXgv/rCEZy6w=
 github.com/filecoin-project/go-ds-versioning v0.1.2/go.mod h1:C9/l9PnB1+mwPa26BBVpCjG/XQCB0yj/q5CK2J8X1I4=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=

gql/resolver.go

@@ -5,7 +5,9 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"math"
 
+	"github.com/dustin/go-humanize"
 	"github.com/filecoin-project/boost/db"
 	"github.com/filecoin-project/boost/fundmanager"
 	gqltypes "github.com/filecoin-project/boost/gql/types"
@@ -139,6 +141,7 @@ func (r *resolver) Deals(ctx context.Context, args dealsArgs) (*dealListResolver
 
 	resolvers := make([]*dealResolver, 0, len(deals))
 	for _, deal := range deals {
+		deal.NBytesReceived = int64(r.provider.NBytesReceived(deal.DealUuid))
 		resolvers = append(resolvers, newDealResolver(&deal, r.provider, r.dealsDB, r.logsDB, r.spApi))
 	}
 
@@ -372,6 +375,14 @@ func (dr *dealResolver) IsVerified() bool {
 	return dr.ProviderDealState.ClientDealProposal.Proposal.VerifiedDeal
 }
 
+func (dr *dealResolver) KeepUnsealedCopy() bool {
+	return dr.ProviderDealState.FastRetrieval
+}
+
+func (dr *dealResolver) AnnounceToIPNI() bool {
+	return dr.ProviderDealState.AnnounceToIPNI
+}
+
 func (dr *dealResolver) ProposalLabel() (string, error) {
 	l := dr.ProviderDealState.ClientDealProposal.Proposal.Label
 	if l.IsString() {
@@ -508,16 +519,25 @@ func (dr *dealResolver) message(ctx context.Context, checkpoint dealcheckpoints.
 		if dr.IsOffline {
 			return "Awaiting Offline Data Import"
 		}
+		var pct uint64 = math.MaxUint64
+		if dr.ProviderDealState.Transfer.Size > 0 {
+			pct = (100 * dr.transferred) / dr.ProviderDealState.Transfer.Size
+		}
 		switch {
 		case dr.transferred == 0 && !dr.provider.IsTransferStalled(dr.DealUuid):
 			return "Transfer Queued"
-		case dr.transferred == 100:
-			return "Transfer Complete"
+		case pct == 100:
+			return "Verifying Commp"
 		default:
-			pct := (100 * dr.transferred) / dr.ProviderDealState.Transfer.Size
 			isStalled := dr.provider.IsTransferStalled(dr.DealUuid)
 			if isStalled {
-				return fmt.Sprintf("Transfer stalled at %d%% ", pct)
+				if pct == math.MaxUint64 {
+					return fmt.Sprintf("Transfer stalled at %s", humanize.Bytes(dr.transferred))
+				}
+				return fmt.Sprintf("Transfer stalled at %d%%", pct)
+			}
+			if pct == math.MaxUint64 {
+				return fmt.Sprintf("Transferring %s", humanize.Bytes(dr.transferred))
 			}
 			return fmt.Sprintf("Transferring %d%%", pct)
 		}

gql/schema.graphql

@@ -45,6 +45,8 @@ type Deal {
   PieceCid: String!
   PieceSize: Uint64!
   IsVerified: Boolean!
+  AnnounceToIPNI: Boolean!
+  KeepUnsealedCopy: Boolean!
   ProposalLabel: String!
   ProviderCollateral: Uint64!
   ClientCollateral: Uint64!

itests/framework/framework.go

@@ -541,7 +541,8 @@ func (f *TestFramework) MakeDummyDeal(dealUuid uuid.UUID, carFilepath string, ro
 			Params: transferParamsJSON,
 			Size:   uint64(carFileinfo.Size()),
 		},
-		FastRetrieval: true,
+		RemoveUnsealedCopy: false,
+		SkipIPNIAnnounce:   false,
 	}
 
 	return f.Client.StorageDeal(f.ctx, dealParams, peerID)