From 65cc6a865e3bd9a2892713fea7579482ea7e7a8a Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 22 Jun 2023 14:23:36 +0200 Subject: [PATCH] feat(gateway): support for order=, dups= parameters from IPIP-412 --- CHANGELOG.md | 5 +++ gateway/blocks_backend.go | 31 +++++++++++++-- gateway/gateway.go | 20 ++++++++-- gateway/gateway_test.go | 4 +- gateway/handler_car.go | 59 ++++++++++++++++++++++++---- gateway/handler_car_test.go | 77 +++++++++++++++++++++++++++++++++---- gateway/metrics.go | 2 +- gateway/utilities_test.go | 2 +- 8 files changed, 173 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8618b7103..64ec2f354 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,11 @@ The following emojis are used to highlight certain changes: ### Added +* ✨ The gateway now supports the optional `order` and `dups` CAR parameters + from [IPIP-412](https://github.com/ipfs/specs/pull/412). `BlocksBackend` only + DFS ordering. If the request explicitly requests an ordering other than `dfs` + and `unk`, the request will return an error. + ### Changed * 🛠 The `ipns` package has been refactored. You should no longer use the direct Protobuf diff --git a/gateway/blocks_backend.go b/gateway/blocks_backend.go index 9586352c9..f7e760609 100644 --- a/gateway/blocks_backend.go +++ b/gateway/blocks_backend.go @@ -232,7 +232,25 @@ func (bb *BlocksBackend) Head(ctx context.Context, path ImmutablePath) (ContentP return md, fileNode, nil } -func (bb *BlocksBackend) GetCAR(ctx context.Context, p ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) { +func (bb *BlocksBackend) GetCAR(ctx context.Context, p ImmutablePath, params *CarParams) (ContentPathMetadata, io.ReadCloser, error) { + // Check if we support the request order. On unknown, change it to DFS. We change + // the parameter directly, which means that the caller can use the value to later construct + // the Content-Type header. + switch params.Order { + case DagOrderUnknown: + params.Order = DagOrderDFS + case DagOrderDFS: + // Do nothing + default: + return ContentPathMetadata{}, nil, fmt.Errorf("unsupported order: %s", params.Order) + } + + // Similarly, if params.Duplicates is not set, let's set it to false. + if params.Duplicates == nil { + v := false + params.Duplicates = &v + } + pathMetadata, err := bb.ResolvePath(ctx, p) if err != nil { return ContentPathMetadata{}, nil, err @@ -245,7 +263,12 @@ func (bb *BlocksBackend) GetCAR(ctx context.Context, p ImmutablePath, params Car r, w := io.Pipe() go func() { - cw, err := storage.NewWritable(w, []cid.Cid{pathMetadata.LastSegment.Cid()}, car.WriteAsCarV1(true)) + cw, err := storage.NewWritable( + w, + []cid.Cid{pathMetadata.LastSegment.Cid()}, + car.WriteAsCarV1(true), + car.AllowDuplicatePuts(*params.Duplicates), + ) if err != nil { // io.PipeWriter.CloseWithError always returns nil. _ = w.CloseWithError(err) @@ -279,7 +302,7 @@ func (bb *BlocksBackend) GetCAR(ctx context.Context, p ImmutablePath, params Car } // walkGatewaySimpleSelector walks the subgraph described by the path and terminal element parameters -func walkGatewaySimpleSelector(ctx context.Context, p ipfspath.Path, params CarParams, lsys *ipld.LinkSystem, pathResolver resolver.Resolver) error { +func walkGatewaySimpleSelector(ctx context.Context, p ipfspath.Path, params *CarParams, lsys *ipld.LinkSystem, pathResolver resolver.Resolver) error { // First resolve the path since we always need to. lastCid, remainder, err := pathResolver.ResolveToLastNode(ctx, p) if err != nil { @@ -312,7 +335,7 @@ func walkGatewaySimpleSelector(ctx context.Context, p ipfspath.Path, params CarP Ctx: ctx, LinkSystem: *lsys, LinkTargetNodePrototypeChooser: bsfetcher.DefaultPrototypeChooser, - LinkVisitOnlyOnce: true, // This is safe for the "all" selector + LinkVisitOnlyOnce: !*params.Duplicates, }, } diff --git a/gateway/gateway.go b/gateway/gateway.go index cf2ca9104..f872caa94 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -121,8 +121,10 @@ func (i ImmutablePath) IsValid() error { var _ path.Path = (*ImmutablePath)(nil) type CarParams struct { - Range *DagByteRange - Scope DagScope + Range *DagByteRange + Scope DagScope + Order DagOrder + Duplicates *bool } // DagByteRange describes a range request within a UnixFS file. "From" and @@ -189,6 +191,13 @@ const ( DagScopeBlock DagScope = "block" ) +type DagOrder string + +const ( + DagOrderDFS DagOrder = "dfs" + DagOrderUnknown DagOrder = "unk" +) + type ContentPathMetadata struct { PathSegmentRoots []cid.Cid LastSegment path.Resolved @@ -278,8 +287,11 @@ type IPFSBackend interface { ResolvePath(context.Context, ImmutablePath) (ContentPathMetadata, error) // GetCAR returns a CAR file for the given immutable path. It returns an error - // if there was an issue before the CAR streaming begins. - GetCAR(context.Context, ImmutablePath, CarParams) (ContentPathMetadata, io.ReadCloser, error) + // if there was an issue before the CAR streaming begins. If [CarParams.Duplicates] + // is nil, or if [CaraParams.Order] is Unknown, the implementer should change it + // such that the caller can form the response "Content-Type" header with the most + // amount of information. + GetCAR(context.Context, ImmutablePath, *CarParams) (ContentPathMetadata, io.ReadCloser, error) // IsCached returns whether or not the path exists locally. IsCached(context.Context, path.Path) bool diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index cc36da68f..d7896f748 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -669,7 +669,7 @@ func (mb *errorMockBackend) Head(ctx context.Context, path ImmutablePath) (Conte return ContentPathMetadata{}, nil, mb.err } -func (mb *errorMockBackend) GetCAR(ctx context.Context, path ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) { +func (mb *errorMockBackend) GetCAR(ctx context.Context, path ImmutablePath, params *CarParams) (ContentPathMetadata, io.ReadCloser, error) { return ContentPathMetadata{}, nil, mb.err } @@ -753,7 +753,7 @@ func (mb *panicMockBackend) Head(ctx context.Context, immutablePath ImmutablePat panic("i am panicking") } -func (mb *panicMockBackend) GetCAR(ctx context.Context, immutablePath ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) { +func (mb *panicMockBackend) GetCAR(ctx context.Context, immutablePath ImmutablePath, params *CarParams) (ContentPathMetadata, io.ReadCloser, error) { panic("i am panicking") } diff --git a/gateway/handler_car.go b/gateway/handler_car.go index e773c920e..3e38cb09b 100644 --- a/gateway/handler_car.go +++ b/gateway/handler_car.go @@ -39,7 +39,7 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R return false } - params, err := getCarParams(r) + params, err := getCarParams(r, rq.responseParams) if err != nil { i.webError(w, r, err, http.StatusBadRequest) return false @@ -90,7 +90,7 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R // sub-DAGs and IPLD selectors: https://github.com/ipfs/go-ipfs/issues/8769 w.Header().Set("Accept-Ranges", "none") - w.Header().Set("Content-Type", carResponseFormat+"; version=1") + w.Header().Set("Content-Type", getContentTypeFromCarParams(params)) w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^) _, copyErr := io.Copy(w, carFile) @@ -113,7 +113,7 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R return true } -func getCarParams(r *http.Request) (CarParams, error) { +func getCarParams(r *http.Request, formatParams map[string]string) (*CarParams, error) { queryParams := r.URL.Query() rangeStr, hasRange := queryParams.Get(carRangeBytesKey), queryParams.Has(carRangeBytesKey) scopeStr, hasScope := queryParams.Get(carTerminalElementTypeKey), queryParams.Has(carTerminalElementTypeKey) @@ -123,7 +123,7 @@ func getCarParams(r *http.Request) (CarParams, error) { rng, err := NewDagByteRange(rangeStr) if err != nil { err = fmt.Errorf("invalid entity-bytes: %w", err) - return CarParams{}, err + return nil, err } params.Range = &rng } @@ -134,13 +134,58 @@ func getCarParams(r *http.Request) (CarParams, error) { params.Scope = s default: err := fmt.Errorf("unsupported dag-scope %s", scopeStr) - return CarParams{}, err + return nil, err } } else { params.Scope = DagScopeAll } - return params, nil + switch order := DagOrder(formatParams["order"]); order { + case DagOrderUnknown, DagOrderDFS: + params.Order = order + case "": + params.Order = DagOrderUnknown + default: + return nil, fmt.Errorf("unsupported order %s", order) + } + + switch dups := formatParams["dups"]; dups { + case "y": + v := true + params.Duplicates = &v + case "n": + v := false + params.Duplicates = &v + case "": + // Acceptable, we do not set anything. + default: + return nil, fmt.Errorf("unsupported dups %s", dups) + } + + return ¶ms, nil +} + +func getContentTypeFromCarParams(params *CarParams) string { + h := strings.Builder{} + h.WriteString(carResponseFormat) + h.WriteString("; version=1; order=") + + if params.Order != "" { + h.WriteString(string(params.Order)) + } else { + h.WriteString(string(DagOrderUnknown)) + } + + if params.Duplicates != nil { + h.WriteString("; dups=") + if *params.Duplicates { + h.WriteString("y") + } else { + h.WriteString("n") + } + } + + return h.String() } func getCarRootCidAndLastSegment(imPath ImmutablePath) (cid.Cid, string, error) { @@ -164,7 +209,7 @@ func getCarRootCidAndLastSegment(imPath ImmutablePath) (cid.Cid, string, error) return rootCid, lastSegment, err } -func getCarEtag(imPath ImmutablePath, params CarParams, rootCid cid.Cid) string { +func getCarEtag(imPath ImmutablePath, params *CarParams, rootCid cid.Cid) string { data := imPath.String() if params.Scope != DagScopeAll { data += "." + string(params.Scope) diff --git a/gateway/handler_car_test.go b/gateway/handler_car_test.go index 858ccb85d..ad3ced51b 100644 --- a/gateway/handler_car_test.go +++ b/gateway/handler_car_test.go @@ -28,7 +28,7 @@ func TestCarParams(t *testing.T) { } for _, test := range tests { r := mustNewRequest(t, http.MethodGet, "http://example.com/?"+test.query, nil) - params, err := getCarParams(r) + params, err := getCarParams(r, map[string]string{}) if test.expectedError { assert.Error(t, err) } else { @@ -60,7 +60,7 @@ func TestCarParams(t *testing.T) { } for _, test := range tests { r := mustNewRequest(t, http.MethodGet, "http://example.com/?"+test.query, nil) - params, err := getCarParams(r) + params, err := getCarParams(r, map[string]string{}) if test.hasError { assert.Error(t, err) } else { @@ -73,6 +73,67 @@ func TestCarParams(t *testing.T) { } } }) + + t.Run("order and duplicates parsing", func(t *testing.T) { + t.Parallel() + + T := true + F := false + + tests := []struct { + acceptHeader string + expectedOrder DagOrder + expectedDuplicates *bool + }{ + {"application/vnd.ipld.car; order=dfs; dups=y", DagOrderDFS, &T}, + {"application/vnd.ipld.car; order=unk; dups=n", DagOrderUnknown, &F}, + {"application/vnd.ipld.car; order=unk", DagOrderUnknown, nil}, + {"application/vnd.ipld.car; dups=y", DagOrderUnknown, &T}, + {"application/vnd.ipld.car; dups=n", DagOrderUnknown, &F}, + {"application/vnd.ipld.car", DagOrderUnknown, nil}, + } + for _, test := range tests { + r := mustNewRequest(t, http.MethodGet, "http://example.com/", nil) + r.Header.Set("Accept", test.acceptHeader) + + mediaType, formatParams, err := customResponseFormat(r) + assert.NoError(t, err) + assert.Equal(t, carResponseFormat, mediaType) + + params, err := getCarParams(r, formatParams) + assert.NoError(t, err) + require.Equal(t, test.expectedOrder, params.Order) + + if test.expectedDuplicates == nil { + require.Nil(t, params.Duplicates) + } else { + require.Equal(t, *test.expectedDuplicates, *params.Duplicates) + } + } + }) +} + +func TestContentTypeFromCarParams(t *testing.T) { + t.Parallel() + + T := true + F := false + + tests := []struct { + params CarParams + header string + }{ + {CarParams{}, "application/vnd.ipld.car; version=1; order=unk"}, + {CarParams{Order: DagOrderDFS, Duplicates: &T}, "application/vnd.ipld.car; version=1; order=dfs; dups=y"}, + {CarParams{Order: DagOrderUnknown, Duplicates: &T}, "application/vnd.ipld.car; version=1; order=unk; dups=y"}, + {CarParams{Order: DagOrderUnknown}, "application/vnd.ipld.car; version=1; order=unk"}, + {CarParams{Duplicates: &T}, "application/vnd.ipld.car; version=1; order=unk; dups=y"}, + {CarParams{Duplicates: &F}, "application/vnd.ipld.car; version=1; order=unk; dups=n"}, + } + for _, test := range tests { + header := getContentTypeFromCarParams(&test.params) + assert.Equal(t, test.header, header) + } } func TestGetCarEtag(t *testing.T) { @@ -87,24 +148,24 @@ func TestGetCarEtag(t *testing.T) { t.Run("Etag with entity-bytes=0:* is the same as without query param", func(t *testing.T) { t.Parallel() - noRange := getCarEtag(imPath, CarParams{}, cid) - withRange := getCarEtag(imPath, CarParams{Range: &DagByteRange{From: 0}}, cid) + noRange := getCarEtag(imPath, &CarParams{}, cid) + withRange := getCarEtag(imPath, &CarParams{Range: &DagByteRange{From: 0}}, cid) require.Equal(t, noRange, withRange) }) t.Run("Etag with entity-bytes=1:* is different than without query param", func(t *testing.T) { t.Parallel() - noRange := getCarEtag(imPath, CarParams{}, cid) - withRange := getCarEtag(imPath, CarParams{Range: &DagByteRange{From: 1}}, cid) + noRange := getCarEtag(imPath, &CarParams{}, cid) + withRange := getCarEtag(imPath, &CarParams{Range: &DagByteRange{From: 1}}, cid) require.NotEqual(t, noRange, withRange) }) t.Run("Etags with different dag-scope are different", func(t *testing.T) { t.Parallel() - a := getCarEtag(imPath, CarParams{Scope: DagScopeAll}, cid) - b := getCarEtag(imPath, CarParams{Scope: DagScopeEntity}, cid) + a := getCarEtag(imPath, &CarParams{Scope: DagScopeAll}, cid) + b := getCarEtag(imPath, &CarParams{Scope: DagScopeEntity}, cid) require.NotEqual(t, a, b) }) } diff --git a/gateway/metrics.go b/gateway/metrics.go index 69e81425f..371fc7864 100644 --- a/gateway/metrics.go +++ b/gateway/metrics.go @@ -120,7 +120,7 @@ func (b *ipfsBackendWithMetrics) ResolvePath(ctx context.Context, path Immutable return md, err } -func (b *ipfsBackendWithMetrics) GetCAR(ctx context.Context, path ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) { +func (b *ipfsBackendWithMetrics) GetCAR(ctx context.Context, path ImmutablePath, params *CarParams) (ContentPathMetadata, io.ReadCloser, error) { begin := time.Now() name := "IPFSBackend.GetCAR" ctx, span := spanTrace(ctx, name, trace.WithAttributes(attribute.String("path", path.String()))) diff --git a/gateway/utilities_test.go b/gateway/utilities_test.go index 1b9f81d32..3d2e7ce1b 100644 --- a/gateway/utilities_test.go +++ b/gateway/utilities_test.go @@ -149,7 +149,7 @@ func (mb *mockBackend) Head(ctx context.Context, immutablePath ImmutablePath) (C return mb.gw.Head(ctx, immutablePath) } -func (mb *mockBackend) GetCAR(ctx context.Context, immutablePath ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) { +func (mb *mockBackend) GetCAR(ctx context.Context, immutablePath ImmutablePath, params *CarParams) (ContentPathMetadata, io.ReadCloser, error) { return mb.gw.GetCAR(ctx, immutablePath, params) }