From f6b448b4263acc7e7a39942fa24e5ada1b09a408 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 24 Jul 2023 10:47:31 +0200 Subject: [PATCH] feat(gateway): support for order=, dups= parameters from IPIP-412 (#370) Co-authored-by: Marcin Rataj --- CHANGELOG.md | 27 +++++++-- gateway/blocks_backend.go | 9 ++- gateway/gateway.go | 50 ++++++++++++++++- gateway/handler.go | 47 ++++++++-------- gateway/handler_car.go | 106 ++++++++++++++++++++++++++++++------ gateway/handler_car_test.go | 66 +++++++++++++++++++++- 6 files changed, 255 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 231c110b7..d2d4b2536 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,13 +16,30 @@ The following emojis are used to highlight certain changes: ### Added +* ✨ The gateway now supports the optional `order` and `dups` CAR parameters + from [IPIP-412](https://github.com/ipfs/specs/pull/412). + * The `BlocksBackend` only implements `order=dfs` (Depth-First Search) + ordering, which was already the default behavior. + * If a request specifies no `dups`, response with `dups=n` is returned, which + was already the default behavior. + * If a request explicitly specifies a CAR `order` other than `dfs`, it will + result in an error. + * The only change to the default behavior on CAR responses is that we follow + IPIP-412 and make `order=dfs;dups=n` explicit in the returned + `Content-Type` HTTP header. + ### Changed -* 🛠 The `ipns` package has been refactored. You should no longer use the direct Protobuf - version of the IPNS Record. Instead, we have a shiny new `ipns.Record` type that wraps - all the required functionality to work the best as possible with IPNS v2 Records. Please - check the [documentation](https://pkg.go.dev/github.com/ipfs/boxo/ipns) for more information, - and follow [ipfs/specs#376](https://github.com/ipfs/specs/issues/376) for related IPIP. +* 🛠 The `ipns` package has been refactored. + * You should no longer use the direct Protobuf version of the IPNS Record. + Instead, we have a shiny new `ipns.Record` type that wraps all the required + functionality to work the best as possible with IPNS v2 Records. Please + check the [documentation](https://pkg.go.dev/github.com/ipfs/boxo/ipns) for + more information, and follow + [ipfs/specs#376](https://github.com/ipfs/specs/issues/376) for related + IPIP. + * There is no change to IPNS Records produced by `boxo/ipns`, it still + produces both V1 and V2 signatures by default, it is still backward-compatible. ### Removed diff --git a/gateway/blocks_backend.go b/gateway/blocks_backend.go index 6d76bb013..01ca49fec 100644 --- a/gateway/blocks_backend.go +++ b/gateway/blocks_backend.go @@ -245,7 +245,12 @@ func (bb *BlocksBackend) GetCAR(ctx context.Context, p ImmutablePath, params Car r, w := io.Pipe() go func() { - cw, err := storage.NewWritable(w, []cid.Cid{pathMetadata.LastSegment.Cid()}, car.WriteAsCarV1(true)) + cw, err := storage.NewWritable( + w, + []cid.Cid{pathMetadata.LastSegment.Cid()}, + car.WriteAsCarV1(true), + car.AllowDuplicatePuts(params.Duplicates.Bool()), + ) if err != nil { // io.PipeWriter.CloseWithError always returns nil. _ = w.CloseWithError(err) @@ -312,7 +317,7 @@ func walkGatewaySimpleSelector(ctx context.Context, p ipfspath.Path, params CarP Ctx: ctx, LinkSystem: *lsys, LinkTargetNodePrototypeChooser: bsfetcher.DefaultPrototypeChooser, - LinkVisitOnlyOnce: true, // This is safe for the "all" selector + LinkVisitOnlyOnce: !params.Duplicates.Bool(), }, } diff --git a/gateway/gateway.go b/gateway/gateway.go index cf2ca9104..780691a45 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -121,8 +121,10 @@ func (i ImmutablePath) IsValid() error { var _ path.Path = (*ImmutablePath)(nil) type CarParams struct { - Range *DagByteRange - Scope DagScope + Range *DagByteRange + Scope DagScope + Order DagOrder + Duplicates DuplicateBlocksPolicy } // DagByteRange describes a range request within a UnixFS file. "From" and @@ -189,6 +191,50 @@ const ( DagScopeBlock DagScope = "block" ) +type DagOrder string + +const ( + DagOrderUnspecified DagOrder = "" + DagOrderUnknown DagOrder = "unk" + DagOrderDFS DagOrder = "dfs" +) + +// DuplicateBlocksPolicy represents the content type parameter 'dups' (IPIP-412) +type DuplicateBlocksPolicy int + +const ( + DuplicateBlocksUnspecified DuplicateBlocksPolicy = iota // 0 - implicit default + DuplicateBlocksIncluded // 1 - explicitly include duplicates + DuplicateBlocksExcluded // 2 - explicitly NOT include duplicates +) + +// NewDuplicateBlocksPolicy returns DuplicateBlocksPolicy based on the content type parameter 'dups' (IPIP-412) +func NewDuplicateBlocksPolicy(dupsValue string) DuplicateBlocksPolicy { + switch dupsValue { + case "y": + return DuplicateBlocksIncluded + case "n": + return DuplicateBlocksExcluded + } + return DuplicateBlocksUnspecified +} + +func (d DuplicateBlocksPolicy) Bool() bool { + // duplicates should be returned only when explicitly requested, + // so any other state than DuplicateBlocksIncluded should return false + return d == DuplicateBlocksIncluded +} + +func (d DuplicateBlocksPolicy) String() string { + switch d { + case DuplicateBlocksIncluded: + return "y" + case DuplicateBlocksExcluded: + return "n" + } + return "" +} + type ContentPathMetadata struct { PathSegmentRoots []cid.Cid LastSegment path.Resolved diff --git a/gateway/handler.go b/gateway/handler.go index 102593449..8c1358553 100644 --- a/gateway/handler.go +++ b/gateway/handler.go @@ -637,28 +637,9 @@ const ( // return explicit response format if specified in request as query parameter or via Accept HTTP header func customResponseFormat(r *http.Request) (mediaType string, params map[string]string, err error) { - // Translate query param to a content type, if present. - if formatParam := r.URL.Query().Get("format"); formatParam != "" { - switch formatParam { - case "raw": - return rawResponseFormat, nil, nil - case "car": - return carResponseFormat, nil, nil - case "tar": - return tarResponseFormat, nil, nil - case "json": - return jsonResponseFormat, nil, nil - case "cbor": - return cborResponseFormat, nil, nil - case "dag-json": - return dagJsonResponseFormat, nil, nil - case "dag-cbor": - return dagCborResponseFormat, nil, nil - case "ipns-record": - return ipnsRecordResponseFormat, nil, nil - } - } - + // First, inspect Accept header, as it may not only include content type, but also optional parameters. + // such as CAR version or additional ones from IPIP-412. + // // Browsers and other user agents will send Accept header with generic types like: // Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8 // We only care about explicit, vendor-specific content-types and respond to the first match (in order). @@ -681,6 +662,28 @@ func customResponseFormat(r *http.Request) (mediaType string, params map[string] } } + // If no Accept header, translate query param to a content type, if present. + if formatParam := r.URL.Query().Get("format"); formatParam != "" { + switch formatParam { + case "raw": + return rawResponseFormat, nil, nil + case "car": + return carResponseFormat, nil, nil + case "tar": + return tarResponseFormat, nil, nil + case "json": + return jsonResponseFormat, nil, nil + case "cbor": + return cborResponseFormat, nil, nil + case "dag-json": + return dagJsonResponseFormat, nil, nil + case "dag-cbor": + return dagCborResponseFormat, nil, nil + case "ipns-record": + return ipnsRecordResponseFormat, nil, nil + } + } + // If none of special-cased content types is found, return empty string // to indicate default, implicit UnixFS response should be prepared return "", nil, nil diff --git a/gateway/handler_car.go b/gateway/handler_car.go index e773c920e..553519988 100644 --- a/gateway/handler_car.go +++ b/gateway/handler_car.go @@ -30,16 +30,7 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R ctx, cancel := context.WithCancel(ctx) defer cancel() - switch rq.responseParams["version"] { - case "": // noop, client does not care about version - case "1": // noop, we support this - default: - err := fmt.Errorf("unsupported CAR version: only version=1 is supported") - i.webError(w, r, err, http.StatusBadRequest) - return false - } - - params, err := getCarParams(r) + params, err := buildCarParams(r, rq.responseParams) if err != nil { i.webError(w, r, err, http.StatusBadRequest) return false @@ -90,7 +81,7 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R // sub-DAGs and IPLD selectors: https://github.com/ipfs/go-ipfs/issues/8769 w.Header().Set("Accept-Ranges", "none") - w.Header().Set("Content-Type", carResponseFormat+"; version=1") + w.Header().Set("Content-Type", buildContentTypeFromCarParams(params)) w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^) _, copyErr := io.Copy(w, carFile) @@ -113,7 +104,15 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R return true } -func getCarParams(r *http.Request) (CarParams, error) { +// buildCarParams returns CarParams based on the request, any optional parameters +// passed in URL, Accept header and the implicit defaults specific to boxo +// implementation, such as block order and duplicates status. +// +// If any of the optional content type parameters (e.g., CAR order or +// duplicates) are unspecified or empty, the function will automatically infer +// default values. +func buildCarParams(r *http.Request, contentTypeParams map[string]string) (CarParams, error) { + // URL query parameters queryParams := r.URL.Query() rangeStr, hasRange := queryParams.Get(carRangeBytesKey), queryParams.Has(carRangeBytesKey) scopeStr, hasScope := queryParams.Get(carTerminalElementTypeKey), queryParams.Has(carTerminalElementTypeKey) @@ -122,7 +121,7 @@ func getCarParams(r *http.Request) (CarParams, error) { if hasRange { rng, err := NewDagByteRange(rangeStr) if err != nil { - err = fmt.Errorf("invalid entity-bytes: %w", err) + err = fmt.Errorf("invalid application/vnd.ipld.car entity-bytes URL parameter: %w", err) return CarParams{}, err } params.Range = &rng @@ -133,16 +132,78 @@ func getCarParams(r *http.Request) (CarParams, error) { case DagScopeEntity, DagScopeAll, DagScopeBlock: params.Scope = s default: - err := fmt.Errorf("unsupported dag-scope %s", scopeStr) + err := fmt.Errorf("unsupported application/vnd.ipld.car dag-scope URL parameter: %q", scopeStr) return CarParams{}, err } } else { params.Scope = DagScopeAll } + // application/vnd.ipld.car content type parameters from Accept header + + // version of CAR format + switch contentTypeParams["version"] { + case "": // noop, client does not care about version + case "1": // noop, we support this + default: + return CarParams{}, fmt.Errorf("unsupported application/vnd.ipld.car version: only version=1 is supported") + } + + // optional order from IPIP-412 + if order := DagOrder(contentTypeParams["order"]); order != DagOrderUnspecified { + switch order { + case DagOrderUnknown, DagOrderDFS: + params.Order = order + default: + return CarParams{}, fmt.Errorf("unsupported application/vnd.ipld.car content type order parameter: %q", order) + } + } else { + // when order is not specified, we use DFS as the implicit default + // as this has always been the default behavior and we should not break + // legacy clients + params.Order = DagOrderDFS + } + + // optional dups from IPIP-412 + if dups := NewDuplicateBlocksPolicy(contentTypeParams["dups"]); dups != DuplicateBlocksUnspecified { + switch dups { + case DuplicateBlocksExcluded, DuplicateBlocksIncluded: + params.Duplicates = dups + default: + return CarParams{}, fmt.Errorf("unsupported application/vnd.ipld.car content type dups parameter: %q", dups) + } + } else { + // when duplicate block preference is not specified, we set it to + // false, as this has always been the default behavior, we should + // not break legacy clients, and responses to requests made via ?format=car + // should benefit from block deduplication + params.Duplicates = DuplicateBlocksExcluded + + } + return params, nil } +// buildContentTypeFromCarParams returns a string for Content-Type header. +// It does not change any values, CarParams are respected as-is. +func buildContentTypeFromCarParams(params CarParams) string { + h := strings.Builder{} + h.WriteString(carResponseFormat) + h.WriteString("; version=1") + + if params.Order != DagOrderUnspecified { + h.WriteString("; order=") + h.WriteString(string(params.Order)) + } + + if params.Duplicates != DuplicateBlocksUnspecified { + h.WriteString("; dups=") + h.WriteString(params.Duplicates.String()) + } + + return h.String() +} + func getCarRootCidAndLastSegment(imPath ImmutablePath) (cid.Cid, string, error) { imPathStr := imPath.String() if !strings.HasPrefix(imPathStr, "/ipfs/") { @@ -167,14 +228,25 @@ func getCarRootCidAndLastSegment(imPath ImmutablePath) (cid.Cid, string, error) func getCarEtag(imPath ImmutablePath, params CarParams, rootCid cid.Cid) string { data := imPath.String() if params.Scope != DagScopeAll { - data += "." + string(params.Scope) + data += string(params.Scope) + } + + // 'order' from IPIP-412 impact Etag only if set to something else + // than DFS (which is the implicit default) + if params.Order != DagOrderDFS { + data += string(params.Order) + } + + // 'dups' from IPIP-412 impact Etag only if 'y' + if dups := params.Duplicates.String(); dups == "y" { + data += dups } if params.Range != nil { if params.Range.From != 0 || params.Range.To != nil { - data += "." + strconv.FormatInt(params.Range.From, 10) + data += strconv.FormatInt(params.Range.From, 10) if params.Range.To != nil { - data += "." + strconv.FormatInt(*params.Range.To, 10) + data += strconv.FormatInt(*params.Range.To, 10) } } } diff --git a/gateway/handler_car_test.go b/gateway/handler_car_test.go index 858ccb85d..65777453d 100644 --- a/gateway/handler_car_test.go +++ b/gateway/handler_car_test.go @@ -28,7 +28,7 @@ func TestCarParams(t *testing.T) { } for _, test := range tests { r := mustNewRequest(t, http.MethodGet, "http://example.com/?"+test.query, nil) - params, err := getCarParams(r) + params, err := buildCarParams(r, map[string]string{}) if test.expectedError { assert.Error(t, err) } else { @@ -60,7 +60,7 @@ func TestCarParams(t *testing.T) { } for _, test := range tests { r := mustNewRequest(t, http.MethodGet, "http://example.com/?"+test.query, nil) - params, err := getCarParams(r) + params, err := buildCarParams(r, map[string]string{}) if test.hasError { assert.Error(t, err) } else { @@ -73,6 +73,68 @@ func TestCarParams(t *testing.T) { } } }) + + t.Run("buildCarParams from Accept header: order and dups parsing", func(t *testing.T) { + t.Parallel() + + // below ensure the implicit default (DFS and no duplicates) is correctly inferred + // from the value read from Accept header + tests := []struct { + acceptHeader string + expectedOrder DagOrder + expectedDuplicates DuplicateBlocksPolicy + }{ + {"application/vnd.ipld.car; order=dfs; dups=y", DagOrderDFS, DuplicateBlocksIncluded}, + {"application/vnd.ipld.car; order=unk; dups=n", DagOrderUnknown, DuplicateBlocksExcluded}, + {"application/vnd.ipld.car; order=unk", DagOrderUnknown, DuplicateBlocksExcluded}, + {"application/vnd.ipld.car; dups=y", DagOrderDFS, DuplicateBlocksIncluded}, + {"application/vnd.ipld.car; dups=n", DagOrderDFS, DuplicateBlocksExcluded}, + {"application/vnd.ipld.car", DagOrderDFS, DuplicateBlocksExcluded}, + {"application/vnd.ipld.car;version=1;order=dfs;dups=y", DagOrderDFS, DuplicateBlocksIncluded}, + } + for _, test := range tests { + r := mustNewRequest(t, http.MethodGet, "http://example.com/", nil) + r.Header.Set("Accept", test.acceptHeader) + + mediaType, formatParams, err := customResponseFormat(r) + assert.NoError(t, err) + assert.Equal(t, carResponseFormat, mediaType) + + params, err := buildCarParams(r, formatParams) + assert.NoError(t, err) + + // order from IPIP-412 + require.Equal(t, test.expectedOrder, params.Order) + + // dups from IPIP-412 + require.Equal(t, test.expectedDuplicates.String(), params.Duplicates.String()) + } + }) +} + +func TestContentTypeFromCarParams(t *testing.T) { + t.Parallel() + + // below ensures buildContentTypeFromCarParams produces correct Content-Type + // at this point we do not do any inferring, it happens in buildCarParams instead + // and tests of *Unspecified here are just present for completenes and to guard + // against regressions between refactors + tests := []struct { + params CarParams + header string + }{ + {CarParams{}, "application/vnd.ipld.car; version=1"}, + {CarParams{Order: DagOrderUnspecified, Duplicates: DuplicateBlocksUnspecified}, "application/vnd.ipld.car; version=1"}, + {CarParams{Order: DagOrderDFS, Duplicates: DuplicateBlocksIncluded}, "application/vnd.ipld.car; version=1; order=dfs; dups=y"}, + {CarParams{Order: DagOrderUnknown, Duplicates: DuplicateBlocksIncluded}, "application/vnd.ipld.car; version=1; order=unk; dups=y"}, + {CarParams{Order: DagOrderUnknown}, "application/vnd.ipld.car; version=1; order=unk"}, + {CarParams{Duplicates: DuplicateBlocksIncluded}, "application/vnd.ipld.car; version=1; dups=y"}, + {CarParams{Duplicates: DuplicateBlocksExcluded}, "application/vnd.ipld.car; version=1; dups=n"}, + } + for _, test := range tests { + header := buildContentTypeFromCarParams(test.params) + assert.Equal(t, test.header, header) + } } func TestGetCarEtag(t *testing.T) {