diff --git a/core/corehttp/gateway_handler_codec.go b/core/corehttp/gateway_handler_codec.go index 736ffc5ab34..95a151c7943 100644 --- a/core/corehttp/gateway_handler_codec.go +++ b/core/corehttp/gateway_handler_codec.go @@ -10,8 +10,10 @@ import ( "strings" "time" + cid "github.com/ipfs/go-cid" ipldlegacy "github.com/ipfs/go-ipld-legacy" ipath "github.com/ipfs/interface-go-ipfs-core/path" + "github.com/ipfs/kubo/assets" dih "github.com/ipfs/kubo/assets/dag-index-html" "github.com/ipfs/kubo/tracing" "github.com/ipld/go-ipld-prime" @@ -54,7 +56,12 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, ctx, span := tracing.Span(ctx, "Gateway", "ServeCodec", trace.WithAttributes(attribute.String("path", resolvedPath.String()), attribute.String("requestedContentType", requestedContentType))) defer span.End() - // If the resolved path still has some remainder, return bad request. + cidCodec := resolvedPath.Cid().Prefix().Codec + responseContentType := requestedContentType + + // If the resolved path still has some remainder, return error for now. + // TODO: handle this when we have IPLD Patch (https://ipld.io/specs/patch/) via HTTP PUT + // TODO: (depends on https://github.com/ipfs/kubo/issues/4801 and https://github.com/ipfs/kubo/issues/4782) if resolvedPath.Remainder() != "" { path := strings.TrimSuffix(resolvedPath.String(), resolvedPath.Remainder()) err := fmt.Errorf("%q of %q could not be returned: reading IPLD Kinds other than Links (CBOR Tag 42) is not implemented: try reading %q instead", resolvedPath.Remainder(), resolvedPath.String(), path) @@ -62,25 +69,35 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, return } + // If no explicit content type was requested, the response will have one based on the codec from the CID + if requestedContentType == "" { + cidContentType, ok := codecToContentType[cidCodec] + if !ok { + // Should not happen unless function is called with wrong parameters. + err := fmt.Errorf("content type not found for codec: %v", cidCodec) + webError(w, "internal error", err, http.StatusInternalServerError) + return + } + responseContentType = cidContentType + } + + // Set HTTP headers (for caching etc) + modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid()) + name := setCodecContentDisposition(w, r, resolvedPath, responseContentType) + w.Header().Set("Content-Type", responseContentType) + w.Header().Set("X-Content-Type-Options", "nosniff") + // No content type is specified by the user (via Accept, or format=). However, // we support this format. Let's handle it. if requestedContentType == "" { - cidCodec := resolvedPath.Cid().Prefix().Codec isDAG := cidCodec == uint64(mc.DagJson) || cidCodec == uint64(mc.DagCbor) acceptsHTML := strings.Contains(r.Header.Get("Accept"), "text/html") + download := r.URL.Query().Get("download") == "true" - if isDAG && acceptsHTML { + if isDAG && acceptsHTML && !download { i.serveCodecHTML(ctx, w, r, resolvedPath, contentPath) } else { - cidContentType, ok := codecToContentType[cidCodec] - if !ok { - // Should not happen unless function is called with wrong parameters. - err := fmt.Errorf("content type not found for codec: %v", cidCodec) - webError(w, "internal error", err, http.StatusInternalServerError) - return - } - - i.serveCodecRaw(ctx, w, r, resolvedPath, contentPath, cidContentType) + i.serveCodecRaw(ctx, w, r, resolvedPath, contentPath, name, modtime) } return @@ -96,10 +113,13 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, return } + // If we need to convert, use the last codec (strict dag- variant) + toCodec := codecs[len(codecs)-1] + // If the requested content type has "dag-", ALWAYS go through the encoding // process in order to validate the content. if strings.Contains(requestedContentType, "dag-") { - i.serveCodecConverted(ctx, w, r, resolvedPath, contentPath, requestedContentType, codecs[len(codecs)-1]) + i.serveCodecConverted(ctx, w, r, resolvedPath, contentPath, toCodec, modtime) return } @@ -108,17 +128,32 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, // used here as it sets different headers. for _, codec := range codecs { if resolvedPath.Cid().Prefix().Codec == codec { - i.serveCodecRaw(ctx, w, r, resolvedPath, contentPath, requestedContentType) + i.serveCodecRaw(ctx, w, r, resolvedPath, contentPath, name, modtime) return } } // Finally, if nothing of the above is true, we have to actually convert the codec. - i.serveCodecConverted(ctx, w, r, resolvedPath, contentPath, requestedContentType, codecs[len(codecs)-1]) + i.serveCodecConverted(ctx, w, r, resolvedPath, contentPath, toCodec, modtime) } func (i *gatewayHandler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path) { - // TODO: cache-control/etag like for DirIndex + // A HTML directory index will be presented, be sure to set the correct + // type instead of relying on autodetection (which may fail). + w.Header().Set("Content-Type", "text/html") + + // Clear Content-Disposition -- we want HTML to be rendered inline + w.Header().Del("Content-Disposition") + + // Generated index requires custom Etag (output may change between Kubo versions) + dagEtag := getDagIndexEtag(resolvedPath.Cid()) + w.Header().Set("Etag", dagEtag) + + // Remove Cache-Control for now to match UnixFS dir-index-html responses + // (we don't want browser to cache HTML forever) + // TODO: if we ever change behavior for UnixFS dir listings, same changes should be applied here + w.Header().Del("Cache-Control") + cidCodec := mc.Code(resolvedPath.Cid().Prefix().Codec) if err := dih.DagIndexTemplate.Execute(w, dih.DagIndexTemplateData{ Path: contentPath.String(), @@ -126,16 +161,11 @@ func (i *gatewayHandler) serveCodecHTML(ctx context.Context, w http.ResponseWrit CodecName: cidCodec.String(), CodecHex: fmt.Sprintf("0x%x", uint64(cidCodec)), }); err != nil { - webError(w, "failed to generate HTML listing for this DAG: retry without 'Accept: text/html'", err, http.StatusInternalServerError) + webError(w, "failed to generate HTML listing for this DAG: try fetching raw block with ?format=raw", err, http.StatusInternalServerError) } } -func (i *gatewayHandler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, contentType string) { - modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid()) - name := setCodecContentDisposition(w, r, resolvedPath, contentType) - w.Header().Set("Content-Type", contentType) - w.Header().Set("X-Content-Type-Options", "nosniff") - +func (i *gatewayHandler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, name string, modtime time.Time) { blockCid := resolvedPath.Cid() blockReader, err := i.api.Block().Get(ctx, resolvedPath) if err != nil { @@ -154,7 +184,7 @@ func (i *gatewayHandler) serveCodecRaw(ctx context.Context, w http.ResponseWrite _, _, _ = ServeContent(w, r, name, modtime, content) } -func (i *gatewayHandler) serveCodecConverted(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, contentType string, codec uint64) { +func (i *gatewayHandler) serveCodecConverted(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, toCodec uint64, modtime time.Time) { obj, err := i.api.Dag().Get(ctx, resolvedPath.Cid()) if err != nil { webError(w, "ipfs dag get "+html.EscapeString(resolvedPath.String()), err, http.StatusInternalServerError) @@ -169,14 +199,13 @@ func (i *gatewayHandler) serveCodecConverted(ctx context.Context, w http.Respons } finalNode := universal.(ipld.Node) - encoder, err := multicodec.LookupEncoder(codec) + encoder, err := multicodec.LookupEncoder(toCodec) if err != nil { webError(w, err.Error(), err, http.StatusInternalServerError) return } - // Keep it in memory so we can detect encoding errors in order to conform - // to the specification. + // Ensure IPLD node conforms to the codec specification. var buf bytes.Buffer err = encoder(finalNode, &buf) if err != nil { @@ -184,12 +213,6 @@ func (i *gatewayHandler) serveCodecConverted(ctx context.Context, w http.Respons return } - // Set Cache-Control and read optional Last-Modified time - modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid()) - setCodecContentDisposition(w, r, resolvedPath, contentType) - w.Header().Set("Content-Type", contentType) - w.Header().Set("X-Content-Type-Options", "nosniff") - // Sets correct Last-Modified header. This code is borrowed from the standard // library (net/http/server.go) as we cannot use serveFile. if !(modtime.IsZero() || modtime.Equal(unixEpochTime)) { @@ -214,12 +237,22 @@ func setCodecContentDisposition(w http.ResponseWriter, r *http.Request, resolved name = resolvedPath.Cid().String() + ext } - switch ext { - case ".json": // codecs that serialize to JSON can be rendered by browsers - dispType = "inline" - default: // everything else is assumed binary / opaque bytes + // JSON should be inlined, but ?download=true should still override + if r.URL.Query().Get("download") == "true" { dispType = "attachment" + } else { + switch ext { + case ".json": // codecs that serialize to JSON can be rendered by browsers + dispType = "inline" + default: // everything else is assumed binary / opaque bytes + dispType = "attachment" + } } + setContentDispositionHeader(w, name, dispType) return name } + +func getDagIndexEtag(dagCid cid.Cid) string { + return `"DagIndex-` + assets.AssetHash + `_CID-` + dagCid.String() + `"` +} diff --git a/test/sharness/t0116-gateway-cache.sh b/test/sharness/t0116-gateway-cache.sh index 2f13f93d896..0af4ec0eb71 100755 --- a/test/sharness/t0116-gateway-cache.sh +++ b/test/sharness/t0116-gateway-cache.sh @@ -16,10 +16,14 @@ test_launch_ipfs_daemon_without_network # - have implicit index.html for a good measure # /ipns/root1/root2/root3/ (/ipns/root1/root2/root3/index.html) -# Note: we cover important edge case here: +# Note: we cover important UnixFS-focused edge case here: +# # ROOT3_CID - dir listing (dir-index-html response) # ROOT4_CID - index.html returned as a root response (dir/), instead of generated dir-index-html # FILE_CID - index.html returned directly, as a file +# +# Caching of things like raw blocks, CARs, dag-json and dag-cbor +# is tested in their respective suites. test_expect_success "Add the test directory" ' mkdir -p root2/root3/root4 && @@ -41,30 +45,36 @@ test_expect_success "Prepare IPNS unixfs content path for testing" ' ' # GET /ipfs/ + # unixfs test_expect_success "GET for /ipfs/ unixfs dir listing succeeds" ' - curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/" >/dev/null 2>curl_ipfs_dir_listing_output && - cat curl_ipfs_dir_listing_output + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/" >/dev/null 2>curl_ipfs_dir_listing_output ' test_expect_success "GET for /ipfs/ unixfs dir with index.html succeeds" ' - curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/root4/" >/dev/null 2>curl_ipfs_dir_index.html_output && - cat curl_ipfs_dir_index.html_output + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/root4/" >/dev/null 2>curl_ipfs_dir_index.html_output ' test_expect_success "GET for /ipfs/ unixfs file succeeds" ' - curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/root4/index.html" >/dev/null 2>curl_ipfs_file_output && - cat curl_ipfs_file_output + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/root4/index.html" >/dev/null 2>curl_ipfs_file_output + ' + # unixfs dir as dag-json + test_expect_success "GET for /ipfs/ unixfs dir as DAG-JSON succeeds" ' + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/root4/?format=dag-json" >/dev/null 2>curl_ipfs_dir_dag-json_output && + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/root4/?format=json" >/dev/null 2>curl_ipfs_dir_json_output ' # GET /ipns/ + # unixfs test_expect_success "GET for /ipns/ unixfs dir listing succeeds" ' - curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/" >/dev/null 2>curl_ipns_dir_listing_output && - cat curl_ipns_dir_listing_output + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/" >/dev/null 2>curl_ipns_dir_listing_output ' test_expect_success "GET for /ipns/ unixfs dir with index.html succeeds" ' - curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/root4/" >/dev/null 2>curl_ipns_dir_index.html_output && - cat curl_ipns_dir_index.html_output + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/root4/" >/dev/null 2>curl_ipns_dir_index.html_output ' test_expect_success "GET for /ipns/ unixfs file succeeds" ' - curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/root4/index.html" >/dev/null 2>curl_ipns_file_output && - cat curl_ipns_file_output + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/root4/index.html" >/dev/null 2>curl_ipns_file_output + ' + # unixfs dir as dag-json + test_expect_success "GET for /ipns/ unixfs dir as DAG-JSON succeeds" ' + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/root4/?format=dag-json" >/dev/null 2>curl_ipns_dir_dag-json_output && + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/root4/?format=json" >/dev/null 2>curl_ipns_dir_json_output ' # Cache-Control @@ -82,19 +92,34 @@ test_expect_success "Prepare IPNS unixfs content path for testing" ' test_expect_success "GET /ipfs/ unixfs dir with index.html has expected Cache-Control" ' test_should_contain "< Cache-Control: public, max-age=29030400, immutable" curl_ipfs_dir_index.html_output ' - +# Cache-Control: immutable /ipfs/ unixfs dir as dag-json + test_expect_success "GET /ipfs/ dag-json has expected Cache-Control" ' + test_should_contain "< Cache-Control: public, max-age=29030400, immutable" curl_ipfs_dir_dag-json_output + ' +# Cache-Control: immutable /ipfs/ unixfs dir as json + test_expect_success "GET /ipfs/ unixfs dir as json has expected Cache-Control" ' + test_should_contain "< Cache-Control: public, max-age=29030400, immutable" curl_ipfs_dir_json_output + ' # Cache-Control: mutable /ipns/ file test_expect_success "GET /ipns/ unixfs file has no Cache-Control" ' test_should_not_contain "< Cache-Control" curl_ipns_file_output ' -# Cache-Control: generated /ipns/dir/ (listing) +# Cache-Control: mutable /ipns/dir/ (generated listing) test_expect_success "GET /ipns/ unixfs dir listing has no Cache-Control" ' test_should_not_contain "< Cache-Control" curl_ipns_dir_listing_output ' -# Cache-Control: immutable /ipns/dir/ (index.html) +# Cache-Control: mutable /ipns/dir/ (index.html) test_expect_success "GET /ipns/ unixfs dir with index.html has no Cache-Control" ' test_should_not_contain "< Cache-Control" curl_ipns_dir_index.html_output ' +# Cache-Control: mutable /ipns/dir/ as dag-json + test_expect_success "GET /ipns/ unixfs dir as dag-json has no Cache-Control" ' + test_should_not_contain "< Cache-Control" curl_ipns_dir_dag-json_output + ' +# Cache-Control: mutable /ipns/dir/ as json + test_expect_success "GET /ipns/ unixfs dir as json has no Cache-Control" ' + test_should_not_contain "< Cache-Control" curl_ipns_dir_json_output + ' # Cache-Control: only-if-cached test_expect_success "HEAD for /ipfs/ with only-if-cached succeeds when in local datastore" ' @@ -122,78 +147,80 @@ test_expect_success "Prepare IPNS unixfs content path for testing" ' ## dir generated listing test_expect_success "GET /ipfs/ dir listing response has original content path in X-Ipfs-Path" ' - grep "< X-Ipfs-Path: /ipfs/$ROOT1_CID/root2/root3" curl_ipfs_dir_listing_output + test_should_contain "< X-Ipfs-Path: /ipfs/$ROOT1_CID/root2/root3" curl_ipfs_dir_listing_output ' test_expect_success "GET /ipns/ dir listing response has original content path in X-Ipfs-Path" ' - grep "< X-Ipfs-Path: /ipns/$TEST_IPNS_ID/root2/root3" curl_ipns_dir_listing_output + test_should_contain "< X-Ipfs-Path: /ipns/$TEST_IPNS_ID/root2/root3" curl_ipns_dir_listing_output ' ## dir static index.html test_expect_success "GET /ipfs/ dir index.html response has original content path in X-Ipfs-Path" ' - grep "< X-Ipfs-Path: /ipfs/$ROOT1_CID/root2/root3/root4/" curl_ipfs_dir_index.html_output + test_should_contain "< X-Ipfs-Path: /ipfs/$ROOT1_CID/root2/root3/root4/" curl_ipfs_dir_index.html_output ' test_expect_success "GET /ipns/ dir index.html response has original content path in X-Ipfs-Path" ' - grep "< X-Ipfs-Path: /ipns/$TEST_IPNS_ID/root2/root3/root4/" curl_ipns_dir_index.html_output + test_should_contain "< X-Ipfs-Path: /ipns/$TEST_IPNS_ID/root2/root3/root4/" curl_ipns_dir_index.html_output ' # file test_expect_success "GET /ipfs/ file response has original content path in X-Ipfs-Path" ' - grep "< X-Ipfs-Path: /ipfs/$ROOT1_CID/root2/root3/root4/index.html" curl_ipfs_file_output + test_should_contain "< X-Ipfs-Path: /ipfs/$ROOT1_CID/root2/root3/root4/index.html" curl_ipfs_file_output ' test_expect_success "GET /ipns/ file response has original content path in X-Ipfs-Path" ' - grep "< X-Ipfs-Path: /ipns/$TEST_IPNS_ID/root2/root3/root4/index.html" curl_ipns_file_output + test_should_contain "< X-Ipfs-Path: /ipns/$TEST_IPNS_ID/root2/root3/root4/index.html" curl_ipns_file_output ' # X-Ipfs-Roots ## dir generated listing test_expect_success "GET /ipfs/ dir listing response has logical CID roots in X-Ipfs-Roots" ' - grep "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID}" curl_ipfs_dir_listing_output + test_should_contain "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID}" curl_ipfs_dir_listing_output ' test_expect_success "GET /ipns/ dir listing response has logical CID roots in X-Ipfs-Roots" ' - grep "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID}" curl_ipns_dir_listing_output + test_should_contain "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID}" curl_ipns_dir_listing_output ' ## dir static index.html test_expect_success "GET /ipfs/ dir index.html response has logical CID roots in X-Ipfs-Roots" ' - grep "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID}" curl_ipfs_dir_index.html_output + test_should_contain "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID}" curl_ipfs_dir_index.html_output ' test_expect_success "GET /ipns/ dir index.html response has logical CID roots in X-Ipfs-Roots" ' - grep "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID}" curl_ipns_dir_index.html_output + test_should_contain "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID}" curl_ipns_dir_index.html_output ' ## file test_expect_success "GET /ipfs/ file response has logical CID roots in X-Ipfs-Roots" ' - grep "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID},${FILE_CID}" curl_ipfs_file_output + test_should_contain "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID},${FILE_CID}" curl_ipfs_file_output ' test_expect_success "GET /ipns/ file response has logical CID roots in X-Ipfs-Roots" ' - grep "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID},${FILE_CID}" curl_ipns_file_output + test_should_contain "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID},${FILE_CID}" curl_ipns_file_output ' # Etag ## dir generated listing test_expect_success "GET /ipfs/ dir response has special Etag for generated dir listing" ' + test_should_contain "< Etag: \"DirIndex" curl_ipfs_dir_listing_output && grep -E "< Etag: \"DirIndex-.+_CID-${ROOT3_CID}\"" curl_ipfs_dir_listing_output ' test_expect_success "GET /ipns/ dir response has special Etag for generated dir listing" ' + test_should_contain "< Etag: \"DirIndex" curl_ipfs_dir_listing_output && grep -E "< Etag: \"DirIndex-.+_CID-${ROOT3_CID}\"" curl_ipns_dir_listing_output ' ## dir static index.html should use CID of the index.html file for improved HTTP caching test_expect_success "GET /ipfs/ dir index.html response has dir CID as Etag" ' - grep "< Etag: \"${ROOT4_CID}\"" curl_ipfs_dir_index.html_output + test_should_contain "< Etag: \"${ROOT4_CID}\"" curl_ipfs_dir_index.html_output ' test_expect_success "GET /ipns/ dir index.html response has dir CID as Etag" ' - grep "< Etag: \"${ROOT4_CID}\"" curl_ipns_dir_index.html_output + test_should_contain "< Etag: \"${ROOT4_CID}\"" curl_ipns_dir_index.html_output ' ## file test_expect_success "GET /ipfs/ response has CID as Etag for a file" ' - grep "< Etag: \"${FILE_CID}\"" curl_ipfs_file_output + test_should_contain "< Etag: \"${FILE_CID}\"" curl_ipfs_file_output ' test_expect_success "GET /ipns/ response has CID as Etag for a file" ' - grep "< Etag: \"${FILE_CID}\"" curl_ipns_file_output + test_should_contain "< Etag: \"${FILE_CID}\"" curl_ipns_file_output ' # If-None-Match (return 304 Not Modified when client sends matching Etag they already have) diff --git a/test/sharness/t0117-gateway-block.sh b/test/sharness/t0117-gateway-block.sh index 87228c1c54f..5f1d080a3d4 100755 --- a/test/sharness/t0117-gateway-block.sh +++ b/test/sharness/t0117-gateway-block.sh @@ -32,43 +32,41 @@ test_expect_success "Create text fixtures" ' test_expect_success "GET response for application/vnd.ipld.raw has expected Content-Type" ' curl -svX GET -H "Accept: application/vnd.ipld.raw" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/dir/ascii.txt" >/dev/null 2>curl_output && - cat curl_output && - grep "< Content-Type: application/vnd.ipld.raw" curl_output + test_should_contain "< Content-Type: application/vnd.ipld.raw" curl_output ' test_expect_success "GET response for application/vnd.ipld.raw includes Content-Length" ' BYTES=$(ipfs block get $FILE_CID | wc --bytes) - grep "< Content-Length: $BYTES" curl_output + test_should_contain "< Content-Length: $BYTES" curl_output ' test_expect_success "GET response for application/vnd.ipld.raw includes Content-Disposition" ' - grep "< Content-Disposition: attachment\; filename=\"${FILE_CID}.bin\"" curl_output + test_should_contain "< Content-Disposition: attachment\; filename=\"${FILE_CID}.bin\"" curl_output ' test_expect_success "GET response for application/vnd.ipld.raw includes nosniff hint" ' - grep "< X-Content-Type-Options: nosniff" curl_output + test_should_contain "< X-Content-Type-Options: nosniff" curl_output ' test_expect_success "GET for application/vnd.ipld.raw with query filename includes Content-Disposition with custom filename" ' curl -svX GET -H "Accept: application/vnd.ipld.raw" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/dir/ascii.txt?filename=foobar.bin" >/dev/null 2>curl_output_filename && - cat curl_output_filename && - grep "< Content-Disposition: attachment\; filename=\"foobar.bin\"" curl_output_filename + test_should_contain "< Content-Disposition: attachment\; filename=\"foobar.bin\"" curl_output_filename ' # Cache control HTTP headers # (basic checks, detailed behavior is tested in t0116-gateway-cache.sh) test_expect_success "GET response for application/vnd.ipld.raw includes Etag" ' - grep "< Etag: \"${FILE_CID}.raw\"" curl_output + test_should_contain "< Etag: \"${FILE_CID}.raw\"" curl_output ' test_expect_success "GET response for application/vnd.ipld.raw includes X-Ipfs-Path and X-Ipfs-Roots" ' - grep "< X-Ipfs-Path" curl_output && - grep "< X-Ipfs-Roots" curl_output + test_should_contain "< X-Ipfs-Path" curl_output && + test_should_contain "< X-Ipfs-Roots" curl_output ' test_expect_success "GET response for application/vnd.ipld.raw includes Cache-Control" ' - grep "< Cache-Control: public, max-age=29030400, immutable" curl_output + test_should_contain "< Cache-Control: public, max-age=29030400, immutable" curl_output ' test_kill_ipfs_daemon diff --git a/test/sharness/t0123-gateway-json-cbor.sh b/test/sharness/t0123-gateway-json-cbor.sh index 535249ab206..812d90f24b9 100755 --- a/test/sharness/t0123-gateway-json-cbor.sh +++ b/test/sharness/t0123-gateway-json-cbor.sh @@ -22,7 +22,9 @@ test_expect_success "Add the test directory" ' echo "$FILE_CID / $FILE_SIZE" ' -test_headers () { +## Reading UnixFS (data encoded with dag-pb codec) as DAG-CBOR and DAG-JSON + +test_dag_pb_headers () { name=$1 format=$2 disposition=$3 @@ -56,8 +58,8 @@ test_headers () { ' } -test_headers "DAG-JSON" "json" "inline" -test_headers "DAG-CBOR" "cbor" "attachment" +test_dag_pb_headers "DAG-JSON" "json" "inline" +test_dag_pb_headers "DAG-CBOR" "cbor" "attachment" test_dag_pb () { name=$1 @@ -85,6 +87,8 @@ test_dag_pb () { test_dag_pb "DAG-JSON" "json" test_dag_pb "DAG-CBOR" "cbor" +## Content-Type response based on Accept header and ?format= parameter + test_cmp_dag_get () { name=$1 format=$2 @@ -136,6 +140,9 @@ test_cmp_dag_get () { test_cmp_dag_get "JSON" "json" "inline" test_cmp_dag_get "CBOR" "cbor" "attachment" + +## Lossless conversion between JSON and CBOR + test_expect_success "GET JSON as CBOR produces DAG-CBOR output" ' CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec json) && curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=cbor" > curl_output 2>&1 && @@ -150,6 +157,9 @@ test_expect_success "GET CBOR as JSON produces DAG-JSON output" ' test_cmp ipfs_dag_get_output curl_output ' + +## Pathing, traversal + DAG_CBOR_TRAVERSAL_CID="bafyreibs4utpgbn7uqegmd2goqz4bkyflre2ek2iwv743fhvylwi4zeeim" DAG_JSON_TRAVERSAL_CID="baguqeeram5ujjqrwheyaty3w5gdsmoz6vittchvhk723jjqxk7hakxkd47xq" DAG_PB_CID="bafybeiegxwlgmoh2cny7qlolykdf7aq7g6dlommarldrbm7c4hbckhfcke" @@ -163,12 +173,6 @@ test_expect_success "Add CARs for path traversal and DAG-PB representation tests test_should_contain $DAG_PB_CID import_output ' -test_expect_success "GET DAG-JSON with Accept: text/html returns HTML" ' - curl -sD - -H "Accept: text/html" "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_JSON_TRAVERSAL_CID" > curl_output 2>&1 && - test_should_not_contain "Content-Disposition" curl_output && - test_should_contain "Content-Type: text/html" curl_output -' - test_expect_success "GET DAG-JSON traversal returns 501 if there is path remainder" ' curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_JSON_TRAVERSAL_CID/foo?format=dag-json" > curl_output 2>&1 && test_should_contain "501 Not Implemented" curl_output && @@ -182,12 +186,6 @@ test_expect_success "GET DAG-JSON traverses multiple links" ' test_cmp expected actual ' -test_expect_success "GET DAG-CBOR with Accept: text/html returns HTML" ' - curl -sD - -H "Accept: text/html" "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID" > curl_output 2>&1 && - test_should_not_contain "Content-Disposition" curl_output && - test_should_contain "Content-Type: text/html" curl_output -' - test_expect_success "GET DAG-CBOR traversal returns 501 if there is path remainder" ' curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID/foo?format=dag-cbor" > curl_output 2>&1 && test_should_contain "501 Not Implemented" curl_output && @@ -207,7 +205,183 @@ test_expect_success "GET DAG-CBOR traverses multiple links" ' # test_cmp ../t0123-gateway-json-cbor/dag-pb.json actual # ' + +## NATIVE TESTS: +## DAG- regression tests for core behaviors when native DAG-(CBOR|JSON) is requested + + +test_native_dag () { + name=$1 + format=$2 + disposition=$3 + CID=$4 + + # GET without explicit format and Accept: text/html returns raw block + + test_expect_success "GET $name from /ipfs without explicit format returns the same payload as the raw block" ' + ipfs block get "/ipfs/$CID" > expected && + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" -o curl_output && + test_cmp expected curl_output + ' + + # GET dag-cbor block via Accept and ?format and ensure both are the same as `ipfs block get` output + + test_expect_success "GET $name from /ipfs with format=dag-$format returns the same payload as the raw block" ' + ipfs block get "/ipfs/$CID" > expected && + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=dag-$format" -o curl_ipfs_dag_param_output && + test_cmp expected curl_ipfs_dag_param_output + ' + + test_expect_success "GET $name from /ipfs with format=$format returns the same payload as format=dag-$format" ' + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=dag-$format" -o expected && + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=dag-$format" -o curl_ipfs_dag_param_output && + test_cmp expected curl_ipfs_dag_param_output + ' + + test_expect_success "GET $name from /ipfs with application/vnd.ipld.dag-$format returns the same payload as the raw block" ' + ipfs block get "/ipfs/$CID" > expected_block && + curl -sX GET -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" -o curl_ipfs_dag_block_accept_output && + test_cmp expected_block curl_ipfs_dag_block_accept_output + ' + + # Make sure expected HTTP headers are returned with the dag- block + + test_expect_success "GET response for application/vnd.ipld.dag-$format has expected Content-Type" ' + curl -svX GET -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" >/dev/null 2>curl_output && + test_should_contain "< Content-Type: application/vnd.ipld.dag-$format" curl_output + ' + + test_expect_success "GET response for application/vnd.ipld.dag-$format includes Content-Length" ' + BYTES=$(ipfs block get $CID | wc --bytes) + test_should_contain "< Content-Length: $BYTES" curl_output + ' + + test_expect_success "GET response for application/vnd.ipld.dag-$format includes Content-Disposition" ' + test_should_contain "< Content-Disposition: ${disposition}\; filename=\"${CID}.${format}\"" curl_output + ' + + test_expect_success "GET response for application/vnd.ipld.dag-$format includes nosniff hint" ' + test_should_contain "< X-Content-Type-Options: nosniff" curl_output + ' + + test_expect_success "GET for application/vnd.ipld.dag-$format with query filename includes Content-Disposition with custom filename" ' + curl -svX GET -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?filename=foobar.$format" >/dev/null 2>curl_output_filename && + test_should_contain "< Content-Disposition: ${disposition}\; filename=\"foobar.$format\"" curl_output_filename + ' + + test_expect_success "GET for application/vnd.ipld.dag-$format with ?download=true forces Content-Disposition: attachment" ' + curl -svX GET -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?filename=foobar.$format&download=true" >/dev/null 2>curl_output_filename && + test_should_contain "< Content-Disposition: attachment" curl_output_filename + ' + + # Cache control HTTP headers + # (basic checks, detailed behavior is tested in t0116-gateway-cache.sh) + + test_expect_success "GET response for application/vnd.ipld.dag-$format includes Etag" ' + test_should_contain "< Etag: \"${CID}.dag-$format\"" curl_output + ' + + test_expect_success "GET response for application/vnd.ipld.dag-$format includes X-Ipfs-Path and X-Ipfs-Roots" ' + test_should_contain "< X-Ipfs-Path" curl_output && + test_should_contain "< X-Ipfs-Roots" curl_output + ' + + test_expect_success "GET response for application/vnd.ipld.dag-$format includes Cache-Control" ' + test_should_contain "< Cache-Control: public, max-age=29030400, immutable" curl_output + ' + + # HTTP HEAD behavior + test_expect_success "HEAD $name with no explicit format returns HTTP 200" ' + curl -I "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" -o output && + test_should_contain "HTTP/1.1 200 OK" output && + test_should_contain "Content-Type: application/vnd.ipld.dag-$format" output && + test_should_contain "Content-Length: " output + ' + test_expect_success "HEAD $name with an explicit JSON format returns HTTP 200" ' + curl -I "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=json" -o output && + test_should_contain "HTTP/1.1 200 OK" output && + test_should_contain "Etag: \"$CID.json\"" output && + test_should_contain "Content-Type: application/json" output && + test_should_contain "Content-Length: " output + ' + test_expect_success "HEAD dag-pb with ?format=$format returns HTTP 200" ' + curl -I "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=$format" -o output && + test_should_contain "HTTP/1.1 200 OK" output && + test_should_contain "Etag: \"$FILE_CID.$format\"" output && + test_should_contain "Content-Type: application/$format" output && + test_should_contain "Content-Length: " output + ' + test_expect_success "HEAD $name with only-if-cached for missing block returns HTTP 412 Precondition Failed" ' + MISSING_CID=$(echo "{\"t\": \"$(date +%s)\"}" | ipfs dag put --store-codec=dag-${format}) && + ipfs block rm -f -q $MISSING_CID && + curl -I -H "Cache-Control: only-if-cached" "http://127.0.0.1:$GWAY_PORT/ipfs/$MISSING_CID" -o output && + test_should_contain "HTTP/1.1 412 Precondition Failed" output + ' + + # IPNS behavior (should be same as immutable /ipfs, but with different caching headers) + # To keep tests small we only confirm payload is the same, and then only test delta around caching headers. + + test_expect_success "Prepare IPNS with dag-$format" ' + IPNS_ID=$(ipfs key gen --ipns-base=base36 --type=ed25519 ${format}_test_key | head -n1 | tr -d "\n") && + ipfs name publish --key ${format}_test_key --allow-offline -Q "/ipfs/$CID" > name_publish_out && + test_check_peerid "${IPNS_ID}" && + ipfs name resolve "${IPNS_ID}" > output && + printf "/ipfs/%s\n" "$CID" > expected && + test_cmp expected output + ' + + test_expect_success "GET $name from /ipns without explicit format returns the same payload as /ipfs" ' + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" -o ipfs_output && + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipns/$IPNS_ID" -o ipns_output && + test_cmp ipfs_output ipns_output + ' + + test_expect_success "GET $name from /ipns without explicit format returns the same payload as /ipfs" ' + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=dag-$format" -o ipfs_output && + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipns/$IPNS_ID?format=dag-$format" -o ipns_output && + test_cmp ipfs_output ipns_output + ' + + test_expect_success "GET $name from /ipns with explicit application/vnd.ipld.dag-$format has expected headers" ' + curl -svX GET -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipns/$IPNS_ID" >/dev/null 2>curl_output && + test_should_not_contain "Cache-Control" curl_output && + test_should_contain "< Content-Type: application/vnd.ipld.dag-$format" curl_output && + test_should_contain "< Etag: \"${CID}.dag-$format\"" curl_output && + test_should_contain "< X-Ipfs-Path" curl_output && + test_should_contain "< X-Ipfs-Roots" curl_output + ' + + + # When Accept header includes text/html and no explicit format is requested for DAG-(CBOR|JSON) + # The gateway returns generated HTML index (see dag-index-html) for web browsers (similar to dir-index-html returned for unixfs dirs) + # As this is generated, we don't return immutable Cache-Control, even on /ipfs (same as for dir-index-html) + + test_expect_success "GET $name on /ipfs with Accept: text/html returns HTML (dag-index-html)" ' + curl -sD - -H "Accept: text/html" "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" > curl_output 2>&1 && + test_should_not_contain "Content-Disposition" curl_output && + test_should_not_contain "Cache-Control" curl_output && + test_should_contain "Etag: \"DagIndex-" curl_output && + test_should_contain "Content-Type: text/html" curl_output && + test_should_contain "" curl_output + ' + + test_expect_success "GET $name on /ipns with Accept: text/html returns HTML (dag-index-html)" ' + curl -sD - -H "Accept: text/html" "http://127.0.0.1:$GWAY_PORT/ipns/$IPNS_ID" > curl_output 2>&1 && + test_should_not_contain "Content-Disposition" curl_output && + test_should_not_contain "Cache-Control" curl_output && + test_should_contain "Etag: \"DagIndex-" curl_output && + test_should_contain "Content-Type: text/html" curl_output && + test_should_contain "" curl_output + ' + + +} + +test_native_dag "DAG-JSON" "json" "inline" "$DAG_JSON_TRAVERSAL_CID" +test_native_dag "DAG-CBOR" "cbor" "attachment" "$DAG_CBOR_TRAVERSAL_CID" + test_kill_ipfs_daemon test_done +# vim: set ts=2 sw=2 et: