Skip to content

Commit

Permalink
fix(dag@gw): content type and cache headers
Browse files Browse the repository at this point in the history
This unifies the way we set headers for DAG- responses, adds missing
ones for HTML response, and adds bunch of regression tests to ensure
desired behaviors are not changed during future refactors.
  • Loading branch information
lidel committed Nov 25, 2022
1 parent 52711d3 commit 7e84856
Show file tree
Hide file tree
Showing 4 changed files with 327 additions and 95 deletions.
107 changes: 70 additions & 37 deletions core/corehttp/gateway_handler_codec.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,10 @@ import (
"strings"
"time"

cid "github.com/ipfs/go-cid"
ipldlegacy "github.com/ipfs/go-ipld-legacy"
ipath "github.com/ipfs/interface-go-ipfs-core/path"
"github.com/ipfs/kubo/assets"
dih "github.com/ipfs/kubo/assets/dag-index-html"
"github.com/ipfs/kubo/tracing"
"github.com/ipld/go-ipld-prime"
Expand Down Expand Up @@ -54,33 +56,48 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter,
ctx, span := tracing.Span(ctx, "Gateway", "ServeCodec", trace.WithAttributes(attribute.String("path", resolvedPath.String()), attribute.String("requestedContentType", requestedContentType)))
defer span.End()

// If the resolved path still has some remainder, return bad request.
cidCodec := resolvedPath.Cid().Prefix().Codec
responseContentType := requestedContentType

// If the resolved path still has some remainder, return error for now.
// TODO: handle this when we have IPLD Patch (https://ipld.io/specs/patch/) via HTTP PUT
// TODO: (depends on https://github.com/ipfs/kubo/issues/4801 and https://github.com/ipfs/kubo/issues/4782)
if resolvedPath.Remainder() != "" {
path := strings.TrimSuffix(resolvedPath.String(), resolvedPath.Remainder())
err := fmt.Errorf("%q of %q could not be returned: reading IPLD Kinds other than Links (CBOR Tag 42) is not implemented: try reading %q instead", resolvedPath.Remainder(), resolvedPath.String(), path)
webError(w, "unsupported pathing", err, http.StatusNotImplemented)
return
}

// If no explicit content type was requested, the response will have one based on the codec from the CID
if requestedContentType == "" {
cidContentType, ok := codecToContentType[cidCodec]
if !ok {
// Should not happen unless function is called with wrong parameters.
err := fmt.Errorf("content type not found for codec: %v", cidCodec)
webError(w, "internal error", err, http.StatusInternalServerError)
return
}
responseContentType = cidContentType
}

// Set HTTP headers (for caching etc)
modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid())
name := setCodecContentDisposition(w, r, resolvedPath, responseContentType)
w.Header().Set("Content-Type", responseContentType)
w.Header().Set("X-Content-Type-Options", "nosniff")

// No content type is specified by the user (via Accept, or format=). However,
// we support this format. Let's handle it.
if requestedContentType == "" {
cidCodec := resolvedPath.Cid().Prefix().Codec
isDAG := cidCodec == uint64(mc.DagJson) || cidCodec == uint64(mc.DagCbor)
acceptsHTML := strings.Contains(r.Header.Get("Accept"), "text/html")
download := r.URL.Query().Get("download") == "true"

if isDAG && acceptsHTML {
if isDAG && acceptsHTML && !download {
i.serveCodecHTML(ctx, w, r, resolvedPath, contentPath)
} else {
cidContentType, ok := codecToContentType[cidCodec]
if !ok {
// Should not happen unless function is called with wrong parameters.
err := fmt.Errorf("content type not found for codec: %v", cidCodec)
webError(w, "internal error", err, http.StatusInternalServerError)
return
}

i.serveCodecRaw(ctx, w, r, resolvedPath, contentPath, cidContentType)
i.serveCodecRaw(ctx, w, r, resolvedPath, contentPath, name, modtime)
}

return
Expand All @@ -96,10 +113,13 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter,
return
}

// If we need to convert, use the last codec (strict dag- variant)
toCodec := codecs[len(codecs)-1]

// If the requested content type has "dag-", ALWAYS go through the encoding
// process in order to validate the content.
if strings.Contains(requestedContentType, "dag-") {
i.serveCodecConverted(ctx, w, r, resolvedPath, contentPath, requestedContentType, codecs[len(codecs)-1])
i.serveCodecConverted(ctx, w, r, resolvedPath, contentPath, toCodec, modtime)
return
}

Expand All @@ -108,34 +128,44 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter,
// used here as it sets different headers.
for _, codec := range codecs {
if resolvedPath.Cid().Prefix().Codec == codec {
i.serveCodecRaw(ctx, w, r, resolvedPath, contentPath, requestedContentType)
i.serveCodecRaw(ctx, w, r, resolvedPath, contentPath, name, modtime)
return
}
}

// Finally, if nothing of the above is true, we have to actually convert the codec.
i.serveCodecConverted(ctx, w, r, resolvedPath, contentPath, requestedContentType, codecs[len(codecs)-1])
i.serveCodecConverted(ctx, w, r, resolvedPath, contentPath, toCodec, modtime)
}

func (i *gatewayHandler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path) {
// TODO: cache-control/etag like for DirIndex
// A HTML directory index will be presented, be sure to set the correct
// type instead of relying on autodetection (which may fail).
w.Header().Set("Content-Type", "text/html")

// Clear Content-Disposition -- we want HTML to be rendered inline
w.Header().Del("Content-Disposition")

// Generated index requires custom Etag (output may change between Kubo versions)
dagEtag := getDagIndexEtag(resolvedPath.Cid())
w.Header().Set("Etag", dagEtag)

// Remove Cache-Control for now to match UnixFS dir-index-html responses
// (we don't want browser to cache HTML forever)
// TODO: if we ever change behavior for UnixFS dir listings, same changes should be applied here
w.Header().Del("Cache-Control")

cidCodec := mc.Code(resolvedPath.Cid().Prefix().Codec)
if err := dih.DagIndexTemplate.Execute(w, dih.DagIndexTemplateData{
Path: contentPath.String(),
CID: resolvedPath.Cid().String(),
CodecName: cidCodec.String(),
CodecHex: fmt.Sprintf("0x%x", uint64(cidCodec)),
}); err != nil {
webError(w, "failed to generate HTML listing for this DAG: retry without 'Accept: text/html'", err, http.StatusInternalServerError)
webError(w, "failed to generate HTML listing for this DAG: try fetching raw block with ?format=raw", err, http.StatusInternalServerError)
}
}

func (i *gatewayHandler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, contentType string) {
modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid())
name := setCodecContentDisposition(w, r, resolvedPath, contentType)
w.Header().Set("Content-Type", contentType)
w.Header().Set("X-Content-Type-Options", "nosniff")

func (i *gatewayHandler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, name string, modtime time.Time) {
blockCid := resolvedPath.Cid()
blockReader, err := i.api.Block().Get(ctx, resolvedPath)
if err != nil {
Expand All @@ -154,7 +184,7 @@ func (i *gatewayHandler) serveCodecRaw(ctx context.Context, w http.ResponseWrite
_, _, _ = ServeContent(w, r, name, modtime, content)
}

func (i *gatewayHandler) serveCodecConverted(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, contentType string, codec uint64) {
func (i *gatewayHandler) serveCodecConverted(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, toCodec uint64, modtime time.Time) {
obj, err := i.api.Dag().Get(ctx, resolvedPath.Cid())
if err != nil {
webError(w, "ipfs dag get "+html.EscapeString(resolvedPath.String()), err, http.StatusInternalServerError)
Expand All @@ -169,27 +199,20 @@ func (i *gatewayHandler) serveCodecConverted(ctx context.Context, w http.Respons
}
finalNode := universal.(ipld.Node)

encoder, err := multicodec.LookupEncoder(codec)
encoder, err := multicodec.LookupEncoder(toCodec)
if err != nil {
webError(w, err.Error(), err, http.StatusInternalServerError)
return
}

// Keep it in memory so we can detect encoding errors in order to conform
// to the specification.
// Ensure IPLD node conforms to the codec specification.
var buf bytes.Buffer
err = encoder(finalNode, &buf)
if err != nil {
webError(w, err.Error(), err, http.StatusInternalServerError)
return
}

// Set Cache-Control and read optional Last-Modified time
modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid())
setCodecContentDisposition(w, r, resolvedPath, contentType)
w.Header().Set("Content-Type", contentType)
w.Header().Set("X-Content-Type-Options", "nosniff")

// Sets correct Last-Modified header. This code is borrowed from the standard
// library (net/http/server.go) as we cannot use serveFile.
if !(modtime.IsZero() || modtime.Equal(unixEpochTime)) {
Expand All @@ -214,12 +237,22 @@ func setCodecContentDisposition(w http.ResponseWriter, r *http.Request, resolved
name = resolvedPath.Cid().String() + ext
}

switch ext {
case ".json": // codecs that serialize to JSON can be rendered by browsers
dispType = "inline"
default: // everything else is assumed binary / opaque bytes
// JSON should be inlined, but ?download=true should still override
if r.URL.Query().Get("download") == "true" {
dispType = "attachment"
} else {
switch ext {
case ".json": // codecs that serialize to JSON can be rendered by browsers
dispType = "inline"
default: // everything else is assumed binary / opaque bytes
dispType = "attachment"
}
}

setContentDispositionHeader(w, name, dispType)
return name
}

func getDagIndexEtag(dagCid cid.Cid) string {
return `"DagIndex-` + assets.AssetHash + `_CID-` + dagCid.String() + `"`
}
Loading

0 comments on commit 7e84856

Please sign in to comment.