diff --git a/gateway/errors_test.go b/gateway/errors_test.go index 223d80fba..4f251822e 100644 --- a/gateway/errors_test.go +++ b/gateway/errors_test.go @@ -8,32 +8,35 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestErrRetryAfterIs(t *testing.T) { + t.Parallel() var err error err = NewErrorRetryAfter(errors.New("test"), 10*time.Second) - assert.True(t, errors.Is(err, &ErrorRetryAfter{}), "pointer to error must be error") + require.True(t, errors.Is(err, &ErrorRetryAfter{}), "pointer to error must be error") err = fmt.Errorf("wrapped: %w", err) - assert.True(t, errors.Is(err, &ErrorRetryAfter{}), "wrapped pointer to error must be error") + require.True(t, errors.Is(err, &ErrorRetryAfter{}), "wrapped pointer to error must be error") } func TestErrRetryAfterAs(t *testing.T) { + t.Parallel() + var ( err error errRA *ErrorRetryAfter ) err = NewErrorRetryAfter(errors.New("test"), 25*time.Second) - assert.True(t, errors.As(err, &errRA), "pointer to error must be error") - assert.EqualValues(t, errRA.RetryAfter, 25*time.Second) + require.True(t, errors.As(err, &errRA), "pointer to error must be error") + require.EqualValues(t, errRA.RetryAfter, 25*time.Second) err = fmt.Errorf("wrapped: %w", err) - assert.True(t, errors.As(err, &errRA), "wrapped pointer to error must be error") - assert.EqualValues(t, errRA.RetryAfter, 25*time.Second) + require.True(t, errors.As(err, &errRA), "wrapped pointer to error must be error") + require.EqualValues(t, errRA.RetryAfter, 25*time.Second) } func TestWebError(t *testing.T) { @@ -43,37 +46,45 @@ func TestWebError(t *testing.T) { config := &Config{Headers: map[string][]string{}} t.Run("429 Too Many Requests", func(t *testing.T) { + t.Parallel() + err := fmt.Errorf("wrapped for testing: %w", NewErrorRetryAfter(ErrTooManyRequests, 0)) w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "/blah", nil) webError(w, r, config, err, http.StatusInternalServerError) - assert.Equal(t, http.StatusTooManyRequests, w.Result().StatusCode) - assert.Zero(t, len(w.Result().Header.Values("Retry-After"))) + require.Equal(t, http.StatusTooManyRequests, w.Result().StatusCode) + require.Zero(t, len(w.Result().Header.Values("Retry-After"))) }) t.Run("429 Too Many Requests with Retry-After header", func(t *testing.T) { + t.Parallel() + err := NewErrorRetryAfter(ErrTooManyRequests, 25*time.Second) w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "/blah", nil) webError(w, r, config, err, http.StatusInternalServerError) - assert.Equal(t, http.StatusTooManyRequests, w.Result().StatusCode) - assert.Equal(t, "25", w.Result().Header.Get("Retry-After")) + require.Equal(t, http.StatusTooManyRequests, w.Result().StatusCode) + require.Equal(t, "25", w.Result().Header.Get("Retry-After")) }) t.Run("503 Service Unavailable with Retry-After header", func(t *testing.T) { + t.Parallel() + err := NewErrorRetryAfter(ErrServiceUnavailable, 50*time.Second) w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "/blah", nil) webError(w, r, config, err, http.StatusInternalServerError) - assert.Equal(t, http.StatusServiceUnavailable, w.Result().StatusCode) - assert.Equal(t, "50", w.Result().Header.Get("Retry-After")) + require.Equal(t, http.StatusServiceUnavailable, w.Result().StatusCode) + require.Equal(t, "50", w.Result().Header.Get("Retry-After")) }) t.Run("ErrorStatusCode propagates HTTP Status Code", func(t *testing.T) { + t.Parallel() + err := NewErrorStatusCodeFromStatus(http.StatusTeapot) w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "/blah", nil) webError(w, r, config, err, http.StatusInternalServerError) - assert.Equal(t, http.StatusTeapot, w.Result().StatusCode) + require.Equal(t, http.StatusTeapot, w.Result().StatusCode) }) } diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index 46ce75113..04b80a118 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -4,236 +4,30 @@ import ( "context" "errors" "fmt" - "html" "io" "net/http" - "net/http/httptest" - "os" - "regexp" - "strings" "testing" + "time" - "github.com/ipfs/boxo/blockservice" - nsopts "github.com/ipfs/boxo/coreiface/options/namesys" ipath "github.com/ipfs/boxo/coreiface/path" - offline "github.com/ipfs/boxo/exchange/offline" "github.com/ipfs/boxo/files" - carblockstore "github.com/ipfs/boxo/ipld/car/v2/blockstore" "github.com/ipfs/boxo/namesys" path "github.com/ipfs/boxo/path" + "github.com/ipfs/boxo/path/resolver" "github.com/ipfs/go-cid" - "github.com/libp2p/go-libp2p/core/crypto" - "github.com/libp2p/go-libp2p/core/routing" + ipld "github.com/ipfs/go-ipld-format" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -type mockNamesys map[string]path.Path - -func (m mockNamesys) Resolve(ctx context.Context, name string, opts ...nsopts.ResolveOpt) (value path.Path, err error) { - cfg := nsopts.DefaultResolveOpts() - for _, o := range opts { - o(&cfg) - } - depth := cfg.Depth - if depth == nsopts.UnlimitedDepth { - // max uint - depth = ^uint(0) - } - for strings.HasPrefix(name, "/ipns/") { - if depth == 0 { - return value, namesys.ErrResolveRecursion - } - depth-- - - var ok bool - value, ok = m[name] - if !ok { - return "", namesys.ErrResolveFailed - } - name = value.String() - } - return value, nil -} - -func (m mockNamesys) ResolveAsync(ctx context.Context, name string, opts ...nsopts.ResolveOpt) <-chan namesys.Result { - out := make(chan namesys.Result, 1) - v, err := m.Resolve(ctx, name, opts...) - out <- namesys.Result{Path: v, Err: err} - close(out) - return out -} - -func (m mockNamesys) Publish(ctx context.Context, name crypto.PrivKey, value path.Path, opts ...nsopts.PublishOption) error { - return errors.New("not implemented for mockNamesys") -} - -func (m mockNamesys) GetResolver(subs string) (namesys.Resolver, bool) { - return nil, false -} - -type mockBackend struct { - gw IPFSBackend - namesys mockNamesys -} - -var _ IPFSBackend = (*mockBackend)(nil) - -func newMockBackend(t *testing.T) (*mockBackend, cid.Cid) { - r, err := os.Open("./testdata/fixtures.car") - assert.NoError(t, err) - - blockStore, err := carblockstore.NewReadOnly(r, nil) - assert.NoError(t, err) - - t.Cleanup(func() { - blockStore.Close() - r.Close() - }) - - cids, err := blockStore.Roots() - assert.NoError(t, err) - assert.Len(t, cids, 1) - - blockService := blockservice.New(blockStore, offline.Exchange(blockStore)) - - n := mockNamesys{} - backend, err := NewBlocksBackend(blockService, WithNameSystem(n)) - if err != nil { - t.Fatal(err) - } - - return &mockBackend{ - gw: backend, - namesys: n, - }, cids[0] -} - -func (mb *mockBackend) Get(ctx context.Context, immutablePath ImmutablePath, ranges ...ByteRange) (ContentPathMetadata, *GetResponse, error) { - return mb.gw.Get(ctx, immutablePath, ranges...) -} - -func (mb *mockBackend) GetAll(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.Node, error) { - return mb.gw.GetAll(ctx, immutablePath) -} - -func (mb *mockBackend) GetBlock(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.File, error) { - return mb.gw.GetBlock(ctx, immutablePath) -} - -func (mb *mockBackend) Head(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.Node, error) { - return mb.gw.Head(ctx, immutablePath) -} - -func (mb *mockBackend) GetCAR(ctx context.Context, immutablePath ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) { - return mb.gw.GetCAR(ctx, immutablePath, params) -} - -func (mb *mockBackend) ResolveMutable(ctx context.Context, p ipath.Path) (ImmutablePath, error) { - return mb.gw.ResolveMutable(ctx, p) -} - -func (mb *mockBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { - return nil, routing.ErrNotSupported -} - -func (mb *mockBackend) GetDNSLinkRecord(ctx context.Context, hostname string) (ipath.Path, error) { - if mb.namesys != nil { - p, err := mb.namesys.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1)) - if err == namesys.ErrResolveRecursion { - err = nil - } - return ipath.New(p.String()), err - } - - return nil, errors.New("not implemented") -} - -func (mb *mockBackend) IsCached(ctx context.Context, p ipath.Path) bool { - return mb.gw.IsCached(ctx, p) -} - -func (mb *mockBackend) ResolvePath(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, error) { - return mb.gw.ResolvePath(ctx, immutablePath) -} - -func (mb *mockBackend) resolvePathNoRootsReturned(ctx context.Context, ip ipath.Path) (ipath.Resolved, error) { - var imPath ImmutablePath - var err error - if ip.Mutable() { - imPath, err = mb.ResolveMutable(ctx, ip) - if err != nil { - return nil, err - } - } else { - imPath, err = NewImmutablePath(ip) - if err != nil { - return nil, err - } - } - - md, err := mb.ResolvePath(ctx, imPath) - if err != nil { - return nil, err - } - return md.LastSegment, nil -} - -func doWithoutRedirect(req *http.Request) (*http.Response, error) { - tag := "without-redirect" - c := &http.Client{ - CheckRedirect: func(req *http.Request, via []*http.Request) error { - return errors.New(tag) - }, - } - res, err := c.Do(req) - if err != nil && !strings.Contains(err.Error(), tag) { - return nil, err - } - return res, nil -} - -func newTestServerAndNode(t *testing.T, ns mockNamesys) (*httptest.Server, *mockBackend, cid.Cid) { - backend, root := newMockBackend(t) - ts := newTestServer(t, backend) - return ts, backend, root -} - -func newTestServer(t *testing.T, backend IPFSBackend) *httptest.Server { - return newTestServerWithConfig(t, backend, Config{ - Headers: map[string][]string{}, - DeserializedResponses: true, - }) -} - -func newTestServerWithConfig(t *testing.T, backend IPFSBackend, config Config) *httptest.Server { - AddAccessControlHeaders(config.Headers) - - handler := NewHandler(config, backend) - mux := http.NewServeMux() - mux.Handle("/ipfs/", handler) - mux.Handle("/ipns/", handler) - handler = NewHostnameHandler(config, backend, mux) - - ts := httptest.NewServer(handler) - t.Cleanup(func() { ts.Close() }) - - return ts -} - -func matchPathOrBreadcrumbs(s string, expected string) bool { - matched, _ := regexp.MatchString("Index of(\n|\r\n)[\t ]*"+regexp.QuoteMeta(expected), s) - return matched -} - func TestGatewayGet(t *testing.T) { - ts, backend, root := newTestServerAndNode(t, nil) - t.Logf("test server url: %s", ts.URL) + ts, backend, root := newTestServerAndNode(t, nil, "fixtures.car") ctx, cancel := context.WithCancel(context.Background()) defer cancel() - k, err := backend.resolvePathNoRootsReturned(ctx, ipath.Join(ipath.IpfsPath(root), t.Name(), "fnord")) - assert.NoError(t, err) + k, err := backend.resolvePathNoRootsReturned(ctx, ipath.Join(ipath.IpfsPath(root), "subdir", "fnord")) + require.NoError(t, err) backend.namesys["/ipns/example.com"] = path.FromCid(k.Cid()) backend.namesys["/ipns/working.example.com"] = path.FromString(k.String()) @@ -249,7 +43,6 @@ func TestGatewayGet(t *testing.T) { // detection is platform dependent. backend.namesys["/ipns/example.man"] = path.FromString(k.String()) - t.Log(ts.URL) for _, test := range []struct { host string path string @@ -279,212 +72,25 @@ func TestGatewayGet(t *testing.T) { } { testName := "http://" + test.host + test.path t.Run(testName, func(t *testing.T) { - var c http.Client - r, err := http.NewRequest(http.MethodGet, ts.URL+test.path, nil) - assert.NoError(t, err) - r.Host = test.host - resp, err := c.Do(r) - assert.NoError(t, err) + req := mustNewRequest(t, http.MethodGet, ts.URL+test.path, nil) + req.Host = test.host + resp := mustDo(t, req) defer resp.Body.Close() - assert.Equal(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type")) + require.Equal(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type")) body, err := io.ReadAll(resp.Body) - assert.NoError(t, err) - assert.Equal(t, test.status, resp.StatusCode, "body", body) - assert.Equal(t, test.text, string(body)) - }) - } -} - -func TestUriQueryRedirect(t *testing.T) { - ts, _, _ := newTestServerAndNode(t, mockNamesys{}) - - cid := "QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR" - for _, test := range []struct { - path string - status int - location string - }{ - // - Browsers will send original URI in URL-escaped form - // - We expect query parameters to be persisted - // - We drop fragments, as those should not be sent by a browser - {"/ipfs/?uri=ipfs%3A%2F%2FQmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco%2Fwiki%2FFoo_%C4%85%C4%99.html%3Ffilename%3Dtest-%C4%99.html%23header-%C4%85", http.StatusMovedPermanently, "/ipfs/QmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco/wiki/Foo_%c4%85%c4%99.html?filename=test-%c4%99.html"}, - {"/ipfs/?uri=ipns%3A%2F%2Fexample.com%2Fwiki%2FFoo_%C4%85%C4%99.html%3Ffilename%3Dtest-%C4%99.html", http.StatusMovedPermanently, "/ipns/example.com/wiki/Foo_%c4%85%c4%99.html?filename=test-%c4%99.html"}, - {"/ipfs/?uri=ipfs://" + cid, http.StatusMovedPermanently, "/ipfs/" + cid}, - {"/ipfs?uri=ipfs://" + cid, http.StatusMovedPermanently, "/ipfs/" + cid}, - {"/ipfs/?uri=ipns://" + cid, http.StatusMovedPermanently, "/ipns/" + cid}, - {"/ipns/?uri=ipfs%3A%2F%2FQmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco%2Fwiki%2FFoo_%C4%85%C4%99.html%3Ffilename%3Dtest-%C4%99.html%23header-%C4%85", http.StatusMovedPermanently, "/ipfs/QmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco/wiki/Foo_%c4%85%c4%99.html?filename=test-%c4%99.html"}, - {"/ipns/?uri=ipns%3A%2F%2Fexample.com%2Fwiki%2FFoo_%C4%85%C4%99.html%3Ffilename%3Dtest-%C4%99.html", http.StatusMovedPermanently, "/ipns/example.com/wiki/Foo_%c4%85%c4%99.html?filename=test-%c4%99.html"}, - {"/ipns?uri=ipns://" + cid, http.StatusMovedPermanently, "/ipns/" + cid}, - {"/ipns/?uri=ipns://" + cid, http.StatusMovedPermanently, "/ipns/" + cid}, - {"/ipns/?uri=ipfs://" + cid, http.StatusMovedPermanently, "/ipfs/" + cid}, - {"/ipfs/?uri=unsupported://" + cid, http.StatusBadRequest, ""}, - {"/ipfs/?uri=invaliduri", http.StatusBadRequest, ""}, - {"/ipfs/?uri=" + cid, http.StatusBadRequest, ""}, - } { - testName := ts.URL + test.path - t.Run(testName, func(t *testing.T) { - r, err := http.NewRequest(http.MethodGet, ts.URL+test.path, nil) - assert.NoError(t, err) - resp, err := doWithoutRedirect(r) - assert.NoError(t, err) - defer resp.Body.Close() - assert.Equal(t, test.status, resp.StatusCode) - assert.Equal(t, test.location, resp.Header.Get("Location")) + require.NoError(t, err) + require.Equal(t, test.status, resp.StatusCode, "body", body) + require.Equal(t, test.text, string(body)) }) } } -func TestIPNSHostnameRedirect(t *testing.T) { - ts, backend, root := newTestServerAndNode(t, nil) - t.Logf("test server url: %s", ts.URL) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - k, err := backend.resolvePathNoRootsReturned(ctx, ipath.Join(ipath.IpfsPath(root), t.Name())) - assert.NoError(t, err) - - t.Logf("k: %s\n", k) - backend.namesys["/ipns/example.net"] = path.FromString(k.String()) - - // make request to directory containing index.html - req, err := http.NewRequest(http.MethodGet, ts.URL+"/foo", nil) - assert.NoError(t, err) - req.Host = "example.net" - - res, err := doWithoutRedirect(req) - assert.NoError(t, err) - - // expect 301 redirect to same path, but with trailing slash - assert.Equal(t, http.StatusMovedPermanently, res.StatusCode) - hdr := res.Header["Location"] - assert.Positive(t, len(hdr), "location header not present") - assert.Equal(t, hdr[0], "/foo/") - - // make request with prefix to directory containing index.html - req, err = http.NewRequest(http.MethodGet, ts.URL+"/foo", nil) - assert.NoError(t, err) - req.Host = "example.net" - - res, err = doWithoutRedirect(req) - assert.NoError(t, err) - // expect 301 redirect to same path, but with prefix and trailing slash - assert.Equal(t, http.StatusMovedPermanently, res.StatusCode) - - hdr = res.Header["Location"] - assert.Positive(t, len(hdr), "location header not present") - assert.Equal(t, hdr[0], "/foo/") - - // make sure /version isn't exposed - req, err = http.NewRequest(http.MethodGet, ts.URL+"/version", nil) - assert.NoError(t, err) - req.Host = "example.net" - - res, err = doWithoutRedirect(req) - assert.NoError(t, err) - assert.Equal(t, http.StatusNotFound, res.StatusCode) -} - -// Test directory listing on DNSLink website -// (scenario when Host header is the same as URL hostname) -// This is basic regression test: additional end-to-end tests -// can be found in test/sharness/t0115-gateway-dir-listing.sh -func TestIPNSHostnameBacklinks(t *testing.T) { - ts, backend, root := newTestServerAndNode(t, nil) - t.Logf("test server url: %s", ts.URL) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - k, err := backend.resolvePathNoRootsReturned(ctx, ipath.Join(ipath.IpfsPath(root), t.Name())) - assert.NoError(t, err) - - // create /ipns/example.net/foo/ - k2, err := backend.resolvePathNoRootsReturned(ctx, ipath.Join(k, "foo? #<'")) - assert.NoError(t, err) - - k3, err := backend.resolvePathNoRootsReturned(ctx, ipath.Join(k, "foo? #<'/bar")) - assert.NoError(t, err) - - t.Logf("k: %s\n", k) - backend.namesys["/ipns/example.net"] = path.FromString(k.String()) - - // make request to directory listing - req, err := http.NewRequest(http.MethodGet, ts.URL+"/foo%3F%20%23%3C%27/", nil) - assert.NoError(t, err) - req.Host = "example.net" - - res, err := doWithoutRedirect(req) - assert.NoError(t, err) - - // expect correct links - body, err := io.ReadAll(res.Body) - assert.NoError(t, err) - s := string(body) - t.Logf("body: %s\n", string(body)) - - assert.True(t, matchPathOrBreadcrumbs(s, "/ipns/example.net/foo? #<'"), "expected a path in directory listing") - // https://github.com/ipfs/dir-index-html/issues/42 - assert.Contains(t, s, "", "expected backlink in directory listing") - assert.Contains(t, s, "", "expected file in directory listing") - assert.Contains(t, s, s, k2.Cid().String(), "expected hash in directory listing") - - // make request to directory listing at root - req, err = http.NewRequest(http.MethodGet, ts.URL, nil) - assert.NoError(t, err) - req.Host = "example.net" - - res, err = doWithoutRedirect(req) - assert.NoError(t, err) - - // expect correct backlinks at root - body, err = io.ReadAll(res.Body) - assert.NoError(t, err) - - s = string(body) - t.Logf("body: %s\n", string(body)) - - assert.True(t, matchPathOrBreadcrumbs(s, "/"), "expected a path in directory listing") - assert.NotContains(t, s, "", "expected no backlink in directory listing of the root CID") - assert.Contains(t, s, "", "expected file in directory listing") - // https://github.com/ipfs/dir-index-html/issues/42 - assert.Contains(t, s, "example.net/foo? #<'/bar"), "expected a path in directory listing") - assert.Contains(t, s, "", "expected backlink in directory listing") - assert.Contains(t, s, "", "expected file in directory listing") - assert.Contains(t, s, k3.Cid().String(), "expected hash in directory listing") -} - func TestPretty404(t *testing.T) { - ts, backend, root := newTestServerAndNode(t, nil) + ts, backend, root := newTestServerAndNode(t, nil, "pretty-404.car") t.Logf("test server url: %s", ts.URL) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - k, err := backend.resolvePathNoRootsReturned(ctx, ipath.Join(ipath.IpfsPath(root), t.Name())) - assert.NoError(t, err) - host := "example.net" - backend.namesys["/ipns/"+host] = path.FromString(k.String()) + backend.namesys["/ipns/"+host] = path.FromCid(root) for _, test := range []struct { path string @@ -496,7 +102,7 @@ func TestPretty404(t *testing.T) { {"/nope", "text/html", http.StatusNotFound, "Custom 404"}, {"/nope", "text/*", http.StatusNotFound, "Custom 404"}, {"/nope", "*/*", http.StatusNotFound, "Custom 404"}, - {"/nope", "application/json", http.StatusNotFound, fmt.Sprintf("failed to resolve /ipns/example.net/nope: no link named \"nope\" under %s\n", k.Cid().String())}, + {"/nope", "application/json", http.StatusNotFound, fmt.Sprintf("failed to resolve /ipns/example.net/nope: no link named \"nope\" under %s\n", root.String())}, {"/deeper/nope", "text/html", http.StatusNotFound, "Deep custom 404"}, {"/deeper/", "text/html", http.StatusOK, ""}, {"/deeper", "text/html", http.StatusOK, ""}, @@ -504,317 +110,652 @@ func TestPretty404(t *testing.T) { } { testName := fmt.Sprintf("%s %s", test.path, test.accept) t.Run(testName, func(t *testing.T) { - var c http.Client - req, err := http.NewRequest("GET", ts.URL+test.path, nil) - assert.NoError(t, err) + req := mustNewRequest(t, "GET", ts.URL+test.path, nil) req.Header.Add("Accept", test.accept) req.Host = host - resp, err := c.Do(req) - assert.NoError(t, err) + resp := mustDo(t, req) defer resp.Body.Close() - assert.Equal(t, test.status, resp.StatusCode) + require.Equal(t, test.status, resp.StatusCode) body, err := io.ReadAll(resp.Body) - assert.NoError(t, err) + require.NoError(t, err) if test.text != "" { - assert.Equal(t, test.text, string(body)) + require.Equal(t, test.text, string(body)) } }) } } -func TestBrowserErrorHTML(t *testing.T) { - ts, _, root := newTestServerAndNode(t, nil) - t.Logf("test server url: %s", ts.URL) +func TestHeaders(t *testing.T) { + t.Parallel() - t.Run("plain error if request does not have Accept: text/html", func(t *testing.T) { - t.Parallel() + ts, _, _ := newTestServerAndNode(t, nil, "headers-test.car") - req, err := http.NewRequest(http.MethodGet, ts.URL+"/ipfs/"+root.String()+"/nonexisting-link", nil) - assert.Nil(t, err) + var ( + rootCID = "bafybeidbcy4u6y55gsemlubd64zk53xoxs73ifd6rieejxcr7xy46mjvky" - res, err := doWithoutRedirect(req) - assert.Nil(t, err) - assert.Equal(t, http.StatusNotFound, res.StatusCode) - assert.NotContains(t, res.Header.Get("Content-Type"), "text/html") + dirCID = "bafybeihta5xfgxcmyxyq6druvidc7es6ogffdd6zel22l3y4wddju5xxsu" + dirPath = "/ipfs/" + rootCID + "/subdir/" + dirRoots = rootCID + "," + dirCID - body, err := io.ReadAll(res.Body) - assert.Nil(t, err) - assert.NotContains(t, string(body), "") - }) + hamtFileCID = "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa" + hamtFilePath = "/ipfs/" + rootCID + "/hamt/685.txt" + hamtFileRoots = rootCID + ",bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i," + hamtFileCID - t.Run("html error if request has Accept: text/html", func(t *testing.T) { - t.Parallel() + fileCID = "bafkreiba3vpkcqpc6xtp3hsatzcod6iwneouzjoq7ymy4m2js6gc3czt6i" + filePath = "/ipfs/" + rootCID + "/subdir/fnord" + fileRoots = dirRoots + "," + fileCID - req, err := http.NewRequest(http.MethodGet, ts.URL+"/ipfs/"+root.String()+"/nonexisting-link", nil) - assert.Nil(t, err) - req.Header.Set("Accept", "text/html") + dagCborCID = "bafyreiaocls5bt2ha5vszv5pwz34zzcdf3axk3uqa56bgsgvlkbezw67hq" + dagCborPath = "/ipfs/" + rootCID + "/subdir/dag-cbor-document" + dagCborRoots = dirRoots + "," + dagCborCID + ) - res, err := doWithoutRedirect(req) - assert.Nil(t, err) - assert.Equal(t, http.StatusNotFound, res.StatusCode) - assert.Contains(t, res.Header.Get("Content-Type"), "text/html") + t.Run("Cache-Control is not immutable on generated /ipfs/ HTML dir listings", func(t *testing.T) { + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+rootCID+"/", nil) + res := mustDoWithoutRedirect(t, req) - body, err := io.ReadAll(res.Body) - assert.Nil(t, err) - assert.Contains(t, string(body), "") + // check the immutable tag isn't set + hdrs, ok := res.Header["Cache-Control"] + if ok { + for _, hdr := range hdrs { + assert.NotContains(t, hdr, "immutable", "unexpected Cache-Control: immutable on directory listing") + } + } }) -} -func TestCacheControlImmutable(t *testing.T) { - ts, _, root := newTestServerAndNode(t, nil) - t.Logf("test server url: %s", ts.URL) + t.Run("ETag is based on CID and response format", func(t *testing.T) { + test := func(responseFormat string, path string, format string, args ...any) { + t.Run(responseFormat, func(t *testing.T) { + url := ts.URL + path + req := mustNewRequest(t, http.MethodGet, url, nil) + req.Header.Add("Accept", responseFormat) + res := mustDoWithoutRedirect(t, req) + _, err := io.Copy(io.Discard, res.Body) + require.NoError(t, err) + defer res.Body.Close() + require.Equal(t, http.StatusOK, res.StatusCode) + require.Regexp(t, `^`+fmt.Sprintf(format, args...)+`$`, res.Header.Get("Etag")) + }) + } + test("", dirPath, `"DirIndex-(.*)_CID-%s"`, dirCID) + test("text/html", dirPath, `"DirIndex-(.*)_CID-%s"`, dirCID) + test(carResponseFormat, dirPath, `W/"%s.car.7of9u8ojv38vd"`, rootCID) // ETags of CARs on a Path have the root CID in the Etag and hashed information to derive the correct Etag of the full request. + test(rawResponseFormat, dirPath, `"%s.raw"`, dirCID) + test(tarResponseFormat, dirPath, `W/"%s.x-tar"`, dirCID) + + test("", hamtFilePath, `"%s"`, hamtFileCID) + test("text/html", hamtFilePath, `"%s"`, hamtFileCID) + test(carResponseFormat, hamtFilePath, `W/"%s.car.2uq26jdcsk50p"`, rootCID) // ETags of CARs on a Path have the root CID in the Etag and hashed information to derive the correct Etag of the full request. + test(rawResponseFormat, hamtFilePath, `"%s.raw"`, hamtFileCID) + test(tarResponseFormat, hamtFilePath, `W/"%s.x-tar"`, hamtFileCID) + + test("", filePath, `"%s"`, fileCID) + test("text/html", filePath, `"%s"`, fileCID) + test(carResponseFormat, filePath, `W/"%s.car.fgq8i0qnhsq01"`, rootCID) + test(rawResponseFormat, filePath, `"%s.raw"`, fileCID) + test(tarResponseFormat, filePath, `W/"%s.x-tar"`, fileCID) + + test("", dagCborPath, `"%s.dag-cbor"`, dagCborCID) + test("text/html", dagCborPath+"/", `"DagIndex-(.*)_CID-%s"`, dagCborCID) + test(carResponseFormat, dagCborPath, `W/"%s.car.5mg3mekeviba5"`, rootCID) + test(rawResponseFormat, dagCborPath, `"%s.raw"`, dagCborCID) + test(dagJsonResponseFormat, dagCborPath, `"%s.dag-json"`, dagCborCID) + test(dagCborResponseFormat, dagCborPath, `"%s.dag-cbor"`, dagCborCID) + }) - req, err := http.NewRequest(http.MethodGet, ts.URL+"/ipfs/"+root.String()+"/", nil) - assert.NoError(t, err) + t.Run("If-None-Match with previous Etag returns Not Modified", func(t *testing.T) { + test := func(responseFormat string, path string) { + t.Run(responseFormat, func(t *testing.T) { + url := ts.URL + path + req := mustNewRequest(t, http.MethodGet, url, nil) + req.Header.Add("Accept", responseFormat) + res := mustDoWithoutRedirect(t, req) + _, err := io.Copy(io.Discard, res.Body) + require.NoError(t, err) + defer res.Body.Close() + require.Equal(t, http.StatusOK, res.StatusCode) + etag := res.Header.Get("Etag") + require.NotEmpty(t, etag) + + req = mustNewRequest(t, http.MethodGet, url, nil) + req.Header.Add("Accept", responseFormat) + req.Header.Add("If-None-Match", etag) + res = mustDoWithoutRedirect(t, req) + _, err = io.Copy(io.Discard, res.Body) + require.NoError(t, err) + defer res.Body.Close() + require.Equal(t, http.StatusNotModified, res.StatusCode) + }) + } - res, err := doWithoutRedirect(req) - assert.NoError(t, err) + test("", dirPath) + test("text/html", dirPath) + test(carResponseFormat, dirPath) + test(rawResponseFormat, dirPath) + test(tarResponseFormat, dirPath) + + test("", hamtFilePath) + test("text/html", hamtFilePath) + test(carResponseFormat, hamtFilePath) + test(rawResponseFormat, hamtFilePath) + test(tarResponseFormat, hamtFilePath) + + test("", filePath) + test("text/html", filePath) + test(carResponseFormat, filePath) + test(rawResponseFormat, filePath) + test(tarResponseFormat, filePath) + + test("", dagCborPath) + test("text/html", dagCborPath+"/") + test(carResponseFormat, dagCborPath) + test(rawResponseFormat, dagCborPath) + test(dagJsonResponseFormat, dagCborPath) + test(dagCborResponseFormat, dagCborPath) + }) - // check the immutable tag isn't set - hdrs, ok := res.Header["Cache-Control"] - if ok { - for _, hdr := range hdrs { - assert.NotContains(t, hdr, "immutable", "unexpected Cache-Control: immutable on directory listing") + t.Run("X-Ipfs-Roots contains expected values", func(t *testing.T) { + test := func(responseFormat string, path string, roots string) { + t.Run(responseFormat, func(t *testing.T) { + url := ts.URL + path + req := mustNewRequest(t, http.MethodGet, url, nil) + req.Header.Add("Accept", responseFormat) + res := mustDoWithoutRedirect(t, req) + _, err := io.Copy(io.Discard, res.Body) + require.NoError(t, err) + defer res.Body.Close() + require.Equal(t, http.StatusOK, res.StatusCode) + require.Equal(t, roots, res.Header.Get("X-Ipfs-Roots")) + }) } - } + + test("", dirPath, dirRoots) + test("text/html", dirPath, dirRoots) + test(carResponseFormat, dirPath, dirRoots) + test(rawResponseFormat, dirPath, dirRoots) + test(tarResponseFormat, dirPath, dirRoots) + + test("", hamtFilePath, hamtFileRoots) + test("text/html", hamtFilePath, hamtFileRoots) + test(carResponseFormat, hamtFilePath, hamtFileRoots) + test(rawResponseFormat, hamtFilePath, hamtFileRoots) + test(tarResponseFormat, hamtFilePath, hamtFileRoots) + + test("", filePath, fileRoots) + test("text/html", filePath, fileRoots) + test(carResponseFormat, filePath, fileRoots) + test(rawResponseFormat, filePath, fileRoots) + test(tarResponseFormat, filePath, fileRoots) + + test("", dagCborPath, dagCborRoots) + test("text/html", dagCborPath+"/", dagCborRoots) + test(carResponseFormat, dagCborPath, dagCborRoots) + test(rawResponseFormat, dagCborPath, dagCborRoots) + test(dagJsonResponseFormat, dagCborPath, dagCborRoots) + test(dagCborResponseFormat, dagCborPath, dagCborRoots) + }) + + t.Run("If-None-Match with wrong value forces path resolution, but X-Ipfs-Roots is correct (regression)", func(t *testing.T) { + test := func(responseFormat string, path string, roots string) { + t.Run(responseFormat, func(t *testing.T) { + url := ts.URL + path + req := mustNewRequest(t, http.MethodGet, url, nil) + req.Header.Add("Accept", responseFormat) + req.Header.Add("If-None-Match", "just-some-gibberish") + res := mustDoWithoutRedirect(t, req) + _, err := io.Copy(io.Discard, res.Body) + require.NoError(t, err) + defer res.Body.Close() + require.Equal(t, http.StatusOK, res.StatusCode) + require.Equal(t, roots, res.Header.Get("X-Ipfs-Roots")) + }) + } + + test("", dirPath, dirRoots) + test("text/html", dirPath, dirRoots) + test(carResponseFormat, dirPath, dirRoots) + test(rawResponseFormat, dirPath, dirRoots) + test(tarResponseFormat, dirPath, dirRoots) + + test("", hamtFilePath, hamtFileRoots) + test("text/html", hamtFilePath, hamtFileRoots) + test(carResponseFormat, hamtFilePath, hamtFileRoots) + test(rawResponseFormat, hamtFilePath, hamtFileRoots) + test(tarResponseFormat, hamtFilePath, hamtFileRoots) + + test("", filePath, fileRoots) + test("text/html", filePath, fileRoots) + test(carResponseFormat, filePath, fileRoots) + test(rawResponseFormat, filePath, fileRoots) + test(tarResponseFormat, filePath, fileRoots) + + test("", dagCborPath, dagCborRoots) + test("text/html", dagCborPath+"/", dagCborRoots) + test(carResponseFormat, dagCborPath, dagCborRoots) + test(rawResponseFormat, dagCborPath, dagCborRoots) + test(dagJsonResponseFormat, dagCborPath, dagCborRoots) + test(dagCborResponseFormat, dagCborPath, dagCborRoots) + }) } func TestGoGetSupport(t *testing.T) { - ts, _, root := newTestServerAndNode(t, nil) - t.Logf("test server url: %s", ts.URL) + ts, _, root := newTestServerAndNode(t, nil, "fixtures.car") // mimic go-get - req, err := http.NewRequest(http.MethodGet, ts.URL+"/ipfs/"+root.String()+"?go-get=1", nil) - assert.NoError(t, err) - - res, err := doWithoutRedirect(req) - assert.NoError(t, err) - assert.Equal(t, http.StatusOK, res.StatusCode) + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+root.String()+"?go-get=1", nil) + res := mustDoWithoutRedirect(t, req) + require.Equal(t, http.StatusOK, res.StatusCode) } -func TestIpnsBase58MultihashRedirect(t *testing.T) { - ts, _, _ := newTestServerAndNode(t, nil) - t.Logf("test server url: %s", ts.URL) +func TestRedirects(t *testing.T) { + t.Parallel() - t.Run("ED25519 Base58-encoded key", func(t *testing.T) { - t.Parallel() + t.Run("IPNS Base58 Multihash Redirect", func(t *testing.T) { + ts, _, _ := newTestServerAndNode(t, nil, "fixtures.car") - req, err := http.NewRequest(http.MethodGet, ts.URL+"/ipns/12D3KooWRBy97UB99e3J6hiPesre1MZeuNQvfan4gBziswrRJsNK?keep=query", nil) - assert.Nil(t, err) + t.Run("ED25519 Base58-encoded key", func(t *testing.T) { + t.Parallel() - res, err := doWithoutRedirect(req) - assert.Nil(t, err) - assert.Equal(t, "/ipns/k51qzi5uqu5dlvj2baxnqndepeb86cbk3ng7n3i46uzyxzyqj2xjonzllnv0v8?keep=query", res.Header.Get("Location")) + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipns/12D3KooWRBy97UB99e3J6hiPesre1MZeuNQvfan4gBziswrRJsNK?keep=query", nil) + res := mustDoWithoutRedirect(t, req) + require.Equal(t, "/ipns/k51qzi5uqu5dlvj2baxnqndepeb86cbk3ng7n3i46uzyxzyqj2xjonzllnv0v8?keep=query", res.Header.Get("Location")) + }) + + t.Run("RSA Base58-encoded key", func(t *testing.T) { + t.Parallel() + + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipns/QmcJM7PRfkSbcM5cf1QugM5R37TLRKyJGgBEhXjLTB8uA2?keep=query", nil) + res := mustDoWithoutRedirect(t, req) + require.Equal(t, "/ipns/k2k4r8ol4m8kkcqz509c1rcjwunebj02gcnm5excpx842u736nja8ger?keep=query", res.Header.Get("Location")) + }) }) - t.Run("RSA Base58-encoded key", func(t *testing.T) { + t.Run("URI Query Redirects", func(t *testing.T) { t.Parallel() + ts, _, _ := newTestServerAndNode(t, mockNamesys{}, "fixtures.car") + + cid := "QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR" + for _, test := range []struct { + path string + status int + location string + }{ + // - Browsers will send original URI in URL-escaped form + // - We expect query parameters to be persisted + // - We drop fragments, as those should not be sent by a browser + {"/ipfs/?uri=ipfs%3A%2F%2FQmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco%2Fwiki%2FFoo_%C4%85%C4%99.html%3Ffilename%3Dtest-%C4%99.html%23header-%C4%85", http.StatusMovedPermanently, "/ipfs/QmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco/wiki/Foo_%c4%85%c4%99.html?filename=test-%c4%99.html"}, + {"/ipfs/?uri=ipns%3A%2F%2Fexample.com%2Fwiki%2FFoo_%C4%85%C4%99.html%3Ffilename%3Dtest-%C4%99.html", http.StatusMovedPermanently, "/ipns/example.com/wiki/Foo_%c4%85%c4%99.html?filename=test-%c4%99.html"}, + {"/ipfs/?uri=ipfs://" + cid, http.StatusMovedPermanently, "/ipfs/" + cid}, + {"/ipfs?uri=ipfs://" + cid, http.StatusMovedPermanently, "/ipfs/" + cid}, + {"/ipfs/?uri=ipns://" + cid, http.StatusMovedPermanently, "/ipns/" + cid}, + {"/ipns/?uri=ipfs%3A%2F%2FQmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco%2Fwiki%2FFoo_%C4%85%C4%99.html%3Ffilename%3Dtest-%C4%99.html%23header-%C4%85", http.StatusMovedPermanently, "/ipfs/QmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco/wiki/Foo_%c4%85%c4%99.html?filename=test-%c4%99.html"}, + {"/ipns/?uri=ipns%3A%2F%2Fexample.com%2Fwiki%2FFoo_%C4%85%C4%99.html%3Ffilename%3Dtest-%C4%99.html", http.StatusMovedPermanently, "/ipns/example.com/wiki/Foo_%c4%85%c4%99.html?filename=test-%c4%99.html"}, + {"/ipns?uri=ipns://" + cid, http.StatusMovedPermanently, "/ipns/" + cid}, + {"/ipns/?uri=ipns://" + cid, http.StatusMovedPermanently, "/ipns/" + cid}, + {"/ipns/?uri=ipfs://" + cid, http.StatusMovedPermanently, "/ipfs/" + cid}, + {"/ipfs/?uri=unsupported://" + cid, http.StatusBadRequest, ""}, + {"/ipfs/?uri=invaliduri", http.StatusBadRequest, ""}, + {"/ipfs/?uri=" + cid, http.StatusBadRequest, ""}, + } { + testName := ts.URL + test.path + t.Run(testName, func(t *testing.T) { + req := mustNewRequest(t, http.MethodGet, ts.URL+test.path, nil) + resp := mustDoWithoutRedirect(t, req) + defer resp.Body.Close() + require.Equal(t, test.status, resp.StatusCode) + require.Equal(t, test.location, resp.Header.Get("Location")) + }) + } + }) - req, err := http.NewRequest(http.MethodGet, ts.URL+"/ipns/QmcJM7PRfkSbcM5cf1QugM5R37TLRKyJGgBEhXjLTB8uA2?keep=query", nil) - assert.Nil(t, err) + t.Run("IPNS Hostname Redirects", func(t *testing.T) { + t.Parallel() - res, err := doWithoutRedirect(req) - assert.Nil(t, err) - assert.Equal(t, "/ipns/k2k4r8ol4m8kkcqz509c1rcjwunebj02gcnm5excpx842u736nja8ger?keep=query", res.Header.Get("Location")) + ts, backend, root := newTestServerAndNode(t, nil, "ipns-hostname-redirects.car") + backend.namesys["/ipns/example.net"] = path.FromCid(root) + + // make request to directory containing index.html + req := mustNewRequest(t, http.MethodGet, ts.URL+"/foo", nil) + req.Host = "example.net" + res := mustDoWithoutRedirect(t, req) + + // expect 301 redirect to same path, but with trailing slash + require.Equal(t, http.StatusMovedPermanently, res.StatusCode) + hdr := res.Header["Location"] + require.Positive(t, len(hdr), "location header not present") + require.Equal(t, hdr[0], "/foo/") + + // make request with prefix to directory containing index.html + req = mustNewRequest(t, http.MethodGet, ts.URL+"/foo", nil) + req.Host = "example.net" + res = mustDoWithoutRedirect(t, req) + // expect 301 redirect to same path, but with prefix and trailing slash + require.Equal(t, http.StatusMovedPermanently, res.StatusCode) + + hdr = res.Header["Location"] + require.Positive(t, len(hdr), "location header not present") + require.Equal(t, hdr[0], "/foo/") + + // make sure /version isn't exposed + req = mustNewRequest(t, http.MethodGet, ts.URL+"/version", nil) + req.Host = "example.net" + res = mustDoWithoutRedirect(t, req) + require.Equal(t, http.StatusNotFound, res.StatusCode) }) } -func TestIpfsTrustlessMode(t *testing.T) { - backend, root := newMockBackend(t) +func TestDeserializedResponses(t *testing.T) { + t.Parallel() - ts := newTestServerWithConfig(t, backend, Config{ - Headers: map[string][]string{}, - NoDNSLink: false, - PublicGateways: map[string]*PublicGateway{ - "trustless.com": { - Paths: []string{"/ipfs", "/ipns"}, - }, - "trusted.com": { - Paths: []string{"/ipfs", "/ipns"}, - DeserializedResponses: true, + t.Run("IPFS", func(t *testing.T) { + t.Parallel() + + backend, root := newMockBackend(t, "fixtures.car") + + ts := newTestServerWithConfig(t, backend, Config{ + Headers: map[string][]string{}, + NoDNSLink: false, + PublicGateways: map[string]*PublicGateway{ + "trustless.com": { + Paths: []string{"/ipfs", "/ipns"}, + }, + "trusted.com": { + Paths: []string{"/ipfs", "/ipns"}, + DeserializedResponses: true, + }, }, - }, - }) - t.Logf("test server url: %s", ts.URL) + }) - trustedFormats := []string{"", "dag-json", "dag-cbor", "tar", "json", "cbor"} - trustlessFormats := []string{"raw", "car"} + trustedFormats := []string{"", "dag-json", "dag-cbor", "tar", "json", "cbor"} + trustlessFormats := []string{"raw", "car"} - doRequest := func(t *testing.T, path, host string, expectedStatus int) { - req, err := http.NewRequest(http.MethodGet, ts.URL+path, nil) - assert.Nil(t, err) + doRequest := func(t *testing.T, path, host string, expectedStatus int) { + req := mustNewRequest(t, http.MethodGet, ts.URL+path, nil) + if host != "" { + req.Host = host + } + res := mustDoWithoutRedirect(t, req) + defer res.Body.Close() + assert.Equal(t, expectedStatus, res.StatusCode) + } - if host != "" { - req.Host = host + doIpfsCidRequests := func(t *testing.T, formats []string, host string, expectedStatus int) { + for _, format := range formats { + doRequest(t, "/ipfs/"+root.String()+"/?format="+format, host, expectedStatus) + } } - res, err := doWithoutRedirect(req) - assert.Nil(t, err) - defer res.Body.Close() - assert.Equal(t, expectedStatus, res.StatusCode) - } + doIpfsCidPathRequests := func(t *testing.T, formats []string, host string, expectedStatus int) { + for _, format := range formats { + doRequest(t, "/ipfs/"+root.String()+"/empty-dir/?format="+format, host, expectedStatus) + } + } - doIpfsCidRequests := func(t *testing.T, formats []string, host string, expectedStatus int) { - for _, format := range formats { - doRequest(t, "/ipfs/"+root.String()+"/?format="+format, host, expectedStatus) + trustedTests := func(t *testing.T, host string) { + doIpfsCidRequests(t, trustlessFormats, host, http.StatusOK) + doIpfsCidRequests(t, trustedFormats, host, http.StatusOK) + doIpfsCidPathRequests(t, trustlessFormats, host, http.StatusOK) + doIpfsCidPathRequests(t, trustedFormats, host, http.StatusOK) } - } - doIpfsCidPathRequests := func(t *testing.T, formats []string, host string, expectedStatus int) { - for _, format := range formats { - doRequest(t, "/ipfs/"+root.String()+"/EmptyDir/?format="+format, host, expectedStatus) + trustlessTests := func(t *testing.T, host string) { + doIpfsCidRequests(t, trustlessFormats, host, http.StatusOK) + doIpfsCidRequests(t, trustedFormats, host, http.StatusNotAcceptable) + doIpfsCidPathRequests(t, trustedFormats, host, http.StatusNotAcceptable) + doIpfsCidPathRequests(t, []string{"raw"}, host, http.StatusNotAcceptable) + doIpfsCidPathRequests(t, []string{"car"}, host, http.StatusOK) } - } - trustedTests := func(t *testing.T, host string) { - doIpfsCidRequests(t, trustlessFormats, host, http.StatusOK) - doIpfsCidRequests(t, trustedFormats, host, http.StatusOK) - doIpfsCidPathRequests(t, trustlessFormats, host, http.StatusOK) - doIpfsCidPathRequests(t, trustedFormats, host, http.StatusOK) - } + t.Run("Explicit Trustless Gateway", func(t *testing.T) { + t.Parallel() + trustlessTests(t, "trustless.com") + }) - trustlessTests := func(t *testing.T, host string) { - doIpfsCidRequests(t, trustlessFormats, host, http.StatusOK) - doIpfsCidRequests(t, trustedFormats, host, http.StatusNotAcceptable) - doIpfsCidPathRequests(t, trustedFormats, host, http.StatusNotAcceptable) - doIpfsCidPathRequests(t, []string{"raw"}, host, http.StatusNotAcceptable) - doIpfsCidPathRequests(t, []string{"car"}, host, http.StatusOK) - } + t.Run("Explicit Trusted Gateway", func(t *testing.T) { + t.Parallel() + trustedTests(t, "trusted.com") + }) - t.Run("Explicit Trustless Gateway", func(t *testing.T) { - t.Parallel() - trustlessTests(t, "trustless.com") + t.Run("Implicit Default Trustless Gateway", func(t *testing.T) { + t.Parallel() + trustlessTests(t, "not.configured.com") + trustlessTests(t, "localhost") + trustlessTests(t, "127.0.0.1") + trustlessTests(t, "::1") + }) }) - t.Run("Explicit Trusted Gateway", func(t *testing.T) { + t.Run("IPNS", func(t *testing.T) { t.Parallel() - trustedTests(t, "trusted.com") - }) - t.Run("Implicit Default Trustless Gateway", func(t *testing.T) { - t.Parallel() - trustlessTests(t, "not.configured.com") - trustlessTests(t, "localhost") - trustlessTests(t, "127.0.0.1") - trustlessTests(t, "::1") + backend, root := newMockBackend(t, "fixtures.car") + backend.namesys["/ipns/trustless.com"] = path.FromCid(root) + backend.namesys["/ipns/trusted.com"] = path.FromCid(root) + + ts := newTestServerWithConfig(t, backend, Config{ + Headers: map[string][]string{}, + NoDNSLink: false, + PublicGateways: map[string]*PublicGateway{ + "trustless.com": { + Paths: []string{"/ipfs", "/ipns"}, + }, + "trusted.com": { + Paths: []string{"/ipfs", "/ipns"}, + DeserializedResponses: true, + }, + }, + }) + + doRequest := func(t *testing.T, path, host string, expectedStatus int) { + req := mustNewRequest(t, http.MethodGet, ts.URL+path, nil) + if host != "" { + req.Host = host + } + res := mustDoWithoutRedirect(t, req) + defer res.Body.Close() + assert.Equal(t, expectedStatus, res.StatusCode) + } + + // DNSLink only. Not supported for trustless. Supported for trusted, except + // format=ipns-record which is unavailable for DNSLink. + doRequest(t, "/", "trustless.com", http.StatusNotAcceptable) + doRequest(t, "/empty-dir/", "trustless.com", http.StatusNotAcceptable) + doRequest(t, "/?format=ipns-record", "trustless.com", http.StatusNotAcceptable) + + doRequest(t, "/", "trusted.com", http.StatusOK) + doRequest(t, "/empty-dir/", "trusted.com", http.StatusOK) + doRequest(t, "/?format=ipns-record", "trusted.com", http.StatusBadRequest) }) } -func TestIpnsTrustlessMode(t *testing.T) { - backend, root := newMockBackend(t) - backend.namesys["/ipns/trustless.com"] = path.FromCid(root) - backend.namesys["/ipns/trusted.com"] = path.FromCid(root) - - ts := newTestServerWithConfig(t, backend, Config{ - Headers: map[string][]string{}, - NoDNSLink: false, - PublicGateways: map[string]*PublicGateway{ - "trustless.com": { - Paths: []string{"/ipfs", "/ipns"}, - }, - "trusted.com": { - Paths: []string{"/ipfs", "/ipns"}, - DeserializedResponses: true, - }, - }, - }) - t.Logf("test server url: %s", ts.URL) +type errorMockBackend struct { + err error +} - doRequest := func(t *testing.T, path, host string, expectedStatus int) { - req, err := http.NewRequest(http.MethodGet, ts.URL+path, nil) - assert.Nil(t, err) +func (mb *errorMockBackend) Get(ctx context.Context, path ImmutablePath, getRange ...ByteRange) (ContentPathMetadata, *GetResponse, error) { + return ContentPathMetadata{}, nil, mb.err +} - if host != "" { - req.Host = host - } +func (mb *errorMockBackend) GetAll(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.Node, error) { + return ContentPathMetadata{}, nil, mb.err +} - res, err := doWithoutRedirect(req) - assert.Nil(t, err) - defer res.Body.Close() - assert.Equal(t, expectedStatus, res.StatusCode) +func (mb *errorMockBackend) GetBlock(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.File, error) { + return ContentPathMetadata{}, nil, mb.err +} + +func (mb *errorMockBackend) Head(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.Node, error) { + return ContentPathMetadata{}, nil, mb.err +} + +func (mb *errorMockBackend) GetCAR(ctx context.Context, path ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) { + return ContentPathMetadata{}, nil, mb.err +} + +func (mb *errorMockBackend) ResolveMutable(ctx context.Context, path ipath.Path) (ImmutablePath, error) { + return ImmutablePath{}, mb.err +} + +func (mb *errorMockBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { + return nil, mb.err +} + +func (mb *errorMockBackend) GetDNSLinkRecord(ctx context.Context, hostname string) (ipath.Path, error) { + return nil, mb.err +} + +func (mb *errorMockBackend) IsCached(ctx context.Context, p ipath.Path) bool { + return false +} + +func (mb *errorMockBackend) ResolvePath(ctx context.Context, path ImmutablePath) (ContentPathMetadata, error) { + return ContentPathMetadata{}, mb.err +} + +func TestErrorBubblingFromBackend(t *testing.T) { + t.Parallel() + + testError := func(name string, err error, status int) { + t.Run(name, func(t *testing.T) { + t.Parallel() + + backend := &errorMockBackend{err: fmt.Errorf("wrapped for testing purposes: %w", err)} + ts := newTestServer(t, backend) + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipns/en.wikipedia-on-ipfs.org", nil) + res := mustDo(t, req) + require.Equal(t, status, res.StatusCode) + }) } - // DNSLink only. Not supported for trustless. Supported for trusted, except - // format=ipns-record which is unavailable for DNSLink. - doRequest(t, "/", "trustless.com", http.StatusNotAcceptable) - doRequest(t, "/EmptyDir/", "trustless.com", http.StatusNotAcceptable) - doRequest(t, "/?format=ipns-record", "trustless.com", http.StatusNotAcceptable) + testError("404 Not Found from IPLD", &ipld.ErrNotFound{}, http.StatusNotFound) + testError("404 Not Found from path resolver", resolver.ErrNoLink{}, http.StatusNotFound) + testError("502 Bad Gateway", ErrBadGateway, http.StatusBadGateway) + testError("504 Gateway Timeout", ErrGatewayTimeout, http.StatusGatewayTimeout) + + testErrorRetryAfter := func(name string, err error, status int, headerValue string, headerLength int) { + t.Run(name, func(t *testing.T) { + t.Parallel() + + backend := &errorMockBackend{err: fmt.Errorf("wrapped for testing purposes: %w", err)} + ts := newTestServer(t, backend) - doRequest(t, "/", "trusted.com", http.StatusOK) - doRequest(t, "/EmptyDir/", "trusted.com", http.StatusOK) - doRequest(t, "/?format=ipns-record", "trusted.com", http.StatusBadRequest) + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipns/en.wikipedia-on-ipfs.org", nil) + res := mustDo(t, req) + require.Equal(t, status, res.StatusCode) + require.Equal(t, headerValue, res.Header.Get("Retry-After")) + require.Equal(t, headerLength, len(res.Header.Values("Retry-After"))) + }) + } + + testErrorRetryAfter("429 Too Many Requests without Retry-After header", ErrTooManyRequests, http.StatusTooManyRequests, "", 0) + testErrorRetryAfter("429 Too Many Requests without Retry-After header", NewErrorRetryAfter(ErrTooManyRequests, 0*time.Second), http.StatusTooManyRequests, "", 0) + testErrorRetryAfter("429 Too Many Requests with Retry-After header", NewErrorRetryAfter(ErrTooManyRequests, 3600*time.Second), http.StatusTooManyRequests, "3600", 1) } -func TestDagJsonCborPreview(t *testing.T) { - backend, root := newMockBackend(t) - - ts := newTestServerWithConfig(t, backend, Config{ - Headers: map[string][]string{}, - NoDNSLink: false, - PublicGateways: map[string]*PublicGateway{ - "example.com": { - Paths: []string{"/ipfs", "/ipns"}, - UseSubdomains: true, - DeserializedResponses: true, - }, - }, - DeserializedResponses: true, - }) - t.Logf("test server url: %s", ts.URL) +type panicMockBackend struct { + panicOnHostnameHandler bool +} - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() +func (mb *panicMockBackend) Get(ctx context.Context, immutablePath ImmutablePath, ranges ...ByteRange) (ContentPathMetadata, *GetResponse, error) { + panic("i am panicking") +} + +func (mb *panicMockBackend) GetAll(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.Node, error) { + panic("i am panicking") +} + +func (mb *panicMockBackend) GetBlock(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.File, error) { + panic("i am panicking") +} + +func (mb *panicMockBackend) Head(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.Node, error) { + panic("i am panicking") +} + +func (mb *panicMockBackend) GetCAR(ctx context.Context, immutablePath ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) { + panic("i am panicking") +} + +func (mb *panicMockBackend) ResolveMutable(ctx context.Context, p ipath.Path) (ImmutablePath, error) { + panic("i am panicking") +} + +func (mb *panicMockBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { + panic("i am panicking") +} + +func (mb *panicMockBackend) GetDNSLinkRecord(ctx context.Context, hostname string) (ipath.Path, error) { + // GetDNSLinkRecord is also called on the WithHostname handler. We have this option + // to disable panicking here so we can test if both the regular gateway handler + // and the hostname handler can handle panics. + if mb.panicOnHostnameHandler { + panic("i am panicking") + } - resolvedPath, err := backend.resolvePathNoRootsReturned(ctx, ipath.Join(ipath.IpfsPath(root), t.Name(), "example")) - assert.NoError(t, err) + return nil, errors.New("not implemented") +} - cidStr := resolvedPath.Cid().String() +func (mb *panicMockBackend) IsCached(ctx context.Context, p ipath.Path) bool { + panic("i am panicking") +} - t.Run("path gateway normalizes to trailing slash", func(t *testing.T) { +func (mb *panicMockBackend) ResolvePath(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, error) { + panic("i am panicking") +} + +func TestPanicStatusCode(t *testing.T) { + t.Parallel() + + t.Run("Panic on Handler", func(t *testing.T) { t.Parallel() - req, err := http.NewRequest(http.MethodGet, ts.URL+"/ipfs/"+cidStr, nil) - req.Header.Add("Accept", "text/html") - assert.NoError(t, err) + backend := &panicMockBackend{} + ts := newTestServer(t, backend) + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e", nil) + res := mustDo(t, req) + require.Equal(t, http.StatusInternalServerError, res.StatusCode) + }) + + t.Run("Panic on Hostname Handler", func(t *testing.T) { + t.Parallel() - res, err := doWithoutRedirect(req) - assert.NoError(t, err) - assert.Equal(t, http.StatusMovedPermanently, res.StatusCode) - assert.Equal(t, "/ipfs/"+cidStr+"/", res.Header.Get("Location")) + backend := &panicMockBackend{panicOnHostnameHandler: true} + ts := newTestServer(t, backend) + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e", nil) + res := mustDo(t, req) + require.Equal(t, http.StatusInternalServerError, res.StatusCode) }) +} - t.Run("subdomain gateway correctly redirects", func(t *testing.T) { +func TestBrowserErrorHTML(t *testing.T) { + t.Parallel() + ts, _, root := newTestServerAndNode(t, nil, "fixtures.car") + + t.Run("plain error if request does not have Accept: text/html", func(t *testing.T) { t.Parallel() - req, err := http.NewRequest(http.MethodGet, ts.URL+"/ipfs/"+cidStr, nil) - req.Header.Add("Accept", "text/html") - req.Host = "example.com" - assert.NoError(t, err) + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+root.String()+"/nonexisting-link", nil) + res := mustDoWithoutRedirect(t, req) + require.Equal(t, http.StatusNotFound, res.StatusCode) + require.NotContains(t, res.Header.Get("Content-Type"), "text/html") - res, err := doWithoutRedirect(req) - assert.NoError(t, err) - assert.Equal(t, http.StatusMovedPermanently, res.StatusCode) - assert.Equal(t, "http://"+cidStr+".ipfs.example.com/", res.Header.Get("Location")) + body, err := io.ReadAll(res.Body) + require.NoError(t, err) + require.NotContains(t, string(body), "") }) - t.Run("preview strings are correctly escaped", func(t *testing.T) { + t.Run("html error if request has Accept: text/html", func(t *testing.T) { t.Parallel() - req, err := http.NewRequest(http.MethodGet, ts.URL+resolvedPath.String()+"/", nil) - req.Header.Add("Accept", "text/html") - assert.NoError(t, err) + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+root.String()+"/nonexisting-link", nil) + req.Header.Set("Accept", "text/html") - res, err := doWithoutRedirect(req) - assert.NoError(t, err) - assert.Equal(t, http.StatusOK, res.StatusCode) + res := mustDoWithoutRedirect(t, req) + require.Equal(t, http.StatusNotFound, res.StatusCode) + require.Contains(t, res.Header.Get("Content-Type"), "text/html") body, err := io.ReadAll(res.Body) - assert.NoError(t, err) - - script := "window.alert('hacked')" - escaped := html.EscapeString(script) - - assert.Contains(t, string(body), escaped) - assert.NotContains(t, string(body), script) + require.NoError(t, err) + require.Contains(t, string(body), "") }) } diff --git a/gateway/handler.go b/gateway/handler.go index cb695408d..c53dc8e5c 100644 --- a/gateway/handler.go +++ b/gateway/handler.go @@ -23,6 +23,7 @@ import ( logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multibase" + mc "github.com/multiformats/go-multicodec" prometheus "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -171,6 +172,37 @@ func (i *handler) optionsHandler(w http.ResponseWriter, r *http.Request) { i.addUserHeaders(w) // return all custom headers (including CORS ones, if set) } +type requestData struct { + // Defined for all requests. + begin time.Time + logger *zap.SugaredLogger + contentPath ipath.Path + responseFormat string + responseParams map[string]string + + // Defined for non IPNS Record requests. + immutablePath ImmutablePath + + // Defined if resolution has already happened. + pathMetadata *ContentPathMetadata +} + +// mostlyResolvedPath is an opportunistic optimization that returns the mostly +// resolved version of ImmutablePath available. It does not guarantee it is fully +// resolved, nor that it is the original. +func (rq *requestData) mostlyResolvedPath() ImmutablePath { + if rq.pathMetadata != nil { + imPath, err := NewImmutablePath(rq.pathMetadata.LastSegment) + if err != nil { + // This will never happen. This error has previously been checked in + // [handleIfNoneMatch] and the request will have returned 500. + panic(err) + } + return imPath + } + return rq.immutablePath +} + func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) { begin := time.Now() @@ -223,25 +255,32 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) { return } + rq := &requestData{ + begin: begin, + logger: logger, + contentPath: contentPath, + responseFormat: responseFormat, + responseParams: formatParams, + } + // IPNS Record response format can be handled now, since (1) it needs the // non-resolved mutable path, and (2) has custom If-None-Match header handling // due to custom ETag. if responseFormat == ipnsRecordResponseFormat { logger.Debugw("serving ipns record", "path", contentPath) - success = i.serveIpnsRecord(r.Context(), w, r, contentPath, begin, logger) + success = i.serveIpnsRecord(r.Context(), w, r, rq) return } - var immutableContentPath ImmutablePath if contentPath.Mutable() { - immutableContentPath, err = i.backend.ResolveMutable(r.Context(), contentPath) + rq.immutablePath, err = i.backend.ResolveMutable(r.Context(), contentPath) if err != nil { err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err) i.webError(w, r, err, http.StatusInternalServerError) return } } else { - immutableContentPath, err = NewImmutablePath(contentPath) + rq.immutablePath, err = NewImmutablePath(contentPath) if err != nil { err = fmt.Errorf("path was expected to be immutable, but was not %s: %w", debugStr(contentPath.String()), err) i.webError(w, r, err, http.StatusInternalServerError) @@ -254,36 +293,28 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) { // header handling due to custom ETag. if responseFormat == carResponseFormat { logger.Debugw("serving car stream", "path", contentPath) - carVersion := formatParams["version"] - success = i.serveCAR(r.Context(), w, r, immutableContentPath, contentPath, carVersion, begin) + success = i.serveCAR(r.Context(), w, r, rq) return } // Detect when If-None-Match HTTP header allows returning HTTP 304 Not Modified. - ifNoneMatchResolvedPath, handled := i.handleIfNoneMatch(w, r, responseFormat, contentPath, immutableContentPath) - if handled { + if i.handleIfNoneMatch(w, r, rq) { return } - // If we already did the path resolution no need to do it again - maybeResolvedImPath := immutableContentPath - if ifNoneMatchResolvedPath != nil { - maybeResolvedImPath = *ifNoneMatchResolvedPath - } - // Support custom response formats passed via ?format or Accept HTTP header switch responseFormat { case "", jsonResponseFormat, cborResponseFormat: - success = i.serveDefaults(r.Context(), w, r, maybeResolvedImPath, immutableContentPath, contentPath, begin, responseFormat, logger) + success = i.serveDefaults(r.Context(), w, r, rq) case rawResponseFormat: logger.Debugw("serving raw block", "path", contentPath) - success = i.serveRawBlock(r.Context(), w, r, maybeResolvedImPath, contentPath, begin) + success = i.serveRawBlock(r.Context(), w, r, rq) case tarResponseFormat: logger.Debugw("serving tar file", "path", contentPath) - success = i.serveTAR(r.Context(), w, r, maybeResolvedImPath, contentPath, begin, logger) + success = i.serveTAR(r.Context(), w, r, rq) case dagJsonResponseFormat, dagCborResponseFormat: logger.Debugw("serving codec", "path", contentPath) - success = i.serveCodec(r.Context(), w, r, maybeResolvedImPath, contentPath, begin, responseFormat) + success = i.serveCodec(r.Context(), w, r, rq) default: // catch-all for unsuported application/vnd.* err := fmt.Errorf("unsupported format %q", responseFormat) i.webError(w, r, err, http.StatusBadRequest) @@ -446,7 +477,12 @@ func setContentDispositionHeader(w http.ResponseWriter, filename string, disposi // setIpfsRootsHeader sets the X-Ipfs-Roots header with logical CID array for // efficient HTTP cache invalidation. -func setIpfsRootsHeader(w http.ResponseWriter, pathMetadata ContentPathMetadata) { +func setIpfsRootsHeader(w http.ResponseWriter, rq *requestData, md *ContentPathMetadata) { + // Update requestData with the latest ContentPathMetadata if it wasn't set yet. + if rq.pathMetadata == nil { + rq.pathMetadata = md + } + // These are logical roots where each CID represent one path segment // and resolves to either a directory or the root block of a file. // The main purpose of this header is allow HTTP caches to do smarter decisions @@ -469,10 +505,10 @@ func setIpfsRootsHeader(w http.ResponseWriter, pathMetadata ContentPathMetadata) // the last root (responsible for specific article) may not change at all. var pathRoots []string - for _, c := range pathMetadata.PathSegmentRoots { + for _, c := range rq.pathMetadata.PathSegmentRoots { pathRoots = append(pathRoots, c.String()) } - pathRoots = append(pathRoots, pathMetadata.LastSegment.Cid().String()) + pathRoots = append(pathRoots, rq.pathMetadata.LastSegment.Cid().String()) rootCidList := strings.Join(pathRoots, ",") // convention from rfc2616#sec4.2 w.Header().Set("X-Ipfs-Roots", rootCidList) @@ -552,6 +588,14 @@ func getEtag(r *http.Request, cid cid.Cid, responseFormat string) string { prefix := `"` suffix := `"` + // For Codecs, ensure that we have the right content-type. + if responseFormat == "" { + cidCodec := mc.Code(cid.Prefix().Codec) + if contentType, ok := codecToContentType[cidCodec]; ok { + responseFormat = contentType + } + } + switch responseFormat { case "": // Do nothing. @@ -645,40 +689,42 @@ func debugStr(path string) string { return q } -func (i *handler) handleIfNoneMatch(w http.ResponseWriter, r *http.Request, responseFormat string, contentPath ipath.Path, imPath ImmutablePath) (*ImmutablePath, bool) { +func (i *handler) handleIfNoneMatch(w http.ResponseWriter, r *http.Request, rq *requestData) bool { // Detect when If-None-Match HTTP header allows returning HTTP 304 Not Modified if ifNoneMatch := r.Header.Get("If-None-Match"); ifNoneMatch != "" { - pathMetadata, err := i.backend.ResolvePath(r.Context(), imPath) + pathMetadata, err := i.backend.ResolvePath(r.Context(), rq.immutablePath) if err != nil { - err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err) + err = fmt.Errorf("failed to resolve %s: %w", debugStr(rq.contentPath.String()), err) i.webError(w, r, err, http.StatusInternalServerError) - return nil, true + return true } - resolvedPath := pathMetadata.LastSegment - pathCid := resolvedPath.Cid() + pathCid := pathMetadata.LastSegment.Cid() // Checks against both file, dir listing, and dag index Etags. // This is an inexpensive check, and it happens before we do any I/O. - cidEtag := getEtag(r, pathCid, responseFormat) + cidEtag := getEtag(r, pathCid, rq.responseFormat) dirEtag := getDirListingEtag(pathCid) dagEtag := getDagIndexEtag(pathCid) if etagMatch(ifNoneMatch, cidEtag, dirEtag, dagEtag) { // Finish early if client already has a matching Etag w.WriteHeader(http.StatusNotModified) - return nil, true + return true } - resolvedImPath, err := NewImmutablePath(resolvedPath) + // Check if the resolvedPath is an immutable path. + _, err = NewImmutablePath(pathMetadata.LastSegment) if err != nil { i.webError(w, r, err, http.StatusInternalServerError) - return nil, true + return true } - return &resolvedImPath, true + rq.pathMetadata = &pathMetadata + return false } - return nil, false + + return false } // check if request was for one of known explicit formats, diff --git a/gateway/handler_block.go b/gateway/handler_block.go index 9708a46ef..dbff9a7ad 100644 --- a/gateway/handler_block.go +++ b/gateway/handler_block.go @@ -5,23 +5,22 @@ import ( "net/http" "time" - ipath "github.com/ipfs/boxo/coreiface/path" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) // serveRawBlock returns bytes behind a raw block -func (i *handler) serveRawBlock(ctx context.Context, w http.ResponseWriter, r *http.Request, imPath ImmutablePath, contentPath ipath.Path, begin time.Time) bool { - ctx, span := spanTrace(ctx, "Handler.ServeRawBlock", trace.WithAttributes(attribute.String("path", imPath.String()))) +func (i *handler) serveRawBlock(ctx context.Context, w http.ResponseWriter, r *http.Request, rq *requestData) bool { + ctx, span := spanTrace(ctx, "Handler.ServeRawBlock", trace.WithAttributes(attribute.String("path", rq.immutablePath.String()))) defer span.End() - pathMetadata, data, err := i.backend.GetBlock(ctx, imPath) - if !i.handleRequestErrors(w, r, contentPath, err) { + pathMetadata, data, err := i.backend.GetBlock(ctx, rq.mostlyResolvedPath()) + if !i.handleRequestErrors(w, r, rq.contentPath, err) { return false } defer data.Close() - setIpfsRootsHeader(w, pathMetadata) + setIpfsRootsHeader(w, rq, &pathMetadata) blockCid := pathMetadata.LastSegment.Cid() @@ -35,7 +34,7 @@ func (i *handler) serveRawBlock(ctx context.Context, w http.ResponseWriter, r *h setContentDispositionHeader(w, name, "attachment") // Set remaining headers - modtime := addCacheControlHeaders(w, r, contentPath, blockCid, rawResponseFormat) + modtime := addCacheControlHeaders(w, r, rq.contentPath, blockCid, rawResponseFormat) w.Header().Set("Content-Type", rawResponseFormat) w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^) @@ -45,7 +44,7 @@ func (i *handler) serveRawBlock(ctx context.Context, w http.ResponseWriter, r *h if dataSent { // Update metrics - i.rawBlockGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) + i.rawBlockGetMetric.WithLabelValues(rq.contentPath.Namespace()).Observe(time.Since(rq.begin).Seconds()) } return dataSent diff --git a/gateway/handler_car.go b/gateway/handler_car.go index a8be3ded9..e773c920e 100644 --- a/gateway/handler_car.go +++ b/gateway/handler_car.go @@ -10,7 +10,6 @@ import ( "time" "github.com/cespare/xxhash/v2" - ipath "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/go-cid" "go.opentelemetry.io/otel/attribute" @@ -24,14 +23,14 @@ const ( ) // serveCAR returns a CAR stream for specific DAG+selector -func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.Request, imPath ImmutablePath, contentPath ipath.Path, carVersion string, begin time.Time) bool { - ctx, span := spanTrace(ctx, "Handler.ServeCAR", trace.WithAttributes(attribute.String("path", imPath.String()))) +func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.Request, rq *requestData) bool { + ctx, span := spanTrace(ctx, "Handler.ServeCAR", trace.WithAttributes(attribute.String("path", rq.immutablePath.String()))) defer span.End() ctx, cancel := context.WithCancel(ctx) defer cancel() - switch carVersion { + switch rq.responseParams["version"] { case "": // noop, client does not care about version case "1": // noop, we support this default: @@ -46,7 +45,7 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R return false } - rootCid, lastSegment, err := getCarRootCidAndLastSegment(imPath) + rootCid, lastSegment, err := getCarRootCidAndLastSegment(rq.immutablePath) if err != nil { i.webError(w, r, err, http.StatusInternalServerError) return false @@ -66,10 +65,10 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R setContentDispositionHeader(w, name, "attachment") // Set Cache-Control (same logic as for a regular files) - addCacheControlHeaders(w, r, contentPath, rootCid, carResponseFormat) + addCacheControlHeaders(w, r, rq.contentPath, rootCid, carResponseFormat) // Generate the CAR Etag. - etag := getCarEtag(imPath, params, rootCid) + etag := getCarEtag(rq.immutablePath, params, rootCid) w.Header().Set("Etag", etag) // Terminate early if Etag matches. We cannot rely on handleIfNoneMatch since @@ -79,12 +78,12 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R return false } - md, carFile, err := i.backend.GetCAR(ctx, imPath, params) - if !i.handleRequestErrors(w, r, contentPath, err) { + md, carFile, err := i.backend.GetCAR(ctx, rq.immutablePath, params) + if !i.handleRequestErrors(w, r, rq.contentPath, err) { return false } defer carFile.Close() - setIpfsRootsHeader(w, md) + setIpfsRootsHeader(w, rq, &md) // Make it clear we don't support range-requests over a car stream // Partial downloads and resumes should be handled using requests for @@ -99,7 +98,7 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R streamErr := multierr.Combine(carErr, copyErr) if streamErr != nil { // Update fail metric - i.carStreamFailMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) + i.carStreamFailMetric.WithLabelValues(rq.contentPath.Namespace()).Observe(time.Since(rq.begin).Seconds()) // We return error as a trailer, however it is not something browsers can access // (https://github.com/mdn/browser-compat-data/issues/14703) @@ -110,7 +109,7 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R } // Update metrics - i.carStreamGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) + i.carStreamGetMetric.WithLabelValues(rq.contentPath.Namespace()).Observe(time.Since(rq.begin).Seconds()) return true } diff --git a/gateway/handler_car_test.go b/gateway/handler_car_test.go index d603de11e..858ccb85d 100644 --- a/gateway/handler_car_test.go +++ b/gateway/handler_car_test.go @@ -7,9 +7,12 @@ import ( "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/go-cid" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestCarParams(t *testing.T) { + t.Parallel() + t.Run("dag-scope parsing", func(t *testing.T) { t.Parallel() @@ -24,11 +27,8 @@ func TestCarParams(t *testing.T) { {"dag-scope=what-is-this", "", true}, } for _, test := range tests { - r, err := http.NewRequest(http.MethodGet, "http://example.com/?"+test.query, nil) - assert.NoError(t, err) - + r := mustNewRequest(t, http.MethodGet, "http://example.com/?"+test.query, nil) params, err := getCarParams(r) - if test.expectedError { assert.Error(t, err) } else { @@ -59,11 +59,8 @@ func TestCarParams(t *testing.T) { {"entity-bytes=123:bbb", true, 0, true, 0}, } for _, test := range tests { - r, err := http.NewRequest(http.MethodGet, "http://example.com/?"+test.query, nil) - assert.NoError(t, err) - + r := mustNewRequest(t, http.MethodGet, "http://example.com/?"+test.query, nil) params, err := getCarParams(r) - if test.hasError { assert.Error(t, err) } else { @@ -78,19 +75,21 @@ func TestCarParams(t *testing.T) { }) } -func TestCarEtag(t *testing.T) { +func TestGetCarEtag(t *testing.T) { + t.Parallel() + cid, err := cid.Parse("bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4") - assert.NoError(t, err) + require.NoError(t, err) imPath, err := NewImmutablePath(path.IpfsPath(cid)) - assert.NoError(t, err) + require.NoError(t, err) t.Run("Etag with entity-bytes=0:* is the same as without query param", func(t *testing.T) { t.Parallel() noRange := getCarEtag(imPath, CarParams{}, cid) withRange := getCarEtag(imPath, CarParams{Range: &DagByteRange{From: 0}}, cid) - assert.Equal(t, noRange, withRange) + require.Equal(t, noRange, withRange) }) t.Run("Etag with entity-bytes=1:* is different than without query param", func(t *testing.T) { @@ -98,7 +97,7 @@ func TestCarEtag(t *testing.T) { noRange := getCarEtag(imPath, CarParams{}, cid) withRange := getCarEtag(imPath, CarParams{Range: &DagByteRange{From: 1}}, cid) - assert.NotEqual(t, noRange, withRange) + require.NotEqual(t, noRange, withRange) }) t.Run("Etags with different dag-scope are different", func(t *testing.T) { @@ -106,6 +105,6 @@ func TestCarEtag(t *testing.T) { a := getCarEtag(imPath, CarParams{Scope: DagScopeAll}, cid) b := getCarEtag(imPath, CarParams{Scope: DagScopeEntity}, cid) - assert.NotEqual(t, a, b) + require.NotEqual(t, a, b) }) } diff --git a/gateway/handler_codec.go b/gateway/handler_codec.go index e7e5c3869..007a52fda 100644 --- a/gateway/handler_codec.go +++ b/gateway/handler_codec.go @@ -58,29 +58,28 @@ var contentTypeToExtension = map[string]string{ dagCborResponseFormat: ".cbor", } -func (i *handler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, imPath ImmutablePath, contentPath ipath.Path, begin time.Time, requestedContentType string) bool { - ctx, span := spanTrace(ctx, "Handler.ServeCodec", trace.WithAttributes(attribute.String("path", imPath.String()), attribute.String("requestedContentType", requestedContentType))) +func (i *handler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, rq *requestData) bool { + ctx, span := spanTrace(ctx, "Handler.ServeCodec", trace.WithAttributes(attribute.String("path", rq.immutablePath.String()), attribute.String("requestedContentType", rq.responseFormat))) defer span.End() - pathMetadata, data, err := i.backend.GetBlock(ctx, imPath) - if !i.handleRequestErrors(w, r, contentPath, err) { + pathMetadata, data, err := i.backend.GetBlock(ctx, rq.mostlyResolvedPath()) + if !i.handleRequestErrors(w, r, rq.contentPath, err) { return false } defer data.Close() - setIpfsRootsHeader(w, pathMetadata) - - resolvedPath := pathMetadata.LastSegment - return i.renderCodec(ctx, w, r, resolvedPath, data, contentPath, begin, requestedContentType) + setIpfsRootsHeader(w, rq, &pathMetadata) + return i.renderCodec(ctx, w, r, rq, data) } -func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, blockData io.ReadSeekCloser, contentPath ipath.Path, begin time.Time, requestedContentType string) bool { - ctx, span := spanTrace(ctx, "Handler.RenderCodec", trace.WithAttributes(attribute.String("path", resolvedPath.String()), attribute.String("requestedContentType", requestedContentType))) +func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, rq *requestData, blockData io.ReadSeekCloser) bool { + resolvedPath := rq.pathMetadata.LastSegment + ctx, span := spanTrace(ctx, "Handler.RenderCodec", trace.WithAttributes(attribute.String("path", resolvedPath.String()), attribute.String("requestedContentType", rq.responseFormat))) defer span.End() blockCid := resolvedPath.Cid() cidCodec := mc.Code(blockCid.Prefix().Codec) - responseContentType := requestedContentType + responseContentType := rq.responseFormat // If the resolved path still has some remainder, return error for now. // TODO: handle this when we have IPLD Patch (https://ipld.io/specs/patch/) via HTTP PUT @@ -93,7 +92,7 @@ func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *htt } // If no explicit content type was requested, the response will have one based on the codec from the CID - if requestedContentType == "" { + if rq.responseFormat == "" { cidContentType, ok := codecToContentType[cidCodec] if !ok { // Should not happen unless function is called with wrong parameters. @@ -105,49 +104,49 @@ func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *htt } // Set HTTP headers (for caching, etc). Etag will be replaced if handled by serveCodecHTML. - modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid(), responseContentType) + modtime := addCacheControlHeaders(w, r, rq.contentPath, resolvedPath.Cid(), responseContentType) name := setCodecContentDisposition(w, r, resolvedPath, responseContentType) w.Header().Set("Content-Type", responseContentType) w.Header().Set("X-Content-Type-Options", "nosniff") // No content type is specified by the user (via Accept, or format=). However, // we support this format. Let's handle it. - if requestedContentType == "" { + if rq.responseFormat == "" { isDAG := cidCodec == mc.DagJson || cidCodec == mc.DagCbor acceptsHTML := strings.Contains(r.Header.Get("Accept"), "text/html") download := r.URL.Query().Get("download") == "true" if isDAG && acceptsHTML && !download { - return i.serveCodecHTML(ctx, w, r, blockCid, blockData, resolvedPath, contentPath) + return i.serveCodecHTML(ctx, w, r, blockCid, blockData, resolvedPath, rq.contentPath) } else { // This covers CIDs with codec 'json' and 'cbor' as those do not have // an explicit requested content type. - return i.serveCodecRaw(ctx, w, r, blockData, contentPath, name, modtime, begin) + return i.serveCodecRaw(ctx, w, r, blockData, rq.contentPath, name, modtime, rq.begin) } } // If DAG-JSON or DAG-CBOR was requested using corresponding plain content type // return raw block as-is, without conversion - skipCodecs, ok := contentTypeToRaw[requestedContentType] + skipCodecs, ok := contentTypeToRaw[rq.responseFormat] if ok { for _, skipCodec := range skipCodecs { if skipCodec == cidCodec { - return i.serveCodecRaw(ctx, w, r, blockData, contentPath, name, modtime, begin) + return i.serveCodecRaw(ctx, w, r, blockData, rq.contentPath, name, modtime, rq.begin) } } } // Otherwise, the user has requested a specific content type (a DAG-* variant). // Let's first get the codecs that can be used with this content type. - toCodec, ok := contentTypeToCodec[requestedContentType] + toCodec, ok := contentTypeToCodec[rq.responseFormat] if !ok { - err := fmt.Errorf("converting from %q to %q is not supported", cidCodec.String(), requestedContentType) + err := fmt.Errorf("converting from %q to %q is not supported", cidCodec.String(), rq.responseFormat) i.webError(w, r, err, http.StatusBadRequest) return false } // This handles DAG-* conversions and validations. - return i.serveCodecConverted(ctx, w, r, blockCid, blockData, contentPath, toCodec, modtime, begin) + return i.serveCodecConverted(ctx, w, r, blockCid, blockData, rq.contentPath, toCodec, modtime, rq.begin) } func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r *http.Request, blockCid cid.Cid, blockData io.ReadSeekCloser, resolvedPath ipath.Resolved, contentPath ipath.Path) bool { diff --git a/gateway/handler_codec_test.go b/gateway/handler_codec_test.go new file mode 100644 index 000000000..c79b07689 --- /dev/null +++ b/gateway/handler_codec_test.go @@ -0,0 +1,80 @@ +package gateway + +import ( + "context" + "html" + "io" + "net/http" + "testing" + + ipath "github.com/ipfs/boxo/coreiface/path" + "github.com/stretchr/testify/require" +) + +func TestDagJsonCborPreview(t *testing.T) { + t.Parallel() + backend, root := newMockBackend(t, "fixtures.car") + + ts := newTestServerWithConfig(t, backend, Config{ + Headers: map[string][]string{}, + NoDNSLink: false, + PublicGateways: map[string]*PublicGateway{ + "example.com": { + Paths: []string{"/ipfs", "/ipns"}, + UseSubdomains: true, + DeserializedResponses: true, + }, + }, + DeserializedResponses: true, + }) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + resolvedPath, err := backend.resolvePathNoRootsReturned(ctx, ipath.Join(ipath.IpfsPath(root), "subdir", "dag-cbor-document")) + require.NoError(t, err) + + cidStr := resolvedPath.Cid().String() + + t.Run("path gateway normalizes to trailing slash", func(t *testing.T) { + t.Parallel() + + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+cidStr, nil) + req.Header.Add("Accept", "text/html") + + res := mustDoWithoutRedirect(t, req) + require.Equal(t, http.StatusMovedPermanently, res.StatusCode) + require.Equal(t, "/ipfs/"+cidStr+"/", res.Header.Get("Location")) + }) + + t.Run("subdomain gateway correctly redirects", func(t *testing.T) { + t.Parallel() + + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+cidStr, nil) + req.Header.Add("Accept", "text/html") + req.Host = "example.com" + + res := mustDoWithoutRedirect(t, req) + require.Equal(t, http.StatusMovedPermanently, res.StatusCode) + require.Equal(t, "http://"+cidStr+".ipfs.example.com/", res.Header.Get("Location")) + }) + + t.Run("preview strings are correctly escaped", func(t *testing.T) { + t.Parallel() + + req := mustNewRequest(t, http.MethodGet, ts.URL+resolvedPath.String()+"/", nil) + req.Header.Add("Accept", "text/html") + + res := mustDoWithoutRedirect(t, req) + require.Equal(t, http.StatusOK, res.StatusCode) + + body, err := io.ReadAll(res.Body) + require.NoError(t, err) + + script := "window.alert('hacked')" + escaped := html.EscapeString(script) + + require.Contains(t, string(body), escaped) + require.NotContains(t, string(body), script) + }) +} diff --git a/gateway/handler_defaults.go b/gateway/handler_defaults.go index 5ccfed537..8e96d8b15 100644 --- a/gateway/handler_defaults.go +++ b/gateway/handler_defaults.go @@ -8,19 +8,16 @@ import ( "net/textproto" "strconv" "strings" - "time" "github.com/ipfs/boxo/files" mc "github.com/multiformats/go-multicodec" - ipath "github.com/ipfs/boxo/coreiface/path" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" - "go.uber.org/zap" ) -func (i *handler) serveDefaults(ctx context.Context, w http.ResponseWriter, r *http.Request, maybeResolvedImPath ImmutablePath, immutableContentPath ImmutablePath, contentPath ipath.Path, begin time.Time, requestedContentType string, logger *zap.SugaredLogger) bool { - ctx, span := spanTrace(ctx, "Handler.ServeDefaults", trace.WithAttributes(attribute.String("path", contentPath.String()))) +func (i *handler) serveDefaults(ctx context.Context, w http.ResponseWriter, r *http.Request, rq *requestData) bool { + ctx, span := spanTrace(ctx, "Handler.ServeDefaults", trace.WithAttributes(attribute.String("path", rq.contentPath.String()))) defer span.End() var ( @@ -35,8 +32,8 @@ func (i *handler) serveDefaults(ctx context.Context, w http.ResponseWriter, r *h switch r.Method { case http.MethodHead: var data files.Node - pathMetadata, data, err = i.backend.Head(ctx, maybeResolvedImPath) - if !i.handleRequestErrors(w, r, contentPath, err) { + pathMetadata, data, err = i.backend.Head(ctx, rq.mostlyResolvedPath()) + if !i.handleRequestErrors(w, r, rq.contentPath, err) { return false } defer data.Close() @@ -65,21 +62,21 @@ func (i *handler) serveDefaults(ctx context.Context, w http.ResponseWriter, r *h // allow backend to find providers for parents, even when internal // CIDs are not announced, and will provide better key for caching // related DAGs. - pathMetadata, getResp, err = i.backend.Get(ctx, maybeResolvedImPath, ranges...) + pathMetadata, getResp, err = i.backend.Get(ctx, rq.mostlyResolvedPath(), ranges...) if err != nil { - if isWebRequest(requestedContentType) { - forwardedPath, continueProcessing := i.handleWebRequestErrors(w, r, maybeResolvedImPath, immutableContentPath, contentPath, err, logger) + if isWebRequest(rq.responseFormat) { + forwardedPath, continueProcessing := i.handleWebRequestErrors(w, r, rq.mostlyResolvedPath(), rq.immutablePath, rq.contentPath, err, rq.logger) if !continueProcessing { return false } pathMetadata, getResp, err = i.backend.Get(ctx, forwardedPath, ranges...) if err != nil { - err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err) + err = fmt.Errorf("failed to resolve %s: %w", debugStr(rq.contentPath.String()), err) i.webError(w, r, err, http.StatusInternalServerError) return false } } else { - if !i.handleRequestErrors(w, r, contentPath, err) { + if !i.handleRequestErrors(w, r, rq.contentPath, err) { return false } } @@ -97,8 +94,7 @@ func (i *handler) serveDefaults(ctx context.Context, w http.ResponseWriter, r *h return false } - // TODO: check if we have a bug when maybeResolvedImPath is resolved and i.setIpfsRootsHeader works with pathMetadata returned by Get(maybeResolvedImPath) - setIpfsRootsHeader(w, pathMetadata) + setIpfsRootsHeader(w, rq, &pathMetadata) resolvedPath := pathMetadata.LastSegment switch mc.Code(resolvedPath.Cid().Prefix().Codec) { @@ -107,23 +103,23 @@ func (i *handler) serveDefaults(ctx context.Context, w http.ResponseWriter, r *h i.webError(w, r, fmt.Errorf("decoding error: data not usable as a file"), http.StatusInternalServerError) return false } - logger.Debugw("serving codec", "path", contentPath) - return i.renderCodec(r.Context(), w, r, resolvedPath, bytesResponse, contentPath, begin, requestedContentType) + rq.logger.Debugw("serving codec", "path", rq.contentPath) + return i.renderCodec(r.Context(), w, r, rq, bytesResponse) default: - logger.Debugw("serving unixfs", "path", contentPath) + rq.logger.Debugw("serving unixfs", "path", rq.contentPath) ctx, span := spanTrace(ctx, "Handler.ServeUnixFS", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) defer span.End() // Handling Unixfs file if bytesResponse != nil { - logger.Debugw("serving unixfs file", "path", contentPath) - return i.serveFile(ctx, w, r, resolvedPath, contentPath, bytesResponse, pathMetadata.ContentType, begin) + rq.logger.Debugw("serving unixfs file", "path", rq.contentPath) + return i.serveFile(ctx, w, r, resolvedPath, rq.contentPath, bytesResponse, pathMetadata.ContentType, rq.begin) } // Handling Unixfs directory if directoryMetadata != nil || isDirectoryHeadRequest { - logger.Debugw("serving unixfs directory", "path", contentPath) - return i.serveDirectory(ctx, w, r, resolvedPath, contentPath, isDirectoryHeadRequest, directoryMetadata, ranges, begin, logger) + rq.logger.Debugw("serving unixfs directory", "path", rq.contentPath) + return i.serveDirectory(ctx, w, r, resolvedPath, rq.contentPath, isDirectoryHeadRequest, directoryMetadata, ranges, rq.begin, rq.logger) } i.webError(w, r, fmt.Errorf("unsupported UnixFS type"), http.StatusInternalServerError) diff --git a/gateway/handler_ipns_record.go b/gateway/handler_ipns_record.go index 40804e005..b077fa59a 100644 --- a/gateway/handler_ipns_record.go +++ b/gateway/handler_ipns_record.go @@ -10,25 +10,23 @@ import ( "time" "github.com/cespare/xxhash/v2" - ipath "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/boxo/ipns" "github.com/ipfs/go-cid" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" - "go.uber.org/zap" ) -func (i *handler) serveIpnsRecord(ctx context.Context, w http.ResponseWriter, r *http.Request, contentPath ipath.Path, begin time.Time, logger *zap.SugaredLogger) bool { - ctx, span := spanTrace(ctx, "Handler.ServeIPNSRecord", trace.WithAttributes(attribute.String("path", contentPath.String()))) +func (i *handler) serveIpnsRecord(ctx context.Context, w http.ResponseWriter, r *http.Request, rq *requestData) bool { + ctx, span := spanTrace(ctx, "Handler.ServeIPNSRecord", trace.WithAttributes(attribute.String("path", rq.contentPath.String()))) defer span.End() - if contentPath.Namespace() != "ipns" { - err := fmt.Errorf("%s is not an IPNS link", contentPath.String()) + if rq.contentPath.Namespace() != "ipns" { + err := fmt.Errorf("%s is not an IPNS link", rq.contentPath.String()) i.webError(w, r, err, http.StatusBadRequest) return false } - key := contentPath.String() + key := rq.contentPath.String() key = strings.TrimSuffix(key, "/") key = strings.TrimPrefix(key, "/ipns/") if strings.Count(key, "/") != 0 { @@ -90,7 +88,7 @@ func (i *handler) serveIpnsRecord(ctx context.Context, w http.ResponseWriter, r _, err = w.Write(rawRecord) if err == nil { // Update metrics - i.ipnsRecordGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) + i.ipnsRecordGetMetric.WithLabelValues(rq.contentPath.Namespace()).Observe(time.Since(rq.begin).Seconds()) return true } diff --git a/gateway/handler_tar.go b/gateway/handler_tar.go index a46bb49dd..784e51993 100644 --- a/gateway/handler_tar.go +++ b/gateway/handler_tar.go @@ -6,34 +6,32 @@ import ( "net/http" "time" - ipath "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/boxo/files" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" - "go.uber.org/zap" ) var unixEpochTime = time.Unix(0, 0) -func (i *handler) serveTAR(ctx context.Context, w http.ResponseWriter, r *http.Request, imPath ImmutablePath, contentPath ipath.Path, begin time.Time, logger *zap.SugaredLogger) bool { - ctx, span := spanTrace(ctx, "Handler.ServeTAR", trace.WithAttributes(attribute.String("path", imPath.String()))) +func (i *handler) serveTAR(ctx context.Context, w http.ResponseWriter, r *http.Request, rq *requestData) bool { + ctx, span := spanTrace(ctx, "Handler.ServeTAR", trace.WithAttributes(attribute.String("path", rq.immutablePath.String()))) defer span.End() ctx, cancel := context.WithCancel(ctx) defer cancel() // Get Unixfs file (or directory) - pathMetadata, file, err := i.backend.GetAll(ctx, imPath) - if !i.handleRequestErrors(w, r, contentPath, err) { + pathMetadata, file, err := i.backend.GetAll(ctx, rq.mostlyResolvedPath()) + if !i.handleRequestErrors(w, r, rq.contentPath, err) { return false } defer file.Close() - setIpfsRootsHeader(w, pathMetadata) + setIpfsRootsHeader(w, rq, &pathMetadata) rootCid := pathMetadata.LastSegment.Cid() // Set Cache-Control and read optional Last-Modified time - modtime := addCacheControlHeaders(w, r, contentPath, rootCid, tarResponseFormat) + modtime := addCacheControlHeaders(w, r, rq.contentPath, rootCid, tarResponseFormat) // Set Content-Disposition var name string @@ -65,7 +63,7 @@ func (i *handler) serveTAR(ctx context.Context, w http.ResponseWriter, r *http.R // The TAR has a top-level directory (or file) named by the CID. if err := tarw.WriteFile(file, rootCid.String()); err != nil { // Update fail metric - i.tarStreamFailMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) + i.tarStreamFailMetric.WithLabelValues(rq.contentPath.Namespace()).Observe(time.Since(rq.begin).Seconds()) w.Header().Set("X-Stream-Error", err.Error()) // Trailer headers do not work in web browsers @@ -79,6 +77,6 @@ func (i *handler) serveTAR(ctx context.Context, w http.ResponseWriter, r *http.R } // Update metrics - i.tarStreamGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) + i.tarStreamGetMetric.WithLabelValues(rq.contentPath.Namespace()).Observe(time.Since(rq.begin).Seconds()) return true } diff --git a/gateway/handler_test.go b/gateway/handler_test.go index 28229a901..e5e8a8ecb 100644 --- a/gateway/handler_test.go +++ b/gateway/handler_test.go @@ -1,20 +1,8 @@ package gateway import ( - "context" - "errors" - "fmt" - "io" - "net/http" - "testing" - "time" - ipath "github.com/ipfs/boxo/coreiface/path" - "github.com/ipfs/boxo/files" - "github.com/ipfs/boxo/path/resolver" - cid "github.com/ipfs/go-cid" - ipld "github.com/ipfs/go-ipld-format" "github.com/stretchr/testify/assert" ) @@ -40,192 +28,3 @@ func TestEtagMatch(t *testing.T) { assert.Equalf(t, test.expected, result, "etagMatch(%q, %q, %q)", test.header, test.cidEtag, test.dirEtag) } } - -type errorMockBackend struct { - err error -} - -func (mb *errorMockBackend) Get(ctx context.Context, path ImmutablePath, getRange ...ByteRange) (ContentPathMetadata, *GetResponse, error) { - return ContentPathMetadata{}, nil, mb.err -} - -func (mb *errorMockBackend) GetAll(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.Node, error) { - return ContentPathMetadata{}, nil, mb.err -} - -func (mb *errorMockBackend) GetBlock(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.File, error) { - return ContentPathMetadata{}, nil, mb.err -} - -func (mb *errorMockBackend) Head(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.Node, error) { - return ContentPathMetadata{}, nil, mb.err -} - -func (mb *errorMockBackend) GetCAR(ctx context.Context, path ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) { - return ContentPathMetadata{}, nil, mb.err -} - -func (mb *errorMockBackend) ResolveMutable(ctx context.Context, path ipath.Path) (ImmutablePath, error) { - return ImmutablePath{}, mb.err -} - -func (mb *errorMockBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { - return nil, mb.err -} - -func (mb *errorMockBackend) GetDNSLinkRecord(ctx context.Context, hostname string) (ipath.Path, error) { - return nil, mb.err -} - -func (mb *errorMockBackend) IsCached(ctx context.Context, p ipath.Path) bool { - return false -} - -func (mb *errorMockBackend) ResolvePath(ctx context.Context, path ImmutablePath) (ContentPathMetadata, error) { - return ContentPathMetadata{}, mb.err -} - -func TestGatewayBadRequestInvalidPath(t *testing.T) { - backend, _ := newMockBackend(t) - ts := newTestServer(t, backend) - t.Logf("test server url: %s", ts.URL) - - req, err := http.NewRequest(http.MethodGet, ts.URL+"/ipfs/QmInvalid/Path", nil) - assert.NoError(t, err) - - res, err := ts.Client().Do(req) - assert.NoError(t, err) - - assert.Equal(t, http.StatusBadRequest, res.StatusCode) -} - -func TestErrorBubblingFromBackend(t *testing.T) { - t.Parallel() - - for _, test := range []struct { - name string - err error - status int - }{ - {"404 Not Found from IPLD", &ipld.ErrNotFound{}, http.StatusNotFound}, - {"404 Not Found from path resolver", resolver.ErrNoLink{}, http.StatusNotFound}, - {"502 Bad Gateway", ErrBadGateway, http.StatusBadGateway}, - {"504 Gateway Timeout", ErrGatewayTimeout, http.StatusGatewayTimeout}, - } { - t.Run(test.name, func(t *testing.T) { - backend := &errorMockBackend{err: fmt.Errorf("wrapped for testing purposes: %w", test.err)} - ts := newTestServer(t, backend) - t.Logf("test server url: %s", ts.URL) - - req, err := http.NewRequest(http.MethodGet, ts.URL+"/ipns/en.wikipedia-on-ipfs.org", nil) - assert.NoError(t, err) - - res, err := ts.Client().Do(req) - assert.NoError(t, err) - assert.Equal(t, test.status, res.StatusCode) - }) - } - - for _, test := range []struct { - name string - err error - status int - headerName string - headerValue string - headerLength int // how many times was headerName set - }{ - {"429 Too Many Requests without Retry-After header", ErrTooManyRequests, http.StatusTooManyRequests, "Retry-After", "", 0}, - {"429 Too Many Requests without Retry-After header", NewErrorRetryAfter(ErrTooManyRequests, 0*time.Second), http.StatusTooManyRequests, "Retry-After", "", 0}, - {"429 Too Many Requests with Retry-After header", NewErrorRetryAfter(ErrTooManyRequests, 3600*time.Second), http.StatusTooManyRequests, "Retry-After", "3600", 1}, - } { - backend := &errorMockBackend{err: fmt.Errorf("wrapped for testing purposes: %w", test.err)} - ts := newTestServer(t, backend) - t.Logf("test server url: %s", ts.URL) - - req, err := http.NewRequest(http.MethodGet, ts.URL+"/ipns/en.wikipedia-on-ipfs.org", nil) - assert.NoError(t, err) - - res, err := ts.Client().Do(req) - assert.NoError(t, err) - assert.Equal(t, test.status, res.StatusCode) - assert.Equal(t, test.headerValue, res.Header.Get(test.headerName)) - assert.Equal(t, test.headerLength, len(res.Header.Values(test.headerName))) - } -} - -type panicMockBackend struct { - panicOnHostnameHandler bool -} - -func (mb *panicMockBackend) Get(ctx context.Context, immutablePath ImmutablePath, ranges ...ByteRange) (ContentPathMetadata, *GetResponse, error) { - panic("i am panicking") -} - -func (mb *panicMockBackend) GetAll(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.Node, error) { - panic("i am panicking") -} - -func (mb *panicMockBackend) GetBlock(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.File, error) { - panic("i am panicking") -} - -func (mb *panicMockBackend) Head(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.Node, error) { - panic("i am panicking") -} - -func (mb *panicMockBackend) GetCAR(ctx context.Context, immutablePath ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) { - panic("i am panicking") -} - -func (mb *panicMockBackend) ResolveMutable(ctx context.Context, p ipath.Path) (ImmutablePath, error) { - panic("i am panicking") -} - -func (mb *panicMockBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { - panic("i am panicking") -} - -func (mb *panicMockBackend) GetDNSLinkRecord(ctx context.Context, hostname string) (ipath.Path, error) { - // GetDNSLinkRecord is also called on the WithHostname handler. We have this option - // to disable panicking here so we can test if both the regular gateway handler - // and the hostname handler can handle panics. - if mb.panicOnHostnameHandler { - panic("i am panicking") - } - - return nil, errors.New("not implemented") -} - -func (mb *panicMockBackend) IsCached(ctx context.Context, p ipath.Path) bool { - panic("i am panicking") -} - -func (mb *panicMockBackend) ResolvePath(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, error) { - panic("i am panicking") -} - -func TestGatewayStatusCodeOnPanic(t *testing.T) { - backend := &panicMockBackend{} - ts := newTestServer(t, backend) - t.Logf("test server url: %s", ts.URL) - - req, err := http.NewRequest(http.MethodGet, ts.URL+"/ipfs/bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e", nil) - assert.NoError(t, err) - - res, err := ts.Client().Do(req) - assert.NoError(t, err) - assert.Equal(t, http.StatusInternalServerError, res.StatusCode) -} - -func TestGatewayStatusCodeOnHostnamePanic(t *testing.T) { - backend := &panicMockBackend{panicOnHostnameHandler: true} - ts := newTestServer(t, backend) - t.Logf("test server url: %s", ts.URL) - - req, err := http.NewRequest(http.MethodGet, ts.URL+"/ipfs/bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e", nil) - assert.NoError(t, err) - - res, err := ts.Client().Do(req) - assert.NoError(t, err) - assert.Equal(t, http.StatusInternalServerError, res.StatusCode) -} diff --git a/gateway/handler_unixfs_dir_test.go b/gateway/handler_unixfs_dir_test.go new file mode 100644 index 000000000..a8ce04778 --- /dev/null +++ b/gateway/handler_unixfs_dir_test.go @@ -0,0 +1,87 @@ +package gateway + +import ( + "context" + "io" + "net/http" + "testing" + + ipath "github.com/ipfs/boxo/coreiface/path" + path "github.com/ipfs/boxo/path" + "github.com/stretchr/testify/require" +) + +func TestIPNSHostnameBacklinks(t *testing.T) { + // Test if directory listing on DNSLink Websites have correct backlinks. + ts, backend, root := newTestServerAndNode(t, nil, "dir-special-chars.car") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // create /ipns/example.net/foo/ + k2, err := backend.resolvePathNoRootsReturned(ctx, ipath.Join(ipath.IpfsPath(root), "foo? #<'")) + require.NoError(t, err) + + k3, err := backend.resolvePathNoRootsReturned(ctx, ipath.Join(ipath.IpfsPath(root), "foo? #<'/bar")) + require.NoError(t, err) + + backend.namesys["/ipns/example.net"] = path.FromCid(root) + + // make request to directory listing + req := mustNewRequest(t, http.MethodGet, ts.URL+"/foo%3F%20%23%3C%27/", nil) + req.Host = "example.net" + + res := mustDoWithoutRedirect(t, req) + + // expect correct links + body, err := io.ReadAll(res.Body) + require.NoError(t, err) + s := string(body) + t.Logf("body: %s\n", string(body)) + + require.True(t, matchPathOrBreadcrumbs(s, "/ipns/example.net/foo? #<'"), "expected a path in directory listing") + // https://github.com/ipfs/dir-index-html/issues/42 + require.Contains(t, s, "", "expected backlink in directory listing") + require.Contains(t, s, "", "expected file in directory listing") + require.Contains(t, s, s, k2.Cid().String(), "expected hash in directory listing") + + // make request to directory listing at root + req = mustNewRequest(t, http.MethodGet, ts.URL, nil) + req.Host = "example.net" + + res = mustDoWithoutRedirect(t, req) + require.NoError(t, err) + + // expect correct backlinks at root + body, err = io.ReadAll(res.Body) + require.NoError(t, err) + + s = string(body) + t.Logf("body: %s\n", string(body)) + + require.True(t, matchPathOrBreadcrumbs(s, "/"), "expected a path in directory listing") + require.NotContains(t, s, "", "expected no backlink in directory listing of the root CID") + require.Contains(t, s, "", "expected file in directory listing") + // https://github.com/ipfs/dir-index-html/issues/42 + require.Contains(t, s, "example.net/foo? #<'/bar"), "expected a path in directory listing") + require.Contains(t, s, "", "expected backlink in directory listing") + require.Contains(t, s, "", "expected file in directory listing") + require.Contains(t, s, k3.Cid().String(), "expected hash in directory listing") +} diff --git a/gateway/hostname_test.go b/gateway/hostname_test.go index 272a24866..a58e0d404 100644 --- a/gateway/hostname_test.go +++ b/gateway/hostname_test.go @@ -10,12 +10,15 @@ import ( path "github.com/ipfs/boxo/path" cid "github.com/ipfs/go-cid" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestToSubdomainURL(t *testing.T) { - backend, _ := newMockBackend(t) + t.Parallel() + + backend, _ := newMockBackend(t, "fixtures.car") testCID, err := cid.Decode("bafkqaglimvwgy3zakrsxg5cun5jxkyten5wwc2lokvjeycq") - assert.NoError(t, err) + require.NoError(t, err) backend.namesys["/ipns/dnslink.long-name.example.com"] = path.FromString(testCID.String()) backend.namesys["/ipns/dnslink.too-long.f1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5o.example.com"] = path.FromString(testCID.String()) @@ -62,13 +65,14 @@ func TestToSubdomainURL(t *testing.T) { testName := fmt.Sprintf("%s, %v, %s", test.gwHostname, test.inlineDNSLink, test.path) t.Run(testName, func(t *testing.T) { url, err := toSubdomainURL(test.gwHostname, test.path, test.request, test.inlineDNSLink, backend) - assert.Equal(t, test.url, url) - assert.Equal(t, test.err, err) + require.Equal(t, test.url, url) + require.Equal(t, test.err, err) }) } } func TestToDNSLinkDNSLabel(t *testing.T) { + t.Parallel() for _, test := range []struct { in string out string @@ -79,13 +83,14 @@ func TestToDNSLinkDNSLabel(t *testing.T) { } { t.Run(test.in, func(t *testing.T) { out, err := toDNSLinkDNSLabel(test.in) - assert.Equal(t, test.out, out) - assert.Equal(t, test.err, err) + require.Equal(t, test.out, out) + require.Equal(t, test.err, err) }) } } func TestToDNSLinkFQDN(t *testing.T) { + t.Parallel() for _, test := range []struct { in string out string @@ -96,12 +101,13 @@ func TestToDNSLinkFQDN(t *testing.T) { } { t.Run(test.in, func(t *testing.T) { out := toDNSLinkFQDN(test.in) - assert.Equal(t, test.out, out) + require.Equal(t, test.out, out) }) } } func TestIsHTTPSRequest(t *testing.T) { + t.Parallel() httpRequest := httptest.NewRequest("GET", "http://127.0.0.1:8080", nil) httpsRequest := httptest.NewRequest("GET", "https://https-request-stub.example.com", nil) httpsProxiedRequest := httptest.NewRequest("GET", "http://proxied-https-request-stub.example.com", nil) @@ -122,12 +128,13 @@ func TestIsHTTPSRequest(t *testing.T) { testName := fmt.Sprintf("%+v", test.in) t.Run(testName, func(t *testing.T) { out := isHTTPSRequest(test.in) - assert.Equal(t, test.out, out) + require.Equal(t, test.out, out) }) } } func TestHasPrefix(t *testing.T) { + t.Parallel() for _, test := range []struct { prefixes []string path string @@ -141,12 +148,13 @@ func TestHasPrefix(t *testing.T) { testName := fmt.Sprintf("%+v, %s", test.prefixes, test.path) t.Run(testName, func(t *testing.T) { out := hasPrefix(test.path, test.prefixes...) - assert.Equal(t, test.out, out) + require.Equal(t, test.out, out) }) } } func TestIsDomainNameAndNotPeerID(t *testing.T) { + t.Parallel() for _, test := range []struct { hostname string out bool @@ -160,12 +168,13 @@ func TestIsDomainNameAndNotPeerID(t *testing.T) { } { t.Run(test.hostname, func(t *testing.T) { out := isDomainNameAndNotPeerID(test.hostname) - assert.Equal(t, test.out, out) + require.Equal(t, test.out, out) }) } } func TestPortStripping(t *testing.T) { + t.Parallel() for _, test := range []struct { in string out string @@ -180,12 +189,13 @@ func TestPortStripping(t *testing.T) { } { t.Run(test.in, func(t *testing.T) { out := stripPort(test.in) - assert.Equal(t, test.out, out) + require.Equal(t, test.out, out) }) } } func TestToDNSLabel(t *testing.T) { + t.Parallel() for _, test := range []struct { in string out string @@ -203,13 +213,15 @@ func TestToDNSLabel(t *testing.T) { t.Run(test.in, func(t *testing.T) { inCID, _ := cid.Decode(test.in) out, err := toDNSLabel(test.in, inCID) - assert.Equal(t, test.out, out) - assert.Equal(t, test.err, err) + require.Equal(t, test.out, out) + require.Equal(t, test.err, err) }) } } func TestKnownSubdomainDetails(t *testing.T) { + t.Parallel() + gwLocalhost := &PublicGateway{Paths: []string{"/ipfs", "/ipns", "/api"}, UseSubdomains: true} gwDweb := &PublicGateway{Paths: []string{"/ipfs", "/ipns", "/api"}, UseSubdomains: true} gwLong := &PublicGateway{Paths: []string{"/ipfs", "/ipns", "/api"}, UseSubdomains: true} diff --git a/gateway/lazyseek_test.go b/gateway/lazyseek_test.go index ca4e57d9e..b3ed4e4e2 100644 --- a/gateway/lazyseek_test.go +++ b/gateway/lazyseek_test.go @@ -6,7 +6,7 @@ import ( "strings" "testing" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type badSeeker struct { @@ -30,33 +30,33 @@ func TestLazySeekerError(t *testing.T) { size: underlyingBuffer.Size(), } off, err := s.Seek(0, io.SeekEnd) - assert.NoError(t, err) - assert.Equal(t, s.size, off, "expected to seek to the end") + require.NoError(t, err) + require.Equal(t, s.size, off, "expected to seek to the end") // shouldn't have actually seeked. b, err := io.ReadAll(s) - assert.NoError(t, err) - assert.Equal(t, 0, len(b), "expected to read nothing") + require.NoError(t, err) + require.Equal(t, 0, len(b), "expected to read nothing") // shouldn't need to actually seek. off, err = s.Seek(0, io.SeekStart) - assert.NoError(t, err) - assert.Equal(t, int64(0), off, "expected to seek to the start") + require.NoError(t, err) + require.Equal(t, int64(0), off, "expected to seek to the start") b, err = io.ReadAll(s) - assert.NoError(t, err) - assert.Equal(t, "fubar", string(b), "expected to read string") + require.NoError(t, err) + require.Equal(t, "fubar", string(b), "expected to read string") // should fail the second time. off, err = s.Seek(0, io.SeekStart) - assert.NoError(t, err) - assert.Equal(t, int64(0), off, "expected to seek to the start") + require.NoError(t, err) + require.Equal(t, int64(0), off, "expected to seek to the start") // right here... b, err = io.ReadAll(s) - assert.NotNil(t, err) - assert.Equal(t, errBadSeek, err) - assert.Equal(t, 0, len(b), "expected to read nothing") + require.NotNil(t, err) + require.Equal(t, errBadSeek, err) + require.Equal(t, 0, len(b), "expected to read nothing") } func TestLazySeeker(t *testing.T) { @@ -69,25 +69,25 @@ func TestLazySeeker(t *testing.T) { t.Helper() var buf [1]byte n, err := io.ReadFull(s, buf[:]) - assert.NoError(t, err) - assert.Equal(t, 1, n, "expected to read one byte, read %d", n) - assert.Equal(t, b, buf[0]) + require.NoError(t, err) + require.Equal(t, 1, n, "expected to read one byte, read %d", n) + require.Equal(t, b, buf[0]) } expectSeek := func(whence int, off, expOff int64, expErr string) { t.Helper() n, err := s.Seek(off, whence) if expErr == "" { - assert.NoError(t, err) + require.NoError(t, err) } else { - assert.EqualError(t, err, expErr) + require.EqualError(t, err, expErr) } - assert.Equal(t, expOff, n) + require.Equal(t, expOff, n) } expectSeek(io.SeekEnd, 0, s.size, "") b, err := io.ReadAll(s) - assert.NoError(t, err) - assert.Equal(t, 0, len(b), "expected to read nothing") + require.NoError(t, err) + require.Equal(t, 0, len(b), "expected to read nothing") expectSeek(io.SeekEnd, -1, s.size-1, "") expectByte('r') expectSeek(io.SeekStart, 0, 0, "") diff --git a/gateway/testdata/dir-special-chars.car b/gateway/testdata/dir-special-chars.car new file mode 100644 index 000000000..ac1ce8480 Binary files /dev/null and b/gateway/testdata/dir-special-chars.car differ diff --git a/gateway/testdata/fixtures.car b/gateway/testdata/fixtures.car index cea5d462d..817c7178d 100644 Binary files a/gateway/testdata/fixtures.car and b/gateway/testdata/fixtures.car differ diff --git a/gateway/testdata/headers-test.car b/gateway/testdata/headers-test.car new file mode 100644 index 000000000..6d34e0a93 Binary files /dev/null and b/gateway/testdata/headers-test.car differ diff --git a/gateway/testdata/ipns-hostname-redirects.car b/gateway/testdata/ipns-hostname-redirects.car new file mode 100644 index 000000000..8e56d0fc7 Binary files /dev/null and b/gateway/testdata/ipns-hostname-redirects.car differ diff --git a/gateway/testdata/pretty-404.car b/gateway/testdata/pretty-404.car new file mode 100644 index 000000000..3adec2904 Binary files /dev/null and b/gateway/testdata/pretty-404.car differ diff --git a/gateway/utilities_test.go b/gateway/utilities_test.go new file mode 100644 index 000000000..1b9f81d32 --- /dev/null +++ b/gateway/utilities_test.go @@ -0,0 +1,238 @@ +package gateway + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/ipfs/boxo/blockservice" + nsopts "github.com/ipfs/boxo/coreiface/options/namesys" + ipath "github.com/ipfs/boxo/coreiface/path" + offline "github.com/ipfs/boxo/exchange/offline" + "github.com/ipfs/boxo/files" + carblockstore "github.com/ipfs/boxo/ipld/car/v2/blockstore" + "github.com/ipfs/boxo/namesys" + path "github.com/ipfs/boxo/path" + "github.com/ipfs/go-cid" + "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/routing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func mustNewRequest(t *testing.T, method string, path string, body io.Reader) *http.Request { + r, err := http.NewRequest(http.MethodGet, path, body) + require.NoError(t, err) + return r +} + +func mustDoWithoutRedirect(t *testing.T, req *http.Request) *http.Response { + errNoRedirect := errors.New("without-redirect") + c := &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return errNoRedirect + }, + } + res, err := c.Do(req) + require.True(t, err == nil || errors.Is(err, errNoRedirect)) + return res +} + +func mustDo(t *testing.T, req *http.Request) *http.Response { + c := &http.Client{} + res, err := c.Do(req) + require.NoError(t, err) + return res +} + +type mockNamesys map[string]path.Path + +func (m mockNamesys) Resolve(ctx context.Context, name string, opts ...nsopts.ResolveOpt) (value path.Path, err error) { + cfg := nsopts.DefaultResolveOpts() + for _, o := range opts { + o(&cfg) + } + depth := cfg.Depth + if depth == nsopts.UnlimitedDepth { + // max uint + depth = ^uint(0) + } + for strings.HasPrefix(name, "/ipns/") { + if depth == 0 { + return value, namesys.ErrResolveRecursion + } + depth-- + + var ok bool + value, ok = m[name] + if !ok { + return "", namesys.ErrResolveFailed + } + name = value.String() + } + return value, nil +} + +func (m mockNamesys) ResolveAsync(ctx context.Context, name string, opts ...nsopts.ResolveOpt) <-chan namesys.Result { + out := make(chan namesys.Result, 1) + v, err := m.Resolve(ctx, name, opts...) + out <- namesys.Result{Path: v, Err: err} + close(out) + return out +} + +func (m mockNamesys) Publish(ctx context.Context, name crypto.PrivKey, value path.Path, opts ...nsopts.PublishOption) error { + return errors.New("not implemented for mockNamesys") +} + +func (m mockNamesys) GetResolver(subs string) (namesys.Resolver, bool) { + return nil, false +} + +type mockBackend struct { + gw IPFSBackend + namesys mockNamesys +} + +var _ IPFSBackend = (*mockBackend)(nil) + +func newMockBackend(t *testing.T, fixturesFile string) (*mockBackend, cid.Cid) { + r, err := os.Open(filepath.Join("./testdata", fixturesFile)) + assert.NoError(t, err) + + blockStore, err := carblockstore.NewReadOnly(r, nil) + assert.NoError(t, err) + + t.Cleanup(func() { + blockStore.Close() + r.Close() + }) + + cids, err := blockStore.Roots() + assert.NoError(t, err) + assert.Len(t, cids, 1) + + blockService := blockservice.New(blockStore, offline.Exchange(blockStore)) + + n := mockNamesys{} + backend, err := NewBlocksBackend(blockService, WithNameSystem(n)) + if err != nil { + t.Fatal(err) + } + + return &mockBackend{ + gw: backend, + namesys: n, + }, cids[0] +} + +func (mb *mockBackend) Get(ctx context.Context, immutablePath ImmutablePath, ranges ...ByteRange) (ContentPathMetadata, *GetResponse, error) { + return mb.gw.Get(ctx, immutablePath, ranges...) +} + +func (mb *mockBackend) GetAll(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.Node, error) { + return mb.gw.GetAll(ctx, immutablePath) +} + +func (mb *mockBackend) GetBlock(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.File, error) { + return mb.gw.GetBlock(ctx, immutablePath) +} + +func (mb *mockBackend) Head(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.Node, error) { + return mb.gw.Head(ctx, immutablePath) +} + +func (mb *mockBackend) GetCAR(ctx context.Context, immutablePath ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) { + return mb.gw.GetCAR(ctx, immutablePath, params) +} + +func (mb *mockBackend) ResolveMutable(ctx context.Context, p ipath.Path) (ImmutablePath, error) { + return mb.gw.ResolveMutable(ctx, p) +} + +func (mb *mockBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { + return nil, routing.ErrNotSupported +} + +func (mb *mockBackend) GetDNSLinkRecord(ctx context.Context, hostname string) (ipath.Path, error) { + if mb.namesys != nil { + p, err := mb.namesys.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1)) + if err == namesys.ErrResolveRecursion { + err = nil + } + return ipath.New(p.String()), err + } + + return nil, errors.New("not implemented") +} + +func (mb *mockBackend) IsCached(ctx context.Context, p ipath.Path) bool { + return mb.gw.IsCached(ctx, p) +} + +func (mb *mockBackend) ResolvePath(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, error) { + return mb.gw.ResolvePath(ctx, immutablePath) +} + +func (mb *mockBackend) resolvePathNoRootsReturned(ctx context.Context, ip ipath.Path) (ipath.Resolved, error) { + var imPath ImmutablePath + var err error + if ip.Mutable() { + imPath, err = mb.ResolveMutable(ctx, ip) + if err != nil { + return nil, err + } + } else { + imPath, err = NewImmutablePath(ip) + if err != nil { + return nil, err + } + } + + md, err := mb.ResolvePath(ctx, imPath) + if err != nil { + return nil, err + } + return md.LastSegment, nil +} + +func newTestServerAndNode(t *testing.T, ns mockNamesys, fixturesFile string) (*httptest.Server, *mockBackend, cid.Cid) { + backend, root := newMockBackend(t, fixturesFile) + ts := newTestServer(t, backend) + return ts, backend, root +} + +func newTestServer(t *testing.T, backend IPFSBackend) *httptest.Server { + return newTestServerWithConfig(t, backend, Config{ + Headers: map[string][]string{}, + DeserializedResponses: true, + }) +} + +func newTestServerWithConfig(t *testing.T, backend IPFSBackend, config Config) *httptest.Server { + AddAccessControlHeaders(config.Headers) + + handler := NewHandler(config, backend) + mux := http.NewServeMux() + mux.Handle("/ipfs/", handler) + mux.Handle("/ipns/", handler) + handler = NewHostnameHandler(config, backend, mux) + + ts := httptest.NewServer(handler) + t.Cleanup(func() { ts.Close() }) + t.Logf("test server url: %s", ts.URL) + + return ts +} + +func matchPathOrBreadcrumbs(s string, expected string) bool { + matched, _ := regexp.MatchString("Index of(\n|\r\n)[\t ]*"+regexp.QuoteMeta(expected), s) + return matched +}