From 5d864faac71b877ae30bd7b2f01c9dfaba68d8eb Mon Sep 17 00:00:00 2001 From: Gus Eggert Date: Wed, 14 Dec 2022 11:10:48 -0500 Subject: [PATCH] test: port gateway sharness tests to Go tests --- test/cli/gateway_test.go | 492 ++++++++++++++++++ test/cli/harness/harness.go | 13 +- test/cli/harness/http_client.go | 116 +++++ test/cli/harness/node.go | 80 ++- test/cli/harness/nodes.go | 21 +- test/cli/testutils/strings.go | 24 + test/sharness/t0110-gateway-data/foo.block | 2 - test/sharness/t0110-gateway-data/foofoo.block | Bin 82 -> 0 bytes test/sharness/t0110-gateway.sh | 357 ------------- 9 files changed, 727 insertions(+), 378 deletions(-) create mode 100644 test/cli/gateway_test.go create mode 100644 test/cli/harness/http_client.go delete mode 100644 test/sharness/t0110-gateway-data/foo.block delete mode 100644 test/sharness/t0110-gateway-data/foofoo.block delete mode 100755 test/sharness/t0110-gateway.sh diff --git a/test/cli/gateway_test.go b/test/cli/gateway_test.go new file mode 100644 index 00000000000..6e8ef516a13 --- /dev/null +++ b/test/cli/gateway_test.go @@ -0,0 +1,492 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "regexp" + "testing" + + "github.com/ipfs/kubo/config" + "github.com/ipfs/kubo/test/cli/harness" + . "github.com/ipfs/kubo/test/cli/testutils" + "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGateway(t *testing.T) { + t.Parallel() + h := harness.NewT(t) + node := h.NewNode().Init().StartDaemon("--offline") + cid := node.IPFSAddStr("Hello Worlds!") + + client := node.GatewayClient() + client.TemplateData = map[string]string{ + "CID": cid, + "PeerID": node.PeerID().String(), + } + + t.Run("GET IPFS path succeeds", func(t *testing.T) { + t.Parallel() + resp := client.Get("/ipfs/{{.CID}}") + assert.Equal(t, 200, resp.StatusCode) + }) + + t.Run("GET IPFS path with explicit ?filename succeeds with proper header", func(t *testing.T) { + t.Parallel() + resp := client.Get("/ipfs/{{.CID}}?filename=testтест.pdf") + assert.Equal(t, 200, resp.StatusCode) + assert.Equal(t, + `inline; filename="test____.pdf"; filename*=UTF-8''test%D1%82%D0%B5%D1%81%D1%82.pdf`, + resp.Headers.Get("Content-Disposition"), + ) + }) + + t.Run("GET IPFS path with explicit ?filename and &download=true succeeds with proper header", func(t *testing.T) { + t.Parallel() + resp := client.Get("/ipfs/{{.CID}}?filename=testтест.mp4&download=true") + assert.Equal(t, 200, resp.StatusCode) + assert.Equal(t, + `attachment; filename="test____.mp4"; filename*=UTF-8''test%D1%82%D0%B5%D1%81%D1%82.mp4`, + resp.Headers.Get("Content-Disposition"), + ) + }) + + // https://github.com/ipfs/go-ipfs/issues/4025#issuecomment-342250616 + t.Run("GET for Server Worker registration outside of an IPFS content root errors", func(t *testing.T) { + t.Parallel() + resp := client.Get("/ipfs/{{.CID}}?filename=sw.js", client.WithHeader("Service-Worker", "script")) + assert.Equal(t, 400, resp.StatusCode) + assert.Contains(t, resp.Body, "navigator.serviceWorker: registration is not allowed for this scope") + }) + + t.Run("GET IPFS directory path succeeds", func(t *testing.T) { + t.Parallel() + client := node.GatewayClient().DisableRedirects() + + pageContents := "hello i am a webpage" + fileContents := "12345" + h.WriteFile("dir/test", fileContents) + h.WriteFile("dir/dirwithindex/index.html", pageContents) + cids := node.IPFS("add", "-r", "-q", filepath.Join(h.Dir, "dir")).Stdout.Lines() + + rootCID := cids[len(cids)-1] + client.TemplateData = map[string]string{ + "IndexFileCID": cids[0], + "TestFileCID": cids[1], + "RootCID": rootCID, + } + + t.Run("GET IPFS the index file CID", func(t *testing.T) { + t.Parallel() + resp := client.Get("/ipfs/{{.IndexFileCID}}") + assert.Equal(t, 200, resp.StatusCode) + assert.Equal(t, pageContents, resp.Body) + }) + + t.Run("GET IPFS the test file CID", func(t *testing.T) { + t.Parallel() + resp := client.Get("/ipfs/{{.TestFileCID}}") + assert.Equal(t, 200, resp.StatusCode) + assert.Equal(t, fileContents, resp.Body) + }) + + t.Run("GET IPFS directory with index.html returns redirect to add trailing slash", func(t *testing.T) { + t.Parallel() + resp := client.Head("/ipfs/{{.RootCID}}/dirwithindex?query=to-remember") + assert.Equal(t, 301, resp.StatusCode) + assert.Equal(t, + fmt.Sprintf("/ipfs/%s/dirwithindex/?query=to-remember", rootCID), + resp.Headers.Get("Location"), + ) + }) + + // This enables go get to parse go-import meta tags from index.html files stored in IPFS + // https://github.com/ipfs/kubo/pull/3963 + t.Run("GET IPFS directory with index.html and no trailing slash returns expected output when go-get is passed", func(t *testing.T) { + t.Parallel() + resp := client.Get("/ipfs/{{.RootCID}}/dirwithindex?go-get=1") + assert.Equal(t, pageContents, resp.Body) + }) + + t.Run("GET IPFS directory with index.html and trailing slash returns expected output", func(t *testing.T) { + t.Parallel() + resp := client.Get("/ipfs/{{.RootCID}}/dirwithindex/?query=to-remember") + assert.Equal(t, pageContents, resp.Body) + }) + + t.Run("GET IPFS nonexistent file returns 404 (Not Found)", func(t *testing.T) { + t.Parallel() + resp := client.Get("/ipfs/{{.RootCID}}/pleaseDontAddMe") + assert.Equal(t, 404, resp.StatusCode) + }) + + t.Run("GET IPFS invalid CID returns 400 (Bad Request)", func(t *testing.T) { + t.Parallel() + resp := client.Get("/ipfs/QmInvalid/pleaseDontAddMe") + assert.Equal(t, 400, resp.StatusCode) + }) + + t.Run("GET IPFS inlined zero-length data object returns ok code (200)", func(t *testing.T) { + t.Parallel() + resp := client.Get("/ipfs/bafkqaaa") + assert.Equal(t, 200, resp.StatusCode) + assert.Equal(t, "0", resp.Resp.Header.Get("Content-Length")) + assert.Equal(t, "", resp.Body) + }) + + t.Run("GET IPFS inlined zero-length data object with byte range returns ok code (200)", func(t *testing.T) { + t.Parallel() + resp := client.Get("/ipfs/bafkqaaa", client.WithHeader("Range", "bytes=0-1048575")) + assert.Equal(t, 200, resp.StatusCode) + assert.Equal(t, "0", resp.Resp.Header.Get("Content-Length")) + assert.Equal(t, "text/plain", resp.Resp.Header.Get("Content-Type")) + }) + + t.Run("GET /ipfs/ipfs/{cid} returns redirect to the valid path", func(t *testing.T) { + t.Parallel() + resp := client.Get("/ipfs/ipfs/bafkqaaa?query=to-remember") + assert.Contains(t, + resp.Body, + ``, + ) + assert.Contains(t, + resp.Body, + ``, + ) + }) + }) + + t.Run("IPNS", func(t *testing.T) { + t.Parallel() + node.IPFS("name", "publish", "--allow-offline", cid) + + t.Run("GET invalid IPNS root returns 400 (Bad Request)", func(t *testing.T) { + t.Parallel() + resp := client.Get("/ipns/QmInvalid/pleaseDontAddMe") + assert.Equal(t, 400, resp.StatusCode) + }) + + t.Run("GET IPNS path succeeds", func(t *testing.T) { + t.Parallel() + resp := client.Get("/ipns/{{.PeerID}}") + assert.Equal(t, 200, resp.StatusCode) + assert.Equal(t, "Hello Worlds!", resp.Body) + }) + + t.Run("GET /ipfs/ipns/{peerid} returns redirect to the valid path", func(t *testing.T) { + t.Parallel() + resp := client.Get("/ipfs/ipns/{{.PeerID}}?query=to-remember") + peerID := node.PeerID().String() + assert.Contains(t, + resp.Body, + fmt.Sprintf(``, peerID), + ) + assert.Contains(t, + resp.Body, + fmt.Sprintf(``, peerID), + ) + + }) + + }) + + t.Run("GET invalid IPFS path errors", func(t *testing.T) { + t.Parallel() + assert.Equal(t, 400, client.Get("/ipfs/12345").StatusCode) + }) + + t.Run("GET invalid path errors", func(t *testing.T) { + t.Parallel() + assert.Equal(t, 404, client.Get("/12345").StatusCode) + }) + + // TODO: these tests that use the API URL shouldn't be part of gateway tests... + t.Run("GET /webui returns 301 or 302", func(t *testing.T) { + t.Parallel() + resp := node.APIClient().DisableRedirects().Get("/webui") + assert.Contains(t, []int{302, 301}, resp.StatusCode) + }) + + t.Run("GET /webui/ returns 301 or 302", func(t *testing.T) { + t.Parallel() + resp := node.APIClient().DisableRedirects().Get("/webui/") + assert.Contains(t, []int{302, 301}, resp.StatusCode) + }) + + t.Run("GET /logs returns logs", func(t *testing.T) { + t.Parallel() + apiClient := node.APIClient() + reqURL := apiClient.BuildURL("/logs") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil) + require.NoError(t, err) + + resp, err := apiClient.Client.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + // read the first line of the output and parse its JSON + dec := json.NewDecoder(resp.Body) + event := struct{ Event string }{} + err = dec.Decode(&event) + require.NoError(t, err) + + assert.Equal(t, "log API client connected", event.Event) + }) + + t.Run("POST /api/v0/version succeeds", func(t *testing.T) { + t.Parallel() + resp := node.APIClient().Post("/api/v0/version", nil) + assert.Equal(t, 200, resp.StatusCode) + + assert.Len(t, resp.Resp.TransferEncoding, 1) + assert.Equal(t, "chunked", resp.Resp.TransferEncoding[0]) + + vers := struct{ Version string }{} + err := json.Unmarshal([]byte(resp.Body), &vers) + require.NoError(t, err) + assert.NotEmpty(t, vers.Version) + }) + + t.Run("pprof", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + apiClient := node.APIClient() + t.Run("mutex", func(t *testing.T) { + t.Parallel() + t.Run("setting the mutex fraction works (negative so it doesn't enable)", func(t *testing.T) { + t.Parallel() + resp := apiClient.Post("/debug/pprof-mutex/?fraction=-1", nil) + assert.Equal(t, 200, resp.StatusCode) + }) + t.Run("mutex endpoint doesn't accept a string as an argument", func(t *testing.T) { + t.Parallel() + resp := apiClient.Post("/debug/pprof-mutex/?fraction=that_is_a_string", nil) + assert.Equal(t, 400, resp.StatusCode) + }) + t.Run("mutex endpoint returns 405 on GET", func(t *testing.T) { + t.Parallel() + resp := apiClient.Get("/debug/pprof-mutex/?fraction=-1") + assert.Equal(t, 405, resp.StatusCode) + }) + }) + t.Run("block", func(t *testing.T) { + t.Parallel() + t.Run("setting the block profiler rate works (0 so it doesn't enable)", func(t *testing.T) { + t.Parallel() + resp := apiClient.Post("/debug/pprof-block/?rate=0", nil) + assert.Equal(t, 200, resp.StatusCode) + }) + t.Run("block profiler endpoint doesn't accept a string as an argument", func(t *testing.T) { + t.Parallel() + resp := apiClient.Post("/debug/pprof-block/?rate=that_is_a_string", nil) + assert.Equal(t, 400, resp.StatusCode) + }) + t.Run("block profiler endpoint returns 405 on GET", func(t *testing.T) { + t.Parallel() + resp := apiClient.Get("/debug/pprof-block/?rate=0") + assert.Equal(t, 405, resp.StatusCode) + }) + }) + }) + + t.Run("index content types", func(t *testing.T) { + t.Parallel() + h := harness.NewT(t) + node := h.NewNode().Init().StartDaemon() + + h.WriteFile("index/index.html", "

") + cid := node.IPFS("add", "-Q", "-r", filepath.Join(h.Dir, "index")).Stderr.Trimmed() + + apiClient := node.APIClient() + apiClient.TemplateData = map[string]string{"CID": cid} + + t.Run("GET index.html has correct content type", func(t *testing.T) { + t.Parallel() + res := apiClient.Get("/ipfs/{{.CID}}/") + assert.Equal(t, "text/html; charset=utf-8", res.Resp.Header.Get("Content-Type")) + }) + + t.Run("HEAD index.html has no content", func(t *testing.T) { + t.Parallel() + res := apiClient.Head("/ipfs/{{.CID}}/") + assert.Equal(t, "", res.Body) + assert.Equal(t, "", res.Resp.Header.Get("Content-Length")) + }) + }) + + t.Run("readonly API", func(t *testing.T) { + t.Parallel() + + client := node.GatewayClient() + + fileContents := "12345" + h.WriteFile("readonly/dir/test", fileContents) + cids := node.IPFS("add", "-r", "-q", filepath.Join(h.Dir, "readonly/dir")).Stdout.Lines() + + rootCID := cids[len(cids)-1] + client.TemplateData = map[string]string{"RootCID": rootCID} + + t.Run("Get IPFS directory file through readonly API succeeds", func(t *testing.T) { + t.Parallel() + resp := client.Get("/api/v0/cat?arg={{.RootCID}}/test") + assert.Equal(t, 200, resp.StatusCode) + assert.Equal(t, fileContents, resp.Body) + }) + + t.Run("refs IPFS directory file through readonly API succeeds", func(t *testing.T) { + t.Parallel() + resp := client.Get("/api/v0/refs?arg={{.RootCID}}/test") + assert.Equal(t, 200, resp.StatusCode) + }) + + t.Run("test gateway API is sanitized", func(t *testing.T) { + t.Parallel() + for _, cmd := range []string{ + "add", + "block/put", + "bootstrap", + "config", + "dag/put", + "dag/import", + "dht", + "diag", + "id", + "mount", + "name/publish", + "object/put", + "object/new", + "object/patch", + "pin", + "ping", + "repo", + "stats", + "swarm", + "file", + "update", + "bitswap", + } { + t.Run(cmd, func(t *testing.T) { + cmd := cmd + t.Parallel() + assert.Equal(t, 404, client.Get("/api/v0/"+cmd).StatusCode) + }) + } + }) + }) + + t.Run("refs/local", func(t *testing.T) { + t.Parallel() + gatewayAddr := URLStrToMultiaddr(node.GatewayURL()) + res := node.RunIPFS("--api", gatewayAddr.String(), "refs", "local") + assert.Equal(t, + `Error: invalid path "local": selected encoding not supported`, + res.Stderr.Trimmed(), + ) + }) + + t.Run("raw leaves node", func(t *testing.T) { + t.Parallel() + contents := "This is RAW!" + cid := node.IPFSAddStr(contents, "--raw-leaves") + assert.Equal(t, contents, client.Get("/ipfs/"+cid).Body) + }) + + t.Run("compact blocks", func(t *testing.T) { + t.Parallel() + block1 := "\x0a\x09\x08\x02\x12\x03\x66\x6f\x6f\x18\x03" + block2 := "\x0a\x04\x08\x02\x18\x06\x12\x24\x0a\x22\x12\x20\xcf\x92\xfd\xef\xcd\xc3\x4c\xac\x00\x9c" + + "\x8b\x05\xeb\x66\x2b\xe0\x61\x8d\xb9\xde\x55\xec\xd4\x27\x85\xe9\xec\x67\x12\xf8\xdf\x65" + + "\x12\x24\x0a\x22\x12\x20\xcf\x92\xfd\xef\xcd\xc3\x4c\xac\x00\x9c\x8b\x05\xeb\x66\x2b\xe0" + + "\x61\x8d\xb9\xde\x55\xec\xd4\x27\x85\xe9\xec\x67\x12\xf8\xdf\x65" + + node.PipeStrToIPFS(block1, "block", "put") + block2CID := node.PipeStrToIPFS(block2, "block", "put", "--cid-codec=dag-pb").Stdout.Trimmed() + + resp := client.Get("/ipfs/" + block2CID) + assert.Equal(t, 200, resp.StatusCode) + assert.Equal(t, "foofoo", resp.Body) + }) + + t.Run("verify gateway file", func(t *testing.T) { + t.Parallel() + r := regexp.MustCompile(`Gateway \(readonly\) server listening on (?P.+)\s`) + matches := r.FindStringSubmatch(node.Daemon.Stdout.String()) + ma, err := multiaddr.NewMultiaddr(matches[1]) + require.NoError(t, err) + netAddr, err := manet.ToNetAddr(ma) + require.NoError(t, err) + expURL := "http://" + netAddr.String() + + b, err := os.ReadFile(filepath.Join(node.Dir, "gateway")) + require.NoError(t, err) + + assert.Equal(t, expURL, string(b)) + }) + + t.Run("verify gateway file diallable while on unspecified", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init() + node.UpdateConfig(func(cfg *config.Config) { + cfg.Addresses.Gateway = config.Strings{"/ip4/127.0.0.1/tcp/32563"} + }) + node.StartDaemon() + + b, err := os.ReadFile(filepath.Join(node.Dir, "gateway")) + require.NoError(t, err) + + assert.Equal(t, "http://127.0.0.1:32563", string(b)) + }) + + t.Run("NoFetch", func(t *testing.T) { + t.Parallel() + nodes := harness.NewT(t).NewNodes(2).Init() + node1 := nodes[0] + node2 := nodes[1] + + node1.UpdateConfig(func(cfg *config.Config) { + cfg.Gateway.NoFetch = true + }) + + nodes.StartDaemons().Connect() + + t.Run("not present", func(t *testing.T) { + cidFoo := node2.IPFSAddStr("foo") + + t.Run("not present key from node 1", func(t *testing.T) { + t.Parallel() + assert.Equal(t, 404, node1.GatewayClient().Get("/ipfs/"+cidFoo).StatusCode) + }) + + t.Run("not present IPNS key from node 1", func(t *testing.T) { + t.Parallel() + assert.Equal(t, 400, node1.GatewayClient().Get("/ipns/"+node2.PeerID().String()).StatusCode) + }) + }) + + t.Run("present", func(t *testing.T) { + cidBar := node1.IPFSAddStr("bar") + + t.Run("present key from node 1", func(t *testing.T) { + t.Parallel() + assert.Equal(t, 200, node1.GatewayClient().Get("/ipfs/"+cidBar).StatusCode) + }) + + t.Run("present IPNS key from node 1", func(t *testing.T) { + t.Parallel() + node2.IPFS("name", "publish", "/ipfs/"+cidBar) + assert.Equal(t, 200, node1.GatewayClient().Get("/ipns/"+node2.PeerID().String()).StatusCode) + + }) + }) + }) +} diff --git a/test/cli/harness/harness.go b/test/cli/harness/harness.go index dd9f38ec3f2..de962e1c120 100644 --- a/test/cli/harness/harness.go +++ b/test/cli/harness/harness.go @@ -119,15 +119,19 @@ func (h *Harness) TempFile() *os.File { } // WriteFile writes a file given a filename and its contents. -// The filename should be a relative path. +// The filename must be a relative path, or this panics. func (h *Harness) WriteFile(filename, contents string) { if filepath.IsAbs(filename) { log.Panicf("%s must be a relative path", filename) } absPath := filepath.Join(h.Runner.Dir, filename) - err := os.WriteFile(absPath, []byte(contents), 0644) + err := os.MkdirAll(filepath.Dir(absPath), 0777) if err != nil { - log.Panicf("writing '%s' ('%s'): %s", filename, absPath, err.Error()) + log.Panicf("creating intermediate dirs for %q: %s", filename, err.Error()) + } + err = os.WriteFile(absPath, []byte(contents), 0644) + if err != nil { + log.Panicf("writing %q (%q): %s", filename, absPath, err.Error()) } } @@ -140,8 +144,7 @@ func WaitForFile(path string, timeout time.Duration) error { for { select { case <-timer.C: - end := time.Now() - return fmt.Errorf("timeout waiting for %s after %v", path, end.Sub(start)) + return fmt.Errorf("timeout waiting for %s after %v", path, time.Since(start)) case <-ticker.C: _, err := os.Stat(path) if err == nil { diff --git a/test/cli/harness/http_client.go b/test/cli/harness/http_client.go new file mode 100644 index 00000000000..83aa1bff15e --- /dev/null +++ b/test/cli/harness/http_client.go @@ -0,0 +1,116 @@ +package harness + +import ( + "io" + "net/http" + "strings" + "text/template" + "time" +) + +// HTTPClient is an HTTP client with some conveniences for testing. +// URLs are constructed from a base URL. +// The response body is buffered into a string. +// Internal errors cause panics so that tests don't need to check errors. +// The paths are evaluated as Go templates for readable string interpolation. +type HTTPClient struct { + Client *http.Client + BaseURL string + + Timeout time.Duration + TemplateData any +} + +type HTTPResponse struct { + Body string + StatusCode int + Headers http.Header + + // The raw response. The body will be closed on this response. + Resp *http.Response +} + +func (c *HTTPClient) WithHeader(k, v string) func(h *http.Request) { + return func(h *http.Request) { + h.Header.Add(k, v) + } +} + +func (c *HTTPClient) DisableRedirects() *HTTPClient { + c.Client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + } + return c +} + +// Do executes the request unchanged. +func (c *HTTPClient) Do(req *http.Request) *HTTPResponse { + log.Debugf("making HTTP req %s to %q with headers %+v", req.Method, req.URL.String(), req.Header) + resp, err := c.Client.Do(req) + if resp != nil && resp.Body != nil { + defer resp.Body.Close() + } + if err != nil { + panic(err) + } + bodyStr, err := io.ReadAll(resp.Body) + if err != nil { + panic(err) + } + + return &HTTPResponse{ + Body: string(bodyStr), + StatusCode: resp.StatusCode, + Headers: resp.Header, + Resp: resp, + } +} + +// BuildURL constructs a request URL from the given path by interpolating the string and then appending it to the base URL. +func (c *HTTPClient) BuildURL(urlPath string) string { + sb := &strings.Builder{} + err := template.Must(template.New("test").Parse(urlPath)).Execute(sb, c.TemplateData) + if err != nil { + panic(err) + } + renderedPath := sb.String() + return c.BaseURL + renderedPath +} + +func (c *HTTPClient) Get(urlPath string, opts ...func(*http.Request)) *HTTPResponse { + req, err := http.NewRequest(http.MethodGet, c.BuildURL(urlPath), nil) + if err != nil { + panic(err) + } + for _, o := range opts { + o(req) + } + return c.Do(req) +} + +func (c *HTTPClient) Post(urlPath string, body io.Reader, opts ...func(*http.Request)) *HTTPResponse { + req, err := http.NewRequest(http.MethodPost, c.BuildURL(urlPath), body) + if err != nil { + panic(err) + } + for _, o := range opts { + o(req) + } + return c.Do(req) +} + +func (c *HTTPClient) PostStr(urlpath, body string, opts ...func(*http.Request)) *HTTPResponse { + r := strings.NewReader(body) + return c.Post(urlpath, r, opts...) +} + +func (c *HTTPClient) Head(urlPath string, opts ...func(*http.Request)) *HTTPResponse { + req, err := http.NewRequest(http.MethodHead, c.BuildURL(urlPath), nil) + if err != nil { + panic(err) + } + for _, o := range opts { + o(req) + } + return c.Do(req) +} diff --git a/test/cli/harness/node.go b/test/cli/harness/node.go index 227737eb96c..26a66ddd9fd 100644 --- a/test/cli/harness/node.go +++ b/test/cli/harness/node.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "io/fs" "net/http" "os" "os/exec" @@ -19,6 +20,7 @@ import ( serial "github.com/ipfs/kubo/config/serialize" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" ) var log = logging.Logger("testharness") @@ -29,14 +31,15 @@ type Node struct { ID int Dir string - APIListenAddr multiaddr.Multiaddr - SwarmAddr multiaddr.Multiaddr - EnableMDNS bool + APIListenAddr multiaddr.Multiaddr + GatewayListenAddr multiaddr.Multiaddr + SwarmAddr multiaddr.Multiaddr + EnableMDNS bool IPFSBin string Runner *Runner - daemon *RunResult + Daemon *RunResult } func BuildNode(ipfsBin, baseDir string, id int) *Node { @@ -134,11 +137,19 @@ func (n *Node) Init(ipfsArgs ...string) *Node { n.APIListenAddr = apiAddr } + if n.GatewayListenAddr == nil { + gatewayAddr, err := multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/0") + if err != nil { + panic(err) + } + n.GatewayListenAddr = gatewayAddr + } + n.UpdateConfig(func(cfg *config.Config) { cfg.Bootstrap = []string{} cfg.Addresses.Swarm = []string{n.SwarmAddr.String()} cfg.Addresses.API = []string{n.APIListenAddr.String()} - cfg.Addresses.Gateway = []string{""} + cfg.Addresses.Gateway = []string{n.GatewayListenAddr.String()} cfg.Swarm.DisableNatPortMap = true cfg.Discovery.MDNS.Enabled = n.EnableMDNS }) @@ -159,7 +170,7 @@ func (n *Node) StartDaemon(ipfsArgs ...string) *Node { RunFunc: (*exec.Cmd).Start, }) - n.daemon = &res + n.Daemon = &res log.Debugf("node %d started, checking API", n.ID) n.WaitOnAPI() @@ -167,7 +178,7 @@ func (n *Node) StartDaemon(ipfsArgs ...string) *Node { } func (n *Node) signalAndWait(watch <-chan struct{}, signal os.Signal, t time.Duration) bool { - err := n.daemon.Cmd.Process.Signal(signal) + err := n.Daemon.Cmd.Process.Signal(signal) if err != nil { if errors.Is(err, os.ErrProcessDone) { log.Debugf("process for node %d has already finished", n.ID) @@ -187,13 +198,13 @@ func (n *Node) signalAndWait(watch <-chan struct{}, signal os.Signal, t time.Dur func (n *Node) StopDaemon() *Node { log.Debugf("stopping node %d", n.ID) - if n.daemon == nil { + if n.Daemon == nil { log.Debugf("didn't stop node %d since no daemon present", n.ID) return n } watch := make(chan struct{}, 1) go func() { - _, _ = n.daemon.Cmd.Process.Wait() + _, _ = n.Daemon.Cmd.Process.Wait() watch <- struct{}{} }() log.Debugf("signaling node %d with SIGTERM", n.ID) @@ -224,6 +235,15 @@ func (n *Node) APIAddr() multiaddr.Multiaddr { return ma } +func (n *Node) APIURL() string { + apiAddr := n.APIAddr() + netAddr, err := manet.ToNetAddr(apiAddr) + if err != nil { + panic(err) + } + return "http://" + netAddr.String() +} + func (n *Node) TryAPIAddr() (multiaddr.Multiaddr, error) { b, err := os.ReadFile(filepath.Join(n.Dir, "api")) if err != nil { @@ -305,20 +325,21 @@ func (n *Node) WaitOnAPI() *Node { log.Debugf("waiting on API for node %d", n.ID) for i := 0; i < 50; i++ { if n.checkAPI() { + log.Debugf("daemon API found, daemon stdout: %s", n.Daemon.Stdout.String()) return n } time.Sleep(400 * time.Millisecond) } - log.Panicf("node %d with peer ID %s failed to come online: \n%s\n\n%s", n.ID, n.PeerID(), n.daemon.Stderr.String(), n.daemon.Stdout.String()) + log.Panicf("node %d with peer ID %s failed to come online: \n%s\n\n%s", n.ID, n.PeerID(), n.Daemon.Stderr.String(), n.Daemon.Stdout.String()) return n } func (n *Node) IsAlive() bool { - if n.daemon == nil || n.daemon.Cmd == nil || n.daemon.Cmd.Process == nil { + if n.Daemon == nil || n.Daemon.Cmd == nil || n.Daemon.Cmd.Process == nil { return false } log.Debugf("signaling node %d daemon process for liveness check", n.ID) - err := n.daemon.Cmd.Process.Signal(syscall.Signal(0)) + err := n.Daemon.Cmd.Process.Signal(syscall.Signal(0)) if err == nil { log.Debugf("node %d daemon is alive", n.ID) return true @@ -381,3 +402,38 @@ func (n *Node) Peers() []multiaddr.Multiaddr { } return addrs } + +// GatewayURL waits for the gateway file and then returns its contents or times out. +func (n *Node) GatewayURL() string { + timer := time.NewTimer(1 * time.Second) + defer timer.Stop() + for { + select { + case <-timer.C: + panic("timeout waiting for gateway file") + default: + b, err := os.ReadFile(filepath.Join(n.Dir, "gateway")) + if err == nil { + return strings.TrimSpace(string(b)) + } + if !errors.Is(err, fs.ErrNotExist) { + panic(err) + } + time.Sleep(1 * time.Millisecond) + } + } +} + +func (n *Node) GatewayClient() *HTTPClient { + return &HTTPClient{ + Client: http.DefaultClient, + BaseURL: n.GatewayURL(), + } +} + +func (n *Node) APIClient() *HTTPClient { + return &HTTPClient{ + Client: http.DefaultClient, + BaseURL: n.APIURL(), + } +} diff --git a/test/cli/harness/nodes.go b/test/cli/harness/nodes.go index b142e3d8f43..dbc7de16ba1 100644 --- a/test/cli/harness/nodes.go +++ b/test/cli/harness/nodes.go @@ -1,6 +1,8 @@ package harness import ( + "sync" + "github.com/multiformats/go-multiaddr" ) @@ -15,14 +17,22 @@ func (n Nodes) Init(args ...string) Nodes { } func (n Nodes) Connect() Nodes { + wg := sync.WaitGroup{} for i, node := range n { for j, otherNode := range n { if i == j { continue } - node.Connect(otherNode) + node := node + otherNode := otherNode + wg.Add(1) + go func() { + defer wg.Done() + node.Connect(otherNode) + }() } } + wg.Wait() for _, node := range n { firstPeer := node.Peers()[0] if _, err := firstPeer.ValueForProtocol(multiaddr.P_P2P); err != nil { @@ -33,9 +43,16 @@ func (n Nodes) Connect() Nodes { } func (n Nodes) StartDaemons() Nodes { + wg := sync.WaitGroup{} for _, node := range n { - node.StartDaemon() + wg.Add(1) + node := node + go func() { + defer wg.Done() + node.StartDaemon() + }() } + wg.Wait() return n } diff --git a/test/cli/testutils/strings.go b/test/cli/testutils/strings.go index 529948d3feb..1fb1512485e 100644 --- a/test/cli/testutils/strings.go +++ b/test/cli/testutils/strings.go @@ -3,7 +3,13 @@ package testutils import ( "bufio" "fmt" + "net" + "net/netip" + "net/url" "strings" + + "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" ) // StrCat takes a bunch of strings or string slices @@ -51,3 +57,21 @@ func SplitLines(s string) []string { } return lines } + +// URLStrToMultiaddr converts a URL string like http://localhost:80 to a multiaddr. +func URLStrToMultiaddr(u string) multiaddr.Multiaddr { + parsedURL, err := url.Parse(u) + if err != nil { + panic(err) + } + addrPort, err := netip.ParseAddrPort(parsedURL.Host) + if err != nil { + panic(err) + } + tcpAddr := net.TCPAddrFromAddrPort(addrPort) + ma, err := manet.FromNetAddr(tcpAddr) + if err != nil { + panic(err) + } + return ma +} diff --git a/test/sharness/t0110-gateway-data/foo.block b/test/sharness/t0110-gateway-data/foo.block deleted file mode 100644 index 39c7ef60b82..00000000000 --- a/test/sharness/t0110-gateway-data/foo.block +++ /dev/null @@ -1,2 +0,0 @@ - - foo \ No newline at end of file diff --git a/test/sharness/t0110-gateway-data/foofoo.block b/test/sharness/t0110-gateway-data/foofoo.block deleted file mode 100644 index 9e5177b183ca963f4b15a2e8ee981f55bded1bcd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 82 zcmd;L;b4+r6H?()5>hxn>F@iqhke#C%;{!*ou>UDv3KXa&^K4qTVK9O7y5BOl{i%Z Ds+T9Z diff --git a/test/sharness/t0110-gateway.sh b/test/sharness/t0110-gateway.sh deleted file mode 100755 index 5244bd2145e..00000000000 --- a/test/sharness/t0110-gateway.sh +++ /dev/null @@ -1,357 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2015 Matt Bell -# MIT Licensed; see the LICENSE file in this repository. -# - -test_description="Test HTTP Gateway" - -. lib/test-lib.sh - -test_init_ipfs -test_launch_ipfs_daemon - -port=$GWAY_PORT -apiport=$API_PORT - -# TODO check both 5001 and 5002. -# 5001 should have a readable gateway (part of the API) -# 5002 should have a readable gateway (using ipfs config Addresses.Gateway) -# but ideally we should only write the tests once. so maybe we need to -# define a function to test a gateway, and do so for each port. -# for now we check 5001 here as 5002 will be checked in gateway-writable. - -test_expect_success "Make a file to test with" ' - echo "Hello Worlds!" >expected && - HASH=$(ipfs add -q expected) || - test_fsh cat daemon_err -' - -test_expect_success "GET IPFS path succeeds" ' - curl -sfo actual "http://127.0.0.1:$port/ipfs/$HASH" -' - -test_expect_success "GET IPFS path with explicit ?filename succeeds with proper header" " - curl -fo actual -D actual_headers 'http://127.0.0.1:$port/ipfs/$HASH?filename=testтест.pdf' && - grep -F 'Content-Disposition: inline; filename=\"test____.pdf\"; filename*=UTF-8'\'\''test%D1%82%D0%B5%D1%81%D1%82.pdf' actual_headers -" - -test_expect_success "GET IPFS path with explicit ?filename and &download=true succeeds with proper header" " - curl -fo actual -D actual_headers 'http://127.0.0.1:$port/ipfs/$HASH?filename=testтест.mp4&download=true' && - grep -F 'Content-Disposition: attachment; filename=\"test____.mp4\"; filename*=UTF-8'\'\''test%D1%82%D0%B5%D1%81%D1%82.mp4' actual_headers -" - -# https://github.com/ipfs/go-ipfs/issues/4025#issuecomment-342250616 -test_expect_success "GET for Service Worker registration outside of an IPFS content root errors" " - curl -H 'Service-Worker: script' -svX GET 'http://127.0.0.1:$port/ipfs/$HASH?filename=sw.js' > curl_sw_out 2>&1 && - grep 'HTTP/1.1 400 Bad Request' curl_sw_out && - grep 'navigator.serviceWorker: registration is not allowed for this scope' curl_sw_out -" - -test_expect_success "GET IPFS path output looks good" ' - test_cmp expected actual && - rm actual -' - -test_expect_success "GET IPFS directory path succeeds" ' - mkdir -p dir/dirwithindex && - echo "12345" >dir/test && - echo "hello i am a webpage" >dir/dirwithindex/index.html && - ipfs add -r -q dir >actual && - HASH2=$(tail -n 1 actual) && - curl -sf "http://127.0.0.1:$port/ipfs/$HASH2" -' - -test_expect_success "GET IPFS directory file succeeds" ' - curl -sfo actual "http://127.0.0.1:$port/ipfs/$HASH2/test" -' - -test_expect_success "GET IPFS directory file output looks good" ' - test_cmp dir/test actual -' - -test_expect_success "GET IPFS directory with index.html returns redirect to add trailing slash" " - curl -sI -o response_without_slash \"http://127.0.0.1:$port/ipfs/$HASH2/dirwithindex?query=to-remember\" && - test_should_contain \"HTTP/1.1 301 Moved Permanently\" response_without_slash && - test_should_contain \"Location: /ipfs/$HASH2/dirwithindex/?query=to-remember\" response_without_slash -" - -# This enables go get to parse go-import meta tags from index.html files stored in IPFS -# https://github.com/ipfs/kubo/pull/3963 -test_expect_success "GET IPFS directory with index.html and no trailing slash returns expected output when go-get is passed" " - curl -s -o response_with_slash \"http://127.0.0.1:$port/ipfs/$HASH2/dirwithindex?go-get=1\" && - test_should_contain \"hello i am a webpage\" response_with_slash -" - -test_expect_success "GET IPFS directory with index.html and trailing slash returns expected output" " - curl -s -o response_with_slash \"http://127.0.0.1:$port/ipfs/$HASH2/dirwithindex/?query=to-remember\" && - test_should_contain \"hello i am a webpage\" response_with_slash -" - -test_expect_success "GET IPFS nonexistent file returns 404 (Not Found)" ' - test_curl_resp_http_code "http://127.0.0.1:$port/ipfs/$HASH2/pleaseDontAddMe" "HTTP/1.1 404 Not Found" -' - -test_expect_success "GET IPFS invalid CID returns 400 (Bad Request)" ' - test_curl_resp_http_code "http://127.0.0.1:$port/ipfs/QmInvalid/pleaseDontAddMe" "HTTP/1.1 400 Bad Request" -' - -# https://github.com/ipfs/go-ipfs/issues/8230 -test_expect_success "GET IPFS inlined zero-length data object returns ok code (200)" ' - curl -sD - "http://127.0.0.1:$port/ipfs/bafkqaaa" > empty_ok_response && - test_should_contain "HTTP/1.1 200 OK" empty_ok_response && - test_should_contain "Content-Length: 0" empty_ok_response -' - -# https://github.com/ipfs/kubo/issues/9238 -test_expect_success "GET IPFS inlined zero-length data object with byte range returns ok code (200)" ' - curl -sD - "http://127.0.0.1:$port/ipfs/bafkqaaa" -H "Range: bytes=0-1048575" > empty_ok_response && - test_should_contain "HTTP/1.1 200 OK" empty_ok_response && - test_should_contain "Content-Length: 0" empty_ok_response && - test_should_contain "Content-Type: text/plain" empty_ok_response -' - -test_expect_success "GET /ipfs/ipfs/{cid} returns redirect to the valid path" ' - curl -sD - "http://127.0.0.1:$port/ipfs/ipfs/bafkqaaa?query=to-remember" > response_with_double_ipfs_ns && - test_should_contain "" response_with_double_ipfs_ns && - test_should_contain "" response_with_double_ipfs_ns -' - -test_expect_success "GET invalid IPNS root returns 400 (Bad Request)" ' - test_curl_resp_http_code "http://127.0.0.1:$port/ipns/QmInvalid/pleaseDontAddMe" "HTTP/1.1 400 Bad Request" -' - -test_expect_success "GET IPNS path succeeds" ' - ipfs name publish --allow-offline "$HASH" && - PEERID=$(ipfs config Identity.PeerID) && - test_check_peerid "$PEERID" && - curl -sfo actual "http://127.0.0.1:$port/ipns/$PEERID" -' - -test_expect_success "GET IPNS path output looks good" ' - test_cmp expected actual -' - -test_expect_success "GET /ipfs/ipns/{peerid} returns redirect to the valid path" ' - PEERID=$(ipfs config Identity.PeerID) && - curl -sD - "http://127.0.0.1:$port/ipfs/ipns/${PEERID}?query=to-remember" > response_with_ipfs_ipns_ns && - test_should_contain "" response_with_ipfs_ipns_ns && - test_should_contain "" response_with_ipfs_ipns_ns -' - -test_expect_success "GET invalid IPFS path errors" ' - test_must_fail curl -sf "http://127.0.0.1:$port/ipfs/12345" -' - -test_expect_success "GET invalid path errors" ' - test_must_fail curl -sf "http://127.0.0.1:$port/12345" -' - -test_expect_success "GET /webui returns code expected" ' - test_curl_resp_http_code "http://127.0.0.1:$apiport/webui" "HTTP/1.1 302 Found" "HTTP/1.1 301 Moved Permanently" -' - -test_expect_success "GET /webui/ returns code expected" ' - test_curl_resp_http_code "http://127.0.0.1:$apiport/webui/" "HTTP/1.1 302 Found" "HTTP/1.1 301 Moved Permanently" -' - -test_expect_success "GET /logs returns logs" ' - test_expect_code 28 curl http://127.0.0.1:$apiport/logs -m1 > log_out -' - -test_expect_success "log output looks good" ' - grep "log API client connected" log_out -' - -test_expect_success "GET /api/v0/version succeeds" ' - curl -X POST -v "http://127.0.0.1:$apiport/api/v0/version" 2> version_out -' - -test_expect_success "output only has one transfer encoding header" ' - grep "Transfer-Encoding: chunked" version_out | wc -l | xargs echo > tecount_out && - echo "1" > tecount_exp && - test_cmp tecount_out tecount_exp -' - -curl_pprofmutex() { - curl -f -X POST "http://127.0.0.1:$apiport/debug/pprof-mutex/?fraction=$1" -} - -test_expect_success "set mutex fraction for pprof (negative so it doesn't enable)" ' - curl_pprofmutex -1 -' - -test_expect_success "test failure conditions of mutex pprof endpoint" ' - test_must_fail curl_pprofmutex && - test_must_fail curl_pprofmutex that_is_string && - test_must_fail curl -f -X GET "http://127.0.0.1:$apiport/debug/pprof-mutex/?fraction=-1" -' - -curl_pprofblock() { - curl -f -X POST "http://127.0.0.1:$apiport/debug/pprof-block/?rate=$1" -} - -test_expect_success "set blocking profiler rate for pprof (0 so it doesn't enable)" ' - curl_pprofblock 0 -' - -test_expect_success "test failure conditions of mutex block endpoint" ' - test_must_fail curl_pprofblock && - test_must_fail curl_pprofblock that_is_string && - test_must_fail curl -f -X GET "http://127.0.0.1:$apiport/debug/pprof-block/?rate=0" -' - -test_expect_success "setup index hash" ' - mkdir index && - echo "

" > index/index.html && - INDEXHASH=$(ipfs add -Q -r index) - echo index: $INDEXHASH -' - -test_expect_success "GET 'index.html' has correct content type" ' - curl -I "http://127.0.0.1:$port/ipfs/$INDEXHASH/" > indexout -' - -test_expect_success "output looks good" ' - grep "Content-Type: text/html" indexout -' - -test_expect_success "HEAD 'index.html' has no content" ' - curl -X HEAD --max-time 1 http://127.0.0.1:$port/ipfs/$INDEXHASH/ > output; - [ ! -s output ] -' - -# test ipfs readonly api - -test_curl_gateway_api() { - curl -sfo actual "http://127.0.0.1:$port/api/v0/$1" -} - -test_expect_success "get IPFS directory file through readonly API succeeds" ' - test_curl_gateway_api "cat?arg=$HASH2/test" -' - -test_expect_success "get IPFS directory file through readonly API output looks good" ' - test_cmp dir/test actual -' - -test_expect_success "refs IPFS directory file through readonly API succeeds" ' - test_curl_gateway_api "refs?arg=$HASH2/test" -' - -for cmd in add \ - block/put \ - bootstrap \ - config \ - dag/put \ - dag/import \ - dht \ - diag \ - id \ - mount \ - name/publish \ - object/put \ - object/new \ - object/patch \ - pin \ - ping \ - repo \ - stats \ - swarm \ - file \ - update \ - bitswap -do - test_expect_success "test gateway api is sanitized: $cmd" ' - test_curl_resp_http_code "http://127.0.0.1:$port/api/v0/$cmd" "HTTP/1.1 404 Not Found" - ' -done - -# This one is different. `local` will be interpreted as a path if the command isn't defined. -test_expect_success "test gateway api is sanitized: refs/local" ' - echo "Error: invalid path \"local\": selected encoding not supported" > refs_local_expected && - ! ipfs --api /ip4/127.0.0.1/tcp/$port refs local > refs_local_actual 2>&1 && - test_cmp refs_local_expected refs_local_actual - ' - -test_expect_success "create raw-leaves node" ' - echo "This is RAW!" > rfile && - echo "This is RAW!" | ipfs add --raw-leaves -q > rhash -' - -test_expect_success "try fetching it from gateway" ' - curl http://127.0.0.1:$port/ipfs/$(cat rhash) > ffile && - test_cmp rfile ffile -' - -test_expect_success "Add compact blocks" ' - ipfs block put ../t0110-gateway-data/foo.block && - FOO2_HASH=$(ipfs block put --cid-codec=dag-pb ../t0110-gateway-data/foofoo.block) && - printf "foofoo" > expected -' - -test_expect_success "GET compact blocks succeeds" ' - curl -o actual "http://127.0.0.1:$port/ipfs/$FOO2_HASH" && - test_cmp expected actual -' - -test_expect_success "Verify gateway file" ' - cat "$IPFS_PATH/gateway" > gateway_file_actual && - echo -n "http://$GWAY_ADDR" > gateway_daemon_actual && - test_cmp gateway_daemon_actual gateway_file_actual -' - -test_kill_ipfs_daemon - -GWPORT=32563 - -test_expect_success "Verify gateway file diallable while on unspecified" ' - ipfs config Addresses.Gateway /ip4/0.0.0.0/tcp/$GWPORT && - test_launch_ipfs_daemon && - cat "$IPFS_PATH/gateway" > gateway_file_actual && - echo -n "http://127.0.0.1:$GWPORT" > gateway_file_expected && - test_cmp gateway_file_expected gateway_file_actual -' - -test_kill_ipfs_daemon - -test_expect_success "set up iptb testbed" ' - iptb testbed create -type localipfs -count 5 -force -init && - ipfsi 0 config Addresses.Gateway /ip4/127.0.0.1/tcp/$GWPORT && - PEERID_1=$(iptb attr get 1 id) -' - -test_expect_success "set NoFetch to true in config of node 0" ' - ipfsi 0 config --bool=true Gateway.NoFetch true -' - -test_expect_success "start ipfs nodes" ' - iptb start -wait && - iptb connect 0 1 -' - -test_expect_success "try fetching not present key from node 0" ' - FOO=$(echo "foo" | ipfsi 1 add -Q) && - test_expect_code 22 curl -f "http://127.0.0.1:$GWPORT/ipfs/$FOO" -' - -test_expect_success "try fetching not present ipns key from node 0" ' - ipfsi 1 name publish /ipfs/$FOO && - test_expect_code 22 curl -f "http://127.0.0.1:$GWPORT/ipns/$PEERID_1" -' - -test_expect_success "try fetching present key from node 0" ' - BAR=$(echo "bar" | ipfsi 0 add -Q) && - curl -f "http://127.0.0.1:$GWPORT/ipfs/$BAR" -' - -test_expect_success "try fetching present ipns key from node 0" ' - ipfsi 1 name publish /ipfs/$BAR && - curl "http://127.0.0.1:$GWPORT/ipns/$PEERID_1" -' - -test_expect_success "stop testbed" ' - iptb stop -' - -test_done