Merge pull request #66 from ipfs/ci/gateway-conformance

* ci: add gateway-confromance test suite to the CI
* fix: disable subdomain-gateway tests
* fix: disable TestGatewayCar
* ci: pin gateway-conformance to a SHA because it doesn't have release tags yet
* ci: update triggers for gateway-conformance workflow
* chore: enable subdomain tests, add env variable
* fix: support http.MethodConnect
* docs: environment variables for testing
* fix: exclude TestGatewayCar/GET_response_for_application/vnd.ipld.car only
* ci: exclude only Content-Length check from the TestGatewayCar test
* ci: use v0.0 of gateway-conformance actions

---------

Co-authored-by: Henrique Dias <hacdias@gmail.com>
Co-authored-by: Marcin Rataj <lidel@lidel.org>

.github/workflows/gateway-conformance.yml

@@ -0,0 +1,70 @@
+name: Gateway Conformance
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  gateway-conformance:
+    runs-on: ubuntu-latest
+    steps:
+      # 1. Start the Kubo gateway
+      - name: Download Kubo gateway
+        uses: ipfs/download-ipfs-distribution-action@v1
+      - name: Start Kubo gateway
+        uses: ipfs/start-ipfs-daemon-action@v1
+
+      # 2. Download the gateway-conformance fixtures
+      - name: Download gateway-conformance fixtures
+        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.0
+        with:
+          output: fixtures
+
+      # 3. Populate the Kubo gateway with the gateway-conformance fixtures
+      - name: Import fixtures
+        run: find ./fixtures -name '*.car' -exec ipfs dag import --pin-roots=false {} \;
+
+      # 4. Build the bifrost-gateway
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19.x
+      - name: Checkout bifrost-gateway
+        uses: actions/checkout@v3
+        with:
+          path: bifrost-gateway
+      - name: Build bifrost-gateway
+        run: go build
+        working-directory: bifrost-gateway
+
+      # 5. Start the bifrost-gateway
+      - name: Start bifrost-gateway
+        env:
+          PROXY_GATEWAY_URL: http://127.0.0.1:8080
+          GATEWAY_CONFORMANCE_TEST: true
+        run: ./bifrost-gateway &
+        working-directory: bifrost-gateway
+
+      # 6. Run the gateway-conformance tests
+      - name: Run gateway-conformance tests
+        uses: ipfs/gateway-conformance/.github/actions/test@v0.0
+        with:
+          gateway-url: http://127.0.0.1:8081
+          json: output.json
+          xml: output.xml
+          html: output.html
+          markdown: output.md
+          args: -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length'
+
+      # 7. Upload the results
+      - name: Upload MD summary
+        if: failure() || success()
+        run: cat output.md >> $GITHUB_STEP_SUMMARY
+      - name: Upload HTML report
+        if: failure() || success()
+        uses: actions/upload-artifact@v3
+        with:
+          name: gateway-conformance.html
+          path: output.html

docs/environment-variables.md

@@ -16,6 +16,9 @@
   - [`GOLOG_LOG_FMT`](#golog_log_fmt)
   - [`GOLOG_FILE`](#golog_file)
   - [`GOLOG_TRACING_FILE`](#golog_tracing_file)
+- [Testing](#testing)
+  - [`GATEWAY_CONFORMANCE_TEST`](#gateway_conformance_test)
+  - [`IPFS_NS_MAP`](#ipfs_ns_map)
 
 ## Configuration
 
@@ -108,3 +111,24 @@ Sets the file to which the Bifrost Gateway sends tracing events. By default,
 tracing is disabled.
 
 Warning: Enabling tracing will likely affect performance.
+
+
+## Testing
+
+### `GATEWAY_CONFORMANCE_TEST`
+
+Setting to `true` enables support for test fixtures required by [ipfs/gateway-conformance](https://github.com/ipfs/gateway-conformance) test suite.
+
+### `IPFS_NS_MAP`
+
+Adds static namesys records for deterministic tests and debugging.
+Useful for testing `/ipns/` support without having to do real IPNS/DNS lookup.
+
+Example:
+
+```console
+$ IPFS_NS_MAP="dnslink-test1.example.com:/ipfs/bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am,dnslink-test2.example.com:/ipns/dnslink-test1.example.com" ./gateway-binary
+...
+$ curl -is http://127.0.0.1:8081/dnslink-test2.example.com/ | grep Etag
+Etag: "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am"
+```

handlers.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"math/rand"
 	"net/http"
+	"os"
 	"strconv"
 	"time"
 
@@ -32,6 +33,20 @@ func makeMetricsHandler(port int) (*http.Server, error) {
 	}, nil
 }
 
+func withConnect(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// ServeMux does not support requests with CONNECT method,
+		// so we need to handle them separately
+		// https://golang.org/src/net/http/request.go#L111
+		if r.Method == http.MethodConnect {
+			w.WriteHeader(http.StatusOK)
+			return
+		}
+
+		next.ServeHTTP(w, r)
+	})
+}
+
 func withRequestLogger(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		goLog.Infow(r.Method, "url", r.URL, "host", r.Host)
@@ -96,6 +111,15 @@ func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockC
 		},
 	}
 
+	// If we're doing tests, ensure the right public gateways are enabled.
+	if os.Getenv("GATEWAY_CONFORMANCE_TEST") == "true" {
+		publicGateways["example.com"] = &gateway.Specification{
+			Paths:         []string{"/ipfs", "/ipns"},
+			NoDNSLink:     noDNSLink,
+			UseSubdomains: true,
+		}
+	}
+
 	// Creates metrics handler for total response size. Matches the same metrics
 	// from Kubo:
 	// https://github.com/ipfs/kubo/blob/e550d9e4761ea394357c413c02ade142c0dea88c/core/corehttp/metrics.go#L79-L152
@@ -112,7 +136,8 @@ func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockC
 	}
 
 	// Construct the HTTP handler for the gateway.
-	handler := http.Handler(gateway.WithHostname(mux, gwAPI, publicGateways, noDNSLink))
+	handler := withConnect(mux)
+	handler = http.Handler(gateway.WithHostname(handler, gwAPI, publicGateways, noDNSLink))
 	handler = promhttp.InstrumentHandlerResponseSize(sum, handler)
 
 	// Add logging