feat: add gateway to http over libp2p #1096
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Gateway Conformance | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| paths-ignore: | |
| - '**/*.md' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| gateway-conformance: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| # 1. Download the gateway-conformance fixtures | |
| - name: Download gateway-conformance fixtures | |
| uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.3 | |
| with: | |
| output: fixtures | |
| # 2. Build the kubo-gateway | |
| - name: Setup Go | |
| uses: actions/setup-go@v3 | |
| with: | |
| go-version: 1.20.x | |
| - name: Checkout kubo-gateway | |
| uses: actions/checkout@v3 | |
| with: | |
| path: kubo-gateway | |
| - name: Build kubo-gateway | |
| run: make build | |
| working-directory: kubo-gateway | |
| # 3. Init the kubo-gateway | |
| - name: Init kubo-gateway | |
| env: | |
| GATEWAY_PUBLIC_GATEWAYS: | | |
| { | |
| "example.com": { | |
| "UseSubdomains": true, | |
| "InlineDNSLink": true, | |
| "Paths": ["/ipfs", "/ipns"] | |
| }, | |
| "localhost": { | |
| "UseSubdomains": true, | |
| "InlineDNSLink": true, | |
| "Paths": ["/ipfs", "/ipns"] | |
| } | |
| } | |
| run: | | |
| ./ipfs init --profile=test | |
| ./ipfs config --json Gateway.PublicGateways "$GATEWAY_PUBLIC_GATEWAYS" | |
| ./ipfs config --json Experimental.GatewayOverLibp2p true | |
| ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8080" | |
| ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001" | |
| working-directory: kubo-gateway/cmd/ipfs | |
| # 4. Populate the Kubo gateway with the gateway-conformance fixtures | |
| - name: Import fixtures | |
| run: | | |
| # Import car files | |
| find ./fixtures -name '*.car' -exec kubo-gateway/cmd/ipfs/ipfs dag import --pin-roots=false {} \; | |
| # Import ipns records | |
| records=$(find ./fixtures -name '*.ipns-record') | |
| for record in $records | |
| do | |
| key=$(basename -s .ipns-record "$record" | cut -d'_' -f1) | |
| kubo-gateway/cmd/ipfs/ipfs routing put --allow-offline "/ipns/$key" "$record" | |
| done | |
| # Import dnslink records | |
| # the IPFS_NS_MAP env will be used by the daemon | |
| export IPFS_NS_MAP=$(cat "./fixtures/dnslinks.json" | jq -r '.subdomains | to_entries | map("\(.key).example.com:\(.value)") | join(",")') | |
| export IPFS_NS_MAP="$(cat "./fixtures/dnslinks.json" | jq -r '.domains | to_entries | map("\(.key):\(.value)") | join(",")'),${IPFS_NS_MAP}" | |
| echo "IPFS_NS_MAP=${IPFS_NS_MAP}" >> $GITHUB_ENV | |
| # 5. Start the kubo-gateway | |
| - name: Start kubo-gateway | |
| run: | | |
| ./ipfs daemon & | |
| endpoint="http://127.0.0.1:5001/api/v0/version" | |
| max_retries=5 | |
| retry_interval=3 | |
| check_endpoint() { | |
| curl -X POST --silent --fail "$endpoint" > /dev/null | |
| return $? | |
| } | |
| retries=0 | |
| while ! check_endpoint; do | |
| retries=$((retries+1)) | |
| if [ $retries -ge $max_retries ]; then | |
| echo "daemon took too long to start" | |
| exit 1 | |
| fi | |
| sleep $retry_interval | |
| done | |
| echo "daemon started and ready to receive API calls" | |
| working-directory: kubo-gateway/cmd/ipfs | |
| # 6. Run the gateway-conformance tests | |
| - name: Run gateway-conformance tests | |
| uses: ipfs/gateway-conformance/.github/actions/test@v0.3 | |
| with: | |
| gateway-url: http://127.0.0.1:8080 | |
| json: output.json | |
| xml: output.xml | |
| html: output.html | |
| markdown: output.md | |
| args: -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length' | |
| # 7. Upload the results | |
| - name: Upload MD summary | |
| if: failure() || success() | |
| run: cat output.md >> $GITHUB_STEP_SUMMARY | |
| - name: Upload HTML report | |
| if: failure() || success() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: gateway-conformance.html | |
| path: output.html | |
| - name: Upload JSON report | |
| if: failure() || success() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: gateway-conformance.json | |
| path: output.json | |
| # 8. Setup a kubo http-p2p-proxy to run gateway conformance tests | |
| - name: Init p2p-proxy kubo node | |
| env: | |
| IPFS_PATH: "~/.kubo-p2p-proxy" | |
| run: | | |
| ./ipfs init --profile=test | |
| ./ipfs config --json Experimental.Libp2pStreamMounting true | |
| ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8081" | |
| ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5002" | |
| working-directory: kubo-gateway/cmd/ipfs | |
| # 9. Start the kubo http-p2p-proxy | |
| - name: Start kubo http-p2p-proxy | |
| env: | |
| IPFS_PATH: "~/.kubo-p2p-proxy" | |
| run: | | |
| ./ipfs daemon & | |
| endpoint="http://127.0.0.1:5002/api/v0/version" | |
| max_retries=5 | |
| retry_interval=3 | |
| check_endpoint() { | |
| curl -X POST --silent --fail "$endpoint" > /dev/null | |
| return $? | |
| } | |
| retries=0 | |
| while ! check_endpoint; do | |
| retries=$((retries+1)) | |
| if [ $retries -ge $max_retries ]; then | |
| echo "daemon took too long to start" | |
| exit 1 | |
| fi | |
| sleep $retry_interval | |
| done | |
| echo "daemon started and ready to receive API calls" | |
| working-directory: kubo-gateway/cmd/ipfs | |
| # 10. Start forwarding data from the http-p2p-proxy to the node serving the Gateway API over libp2p | |
| - name: Start http-over-libp2p forwarding proxy | |
| run: | | |
| gatewayNodeId=$(./ipfs --api=/ip4/127.0.0.1/tcp/5001 id -f="<id>") | |
| ./ipfs --api=/ip4/127.0.0.1/tcp/5002 swarm connect $(./ipfs --api=/ip4/127.0.0.1/tcp/5001 swarm addrs local --id | head -n 1) | |
| ./ipfs --api=/ip4/127.0.0.1/tcp/5002 p2p forward --allow-custom-protocol /http/1.1 /ip4/127.0.0.1/tcp/8082 /p2p/$gatewayNodeId | |
| working-directory: kubo-gateway/cmd/ipfs | |
| # 11. Run the gateway-conformance tests over libp2p | |
| - name: Run gateway-conformance tests over libp2p | |
| uses: ipfs/gateway-conformance/.github/actions/test@v0.3 | |
| with: | |
| gateway-url: http://127.0.0.1:8081 | |
| json: output.json | |
| xml: output.xml | |
| html: output.html | |
| markdown: output.md | |
| args: --specs "trustless-gateway,-trustless-ipns-gateway" -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length' | |
| # 11. Upload the results | |
| - name: Upload MD summary | |
| if: failure() || success() | |
| run: cat output.md >> $GITHUB_STEP_SUMMARY | |
| - name: Upload HTML report | |
| if: failure() || success() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: gateway-conformance-libp2p.html | |
| path: output.html | |
| - name: Upload JSON report | |
| if: failure() || success() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: gateway-conformance-libp2p.json | |
| path: output.json |