Merge pull request #10060 from ipfs/release-v0.22.0

Release v0.22.0

.github/workflows/build.yml

@@ -48,45 +48,61 @@ jobs:
         with:
           name: kubo
           path: cmd/ipfs/ipfs
-  interop:
+  helia-interop:
     needs: [interop-prep]
     runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }}
     timeout-minutes: 20
     defaults:
       run:
         shell: bash
+    strategy:
+      matrix:
+        repo-to-test-against: ["helia", "helia-ipns", "helia-unixfs"] # this needs to be manually kept in sync as new helia tests are written
     steps:
       - uses: actions/setup-node@v3
         with:
-          node-version: 16.12.0
+          node-version: lts/*
       - uses: actions/download-artifact@v3
         with:
           name: kubo
           path: cmd/ipfs
       - run: chmod +x cmd/ipfs/ipfs
-      - run: |
-          echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT
+      - run: echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT
         id: npm-cache-dir
       - uses: actions/cache@v3
         with:
           path: ${{ steps.npm-cache-dir.outputs.dir }}
-          key: ${{ runner.os }}-${{ github.job }}-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ github.job }}-
-      - run: mkdir interop
-      - run: |
-          npm init -y
-          npm install ipfs@^0.66.0
-          npm install kubo-rpc-client@^3.0.1
-          npm install ipfs-interop@^10.0.1
+          key: ${{ runner.os }}-${{ github.job }}-${{ matrix.repo-to-test-against }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: ${{ runner.os }}-${{ github.job }}-${{ matrix.repo-to-test-against }}-
+      - run: sudo apt update
+      - run: sudo apt install -y libxkbcommon0 libxdamage1 libgbm1 libpango-1.0-0 libcairo2 # dependencies for playwright
+      - uses: actions/checkout@v3
+        with:
+          repository: ipfs/${{ matrix.repo-to-test-against }}
+          fetch-depth: 0
+          path: interop
+      - name: Checkout latest tag
+        run: |
+          exit 0 # temporary while theses pull requests are released:
+          # https://github.com/ipfs/helia/pull/200
+          # https://github.com/ipfs/helia-unixfs/pull/68
+          # https://github.com/ipfs/helia-ipns/pull/72
+          export TAG="$(git describe --tags --abbrev=0)"
+          echo "Running tests against: $TAG"
+          git checkout "$TAG"
         working-directory: interop
-      # Run the interop tests while ignoring the js-js interop test cases
-      - run: npx ipfs-interop -- -t node --grep '^(?!.*(js\d? -> js\d?|js-js-js|js-rv\d?-js))' --parallel
-        env:
-          LIBP2P_TCP_REUSEPORT: false
-          LIBP2P_ALLOW_WEAK_RSA_KEYS: 1
-          IPFS_GO_EXEC: ${{ github.workspace }}/cmd/ipfs/ipfs
+      - run: npm install
+        working-directory: interop
+      - run: npm run build
         working-directory: interop
+      - run: npm install
+        working-directory: interop/packages/interop
+      - run: npm install --ignore-scripts --save "ipfs/npm-go-ipfs#4441b8a60f1cfee3035a9e4bb824dfcca08e9b01" # temporary while https://github.com/ipfs/npm-go-ipfs/pull/62 is being bubbled
+        working-directory: interop/packages/interop
+      - run: npm test
+        working-directory: interop/packages/interop
+        env:
+          KUBO_BINARY: ${{ github.workspace }}/cmd/ipfs/ipfs
   go-ipfs-api:
     needs: [interop-prep]
     runs-on: ubuntu-latest

.github/workflows/docker-build.yml

@@ -1,3 +1,4 @@
+# If we decide to run build-image.yml on every PR, we could deprecate this workflow.
 name: Docker Build
 
 on:
@@ -30,3 +31,4 @@ jobs:
           go-version: 1.19.x
       - uses: actions/checkout@v3
       - run: docker build -t $IMAGE_NAME:$WIP_IMAGE_TAG .
+      - run: docker run --rm $IMAGE_NAME:$WIP_IMAGE_TAG --version

.github/workflows/docker-image.yml

@@ -2,6 +2,16 @@ name: Docker Push
 
 on:
   workflow_dispatch:
+    inputs:
+      push:
+        description: 'Push to Docker Hub'
+        required: true
+        default: 'false'
+  # # If we decide to build all images on every PR, we should make sure that
+  # # they are NOT pushed to Docker Hub.
+  # pull_request:
+  #   paths-ignore:
+  #     - '**/*.md'
   push:
     branches:
       - 'master'
@@ -53,15 +63,62 @@ jobs:
           username: ${{ vars.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
-      - name: Build Docker image and publish to Docker Hub
+      # We have to build each platform separately because when using multi-arch
+      # builds, only one platform is being loaded into the cache. This would
+      # prevent us from testing the other platforms.
+      - name: Build Docker image (linux/amd64)
+        uses: docker/build-push-action@v4
+        with:
+          platforms: linux/amd64
+          context: .
+          push: false
+          load: true
+          file: ./Dockerfile
+          tags: ${{ env.IMAGE_NAME }}:linux-amd64
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+
+      - name: Build Docker image (linux/arm/v7)
+        uses: docker/build-push-action@v4
+        with:
+          platforms: linux/arm/v7
+          context: .
+          push: false
+          load: true
+          file: ./Dockerfile
+          tags: ${{ env.IMAGE_NAME }}:linux-arm-v7
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+
+      - name: Build Docker image (linux/arm64/v8)
+        uses: docker/build-push-action@v4
+        with:
+          platforms: linux/arm64/v8
+          context: .
+          push: false
+          load: true
+          file: ./Dockerfile
+          tags: ${{ env.IMAGE_NAME }}:linux-arm64-v8
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+
+      # We test all the images on amd64 host here. This uses QEMU to emulate
+      # the other platforms.
+      - run: docker run --rm $IMAGE_NAME:linux-amd64 --version
+      - run: docker run --rm $IMAGE_NAME:linux-arm-v7 --version
+      - run: docker run --rm $IMAGE_NAME:linux-arm64-v8 --version
+
+      # This will only push the previously built images.
+      - if: github.event_name != 'workflow_dispatch' || github.event.inputs.push == 'true'
+        name: Publish to Docker Hub
         uses: docker/build-push-action@v4
         with:
           platforms: linux/amd64,linux/arm/v7,linux/arm64/v8
           context: .
           push: true
           file: ./Dockerfile
           tags: "${{ steps.tags.outputs.value }}"
-          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-from: type=local,src=/tmp/.buildx-cache-new
           cache-to: type=local,dest=/tmp/.buildx-cache-new
 
       # https://github.com/docker/build-push-action/issues/252

.github/workflows/gateway-conformance.yml

@@ -1,6 +1,7 @@
 name: Gateway Conformance
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - master
@@ -23,7 +24,7 @@ jobs:
     steps:
       # 1. Download the gateway-conformance fixtures
       - name: Download gateway-conformance fixtures
-        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.2
+        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.3
         with:
           output: fixtures
 
@@ -89,7 +90,7 @@ jobs:
 
       # 6. Run the gateway-conformance tests
       - name: Run gateway-conformance tests
-        uses: ipfs/gateway-conformance/.github/actions/test@v0.2
+        uses: ipfs/gateway-conformance/.github/actions/test@v0.3
         with:
           gateway-url: http://127.0.0.1:8080
           json: output.json

Dockerfile

@@ -1,16 +1,6 @@
-FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.19-buster
-LABEL maintainer="Steven Allen <steven@stebalien.com>"
+FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.19-buster AS builder
 
-ARG TARGETPLATFORM
-ARG BUILDPLATFORM
-ARG TARGETOS
-ARG TARGETARCH
-
-# Install deps
-RUN apt-get update && apt-get install -y \
-  libssl-dev \
-  ca-certificates \
-  fuse
+ARG TARGETPLATFORM TARGETOS TARGETARCH
 
 ENV SRC_DIR /kubo
 
@@ -31,38 +21,40 @@ RUN cd $SRC_DIR \
   && mkdir -p .git/objects \
   && GOOS=$TARGETOS GOARCH=$TARGETARCH GOFLAGS=-buildvcs=false make build GOTAGS=openssl IPFS_PLUGINS=$IPFS_PLUGINS
 
-# Get su-exec, a very minimal tool for dropping privileges,
-# and tini, a very minimal init daemon for containers
-ENV SUEXEC_VERSION v0.2
-ENV TINI_VERSION v0.19.0
+# Using Debian Buster because the version of busybox we're using is based on it
+# and we want to make sure the libraries we're using are compatible. That's also
+# why we're running this for the target platform.
+FROM debian:buster-slim AS utilities
 RUN set -eux; \
-    dpkgArch="$(dpkg --print-architecture)"; \
-    case "${dpkgArch##*-}" in \
-        "amd64" | "armhf" | "arm64") tiniArch="tini-static-$dpkgArch" ;;\
-        *) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \
-    esac; \
-  cd /tmp \
-  && git clone https://github.com/ncopa/su-exec.git \
-  && cd su-exec \
-  && git checkout -q $SUEXEC_VERSION \
-  && make su-exec-static \
-  && cd /tmp \
-  && wget -q -O tini https://github.com/krallin/tini/releases/download/$TINI_VERSION/$tiniArch \
-  && chmod +x tini
+	apt-get update; \
+	apt-get install -y \
+		tini \
+    # Using gosu (~2MB) instead of su-exec (~20KB) because it's easier to
+    # install on Debian. Useful links:
+    # - https://github.com/ncopa/su-exec#why-reinvent-gosu
+    # - https://github.com/tianon/gosu/issues/52#issuecomment-441946745
+		gosu \
+    # This installs fusermount which we later copy over to the target image.
+    fuse \
+    ca-certificates \
+    # This installs libssl.so and libcrypto.so which we later copy over to the
+    # target image. We need these to be able to use the OpenSSL plugin.
+    libssl-dev \
+	; \
+	rm -rf /var/lib/apt/lists/*
 
 # Now comes the actual target image, which aims to be as small as possible.
-FROM --platform=${BUILDPLATFORM:-linux/amd64} busybox:1.31.1-glibc
-LABEL maintainer="Steven Allen <steven@stebalien.com>"
+FROM busybox:1.31.1-glibc
 
 # Get the ipfs binary, entrypoint script, and TLS CAs from the build container.
 ENV SRC_DIR /kubo
-COPY --from=0 $SRC_DIR/cmd/ipfs/ipfs /usr/local/bin/ipfs
-COPY --from=0 $SRC_DIR/bin/container_daemon /usr/local/bin/start_ipfs
-COPY --from=0 $SRC_DIR/bin/container_init_run /usr/local/bin/container_init_run
-COPY --from=0 /tmp/su-exec/su-exec-static /sbin/su-exec
-COPY --from=0 /tmp/tini /sbin/tini
-COPY --from=0 /bin/fusermount /usr/local/bin/fusermount
-COPY --from=0 /etc/ssl/certs /etc/ssl/certs
+COPY --from=builder $SRC_DIR/cmd/ipfs/ipfs /usr/local/bin/ipfs
+COPY --from=builder $SRC_DIR/bin/container_daemon /usr/local/bin/start_ipfs
+COPY --from=builder $SRC_DIR/bin/container_init_run /usr/local/bin/container_init_run
+COPY --from=utilities /usr/sbin/gosu /sbin/gosu
+COPY --from=utilities /usr/bin/tini /sbin/tini
+COPY --from=utilities /bin/fusermount /usr/local/bin/fusermount
+COPY --from=utilities /etc/ssl/certs /etc/ssl/certs
 
 # Add suid bit on fusermount so it will run properly
 RUN chmod 4755 /usr/local/bin/fusermount
@@ -71,11 +63,11 @@ RUN chmod 4755 /usr/local/bin/fusermount
 RUN chmod 0755 /usr/local/bin/start_ipfs
 
 # This shared lib (part of glibc) doesn't seem to be included with busybox.
-COPY --from=0 /lib/*-linux-gnu*/libdl.so.2 /lib/
+COPY --from=utilities /lib/*-linux-gnu*/libdl.so.2 /lib/
 
 # Copy over SSL libraries.
-COPY --from=0 /usr/lib/*-linux-gnu*/libssl.so* /usr/lib/
-COPY --from=0 /usr/lib/*-linux-gnu*/libcrypto.so* /usr/lib/
+COPY --from=utilities /usr/lib/*-linux-gnu*/libssl.so* /usr/lib/
+COPY --from=utilities /usr/lib/*-linux-gnu*/libcrypto.so* /usr/lib/
 
 # Swarm TCP; should be exposed to the public
 EXPOSE 4001

README.md

@@ -165,8 +165,12 @@ $ ipfs get /ipns/dist.ipfs.tech/kubo/$VERSION/kubo_$VERSION_windows-amd64.zip
 
 ### Unofficial Linux packages
 
+<a href="https://repology.org/project/kubo/versions">
+    <img src="https://repology.org/badge/vertical-allrepos/kubo.svg" alt="Packaging status" align="right">
+</a>
+
 - [ArchLinux](#arch-linux)
-- [Nix](#nix)
+- [Nix](#nix-linux)
 - [Solus](#solus)
 - [openSUSE](#opensuse)
 - [Guix](#guix)

bin/container_daemon

@@ -7,9 +7,9 @@ repo="$IPFS_PATH"
 if [ "$(id -u)" -eq 0 ]; then
   echo "Changing user to $user"
   # ensure folder is writable
-  su-exec "$user" test -w "$repo" || chown -R -- "$user" "$repo"
+  gosu "$user" test -w "$repo" || chown -R -- "$user" "$repo"
   # restart script with new privileges
-  exec su-exec "$user" "$0" "$@"
+  exec gosu "$user" "$0" "$@"
 fi
 
 # 2nd invocation with regular user

core/node/libp2p/rcmgr.go

@@ -319,11 +319,11 @@ func mergeResourceLimitsAndScopeStatToResourceLimitsAndUsage(rl rcmgr.ResourceLi
 		ConnsInbound:         rl.ConnsInbound,
 		ConnsInboundUsage:    ss.NumConnsInbound,
 		Streams:              rl.Streams,
-		StreamsUsage:         ss.NumStreamsOutbound + ss.NumConnsInbound,
+		StreamsUsage:         ss.NumStreamsOutbound + ss.NumStreamsInbound,
 		StreamsOutbound:      rl.StreamsOutbound,
-		StreamsOutboundUsage: ss.NumConnsOutbound,
+		StreamsOutboundUsage: ss.NumStreamsOutbound,
 		StreamsInbound:       rl.StreamsInbound,
-		StreamsInboundUsage:  ss.NumConnsInbound,
+		StreamsInboundUsage:  ss.NumStreamsInbound,
 	}
 }
 

docs/EARLY_TESTERS.md

@@ -25,11 +25,10 @@ We will ask early testers to participate at two points in the process:
 - [ ] Charity Engine (@rytiss, @tristanolive)
 - [ ] Fission (@bmann)
 - [ ] Infura (@MichaelMure)
-- [ ] OrbitDB (@aphelionz)
+- [ ] OrbitDB (@haydenyoung)
 - [ ] pacman.store (@RubenKelevra)
 - [ ] Pinata (@obo20)
 - [ ] PL EngRes bifrost (@gmasgras)
-- [ ] RTrade (@postables)
 - [ ] Siderus (@koalalorenzo)
 - [ ] Textile (@sanderpick)
 

docs/RELEASE_ISSUE_TEMPLATE.md

@@ -174,12 +174,6 @@ This section covers tasks to be done during each release.
     - use `vX.Y.Z(-RCN)` as the Kubo image version
   - [ ] wait for the [e2e](https://github.com/ipfs/ipfs-companion/actions/workflows/e2e.yml) workflow run to finish
   </details>
-- [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) Update Kubo in [interop](https://github.com/ipfs/interop) <details><summary>using `./kuboreleaser release --version vX.Y.Z(-rcN) update-interop` or ...</summary>
-  - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) check out [ipfs/interop](https://github.com/ipfs/interop)
-  - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) run `npm install`
-  - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) create a PR which updates `package.json` and `package-lock.json`
-  - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) merge the PR
-  </details>
 - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) Update Kubo in [ipfs-desktop](https://github.com/ipfs/ipfs-desktop) <details><summary>using `./kuboreleaser release --version vX.Y.Z(-rcN) update-ipfs-desktop` or ...</summary>
   - [ ] check out [ipfs/ipfs-desktop](https://github.com/ipfs/ipfs-desktop)
   - [ ] run `npm install`