test(e2e): add zot as a testing backend (#1072)

Signed-off-by: Billy Zha <jinzha1@microsoft.com>

Makefile

@@ -37,7 +37,7 @@ test: tidy vendor check-encoding  ## tidy and run tests
 
 .PHONY: teste2e
 teste2e:  ## run end to end tests
-	./test/e2e/scripts/e2e.sh $(shell git rev-parse --show-toplevel) --clean 
+	./test/e2e/scripts/e2e.sh $(shell git rev-parse --show-toplevel) --clean
 
 .PHONY: covhtml
 covhtml:  ## look at code coverage

test/e2e/README.md

@@ -5,8 +5,8 @@ Install [git](https://git-scm.com/download/linux), [docker](https://docs.docker.
 
 ## Run E2E Script
 ```shell
-REPO_ROOT=$(git rev-parse --show-toplevel) # REPO_ROOT is root folder of oras CLI code
-$REPO_ROOT/test/e2e/scripts/e2e.sh $REPO_ROOT --clean 
+# in root folder of oras CLI code
+make teste2e
 ```
 
 If the tests fails with errors like `ginkgo: not found`, use below command to add GOPATH into the PATH variable
@@ -15,36 +15,46 @@ PATH+=:$(go env GOPATH)/bin
 ```
 
 ## Development
-### 1. Using IDE
+### 1. Prepare E2E Test Environment
+You may use the prepare script to setup an E2E test environment before developing E2E tests:
+```shell
+REPO_ROOT=$(git rev-parse --show-toplevel) # REPO_ROOT is root folder of oras CLI code
+$REPO_ROOT/test/e2e/scripts/prepare.sh $REPO_ROOT
+```
+
+### 2. Using IDE
 Since E2E test suites are added as an nested module, the module file and checksum file are separated from oras CLI. To develop E2E tests, it's better to set the working directory to `$REPO_ROOT/test/e2e/` or open your IDE at it.
 
-### 2. Testing pre-built ORAS Binary
+### 3. Testing pre-built ORAS Binary
 By default, Gomega builds a temp binary every time before running e2e tests, which makes sure that latest code changes in the working directory are covered. If you are making changes to E2E test code only, set `ORAS_PATH` towards your pre-built ORAS binary to skip building and speed up the test.
 
-### 3. Debugging via `go test`
+### 4. Debugging via `go test`
 E2E specs can be ran natively without `ginkgo`:
 ```shell
 # run below command in the target test suite folder
 go test oras.land/oras/test/e2e/suite/${suite_name}
 ```
 This is super handy when you want to do step-by-step debugging from command-line or via an IDE. If you need to debug certain specs, use [focused specs](https://onsi.github.io/ginkgo/#focused-specs) but don't check it in.
 
-### 4. Testing Registry Services
-The backend of E2E tests are two registry services: [oras-distribution](https://github.com/oras-project/distribution) and [upstream distribution](https://github.com/distribution/distribution). The former is expected to support image and artifact media types and referrer API; The latter is expected to only support image media type with subject and provide referrers via [tag schema](https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#referrers-tag-schema). 
+### 5. Testing Registry Services
+The backend of E2E tests are three registry services: 
+- [oras-distribution](https://github.com/oras-project/distribution): registry service supports artifact and image types in [image-spec 1.1.0 rc2](https://github.com/opencontainers/image-spec/tree/v1.1.0-rc2) and referrer API
+- [upstream distribution](https://github.com/distribution/distribution): registry service supports image media type with subject and provide referrers via [tag schema](https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#referrers-tag-schema). 
+- [zot](https://github.com/project-zot/zot): registry service supports artifact and image types in [image-spec 1.1.0 rc4](https://github.com/opencontainers/image-spec/tree/v1.1.0-rc4) and referrer API
 
-You can run scenario test suite against your own registry services via setting `ORAS_REGISTRY_HOST` or `ORAS_REGISTRY_FALLBACK_HOST` environmental variables.
+You can run scenario test suite against your own registry services via setting `ORAS_REGISTRY_HOST`, `ORAS_REGISTRY_FALLBACK_HOST` and `ZOT_REGISTRY_HOST` environmental variables.
 
-### 5. Constant Build & Watch
+### 6. Constant Build & Watch
 This is a good choice if you want to debug certain re-runnable specs:
 ```shell
 cd $REPO_ROOT/test/e2e
 ginkgo watch -r
 ```
 
-### 6. Trouble-shooting CLI
+### 7. Trouble-shooting CLI
 The executed commands should be shown in the ginkgo logs after `[It]`, with full execution output in the E2E log.
 
-### 7. Adding New Tests
+### 8. Adding New Tests
 Three suites will be maintained for E2E testing:
 - command: contains test specs for single oras command execution
 - auth: contains test specs similar to command specs but specific to auth. It cannot be ran in parallel with command suite specs
@@ -59,9 +69,9 @@ Describe: <Role>
        Expect: <Result> (detailed checks for execution results)
 ```
 
-### 8. Adding New Test Data
+### 9. Adding New Test Data
 
-#### 8.1 Command Suite
+#### 9.1 Command Suite
 Command suite uses pre-baked test data, which is a bunch of layered archive files compressed from registry storage. Test data are all stored in `$REPO_ROOT/test/e2e/testdata/distribution/` but separated in different sub-folders: oras distribution uses `mount` and upstream distribution uses `mount_fallback`.
 
 For both registries, the repository name should follow the convention of `command/$repo_suffix`. To add a new layer to the test data, use the below command to compress the `docker` folder from the root directory of the registry storage and copy it to the corresponding subfolder in `$REPO_ROOT/test/e2e/testdata/distribution/mount`.
@@ -135,5 +145,5 @@ graph TD;
         end
     end
 ```
-#### 8.2 Scenario Suite
+#### 9.2 Scenario Suite
 Test files used by scenario-based specs are placed in `$REPO_ROOT/test/e2e/testdata/files`.

test/e2e/internal/utils/init.go

@@ -36,6 +36,9 @@ var Host string
 // FallbackHost points to the registry service where fallback E2E specs will be run against.
 var FallbackHost string
 
+// ZOTHost points to the zot service where E2E specs will be run against.
+var ZOTHost string
+
 func init() {
 	Host = os.Getenv(RegHostKey)
 	if Host == "" {
@@ -59,6 +62,16 @@ func init() {
 		panic(err)
 	}
 
+	ZOTHost = os.Getenv(ZOTHostKey)
+	if ZOTHost == "" {
+		ZOTHost = "localhost:7000"
+		fmt.Fprintf(os.Stderr, "cannot find zot host name in %s, using %s instead\n", ZOTHostKey, ZOTHost)
+	}
+	ref.Registry = ZOTHost
+	if err := ref.ValidateRegistry(); err != nil {
+		panic(err)
+	}
+
 	// setup test data
 	pwd, err := os.Getwd()
 	if err != nil {
@@ -111,5 +124,7 @@ func init() {
 		gomega.Expect(cmd.Run()).ShouldNot(gomega.HaveOccurred())
 		cmd = exec.Command(ORASPath, "login", FallbackHost, "-u", Username, "-p", Password)
 		gomega.Expect(cmd.Run()).ShouldNot(gomega.HaveOccurred())
+		cmd = exec.Command(ORASPath, "login", ZOTHost, "-u", Username, "-p", Password)
+		gomega.Expect(cmd.Run()).ShouldNot(gomega.HaveOccurred())
 	})
 }

test/e2e/internal/utils/testdata.go

@@ -25,6 +25,7 @@ const (
 	// env
 	RegHostKey         = "ORAS_REGISTRY_HOST"
 	FallbackRegHostKey = "ORAS_REGISTRY_FALLBACK_HOST"
+	ZOTHostKey         = "ZOT_REGISTRY_HOST"
 )
 
 var (

test/e2e/scripts/e2e.sh

@@ -13,57 +13,42 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-export ORAS_REGISTRY_PORT="5000"
-export ORAS_REGISTRY_HOST="localhost:${ORAS_REGISTRY_PORT}"
-export ORAS_REGISTRY_FALLBACK_PORT="6000"
-export ORAS_REGISTRY_FALLBACK_HOST="localhost:${ORAS_REGISTRY_FALLBACK_PORT}"
+# help
+help () {
+    echo "Usage"
+    echo "  e2e.sh <repo_root> [--clean]"
+    exit 1
+}
 
+# 1. Prepare
 repo_root=$1
 if [ -z "${repo_root}" ]; then
     echo "repository root path is not provided."
-    echo "Usage"
-    echo "  e2e.sh <repo_root> [--clean]"
-    exit 1
+    help
 fi
-clean_up=$2
 
-echo " === installing ginkgo  === "
-repo_root=$(realpath --canonicalize-existing ${repo_root})
-cwd=$(pwd)
-cd ${repo_root}/test/e2e && go install github.com/onsi/ginkgo/v2/ginkgo@latest
-trap "cd $cwd" EXIT
+clean=$2
+if [ "${clean}" != '--clean' ] && [ -n "${clean}" ]; then
+    echo "invalid flag found: ${clean}"
+    help
+fi
 
-# start registries
-. ${repo_root}/test/e2e/scripts/common.sh
+. ${repo_root}/test/e2e/scripts/prepare.sh $1 $2
 
-if [ "$clean_up" = '--clean' ]; then
+if [ "${clean}" = '--clean' ]; then
     echo " === setting deferred clean up jobs  === "
-    trap "try_clean_up oras-e2e oras-e2e-fallback" EXIT
+    trap "try_clean_up $ORAS_CTR_NAME $UPSTREAM_CTR_NAME $ZOT_CTR_NAME" EXIT
 fi
 
-oras_container_name="oras-e2e"
-upstream_container_name="oras-e2e-fallback"
-e2e_root="${repo_root}/test/e2e"
-echo " === preparing oras distribution === "
-run_registry \
-  ${e2e_root}/testdata/distribution/mount \
-  ghcr.io/oras-project/registry:v1.0.0-rc.4 \
-  $oras_container_name \
-  $ORAS_REGISTRY_PORT
-
-echo " === preparing upstream distribution === "
-run_registry \
-  ${e2e_root}/testdata/distribution/mount_fallback \
-  registry:2.8.1 \
-  $upstream_container_name \
-  $ORAS_REGISTRY_FALLBACK_PORT
-
+# 2. Test
 echo " === run tests === "
 if ! ginkgo -r -p --succinct suite; then 
   echo " === retriving registry error logs === "
   echo '-------- oras distribution trace -------------'
   docker logs -t --tail 200 $oras_container_name
   echo '-------- upstream distribution trace -------------'
   docker logs -t --tail 200 $upstream_container_name
+  echo '-------- zot trace -------------'
+  docker logs -t --tail 200 $zot_container_name
   exit 1
 fi

test/e2e/scripts/prepare.sh

@@ -0,0 +1,66 @@
+#!/bin/sh -e
+
+# Copyright The ORAS Authors.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export ORAS_REGISTRY_PORT="5000"
+export ORAS_REGISTRY_HOST="localhost:${ORAS_REGISTRY_PORT}"
+export ORAS_REGISTRY_FALLBACK_PORT="6000"
+export ORAS_REGISTRY_FALLBACK_HOST="localhost:${ORAS_REGISTRY_FALLBACK_PORT}"
+export ZOT_REGISTRY_PORT="7000"
+export ZOT_REGISTRY_HOST="localhost:${ZOT_REGISTRY_PORT}"
+export ORAS_CTR_NAME="oras-e2e"
+export UPSTREAM_CTR_NAME="oras-e2e-fallback"
+export ZOT_CTR_NAME="oras-e2e-zot"
+
+repo_root=$1
+if [ -z "${repo_root}" ]; then
+    echo "repository root path is not provided."
+    echo "Usage"
+    echo "  prepare.sh <repo_root>"
+    exit 1
+fi
+
+echo " === installing ginkgo  === "
+repo_root=$(realpath --canonicalize-existing ${repo_root})
+cwd=$(pwd)
+cd ${repo_root}/test/e2e && go install github.com/onsi/ginkgo/v2/ginkgo@latest
+trap "cd $cwd" EXIT
+
+# start registries
+. ${repo_root}/test/e2e/scripts/common.sh
+echo " >>> preparing: oras distribution >>> "
+e2e_root="${repo_root}/test/e2e"
+run_registry \
+  ${e2e_root}/testdata/distribution/mount \
+  ghcr.io/oras-project/registry:v1.0.0-rc.4 \
+  $ORAS_CTR_NAME \
+  $ORAS_REGISTRY_PORT
+echo " <<< prepared : oras distribution <<< "
+
+echo " >>> preparing: upstream distribution >>> "
+run_registry \
+  ${e2e_root}/testdata/distribution/mount_fallback \
+  registry:2.8.1 \
+  $UPSTREAM_CTR_NAME \
+  $ORAS_REGISTRY_FALLBACK_PORT
+echo "  prepared : upstream distribution  "
+
+echo " >>> preparing: zot >>> "
+try_clean_up $ZOT_CTR_NAME
+docker run --pull always -dp $ZOT_REGISTRY_PORT:5000 \
+  --name $ZOT_CTR_NAME \
+  -u $(id -u $(whoami)) \
+  --mount type=bind,source="${e2e_root}/testdata/zot/",target=/etc/zot \
+  --rm ghcr.io/project-zot/zot-linux-amd64:v2.0.0-rc6
+echo " <<< prepared : zot <<< "

test/e2e/suite/auth/auth.go

@@ -27,38 +27,38 @@ import (
 var _ = Describe("Common registry user", Ordered, func() {
 	When("logging out", Ordered, func() {
 		It("should use logout command to logout", func() {
-			ORAS("logout", Host, "--registry-config", AuthConfigPath).Exec()
+			ORAS("logout", ZOTHost, "--registry-config", AuthConfigPath).Exec()
 		})
 
 		It("should run commands without logging in", func() {
-			RunWithoutLogin("attach", Host+"/repo:tag", "-a", "test=true", "--artifact-type", "doc/example")
-			ORAS("copy", Host+"/repo:from", Host+"/repo:to", "--from-registry-config", AuthConfigPath, "--to-registry-config", AuthConfigPath).
+			RunWithoutLogin("attach", ZOTHost+"/repo:tag", "-a", "test=true", "--artifact-type", "doc/example")
+			ORAS("copy", ZOTHost+"/repo:from", ZOTHost+"/repo:to", "--from-registry-config", AuthConfigPath, "--to-registry-config", AuthConfigPath).
 				ExpectFailure().
 				MatchErrKeyWords("Error:", "credential required").
 				WithDescription("fail without logging in").Exec()
-			RunWithoutLogin("discover", Host+"/repo:tag")
-			RunWithoutLogin("push", "-a", "key=value", Host+"/repo:tag")
-			RunWithoutLogin("pull", Host+"/repo:tag")
-			RunWithoutLogin("manifest", "fetch", Host+"/repo:tag")
-			RunWithoutLogin("blob", "delete", Host+"/repo@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
-			RunWithoutLogin("blob", "push", Host+"/repo", WriteTempFile("blob", "test"))
-			RunWithoutLogin("tag", Host+"/repo:tag", "tag1")
-			RunWithoutLogin("repo", "ls", Host)
-			RunWithoutLogin("repo", "tags", RegistryRef(Host, "repo", ""))
-			RunWithoutLogin("manifest", "fetch-config", Host+"/repo:tag")
+			RunWithoutLogin("discover", ZOTHost+"/repo:tag")
+			RunWithoutLogin("push", "-a", "key=value", ZOTHost+"/repo:tag")
+			RunWithoutLogin("pull", ZOTHost+"/repo:tag")
+			RunWithoutLogin("manifest", "fetch", ZOTHost+"/repo:tag")
+			RunWithoutLogin("blob", "delete", ZOTHost+"/repo@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
+			RunWithoutLogin("blob", "push", ZOTHost+"/repo", WriteTempFile("blob", "test"))
+			RunWithoutLogin("tag", ZOTHost+"/repo:tag", "tag1")
+			RunWithoutLogin("repo", "ls", ZOTHost)
+			RunWithoutLogin("repo", "tags", RegistryRef(ZOTHost, "repo", ""))
+			RunWithoutLogin("manifest", "fetch-config", ZOTHost+"/repo:tag")
 		})
 	})
 
 	When("logging in", func() {
 		It("should use basic auth", func() {
-			ORAS("login", Host, "-u", Username, "-p", Password, "--registry-config", AuthConfigPath).
+			ORAS("login", ZOTHost, "-u", Username, "-p", Password, "--registry-config", AuthConfigPath).
 				WithTimeOut(20*time.Second).
 				MatchContent("Login Succeeded\n").
 				MatchErrKeyWords("WARNING", "Using --password via the CLI is insecure", "Use --password-stdin").Exec()
 		})
 
 		It("should fail if no username input", func() {
-			ORAS("login", Host, "--registry-config", AuthConfigPath).
+			ORAS("login", ZOTHost, "--registry-config", AuthConfigPath).
 				WithTimeOut(20 * time.Second).
 				WithInput(strings.NewReader("")).
 				MatchKeyWords("username:").
@@ -67,37 +67,37 @@ var _ = Describe("Common registry user", Ordered, func() {
 		})
 
 		It("should fail if no password input", func() {
-			ORAS("login", Host, "--registry-config", AuthConfigPath).
+			ORAS("login", ZOTHost, "--registry-config", AuthConfigPath).
 				WithTimeOut(20*time.Second).
 				MatchKeyWords("Username: ", "Password: ").
 				WithInput(strings.NewReader(fmt.Sprintf("%s\n", Username))).ExpectFailure().Exec()
 		})
 
 		It("should fail if password is empty", func() {
-			ORAS("login", Host, "--registry-config", AuthConfigPath).
+			ORAS("login", ZOTHost, "--registry-config", AuthConfigPath).
 				WithTimeOut(20*time.Second).
 				MatchKeyWords("Username: ", "Password: ").
 				MatchErrKeyWords("Error: password required").
 				WithInput(strings.NewReader(fmt.Sprintf("%s\n\n", Username))).ExpectFailure().Exec()
 		})
 
 		It("should fail if no token input", func() {
-			ORAS("login", Host, "--registry-config", AuthConfigPath).
+			ORAS("login", ZOTHost, "--registry-config", AuthConfigPath).
 				WithTimeOut(20*time.Second).
 				MatchKeyWords("Username: ", "Token: ").
 				WithInput(strings.NewReader("\n")).ExpectFailure().Exec()
 		})
 
 		It("should fail if token is empty", func() {
-			ORAS("login", Host, "--registry-config", AuthConfigPath).
+			ORAS("login", ZOTHost, "--registry-config", AuthConfigPath).
 				WithTimeOut(20*time.Second).
 				MatchKeyWords("Username: ", "Token: ").
 				MatchErrKeyWords("Error: token required").
 				WithInput(strings.NewReader("\n\n")).ExpectFailure().Exec()
 		})
 
 		It("should use prompted input", func() {
-			ORAS("login", Host, "--registry-config", AuthConfigPath).
+			ORAS("login", ZOTHost, "--registry-config", AuthConfigPath).
 				WithTimeOut(20*time.Second).
 				WithInput(strings.NewReader(fmt.Sprintf("%s\n%s\n", Username, Password))).
 				MatchKeyWords("Username: ", "Password: ", "Login Succeeded\n").Exec()