E2E test framework for KIM

.gitignore

@@ -27,4 +27,10 @@ Dockerfile.cross
 *.swo
 *~
 
-vendor
+vendor
+
+# KIND log dumps
+logs_*
+
+# Debug files of IDEs
+__debug*

Makefile

@@ -74,6 +74,10 @@ build: manifests generate fmt vet ## Build manager binary.
 run: manifests generate fmt vet ## Run a controller from your host.
 	go run ./cmd/main.go
 
+.PHONY: docker-build-notest
+docker-build-notest: manifests generate fmt vet ## Build docker image with the manager but NOT executing tests (required for E2E test to avoid enless loops!!!)
+	$(CONTAINER_TOOL) build -t ${IMG} .
+
 # If you wish built the manager image targeting other platforms you can use the --platform flag.
 # (i.e. docker build --platform linux/arm64 ). However, you must enable docker buildKit for it.
 # More info: https://docs.docker.com/develop/develop-images/build_enhancements/

config/manager/kustomization.yaml

@@ -6,4 +6,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: controller
-  newTag: latest
+  newTag: e2e

go.mod

@@ -19,12 +19,14 @@ require (
 	k8s.io/client-go v0.30.3
 	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
 	sigs.k8s.io/controller-runtime v0.18.4
+	sigs.k8s.io/e2e-framework v0.4.0
 	sigs.k8s.io/yaml v1.4.0
 )
 
 require (
 	github.com/Masterminds/semver/v3 v3.2.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
@@ -47,22 +49,26 @@ require (
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect
 	github.com/google/uuid v1.6.0 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.6.0 // indirect
 	github.com/prometheus/common v0.48.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
+	github.com/vladimirvivien/gexe v0.2.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/crypto v0.25.0 // indirect
@@ -80,6 +86,7 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.30.1 // indirect
+	k8s.io/component-base v0.30.1 // indirect
 	k8s.io/klog/v2 v2.120.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect

go.sum

@@ -2,8 +2,12 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0
 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
 github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
 github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
+github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@@ -69,6 +73,9 @@ github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQN
 github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
+github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -93,13 +100,17 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
+github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/onsi/ginkgo/v2 v2.19.1 h1:QXgq3Z8Crl5EL1WBAC98A5sEBHARrAJNzAmMxzLcRF0=
 github.com/onsi/ginkgo/v2 v2.19.1/go.mod h1:O3DtEWQkPa/F7fBMgmZQKKsluAy8pd3rEQdrjkPb9zA=
 github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
@@ -131,6 +142,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/vladimirvivien/gexe v0.2.0 h1:nbdAQ6vbZ+ZNsolCgSVb9Fno60kzSuvtzVh6Ytqi/xY=
+github.com/vladimirvivien/gexe v0.2.0/go.mod h1:LHQL00w/7gDUKIak24n801ABp8C+ni6eBht9vGVst8w=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
@@ -187,6 +200,8 @@ k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc=
 k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc=
 k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k=
 k8s.io/client-go v0.30.3/go.mod h1:8d4pf8vYu665/kUbsxWAQ/JDBNWqfFeZnvFiVdmx89U=
+k8s.io/component-base v0.30.1 h1:bvAtlPh1UrdaZL20D9+sWxsJljMi0QZ3Lmw+kmZAaxQ=
+k8s.io/component-base v0.30.1/go.mod h1:e/X9kDiOebwlI41AvBHuWdqFriSRrX50CdwA9TFaHLI=
 k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw=
 k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
 k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag=
@@ -195,6 +210,8 @@ k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1
 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw=
 sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg=
+sigs.k8s.io/e2e-framework v0.4.0 h1:4yYmFDNNoTnazqmZJXQ6dlQF1vrnDbutmxlyvBpC5rY=
+sigs.k8s.io/e2e-framework v0.4.0/go.mod h1:JilFQPF1OL1728ABhMlf9huse7h+uBJDXl9YeTs49A8=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=

pkg/test/assets/runtime-example.yaml

@@ -0,0 +1,80 @@
+apiVersion: infrastructuremanager.kyma-project.io/v1
+kind: Runtime
+metadata:
+  labels:
+    kyma-project.io/broker-plan-id: 5cb3d976-b85c-42ea-a636-79cadda109a9
+    kyma-project.io/broker-plan-name: preview
+    kyma-project.io/controlled-by-provisioner: "false"
+    kyma-project.io/global-account-id: e449f875-b5b2-4485-b7c0-98725c0571bf
+    kyma-project.io/instance-id: jp-010
+    kyma-project.io/region: eu-west-2
+    kyma-project.io/runtime-id: 27e7d1a2-ae05-45ba-9164-mdmdmdmkim04
+    kyma-project.io/shoot-name: unittest-kim01
+    kyma-project.io/subaccount-id: 123456-865d-42fd-8d6f-3e3e2dcce3c8
+    operator.kyma-project.io/kyma-name: 27e7d1a2-ae05-45ba-9164-mdmdmdmkim04
+  name: runtime-example
+  namespace: default
+spec:
+  security:
+    administrators:
+    - admin@myorg.com
+    networking:
+      filter:
+        egress:
+          enabled: false
+        ingress:
+          enabled: true
+  shoot:
+    controlPlane:
+      highAvailability:
+        failureTolerance:
+          type: zone
+    enforceSeedLocation: true
+    kubernetes:
+      kubeAPIServer:
+        additionalOidcConfig:
+        - clientID: 9bd05ed7-a930-44e6-8c79-e6defeb7dec9
+          groupsClaim: groups
+          issuerURL: https://kymatest.accounts400.ondemand.com
+          signingAlgs:
+          - RS256
+          usernameClaim: sub
+          usernamePrefix: '-'
+        oidcConfig:
+          clientID: 9bd05ed7-a930-44e6-8c79-e6defeb7dec9
+          groupsClaim: groups
+          issuerURL: https://kymatest.accounts400.ondemand.com
+          signingAlgs:
+          - RS256
+          usernameClaim: sub
+          usernamePrefix: '-'
+      version: 1.28.7
+    name: unittest-kim01
+    networking:
+      nodes: 10.250.0.0/16
+      pods: 100.64.0.0/12
+      services: 100.104.0.0/13
+    platformRegion: cd-eu11
+    provider:
+      type: aws
+      workers:
+      - machine:
+          image:
+            name: gardenlinux
+            version: 1312.3.0
+          type: m6i.large
+        maxSurge: 3
+        maxUnavailable: 0
+        maximum: 20
+        minimum: 3
+        name: cpu-worker-0
+        volume:
+          size: 50Gi
+          type: gp2
+        zones:
+        - eu-central-1a
+        - eu-central-1b
+        - eu-central-1c
+    purpose: production
+    region: eu-central-1
+    secretBindingName: hyperscaler_secret

pkg/test/example_test.go

@@ -0,0 +1,60 @@
+package test
+
+import (
+	"context"
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	v1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/e2e-framework/klient"
+)
+
+var s *Suite
+
+func TestMain(m *testing.M) {
+	s = NewSuite(m, NewEnvConf(""), WithCRDsInstalled, WithKindCluster, WithDockerBuild, WithKIMDeployed, WithExportOfClusterLogs)
+	s.Run()
+}
+
+func TestKCPSystem(t *testing.T) {
+	f := s.NewFeature(t, "Get list of kcp-system pods and check for KIM")
+
+	f.Assert("KCP-system namespace exists", func(client klient.Client) {
+		var ns v1.Namespace
+		err := client.Resources(KCPNamespace).Get(context.TODO(), "kcp-system", "", &ns)
+		assert.NoError(t, err)
+		assert.Equal(t, ns.Name, "kcp-system")
+	})
+
+	f.Assert("KIM Pod exists", func(client klient.Client) {
+		var pods v1.PodList
+		err := client.Resources(KCPNamespace).List(context.TODO(), &pods)
+		assert.NoError(t, err)
+		assert.Len(t, pods.Items, 1)
+		assert.Contains(t, pods.Items[0].Name, "infrastructure-manager")
+	})
+	f.Run()
+}
+
+func TestRuntimeCR(t *testing.T) {
+	f := s.NewFeature(t, "Compare Runtime CR with Shoot")
+	f.WithRuntimeCRs(path.Join("assets", "runtime-example.yaml"))
+	f.Assert("Check for the correct Shoot", func(client klient.Client) {
+		var ns v1.Namespace
+		err := client.Resources(KCPNamespace).Get(context.TODO(), "kcp-system", "", &ns)
+		assert.NoError(t, err)
+	})
+	f.Run()
+}
+
+func TestKubeSytem(t *testing.T) {
+	f := s.NewFeature(t, "Get list of kube-system pods")
+	f.Assert("Kube-system pods exist", func(client klient.Client) {
+		var pods v1.PodList
+		err := client.Resources("kube-system").List(context.TODO(), &pods)
+		assert.NoError(t, err)
+		assert.True(t, len(pods.Items) > 5)
+	})
+	f.Run()
+}

pkg/test/feature.go

@@ -0,0 +1,65 @@
+package test
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	imv1 "github.com/kyma-project/infrastructure-manager/api/v1"
+	"github.com/stretchr/testify/require"
+	"k8s.io/apimachinery/pkg/util/json"
+	"k8s.io/apimachinery/pkg/util/yaml"
+
+	"sigs.k8s.io/e2e-framework/klient"
+	"sigs.k8s.io/e2e-framework/pkg/env"
+	"sigs.k8s.io/e2e-framework/pkg/features"
+
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+)
+
+type Feature struct {
+	feature          *features.FeatureBuilder
+	testEnv          env.Environment
+	clusterName      string
+	clusterNamespace string
+	t                *testing.T
+}
+
+func (f *Feature) WithRuntimeCRs(runtimeCRFiles ...string) *Feature {
+	f.feature.Assess("Installing Runtime CRs", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+		for _, runtimeCRFile := range runtimeCRFiles {
+			rawRuntimeCR, err := os.ReadFile(runtimeCRFile)
+			require.NoError(t, err)
+
+			runtime := &imv1.Runtime{}
+			yaml.Unmarshal(rawRuntimeCR, runtime)
+			switch filepath.Ext(runtimeCRFile) {
+			case ".json":
+				require.NoError(t, json.Unmarshal(rawRuntimeCR, runtime))
+			case ".yaml", ".yml":
+				require.NoError(t, yaml.Unmarshal(rawRuntimeCR, runtime))
+			default:
+				t.Logf("Cannot read Runtime CR file '%s' because only file extesnion .json or .yaml is supported", runtimeCRFile)
+				t.Fail()
+			}
+
+			err = cfg.Client().Resources(KCPNamespace).Create(context.TODO(), runtime)
+			require.NoError(t, err)
+		}
+		return ctx
+	})
+	return f
+}
+
+func (f *Feature) Assert(desc string, assertFct func(client klient.Client)) *Feature {
+	f.feature.Assess(desc, func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+		assertFct(cfg.Client())
+		return ctx
+	})
+	return f
+}
+
+func (f *Feature) Run() {
+	f.testEnv.Test(f.t, f.feature.Feature())
+}