Upgrade test failing draft

tests/end-to-end/upgrade/README.md

@@ -35,6 +35,8 @@ Use the following environment variables to configure the application:
 | **APP_KUBECONFIG_PATH** | NO | None | A path to the `kubeconfig` file needed to run an application outside of the cluster. |
 | **APP_MAX_CONCURRENCY_LEVEL** | NO | `1` | A maximum concurrency level used for running tests. |
 | **APP_TESTING_ADDONS_URL** | YES | None | An external link to testing addons. |
+| **APP_WORKING_NAMESPACE** | No | `e2e-upgrade-test` | A namespace where the test is run. |
+| **APP_TESTS_INFO_CONFIG_MAP_NAME** | No | `upgrade-tests-info` | A name of a config map, where the upgrade test framework stores information about passed tests. |
 
 ### Use flags
 
@@ -97,13 +99,14 @@ Run the application without building a binary file. To do so:
 1. Prepare the upgrade data:
 
    ```bash
-   env APP_KUBECONFIG_PATH=/Users/$USER/.kube/config APP_LOGGER_LEVEL=debug APP_TESTING_ADDONS_URL="https://github.com/kyma-project/addons/releases/download/0.8.0/index-testing.yaml" go run main.go --action prepareData
+   kubectl create configmap tests-info -n kyma-system
+   env APP_WORKING_NAMESPACE=kyma-system APP_TESTS_INFO_CONFIG_MAP_NAME=tests-info APP_KUBECONFIG_PATH=/Users/$USER/.kube/config APP_LOGGER_LEVEL=debug APP_TESTING_ADDONS_URL="https://github.com/kyma-project/addons/releases/download/0.8.0/index-testing.yaml" go run main.go --action prepareData
    ```
 
 2. Run tests:
 
    ```bash
-   env APP_KUBECONFIG_PATH=/Users/$USER/.kube/config APP_LOGGER_LEVEL=debug APP_TESTING_ADDONS_URL="https://github.com/kyma-project/addons/releases/download/0.8.0/index-testing.yaml" go run main.go --action executeTests
+   env APP_WORKING_NAMESPACE=kyma-system APP_TESTS_INFO_CONFIG_MAP_NAME=tests-info APP_KUBECONFIG_PATH=/Users/$USER/.kube/config APP_LOGGER_LEVEL=debug APP_TESTING_ADDONS_URL="https://github.com/kyma-project/addons/releases/download/0.8.0/index-testing.yaml" go run main.go --action executeTests
    ```
 
 For the description of the available environment variables, see [this](#use-environment-variables) section.

tests/end-to-end/upgrade/chart/upgrade/templates/pod.yaml

@@ -32,6 +32,10 @@ spec:
         sleep 5
         exit $exit_code
     env:
+    - name: APP_WORKING_NAMESPACE
+      value: kyma-system
+    - name: APP_TESTS_INFO_CONFIG_MAP_NAME
+      value: test-{{ template "fullname" . }}-info
     - name: APP_MAX_CONCURRENCY_LEVEL
       value: "1"
     - name: APP_LOGGER_LEVEL

tests/end-to-end/upgrade/chart/upgrade/templates/rbac.yaml

@@ -24,7 +24,7 @@ rules:
   verbs: ["create", "delete", "get", "list", "patch", "update"]
 - apiGroups: [""]
   resources: ["configmaps"]
-  verbs: ["list", "get", "create", "delete"]
+  verbs: ["list", "get", "create", "delete", "update"]
 - apiGroups: ["apps"]
   resources: ["deployments"]
   verbs: ["create", "get", "delete", "list"]

tests/end-to-end/upgrade/chart/upgrade/templates/test/test-upgrade.yaml

@@ -31,6 +31,10 @@ spec:
             sleep 5
             exit $exit_code
         env:
+        - name: APP_WORKING_NAMESPACE
+          value: kyma-system
+        - name: APP_TESTS_INFO_CONFIG_MAP_NAME
+          value: test-{{ template "fullname" . }}-info
         - name: APP_MAX_CONCURRENCY_LEVEL
           value: "1"
         - name: APP_LOGGER_LEVEL

tests/end-to-end/upgrade/chart/upgrade/templates/tests-info.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: test-{{ template "fullname" . }}-info
+  namespace: kyma-system
+data: {}

tests/end-to-end/upgrade/chart/upgrade/values.yaml

@@ -3,7 +3,7 @@ containerRegistry:
 
 image:
   dir:
-  version: 138deae2
+  version: PR-8070
   pullPolicy: "IfNotPresent"
 
 dex:

tests/end-to-end/upgrade/internal/runner/cm_test_registry.go

@@ -0,0 +1,60 @@
+package runner
+
+import (
+	"github.com/pkg/errors"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/util/retry"
+)
+
+// ConfigMapTestRegistry allows to store information about passed tests in a config map
+type ConfigMapTestRegistry struct {
+	passed map[string]struct{}
+	k8s    kubernetes.Interface
+
+	cmName      string
+	cmNamespace string
+}
+
+func NewConfigMapTestRegistry(k8s kubernetes.Interface, namespace, name string) (*ConfigMapTestRegistry, error) {
+	cm, err := k8s.CoreV1().ConfigMaps(namespace).Get(name, v1.GetOptions{})
+	if err != nil {
+		return nil, errors.Wrapf(err, "while reading ConfigMap %s/%s with list of passed tests", namespace, name)
+	}
+	passed := make(map[string]struct{})
+	for key := range cm.Data {
+		passed[key] = struct{}{}
+	}
+	return &ConfigMapTestRegistry{
+		passed:      passed,
+		k8s:         k8s,
+		cmName:      name,
+		cmNamespace: namespace,
+	}, nil
+}
+
+func (r *ConfigMapTestRegistry) IsTestPassed(name string) bool {
+	_, found := r.passed[name]
+	return found
+}
+
+func (r *ConfigMapTestRegistry) MarkTestPassed(name string) error {
+	// Try to update, in case of error - retry
+	// Every retry covers getting config map to avoid conflicts.
+	return wait.ExponentialBackoff(retry.DefaultRetry, func() (bool, error) {
+		cm, err := r.k8s.CoreV1().ConfigMaps(r.cmNamespace).Get(r.cmName, v1.GetOptions{})
+		if err != nil {
+			return false, nil
+		}
+		if cm.Data == nil {
+			cm.Data = map[string]string{}
+		}
+		cm.Data[name] = "passed"
+		_, err = r.k8s.CoreV1().ConfigMaps(r.cmNamespace).Update(cm)
+		if err != nil {
+			return false, nil
+		}
+		return true, nil
+	})
+}

tests/end-to-end/upgrade/internal/runner/runner.go

@@ -31,18 +31,24 @@ type NamespaceCreator interface {
 	Create(*v1.Namespace) (*v1.Namespace, error)
 }
 
+type TestRegistry interface {
+	IsTestPassed(name string) bool
+	MarkTestPassed(name string) error
+}
+
 // TestRunner executes registered tests
 type TestRunner struct {
 	log                 *logrus.Entry
 	nsCreator           NamespaceCreator
+	testRegistry        TestRegistry
 	tests               map[string]UpgradeTest
 	maxConcurrencyLevel int
 	sanitizeRegex       *regexp.Regexp
 	verbose             bool
 }
 
 // NewTestRunner is a constructor for TestRunner
-func NewTestRunner(log *logrus.Entry, nsCreator NamespaceCreator, tests map[string]UpgradeTest, maxConcurrencyLevel int, verbose bool) (*TestRunner, error) {
+func NewTestRunner(log *logrus.Entry, nsCreator NamespaceCreator, testRegistry TestRegistry, tests map[string]UpgradeTest, maxConcurrencyLevel int, verbose bool) (*TestRunner, error) {
 	sanitizeRegex, err := regexp.Compile(regexSanitize)
 	if err != nil {
 		return nil, errors.Wrap(err, "while compiling sanitize regexp")
@@ -51,6 +57,7 @@ func NewTestRunner(log *logrus.Entry, nsCreator NamespaceCreator, tests map[stri
 	return &TestRunner{
 		log:                 log.WithField("service", "test:runner"),
 		nsCreator:           nsCreator,
+		testRegistry:        testRegistry,
 		tests:               tests,
 		maxConcurrencyLevel: maxConcurrencyLevel,
 		sanitizeRegex:       sanitizeRegex,
@@ -109,14 +116,24 @@ func (r *TestRunner) ExecuteTests(stopCh <-chan struct{}) error {
 				failed := r.executeTaskFunc(test.TestResources, stopCh, "TestResources", test.Name(), false)
 				if failed {
 					failedTaskCnt++
+				} else {
+					err := r.testRegistry.MarkTestPassed(test.Name())
+					if err != nil {
+						r.log.Errorf("Unable to mark test passed: %s", err.Error())
+					}
 				}
 			}
 			wg.Done()
 		}()
 	}
 
 	// populate all tests
+	numOfTests := 0
 	for name, test := range r.tests {
+		if r.testRegistry.IsTestPassed(name) {
+			continue
+		}
+		numOfTests++
 		queue <- task{name, test}
 	}
 	close(queue)
@@ -125,7 +142,7 @@ func (r *TestRunner) ExecuteTests(stopCh <-chan struct{}) error {
 	r.wgWait(stopCh, &wg)
 
 	if failedTaskCnt > 0 {
-		return fmt.Errorf("executed %d task and %d of them failed", len(r.tests), failedTaskCnt)
+		return fmt.Errorf("executed %d task and %d of them failed", numOfTests, failedTaskCnt)
 	}
 
 	return nil

tests/end-to-end/upgrade/main.go

@@ -47,13 +47,15 @@ import (
 
 // Config holds application configuration
 type Config struct {
-	Logger              logger.Config
-	DexUserSecret       string `envconfig:"default=admin-user"`
-	DexNamespace        string `envconfig:"default=kyma-system"`
-	KubeNamespace       string `envconfig:"default=kube-system"`
-	MaxConcurrencyLevel int    `envconfig:"default=1"`
-	KubeconfigPath      string `envconfig:"optional"`
-	TestingAddonsURL    string
+	Logger                 logger.Config
+	DexUserSecret          string `envconfig:"default=admin-user"`
+	DexNamespace           string `envconfig:"default=kyma-system"`
+	KubeNamespace          string `envconfig:"default=kube-system"`
+	MaxConcurrencyLevel    int    `envconfig:"default=1"`
+	KubeconfigPath         string `envconfig:"optional"`
+	TestingAddonsURL       string
+	WorkingNamespace       string `envconfig:"default=e2e-upgrade-test"`
+	TestsInfoConfigMapName string `envconfig:"default=upgrade-tests-info"`
 }
 
 const (
@@ -153,9 +155,11 @@ func main() {
 		"EventMeshUpgradeTest":            eventmesh.NewEventMeshUpgradeTest(appConnectorCli, k8sCli, messagingCli, servingCli, appBrokerCli, scCli, eventingCli),
 		//"LoggingUpgradeTest":              logging.NewLoggingTest(k8sCli, domainName, dexConfig.IdProviderConfig()),
 	}
+	tRegistry, err := runner.NewConfigMapTestRegistry(k8sCli, cfg.WorkingNamespace, cfg.TestsInfoConfigMapName)
+	fatalOnError(err, "while creating Test Registry")
 
 	// Execute requested action
-	testRunner, err := runner.NewTestRunner(log, k8sCli.CoreV1().Namespaces(), tests, cfg.MaxConcurrencyLevel, verbose)
+	testRunner, err := runner.NewTestRunner(log, k8sCli.CoreV1().Namespaces(), tRegistry, tests, cfg.MaxConcurrencyLevel, verbose)
 	fatalOnError(err, "while creating test runner")
 
 	switch action {

tests/end-to-end/upgrade/pkg/tests/api-gateway/apigateway.go

@@ -141,7 +141,7 @@ func (t ApiGatewayTest) TestResourcesError(namespace string) error {
 		return errors.Wrap(err, "cannot call deployment with oauth2 access token")
 	}
 
-	return nil
+	return errors.New("failing test")
 }
 
 func (t ApiGatewayTest) callTestServiceWithoutToken() error {

tests/end-to-end/upgrade/pkg/tests/function/function.go

@@ -85,6 +85,8 @@ func (f *LambdaFunctionUpgradeTest) CreateResources(stop <-chan struct{}, log lo
 // TestResources tests resources after the upgrade test
 func (f *LambdaFunctionUpgradeTest) TestResources(stop <-chan struct{}, log logrus.FieldLogger, namespace string) error {
 	log.Println("FunctionUpgradeTest testing resources")
+	log.Infof("FAiling test for testing upgrade test framework")
+
 	f.stop = stop
 	err := f.getFunctionPodStatus(10 * time.Minute)
 	if err != nil {