Add QE tests

tests/interop/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "0.1.0"
+__loggername__ = "css_logger"

tests/interop/conftest.py

@@ -0,0 +1,2 @@
+from validatedpatterns_tests.interop.conftest_logger import *  # noqa: F401, F403
+from validatedpatterns_tests.interop.conftest_openshift import *  # noqa: F401, F403

tests/interop/test_subscription_status_devel.py

@@ -0,0 +1,55 @@
+import logging
+
+import pytest
+from validatedpatterns_tests.interop import subscription
+
+from . import __loggername__
+
+logger = logging.getLogger(__loggername__)
+
+
+@pytest.mark.subscription_status_devel
+def test_subscription_status_devel(openshift_dyn_client):
+    # These are the operator subscriptions and their associated namespaces
+    expected_subs = {
+        "openshift-gitops-operator": ["openshift-operators"],
+        "openshift-pipelines-operator-rh": ["openshift-operators"],
+        "quay-bridge-operator": ["openshift-operators"],
+        "rhacs-operator": ["openshift-operators"],
+    }
+
+    (
+        operator_versions,
+        missing_subs,
+        unhealthy_subs,
+        missing_installplans,
+        upgrades_pending,
+    ) = subscription.subscription_status(openshift_dyn_client, expected_subs)
+
+    if missing_subs:
+        logger.error(f"FAIL: The following subscriptions are missing: {missing_subs}")
+    if unhealthy_subs:
+        logger.error(
+            f"FAIL: The following subscriptions are unhealthy: {unhealthy_subs}"
+        )
+    if missing_installplans:
+        logger.error(
+            f"FAIL: The install plan for the following subscriptions is missing: {missing_installplans}"
+        )
+    if upgrades_pending:
+        logger.error(
+            f"FAIL: The following subscriptions are in UpgradePending state: {upgrades_pending}"
+        )
+
+    cluster_version = subscription.openshift_version(openshift_dyn_client)
+    logger.info(f"Openshift version:\n{cluster_version.instance.status.history}")
+
+    for line in operator_versions:
+        logger.info(line)
+
+    if missing_subs or unhealthy_subs or missing_installplans or upgrades_pending:
+        err_msg = "Subscription status check failed"
+        logger.error(f"FAIL: {err_msg}")
+        assert False, err_msg
+    else:
+        logger.info("PASS: Subscription status check passed")

tests/interop/test_subscription_status_hub.py

@@ -0,0 +1,123 @@
+import difflib
+import logging
+import os
+import re
+import subprocess
+
+import pytest
+from validatedpatterns_tests.interop import subscription
+
+from . import __loggername__
+
+logger = logging.getLogger(__loggername__)
+
+
+@pytest.mark.subscription_status_hub
+def test_subscription_status_hub(openshift_dyn_client):
+    # These are the operator subscriptions and their associated namespaces
+    expected_subs = {
+        "openshift-gitops-operator": ["openshift-operators"],
+        "advanced-cluster-management": ["open-cluster-management"],
+        "odf-operator": ["openshift-storage"],
+        "multicluster-engine": ["multicluster-engine"],
+        "quay-operator": ["openshift-operators"],
+        "rhacs-operator": ["openshift-operators"],
+    }
+
+    (
+        operator_versions,
+        missing_subs,
+        unhealthy_subs,
+        missing_installplans,
+        upgrades_pending,
+    ) = subscription.subscription_status(openshift_dyn_client, expected_subs)
+
+    if missing_subs:
+        logger.error(f"FAIL: The following subscriptions are missing: {missing_subs}")
+    if unhealthy_subs:
+        logger.error(
+            f"FAIL: The following subscriptions are unhealthy: {unhealthy_subs}"
+        )
+    if missing_installplans:
+        logger.error(
+            f"FAIL: The install plan for the following subscriptions is missing: {missing_installplans}"
+        )
+    if upgrades_pending:
+        logger.error(
+            f"FAIL: The following subscriptions are in UpgradePending state: {upgrades_pending}"
+        )
+
+    cluster_version = subscription.openshift_version(openshift_dyn_client)
+    logger.info(f"Openshift version:\n{cluster_version.instance.status.history}")
+
+    if os.getenv("EXTERNAL_TEST") != "true":
+        shortversion = re.sub("(.[0-9]+$)", "", os.getenv("OPENSHIFT_VER"))
+        currentfile = os.getcwd() + "/operators_hub_current"
+        sourceFile = open(currentfile, "w")
+        for line in operator_versions:
+            logger.info(line)
+            print(line, file=sourceFile)
+        sourceFile.close()
+
+        logger.info("Clone operator-versions repo")
+        try:
+            operator_versions_repo = (
+                "git@gitlab.cee.redhat.com:mpqe/mps/vp/operator-versions.git"
+            )
+            clone = subprocess.run(
+                ["git", "clone", operator_versions_repo], capture_output=True, text=True
+            )
+            logger.info(clone.stdout)
+            logger.info(clone.stderr)
+        except Exception:
+            pass
+
+        previouspath = os.getcwd() + f"/operator-versions/devsecops_hub_{shortversion}"
+        previousfile = f"devsecops_hub_{shortversion}"
+
+        logger.info("Ensure previous file exists")
+        checkpath = os.path.exists(previouspath)
+        logger.info(checkpath)
+
+        if checkpath is True:
+            logger.info("Diff current operator list with previous file")
+            diff = opdiff(open(previouspath).readlines(), open(currentfile).readlines())
+            diffstring = "".join(diff)
+            logger.info(diffstring)
+
+            logger.info("Write diff to file")
+            sourceFile = open("operator_diffs_hub.log", "w")
+            print(diffstring, file=sourceFile)
+            sourceFile.close()
+        else:
+            logger.info("Skipping operator diff - previous file not found")
+
+    if missing_subs or unhealthy_subs or missing_installplans or upgrades_pending:
+        err_msg = "Subscription status check failed"
+        logger.error(f"FAIL: {err_msg}")
+        assert False, err_msg
+    else:
+        # Only push the new operarator list if the test passed
+        # and we are not testing a pre-release operator nor
+        # running externally
+        if os.getenv("EXTERNAL_TEST") != "true":
+            if checkpath is True and not os.environ["INDEX_IMAGE"]:
+                os.remove(previouspath)
+                os.rename(currentfile, previouspath)
+
+                cwd = os.getcwd() + "/operator-versions"
+                logger.info(f"CWD: {cwd}")
+
+                logger.info("Push new operator list")
+                subprocess.run(["git", "add", previousfile], cwd=cwd)
+                subprocess.run(
+                    ["git", "commit", "-m", "Update operator versions list"],
+                    cwd=cwd,
+                )
+                subprocess.run(["git", "push"], cwd=cwd)
+
+        logger.info("PASS: Subscription status check passed")
+
+
+def opdiff(*args):
+    return filter(lambda x: not x.startswith(" "), difflib.ndiff(*args))

tests/interop/test_subscription_status_prod.py

@@ -0,0 +1,54 @@
+import logging
+
+import pytest
+from validatedpatterns_tests.interop import subscription
+
+from . import __loggername__
+
+logger = logging.getLogger(__loggername__)
+
+
+@pytest.mark.subscription_status_prod
+def test_subscription_status_prod(openshift_dyn_client):
+    # These are the operator subscriptions and their associated namespaces
+    expected_subs = {
+        "openshift-gitops-operator": ["openshift-operators"],
+        "quay-bridge-operator": ["openshift-operators"],
+        "rhacs-operator": ["openshift-operators"],
+    }
+
+    (
+        operator_versions,
+        missing_subs,
+        unhealthy_subs,
+        missing_installplans,
+        upgrades_pending,
+    ) = subscription.subscription_status(openshift_dyn_client, expected_subs)
+
+    if missing_subs:
+        logger.error(f"FAIL: The following subscriptions are missing: {missing_subs}")
+    if unhealthy_subs:
+        logger.error(
+            f"FAIL: The following subscriptions are unhealthy: {unhealthy_subs}"
+        )
+    if missing_installplans:
+        logger.error(
+            f"FAIL: The install plan for the following subscriptions is missing: {missing_installplans}"
+        )
+    if upgrades_pending:
+        logger.error(
+            f"FAIL: The following subscriptions are in UpgradePending state: {upgrades_pending}"
+        )
+
+    cluster_version = subscription.openshift_version(openshift_dyn_client)
+    logger.info(f"Openshift version:\n{cluster_version.instance.status.history}")
+
+    for line in operator_versions:
+        logger.info(line)
+
+    if missing_subs or unhealthy_subs or missing_installplans or upgrades_pending:
+        err_msg = "Subscription status check failed"
+        logger.error(f"FAIL: {err_msg}")
+        assert False, err_msg
+    else:
+        logger.info("PASS: Subscription status check passed")

tests/interop/test_validate_devel_site_components.py

@@ -0,0 +1,138 @@
+import logging
+import os
+
+import pytest
+from validatedpatterns_tests.interop import application, components
+
+from . import __loggername__
+
+logger = logging.getLogger(__loggername__)
+
+oc = os.environ["HOME"] + "/oc_client/oc"
+
+"""
+Validate following multicluster-devsecops components pods and
+endpoints on edge site (line server):
+
+1) argocd
+2) ACM agents
+3) applications health (Applications deployed through argocd)
+"""
+
+
+@pytest.mark.test_validate_devel_site_components
+def test_validate_devel_site_components():
+    logger.info("Checking Openshift version on devel site")
+    version_out = components.dump_openshift_version()
+    logger.info(f"Openshift version:\n{version_out}")
+
+
+@pytest.mark.validate_devel_site_reachable
+def test_validate_devel_site_reachable(kube_config, openshift_dyn_client):
+    logger.info("Check if devel site API end point is reachable")
+    namespace = "openshift-gitops"
+    sub_string = "argocd-dex-server-token"
+    try:
+        devel_api_url = application.get_site_api_url(kube_config)
+        devel_api_response = application.get_site_api_response(
+            openshift_dyn_client, devel_api_url, namespace, sub_string
+        )
+    except AssertionError as e:
+        logger.error(f"FAIL: {e}")
+        assert False, e
+
+    if devel_api_response.status_code != 200:
+        err_msg = "Devel site is not reachable. Please check the deployment."
+        logger.error(f"FAIL: {err_msg}")
+        assert False, err_msg
+    else:
+        logger.info("PASS: Devel site is reachable")
+
+
+@pytest.mark.check_pod_status_devel
+def test_check_pod_status(openshift_dyn_client):
+    logger.info("Checking pod status")
+
+    err_msg = []
+    projects = [
+        "openshift-operators",
+        "openshift-gitops",
+        "multicluster-devsecops-development",
+        "open-cluster-management-agent",
+        "open-cluster-management-agent-addon",
+    ]
+
+    missing_projects = components.check_project_absense(openshift_dyn_client, projects)
+    missing_pods = []
+    failed_pods = []
+
+    for project in projects:
+        logger.info(f"Checking pods in namespace '{project}'")
+        missing_pods += components.check_pod_absence(openshift_dyn_client, project)
+        failed_pods += components.check_pod_status(openshift_dyn_client, projects)
+
+    if missing_projects:
+        err_msg.append(f"The following namespaces are missing: {missing_projects}")
+
+    if missing_pods:
+        err_msg.append(
+            f"The following namespaces have no pods deployed: {missing_pods}"
+        )
+
+    if failed_pods:
+        err_msg.append(f"The following pods are failed: {failed_pods}")
+
+    if err_msg:
+        logger.error(f"FAIL: {err_msg}")
+        assert False, err_msg
+    else:
+        logger.info("PASS: Pod status check succeeded.")
+
+
+@pytest.mark.validate_argocd_reachable_devel_site
+def test_validate_argocd_reachable_devel_site(openshift_dyn_client):
+    namespace = "openshift-gitops"
+    name = "openshift-gitops-server"
+    sub_string = "argocd-dex-server-token"
+    logger.info("Check if argocd route/url on devel site is reachable")
+    try:
+        argocd_route_url = application.get_argocd_route_url(
+            openshift_dyn_client, namespace, name
+        )
+        argocd_route_response = application.get_site_api_response(
+            openshift_dyn_client, argocd_route_url, namespace, sub_string
+        )
+    except StopIteration:
+        err_msg = "Argocd url/route is missing in open-cluster-management namespace"
+        logger.error(f"FAIL: {err_msg}")
+        assert False, err_msg
+    except AssertionError:
+        err_msg = "Bearer token is missing for argocd-dex-server"
+        logger.error(f"FAIL: {err_msg}")
+        assert False, err_msg
+
+    logger.info(f"Argocd route response : {argocd_route_response}")
+
+    if argocd_route_response.status_code != 200:
+        err_msg = "Argocd is not reachable. Please check the deployment"
+        logger.error(f"FAIL: {err_msg}")
+        assert False, err_msg
+    else:
+        logger.info("PASS: Argocd is reachable")
+
+
+@pytest.mark.validate_argocd_applications_health_devel_site
+def test_validate_argocd_applications_health_devel_site(openshift_dyn_client):
+    unhealthy_apps = []
+    logger.info("Get all applications deployed by argocd on devel site")
+    projects = ["openshift-gitops"]
+    for project in projects:
+        unhealthy_apps += application.get_argocd_application_status(
+            openshift_dyn_client, project
+        )
+    if unhealthy_apps:
+        err_msg = "Some or all applications deployed on devel site are unhealthy"
+        logger.error(f"FAIL: {err_msg}:\n{unhealthy_apps}")
+        assert False, err_msg
+    else:
+        logger.info("PASS: All applications deployed on devel site are healthy.")