test(uat): migrate tests to use gdk_cli abstraction (#76)

* test(uat): add abstraction for instrumented gdk

* test(uat): migrate tests to use gdk_cli abstraction

* ci: track coverage for each test suite

* refactor(test): rename --gdkimpl arg to --instrumented

* refactor(test): control output capturing with arg

.github/workflows/CI.yml

@@ -43,10 +43,22 @@ jobs:
           pip3 install flake8
           # The GitHub editor is 127 chars wide
           flake8 . --count --max-complexity=10 --max-line-length=127 --show-source --statistics
-      - name: Testing CLI (Runs both unit and integration tests)
+      - name: Unit Testing CLI
         run: |
           pip install -r test-requirements.txt
-          coverage run --source=gdk -m pytest -v -s tests integration_tests && coverage xml --fail-under=90
+          coverage run --source=gdk -m pytest -v -s tests && coverage xml --fail-under=99
+      - name: Upload tests coverage to Codecov
+        uses: codecov/codecov-action@v2
+        with:
+          flags: unit
+      - name: Integration Testing CLI
+        run: |
+          pip install -r test-requirements.txt
+          coverage run --source=gdk -m pytest -v -s integration_tests && coverage xml --fail-under=90
+      - name: Upload integration tests coverage to Codecov
+        uses: codecov/codecov-action@v2
+        with:
+          flags: integ
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
@@ -56,8 +68,13 @@ jobs:
         uses: actions/setup-java@v1
         with:
           java-version: 1.8
-      - name: Run UATs 
-        run: | 
+      - name: Run UATs
+        run: |
           coverage run --source=gdk -m pytest -v -s uat
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v2
+      - name: Run UATs with code coverage
+        run: |
+          coverage run --source=gdk -m pytest -v -s uat --instrumented && coverage xml --fail-under=77
+      - name: Upload UAT coverage to Codecov
+        uses: codecov/codecov-action@v2
+        with:
+          flags: uat

README.md

@@ -1,5 +1,6 @@
 # Greengrass Development Kit - Command Line Interface
 ![CI](https://github.com/aws-greengrass/aws-greengrass-gdk-cli/workflows/CI/badge.svg?branch=main)
+[![codecov](https://codecov.io/gh/aws-greengrass/aws-greengrass-gdk-cli/branch/main/graph/badge.svg)](https://codecov.io/gh/aws-greengrass/aws-greengrass-gdk-cli)
 
 ### *Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.*
 #### *SPDX-License-Identifier: Apache-2.0*

uat/conftest.py

@@ -1,7 +1,23 @@
 import pytest
 
+from t_setup import GdkProcess, GdkInstrumentedProcess
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--instrumented", action="store_true", default=False, help="run tests against code instead of installed gdk cli"
+    )
+
 
 @pytest.fixture()
 def change_test_dir(tmpdir, monkeypatch):
     monkeypatch.chdir(tmpdir)
     return tmpdir
+
+
+@pytest.fixture()
+def gdk_cli(request):
+    if request.config.getoption("--instrumented"):
+        return GdkInstrumentedProcess()
+    else:
+        return GdkProcess()

uat/t_setup.py

@@ -0,0 +1,70 @@
+import subprocess as sp
+import io
+from contextlib import redirect_stdout
+from importlib import reload
+
+import gdk.CLIParser as CLIParser
+import gdk.common.parse_args_actions as parse_args_actions
+import gdk.common.utils as utils
+
+
+class ProcessOutput:
+    def __init__(self, exit_code, output) -> None:
+        self.returncode = exit_code
+        self.output = output
+
+
+class GdkProcess:
+    def __init__(self) -> None:
+        pass
+
+    @classmethod
+    def run(self, arguments=None, capture_output=True) -> ProcessOutput:
+        if arguments is None:
+            arguments = []
+        try:
+            if capture_output:
+                output = sp.run(["gdk"] + arguments, check=True, stdout=sp.PIPE)
+                return ProcessOutput(output.returncode, output.stdout.decode())
+            else:
+                output = sp.run(["gdk"] + arguments)
+                return ProcessOutput(output.returncode, "")
+        except sp.CalledProcessError as e:
+            return ProcessOutput(e.returncode, e.stdout.decode())
+
+
+class GdkInstrumentedProcess(GdkProcess):
+    def __init__(self) -> None:
+        pass
+
+    @classmethod
+    def run(self, arguments=None, capture_output=True) -> ProcessOutput:
+        if arguments is None:
+            arguments = []
+
+        #
+        # In each cli execution, python interpreter reloads all gdk modules, here this simulates that effect.
+        #
+        reload(CLIParser)
+        reload(parse_args_actions)
+        reload(utils)
+
+        # parsed args
+        args = CLIParser.cli_parser.parse_args(arguments)
+
+        exit_code = 0
+        output = ""
+
+        try:
+            if capture_output:
+                f = io.StringIO()
+                with redirect_stdout(f):
+                    parse_args_actions.run_command(args)
+                output = f.getvalue()
+            else:
+                parse_args_actions.run_command(args)
+        except Exception as e:
+            exit_code = 1
+            output = str(e)
+
+        return ProcessOutput(exit_code, output)

uat/test_uat_build.py

@@ -1,19 +1,18 @@
 import os
 import shutil
-import subprocess as sp
 from pathlib import Path
 
 import t_utils
 
 
-def test_build_template_zip(change_test_dir):
+def test_build_template_zip(change_test_dir, gdk_cli):
     # Recipe contains HelloWorld.zip artifact. So, create HelloWorld directory inside temporary directory.
     path_HelloWorld = Path(change_test_dir).joinpath("HelloWorld")
     component_name = "com.example.PythonHelloWorld"
     region = "us-east-1"
     # Check if init downloads templates with necessary files.
-    check_init_template = sp.run(
-        ["gdk", "component", "init", "-t", "HelloWorld", "-l", "python", "-n", "HelloWorld"], check=True, stdout=sp.PIPE
+    check_init_template = gdk_cli.run(
+        ["component", "init", "-t", "HelloWorld", "-l", "python", "-n", "HelloWorld"]
     )
     assert check_init_template.returncode == 0
     assert Path(path_HelloWorld).joinpath("recipe.yaml").resolve().exists()
@@ -25,7 +24,7 @@ def test_build_template_zip(change_test_dir):
 
     os.chdir(path_HelloWorld)
     # Check if build works as expected.
-    check_build_template = sp.run(["gdk", "component", "build"], check=True, stdout=sp.PIPE)
+    check_build_template = gdk_cli.run(["component", "build"])
     assert check_build_template.returncode == 0
     assert Path(path_HelloWorld).joinpath("zip-build").resolve().exists()
     assert Path(path_HelloWorld).joinpath("greengrass-build").resolve().exists()
@@ -42,17 +41,15 @@ def test_build_template_zip(change_test_dir):
     assert artifact_path.exists()
 
 
-def test_build_template_zip_fail_with_no_artifact(change_test_dir):
+def test_build_template_zip_fail_with_no_artifact(change_test_dir, gdk_cli):
     # Recipe contains HelloWorld.zip artifact. So, create a directory with different name.
     dir_name = "artifact-not-exists"
     dir_path = Path(change_test_dir).joinpath(dir_name)
     component_name = "com.example.PythonHelloWorld"
     region = "us-east-1"
     # Check if init downloads templates with necessary files.
-    check_init_template = sp.run(
-        ["gdk", "component", "init", "-t", "HelloWorld", "-l", "python", "-n", dir_name],
-        check=True,
-        stdout=sp.PIPE,
+    check_init_template = gdk_cli.run(
+        ["component", "init", "-t", "HelloWorld", "-l", "python", "-n", dir_name]
     )
     assert check_init_template.returncode == 0
     assert Path(dir_path).joinpath("recipe.yaml").resolve().exists()
@@ -64,8 +61,8 @@ def test_build_template_zip_fail_with_no_artifact(change_test_dir):
 
     os.chdir(dir_path)
     # Check if build works as expected.
-    check_build_template = sp.run(["gdk", "component", "build"], stdout=sp.PIPE)
-    output = check_build_template.stdout.decode()
+    check_build_template = gdk_cli.run(["component", "build"])
+    output = check_build_template.output
     assert check_build_template.returncode == 1
     assert Path(dir_path).joinpath("zip-build").resolve().exists()
     assert (
@@ -76,13 +73,13 @@ def test_build_template_zip_fail_with_no_artifact(change_test_dir):
     assert "Could not build the project due to the following error." in output
 
 
-def test_build_template_maven(change_test_dir):
+def test_build_template_maven(change_test_dir, gdk_cli):
     path_HelloWorld = Path(change_test_dir).joinpath("HelloWorld")
     component_name = "com.example.JavaHelloWorld"
     region = "us-east-1"
     # Check if init downloads templates with necessary files.
-    check_init_template = sp.run(
-        ["gdk", "component", "init", "-t", "HelloWorld", "-l", "java", "-n", "HelloWorld"], check=True, stdout=sp.PIPE
+    check_init_template = gdk_cli.run(
+        ["component", "init", "-t", "HelloWorld", "-l", "java", "-n", "HelloWorld"]
     )
     assert check_init_template.returncode == 0
     assert Path(path_HelloWorld).joinpath("recipe.yaml").resolve().exists()
@@ -93,12 +90,12 @@ def test_build_template_maven(change_test_dir):
 
     os.chdir(path_HelloWorld)
     # Check if build works as expected.
-    check_build_template = sp.run(["gdk", "component", "build"])
+    check_build_template = gdk_cli.run(["component", "build"], capture_output=False)
     assert check_build_template.returncode == 0
     assert Path(path_HelloWorld).joinpath("greengrass-build").resolve().exists()
 
 
-def test_build_template_gradle_multi_project(change_test_dir):
+def test_build_template_gradle_multi_project(change_test_dir, gdk_cli):
     path_multi_gradle_project = Path(change_test_dir).joinpath("gradle-build-test").resolve()
     zip_file = "gradle-build-test.zip"
     component_name = "com.example.Multi.Gradle"
@@ -125,12 +122,12 @@ def test_build_template_gradle_multi_project(change_test_dir):
     t_utils.update_config(config_file, component_name, region, bucket="", author="")
 
     # Check if build works as expected.
-    check_build_template = sp.run(["gdk", "component", "build"])
+    check_build_template = gdk_cli.run(["component", "build"], capture_output=False)
     assert check_build_template.returncode == 0
     assert Path(path_multi_gradle_project).joinpath("greengrass-build").resolve().exists()
 
 
-def test_build_template_maven_multi_project(change_test_dir):
+def test_build_template_maven_multi_project(change_test_dir, gdk_cli):
     path_multi_gradle_project = Path(change_test_dir).joinpath("maven-build-test").resolve()
     zip_file = "maven-build-test.zip"
     component_name = "com.example.Multi.Maven"
@@ -157,6 +154,6 @@ def test_build_template_maven_multi_project(change_test_dir):
     t_utils.update_config(config_file, component_name, region, bucket="", author="")
 
     # Check if build works as expected.
-    check_build_template = sp.run(["gdk", "component", "build"])
+    check_build_template = gdk_cli.run(["component", "build"], capture_output=False)
     assert check_build_template.returncode == 0
     assert Path(path_multi_gradle_project).joinpath("greengrass-build").resolve().exists()

uat/test_uat_init.py

@@ -1,47 +1,46 @@
-import subprocess as sp
 from pathlib import Path
 
 
-def test_init_template_non_empty_dir():
-    check_init_template = sp.run(["gdk", "component", "init", "-t", "HelloWorld", "-l", "python"], stdout=sp.PIPE)
+def test_init_template_non_empty_dir(gdk_cli):
+    check_init_template = gdk_cli.run(["component", "init", "-t", "HelloWorld", "-l", "python"])
     assert check_init_template.returncode == 1
-    assert "Try `gdk component init --help`" in check_init_template.stdout.decode()
+    assert "Try `gdk component init --help`" in check_init_template.output
 
 
-def test_init_template(change_test_dir):
+def test_init_template(change_test_dir, gdk_cli):
     dirpath = Path(change_test_dir)
-    check_init_template = sp.run(["gdk", "component", "init", "-t", "HelloWorld", "-l", "python"], check=True, stdout=sp.PIPE)
+    check_init_template = gdk_cli.run(["component", "init", "-t", "HelloWorld", "-l", "python"])
     assert check_init_template.returncode == 0
     assert Path(dirpath).joinpath("recipe.yaml").resolve().exists()
     assert Path(dirpath).joinpath("gdk-config.json").resolve().exists()
 
 
-def test_init_template_with_new_directory(change_test_dir):
+def test_init_template_with_new_directory(change_test_dir, gdk_cli):
     dir = "test-dir"
     dirpath = Path(change_test_dir).joinpath(dir)
-    check_init_template = sp.run(
-        ["gdk", "component", "init", "-t", "HelloWorld", "-l", "python", "-n", dir], check=True, stdout=sp.PIPE
+    check_init_template = gdk_cli.run(
+        ["component", "init", "-t", "HelloWorld", "-l", "python", "-n", dir]
     )
     assert check_init_template.returncode == 0
     assert Path(dirpath).joinpath("recipe.yaml").resolve().exists()
     assert Path(dirpath).joinpath("gdk-config.json").resolve().exists()
 
 
-def test_init_repository(change_test_dir):
+def test_init_repository(change_test_dir, gdk_cli):
     dirpath = Path(change_test_dir)
-    check_init_repo = sp.run(
-        ["gdk", "component", "init", "-r", "aws-greengrass-labs-database-influxdb"], check=True, stdout=sp.PIPE
+    check_init_repo = gdk_cli.run(
+        ["component", "init", "-r", "aws-greengrass-labs-database-influxdb"]
     )
     assert check_init_repo.returncode == 0
     assert Path(dirpath).joinpath("recipe.yaml").exists()
     assert Path(dirpath).joinpath("gdk-config.json").exists()
 
 
-def test_init_repository_with_new_dir(change_test_dir):
+def test_init_repository_with_new_dir(change_test_dir, gdk_cli):
     dir = "test-dir"
     dirpath = Path(change_test_dir).joinpath(dir)
-    check_init_repo = sp.run(
-        ["gdk", "component", "init", "-r", "aws-greengrass-labs-database-influxdb", "-n", dir], check=True, stdout=sp.PIPE
+    check_init_repo = gdk_cli.run(
+        ["component", "init", "-r", "aws-greengrass-labs-database-influxdb", "-n", dir]
     )
     assert check_init_repo.returncode == 0
     assert Path(dirpath).joinpath("recipe.yaml").exists()

uat/test_uat_list.py

@@ -1,12 +1,9 @@
-import subprocess as sp
+def test_list_template(gdk_cli):
+    check_list_template = gdk_cli.run(["component", "list", "--template"])
+    assert "HelloWorld-python" in check_list_template.output
+    assert "HelloWorld-java" in check_list_template.output
 
 
-def test_list_template():
-    check_list_template = sp.run(["gdk", "component", "list", "--template"], check=True, stdout=sp.PIPE)
-    assert "HelloWorld-python" in check_list_template.stdout.decode()
-    assert "HelloWorld-java" in check_list_template.stdout.decode()
-
-
-def test_list_repository():
-    check_list_template = sp.run(["gdk", "component", "list", "--repository"], check=True, stdout=sp.PIPE)
-    assert "aws-greengrass-labs-database-influxdb" in check_list_template.stdout.decode()
+def test_list_repository(gdk_cli):
+    check_list_template = gdk_cli.run(["component", "list", "--repository"])
+    assert "aws-greengrass-labs-database-influxdb" in check_list_template.output