From e7d0df9082a90c43e76f6813a00626319fc285c6 Mon Sep 17 00:00:00 2001 From: Dev Naruka Date: Tue, 22 Feb 2022 17:26:08 -0800 Subject: [PATCH] test(uat): migrate tests to use gdk_cli abstraction (#76) * test(uat): add abstraction for instrumented gdk * test(uat): migrate tests to use gdk_cli abstraction * ci: track coverage for each test suite * refactor(test): rename --gdkimpl arg to --instrumented * refactor(test): control output capturing with arg --- .github/workflows/CI.yml | 29 +++++++++++++---- README.md | 1 + uat/conftest.py | 16 +++++++++ uat/t_setup.py | 70 ++++++++++++++++++++++++++++++++++++++++ uat/test_uat_build.py | 37 ++++++++++----------- uat/test_uat_init.py | 29 ++++++++--------- uat/test_uat_list.py | 17 ++++------ uat/test_uat_publish.py | 19 ++++++----- 8 files changed, 157 insertions(+), 61 deletions(-) create mode 100644 uat/t_setup.py diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index d8a0150f..8ba44d8d 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -43,10 +43,22 @@ jobs: pip3 install flake8 # The GitHub editor is 127 chars wide flake8 . --count --max-complexity=10 --max-line-length=127 --show-source --statistics - - name: Testing CLI (Runs both unit and integration tests) + - name: Unit Testing CLI run: | pip install -r test-requirements.txt - coverage run --source=gdk -m pytest -v -s tests integration_tests && coverage xml --fail-under=90 + coverage run --source=gdk -m pytest -v -s tests && coverage xml --fail-under=99 + - name: Upload tests coverage to Codecov + uses: codecov/codecov-action@v2 + with: + flags: unit + - name: Integration Testing CLI + run: | + pip install -r test-requirements.txt + coverage run --source=gdk -m pytest -v -s integration_tests && coverage xml --fail-under=90 + - name: Upload integration tests coverage to Codecov + uses: codecov/codecov-action@v2 + with: + flags: integ - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: @@ -56,8 +68,13 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.8 - - name: Run UATs - run: | + - name: Run UATs + run: | coverage run --source=gdk -m pytest -v -s uat - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 \ No newline at end of file + - name: Run UATs with code coverage + run: | + coverage run --source=gdk -m pytest -v -s uat --instrumented && coverage xml --fail-under=77 + - name: Upload UAT coverage to Codecov + uses: codecov/codecov-action@v2 + with: + flags: uat \ No newline at end of file diff --git a/README.md b/README.md index 6309ec7a..95dd7d22 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # Greengrass Development Kit - Command Line Interface ![CI](https://github.com/aws-greengrass/aws-greengrass-gdk-cli/workflows/CI/badge.svg?branch=main) +[![codecov](https://codecov.io/gh/aws-greengrass/aws-greengrass-gdk-cli/branch/main/graph/badge.svg)](https://codecov.io/gh/aws-greengrass/aws-greengrass-gdk-cli) ### *Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.* #### *SPDX-License-Identifier: Apache-2.0* diff --git a/uat/conftest.py b/uat/conftest.py index 6ace426b..755bb4c7 100644 --- a/uat/conftest.py +++ b/uat/conftest.py @@ -1,7 +1,23 @@ import pytest +from t_setup import GdkProcess, GdkInstrumentedProcess + + +def pytest_addoption(parser): + parser.addoption( + "--instrumented", action="store_true", default=False, help="run tests against code instead of installed gdk cli" + ) + @pytest.fixture() def change_test_dir(tmpdir, monkeypatch): monkeypatch.chdir(tmpdir) return tmpdir + + +@pytest.fixture() +def gdk_cli(request): + if request.config.getoption("--instrumented"): + return GdkInstrumentedProcess() + else: + return GdkProcess() diff --git a/uat/t_setup.py b/uat/t_setup.py new file mode 100644 index 00000000..57b7ef37 --- /dev/null +++ b/uat/t_setup.py @@ -0,0 +1,70 @@ +import subprocess as sp +import io +from contextlib import redirect_stdout +from importlib import reload + +import gdk.CLIParser as CLIParser +import gdk.common.parse_args_actions as parse_args_actions +import gdk.common.utils as utils + + +class ProcessOutput: + def __init__(self, exit_code, output) -> None: + self.returncode = exit_code + self.output = output + + +class GdkProcess: + def __init__(self) -> None: + pass + + @classmethod + def run(self, arguments=None, capture_output=True) -> ProcessOutput: + if arguments is None: + arguments = [] + try: + if capture_output: + output = sp.run(["gdk"] + arguments, check=True, stdout=sp.PIPE) + return ProcessOutput(output.returncode, output.stdout.decode()) + else: + output = sp.run(["gdk"] + arguments) + return ProcessOutput(output.returncode, "") + except sp.CalledProcessError as e: + return ProcessOutput(e.returncode, e.stdout.decode()) + + +class GdkInstrumentedProcess(GdkProcess): + def __init__(self) -> None: + pass + + @classmethod + def run(self, arguments=None, capture_output=True) -> ProcessOutput: + if arguments is None: + arguments = [] + + # + # In each cli execution, python interpreter reloads all gdk modules, here this simulates that effect. + # + reload(CLIParser) + reload(parse_args_actions) + reload(utils) + + # parsed args + args = CLIParser.cli_parser.parse_args(arguments) + + exit_code = 0 + output = "" + + try: + if capture_output: + f = io.StringIO() + with redirect_stdout(f): + parse_args_actions.run_command(args) + output = f.getvalue() + else: + parse_args_actions.run_command(args) + except Exception as e: + exit_code = 1 + output = str(e) + + return ProcessOutput(exit_code, output) diff --git a/uat/test_uat_build.py b/uat/test_uat_build.py index 2ef1eee4..da1341f2 100644 --- a/uat/test_uat_build.py +++ b/uat/test_uat_build.py @@ -1,19 +1,18 @@ import os import shutil -import subprocess as sp from pathlib import Path import t_utils -def test_build_template_zip(change_test_dir): +def test_build_template_zip(change_test_dir, gdk_cli): # Recipe contains HelloWorld.zip artifact. So, create HelloWorld directory inside temporary directory. path_HelloWorld = Path(change_test_dir).joinpath("HelloWorld") component_name = "com.example.PythonHelloWorld" region = "us-east-1" # Check if init downloads templates with necessary files. - check_init_template = sp.run( - ["gdk", "component", "init", "-t", "HelloWorld", "-l", "python", "-n", "HelloWorld"], check=True, stdout=sp.PIPE + check_init_template = gdk_cli.run( + ["component", "init", "-t", "HelloWorld", "-l", "python", "-n", "HelloWorld"] ) assert check_init_template.returncode == 0 assert Path(path_HelloWorld).joinpath("recipe.yaml").resolve().exists() @@ -25,7 +24,7 @@ def test_build_template_zip(change_test_dir): os.chdir(path_HelloWorld) # Check if build works as expected. - check_build_template = sp.run(["gdk", "component", "build"], check=True, stdout=sp.PIPE) + check_build_template = gdk_cli.run(["component", "build"]) assert check_build_template.returncode == 0 assert Path(path_HelloWorld).joinpath("zip-build").resolve().exists() assert Path(path_HelloWorld).joinpath("greengrass-build").resolve().exists() @@ -42,17 +41,15 @@ def test_build_template_zip(change_test_dir): assert artifact_path.exists() -def test_build_template_zip_fail_with_no_artifact(change_test_dir): +def test_build_template_zip_fail_with_no_artifact(change_test_dir, gdk_cli): # Recipe contains HelloWorld.zip artifact. So, create a directory with different name. dir_name = "artifact-not-exists" dir_path = Path(change_test_dir).joinpath(dir_name) component_name = "com.example.PythonHelloWorld" region = "us-east-1" # Check if init downloads templates with necessary files. - check_init_template = sp.run( - ["gdk", "component", "init", "-t", "HelloWorld", "-l", "python", "-n", dir_name], - check=True, - stdout=sp.PIPE, + check_init_template = gdk_cli.run( + ["component", "init", "-t", "HelloWorld", "-l", "python", "-n", dir_name] ) assert check_init_template.returncode == 0 assert Path(dir_path).joinpath("recipe.yaml").resolve().exists() @@ -64,8 +61,8 @@ def test_build_template_zip_fail_with_no_artifact(change_test_dir): os.chdir(dir_path) # Check if build works as expected. - check_build_template = sp.run(["gdk", "component", "build"], stdout=sp.PIPE) - output = check_build_template.stdout.decode() + check_build_template = gdk_cli.run(["component", "build"]) + output = check_build_template.output assert check_build_template.returncode == 1 assert Path(dir_path).joinpath("zip-build").resolve().exists() assert ( @@ -76,13 +73,13 @@ def test_build_template_zip_fail_with_no_artifact(change_test_dir): assert "Could not build the project due to the following error." in output -def test_build_template_maven(change_test_dir): +def test_build_template_maven(change_test_dir, gdk_cli): path_HelloWorld = Path(change_test_dir).joinpath("HelloWorld") component_name = "com.example.JavaHelloWorld" region = "us-east-1" # Check if init downloads templates with necessary files. - check_init_template = sp.run( - ["gdk", "component", "init", "-t", "HelloWorld", "-l", "java", "-n", "HelloWorld"], check=True, stdout=sp.PIPE + check_init_template = gdk_cli.run( + ["component", "init", "-t", "HelloWorld", "-l", "java", "-n", "HelloWorld"] ) assert check_init_template.returncode == 0 assert Path(path_HelloWorld).joinpath("recipe.yaml").resolve().exists() @@ -93,12 +90,12 @@ def test_build_template_maven(change_test_dir): os.chdir(path_HelloWorld) # Check if build works as expected. - check_build_template = sp.run(["gdk", "component", "build"]) + check_build_template = gdk_cli.run(["component", "build"], capture_output=False) assert check_build_template.returncode == 0 assert Path(path_HelloWorld).joinpath("greengrass-build").resolve().exists() -def test_build_template_gradle_multi_project(change_test_dir): +def test_build_template_gradle_multi_project(change_test_dir, gdk_cli): path_multi_gradle_project = Path(change_test_dir).joinpath("gradle-build-test").resolve() zip_file = "gradle-build-test.zip" component_name = "com.example.Multi.Gradle" @@ -125,12 +122,12 @@ def test_build_template_gradle_multi_project(change_test_dir): t_utils.update_config(config_file, component_name, region, bucket="", author="") # Check if build works as expected. - check_build_template = sp.run(["gdk", "component", "build"]) + check_build_template = gdk_cli.run(["component", "build"], capture_output=False) assert check_build_template.returncode == 0 assert Path(path_multi_gradle_project).joinpath("greengrass-build").resolve().exists() -def test_build_template_maven_multi_project(change_test_dir): +def test_build_template_maven_multi_project(change_test_dir, gdk_cli): path_multi_gradle_project = Path(change_test_dir).joinpath("maven-build-test").resolve() zip_file = "maven-build-test.zip" component_name = "com.example.Multi.Maven" @@ -157,6 +154,6 @@ def test_build_template_maven_multi_project(change_test_dir): t_utils.update_config(config_file, component_name, region, bucket="", author="") # Check if build works as expected. - check_build_template = sp.run(["gdk", "component", "build"]) + check_build_template = gdk_cli.run(["component", "build"], capture_output=False) assert check_build_template.returncode == 0 assert Path(path_multi_gradle_project).joinpath("greengrass-build").resolve().exists() diff --git a/uat/test_uat_init.py b/uat/test_uat_init.py index 15c469fb..04d14a4a 100644 --- a/uat/test_uat_init.py +++ b/uat/test_uat_init.py @@ -1,47 +1,46 @@ -import subprocess as sp from pathlib import Path -def test_init_template_non_empty_dir(): - check_init_template = sp.run(["gdk", "component", "init", "-t", "HelloWorld", "-l", "python"], stdout=sp.PIPE) +def test_init_template_non_empty_dir(gdk_cli): + check_init_template = gdk_cli.run(["component", "init", "-t", "HelloWorld", "-l", "python"]) assert check_init_template.returncode == 1 - assert "Try `gdk component init --help`" in check_init_template.stdout.decode() + assert "Try `gdk component init --help`" in check_init_template.output -def test_init_template(change_test_dir): +def test_init_template(change_test_dir, gdk_cli): dirpath = Path(change_test_dir) - check_init_template = sp.run(["gdk", "component", "init", "-t", "HelloWorld", "-l", "python"], check=True, stdout=sp.PIPE) + check_init_template = gdk_cli.run(["component", "init", "-t", "HelloWorld", "-l", "python"]) assert check_init_template.returncode == 0 assert Path(dirpath).joinpath("recipe.yaml").resolve().exists() assert Path(dirpath).joinpath("gdk-config.json").resolve().exists() -def test_init_template_with_new_directory(change_test_dir): +def test_init_template_with_new_directory(change_test_dir, gdk_cli): dir = "test-dir" dirpath = Path(change_test_dir).joinpath(dir) - check_init_template = sp.run( - ["gdk", "component", "init", "-t", "HelloWorld", "-l", "python", "-n", dir], check=True, stdout=sp.PIPE + check_init_template = gdk_cli.run( + ["component", "init", "-t", "HelloWorld", "-l", "python", "-n", dir] ) assert check_init_template.returncode == 0 assert Path(dirpath).joinpath("recipe.yaml").resolve().exists() assert Path(dirpath).joinpath("gdk-config.json").resolve().exists() -def test_init_repository(change_test_dir): +def test_init_repository(change_test_dir, gdk_cli): dirpath = Path(change_test_dir) - check_init_repo = sp.run( - ["gdk", "component", "init", "-r", "aws-greengrass-labs-database-influxdb"], check=True, stdout=sp.PIPE + check_init_repo = gdk_cli.run( + ["component", "init", "-r", "aws-greengrass-labs-database-influxdb"] ) assert check_init_repo.returncode == 0 assert Path(dirpath).joinpath("recipe.yaml").exists() assert Path(dirpath).joinpath("gdk-config.json").exists() -def test_init_repository_with_new_dir(change_test_dir): +def test_init_repository_with_new_dir(change_test_dir, gdk_cli): dir = "test-dir" dirpath = Path(change_test_dir).joinpath(dir) - check_init_repo = sp.run( - ["gdk", "component", "init", "-r", "aws-greengrass-labs-database-influxdb", "-n", dir], check=True, stdout=sp.PIPE + check_init_repo = gdk_cli.run( + ["component", "init", "-r", "aws-greengrass-labs-database-influxdb", "-n", dir] ) assert check_init_repo.returncode == 0 assert Path(dirpath).joinpath("recipe.yaml").exists() diff --git a/uat/test_uat_list.py b/uat/test_uat_list.py index d6d62918..7b2caa76 100644 --- a/uat/test_uat_list.py +++ b/uat/test_uat_list.py @@ -1,12 +1,9 @@ -import subprocess as sp +def test_list_template(gdk_cli): + check_list_template = gdk_cli.run(["component", "list", "--template"]) + assert "HelloWorld-python" in check_list_template.output + assert "HelloWorld-java" in check_list_template.output -def test_list_template(): - check_list_template = sp.run(["gdk", "component", "list", "--template"], check=True, stdout=sp.PIPE) - assert "HelloWorld-python" in check_list_template.stdout.decode() - assert "HelloWorld-java" in check_list_template.stdout.decode() - - -def test_list_repository(): - check_list_template = sp.run(["gdk", "component", "list", "--repository"], check=True, stdout=sp.PIPE) - assert "aws-greengrass-labs-database-influxdb" in check_list_template.stdout.decode() +def test_list_repository(gdk_cli): + check_list_template = gdk_cli.run(["component", "list", "--repository"]) + assert "aws-greengrass-labs-database-influxdb" in check_list_template.output diff --git a/uat/test_uat_publish.py b/uat/test_uat_publish.py index f0dfa5c2..cc346098 100644 --- a/uat/test_uat_publish.py +++ b/uat/test_uat_publish.py @@ -1,11 +1,10 @@ import os -import subprocess as sp from pathlib import Path import t_utils -def test_publish_template_zip(change_test_dir): +def test_publish_template_zip(change_test_dir, gdk_cli): # Recipe contains HelloWorld.zip artifact. So, create HelloWorld directory inside temporary directory. path_HelloWorld = Path(change_test_dir).joinpath("HelloWorld") component_name = "com.example.PythonHelloWorld" @@ -13,8 +12,8 @@ def test_publish_template_zip(change_test_dir): bucket = "gdk-cli-uat" author = "gdk-cli-uat" # Check if init downloads templates with necessary files. - check_init_template = sp.run( - ["gdk", "component", "init", "-t", "HelloWorld", "-l", "python", "-n", "HelloWorld"], check=True, stdout=sp.PIPE + check_init_template = gdk_cli.run( + ["component", "init", "-t", "HelloWorld", "-l", "python", "-n", "HelloWorld"] ) assert check_init_template.returncode == 0 assert Path(path_HelloWorld).joinpath("recipe.yaml").resolve().exists() @@ -24,7 +23,7 @@ def test_publish_template_zip(change_test_dir): t_utils.update_config(config_file, component_name, region, bucket, author) os.chdir(path_HelloWorld) # Check if build works as expected. - check_build_template = sp.run(["gdk", "component", "build"], check=True, stdout=sp.PIPE) + check_build_template = gdk_cli.run(["component", "build"]) assert check_build_template.returncode == 0 assert Path(path_HelloWorld).joinpath("zip-build").resolve().exists() assert Path(path_HelloWorld).joinpath("greengrass-build").resolve().exists() @@ -39,13 +38,13 @@ def test_publish_template_zip(change_test_dir): ) assert artifact_path.exists() - check_publish_component = sp.run(["gdk", "component", "publish"], stdout=sp.PIPE) + check_publish_component = gdk_cli.run(["component", "publish"]) assert check_publish_component.returncode == 0 recipes_path = Path(path_HelloWorld).joinpath("greengrass-build").joinpath("recipes").resolve() t_utils.clean_up_aws_resources(component_name, t_utils.get_version_created(recipes_path, component_name), region) -def test_publish_without_build_template_zip(change_test_dir): +def test_publish_without_build_template_zip(change_test_dir, gdk_cli): # Recipe contains HelloWorld.zip artifact. So, create HelloWorld directory inside temporary directory. path_HelloWorld = Path(change_test_dir).joinpath("HelloWorld") component_name = "com.example.PythonHelloWorld" @@ -53,8 +52,8 @@ def test_publish_without_build_template_zip(change_test_dir): bucket = "gdk-cli-uat" author = "gdk-cli-uat" # Check if init downloads templates with necessary files. - check_init_template = sp.run( - ["gdk", "component", "init", "-t", "HelloWorld", "-l", "python", "-n", "HelloWorld"], check=True, stdout=sp.PIPE + check_init_template = gdk_cli.run( + ["component", "init", "-t", "HelloWorld", "-l", "python", "-n", "HelloWorld"] ) assert check_init_template.returncode == 0 assert Path(path_HelloWorld).joinpath("recipe.yaml").resolve().exists() @@ -66,7 +65,7 @@ def test_publish_without_build_template_zip(change_test_dir): os.chdir(path_HelloWorld) - check_publish_component = sp.run(["gdk", "component", "publish"], stdout=sp.PIPE) + check_publish_component = gdk_cli.run(["component", "publish"]) assert check_publish_component.returncode == 0 assert Path(path_HelloWorld).joinpath("zip-build").resolve().exists() assert Path(path_HelloWorld).joinpath("greengrass-build").resolve().exists()