From d79b25ddcc8b03ddbf274d2aba2b57892ca0d83d Mon Sep 17 00:00:00 2001 From: Sriram Madapusi Vasudevan Date: Tue, 3 Dec 2019 12:57:24 -0800 Subject: [PATCH 1/7] fix: appveyor build badge --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 87ea496f5..bf1e12779 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ ## Lambda Builders -[![AppVeyor Status](https://ci.appveyor.com/api/projects/status/txv6gk69614727nu?svg=true)](https://ci.appveyor.com/project/sanathkr/aws-lambda-builders-3bxwl) -[![Travis Status](https://travis-ci.org/awslabs/aws-lambda-builders.svg?branch=develop)](https://travis-ci.org/awslabs/aws-lambda-builders) +[![Build status](https://ci.appveyor.com/api/projects/status/mrehrn5im0305lje/branch/develop?svg=true)](https://ci.appveyor.com/project/AWSSAMCLI/aws-lambda-builders/branch/develop) Lambda Builders is a Python library to compile, build and package AWS Lambda functions for several runtimes & frameworks. From 24f0aca6ed41ec2b08333655adbd1433f14cdc9e Mon Sep 17 00:00:00 2001 From: Jacob Fuss <32497805+jfuss@users.noreply.github.com> Date: Tue, 17 Dec 2019 17:02:44 -0600 Subject: [PATCH 2/7] chore: Adopt Black as an autoformatter (#143) * chore: Adopt Black as an autoformatter * Remove unneeded excludes in pyproject for black * Add python3.8 and python2.7 to black formatting * Update 3.6.8 to 3.6.9 * Update 2.7.16 to 2.7.17 * force unicode string for python2.7 --- .appveyor.yml | 28 +- Makefile | 15 +- aws_lambda_builders/__init__.py | 2 +- aws_lambda_builders/__main__.py | 58 +- aws_lambda_builders/actions.py | 8 +- aws_lambda_builders/binary_path.py | 1 - aws_lambda_builders/builder.py | 44 +- aws_lambda_builders/exceptions.py | 21 +- aws_lambda_builders/path_resolver.py | 6 +- aws_lambda_builders/registry.py | 16 +- aws_lambda_builders/utils.py | 1 + aws_lambda_builders/validator.py | 1 - aws_lambda_builders/workflow.py | 63 +- .../workflows/dotnet_clipackage/actions.py | 21 +- .../workflows/dotnet_clipackage/dotnetcli.py | 19 +- .../dotnet_clipackage/dotnetcli_resolver.py | 8 +- .../workflows/dotnet_clipackage/utils.py | 6 +- .../workflows/dotnet_clipackage/workflow.py | 36 +- .../workflows/go_dep/actions.py | 8 +- .../workflows/go_dep/subproc_exec.py | 12 +- aws_lambda_builders/workflows/go_dep/utils.py | 2 +- .../workflows/go_dep/workflow.py | 27 +- .../workflows/go_modules/actions.py | 5 +- .../workflows/go_modules/builder.py | 8 +- .../workflows/go_modules/utils.py | 1 + .../workflows/go_modules/validator.py | 19 +- .../workflows/go_modules/workflow.py | 27 +- .../workflows/java_gradle/actions.py | 33 +- .../workflows/java_gradle/gradle.py | 19 +- .../workflows/java_gradle/gradle_resolver.py | 7 +- .../workflows/java_gradle/gradle_validator.py | 28 +- .../workflows/java_gradle/utils.py | 2 +- .../workflows/java_gradle/workflow.py | 33 +- .../workflows/java_maven/actions.py | 30 +- .../workflows/java_maven/maven.py | 16 +- .../workflows/java_maven/maven_resolver.py | 5 +- .../workflows/java_maven/maven_validator.py | 28 +- .../workflows/java_maven/utils.py | 2 +- .../workflows/java_maven/workflow.py | 31 +- .../workflows/nodejs_npm/actions.py | 15 +- .../workflows/nodejs_npm/npm.py | 17 +- .../workflows/nodejs_npm/utils.py | 4 +- .../workflows/nodejs_npm/workflow.py | 42 +- .../workflows/python_pip/actions.py | 15 +- .../workflows/python_pip/compat.py | 36 +- .../workflows/python_pip/packager.py | 206 ++-- .../workflows/python_pip/utils.py | 11 +- .../workflows/python_pip/validator.py | 25 +- .../workflows/python_pip/workflow.py | 95 +- .../workflows/ruby_bundler/actions.py | 14 +- .../workflows/ruby_bundler/bundler.py | 19 +- .../workflows/ruby_bundler/utils.py | 4 +- .../workflows/ruby_bundler/workflow.py | 31 +- pyproject.toml | 17 + setup.py | 79 +- tests/functional/test_builder.py | 33 +- tests/functional/test_cli.py | 99 +- tests/functional/test_utils.py | 4 +- tests/functional/testdata/cwd.py | 2 +- .../workflows/hello_workflow/write_hello.py | 5 +- .../workflows/go_dep/test_godep_utils.py | 28 +- .../workflows/go_modules/test_go_utils.py | 24 +- .../workflows/java_gradle/test_java_utils.py | 34 +- .../workflows/java_maven/test_java_utils.py | 34 +- .../workflows/nodejs_npm/test_utils.py | 30 +- .../workflows/python_pip/test_packager.py | 887 +++++++----------- .../workflows/python_pip/test_utils.py | 9 +- .../workflows/ruby_bundler/test_ruby_utils.py | 28 +- .../dotnet_clipackage/test_dotnet.py | 54 +- .../workflows/go_dep/test_go_dep.py | 61 +- .../workflows/go_modules/test_go.py | 45 +- .../workflows/java_gradle/test_java_gradle.py | 89 +- .../workflows/java_maven/test_java_maven.py | 33 +- .../workflows/nodejs_npm/test_nodejs_npm.py | 54 +- .../workflows/python_pip/test_python_pip.py | 70 +- .../workflows/ruby_bundler/test_ruby.py | 35 +- tests/unit/test_actions.py | 7 +- tests/unit/test_builder.py | 109 ++- tests/unit/test_path_resolver.py | 3 +- tests/unit/test_registry.py | 16 +- tests/unit/test_validator.py | 1 - tests/unit/test_workflow.py | 137 +-- .../dotnet_clipackage/test_actions.py | 65 +- .../dotnet_clipackage/test_dotnetcli.py | 5 +- .../test_dotnetcli_resolver.py | 1 - .../dotnet_clipackage/test_workflow.py | 1 - tests/unit/workflows/go_dep/test_actions.py | 6 +- tests/unit/workflows/go_dep/test_exec.py | 11 +- tests/unit/workflows/go_dep/test_workflow.py | 8 +- .../unit/workflows/go_modules/test_builder.py | 22 +- .../workflows/go_modules/test_validator.py | 6 +- .../workflows/go_modules/test_workflow.py | 8 +- .../workflows/java_gradle/test_actions.py | 88 +- .../unit/workflows/java_gradle/test_gradle.py | 78 +- .../java_gradle/test_gradle_validator.py | 33 +- .../java_gradle/test_gradlew_resolver.py | 17 +- .../workflows/java_gradle/test_workflow.py | 2 +- .../unit/workflows/java_maven/test_actions.py | 79 +- tests/unit/workflows/java_maven/test_maven.py | 34 +- .../java_maven/test_maven_resolver.py | 5 +- .../java_maven/test_maven_validator.py | 35 +- .../workflows/java_maven/test_workflow.py | 7 +- .../unit/workflows/nodejs_npm/test_actions.py | 61 +- tests/unit/workflows/nodejs_npm/test_npm.py | 36 +- .../workflows/nodejs_npm/test_workflow.py | 8 +- .../unit/workflows/python_pip/test_actions.py | 33 +- .../workflows/python_pip/test_packager.py | 207 ++-- .../workflows/python_pip/test_validator.py | 30 +- .../workflows/python_pip/test_workflow.py | 1 - .../workflows/ruby_bundler/test_actions.py | 20 +- .../workflows/ruby_bundler/test_bundler.py | 48 +- 111 files changed, 1949 insertions(+), 2264 deletions(-) create mode 100644 pyproject.toml diff --git a/.appveyor.yml b/.appveyor.yml index 476d03a3e..68f61232d 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -12,13 +12,13 @@ environment: matrix: - PYTHON: "C:\\Python27-x64" - PYTHON_VERSION: '2.7.16' + PYTHON_VERSION: '2.7.17' PYTHON_ARCH: '64' LINE_COVERAGE: '91' NEW_FLAKE8: 0 JAVA_HOME: "C:\\Program Files\\Java\\jdk11" - PYTHON: "C:\\Python36-x64" - PYTHON_VERSION: '3.6.8' + PYTHON_VERSION: '3.6.9' PYTHON_ARCH: '64' LINE_COVERAGE: '91' NEW_FLAKE8: 0 @@ -100,6 +100,11 @@ for: - sh: "sudo unzip -d /opt/gradle /tmp/gradle-*.zip" - sh: "PATH=/opt/gradle/gradle-5.5/bin:$PATH" + # Install black + - sh: "wget -O /tmp/black https://github.com/python/black/releases/download/19.3b0/black" + - sh: "chmod +x /tmp/black" + - sh: "/tmp/black --version" + build_script: - "python -c \"import sys; print(sys.executable)\"" - "LAMBDA_BUILDERS_DEV=1 pip install -e \".[dev]\"" @@ -111,20 +116,5 @@ for: # Runs only in Linux - "LAMBDA_BUILDERS_DEV=1 pytest -vv tests/integration" - - - matrix: - only: - - OLD_FLAKE8: 1 - - test_script: - - "flake8 lambda_builders" - - "flake8 tests/unit tests/integration" - - - - matrix: - only: - - NEW_FLAKE8: 0 - - test_script: - - "flake8 lambda_builders" - - "flake8 tests/unit tests/integration --extend-ignore=W504" + # Validate Code was formatted with Black + - "/tmp/black --check setup.py tests aws_lambda_builders" diff --git a/Makefile b/Makefile index fead70b1d..883aba1b1 100644 --- a/Makefile +++ b/Makefile @@ -13,17 +13,18 @@ integ-test: # Integration tests don't need code coverage LAMBDA_BUILDERS_DEV=1 pytest tests/integration -flake: - # Make sure code conforms to PEP8 standards - flake8 lambda_builders - flake8 tests/unit tests/integration --extend-ignore=W504 - lint: # Liner performs static analysis to catch latent bugs pylint --rcfile .pylintrc aws_lambda_builders # Command to run everytime you make changes to verify everything works -dev: flake lint test +dev: lint test + +black: + black setup.py aws_lambda_builders/* tests/* + +black-check: + black --check setup.py aws_lambda_builders/* tests/* # Verifications to run before sending a pull request -pr: init dev +pr: init dev black-check diff --git a/aws_lambda_builders/__init__.py b/aws_lambda_builders/__init__.py index f36ff719e..b5c061292 100644 --- a/aws_lambda_builders/__init__.py +++ b/aws_lambda_builders/__init__.py @@ -1,5 +1,5 @@ """ AWS Lambda Builder Library """ -__version__ = '0.6.0' +__version__ = "0.6.0" RPC_PROTOCOL_VERSION = "0.3" diff --git a/aws_lambda_builders/__main__.py b/aws_lambda_builders/__main__.py index b5fe009bc..b4fc80cfa 100644 --- a/aws_lambda_builders/__main__.py +++ b/aws_lambda_builders/__main__.py @@ -19,9 +19,7 @@ log_level = int(os.environ.get("LAMBDA_BUILDERS_LOG_LEVEL", logging.INFO)) # Write output to stderr because stdout is used for command response -logging.basicConfig(stream=sys.stderr, - level=log_level, - format='%(message)s') +logging.basicConfig(stream=sys.stderr, level=log_level, format="%(message)s") LOG = logging.getLogger(__name__) @@ -29,24 +27,11 @@ def _success_response(request_id, artifacts_dir): - return json.dumps({ - "jsonrpc": "2.0", - "id": request_id, - "result": { - "artifacts_dir": artifacts_dir - } - }) + return json.dumps({"jsonrpc": "2.0", "id": request_id, "result": {"artifacts_dir": artifacts_dir}}) def _error_response(request_id, http_status_code, message): - return json.dumps({ - "jsonrpc": "2.0", - "id": request_id, - "error": { - "code": http_status_code, - "message": message - } - }) + return json.dumps({"jsonrpc": "2.0", "id": request_id, "error": {"code": http_status_code, "message": message}}) def _parse_version(version_string): @@ -68,8 +53,9 @@ def version_compatibility_check(version): # 0.2 < 0.1 comparison will fail, don't throw a value Error saying incompatible version if _parse_version(lambda_builders_protocol_version) < version: - ex = "Incompatible Protocol Version : {}, " \ - "Current Protocol Version: {}".format(version, lambda_builders_protocol_version) + ex = "Incompatible Protocol Version : {}, " "Current Protocol Version: {}".format( + version, lambda_builders_protocol_version + ) LOG.error(ex) raise ValueError(ex) @@ -120,21 +106,25 @@ def main(): # pylint: disable=too-many-statements response = None try: - builder = LambdaBuilder(language=capabilities["language"], - dependency_manager=capabilities["dependency_manager"], - application_framework=capabilities["application_framework"], - supported_workflows=supported_workflows) + builder = LambdaBuilder( + language=capabilities["language"], + dependency_manager=capabilities["dependency_manager"], + application_framework=capabilities["application_framework"], + supported_workflows=supported_workflows, + ) artifacts_dir = params["artifacts_dir"] - builder.build(params["source_dir"], - params["artifacts_dir"], - params["scratch_dir"], - params["manifest_path"], - executable_search_paths=params.get('executable_search_paths', None), - runtime=params["runtime"], - optimizations=params["optimizations"], - options=params["options"], - mode=params.get('mode', None)) + builder.build( + params["source_dir"], + params["artifacts_dir"], + params["scratch_dir"], + params["manifest_path"], + executable_search_paths=params.get("executable_search_paths", None), + runtime=params["runtime"], + optimizations=params["optimizations"], + options=params["options"], + mode=params.get("mode", None), + ) # Return a success response response = _success_response(request_id, artifacts_dir) @@ -152,5 +142,5 @@ def main(): # pylint: disable=too-many-statements _write_response(response, exit_code) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/aws_lambda_builders/actions.py b/aws_lambda_builders/actions.py index d37d361c1..89ae3801a 100644 --- a/aws_lambda_builders/actions.py +++ b/aws_lambda_builders/actions.py @@ -15,6 +15,7 @@ class ActionFailedError(Exception): """ Base class for exception raised when action failed to complete. Use this to express well-known failure scenarios. """ + pass @@ -38,12 +39,11 @@ def has_value(item): class _ActionMetaClass(type): - def __new__(mcs, name, bases, class_dict): cls = type.__new__(mcs, name, bases, class_dict) - if cls.__name__ == 'BaseAction': + if cls.__name__ == "BaseAction": return cls # Validate class variables @@ -78,7 +78,7 @@ def execute(self): :raises lambda_builders.actions.ActionFailedError: Instance of this class if something went wrong with the action """ - raise NotImplementedError('execute') + raise NotImplementedError("execute") def __repr__(self): return "Name={}, Purpose={}, Description={}".format(self.NAME, self.PURPOSE, self.DESCRIPTION) @@ -86,7 +86,7 @@ def __repr__(self): class CopySourceAction(BaseAction): - NAME = 'CopySource' + NAME = "CopySource" DESCRIPTION = "Copying source code while skipping certain commonly excluded files" diff --git a/aws_lambda_builders/binary_path.py b/aws_lambda_builders/binary_path.py index 78467666f..541299c0d 100644 --- a/aws_lambda_builders/binary_path.py +++ b/aws_lambda_builders/binary_path.py @@ -4,7 +4,6 @@ class BinaryPath(object): - def __init__(self, resolver, validator, binary, binary_path=None): self.resolver = resolver self.validator = validator diff --git a/aws_lambda_builders/builder.py b/aws_lambda_builders/builder.py index 9518be6fd..9a9a3d04d 100644 --- a/aws_lambda_builders/builder.py +++ b/aws_lambda_builders/builder.py @@ -11,9 +11,7 @@ LOG = logging.getLogger(__name__) -_SUPPORTED_WORKFLOWS = [ - "aws_lambda_builders.workflows" -] +_SUPPORTED_WORKFLOWS = ["aws_lambda_builders.workflows"] class LambdaBuilder(object): @@ -49,14 +47,24 @@ def __init__(self, language, dependency_manager, application_framework, supporte # If a module is already loaded, this call is pretty much a no-op. So it is okay to keep loading again. importlib.import_module(workflow_module) - self.capability = Capability(language=language, - dependency_manager=dependency_manager, - application_framework=application_framework) + self.capability = Capability( + language=language, dependency_manager=dependency_manager, application_framework=application_framework + ) self.selected_workflow_cls = get_workflow(self.capability) LOG.debug("Found workflow '%s' to support capabilities '%s'", self.selected_workflow_cls.NAME, self.capability) - def build(self, source_dir, artifacts_dir, scratch_dir, manifest_path, - runtime=None, optimizations=None, options=None, executable_search_paths=None, mode=None): + def build( + self, + source_dir, + artifacts_dir, + scratch_dir, + manifest_path, + runtime=None, + optimizations=None, + options=None, + executable_search_paths=None, + mode=None, + ): """ Actually build the code by running workflows @@ -102,15 +110,17 @@ def build(self, source_dir, artifacts_dir, scratch_dir, manifest_path, if not os.path.exists(scratch_dir): os.makedirs(scratch_dir) - workflow = self.selected_workflow_cls(source_dir, - artifacts_dir, - scratch_dir, - manifest_path, - runtime=runtime, - optimizations=optimizations, - options=options, - executable_search_paths=executable_search_paths, - mode=mode) + workflow = self.selected_workflow_cls( + source_dir, + artifacts_dir, + scratch_dir, + manifest_path, + runtime=runtime, + optimizations=optimizations, + options=options, + executable_search_paths=executable_search_paths, + mode=mode, + ) return workflow.run() diff --git a/aws_lambda_builders/exceptions.py b/aws_lambda_builders/exceptions.py index 737188510..646b914dc 100644 --- a/aws_lambda_builders/exceptions.py +++ b/aws_lambda_builders/exceptions.py @@ -5,7 +5,7 @@ class LambdaBuilderError(Exception): - MESSAGE = '' + MESSAGE = "" def __init__(self, **kwargs): Exception.__init__(self, self.MESSAGE.format(**kwargs)) @@ -16,24 +16,30 @@ class UnsupportedManifestError(LambdaBuilderError): class MisMatchRuntimeError(LambdaBuilderError): - MESSAGE = "{language} executable found in your path does not " \ - "match runtime. " \ - "\n Expected version: {required_runtime}, Found version: {runtime_path}. " \ - "\n Possibly related: https://github.com/awslabs/aws-lambda-builders/issues/30" + MESSAGE = ( + "{language} executable found in your path does not " + "match runtime. " + "\n Expected version: {required_runtime}, Found version: {runtime_path}. " + "\n Possibly related: https://github.com/awslabs/aws-lambda-builders/issues/30" + ) class WorkflowNotFoundError(LambdaBuilderError): """ Raised when a workflow matching the given capabilities was not found """ - MESSAGE = "Unable to find a workflow matching given capability: " \ - "{language}, {dependency_manager}, {application_framework}" + + MESSAGE = ( + "Unable to find a workflow matching given capability: " + "{language}, {dependency_manager}, {application_framework}" + ) class WorkflowFailedError(LambdaBuilderError): """ Raised when the build failed, for well-known cases """ + MESSAGE = "{workflow_name}:{action_name} - {reason}" @@ -41,4 +47,5 @@ class WorkflowUnknownError(LambdaBuilderError): """ Raised when the build ran into an unexpected error """ + MESSAGE = "{workflow_name}:{action_name} - {reason}" diff --git a/aws_lambda_builders/path_resolver.py b/aws_lambda_builders/path_resolver.py index 3d05e135c..e8f9736ee 100644 --- a/aws_lambda_builders/path_resolver.py +++ b/aws_lambda_builders/path_resolver.py @@ -6,7 +6,6 @@ class PathResolver(object): - def __init__(self, binary, runtime, executable_search_paths=None): self.binary = binary self.runtime = runtime @@ -20,8 +19,9 @@ def _which(self): exec_paths.extend(paths) if not exec_paths: - raise ValueError("Path resolution for runtime: {} of binary: " - "{} was not successful".format(self.runtime, self.binary)) + raise ValueError( + "Path resolution for runtime: {} of binary: " "{} was not successful".format(self.runtime, self.binary) + ) return exec_paths @property diff --git a/aws_lambda_builders/registry.py b/aws_lambda_builders/registry.py index cb59217d7..60348d292 100644 --- a/aws_lambda_builders/registry.py +++ b/aws_lambda_builders/registry.py @@ -61,11 +61,9 @@ def _make_key(capability): # Key is created by concatenating the capabilites data with underscore. # This delimiter is positional ie. if a value is not provided, the delimiter still needs to exist in the key. # This helps us be forwards compatible with new capabilities - return "_".join([ - capability.language or "", - capability.dependency_manager or "", - capability.application_framework or "" - ]).lower() + return "_".join( + [capability.language or "", capability.dependency_manager or "", capability.application_framework or ""] + ).lower() # Built-in registry of workflows. @@ -92,8 +90,10 @@ def get_workflow(capability, registry=DEFAULT_REGISTRY): """ if capability not in registry: - raise WorkflowNotFoundError(language=capability.language, - dependency_manager=capability.dependency_manager, - application_framework=capability.application_framework) + raise WorkflowNotFoundError( + language=capability.language, + dependency_manager=capability.dependency_manager, + application_framework=capability.application_framework, + ) return registry[capability] diff --git a/aws_lambda_builders/utils.py b/aws_lambda_builders/utils.py index b8e6e3ea7..1ccd6dce5 100644 --- a/aws_lambda_builders/utils.py +++ b/aws_lambda_builders/utils.py @@ -59,6 +59,7 @@ def copytree(source, destination, ignore=None): else: shutil.copy2(new_source, new_destination) + # NOTE: The below function is copied from Python source code and modified # slightly to return a list of paths that match a given command # instead of returning just the first match diff --git a/aws_lambda_builders/validator.py b/aws_lambda_builders/validator.py index aa0fa1528..4e014c103 100644 --- a/aws_lambda_builders/validator.py +++ b/aws_lambda_builders/validator.py @@ -8,7 +8,6 @@ class RuntimeValidator(object): - def __init__(self, runtime): self.runtime = runtime self._runtime_path = None diff --git a/aws_lambda_builders/workflow.py b/aws_lambda_builders/workflow.py index 0c3541973..a659f817d 100644 --- a/aws_lambda_builders/workflow.py +++ b/aws_lambda_builders/workflow.py @@ -22,7 +22,7 @@ # ``Language`` is the programming language. Ex: Python # ``LangageFramework`` is the framework of particular language. Ex: PIP # ``ApplicationFramework`` is the specific application framework used to write the code. Ex: Chalice -Capability = namedtuple('Capability', ["language", "dependency_manager", "application_framework"]) +Capability = namedtuple("Capability", ["language", "dependency_manager", "application_framework"]) class BuildMode(object): @@ -51,18 +51,16 @@ def wrapper(self, *args, **kwargs): try: valid_path = validator.validate(executable_path) except MisMatchRuntimeError as ex: - LOG.debug("Invalid executable for %s at %s", - binary, executable_path, exc_info=str(ex)) + LOG.debug("Invalid executable for %s at %s", binary, executable_path, exc_info=str(ex)) if valid_path: binary_path.binary_path = valid_path valid_paths.append(valid_path) break self.binaries = binaries_copy if len(self.binaries) != len(valid_paths): - raise WorkflowFailedError(workflow_name=self.NAME, - action_name=None, - reason='Binary validation failed!') + raise WorkflowFailedError(workflow_name=self.NAME, action_name=None, reason="Binary validation failed!") func(self, *args, **kwargs) + return wrapper @@ -80,7 +78,7 @@ def __new__(mcs, name, bases, class_dict): # We don't want to register the base classes, so we simply return here. # Also, skip further steps if the class is marked for testing - if cls.__name__ == 'BaseWorkflow' or cls.__TESTING__: + if cls.__name__ == "BaseWorkflow" or cls.__TESTING__: return cls # Validate class variables @@ -117,16 +115,18 @@ class BaseWorkflow(six.with_metaclass(_WorkflowMetaClass, object)): # Optional list of manifests file/folder names supported by this workflow. SUPPORTED_MANIFESTS = [] - def __init__(self, - source_dir, - artifacts_dir, - scratch_dir, - manifest_path, - runtime=None, - executable_search_paths=None, - optimizations=None, - options=None, - mode=BuildMode.RELEASE): + def __init__( + self, + source_dir, + artifacts_dir, + scratch_dir, + manifest_path, + runtime=None, + executable_search_paths=None, + optimizations=None, + options=None, + mode=BuildMode.RELEASE, + ): """ Initialize the builder with given arguments. These arguments together form the "public API" that each build action must support at the minimum. @@ -199,8 +199,13 @@ def get_resolvers(self): """ Non specialized path resolver that just returns the list of executable for the runtime on the path. """ - return [PathResolver(runtime=self.runtime, binary=self.CAPABILITY.language, - executable_search_paths=self.executable_search_paths)] + return [ + PathResolver( + runtime=self.runtime, + binary=self.CAPABILITY.language, + executable_search_paths=self.executable_search_paths, + ) + ] def get_validators(self): """ @@ -213,8 +218,10 @@ def binaries(self): if not self._binaries: resolvers = self.get_resolvers() validators = self.get_validators() - self._binaries = {resolver.binary: BinaryPath(resolver=resolver, validator=validator, binary=resolver.binary) - for resolver, validator in zip(resolvers, validators)} + self._binaries = { + resolver.binary: BinaryPath(resolver=resolver, validator=validator, binary=resolver.binary) + for resolver, validator in zip(resolvers, validators) + } return self._binaries @binaries.setter @@ -235,9 +242,9 @@ def run(self): LOG.debug("Running workflow '%s'", self.NAME) if not self.actions: - raise WorkflowFailedError(workflow_name=self.NAME, - action_name=None, - reason="Workflow does not have any actions registered") + raise WorkflowFailedError( + workflow_name=self.NAME, action_name=None, reason="Workflow does not have any actions registered" + ) for action in self.actions: action_info = "{}:{}".format(self.NAME, action.NAME) @@ -252,15 +259,11 @@ def run(self): except ActionFailedError as ex: LOG.debug("%s failed", action_info, exc_info=ex) - raise WorkflowFailedError(workflow_name=self.NAME, - action_name=action.NAME, - reason=str(ex)) + raise WorkflowFailedError(workflow_name=self.NAME, action_name=action.NAME, reason=str(ex)) except Exception as ex: LOG.debug("%s raised unhandled exception", action_info, exc_info=ex) - raise WorkflowUnknownError(workflow_name=self.NAME, - action_name=action.NAME, - reason=str(ex)) + raise WorkflowUnknownError(workflow_name=self.NAME, action_name=action.NAME, reason=str(ex)) def __repr__(self): """ diff --git a/aws_lambda_builders/workflows/dotnet_clipackage/actions.py b/aws_lambda_builders/workflows/dotnet_clipackage/actions.py index 0d67eaed1..accaaad08 100644 --- a/aws_lambda_builders/workflows/dotnet_clipackage/actions.py +++ b/aws_lambda_builders/workflows/dotnet_clipackage/actions.py @@ -12,13 +12,14 @@ LOG = logging.getLogger(__name__) + class GlobalToolInstallAction(BaseAction): """ A Lambda Builder Action which installs the Amazon.Lambda.Tools .NET Core Global Tool """ - NAME = 'GlobalToolInstall' + NAME = "GlobalToolInstall" DESCRIPTION = "Install or update the Amazon.Lambda.Tools .NET Core Global Tool." PURPOSE = Purpose.COMPILE_SOURCE @@ -29,24 +30,21 @@ def __init__(self, subprocess_dotnet): def execute(self): try: LOG.debug("Installing Amazon.Lambda.Tools Global Tool") - self.subprocess_dotnet.run( - ['tool', 'install', '-g', 'Amazon.Lambda.Tools'], - ) + self.subprocess_dotnet.run(["tool", "install", "-g", "Amazon.Lambda.Tools"]) except DotnetCLIExecutionError as ex: LOG.debug("Error installing probably due to already installed. Attempt to update to latest version.") try: - self.subprocess_dotnet.run( - ['tool', 'update', '-g', 'Amazon.Lambda.Tools'], - ) + self.subprocess_dotnet.run(["tool", "update", "-g", "Amazon.Lambda.Tools"]) except DotnetCLIExecutionError as ex: raise ActionFailedError("Error configuring the Amazon.Lambda.Tools .NET Core Global Tool: " + str(ex)) + class RunPackageAction(BaseAction): """ A Lambda Builder Action which builds the .NET Core project using the Amazon.Lambda.Tools .NET Core Global Tool """ - NAME = 'RunPackageAction' + NAME = "RunPackageAction" DESCRIPTION = "Execute the `dotnet lambda package` command." PURPOSE = Purpose.COMPILE_SOURCE @@ -66,7 +64,7 @@ def execute(self): zipfilename = os.path.basename(os.path.normpath(self.source_dir)) + ".zip" zipfullpath = os.path.join(self.artifacts_dir, zipfilename) - arguments = ['lambda', 'package', '--output-package', zipfullpath] + arguments = ["lambda", "package", "--output-package", zipfullpath] if self.mode and self.mode.lower() == BuildMode.DEBUG: LOG.debug("Debug build requested: Setting configuration to Debug") @@ -78,10 +76,7 @@ def execute(self): arguments.append(key) arguments.append(self.options[key]) - self.subprocess_dotnet.run( - arguments, - cwd=self.source_dir - ) + self.subprocess_dotnet.run(arguments, cwd=self.source_dir) # The dotnet lambda package command outputs a zip file for the package. To make this compatible # with the workflow, unzip the zip file into the artifacts directory and then delete the zip archive. diff --git a/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py b/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py index 41e2c2d3b..b1fe104f1 100644 --- a/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py +++ b/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py @@ -9,6 +9,7 @@ LOG = logging.getLogger(__name__) + class DotnetCLIExecutionError(Exception): """ Exception raised when dotnet CLI fails. @@ -20,6 +21,7 @@ class DotnetCLIExecutionError(Exception): def __init__(self, **kwargs): Exception.__init__(self, self.MESSAGE.format(**kwargs)) + class SubprocessDotnetCLI(object): """ Wrapper around the Dotnet CLI, encapsulating @@ -30,34 +32,31 @@ def __init__(self, dotnet_exe=None, os_utils=None): self.os_utils = os_utils if os_utils else OSUtils() if dotnet_exe is None: if self.os_utils.is_windows(): - dotnet_exe = 'dotnet.exe' + dotnet_exe = "dotnet.exe" else: - dotnet_exe = 'dotnet' + dotnet_exe = "dotnet" self.dotnet_exe = dotnet_exe def run(self, args, cwd=None): if not isinstance(args, list): - raise ValueError('args must be a list') + raise ValueError("args must be a list") if not args: - raise ValueError('requires at least one arg') + raise ValueError("requires at least one arg") invoke_dotnet = [self.dotnet_exe] + args LOG.debug("executing dotnet: %s", invoke_dotnet) - p = self.os_utils.popen(invoke_dotnet, - stdout=self.os_utils.pipe, - stderr=self.os_utils.pipe, - cwd=cwd) + p = self.os_utils.popen(invoke_dotnet, stdout=self.os_utils.pipe, stderr=self.os_utils.pipe, cwd=cwd) out, err = p.communicate() # The package command contains lots of useful information on how the package was created and # information when the package command was not successful. For that reason the output is # always written to the output to help developers diagnose issues. - LOG.info(out.decode('utf8').strip()) + LOG.info(out.decode("utf8").strip()) if p.returncode != 0: - raise DotnetCLIExecutionError(message=err.decode('utf8').strip()) + raise DotnetCLIExecutionError(message=err.decode("utf8").strip()) diff --git a/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli_resolver.py b/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli_resolver.py index af7a7b980..d1db0a8d7 100644 --- a/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli_resolver.py +++ b/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli_resolver.py @@ -4,10 +4,10 @@ from .utils import OSUtils -class DotnetCliResolver(object): +class DotnetCliResolver(object): def __init__(self, executable_search_paths=None, os_utils=None): - self.binary = 'dotnet' + self.binary = "dotnet" self.executable_search_paths = executable_search_paths self.os_utils = os_utils if os_utils else OSUtils() @@ -15,10 +15,10 @@ def __init__(self, executable_search_paths=None, os_utils=None): def exec_paths(self): # look for the windows executable - paths = self.os_utils.which('dotnet.exe', executable_search_paths=self.executable_search_paths) + paths = self.os_utils.which("dotnet.exe", executable_search_paths=self.executable_search_paths) if not paths: # fallback to the non windows name without the .exe suffix - paths = self.os_utils.which('dotnet', executable_search_paths=self.executable_search_paths) + paths = self.os_utils.which("dotnet", executable_search_paths=self.executable_search_paths) if not paths: raise ValueError("No dotnet cli executable found!") diff --git a/aws_lambda_builders/workflows/dotnet_clipackage/utils.py b/aws_lambda_builders/workflows/dotnet_clipackage/utils.py index efd344f1a..0df1a3fcd 100644 --- a/aws_lambda_builders/workflows/dotnet_clipackage/utils.py +++ b/aws_lambda_builders/workflows/dotnet_clipackage/utils.py @@ -20,13 +20,13 @@ def popen(self, command, stdout=None, stderr=None, env=None, cwd=None): return p def is_windows(self): - return platform.system().lower() == 'windows' + return platform.system().lower() == "windows" def which(self, executable, executable_search_paths=None): return which(executable, executable_search_paths=executable_search_paths) - def expand_zip(self, zipfullpath,destination_dir): - ziparchive = zipfile.ZipFile(zipfullpath, 'r') + def expand_zip(self, zipfullpath, destination_dir): + ziparchive = zipfile.ZipFile(zipfullpath, "r") ziparchive.extractall(destination_dir) ziparchive.close() os.remove(zipfullpath) diff --git a/aws_lambda_builders/workflows/dotnet_clipackage/workflow.py b/aws_lambda_builders/workflows/dotnet_clipackage/workflow.py index 0c92462af..6756beffc 100644 --- a/aws_lambda_builders/workflows/dotnet_clipackage/workflow.py +++ b/aws_lambda_builders/workflows/dotnet_clipackage/workflow.py @@ -14,43 +14,25 @@ class DotnetCliPackageWorkflow(BaseWorkflow): """ A Lambda builder workflow that knows to build and package .NET Core Lambda functions """ + NAME = "DotnetCliPackageBuilder" - CAPABILITY = Capability(language="dotnet", - dependency_manager="cli-package", - application_framework=None) + CAPABILITY = Capability(language="dotnet", dependency_manager="cli-package", application_framework=None) - def __init__(self, - source_dir, - artifacts_dir, - scratch_dir, - manifest_path, - runtime=None, - mode=None, - **kwargs): + def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=None, mode=None, **kwargs): super(DotnetCliPackageWorkflow, self).__init__( - source_dir, - artifacts_dir, - scratch_dir, - manifest_path, - runtime=runtime, - mode=mode, - **kwargs) + source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=runtime, mode=mode, **kwargs + ) options = kwargs["options"] if "options" in kwargs else {} subprocess_dotnetcli = SubprocessDotnetCLI(os_utils=OSUtils()) dotnetcli_install = GlobalToolInstallAction(subprocess_dotnet=subprocess_dotnetcli) - dotnetcli_deployment = RunPackageAction(source_dir, - subprocess_dotnet=subprocess_dotnetcli, - artifacts_dir=artifacts_dir, - options=options, - mode=mode) - self.actions = [ - dotnetcli_install, - dotnetcli_deployment, - ] + dotnetcli_deployment = RunPackageAction( + source_dir, subprocess_dotnet=subprocess_dotnetcli, artifacts_dir=artifacts_dir, options=options, mode=mode + ) + self.actions = [dotnetcli_install, dotnetcli_deployment] def get_resolvers(self): return [DotnetCliResolver(executable_search_paths=self.executable_search_paths)] diff --git a/aws_lambda_builders/workflows/go_dep/actions.py b/aws_lambda_builders/workflows/go_dep/actions.py index 067aac58f..f424e7c0b 100644 --- a/aws_lambda_builders/workflows/go_dep/actions.py +++ b/aws_lambda_builders/workflows/go_dep/actions.py @@ -12,6 +12,7 @@ LOG = logging.getLogger(__name__) + class DepEnsureAction(BaseAction): """ @@ -30,11 +31,11 @@ def __init__(self, base_dir, subprocess_dep): def execute(self): try: - self.subprocess_dep.run(["ensure"], - cwd=self.base_dir) + self.subprocess_dep.run(["ensure"], cwd=self.base_dir) except ExecutionError as ex: raise ActionFailedError(str(ex)) + class GoBuildAction(BaseAction): """ @@ -60,7 +61,6 @@ def execute(self): env.update({"GOOS": "linux", "GOARCH": "amd64"}) try: - self.subprocess_go.run(["build", "-o", self.output_path, self.source_path], - cwd=self.source_path, env=env) + self.subprocess_go.run(["build", "-o", self.output_path, self.source_path], cwd=self.source_path, env=env) except ExecutionError as ex: raise ActionFailedError(str(ex)) diff --git a/aws_lambda_builders/workflows/go_dep/subproc_exec.py b/aws_lambda_builders/workflows/go_dep/subproc_exec.py index 1bb9f9746..ac7fd9239 100644 --- a/aws_lambda_builders/workflows/go_dep/subproc_exec.py +++ b/aws_lambda_builders/workflows/go_dep/subproc_exec.py @@ -18,13 +18,14 @@ class ExecutionError(Exception): def __init__(self, message): raw_message = message if isinstance(message, bytes): - message = message.decode('utf-8') + message = message.decode("utf-8") try: Exception.__init__(self, self.MESSAGE.format(message.strip())) except UnicodeError: Exception.__init__(self, self.MESSAGE.format(raw_message.strip())) + class SubprocessExec(object): """ @@ -45,7 +46,6 @@ def __init__(self, osutils, binary=None): self.binary = binary - def run(self, args, cwd=None, env=None): """ @@ -77,17 +77,13 @@ def run(self, args, cwd=None, env=None): LOG.debug("executing binary: %s", invoke_bin) - p = self.osutils.popen(invoke_bin, - stdout=self.osutils.pipe, - stderr=self.osutils.pipe, - cwd=cwd, - env=env) + p = self.osutils.popen(invoke_bin, stdout=self.osutils.pipe, stderr=self.osutils.pipe, cwd=cwd, env=env) out, err = p.communicate() if p.returncode != 0: raise ExecutionError(message=err) - out = out.decode('utf-8') if isinstance(out, bytes) else out + out = out.decode("utf-8") if isinstance(out, bytes) else out return out.strip() diff --git a/aws_lambda_builders/workflows/go_dep/utils.py b/aws_lambda_builders/workflows/go_dep/utils.py index 7c0d0052e..010a69827 100644 --- a/aws_lambda_builders/workflows/go_dep/utils.py +++ b/aws_lambda_builders/workflows/go_dep/utils.py @@ -39,4 +39,4 @@ def abspath(self, path): return os.path.abspath(path) def is_windows(self): - return platform.system().lower() == 'windows' + return platform.system().lower() == "windows" diff --git a/aws_lambda_builders/workflows/go_dep/workflow.py b/aws_lambda_builders/workflows/go_dep/workflow.py index c2f58ab14..7d1cd81e3 100644 --- a/aws_lambda_builders/workflows/go_dep/workflow.py +++ b/aws_lambda_builders/workflows/go_dep/workflow.py @@ -14,6 +14,7 @@ LOG = logging.getLogger(__name__) + class GoDepWorkflow(BaseWorkflow): """ A Lambda builder workflow that knows how to build @@ -22,27 +23,15 @@ class GoDepWorkflow(BaseWorkflow): NAME = "GoDepBuilder" - CAPABILITY = Capability(language="go", - dependency_manager="dep", - application_framework=None) + CAPABILITY = Capability(language="go", dependency_manager="dep", application_framework=None) - EXCLUDED_FILES = (".aws-sam") + EXCLUDED_FILES = ".aws-sam" - def __init__(self, - source_dir, - artifacts_dir, - scratch_dir, - manifest_path, - runtime=None, - osutils=None, - **kwargs): + def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=None, osutils=None, **kwargs): - super(GoDepWorkflow, self).__init__(source_dir, - artifacts_dir, - scratch_dir, - manifest_path, - runtime=runtime, - **kwargs) + super(GoDepWorkflow, self).__init__( + source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=runtime, **kwargs + ) options = kwargs["options"] if "options" in kwargs else {} handler = options.get("artifact_executable_name", None) @@ -59,5 +48,5 @@ def __init__(self, self.actions = [ DepEnsureAction(base_dir, subprocess_dep), - GoBuildAction(base_dir, osutils.abspath(source_dir), output_path, subprocess_go, env=osutils.environ) + GoBuildAction(base_dir, osutils.abspath(source_dir), output_path, subprocess_go, env=osutils.environ), ] diff --git a/aws_lambda_builders/workflows/go_modules/actions.py b/aws_lambda_builders/workflows/go_modules/actions.py index 39ddedc6c..1354a2704 100644 --- a/aws_lambda_builders/workflows/go_modules/actions.py +++ b/aws_lambda_builders/workflows/go_modules/actions.py @@ -19,9 +19,6 @@ def __init__(self, source_dir, output_path, builder): def execute(self): try: - self.builder.build( - self.source_dir, - self.output_path, - ) + self.builder.build(self.source_dir, self.output_path) except BuilderError as ex: raise ActionFailedError(str(ex)) diff --git a/aws_lambda_builders/workflows/go_modules/builder.py b/aws_lambda_builders/workflows/go_modules/builder.py index 35f649b26..38e220768 100644 --- a/aws_lambda_builders/workflows/go_modules/builder.py +++ b/aws_lambda_builders/workflows/go_modules/builder.py @@ -46,13 +46,7 @@ def build(self, source_dir_path, output_path): runtime_path = self.binaries[self.LANGUAGE].binary_path cmd = [runtime_path, "build", "-o", output_path, source_dir_path] - p = self.osutils.popen( - cmd, - cwd=source_dir_path, - env=env, - stdout=self.osutils.pipe, - stderr=self.osutils.pipe, - ) + p = self.osutils.popen(cmd, cwd=source_dir_path, env=env, stdout=self.osutils.pipe, stderr=self.osutils.pipe) out, err = p.communicate() if p.returncode != 0: diff --git a/aws_lambda_builders/workflows/go_modules/utils.py b/aws_lambda_builders/workflows/go_modules/utils.py index 2efc79aca..4e546c840 100644 --- a/aws_lambda_builders/workflows/go_modules/utils.py +++ b/aws_lambda_builders/workflows/go_modules/utils.py @@ -11,6 +11,7 @@ class OSUtils(object): Wrapper around file system functions, to make it easy to unit test actions in memory """ + @property def environ(self): return os.environ.copy() diff --git a/aws_lambda_builders/workflows/go_modules/validator.py b/aws_lambda_builders/workflows/go_modules/validator.py index 872701137..f4143bb91 100644 --- a/aws_lambda_builders/workflows/go_modules/validator.py +++ b/aws_lambda_builders/workflows/go_modules/validator.py @@ -14,9 +14,7 @@ class GoRuntimeValidator(object): LANGUAGE = "go" - SUPPORTED_RUNTIMES = { - "go1.x" - } + SUPPORTED_RUNTIMES = {"go1.x"} def __init__(self, runtime): self.runtime = runtime @@ -37,31 +35,26 @@ def validate(self, runtime_path): :raises MisMatchRuntimeError: Version mismatch of the language vs the required runtime """ if not self.has_runtime(): - LOG.warning("'%s' runtime is not " - "a supported runtime", self.runtime) + LOG.warning("'%s' runtime is not " "a supported runtime", self.runtime) return None - expected_major_version = int(self.runtime.replace(self.LANGUAGE, "").split('.')[0]) + expected_major_version = int(self.runtime.replace(self.LANGUAGE, "").split(".")[0]) min_expected_minor_version = 11 if expected_major_version == 1 else 0 - p = subprocess.Popen([runtime_path, "version"], - cwd=os.getcwd(), - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p = subprocess.Popen([runtime_path, "version"], cwd=os.getcwd(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, _ = p.communicate() if p.returncode == 0: out_parts = out.decode().split() if len(out_parts) >= 3: - version_parts = [int(x) for x in out_parts[2].replace(self.LANGUAGE, "").split('.')] + version_parts = [int(x) for x in out_parts[2].replace(self.LANGUAGE, "").split(".")] if len(version_parts) >= 2: if version_parts[0] == expected_major_version and version_parts[1] >= min_expected_minor_version: self._valid_runtime_path = runtime_path return self._valid_runtime_path # otherwise, raise mismatch exception - raise MisMatchRuntimeError(language=self.LANGUAGE, - required_runtime=self.runtime, - runtime_path=runtime_path) + raise MisMatchRuntimeError(language=self.LANGUAGE, required_runtime=self.runtime, runtime_path=runtime_path) @property def validated_runtime_path(self): diff --git a/aws_lambda_builders/workflows/go_modules/workflow.py b/aws_lambda_builders/workflows/go_modules/workflow.py index cbfd6d099..a80c1d65a 100644 --- a/aws_lambda_builders/workflows/go_modules/workflow.py +++ b/aws_lambda_builders/workflows/go_modules/workflow.py @@ -13,26 +13,13 @@ class GoModulesWorkflow(BaseWorkflow): NAME = "GoModulesBuilder" - CAPABILITY = Capability(language="go", - dependency_manager="modules", - application_framework=None) - - def __init__(self, - source_dir, - artifacts_dir, - scratch_dir, - manifest_path, - runtime=None, - osutils=None, - **kwargs): + CAPABILITY = Capability(language="go", dependency_manager="modules", application_framework=None) + + def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=None, osutils=None, **kwargs): super(GoModulesWorkflow, self).__init__( - source_dir, - artifacts_dir, - scratch_dir, - manifest_path, - runtime=runtime, - **kwargs) + source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=runtime, **kwargs + ) if osutils is None: osutils = OSUtils() @@ -43,9 +30,7 @@ def __init__(self, output_path = osutils.joinpath(artifacts_dir, handler) builder = GoModulesBuilder(osutils, binaries=self.binaries) - self.actions = [ - GoModulesBuildAction(source_dir, output_path, builder), - ] + self.actions = [GoModulesBuildAction(source_dir, output_path, builder)] def get_validators(self): return [GoRuntimeValidator(runtime=self.runtime)] diff --git a/aws_lambda_builders/workflows/java_gradle/actions.py b/aws_lambda_builders/workflows/java_gradle/actions.py index 3108c8aff..ddf8ded9c 100644 --- a/aws_lambda_builders/workflows/java_gradle/actions.py +++ b/aws_lambda_builders/workflows/java_gradle/actions.py @@ -12,16 +12,11 @@ class JavaGradleBuildAction(BaseAction): DESCRIPTION = "Building the project using Gradle" PURPOSE = Purpose.COMPILE_SOURCE - INIT_SCRIPT = 'lambda-build-init.gradle' - SCRATCH_DIR_PROPERTY = 'software.amazon.aws.lambdabuilders.scratch-dir' - GRADLE_CACHE_DIR_NAME = 'gradle-cache' + INIT_SCRIPT = "lambda-build-init.gradle" + SCRATCH_DIR_PROPERTY = "software.amazon.aws.lambdabuilders.scratch-dir" + GRADLE_CACHE_DIR_NAME = "gradle-cache" - def __init__(self, - source_dir, - build_file, - subprocess_gradle, - scratch_dir, - os_utils): + def __init__(self, source_dir, build_file, subprocess_gradle, scratch_dir, os_utils): self.source_dir = source_dir self.build_file = build_file self.scratch_dir = scratch_dir @@ -39,7 +34,7 @@ def gradle_cache_dir(self): def _copy_init_script(self): try: - src = os.path.join(os.path.dirname(__file__), 'resources', self.INIT_SCRIPT) + src = os.path.join(os.path.dirname(__file__), "resources", self.INIT_SCRIPT) dst = os.path.join(self.scratch_dir, self.INIT_SCRIPT) return self.os_utils.copy(src, dst) except Exception as ex: @@ -49,9 +44,13 @@ def _build_project(self, init_script_file): try: if not self.os_utils.exists(self.scratch_dir): self.os_utils.makedirs(self.scratch_dir) - self.subprocess_gradle.build(self.source_dir, self.build_file, self.gradle_cache_dir, - init_script_file, - {self.SCRATCH_DIR_PROPERTY: os.path.abspath(self.scratch_dir)}) + self.subprocess_gradle.build( + self.source_dir, + self.build_file, + self.gradle_cache_dir, + init_script_file, + {self.SCRATCH_DIR_PROPERTY: os.path.abspath(self.scratch_dir)}, + ) except GradleExecutionError as ex: raise ActionFailedError(str(ex)) @@ -61,11 +60,7 @@ class JavaGradleCopyArtifactsAction(BaseAction): DESCRIPTION = "Copying the built artifacts" PURPOSE = Purpose.COPY_SOURCE - def __init__(self, - source_dir, - artifacts_dir, - build_dir, - os_utils): + def __init__(self, source_dir, artifacts_dir, build_dir, os_utils): self.source_dir = source_dir self.artifacts_dir = artifacts_dir self.build_dir = build_dir @@ -75,7 +70,7 @@ def execute(self): self._copy_artifacts() def _copy_artifacts(self): - lambda_build_output = os.path.join(self.build_dir, 'build', 'distributions', 'lambda-build') + lambda_build_output = os.path.join(self.build_dir, "build", "distributions", "lambda-build") try: if not self.os_utils.exists(self.artifacts_dir): self.os_utils.makedirs(self.artifacts_dir) diff --git a/aws_lambda_builders/workflows/java_gradle/gradle.py b/aws_lambda_builders/workflows/java_gradle/gradle.py index e1222ba99..ffdf38c4d 100644 --- a/aws_lambda_builders/workflows/java_gradle/gradle.py +++ b/aws_lambda_builders/workflows/java_gradle/gradle.py @@ -17,12 +17,10 @@ def __init__(self, **kwargs): class BuildFileNotFoundError(GradleExecutionError): def __init__(self, build_file_path): - super(BuildFileNotFoundError, self).__init__( - message='Gradle build file not found: %s' % build_file_path) + super(BuildFileNotFoundError, self).__init__(message="Gradle build file not found: %s" % build_file_path) class SubprocessGradle(object): - def __init__(self, gradle_binary, os_utils=None): if gradle_binary is None: raise ValueError("Must provide Gradle BinaryPath") @@ -35,19 +33,20 @@ def build(self, source_dir, build_file, cache_dir=None, init_script_path=None, p if not self.os_utils.exists(build_file): raise BuildFileNotFoundError(build_file) - args = ['build', '--build-file', build_file] + args = ["build", "--build-file", build_file] if cache_dir is not None: - args.extend(['--project-cache-dir', cache_dir]) + args.extend(["--project-cache-dir", cache_dir]) if properties is not None: - args.extend(['-D%s=%s' % (n, v) for n, v in properties.items()]) + args.extend(["-D%s=%s" % (n, v) for n, v in properties.items()]) if init_script_path is not None: - args.extend(['--init-script', init_script_path]) + args.extend(["--init-script", init_script_path]) ret_code, _, stderr = self._run(args, source_dir) if ret_code != 0: - raise GradleExecutionError(message=stderr.decode('utf8').strip()) + raise GradleExecutionError(message=stderr.decode("utf8").strip()) def _run(self, args, cwd=None): - p = self.os_utils.popen([self.gradle_binary.binary_path] + args, cwd=cwd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + p = self.os_utils.popen( + [self.gradle_binary.binary_path] + args, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) stdout, stderr = p.communicate() return p.returncode, stdout, stderr diff --git a/aws_lambda_builders/workflows/java_gradle/gradle_resolver.py b/aws_lambda_builders/workflows/java_gradle/gradle_resolver.py index 7827d5083..f629c993f 100644 --- a/aws_lambda_builders/workflows/java_gradle/gradle_resolver.py +++ b/aws_lambda_builders/workflows/java_gradle/gradle_resolver.py @@ -6,9 +6,8 @@ class GradleResolver(object): - def __init__(self, executable_search_paths=None, os_utils=None): - self.binary = 'gradle' + self.binary = "gradle" self.executables = [self.binary] self.executable_search_paths = executable_search_paths self.os_utils = os_utils if os_utils else OSUtils() @@ -19,7 +18,7 @@ def exec_paths(self): paths = self.os_utils.which(self.wrapper_name, executable_search_paths=self.executable_search_paths) if not paths: # fallback to the gradle binary - paths = self.os_utils.which('gradle', executable_search_paths=self.executable_search_paths) + paths = self.os_utils.which("gradle", executable_search_paths=self.executable_search_paths) if not paths: raise ValueError("No Gradle executable found!") @@ -28,4 +27,4 @@ def exec_paths(self): @property def wrapper_name(self): - return 'gradlew.bat' if self.os_utils.is_windows() else 'gradlew' + return "gradlew.bat" if self.os_utils.is_windows() else "gradlew" diff --git a/aws_lambda_builders/workflows/java_gradle/gradle_validator.py b/aws_lambda_builders/workflows/java_gradle/gradle_validator.py index 95d734c1e..36dcb4817 100644 --- a/aws_lambda_builders/workflows/java_gradle/gradle_validator.py +++ b/aws_lambda_builders/workflows/java_gradle/gradle_validator.py @@ -11,15 +11,19 @@ class GradleValidator(object): - VERSION_STRING_WARNING = "%s failed to return a version string using the '-v' option. The workflow is unable to " \ - "check that the version of the JVM used is compatible with AWS Lambda." + VERSION_STRING_WARNING = ( + "%s failed to return a version string using the '-v' option. The workflow is unable to " + "check that the version of the JVM used is compatible with AWS Lambda." + ) - MAJOR_VERSION_WARNING = "%s is using a JVM with major version %s which is newer than %s that is supported by AWS " \ - "Lambda. The compiled function code may not run in AWS Lambda unless the project has " \ - "been configured to be compatible with Java %s using 'targetCompatibility' in Gradle." + MAJOR_VERSION_WARNING = ( + "%s is using a JVM with major version %s which is newer than %s that is supported by AWS " + "Lambda. The compiled function code may not run in AWS Lambda unless the project has " + "been configured to be compatible with Java %s using 'targetCompatibility' in Gradle." + ) def __init__(self, runtime, os_utils=None, log=None): - self.language = 'java' + self.language = "java" self._valid_binary_path = None self._runtime = runtime self.os_utils = OSUtils() if not os_utils else os_utils @@ -28,7 +32,7 @@ def __init__(self, runtime, os_utils=None, log=None): def validate(self, gradle_path): jvm_mv = self._get_major_version(gradle_path) - language_version = self._runtime.replace('java', '') + language_version = self._runtime.replace("java", "") if jvm_mv: if int(jvm_mv) > int(language_version): @@ -46,16 +50,16 @@ def validated_binary_path(self): def _get_major_version(self, gradle_path): vs = self._get_jvm_string(gradle_path) if vs: - m = re.search(r'JVM:\s+([\d\.]+)', vs) - version = m.group(1).split('.') + m = re.search(r"JVM:\s+([\d\.]+)", vs) + version = m.group(1).split(".") # For Java 8 or earlier, version strings begin with 1.{Major Version} - if version[0] == '1': + if version[0] == "1": return version[1] # Starting with Java 9, the major version is first return version[0] def _get_jvm_string(self, gradle_path): - p = self.os_utils.popen([gradle_path, '-version'], stdout=self.os_utils.pipe, stderr=self.os_utils.pipe) + p = self.os_utils.popen([gradle_path, "-version"], stdout=self.os_utils.pipe, stderr=self.os_utils.pipe) stdout, _ = p.communicate() if p.returncode != 0: @@ -63,5 +67,5 @@ def _get_jvm_string(self, gradle_path): for l in stdout.splitlines(): l_dec = l.decode() - if l_dec.startswith('JVM'): + if l_dec.startswith("JVM"): return l_dec diff --git a/aws_lambda_builders/workflows/java_gradle/utils.py b/aws_lambda_builders/workflows/java_gradle/utils.py index 2e178a320..a61a7bee6 100644 --- a/aws_lambda_builders/workflows/java_gradle/utils.py +++ b/aws_lambda_builders/workflows/java_gradle/utils.py @@ -19,7 +19,7 @@ def popen(self, command, stdout=None, stderr=None, env=None, cwd=None): return p def is_windows(self): - return platform.system().lower() == 'windows' + return platform.system().lower() == "windows" def copy(self, src, dst): shutil.copy2(src, dst) diff --git a/aws_lambda_builders/workflows/java_gradle/workflow.py b/aws_lambda_builders/workflows/java_gradle/workflow.py index c68b93ab5..e28c06c94 100644 --- a/aws_lambda_builders/workflows/java_gradle/workflow.py +++ b/aws_lambda_builders/workflows/java_gradle/workflow.py @@ -15,40 +15,23 @@ class JavaGradleWorkflow(BaseWorkflow): """ A Lambda builder workflow that knows how to build Java projects using Gradle. """ + NAME = "JavaGradleWorkflow" - CAPABILITY = Capability(language="java", - dependency_manager="gradle", - application_framework=None) + CAPABILITY = Capability(language="java", dependency_manager="gradle", application_framework=None) INIT_FILE = "lambda-build-init.gradle" - def __init__(self, - source_dir, - artifacts_dir, - scratch_dir, - manifest_path, - **kwargs): - super(JavaGradleWorkflow, self).__init__(source_dir, - artifacts_dir, - scratch_dir, - manifest_path, - **kwargs) + def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, **kwargs): + super(JavaGradleWorkflow, self).__init__(source_dir, artifacts_dir, scratch_dir, manifest_path, **kwargs) self.os_utils = OSUtils() self.build_dir = None - subprocess_gradle = SubprocessGradle(gradle_binary=self.binaries['gradle'], os_utils=self.os_utils) + subprocess_gradle = SubprocessGradle(gradle_binary=self.binaries["gradle"], os_utils=self.os_utils) self.actions = [ - JavaGradleBuildAction(source_dir, - manifest_path, - subprocess_gradle, - scratch_dir, - self.os_utils), - JavaGradleCopyArtifactsAction(source_dir, - artifacts_dir, - self.build_output_dir, - self.os_utils) + JavaGradleBuildAction(source_dir, manifest_path, subprocess_gradle, scratch_dir, self.os_utils), + JavaGradleCopyArtifactsAction(source_dir, artifacts_dir, self.build_output_dir, self.os_utils), ] def get_resolvers(self): @@ -71,5 +54,5 @@ def _compute_scratch_subdir(self): :return: The path of the buildDir used for building the lambda. """ sha1 = hashlib.sha1() - sha1.update(os.path.abspath(self.source_dir).encode('utf8')) + sha1.update(os.path.abspath(self.source_dir).encode("utf8")) return sha1.hexdigest() diff --git a/aws_lambda_builders/workflows/java_maven/actions.py b/aws_lambda_builders/workflows/java_maven/actions.py index e81804d42..169459cda 100644 --- a/aws_lambda_builders/workflows/java_maven/actions.py +++ b/aws_lambda_builders/workflows/java_maven/actions.py @@ -15,9 +15,8 @@ class JavaMavenBaseAction(object): """ Base class for Java Maven actions. Provides property of the module name """ - def __init__(self, - scratch_dir, - subprocess_maven): + + def __init__(self, scratch_dir, subprocess_maven): self.scratch_dir = scratch_dir self.subprocess_maven = subprocess_maven @@ -27,11 +26,8 @@ class JavaMavenBuildAction(JavaMavenBaseAction, BaseAction): DESCRIPTION = "Building the project using Maven" PURPOSE = Purpose.COMPILE_SOURCE - def __init__(self, - scratch_dir, - subprocess_maven): - super(JavaMavenBuildAction, self).__init__(scratch_dir, - subprocess_maven) + def __init__(self, scratch_dir, subprocess_maven): + super(JavaMavenBuildAction, self).__init__(scratch_dir, subprocess_maven) self.scratch_dir = scratch_dir self.subprocess_maven = subprocess_maven @@ -47,11 +43,8 @@ class JavaMavenCopyDependencyAction(JavaMavenBaseAction, BaseAction): DESCRIPTION = "Copy dependency jars to target directory" PURPOSE = Purpose.COPY_SOURCE - def __init__(self, - scratch_dir, - subprocess_maven): - super(JavaMavenCopyDependencyAction, self).__init__(scratch_dir, - subprocess_maven) + def __init__(self, scratch_dir, subprocess_maven): + super(JavaMavenCopyDependencyAction, self).__init__(scratch_dir, subprocess_maven) self.scratch_dir = scratch_dir self.subprocess_maven = subprocess_maven @@ -67,10 +60,7 @@ class JavaMavenCopyArtifactsAction(BaseAction): DESCRIPTION = "Copying the built artifacts" PURPOSE = Purpose.COPY_SOURCE - def __init__(self, - scratch_dir, - artifacts_dir, - os_utils): + def __init__(self, scratch_dir, artifacts_dir, os_utils): self.scratch_dir = scratch_dir self.artifacts_dir = artifacts_dir self.os_utils = os_utils @@ -79,8 +69,8 @@ def execute(self): self._copy_artifacts() def _copy_artifacts(self): - lambda_build_output = os.path.join(self.scratch_dir, 'target', 'classes') - dependency_output = os.path.join(self.scratch_dir, 'target', 'dependency') + lambda_build_output = os.path.join(self.scratch_dir, "target", "classes") + dependency_output = os.path.join(self.scratch_dir, "target", "dependency") if not self.os_utils.exists(lambda_build_output): raise ActionFailedError("Required target/classes directory was not produced from 'mvn package'") @@ -88,6 +78,6 @@ def _copy_artifacts(self): try: self.os_utils.copytree(lambda_build_output, self.artifacts_dir) if self.os_utils.exists(dependency_output): - self.os_utils.copytree(dependency_output, os.path.join(self.artifacts_dir, 'lib')) + self.os_utils.copytree(dependency_output, os.path.join(self.artifacts_dir, "lib")) except Exception as ex: raise ActionFailedError(str(ex)) diff --git a/aws_lambda_builders/workflows/java_maven/maven.py b/aws_lambda_builders/workflows/java_maven/maven.py index aa6f55ad0..2f10a32e2 100644 --- a/aws_lambda_builders/workflows/java_maven/maven.py +++ b/aws_lambda_builders/workflows/java_maven/maven.py @@ -16,7 +16,6 @@ def __init__(self, **kwargs): class SubprocessMaven(object): - def __init__(self, maven_binary, os_utils=None): if maven_binary is None: raise ValueError("Must provide Maven BinaryPath") @@ -26,23 +25,24 @@ def __init__(self, maven_binary, os_utils=None): self.os_utils = os_utils def build(self, scratch_dir): - args = ['clean', 'install'] + args = ["clean", "install"] ret_code, stdout, _ = self._run(args, scratch_dir) - LOG.debug("Maven logs: %s", stdout.decode('utf8').strip()) + LOG.debug("Maven logs: %s", stdout.decode("utf8").strip()) if ret_code != 0: - raise MavenExecutionError(message=stdout.decode('utf8').strip()) + raise MavenExecutionError(message=stdout.decode("utf8").strip()) def copy_dependency(self, scratch_dir): - args = ['dependency:copy-dependencies', '-DincludeScope=compile'] + args = ["dependency:copy-dependencies", "-DincludeScope=compile"] ret_code, stdout, _ = self._run(args, scratch_dir) if ret_code != 0: - raise MavenExecutionError(message=stdout.decode('utf8').strip()) + raise MavenExecutionError(message=stdout.decode("utf8").strip()) def _run(self, args, cwd=None): - p = self.os_utils.popen([self.maven_binary.binary_path] + args, cwd=cwd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + p = self.os_utils.popen( + [self.maven_binary.binary_path] + args, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) stdout, stderr = p.communicate() return p.returncode, stdout, stderr diff --git a/aws_lambda_builders/workflows/java_maven/maven_resolver.py b/aws_lambda_builders/workflows/java_maven/maven_resolver.py index fdb42a9fe..e26e965ec 100644 --- a/aws_lambda_builders/workflows/java_maven/maven_resolver.py +++ b/aws_lambda_builders/workflows/java_maven/maven_resolver.py @@ -6,16 +6,15 @@ class MavenResolver(object): - def __init__(self, executable_search_paths=None, os_utils=None): - self.binary = 'mvn' + self.binary = "mvn" self.executables = [self.binary] self.executable_search_paths = executable_search_paths self.os_utils = os_utils if os_utils else OSUtils() @property def exec_paths(self): - paths = self.os_utils.which('mvn', executable_search_paths=self.executable_search_paths) + paths = self.os_utils.which("mvn", executable_search_paths=self.executable_search_paths) if not paths: raise ValueError("No Maven executable found!") diff --git a/aws_lambda_builders/workflows/java_maven/maven_validator.py b/aws_lambda_builders/workflows/java_maven/maven_validator.py index 6b01d1412..ad69eb7ec 100644 --- a/aws_lambda_builders/workflows/java_maven/maven_validator.py +++ b/aws_lambda_builders/workflows/java_maven/maven_validator.py @@ -11,15 +11,19 @@ class MavenValidator(object): - VERSION_STRING_WARNING = "%s failed to return a version string using the '-v' option. The workflow is unable to " \ - "check that the version of the JVM used is compatible with AWS Lambda." + VERSION_STRING_WARNING = ( + "%s failed to return a version string using the '-v' option. The workflow is unable to " + "check that the version of the JVM used is compatible with AWS Lambda." + ) - MAJOR_VERSION_WARNING = "%s is using a JVM with major version %s which is newer than %s that is supported by AWS " \ - "Lambda. The compiled function code may not run in AWS Lambda unless the project has " \ - "been configured to be compatible with Java %s using 'maven.compiler.target' in Maven." + MAJOR_VERSION_WARNING = ( + "%s is using a JVM with major version %s which is newer than %s that is supported by AWS " + "Lambda. The compiled function code may not run in AWS Lambda unless the project has " + "been configured to be compatible with Java %s using 'maven.compiler.target' in Maven." + ) def __init__(self, runtime, os_utils=None, log=None): - self.language = 'java' + self.language = "java" self._valid_binary_path = None self._runtime = runtime self.os_utils = OSUtils() if not os_utils else os_utils @@ -28,7 +32,7 @@ def __init__(self, runtime, os_utils=None, log=None): def validate(self, maven_path): jvm_mv = self._get_major_version(maven_path) - language_version = self._runtime.replace('java', '') + language_version = self._runtime.replace("java", "") if jvm_mv: if int(jvm_mv) > int(language_version): @@ -46,16 +50,16 @@ def validated_binary_path(self): def _get_major_version(self, maven_path): vs = self._get_jvm_string(maven_path) if vs: - m = re.search(r'Java version:\s+([\d\.]+)', vs) - version = m.group(1).split('.') + m = re.search(r"Java version:\s+([\d\.]+)", vs) + version = m.group(1).split(".") # For Java 8 or earlier, version strings begin with 1.{Major Version} - if version[0] == '1': + if version[0] == "1": return version[1] # Starting with Java 9, the major version is first return version[0] def _get_jvm_string(self, maven_path): - p = self.os_utils.popen([maven_path, '-version'], stdout=self.os_utils.pipe, stderr=self.os_utils.pipe) + p = self.os_utils.popen([maven_path, "-version"], stdout=self.os_utils.pipe, stderr=self.os_utils.pipe) stdout, _ = p.communicate() if p.returncode != 0: @@ -63,5 +67,5 @@ def _get_jvm_string(self, maven_path): for l in stdout.splitlines(): l_dec = l.decode() - if l_dec.startswith('Java version'): + if l_dec.startswith("Java version"): return l_dec diff --git a/aws_lambda_builders/workflows/java_maven/utils.py b/aws_lambda_builders/workflows/java_maven/utils.py index 2e178a320..a61a7bee6 100644 --- a/aws_lambda_builders/workflows/java_maven/utils.py +++ b/aws_lambda_builders/workflows/java_maven/utils.py @@ -19,7 +19,7 @@ def popen(self, command, stdout=None, stderr=None, env=None, cwd=None): return p def is_windows(self): - return platform.system().lower() == 'windows' + return platform.system().lower() == "windows" def copy(self, src, dst): shutil.copy2(src, dst) diff --git a/aws_lambda_builders/workflows/java_maven/workflow.py b/aws_lambda_builders/workflows/java_maven/workflow.py index d7ec69f0e..6586ee4f0 100644 --- a/aws_lambda_builders/workflows/java_maven/workflow.py +++ b/aws_lambda_builders/workflows/java_maven/workflow.py @@ -14,41 +14,26 @@ class JavaMavenWorkflow(BaseWorkflow): """ A Lambda builder workflow that knows how to build Java projects using Maven. """ + NAME = "JavaMavenWorkflow" - CAPABILITY = Capability(language="java", - dependency_manager="maven", - application_framework=None) + CAPABILITY = Capability(language="java", dependency_manager="maven", application_framework=None) EXCLUDED_FILES = (".aws-sam", ".git") - def __init__(self, - source_dir, - artifacts_dir, - scratch_dir, - manifest_path, - **kwargs): - super(JavaMavenWorkflow, self).__init__(source_dir, - artifacts_dir, - scratch_dir, - manifest_path, - **kwargs) + def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, **kwargs): + super(JavaMavenWorkflow, self).__init__(source_dir, artifacts_dir, scratch_dir, manifest_path, **kwargs) self.os_utils = OSUtils() # Assuming root_dir is the same as source_dir for now root_dir = source_dir - subprocess_maven = SubprocessMaven(maven_binary=self.binaries['mvn'], os_utils=self.os_utils) + subprocess_maven = SubprocessMaven(maven_binary=self.binaries["mvn"], os_utils=self.os_utils) self.actions = [ CopySourceAction(root_dir, scratch_dir, excludes=self.EXCLUDED_FILES), - - JavaMavenBuildAction(scratch_dir, - subprocess_maven), - JavaMavenCopyDependencyAction(scratch_dir, - subprocess_maven), - JavaMavenCopyArtifactsAction(scratch_dir, - artifacts_dir, - self.os_utils) + JavaMavenBuildAction(scratch_dir, subprocess_maven), + JavaMavenCopyDependencyAction(scratch_dir, subprocess_maven), + JavaMavenCopyArtifactsAction(scratch_dir, artifacts_dir, self.os_utils), ] def get_resolvers(self): diff --git a/aws_lambda_builders/workflows/nodejs_npm/actions.py b/aws_lambda_builders/workflows/nodejs_npm/actions.py index ad6e7a927..5ba7bec3a 100644 --- a/aws_lambda_builders/workflows/nodejs_npm/actions.py +++ b/aws_lambda_builders/workflows/nodejs_npm/actions.py @@ -16,7 +16,7 @@ class NodejsNpmPackAction(BaseAction): A Lambda Builder Action that packages a Node.js package using NPM to extract the source and remove test resources """ - NAME = 'NpmPack' + NAME = "NpmPack" DESCRIPTION = "Packaging source using NPM" PURPOSE = Purpose.COPY_SOURCE @@ -56,7 +56,7 @@ def execute(self): LOG.debug("NODEJS packaging %s to %s", package_path, self.scratch_dir) - tarfile_name = self.subprocess_npm.run(['pack', '-q', package_path], cwd=self.scratch_dir) + tarfile_name = self.subprocess_npm.run(["pack", "-q", package_path], cwd=self.scratch_dir) LOG.debug("NODEJS packed to %s", tarfile_name) @@ -76,7 +76,7 @@ class NodejsNpmInstallAction(BaseAction): A Lambda Builder Action that installs NPM project dependencies """ - NAME = 'NpmInstall' + NAME = "NpmInstall" DESCRIPTION = "Installing dependencies from NPM" PURPOSE = Purpose.RESOLVE_DEPENDENCIES @@ -105,20 +105,20 @@ def execute(self): LOG.debug("NODEJS installing in: %s", self.artifacts_dir) self.subprocess_npm.run( - ['install', '-q', '--no-audit', '--no-save', '--production'], - cwd=self.artifacts_dir + ["install", "-q", "--no-audit", "--no-save", "--production"], cwd=self.artifacts_dir ) except NpmExecutionError as ex: raise ActionFailedError(str(ex)) + class NodejsNpmrcCopyAction(BaseAction): """ A Lambda Builder Action that copies NPM config file .npmrc """ - NAME = 'CopyNpmrc' + NAME = "CopyNpmrc" DESCRIPTION = "Copying configuration from .npmrc" PURPOSE = Purpose.COPY_SOURCE @@ -156,13 +156,14 @@ def execute(self): except OSError as ex: raise ActionFailedError(str(ex)) + class NodejsNpmrcCleanUpAction(BaseAction): """ A Lambda Builder Action that cleans NPM config file .npmrc """ - NAME = 'CleanUpNpmrc' + NAME = "CleanUpNpmrc" DESCRIPTION = "Cleans artifacts dir" PURPOSE = Purpose.COPY_SOURCE diff --git a/aws_lambda_builders/workflows/nodejs_npm/npm.py b/aws_lambda_builders/workflows/nodejs_npm/npm.py index b737ff4d1..0515b1809 100644 --- a/aws_lambda_builders/workflows/nodejs_npm/npm.py +++ b/aws_lambda_builders/workflows/nodejs_npm/npm.py @@ -40,9 +40,9 @@ def __init__(self, osutils, npm_exe=None): if npm_exe is None: if osutils.is_windows(): - npm_exe = 'npm.cmd' + npm_exe = "npm.cmd" else: - npm_exe = 'npm' + npm_exe = "npm" self.npm_exe = npm_exe @@ -68,23 +68,20 @@ def run(self, args, cwd=None): """ if not isinstance(args, list): - raise ValueError('args must be a list') + raise ValueError("args must be a list") if not args: - raise ValueError('requires at least one arg') + raise ValueError("requires at least one arg") invoke_npm = [self.npm_exe] + args LOG.debug("executing NPM: %s", invoke_npm) - p = self.osutils.popen(invoke_npm, - stdout=self.osutils.pipe, - stderr=self.osutils.pipe, - cwd=cwd) + p = self.osutils.popen(invoke_npm, stdout=self.osutils.pipe, stderr=self.osutils.pipe, cwd=cwd) out, err = p.communicate() if p.returncode != 0: - raise NpmExecutionError(message=err.decode('utf8').strip()) + raise NpmExecutionError(message=err.decode("utf8").strip()) - return out.decode('utf8').strip() + return out.decode("utf8").strip() diff --git a/aws_lambda_builders/workflows/nodejs_npm/utils.py b/aws_lambda_builders/workflows/nodejs_npm/utils.py index b34863788..ad92cfd23 100644 --- a/aws_lambda_builders/workflows/nodejs_npm/utils.py +++ b/aws_lambda_builders/workflows/nodejs_npm/utils.py @@ -20,7 +20,7 @@ def copy_file(self, file_path, destination_path): return shutil.copy2(file_path, destination_path) def extract_tarfile(self, tarfile_path, unpack_dir): - with tarfile.open(tarfile_path, 'r:*') as tar: + with tarfile.open(tarfile_path, "r:*") as tar: tar.extractall(unpack_dir) def file_exists(self, filename): @@ -47,4 +47,4 @@ def abspath(self, path): return os.path.abspath(path) def is_windows(self): - return platform.system().lower() == 'windows' + return platform.system().lower() == "windows" diff --git a/aws_lambda_builders/workflows/nodejs_npm/workflow.py b/aws_lambda_builders/workflows/nodejs_npm/workflow.py index dc6be8ea4..1873d366d 100644 --- a/aws_lambda_builders/workflows/nodejs_npm/workflow.py +++ b/aws_lambda_builders/workflows/nodejs_npm/workflow.py @@ -15,46 +15,32 @@ class NodejsNpmWorkflow(BaseWorkflow): A Lambda builder workflow that knows how to pack NodeJS projects using NPM. """ + NAME = "NodejsNpmBuilder" - CAPABILITY = Capability(language="nodejs", - dependency_manager="npm", - application_framework=None) + CAPABILITY = Capability(language="nodejs", dependency_manager="npm", application_framework=None) - EXCLUDED_FILES = (".aws-sam") + EXCLUDED_FILES = ".aws-sam" - def __init__(self, - source_dir, - artifacts_dir, - scratch_dir, - manifest_path, - runtime=None, - osutils=None, - **kwargs): + def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=None, osutils=None, **kwargs): - super(NodejsNpmWorkflow, self).__init__(source_dir, - artifacts_dir, - scratch_dir, - manifest_path, - runtime=runtime, - **kwargs) + super(NodejsNpmWorkflow, self).__init__( + source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=runtime, **kwargs + ) if osutils is None: osutils = OSUtils() subprocess_npm = SubprocessNpm(osutils) - tar_dest_dir = osutils.joinpath(scratch_dir, 'unpacked') - tar_package_dir = osutils.joinpath(tar_dest_dir, 'package') + tar_dest_dir = osutils.joinpath(scratch_dir, "unpacked") + tar_package_dir = osutils.joinpath(tar_dest_dir, "package") - npm_pack = NodejsNpmPackAction(tar_dest_dir, - scratch_dir, - manifest_path, - osutils=osutils, - subprocess_npm=subprocess_npm) + npm_pack = NodejsNpmPackAction( + tar_dest_dir, scratch_dir, manifest_path, osutils=osutils, subprocess_npm=subprocess_npm + ) - npm_install = NodejsNpmInstallAction(artifacts_dir, - subprocess_npm=subprocess_npm) + npm_install = NodejsNpmInstallAction(artifacts_dir, subprocess_npm=subprocess_npm) npm_copy_npmrc = NodejsNpmrcCopyAction(tar_package_dir, source_dir, osutils=osutils) @@ -63,7 +49,7 @@ def __init__(self, npm_copy_npmrc, CopySourceAction(tar_package_dir, artifacts_dir, excludes=self.EXCLUDED_FILES), npm_install, - NodejsNpmrcCleanUpAction(artifacts_dir, osutils=osutils) + NodejsNpmrcCleanUpAction(artifacts_dir, osutils=osutils), ] def get_resolvers(self): diff --git a/aws_lambda_builders/workflows/python_pip/actions.py b/aws_lambda_builders/workflows/python_pip/actions.py index 7886edc65..3c21acf53 100644 --- a/aws_lambda_builders/workflows/python_pip/actions.py +++ b/aws_lambda_builders/workflows/python_pip/actions.py @@ -9,10 +9,10 @@ class PythonPipBuildAction(BaseAction): - NAME = 'ResolveDependencies' + NAME = "ResolveDependencies" DESCRIPTION = "Installing dependencies from PIP" PURPOSE = Purpose.RESOLVE_DEPENDENCIES - LANGUAGE = 'python' + LANGUAGE = "python" def __init__(self, artifacts_dir, scratch_dir, manifest_path, runtime, binaries): self.artifacts_dir = artifacts_dir @@ -26,17 +26,16 @@ def execute(self): python_path = self.binaries[self.LANGUAGE].binary_path pip = SubprocessPip(osutils=os_utils, python_exe=python_path) pip_runner = PipRunner(python_exe=python_path, pip=pip) - dependency_builder = DependencyBuilder(osutils=os_utils, pip_runner=pip_runner, - runtime=self.runtime) + dependency_builder = DependencyBuilder(osutils=os_utils, pip_runner=pip_runner, runtime=self.runtime) - package_builder = PythonPipDependencyBuilder(osutils=os_utils, - runtime=self.runtime, - dependency_builder=dependency_builder) + package_builder = PythonPipDependencyBuilder( + osutils=os_utils, runtime=self.runtime, dependency_builder=dependency_builder + ) try: package_builder.build_dependencies( artifacts_dir_path=self.artifacts_dir, scratch_dir_path=self.scratch_dir, - requirements_path=self.manifest_path + requirements_path=self.manifest_path, ) except PackagerError as ex: raise ActionFailedError(str(ex)) diff --git a/aws_lambda_builders/workflows/python_pip/compat.py b/aws_lambda_builders/workflows/python_pip/compat.py index b3e7c7281..a6c3e0814 100644 --- a/aws_lambda_builders/workflows/python_pip/compat.py +++ b/aws_lambda_builders/workflows/python_pip/compat.py @@ -5,31 +5,27 @@ def pip_import_string(python_exe): os_utils = OSUtils() - cmd = [ - python_exe, - "-c", - "import pip; print(pip.__version__)" - ] + cmd = [python_exe, "-c", "import pip; print(pip.__version__)"] p = os_utils.popen(cmd, stdout=os_utils.pipe, stderr=os_utils.pipe) stdout, stderr = p.communicate() - pip_version = stdout.decode('utf-8').strip() - pip_major_version = int(pip_version.split('.')[0]) - pip_minor_version = int(pip_version.split('.')[1]) + pip_version = stdout.decode("utf-8").strip() + pip_major_version = int(pip_version.split(".")[0]) + pip_minor_version = int(pip_version.split(".")[1]) # Pip moved its internals to an _internal module in version 10. # In order to be compatible with version 9 which has it at at the # top level we need to figure out the correct import path here. if pip_major_version == 9: - return 'from pip import main' + return "from pip import main" # Pip changed their import structure again in 19.3 # https://github.com/pypa/pip/commit/09fd200 - elif (pip_major_version, pip_minor_version) >= (19, 3): - return 'from pip._internal.main import main' + elif (pip_major_version, pip_minor_version) >= (19, 3): + return "from pip._internal.main import main" else: - return 'from pip._internal import main' + return "from pip._internal import main" -if os.name == 'nt': +if os.name == "nt": # windows # This is the actual patch used on windows to prevent distutils from # compiling C extensions. The msvc compiler base class has its compile @@ -48,7 +44,8 @@ def prevent_msvc_compiling_patch(): from distutils.errors import CompileError def raise_compile_error(*args, **kwargs): - raise CompileError('Prevented C extension compiling.') + raise CompileError("Prevented C extension compiling.") + distutils._msvccompiler.MSVCCompiler.compile = raise_compile_error distutils.msvc9compiler.MSVCCompiler.compile = raise_compile_error distutils.msvccompiler.MSVCCompiler.compile = raise_compile_error @@ -99,10 +96,7 @@ def raise_compile_error(*args, **kwargs): # the c extensions before the setup.py file has been executed. # The actual patches used are decribed in the comment above # _SETUPTOOLS_SHIM. - pip_no_compile_c_shim = ( - 'import pip;' - 'pip.wheel.SETUPTOOLS_SHIM = """%s""";' - ) % _SETUPTOOLS_SHIM + pip_no_compile_c_shim = ("import pip;" 'pip.wheel.SETUPTOOLS_SHIM = """%s""";') % _SETUPTOOLS_SHIM pip_no_compile_c_env_vars = {} else: # posix @@ -112,7 +106,5 @@ def raise_compile_error(*args, **kwargs): # C extensions, and any fallback processes in place to build a pure python # package will be kicked off. # No need to monkey patch the process. - pip_no_compile_c_shim = '' - pip_no_compile_c_env_vars = { - 'CC': '/var/false' - } + pip_no_compile_c_shim = "" + pip_no_compile_c_env_vars = {"CC": "/var/false"} diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index 4d37dbf8a..4fdd30260 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -36,43 +36,38 @@ class InvalidSourceDistributionNameError(PackagerError): class RequirementsFileNotFoundError(PackagerError): def __init__(self, requirements_path): - super(RequirementsFileNotFoundError, self).__init__( - 'Requirements file not found: %s' % requirements_path) + super(RequirementsFileNotFoundError, self).__init__("Requirements file not found: %s" % requirements_path) class MissingDependencyError(PackagerError): """Raised when some dependencies could not be packaged for any reason.""" + def __init__(self, missing): self.missing = missing class NoSuchPackageError(PackagerError): """Raised when a package name or version could not be found.""" + def __init__(self, package_name): - super(NoSuchPackageError, self).__init__( - 'Could not satisfy the requirement: %s' % package_name) + super(NoSuchPackageError, self).__init__("Could not satisfy the requirement: %s" % package_name) class PackageDownloadError(PackagerError): """Generic networking error during a package download.""" + pass class UnsupportedPythonVersion(PackagerError): """Generic networking error during a package download.""" + def __init__(self, version): - super(UnsupportedPythonVersion, self).__init__( - "'%s' version of python is not supported" % version - ) + super(UnsupportedPythonVersion, self).__init__("'%s' version of python is not supported" % version) def get_lambda_abi(runtime): - supported = { - "python2.7": "cp27mu", - "python3.6": "cp36m", - "python3.7": "cp37m", - "python3.8": "cp38" - } + supported = {"python2.7": "cp27mu", "python3.6": "cp36m", "python3.7": "cp37m", "python3.8": "cp38"} if runtime not in supported: raise UnsupportedPythonVersion(runtime) @@ -105,8 +100,7 @@ def __init__(self, runtime, osutils=None, dependency_builder=None): dependency_builder = DependencyBuilder(self.osutils, runtime) self._dependency_builder = dependency_builder - def build_dependencies(self, artifacts_dir_path, scratch_dir_path, - requirements_path, ui=None, config=None): + def build_dependencies(self, artifacts_dir_path, scratch_dir_path, requirements_path, ui=None, config=None): """Builds a python project's dependencies into an artifact directory. :type artifacts_dir_path: str @@ -140,8 +134,7 @@ def build_dependencies(self, artifacts_dir_path, scratch_dir_path, if not self.osutils.file_exists(requirements_path): raise RequirementsFileNotFoundError(requirements_path) - self._dependency_builder.build_site_packages( - requirements_path, artifacts_dir_path, scratch_dir_path) + self._dependency_builder.build_site_packages(requirements_path, artifacts_dir_path, scratch_dir_path) class DependencyBuilder(object): @@ -156,11 +149,9 @@ class DependencyBuilder(object): into a site-packages directory, to be included in the bundle by the packager. """ - _MANYLINUX_COMPATIBLE_PLATFORM = {'any', 'linux_x86_64', - 'manylinux1_x86_64'} - _COMPATIBLE_PACKAGE_WHITELIST = { - 'sqlalchemy' - } + + _MANYLINUX_COMPATIBLE_PLATFORM = {"any", "linux_x86_64", "manylinux1_x86_64"} + _COMPATIBLE_PACKAGE_WHITELIST = {"sqlalchemy"} def __init__(self, osutils, runtime, pip_runner=None): """Initialize a DependencyBuilder. @@ -182,9 +173,7 @@ def __init__(self, osutils, runtime, pip_runner=None): self._pip = pip_runner self.runtime = runtime - def build_site_packages(self, requirements_filepath, - target_directory, - scratch_directory): + def build_site_packages(self, requirements_filepath, target_directory, scratch_directory): """Build site-packages directory for a set of requiremetns. :type requirements_filepath: str @@ -206,8 +195,7 @@ def build_site_packages(self, requirements_filepath, packages is included in the error object's ``missing`` property. """ if self._has_at_least_one_package(requirements_filepath): - wheels, packages_without_wheels = self._download_dependencies( - scratch_directory, requirements_filepath) + wheels, packages_without_wheels = self._download_dependencies(scratch_directory, requirements_filepath) self._install_wheels(scratch_directory, target_directory, wheels) if packages_without_wheels: raise MissingDependencyError(packages_without_wheels) @@ -215,14 +203,14 @@ def build_site_packages(self, requirements_filepath, def _has_at_least_one_package(self, filename): if not self._osutils.file_exists(filename): return False - with open(filename, 'r') as f: + with open(filename, "r") as f: # This is meant to be a best effort attempt. # This can return True and still have no packages # actually being specified, but those aren't common # cases. for line in f: line = line.strip() - if line and not line.startswith('#'): + if line and not line.startswith("#"): return True return False @@ -231,8 +219,7 @@ def _download_dependencies(self, directory, requirements_filename): # download. # deps should represent the best effort we can make to gather all the # dependencies. - deps = self._download_all_dependencies( - requirements_filename, directory) + deps = self._download_all_dependencies(requirements_filename, directory) # Sort the downloaded packages into three categories: # - sdists (Pip could not get a wheel so it gave us an sdist) @@ -249,7 +236,7 @@ def _download_dependencies(self, directory, requirements_filename): incompatible_wheels = set() sdists = set() for package in deps: - if package.dist_type == 'sdist': + if package.dist_type == "sdist": sdists.add(package) else: if self._is_compatible_wheel_filename(package.filename): @@ -271,12 +258,8 @@ def _download_dependencies(self, directory, requirements_filename): # that has an sdist but not a valid wheel file is still not going to # work on lambda and we must now try and build the sdist into a wheel # file ourselves. - compatible_wheels, incompatible_wheels = self._categorize_wheel_files( - directory) - LOG.debug( - "compatible wheels after second download pass: %s", - compatible_wheels - ) + compatible_wheels, incompatible_wheels = self._categorize_wheel_files(directory) + LOG.debug("compatible wheels after second download pass: %s", compatible_wheels) missing_wheels = sdists - compatible_wheels self._build_sdists(missing_wheels, directory, compile_c=True) @@ -288,12 +271,8 @@ def _download_dependencies(self, directory, requirements_filename): # there are missing dependencies we can try our last ditch effort of # building the package and trying to sever its ability to find a C # compiler. - compatible_wheels, incompatible_wheels = self._categorize_wheel_files( - directory) - LOG.debug( - "compatible after building wheels (no C compiling): %s", - compatible_wheels - ) + compatible_wheels, incompatible_wheels = self._categorize_wheel_files(directory) + LOG.debug("compatible after building wheels (no C compiling): %s", compatible_wheels) missing_wheels = sdists - compatible_wheels self._build_sdists(missing_wheels, directory, compile_c=False) @@ -301,20 +280,15 @@ def _download_dependencies(self, directory, requirements_filename): # any unmet dependencies left over. At this point there is nothing we # can do about any missing wheel files. We tried downloading a # compatible version directly and building from source. - compatible_wheels, incompatible_wheels = self._categorize_wheel_files( - directory) - LOG.debug( - "compatible after building wheels (C compiling): %s", - compatible_wheels - ) + compatible_wheels, incompatible_wheels = self._categorize_wheel_files(directory) + LOG.debug("compatible after building wheels (C compiling): %s", compatible_wheels) # Now there is still the case left over where the setup.py has been # made in such a way to be incompatible with python's setup tools, # causing it to lie about its compatibility. To fix this we have a # manually curated whitelist of packages that will work, despite # claiming otherwise. - compatible_wheels, incompatible_wheels = self._apply_wheel_whitelist( - compatible_wheels, incompatible_wheels) + compatible_wheels, incompatible_wheels = self._apply_wheel_whitelist(compatible_wheels, incompatible_wheels) missing_wheels = deps - compatible_wheels LOG.debug("Final compatible: %s", compatible_wheels) LOG.debug("Final incompatible: %s", incompatible_wheels) @@ -329,8 +303,7 @@ def _download_all_dependencies(self, requirements_filename, directory): # which will serve as the master list of dependencies needed to deploy # successfully. self._pip.download_all_dependencies(requirements_filename, directory) - deps = {Package(directory, filename) for filename - in self._osutils.get_directory_contents(directory)} + deps = {Package(directory, filename) for filename in self._osutils.get_directory_contents(directory)} LOG.debug("Full dependency closure: %s", deps) return deps @@ -338,20 +311,20 @@ def _download_binary_wheels(self, packages, directory): # Try to get binary wheels for each package that isn't compatible. LOG.debug("Downloading missing wheels: %s", packages) lambda_abi = get_lambda_abi(self.runtime) - self._pip.download_manylinux_wheels( - [pkg.identifier for pkg in packages], directory, lambda_abi) + self._pip.download_manylinux_wheels([pkg.identifier for pkg in packages], directory, lambda_abi) def _build_sdists(self, sdists, directory, compile_c=True): - LOG.debug("Build missing wheels from sdists " - "(C compiling %s): %s", compile_c, sdists) + LOG.debug("Build missing wheels from sdists " "(C compiling %s): %s", compile_c, sdists) for sdist in sdists: path_to_sdist = self._osutils.joinpath(directory, sdist.filename) self._pip.build_wheel(path_to_sdist, directory, compile_c) def _categorize_wheel_files(self, directory): - final_wheels = [Package(directory, filename) for filename - in self._osutils.get_directory_contents(directory) - if filename.endswith('.whl')] + final_wheels = [ + Package(directory, filename) + for filename in self._osutils.get_directory_contents(directory) + if filename.endswith(".whl") + ] compatible_wheels, incompatible_wheels = set(), set() for wheel in final_wheels: @@ -363,7 +336,7 @@ def _categorize_wheel_files(self, directory): def _is_compatible_wheel_filename(self, filename): wheel = filename[:-4] - implementation, abi, platform = wheel.split('-')[-3:] + implementation, abi, platform = wheel.split("-")[-3:] # Verify platform is compatible if platform not in self._MANYLINUX_COMPATIBLE_PLATFORM: return False @@ -373,17 +346,17 @@ def _is_compatible_wheel_filename(self, filename): # Verify that the ABI is compatible with lambda. Either none or the # correct type for the python version cp27mu for py27 and cp36m for # py36. - if abi == 'none': + if abi == "none": return True prefix_version = implementation[:3] - if prefix_version == 'cp3': + if prefix_version == "cp3": # Deploying python 3 function which means we need cp36m abi # We can also accept abi3 which is the CPython 3 Stable ABI and # will work on any version of python 3. - return abi == lambda_runtime_abi or abi == 'abi3' - elif prefix_version == 'cp2': + return abi == lambda_runtime_abi or abi == "abi3" + elif prefix_version == "cp2": # Deploying to python 2 function which means we need cp27mu abi - return abi == 'cp27mu' + return abi == "cp27mu" # Don't know what we have but it didn't pass compatibility tests. return False @@ -406,7 +379,7 @@ def _install_purelib_and_platlib(self, wheel, root): data_dir = self._osutils.joinpath(root, wheel.data_dir) if not self._osutils.directory_exists(data_dir): return - unpack_dirs = {'purelib', 'platlib'} + unpack_dirs = {"purelib", "platlib"} data_contents = self._osutils.get_directory_contents(data_dir) for content_name in data_contents: if content_name in unpack_dirs: @@ -428,8 +401,9 @@ def _install_wheels(self, src_dir, dst_dir, wheels): class Package(object): """A class to represent a package downloaded but not yet installed.""" + def __init__(self, directory, filename, osutils=None): - self.dist_type = 'wheel' if filename.endswith('.whl') else 'sdist' + self.dist_type = "wheel" if filename.endswith(".whl") else "sdist" self._directory = directory self.filename = filename if osutils is None: @@ -444,7 +418,7 @@ def name(self): @property def data_dir(self): # The directory format is {distribution}-{version}.data - return '%s-%s.data' % (self._name, self._version) + return "%s-%s.data" % (self._name, self._version) def _normalize_name(self, name): # Taken directly from PEP 503 @@ -452,10 +426,10 @@ def _normalize_name(self, name): @property def identifier(self): - return '%s==%s' % (self._name, self._version) + return "%s==%s" % (self._name, self._version) def __str__(self): - return '%s(%s)' % (self.identifier, self.dist_type) + return "%s(%s)" % (self.identifier, self.dist_type) def __repr__(self): return str(self) @@ -469,22 +443,22 @@ def __hash__(self): return hash(self.identifier) def _calculate_name_and_version(self): - if self.dist_type == 'wheel': + if self.dist_type == "wheel": # From the wheel spec (PEP 427) # {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}- # {platform tag}.whl - name, version = self.filename.split('-')[:2] + name, version = self.filename.split("-")[:2] else: info_fetcher = SDistMetadataFetcher(osutils=self._osutils) sdist_path = self._osutils.joinpath(self._directory, self.filename) - name, version = info_fetcher.get_package_name_and_version( - sdist_path) + name, version = info_fetcher.get_package_name_and_version(sdist_path) normalized_name = self._normalize_name(name) return normalized_name, version class SDistMetadataFetcher(object): """This is the "correct" way to get name and version from an sdist.""" + # https://git.io/vQkwV _SETUPTOOLS_SHIM = ( "import setuptools, tokenize;__file__=%r;" @@ -509,25 +483,22 @@ def _parse_pkg_info_file(self, filepath): return parser.close() def _generate_egg_info(self, package_dir): - setup_py = self._osutils.joinpath(package_dir, 'setup.py') + setup_py = self._osutils.joinpath(package_dir, "setup.py") script = self._SETUPTOOLS_SHIM % setup_py - cmd = [sys.executable, '-c', script, '--no-user-cfg', 'egg_info', - '--egg-base', 'egg-info'] - egg_info_dir = self._osutils.joinpath(package_dir, 'egg-info') + cmd = [sys.executable, "-c", script, "--no-user-cfg", "egg_info", "--egg-base", "egg-info"] + egg_info_dir = self._osutils.joinpath(package_dir, "egg-info") self._osutils.makedirs(egg_info_dir) - p = subprocess.Popen(cmd, cwd=package_dir, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p = subprocess.Popen(cmd, cwd=package_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) p.communicate() info_contents = self._osutils.get_directory_contents(egg_info_dir) - pkg_info_path = self._osutils.joinpath( - egg_info_dir, info_contents[0], 'PKG-INFO') + pkg_info_path = self._osutils.joinpath(egg_info_dir, info_contents[0], "PKG-INFO") return pkg_info_path def _unpack_sdist_into_dir(self, sdist_path, unpack_dir): - if sdist_path.endswith('.zip'): + if sdist_path.endswith(".zip"): self._osutils.extract_zipfile(sdist_path, unpack_dir) - elif sdist_path.endswith(('.tar.gz', '.tar.bz2')): + elif sdist_path.endswith((".tar.gz", ".tar.bz2")): self._osutils.extract_tarfile(sdist_path, unpack_dir) else: raise InvalidSourceDistributionNameError(sdist_path) @@ -540,13 +511,14 @@ def get_package_name_and_version(self, sdist_path): package_dir = self._unpack_sdist_into_dir(sdist_path, tempdir) pkg_info_filepath = self._generate_egg_info(package_dir) metadata = self._parse_pkg_info_file(pkg_info_filepath) - name = metadata['Name'] - version = metadata['Version'] + name = metadata["Name"] + version = metadata["Version"] return name, version class SubprocessPip(object): """Wrapper around calling pip through a subprocess.""" + def __init__(self, osutils=None, python_exe=None, import_string=None): if osutils is None: osutils = OSUtils() @@ -560,16 +532,11 @@ def main(self, args, env_vars=None, shim=None): if env_vars is None: env_vars = self._osutils.environ() if shim is None: - shim = '' - run_pip = ( - 'import sys; %s; sys.exit(main(%s))' - ) % (self._import_string, args) - exec_string = '%s%s' % (shim, run_pip) - invoke_pip = [self.python_exe, '-c', exec_string] - p = self._osutils.popen(invoke_pip, - stdout=self._osutils.pipe, - stderr=self._osutils.pipe, - env=env_vars) + shim = "" + run_pip = ("import sys; %s; sys.exit(main(%s))") % (self._import_string, args) + exec_string = "%s%s" % (shim, run_pip) + invoke_pip = [self.python_exe, "-c", exec_string] + p = self._osutils.popen(invoke_pip, stdout=self._osutils.pipe, stderr=self._osutils.pipe, env=env_vars) out, err = p.communicate() rc = p.returncode return rc, out, err @@ -578,9 +545,7 @@ def main(self, args, env_vars=None, shim=None): class PipRunner(object): """Wrapper around pip calls used by chalice.""" - _LINK_IS_DIR_PATTERN = ("Processing (.+?)\n" - " Link is a directory," - " ignoring download_dir") + _LINK_IS_DIR_PATTERN = "Processing (.+?)\n" " Link is a directory," " ignoring download_dir" def __init__(self, python_exe, pip, osutils=None): if osutils is None: @@ -592,29 +557,27 @@ def __init__(self, python_exe, pip, osutils=None): def _execute(self, command, args, env_vars=None, shim=None): """Execute a pip command with the given arguments.""" main_args = [command] + args - LOG.debug("calling pip %s", ' '.join(main_args)) - rc, out, err = self._wrapped_pip.main(main_args, env_vars=env_vars, - shim=shim) + LOG.debug("calling pip %s", " ".join(main_args)) + rc, out, err = self._wrapped_pip.main(main_args, env_vars=env_vars, shim=shim) return rc, out, err def build_wheel(self, wheel, directory, compile_c=True): """Build an sdist into a wheel file.""" - arguments = ['--no-deps', '--wheel-dir', directory, wheel] + arguments = ["--no-deps", "--wheel-dir", directory, wheel] env_vars = self._osutils.environ() - shim = '' + shim = "" if not compile_c: env_vars.update(pip_no_compile_c_env_vars) shim = pip_no_compile_c_shim # Ignore rc and stderr from this command since building the wheels # may fail and we will find out when we categorize the files that were # generated. - self._execute('wheel', arguments, - env_vars=env_vars, shim=shim) + self._execute("wheel", arguments, env_vars=env_vars, shim=shim) def download_all_dependencies(self, requirements_filename, directory): """Download all dependencies as sdist or wheel.""" - arguments = ['-r', requirements_filename, '--dest', directory] - rc, out, err = self._execute('download', arguments) + arguments = ["-r", requirements_filename, "--dest", directory] + rc, out, err = self._execute("download", arguments) # When downloading all dependencies we expect to get an rc of 0 back # since we are casting a wide net here letting pip have options about # what to download. If a package is not found it is likely because it @@ -623,10 +586,9 @@ def download_all_dependencies(self, requirements_filename, directory): # download error where we pass along the stderr. if rc != 0: if err is None: - err = b'Unknown error' + err = b"Unknown error" error = err.decode() - match = re.search(("Could not find a version that satisfies the " - "requirement (.+?) "), error) + match = re.search(("Could not find a version that satisfies the " "requirement (.+?) "), error) if match: package_name = match.group(1) raise NoSuchPackageError(str(package_name)) @@ -656,7 +618,17 @@ def download_manylinux_wheels(self, packages, directory, lambda_abi): # cpython implementation. The compatible abi depends on the python # version and is checked later. for package in packages: - arguments = ['--only-binary=:all:', '--no-deps', '--platform', - 'manylinux1_x86_64', '--implementation', 'cp', - '--abi', lambda_abi, '--dest', directory, package] - self._execute('download', arguments) + arguments = [ + "--only-binary=:all:", + "--no-deps", + "--platform", + "manylinux1_x86_64", + "--implementation", + "cp", + "--abi", + lambda_abi, + "--dest", + directory, + package, + ] + self._execute("download", arguments) diff --git a/aws_lambda_builders/workflows/python_pip/utils.py b/aws_lambda_builders/workflows/python_pip/utils.py index 2cdd1941c..6c5afc73a 100644 --- a/aws_lambda_builders/workflows/python_pip/utils.py +++ b/aws_lambda_builders/workflows/python_pip/utils.py @@ -13,32 +13,31 @@ class OSUtils(object): - def environ(self): return os.environ def file_exists(self, filename): return os.path.isfile(filename) - def get_file_contents(self, filename, binary=True, encoding='utf-8'): + def get_file_contents(self, filename, binary=True, encoding="utf-8"): # It looks like the type definition for io.open is wrong. # the encoding arg is unicode, but the actual type is # Optional[Text]. For now we have to use Any to keep mypy happy. if binary: - mode = 'rb' + mode = "rb" # In binary mode the encoding is not used and most be None. encoding = None else: - mode = 'r' + mode = "r" with io.open(filename, mode, encoding=encoding) as f: return f.read() def extract_zipfile(self, zipfile_path, unpack_dir): - with zipfile.ZipFile(zipfile_path, 'r') as z: + with zipfile.ZipFile(zipfile_path, "r") as z: z.extractall(unpack_dir) def extract_tarfile(self, tarfile_path, unpack_dir): - with tarfile.open(tarfile_path, 'r:*') as tar: + with tarfile.open(tarfile_path, "r:*") as tar: tar.extractall(unpack_dir) def directory_exists(self, path): diff --git a/aws_lambda_builders/workflows/python_pip/validator.py b/aws_lambda_builders/workflows/python_pip/validator.py index 4b2fd67b3..f0a1c06b1 100644 --- a/aws_lambda_builders/workflows/python_pip/validator.py +++ b/aws_lambda_builders/workflows/python_pip/validator.py @@ -12,12 +12,7 @@ class PythonRuntimeValidator(object): - SUPPORTED_RUNTIMES = { - "python2.7", - "python3.6", - "python3.7", - "python3.8" - } + SUPPORTED_RUNTIMES = {"python2.7", "python3.6", "python3.7", "python3.8"} def __init__(self, runtime): self.language = "python" @@ -39,34 +34,28 @@ def validate(self, runtime_path): :raises MisMatchRuntimeError: Version mismatch of the language vs the required runtime """ if not self.has_runtime(): - LOG.warning("'%s' runtime is not " - "a supported runtime", self.runtime) + LOG.warning("'%s' runtime is not " "a supported runtime", self.runtime) return cmd = self._validate_python_cmd(runtime_path) - p = subprocess.Popen(cmd, - cwd=os.getcwd(), - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p = subprocess.Popen(cmd, cwd=os.getcwd(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) p.communicate() if p.returncode != 0: - raise MisMatchRuntimeError(language=self.language, - required_runtime=self.runtime, - runtime_path=runtime_path) + raise MisMatchRuntimeError(language=self.language, required_runtime=self.runtime, runtime_path=runtime_path) else: self._valid_runtime_path = runtime_path return self._valid_runtime_path def _validate_python_cmd(self, runtime_path): - major, minor = self.runtime.replace(self.language, "").split('.') + major, minor = self.runtime.replace(self.language, "").split(".") cmd = [ runtime_path, "-c", "import sys; " "assert sys.version_info.major == {major} " - "and sys.version_info.minor == {minor}".format( - major=major, - minor=minor)] + "and sys.version_info.minor == {minor}".format(major=major, minor=minor), + ] return cmd @property diff --git a/aws_lambda_builders/workflows/python_pip/workflow.py b/aws_lambda_builders/workflows/python_pip/workflow.py index 5d5cee0c7..f623d03cc 100644 --- a/aws_lambda_builders/workflows/python_pip/workflow.py +++ b/aws_lambda_builders/workflows/python_pip/workflow.py @@ -12,60 +12,59 @@ class PythonPipWorkflow(BaseWorkflow): NAME = "PythonPipBuilder" - CAPABILITY = Capability(language="python", - dependency_manager="pip", - application_framework=None) + CAPABILITY = Capability(language="python", dependency_manager="pip", application_framework=None) # Common source files to exclude from build artifacts output # Trimmed version of https://github.com/github/gitignore/blob/master/Python.gitignore EXCLUDED_FILES = ( - ".aws-sam", ".chalice", - - ".git", - - # Compiled files - "*.pyc", "__pycache__", "*.so", - - # Distribution / packaging - ".Python", "*.egg-info", "*.egg", - - # Installer logs - "pip-log.txt", "pip-delete-this-directory.txt", - - # Unit test / coverage reports - "htmlcov", ".tox", ".nox", ".coverage", ".cache", ".pytest_cache", - - # pyenv - ".python-version", - - # mypy, Pyre - ".mypy_cache", ".dmypy.json", ".pyre", - - # environments - ".env", ".venv", "venv", "venv.bak", "env.bak", "ENV", - - # Editors - # TODO: Move the commonly ignored files to base class - ".vscode", ".idea" - ) - - def __init__(self, - source_dir, - artifacts_dir, - scratch_dir, - manifest_path, - runtime=None, **kwargs): - - super(PythonPipWorkflow, self).__init__(source_dir, - artifacts_dir, - scratch_dir, - manifest_path, - runtime=runtime, - **kwargs) + ".aws-sam", + ".chalice", + ".git", + # Compiled files + "*.pyc", + "__pycache__", + "*.so", + # Distribution / packaging + ".Python", + "*.egg-info", + "*.egg", + # Installer logs + "pip-log.txt", + "pip-delete-this-directory.txt", + # Unit test / coverage reports + "htmlcov", + ".tox", + ".nox", + ".coverage", + ".cache", + ".pytest_cache", + # pyenv + ".python-version", + # mypy, Pyre + ".mypy_cache", + ".dmypy.json", + ".pyre", + # environments + ".env", + ".venv", + "venv", + "venv.bak", + "env.bak", + "ENV", + # Editors + # TODO: Move the commonly ignored files to base class + ".vscode", + ".idea", + ) + + def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=None, **kwargs): + + super(PythonPipWorkflow, self).__init__( + source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=runtime, **kwargs + ) self.actions = [ - PythonPipBuildAction(artifacts_dir, scratch_dir, - manifest_path, runtime, binaries=self.binaries), + PythonPipBuildAction(artifacts_dir, scratch_dir, manifest_path, runtime, binaries=self.binaries), CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES), ] diff --git a/aws_lambda_builders/workflows/ruby_bundler/actions.py b/aws_lambda_builders/workflows/ruby_bundler/actions.py index 329309c4c..ffc9ad048 100644 --- a/aws_lambda_builders/workflows/ruby_bundler/actions.py +++ b/aws_lambda_builders/workflows/ruby_bundler/actions.py @@ -9,13 +9,14 @@ LOG = logging.getLogger(__name__) + class RubyBundlerInstallAction(BaseAction): """ A Lambda Builder Action which runs bundle install in order to build a full Gemfile.lock """ - NAME = 'RubyBundle' + NAME = "RubyBundle" DESCRIPTION = "Resolving dependencies using Bundler" PURPOSE = Purpose.RESOLVE_DEPENDENCIES @@ -27,19 +28,17 @@ def __init__(self, source_dir, subprocess_bundler): def execute(self): try: LOG.debug("Running bundle install in %s", self.source_dir) - self.subprocess_bundler.run( - ['install', '--without', 'development', 'test'], - cwd=self.source_dir - ) + self.subprocess_bundler.run(["install", "--without", "development", "test"], cwd=self.source_dir) except BundlerExecutionError as ex: raise ActionFailedError(str(ex)) + class RubyBundlerVendorAction(BaseAction): """ A Lambda Builder Action which vendors dependencies to the vendor/bundle directory. """ - NAME = 'RubyBundleDeployment' + NAME = "RubyBundleDeployment" DESCRIPTION = "Package dependencies for deployment." PURPOSE = Purpose.RESOLVE_DEPENDENCIES @@ -52,8 +51,7 @@ def execute(self): try: LOG.debug("Running bundle install --deployment in %s", self.source_dir) self.subprocess_bundler.run( - ['install', '--deployment', '--without', 'development', 'test'], - cwd=self.source_dir + ["install", "--deployment", "--without", "development", "test"], cwd=self.source_dir ) except BundlerExecutionError as ex: raise ActionFailedError(str(ex)) diff --git a/aws_lambda_builders/workflows/ruby_bundler/bundler.py b/aws_lambda_builders/workflows/ruby_bundler/bundler.py index 64fb299c8..33405f233 100644 --- a/aws_lambda_builders/workflows/ruby_bundler/bundler.py +++ b/aws_lambda_builders/workflows/ruby_bundler/bundler.py @@ -6,6 +6,7 @@ LOG = logging.getLogger(__name__) + class BundlerExecutionError(Exception): """ Exception raised when Bundler fails. @@ -17,6 +18,7 @@ class BundlerExecutionError(Exception): def __init__(self, **kwargs): Exception.__init__(self, self.MESSAGE.format(**kwargs)) + class SubprocessBundler(object): """ Wrapper around the Bundler command line utility, encapsulating @@ -27,31 +29,28 @@ def __init__(self, osutils, bundler_exe=None): self.osutils = osutils if bundler_exe is None: if osutils.is_windows(): - bundler_exe = 'bundler.bat' + bundler_exe = "bundler.bat" else: - bundler_exe = 'bundle' + bundler_exe = "bundle" self.bundler_exe = bundler_exe def run(self, args, cwd=None): if not isinstance(args, list): - raise ValueError('args must be a list') + raise ValueError("args must be a list") if not args: - raise ValueError('requires at least one arg') + raise ValueError("requires at least one arg") invoke_bundler = [self.bundler_exe] + args LOG.debug("executing Bundler: %s", invoke_bundler) - p = self.osutils.popen(invoke_bundler, - stdout=self.osutils.pipe, - stderr=self.osutils.pipe, - cwd=cwd) + p = self.osutils.popen(invoke_bundler, stdout=self.osutils.pipe, stderr=self.osutils.pipe, cwd=cwd) out, err = p.communicate() if p.returncode != 0: - raise BundlerExecutionError(message=err.decode('utf8').strip()) + raise BundlerExecutionError(message=err.decode("utf8").strip()) - return out.decode('utf8').strip() + return out.decode("utf8").strip() diff --git a/aws_lambda_builders/workflows/ruby_bundler/utils.py b/aws_lambda_builders/workflows/ruby_bundler/utils.py index 221e2f905..26c4339f3 100644 --- a/aws_lambda_builders/workflows/ruby_bundler/utils.py +++ b/aws_lambda_builders/workflows/ruby_bundler/utils.py @@ -16,7 +16,7 @@ class OSUtils(object): """ def extract_tarfile(self, tarfile_path, unpack_dir): - with tarfile.open(tarfile_path, 'r:*') as tar: + with tarfile.open(tarfile_path, "r:*") as tar: tar.extractall(unpack_dir) def popen(self, command, stdout=None, stderr=None, env=None, cwd=None): @@ -37,4 +37,4 @@ def abspath(self, path): return os.path.abspath(path) def is_windows(self): - return platform.system().lower() == 'windows' + return platform.system().lower() == "windows" diff --git a/aws_lambda_builders/workflows/ruby_bundler/workflow.py b/aws_lambda_builders/workflows/ruby_bundler/workflow.py index 013c67dd7..f57f6cf60 100644 --- a/aws_lambda_builders/workflows/ruby_bundler/workflow.py +++ b/aws_lambda_builders/workflows/ruby_bundler/workflow.py @@ -15,39 +15,26 @@ class RubyBundlerWorkflow(BaseWorkflow): A Lambda builder workflow that knows how to build Ruby projects using Bundler. """ + NAME = "RubyBundlerBuilder" - CAPABILITY = Capability(language="ruby", - dependency_manager="bundler", - application_framework=None) + CAPABILITY = Capability(language="ruby", dependency_manager="bundler", application_framework=None) EXCLUDED_FILES = (".aws-sam", ".git") - def __init__(self, - source_dir, - artifacts_dir, - scratch_dir, - manifest_path, - runtime=None, - osutils=None, - **kwargs): - - super(RubyBundlerWorkflow, self).__init__(source_dir, - artifacts_dir, - scratch_dir, - manifest_path, - runtime=runtime, - **kwargs) + def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=None, osutils=None, **kwargs): + + super(RubyBundlerWorkflow, self).__init__( + source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=runtime, **kwargs + ) if osutils is None: osutils = OSUtils() subprocess_bundler = SubprocessBundler(osutils) - bundle_install = RubyBundlerInstallAction(artifacts_dir, - subprocess_bundler=subprocess_bundler) + bundle_install = RubyBundlerInstallAction(artifacts_dir, subprocess_bundler=subprocess_bundler) - bundle_deployment = RubyBundlerVendorAction(artifacts_dir, - subprocess_bundler=subprocess_bundler) + bundle_deployment = RubyBundlerVendorAction(artifacts_dir, subprocess_bundler=subprocess_bundler) self.actions = [ CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES), bundle_install, diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..61b1ab85c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,17 @@ +[tool.black] +line-length = 120 +target_version = ['py27', 'py37', 'py36', 'py38'] +exclude = ''' + +( + /( + \.eggs # exclude a few common directories in the + | \.git # root of the project + | \.tox + | \.venv + | build + | dist + | pip-wheel-metadata + )/ +) +''' diff --git a/setup.py b/setup.py index ecbf3d7e1..b10d412ae 100644 --- a/setup.py +++ b/setup.py @@ -6,8 +6,8 @@ def read(*filenames, **kwargs): - encoding = kwargs.get('encoding', 'utf-8') - sep = kwargs.get('sep', os.linesep) + encoding = kwargs.get("encoding", "utf-8") + sep = kwargs.get("sep", os.linesep) buf = [] for filename in filenames: with io.open(filename, encoding=encoding) as f: @@ -15,16 +15,14 @@ def read(*filenames, **kwargs): return sep.join(buf) -def read_requirements(req='base.txt'): - content = read(os.path.join('requirements', req)) - return [line for line in content.split(os.linesep) - if not line.strip().startswith('#')] +def read_requirements(req="base.txt"): + content = read(os.path.join("requirements", req)) + return [line for line in content.split(os.linesep) if not line.strip().startswith("#")] def read_version(): - content = read(os.path.join( - os.path.dirname(__file__), 'aws_lambda_builders', '__init__.py')) - return re.search(r"__version__ = '([^']+)'", content).group(1) + content = read(os.path.join(os.path.dirname(__file__), "aws_lambda_builders", "__init__.py")) + return re.search(r"__version__ = \"([^']+)\"", content).group(1) cmd_name = "lambda-builders" @@ -33,47 +31,38 @@ def read_version(): cmd_name = "lambda-builders-dev" setup( - name='aws_lambda_builders', + name="aws_lambda_builders", version=read_version(), description=( - 'Python library to compile, build & package AWS Lambda functions for ' - 'several runtimes & frameworks.' + "Python library to compile, build & package AWS Lambda functions for " "several runtimes & frameworks." ), - long_description=read('README.md'), - author='Amazon Web Services', - author_email='aws-sam-developers@amazon.com', - url='https://github.com/awslabs/aws-lambda-builders', - license='Apache License 2.0', - packages=find_packages(exclude=['tests.*', 'tests']), + long_description=read("README.md"), + author="Amazon Web Services", + author_email="aws-sam-developers@amazon.com", + url="https://github.com/awslabs/aws-lambda-builders", + license="Apache License 2.0", + packages=find_packages(exclude=["tests.*", "tests"]), keywords="AWS Lambda Functions Building", # Support Python 2.7 and 3.6 or greater - python_requires=( - '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*' - ), - entry_points={ - 'console_scripts': [ - '{}=aws_lambda_builders.__main__:main'.format(cmd_name) - ] - }, - install_requires=read_requirements('base.txt') + read_requirements("python_pip.txt"), - extras_require={ - 'dev': read_requirements('dev.txt') - }, + python_requires=(">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"), + entry_points={"console_scripts": ["{}=aws_lambda_builders.__main__:main".format(cmd_name)]}, + install_requires=read_requirements("base.txt") + read_requirements("python_pip.txt"), + extras_require={"dev": read_requirements("dev.txt")}, include_package_data=True, classifiers=[ - 'Development Status :: 4 - Beta', - 'Environment :: Console', - 'Environment :: Other Environment', - 'Intended Audience :: Developers', - 'Intended Audience :: Information Technology', - 'License :: OSI Approved :: Apache Software License', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Topic :: Internet', - 'Topic :: Software Development :: Build Tools', - 'Topic :: Utilities', - ] + "Development Status :: 4 - Beta", + "Environment :: Console", + "Environment :: Other Environment", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Topic :: Internet", + "Topic :: Software Development :: Build Tools", + "Topic :: Utilities", + ], ) diff --git a/tests/functional/test_builder.py b/tests/functional/test_builder.py index a5a1b172f..ac0fb3e51 100644 --- a/tests/functional/test_builder.py +++ b/tests/functional/test_builder.py @@ -1,4 +1,3 @@ - import sys import os import shutil @@ -25,15 +24,15 @@ def setUp(self): self.source_dir = tempfile.mkdtemp() self.artifacts_dir = tempfile.mkdtemp() self.scratch_dir = os.path.join(tempfile.mkdtemp(), "scratch") - self.hello_builder = LambdaBuilder(language="python", - dependency_manager="test", - application_framework="test", - supported_workflows=[ - self.HELLO_WORKFLOW_MODULE - ]) + self.hello_builder = LambdaBuilder( + language="python", + dependency_manager="test", + application_framework="test", + supported_workflows=[self.HELLO_WORKFLOW_MODULE], + ) # The builder should write a file called hello.txt with contents "Hello World" - self.expected_filename = os.path.join(self.artifacts_dir, 'hello.txt') + self.expected_filename = os.path.join(self.artifacts_dir, "hello.txt") self.expected_contents = "Hello World" def tearDown(self): @@ -47,15 +46,17 @@ def tearDown(self): def test_run_hello_workflow_with_exec_paths(self): - self.hello_builder.build(self.source_dir, - self.artifacts_dir, - self.scratch_dir, - "/ignored", - executable_search_paths=[str(pathlib.Path(sys.executable).parent)]) + self.hello_builder.build( + self.source_dir, + self.artifacts_dir, + self.scratch_dir, + "/ignored", + executable_search_paths=[str(pathlib.Path(sys.executable).parent)], + ) self.assertTrue(os.path.exists(self.expected_filename)) - contents = '' - with open(self.expected_filename, 'r') as fp: + contents = "" + with open(self.expected_filename, "r") as fp: contents = fp.read() - self.assertEquals(contents, self.expected_contents) \ No newline at end of file + self.assertEquals(contents, self.expected_contents) diff --git a/tests/functional/test_cli.py b/tests/functional/test_cli.py index d5ce9301d..7a2285e6b 100644 --- a/tests/functional/test_cli.py +++ b/tests/functional/test_cli.py @@ -1,4 +1,3 @@ - import json import os import shutil @@ -36,25 +35,27 @@ def setUp(self): self.application_framework = "test" # The builder should write a file called hello.txt with contents "Hello World" - self.expected_filename = os.path.join(self.artifacts_dir, 'hello.txt') + self.expected_filename = os.path.join(self.artifacts_dir, "hello.txt") self.expected_contents = "Hello World" self.command_name = "lambda-builders-dev" if os.environ.get("LAMBDA_BUILDERS_DEV") else "lambda-builders" # Make sure the test workflow is in PYTHONPATH to be automatically loaded - self.python_path_list = os.environ.get("PYTHONPATH", '').split(os.pathsep) + [self.TEST_WORKFLOWS_FOLDER] + self.python_path_list = os.environ.get("PYTHONPATH", "").split(os.pathsep) + [self.TEST_WORKFLOWS_FOLDER] self.python_path = os.pathsep.join(filter(bool, self.python_path_list)) def tearDown(self): shutil.rmtree(self.source_dir) shutil.rmtree(self.artifacts_dir) - @parameterized.expand([ - ("request_through_stdin", lambda_builders_protocol_version), - ("request_through_argument", lambda_builders_protocol_version), - ("request_through_stdin", "0.1"), - ("request_through_argument", "0.1"), - ]) + @parameterized.expand( + [ + ("request_through_stdin", lambda_builders_protocol_version), + ("request_through_argument", lambda_builders_protocol_version), + ("request_through_stdin", "0.1"), + ("request_through_argument", "0.1"), + ] + ) def test_run_hello_workflow_with_backcompat(self, flavor, protocol_version): request = { @@ -66,7 +67,7 @@ def test_run_hello_workflow_with_backcompat(self, flavor, protocol_version): "capability": { "language": self.language, "dependency_manager": self.dependency_manager, - "application_framework": self.application_framework + "application_framework": self.application_framework, }, "supported_workflows": [self.HELLO_WORKFLOW_MODULE], "source_dir": self.source_dir, @@ -75,8 +76,8 @@ def test_run_hello_workflow_with_backcompat(self, flavor, protocol_version): "manifest_path": "/ignored", "runtime": "ignored", "optimizations": {}, - "options": {} - } + "options": {}, + }, } if protocol_version == lambda_builders_protocol_version: @@ -90,56 +91,56 @@ def test_run_hello_workflow_with_backcompat(self, flavor, protocol_version): stdout_data = None if flavor == "request_through_stdin": p = subprocess.Popen([self.command_name], env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE) - stdout_data = p.communicate(input=request_json.encode('utf-8'))[0] + stdout_data = p.communicate(input=request_json.encode("utf-8"))[0] elif flavor == "request_through_argument": - p = subprocess.Popen([self.command_name, request_json], env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + p = subprocess.Popen( + [self.command_name, request_json], env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE + ) stdout_data = p.communicate()[0] else: raise ValueError("Invalid test flavor") # Validate the response object. It should be successful response response = json.loads(stdout_data) - self.assertNotIn('error', response) - self.assertIn('result', response) - self.assertEquals(response['result']['artifacts_dir'], self.artifacts_dir) + self.assertNotIn("error", response) + self.assertIn("result", response) + self.assertEquals(response["result"]["artifacts_dir"], self.artifacts_dir) self.assertTrue(os.path.exists(self.expected_filename)) - contents = '' - with open(self.expected_filename, 'r') as fp: + contents = "" + with open(self.expected_filename, "r") as fp: contents = fp.read() self.assertEquals(contents, self.expected_contents) shutil.rmtree(self.scratch_dir) - @parameterized.expand([ - ("request_through_stdin"), - ("request_through_argument") - ]) + @parameterized.expand([("request_through_stdin"), ("request_through_argument")]) def test_run_hello_workflow_incompatible(self, flavor): - request_json = json.dumps({ - "jsonschema": "2.0", - "id": 1234, - "method": "LambdaBuilder.build", - "params": { - "__protocol_version": "2.0", - "capability": { - "language": self.language, - "dependency_manager": self.dependency_manager, - "application_framework": self.application_framework + request_json = json.dumps( + { + "jsonschema": "2.0", + "id": 1234, + "method": "LambdaBuilder.build", + "params": { + "__protocol_version": "2.0", + "capability": { + "language": self.language, + "dependency_manager": self.dependency_manager, + "application_framework": self.application_framework, + }, + "supported_workflows": [self.HELLO_WORKFLOW_MODULE], + "source_dir": self.source_dir, + "artifacts_dir": self.artifacts_dir, + "scratch_dir": self.scratch_dir, + "manifest_path": "/ignored", + "runtime": "ignored", + "optimizations": {}, + "options": {}, + "executable_search_paths": [str(pathlib.Path(sys.executable).parent)], }, - "supported_workflows": [self.HELLO_WORKFLOW_MODULE], - "source_dir": self.source_dir, - "artifacts_dir": self.artifacts_dir, - "scratch_dir": self.scratch_dir, - "manifest_path": "/ignored", - "runtime": "ignored", - "optimizations": {}, - "options": {}, - "executable_search_paths": [str(pathlib.Path(sys.executable).parent)] } - }) - + ) env = copy.deepcopy(os.environ) env["PYTHONPATH"] = self.python_path @@ -147,14 +148,16 @@ def test_run_hello_workflow_incompatible(self, flavor): stdout_data = None if flavor == "request_through_stdin": p = subprocess.Popen([self.command_name], env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE) - stdout_data = p.communicate(input=request_json.encode('utf-8'))[0] + stdout_data = p.communicate(input=request_json.encode("utf-8"))[0] elif flavor == "request_through_argument": - p = subprocess.Popen([self.command_name, request_json], env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + p = subprocess.Popen( + [self.command_name, request_json], env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE + ) stdout_data = p.communicate()[0] else: raise ValueError("Invalid test flavor") # Validate the response object. It should be error response response = json.loads(stdout_data) - self.assertIn('error', response) - self.assertEquals(response['error']['code'], 505) + self.assertIn("error", response) + self.assertEquals(response["error"]["code"], 505) diff --git a/tests/functional/test_utils.py b/tests/functional/test_utils.py index b82032df0..db2657c40 100644 --- a/tests/functional/test_utils.py +++ b/tests/functional/test_utils.py @@ -6,8 +6,8 @@ from aws_lambda_builders.utils import copytree -class TestCopyTree(TestCase): +class TestCopyTree(TestCase): def setUp(self): self.source = tempfile.mkdtemp() self.dest = tempfile.mkdtemp() @@ -48,4 +48,4 @@ def file(*args): os.makedirs(basedir) # empty file - open(path, 'a').close() + open(path, "a").close() diff --git a/tests/functional/testdata/cwd.py b/tests/functional/testdata/cwd.py index ea065561f..ea0aff47e 100644 --- a/tests/functional/testdata/cwd.py +++ b/tests/functional/testdata/cwd.py @@ -1,3 +1,3 @@ import os -print(os.getcwd()) +print (os.getcwd()) diff --git a/tests/functional/testdata/workflows/hello_workflow/write_hello.py b/tests/functional/testdata/workflows/hello_workflow/write_hello.py index 0a8a5e311..a2cf94370 100644 --- a/tests/functional/testdata/workflows/hello_workflow/write_hello.py +++ b/tests/functional/testdata/workflows/hello_workflow/write_hello.py @@ -38,7 +38,4 @@ class WriteHelloWorkflow(BaseWorkflow): def __init__(self, source_dir, artifacts_dir, *args, **kwargs): super(WriteHelloWorkflow, self).__init__(source_dir, artifacts_dir, *args, **kwargs) - self.actions = [ - WriteHelloAction(artifacts_dir) - ] - + self.actions = [WriteHelloAction(artifacts_dir)] diff --git a/tests/functional/workflows/go_dep/test_godep_utils.py b/tests/functional/workflows/go_dep/test_godep_utils.py index e2d610ed6..94431c818 100644 --- a/tests/functional/workflows/go_dep/test_godep_utils.py +++ b/tests/functional/workflows/go_dep/test_godep_utils.py @@ -9,7 +9,6 @@ class TestGoDepOSUtils(TestCase): - def setUp(self): self.osutils = utils.OSUtils() @@ -21,43 +20,40 @@ def test_dirname_returns_directory_for_path(self): def test_abspath_returns_absolute_path(self): - result = self.osutils.abspath('.') + result = self.osutils.abspath(".") self.assertTrue(os.path.isabs(result)) - self.assertEqual(result, os.path.abspath('.')) + self.assertEqual(result, os.path.abspath(".")) def test_joinpath_joins_path_components(self): - result = self.osutils.joinpath('a', 'b', 'c') + result = self.osutils.joinpath("a", "b", "c") - self.assertEqual(result, os.path.join('a', 'b', 'c')) + self.assertEqual(result, os.path.join("a", "b", "c")) def test_popen_runs_a_process_and_returns_outcome(self): - cwd_py = os.path.join(os.path.dirname(__file__), '..', '..', 'testdata', 'cwd.py') + cwd_py = os.path.join(os.path.dirname(__file__), "..", "..", "testdata", "cwd.py") - p = self.osutils.popen([sys.executable, cwd_py], - stdout=self.osutils.pipe, - stderr=self.osutils.pipe) + p = self.osutils.popen([sys.executable, cwd_py], stdout=self.osutils.pipe, stderr=self.osutils.pipe) out, err = p.communicate() self.assertEqual(p.returncode, 0) - self.assertEqual(out.decode('utf8').strip(), os.getcwd()) + self.assertEqual(out.decode("utf8").strip(), os.getcwd()) def test_popen_can_accept_cwd(self): - testdata_dir = os.path.join(os.path.dirname(__file__), '..', '..', 'testdata') + testdata_dir = os.path.join(os.path.dirname(__file__), "..", "..", "testdata") - p = self.osutils.popen([sys.executable, 'cwd.py'], - stdout=self.osutils.pipe, - stderr=self.osutils.pipe, - cwd=testdata_dir) + p = self.osutils.popen( + [sys.executable, "cwd.py"], stdout=self.osutils.pipe, stderr=self.osutils.pipe, cwd=testdata_dir + ) out, err = p.communicate() self.assertEqual(p.returncode, 0) - self.assertEqual(out.decode('utf8').strip(), os.path.abspath(testdata_dir)) + self.assertEqual(out.decode("utf8").strip(), os.path.abspath(testdata_dir)) diff --git a/tests/functional/workflows/go_modules/test_go_utils.py b/tests/functional/workflows/go_modules/test_go_utils.py index 1264af818..69950a2e3 100644 --- a/tests/functional/workflows/go_modules/test_go_utils.py +++ b/tests/functional/workflows/go_modules/test_go_utils.py @@ -7,7 +7,6 @@ class TestOSUtils(TestCase): - def setUp(self): self.osutils = utils.OSUtils() @@ -16,24 +15,21 @@ def test_environ_returns_environment(self): self.assertEqual(result, os.environ) def test_joinpath_joins_path_components(self): - result = self.osutils.joinpath('a', 'b', 'c') - self.assertEqual(result, os.path.join('a', 'b', 'c')) + result = self.osutils.joinpath("a", "b", "c") + self.assertEqual(result, os.path.join("a", "b", "c")) def test_popen_runs_a_process_and_returns_outcome(self): - cwd_py = os.path.join(os.path.dirname(__file__), '..', '..', 'testdata', 'cwd.py') - p = self.osutils.popen([sys.executable, cwd_py], - stdout=self.osutils.pipe, - stderr=self.osutils.pipe) + cwd_py = os.path.join(os.path.dirname(__file__), "..", "..", "testdata", "cwd.py") + p = self.osutils.popen([sys.executable, cwd_py], stdout=self.osutils.pipe, stderr=self.osutils.pipe) out, err = p.communicate() self.assertEqual(p.returncode, 0) - self.assertEqual(out.decode('utf8').strip(), os.getcwd()) + self.assertEqual(out.decode("utf8").strip(), os.getcwd()) def test_popen_can_accept_cwd(self): - testdata_dir = os.path.join(os.path.dirname(__file__), '..', '..', 'testdata') - p = self.osutils.popen([sys.executable, 'cwd.py'], - stdout=self.osutils.pipe, - stderr=self.osutils.pipe, - cwd=testdata_dir) + testdata_dir = os.path.join(os.path.dirname(__file__), "..", "..", "testdata") + p = self.osutils.popen( + [sys.executable, "cwd.py"], stdout=self.osutils.pipe, stderr=self.osutils.pipe, cwd=testdata_dir + ) out, err = p.communicate() self.assertEqual(p.returncode, 0) - self.assertEqual(out.decode('utf8').strip(), os.path.abspath(testdata_dir)) + self.assertEqual(out.decode("utf8").strip(), os.path.abspath(testdata_dir)) diff --git a/tests/functional/workflows/java_gradle/test_java_utils.py b/tests/functional/workflows/java_gradle/test_java_utils.py index a071dbab7..e8d4d0478 100644 --- a/tests/functional/workflows/java_gradle/test_java_utils.py +++ b/tests/functional/workflows/java_gradle/test_java_utils.py @@ -8,50 +8,46 @@ class TestOSUtils(TestCase): - def setUp(self): self.src = tempfile.mkdtemp() self.dst = tempfile.mkdtemp() self.os_utils = utils.OSUtils() def test_popen_runs_a_process_and_returns_outcome(self): - cwd_py = os.path.join(os.path.dirname(__file__), '..', '..', 'testdata', 'cwd.py') - p = self.os_utils.popen([sys.executable, cwd_py], - stdout=self.os_utils.pipe, - stderr=self.os_utils.pipe) + cwd_py = os.path.join(os.path.dirname(__file__), "..", "..", "testdata", "cwd.py") + p = self.os_utils.popen([sys.executable, cwd_py], stdout=self.os_utils.pipe, stderr=self.os_utils.pipe) out, err = p.communicate() self.assertEqual(p.returncode, 0) - self.assertEqual(out.decode('utf8').strip(), os.getcwd()) + self.assertEqual(out.decode("utf8").strip(), os.getcwd()) def test_popen_can_accept_cwd(self): - testdata_dir = os.path.join(os.path.dirname(__file__), '..', '..', 'testdata') - p = self.os_utils.popen([sys.executable, 'cwd.py'], - stdout=self.os_utils.pipe, - stderr=self.os_utils.pipe, - cwd=testdata_dir) + testdata_dir = os.path.join(os.path.dirname(__file__), "..", "..", "testdata") + p = self.os_utils.popen( + [sys.executable, "cwd.py"], stdout=self.os_utils.pipe, stderr=self.os_utils.pipe, cwd=testdata_dir + ) out, err = p.communicate() self.assertEqual(p.returncode, 0) - self.assertEqual(out.decode('utf8').strip(), os.path.abspath(testdata_dir)) + self.assertEqual(out.decode("utf8").strip(), os.path.abspath(testdata_dir)) def test_listdir(self): - names = ['a', 'b', 'c'] + names = ["a", "b", "c"] for n in names: self.new_file(self.src, n) self.assertEquals(set(names), set(self.os_utils.listdir(self.src))) def test_copy(self): - f = self.new_file(self.src, 'a') - expected = os.path.join(self.dst, 'a') + f = self.new_file(self.src, "a") + expected = os.path.join(self.dst, "a") copy_ret = self.os_utils.copy(f, expected) self.assertEquals(expected, copy_ret) - self.assertTrue('a' in os.listdir(self.dst)) + self.assertTrue("a" in os.listdir(self.dst)) def test_exists(self): - self.new_file(self.src, 'foo') - self.assertTrue(self.os_utils.exists(os.path.join(self.src, 'foo'))) + self.new_file(self.src, "foo") + self.assertTrue(self.os_utils.exists(os.path.join(self.src, "foo"))) def new_file(self, d, name): p = os.path.join(d, name) - with open(p, 'w') as f: + with open(p, "w") as f: f.close() return p diff --git a/tests/functional/workflows/java_maven/test_java_utils.py b/tests/functional/workflows/java_maven/test_java_utils.py index 621e55e68..9f0bb4c2e 100644 --- a/tests/functional/workflows/java_maven/test_java_utils.py +++ b/tests/functional/workflows/java_maven/test_java_utils.py @@ -8,50 +8,46 @@ class TestOSUtils(TestCase): - def setUp(self): self.src = tempfile.mkdtemp() self.dst = tempfile.mkdtemp() self.os_utils = utils.OSUtils() def test_popen_runs_a_process_and_returns_outcome(self): - cwd_py = os.path.join(os.path.dirname(__file__), '..', '..', 'testdata', 'cwd.py') - p = self.os_utils.popen([sys.executable, cwd_py], - stdout=self.os_utils.pipe, - stderr=self.os_utils.pipe) + cwd_py = os.path.join(os.path.dirname(__file__), "..", "..", "testdata", "cwd.py") + p = self.os_utils.popen([sys.executable, cwd_py], stdout=self.os_utils.pipe, stderr=self.os_utils.pipe) out, err = p.communicate() self.assertEqual(p.returncode, 0) - self.assertEqual(out.decode('utf8').strip(), os.getcwd()) + self.assertEqual(out.decode("utf8").strip(), os.getcwd()) def test_popen_can_accept_cwd(self): - testdata_dir = os.path.join(os.path.dirname(__file__), '..', '..', 'testdata') - p = self.os_utils.popen([sys.executable, 'cwd.py'], - stdout=self.os_utils.pipe, - stderr=self.os_utils.pipe, - cwd=testdata_dir) + testdata_dir = os.path.join(os.path.dirname(__file__), "..", "..", "testdata") + p = self.os_utils.popen( + [sys.executable, "cwd.py"], stdout=self.os_utils.pipe, stderr=self.os_utils.pipe, cwd=testdata_dir + ) out, err = p.communicate() self.assertEqual(p.returncode, 0) - self.assertEqual(out.decode('utf8').strip(), os.path.abspath(testdata_dir)) + self.assertEqual(out.decode("utf8").strip(), os.path.abspath(testdata_dir)) def test_listdir(self): - names = ['a', 'b', 'c'] + names = ["a", "b", "c"] for n in names: self.new_file(self.src, n) self.assertEquals(set(names), set(self.os_utils.listdir(self.src))) def test_copy(self): - f = self.new_file(self.src, 'a') - expected = os.path.join(self.dst, 'a') + f = self.new_file(self.src, "a") + expected = os.path.join(self.dst, "a") copy_ret = self.os_utils.copy(f, expected) self.assertEquals(expected, copy_ret) - self.assertTrue('a' in os.listdir(self.dst)) + self.assertTrue("a" in os.listdir(self.dst)) def test_exists(self): - self.new_file(self.src, 'foo') - self.assertTrue(self.os_utils.exists(os.path.join(self.src, 'foo'))) + self.new_file(self.src, "foo") + self.assertTrue(self.os_utils.exists(os.path.join(self.src, "foo"))) def new_file(self, d, name): p = os.path.join(d, name) - with open(p, 'w') as f: + with open(p, "w") as f: f.close() return p diff --git a/tests/functional/workflows/nodejs_npm/test_utils.py b/tests/functional/workflows/nodejs_npm/test_utils.py index 01fd52f01..bd39e0ff3 100644 --- a/tests/functional/workflows/nodejs_npm/test_utils.py +++ b/tests/functional/workflows/nodejs_npm/test_utils.py @@ -9,7 +9,6 @@ class TestOSUtils(TestCase): - def setUp(self): self.osutils = utils.OSUtils() @@ -65,7 +64,7 @@ def test_file_exists_checking_if_file_exists_in_a_dir(self): self.assertTrue(self.osutils.file_exists(existing_file)) self.assertFalse(self.osutils.file_exists(nonexisting_file)) - + def test_extract_tarfile_unpacks_a_tar(self): test_tar = os.path.join(os.path.dirname(__file__), "test_data", "test.tgz") @@ -87,43 +86,40 @@ def test_dirname_returns_directory_for_path(self): def test_abspath_returns_absolute_path(self): - result = self.osutils.abspath('.') + result = self.osutils.abspath(".") self.assertTrue(os.path.isabs(result)) - self.assertEqual(result, os.path.abspath('.')) + self.assertEqual(result, os.path.abspath(".")) def test_joinpath_joins_path_components(self): - result = self.osutils.joinpath('a', 'b', 'c') + result = self.osutils.joinpath("a", "b", "c") - self.assertEqual(result, os.path.join('a', 'b', 'c')) + self.assertEqual(result, os.path.join("a", "b", "c")) def test_popen_runs_a_process_and_returns_outcome(self): - cwd_py = os.path.join(os.path.dirname(__file__), '..', '..', 'testdata', 'cwd.py') + cwd_py = os.path.join(os.path.dirname(__file__), "..", "..", "testdata", "cwd.py") - p = self.osutils.popen([sys.executable, cwd_py], - stdout=self.osutils.pipe, - stderr=self.osutils.pipe) + p = self.osutils.popen([sys.executable, cwd_py], stdout=self.osutils.pipe, stderr=self.osutils.pipe) out, err = p.communicate() self.assertEqual(p.returncode, 0) - self.assertEqual(out.decode('utf8').strip(), os.getcwd()) + self.assertEqual(out.decode("utf8").strip(), os.getcwd()) def test_popen_can_accept_cwd(self): - testdata_dir = os.path.join(os.path.dirname(__file__), '..', '..', 'testdata') + testdata_dir = os.path.join(os.path.dirname(__file__), "..", "..", "testdata") - p = self.osutils.popen([sys.executable, 'cwd.py'], - stdout=self.osutils.pipe, - stderr=self.osutils.pipe, - cwd=testdata_dir) + p = self.osutils.popen( + [sys.executable, "cwd.py"], stdout=self.osutils.pipe, stderr=self.osutils.pipe, cwd=testdata_dir + ) out, err = p.communicate() self.assertEqual(p.returncode, 0) - self.assertEqual(out.decode('utf8').strip(), os.path.abspath(testdata_dir)) + self.assertEqual(out.decode("utf8").strip(), os.path.abspath(testdata_dir)) diff --git a/tests/functional/workflows/python_pip/test_packager.py b/tests/functional/workflows/python_pip/test_packager.py index 58618b768..4c73efa9c 100644 --- a/tests/functional/workflows/python_pip/test_packager.py +++ b/tests/functional/workflows/python_pip/test_packager.py @@ -11,24 +11,22 @@ from aws_lambda_builders.workflows.python_pip.packager import PipRunner from aws_lambda_builders.workflows.python_pip.packager import DependencyBuilder from aws_lambda_builders.workflows.python_pip.packager import Package -from aws_lambda_builders.workflows.python_pip.packager import \ - MissingDependencyError +from aws_lambda_builders.workflows.python_pip.packager import MissingDependencyError from aws_lambda_builders.workflows.python_pip.packager import SubprocessPip from aws_lambda_builders.workflows.python_pip.packager import SDistMetadataFetcher -from aws_lambda_builders.workflows.python_pip.packager import \ - InvalidSourceDistributionNameError +from aws_lambda_builders.workflows.python_pip.packager import InvalidSourceDistributionNameError from aws_lambda_builders.workflows.python_pip.packager import get_lambda_abi from aws_lambda_builders.workflows.python_pip.compat import pip_no_compile_c_env_vars from aws_lambda_builders.workflows.python_pip.compat import pip_no_compile_c_shim from aws_lambda_builders.workflows.python_pip.utils import OSUtils -FakePipCall = namedtuple('FakePipEntry', ['args', 'env_vars', 'shim']) +FakePipCall = namedtuple("FakePipEntry", ["args", "env_vars", "shim"]) def _create_app_structure(tmpdir): - appdir = tmpdir.mkdir('app') - appdir.join('app.py').write('# Test app') + appdir = tmpdir.mkdir("app") + appdir.join("app.py").write("# Test app") return appdir @@ -44,20 +42,13 @@ def sdist_builder(): class FakeSdistBuilder(object): - _SETUP_PY = ( - 'from setuptools import setup\n' - 'setup(\n' - ' name="%s",\n' - ' version="%s"\n' - ')\n' - ) + _SETUP_PY = "from setuptools import setup\n" "setup(\n" ' name="%s",\n' ' version="%s"\n' ")\n" def write_fake_sdist(self, directory, name, version): - filename = '%s-%s.zip' % (name, version) - path = '%s/%s' % (directory, filename) - with zipfile.ZipFile(path, 'w', - compression=zipfile.ZIP_DEFLATED) as z: - z.writestr('sdist/setup.py', self._SETUP_PY % (name, version)) + filename = "%s-%s.zip" % (name, version) + path = "%s/%s" % (directory, filename) + with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as z: + z.writestr("sdist/setup.py", self._SETUP_PY % (name, version)) return directory, filename @@ -77,7 +68,7 @@ def __init__(self): self._calls = defaultdict(lambda: []) self._call_history = [] self._side_effects = defaultdict(lambda: []) - self._return_tuple = (0, b'', b'') + self._return_tuple = (0, b"", b"") def main(self, args, env_vars=None, shim=None): cmd, args = args[0], args[1:] @@ -85,11 +76,14 @@ def main(self, args, env_vars=None, shim=None): try: side_effects = self._side_effects[cmd].pop(0) for side_effect in side_effects: - self._call_history.append(( - FakePipCall(args, env_vars, shim), - FakePipCall(side_effect.expected_args, - side_effect.expected_env_vars, - side_effect.expected_shim))) + self._call_history.append( + ( + FakePipCall(args, env_vars, shim), + FakePipCall( + side_effect.expected_args, side_effect.expected_env_vars, side_effect.expected_shim + ), + ) + ) side_effect.execute(args) except IndexError: pass @@ -99,15 +93,10 @@ def set_return_tuple(self, rc, out, err): self._return_tuple = (rc, out, err) def packages_to_download(self, expected_args, packages, whl_contents=None): - side_effects = [PipSideEffect(pkg, - '--dest', - expected_args, - whl_contents) - for pkg in packages] - self._side_effects['download'].append(side_effects) - - def wheels_to_build(self, expected_args, wheels_to_build, - expected_env_vars=None, expected_shim=None): + side_effects = [PipSideEffect(pkg, "--dest", expected_args, whl_contents) for pkg in packages] + self._side_effects["download"].append(side_effects) + + def wheels_to_build(self, expected_args, wheels_to_build, expected_env_vars=None, expected_shim=None): # The SubprocessPip class handles injecting the # subprocess_python_base_environ into the env vars if needed, # so at this level of abstraction the env vars just default @@ -115,12 +104,14 @@ def wheels_to_build(self, expected_args, wheels_to_build, if expected_env_vars is None: expected_env_vars = {} if expected_shim is None: - expected_shim = '' - side_effects = [PipSideEffect(pkg, '--wheel-dir', expected_args, - expected_env_vars=expected_env_vars, - expected_shim=expected_shim) - for pkg in wheels_to_build] - self._side_effects['wheel'].append(side_effects) + expected_shim = "" + side_effects = [ + PipSideEffect( + pkg, "--wheel-dir", expected_args, expected_env_vars=expected_env_vars, expected_shim=expected_shim + ) + for pkg in wheels_to_build + ] + self._side_effects["wheel"].append(side_effects) @property def calls(self): @@ -135,38 +126,34 @@ def validate(self): class PipSideEffect(object): - def __init__(self, filename, dirarg, expected_args, whl_contents=None, - expected_env_vars=None, expected_shim=None): + def __init__(self, filename, dirarg, expected_args, whl_contents=None, expected_env_vars=None, expected_shim=None): self._filename = filename - self._package_name = filename.split('-')[0] + self._package_name = filename.split("-")[0] self._dirarg = dirarg self.expected_args = expected_args self.expected_env_vars = expected_env_vars self.expected_shim = expected_shim if whl_contents is None: - whl_contents = ['{package_name}/placeholder'] + whl_contents = ["{package_name}/placeholder"] self._whl_contents = whl_contents def _build_fake_whl(self, directory, filename): filepath = os.path.join(directory, filename) if not os.path.isfile(filepath): package = Package(directory, filename) - with zipfile.ZipFile(filepath, 'w') as z: + with zipfile.ZipFile(filepath, "w") as z: for content_path in self._whl_contents: - z.writestr(content_path.format( - package_name=self._package_name, - data_dir=package.data_dir - ), b'') + z.writestr(content_path.format(package_name=self._package_name, data_dir=package.data_dir), b"") def _build_fake_sdist(self, filepath): # tar.gz is the same no reason to test it here as it is tested in # unit.deploy.TestSdistMetadataFetcher - assert filepath.endswith('.zip') + assert filepath.endswith(".zip") components = os.path.split(filepath) prefix, filename = components[:-1], components[-1] directory = os.path.join(*prefix) filename_without_ext = filename[:-4] - pkg_name, pkg_version = filename_without_ext.split('-') + pkg_name, pkg_version = filename_without_ext.split("-") builder = FakeSdistBuilder() builder.write_fake_sdist(directory, pkg_name, pkg_version) @@ -179,7 +166,7 @@ def execute(self, args): target_dir = args[i + 1] if target_dir: filepath = os.path.join(target_dir, self._filename) - if filepath.endswith('.whl'): + if filepath.endswith(".whl"): self._build_fake_whl(target_dir, self._filename) else: self._build_fake_sdist(filepath) @@ -195,23 +182,22 @@ def empty_env_osutils(): class EmptyEnv(object): def environ(self): return {} + return EmptyEnv() @pytest.fixture def pip_runner(empty_env_osutils): pip = FakePip() - pip_runner = PipRunner(python_exe=sys.executable, - pip=pip, - osutils=empty_env_osutils) + pip_runner = PipRunner(python_exe=sys.executable, pip=pip, osutils=empty_env_osutils) return pip, pip_runner class TestDependencyBuilder(object): def _write_requirements_txt(self, packages, directory): - contents = '\n'.join(packages) - filepath = os.path.join(directory, 'requirements.txt') - with open(filepath, 'w') as f: + contents = "\n".join(packages) + filepath = os.path.join(directory, "requirements.txt") + with open(filepath, "w") as f: f.write(contents) def _make_appdir_and_dependency_builder(self, reqs, tmpdir, runner): @@ -221,75 +207,61 @@ def _make_appdir_and_dependency_builder(self, reqs, tmpdir, runner): return appdir, builder def test_can_build_local_dir_as_whl(self, tmpdir, pip_runner, osutils): - reqs = ['../foo'] + reqs = ["../foo"] pip, runner = pip_runner - appdir, builder = self._make_appdir_and_dependency_builder( - reqs, tmpdir, runner) - requirements_file = os.path.join(appdir, 'requirements.txt') - pip.set_return_tuple(0, (b"Processing ../foo\n" - b" Link is a directory," - b" ignoring download_dir"), b'') + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") + pip.set_return_tuple(0, (b"Processing ../foo\n" b" Link is a directory," b" ignoring download_dir"), b"") pip.wheels_to_build( - expected_args=['--no-deps', '--wheel-dir', mock.ANY, '../foo'], - wheels_to_build=[ - 'foo-1.2-cp36-none-any.whl' - ] + expected_args=["--no-deps", "--wheel-dir", mock.ANY, "../foo"], + wheels_to_build=["foo-1.2-cp36-none-any.whl"], ) - site_packages = os.path.join(appdir, 'site-packages') + site_packages = os.path.join(appdir, "site-packages") with osutils.tempdir() as scratch_dir: - builder.build_site_packages( - requirements_file, site_packages, scratch_dir) + builder.build_site_packages(requirements_file, site_packages, scratch_dir) installed_packages = os.listdir(site_packages) pip.validate() - assert ['foo'] == installed_packages + assert ["foo"] == installed_packages def test_can_get_whls_all_manylinux(self, tmpdir, pip_runner, osutils): - reqs = ['foo', 'bar'] + reqs = ["foo", "bar"] pip, runner = pip_runner - appdir, builder = self._make_appdir_and_dependency_builder( - reqs, tmpdir, runner) - requirements_file = os.path.join(appdir, 'requirements.txt') + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( - expected_args=['-r', requirements_file, '--dest', mock.ANY], - packages=[ - 'foo-1.2-cp36-cp36m-manylinux1_x86_64.whl', - 'bar-1.2-cp36-cp36m-manylinux1_x86_64.whl' - ] + expected_args=["-r", requirements_file, "--dest", mock.ANY], + packages=["foo-1.2-cp36-cp36m-manylinux1_x86_64.whl", "bar-1.2-cp36-cp36m-manylinux1_x86_64.whl"], ) - site_packages = os.path.join(appdir, '.chalice.', 'site-packages') + site_packages = os.path.join(appdir, ".chalice.", "site-packages") with osutils.tempdir() as scratch_dir: - builder.build_site_packages( - requirements_file, site_packages, scratch_dir) + builder.build_site_packages(requirements_file, site_packages, scratch_dir) installed_packages = os.listdir(site_packages) pip.validate() for req in reqs: assert req in installed_packages - def test_can_use_abi3_whl_for_any_python3(self, tmpdir, - pip_runner, osutils): - reqs = ['foo', 'bar', 'baz', 'qux'] + def test_can_use_abi3_whl_for_any_python3(self, tmpdir, pip_runner, osutils): + reqs = ["foo", "bar", "baz", "qux"] pip, runner = pip_runner - appdir, builder = self._make_appdir_and_dependency_builder( - reqs, tmpdir, runner) - requirements_file = os.path.join(appdir, 'requirements.txt') + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( - expected_args=['-r', requirements_file, '--dest', mock.ANY], + expected_args=["-r", requirements_file, "--dest", mock.ANY], packages=[ - 'foo-1.2-cp33-abi3-manylinux1_x86_64.whl', - 'bar-1.2-cp34-abi3-manylinux1_x86_64.whl', - 'baz-1.2-cp35-abi3-manylinux1_x86_64.whl', - 'qux-1.2-cp36-abi3-manylinux1_x86_64.whl', - ] + "foo-1.2-cp33-abi3-manylinux1_x86_64.whl", + "bar-1.2-cp34-abi3-manylinux1_x86_64.whl", + "baz-1.2-cp35-abi3-manylinux1_x86_64.whl", + "qux-1.2-cp36-abi3-manylinux1_x86_64.whl", + ], ) - site_packages = os.path.join(appdir, '.chalice.', 'site-packages') + site_packages = os.path.join(appdir, ".chalice.", "site-packages") with osutils.tempdir() as scratch_dir: - builder.build_site_packages( - requirements_file, site_packages, scratch_dir) + builder.build_site_packages(requirements_file, site_packages, scratch_dir) installed_packages = os.listdir(site_packages) pip.validate() @@ -297,23 +269,19 @@ def test_can_use_abi3_whl_for_any_python3(self, tmpdir, assert req in installed_packages def test_can_expand_purelib_whl(self, tmpdir, pip_runner, osutils): - reqs = ['foo'] + reqs = ["foo"] pip, runner = pip_runner - appdir, builder = self._make_appdir_and_dependency_builder( - reqs, tmpdir, runner) - requirements_file = os.path.join(appdir, 'requirements.txt') + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( - expected_args=['-r', requirements_file, '--dest', mock.ANY], - packages=[ - 'foo-1.2-cp36-cp36m-manylinux1_x86_64.whl' - ], - whl_contents=['foo-1.2.data/purelib/foo/'] + expected_args=["-r", requirements_file, "--dest", mock.ANY], + packages=["foo-1.2-cp36-cp36m-manylinux1_x86_64.whl"], + whl_contents=["foo-1.2.data/purelib/foo/"], ) - site_packages = os.path.join(appdir, '.chalice.', 'site-packages') + site_packages = os.path.join(appdir, ".chalice.", "site-packages") with osutils.tempdir() as scratch_dir: - builder.build_site_packages( - requirements_file, site_packages, scratch_dir) + builder.build_site_packages(requirements_file, site_packages, scratch_dir) installed_packages = os.listdir(site_packages) pip.validate() @@ -321,23 +289,19 @@ def test_can_expand_purelib_whl(self, tmpdir, pip_runner, osutils): assert req in installed_packages def test_can_expand_platlib_whl(self, tmpdir, pip_runner, osutils): - reqs = ['foo'] + reqs = ["foo"] pip, runner = pip_runner - appdir, builder = self._make_appdir_and_dependency_builder( - reqs, tmpdir, runner) - requirements_file = os.path.join(appdir, 'requirements.txt') + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( - expected_args=['-r', requirements_file, '--dest', mock.ANY], - packages=[ - 'foo-1.2-cp36-cp36m-manylinux1_x86_64.whl' - ], - whl_contents=['foo-1.2.data/platlib/foo/'] + expected_args=["-r", requirements_file, "--dest", mock.ANY], + packages=["foo-1.2-cp36-cp36m-manylinux1_x86_64.whl"], + whl_contents=["foo-1.2.data/platlib/foo/"], ) - site_packages = os.path.join(appdir, '.chalice.', 'site-packages') + site_packages = os.path.join(appdir, ".chalice.", "site-packages") with osutils.tempdir() as scratch_dir: - builder.build_site_packages( - requirements_file, site_packages, scratch_dir) + builder.build_site_packages(requirements_file, site_packages, scratch_dir) installed_packages = os.listdir(site_packages) pip.validate() @@ -347,26 +311,19 @@ def test_can_expand_platlib_whl(self, tmpdir, pip_runner, osutils): def test_can_expand_platlib_and_purelib(self, tmpdir, pip_runner, osutils): # This wheel installs two importable libraries foo and bar, one from # the wheels purelib and one from its platlib. - reqs = ['foo', 'bar'] + reqs = ["foo", "bar"] pip, runner = pip_runner - appdir, builder = self._make_appdir_and_dependency_builder( - reqs, tmpdir, runner) - requirements_file = os.path.join(appdir, 'requirements.txt') + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( - expected_args=['-r', requirements_file, '--dest', mock.ANY], - packages=[ - 'foo-1.2-cp36-cp36m-manylinux1_x86_64.whl' - ], - whl_contents=[ - 'foo-1.2.data/platlib/foo/', - 'foo-1.2.data/purelib/bar/' - ] + expected_args=["-r", requirements_file, "--dest", mock.ANY], + packages=["foo-1.2-cp36-cp36m-manylinux1_x86_64.whl"], + whl_contents=["foo-1.2.data/platlib/foo/", "foo-1.2.data/purelib/bar/"], ) - site_packages = os.path.join(appdir, '.chalice.', 'site-packages') + site_packages = os.path.join(appdir, ".chalice.", "site-packages") with osutils.tempdir() as scratch_dir: - builder.build_site_packages( - requirements_file, site_packages, scratch_dir) + builder.build_site_packages(requirements_file, site_packages, scratch_dir) installed_packages = os.listdir(site_packages) pip.validate() @@ -376,119 +333,93 @@ def test_can_expand_platlib_and_purelib(self, tmpdir, pip_runner, osutils): def test_does_ignore_data(self, tmpdir, pip_runner, osutils): # Make sure the wheel installer does not copy the data directory # up to the root. - reqs = ['foo'] + reqs = ["foo"] pip, runner = pip_runner - appdir, builder = self._make_appdir_and_dependency_builder( - reqs, tmpdir, runner) - requirements_file = os.path.join(appdir, 'requirements.txt') + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( - expected_args=['-r', requirements_file, '--dest', mock.ANY], - packages=[ - 'foo-1.2-cp36-cp36m-manylinux1_x86_64.whl' - ], - whl_contents=[ - 'foo/placeholder', - 'foo-1.2.data/data/bar/' - ] + expected_args=["-r", requirements_file, "--dest", mock.ANY], + packages=["foo-1.2-cp36-cp36m-manylinux1_x86_64.whl"], + whl_contents=["foo/placeholder", "foo-1.2.data/data/bar/"], ) - site_packages = os.path.join(appdir, '.chalice.', 'site-packages') + site_packages = os.path.join(appdir, ".chalice.", "site-packages") with osutils.tempdir() as scratch_dir: - builder.build_site_packages( - requirements_file, site_packages, scratch_dir) + builder.build_site_packages(requirements_file, site_packages, scratch_dir) installed_packages = os.listdir(site_packages) pip.validate() for req in reqs: assert req in installed_packages - assert 'bar' not in installed_packages + assert "bar" not in installed_packages def test_does_ignore_include(self, tmpdir, pip_runner, osutils): # Make sure the wheel installer does not copy the includes directory # up to the root. - reqs = ['foo'] + reqs = ["foo"] pip, runner = pip_runner - appdir, builder = self._make_appdir_and_dependency_builder( - reqs, tmpdir, runner) - requirements_file = os.path.join(appdir, 'requirements.txt') + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( - expected_args=['-r', requirements_file, '--dest', mock.ANY], - packages=[ - 'foo-1.2-cp36-cp36m-manylinux1_x86_64.whl' - ], - whl_contents=[ - 'foo/placeholder', - 'foo.1.2.data/includes/bar/' - ] + expected_args=["-r", requirements_file, "--dest", mock.ANY], + packages=["foo-1.2-cp36-cp36m-manylinux1_x86_64.whl"], + whl_contents=["foo/placeholder", "foo.1.2.data/includes/bar/"], ) - site_packages = os.path.join(appdir, '.chalice.', 'site-packages') + site_packages = os.path.join(appdir, ".chalice.", "site-packages") with osutils.tempdir() as scratch_dir: - builder.build_site_packages( - requirements_file, site_packages, scratch_dir) + builder.build_site_packages(requirements_file, site_packages, scratch_dir) installed_packages = os.listdir(site_packages) pip.validate() for req in reqs: assert req in installed_packages - assert 'bar' not in installed_packages + assert "bar" not in installed_packages def test_does_ignore_scripts(self, tmpdir, pip_runner, osutils): # Make sure the wheel isntaller does not copy the scripts directory # up to the root. - reqs = ['foo'] + reqs = ["foo"] pip, runner = pip_runner - appdir, builder = self._make_appdir_and_dependency_builder( - reqs, tmpdir, runner) - requirements_file = os.path.join(appdir, 'requirements.txt') + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( - expected_args=['-r', requirements_file, '--dest', mock.ANY], - packages=[ - 'foo-1.2-cp36-cp36m-manylinux1_x86_64.whl' - ], - whl_contents=[ - '{package_name}/placeholder', - '{data_dir}/scripts/bar/placeholder' - ] + expected_args=["-r", requirements_file, "--dest", mock.ANY], + packages=["foo-1.2-cp36-cp36m-manylinux1_x86_64.whl"], + whl_contents=["{package_name}/placeholder", "{data_dir}/scripts/bar/placeholder"], ) - site_packages = os.path.join(appdir, '.chalice.', 'site-packages') + site_packages = os.path.join(appdir, ".chalice.", "site-packages") with osutils.tempdir() as scratch_dir: - builder.build_site_packages( - requirements_file, site_packages, scratch_dir) + builder.build_site_packages(requirements_file, site_packages, scratch_dir) installed_packages = os.listdir(site_packages) pip.validate() for req in reqs: assert req in installed_packages - assert 'bar' not in installed_packages + assert "bar" not in installed_packages - def test_can_expand_platlib_and_platlib_and_root(self, tmpdir, - pip_runner, osutils): + def test_can_expand_platlib_and_platlib_and_root(self, tmpdir, pip_runner, osutils): # This wheel installs three import names foo, bar and baz. # they are from the root install directory and the platlib and purelib # subdirectories in the platlib. - reqs = ['foo', 'bar', 'baz'] + reqs = ["foo", "bar", "baz"] pip, runner = pip_runner - appdir, builder = self._make_appdir_and_dependency_builder( - reqs, tmpdir, runner) - requirements_file = os.path.join(appdir, 'requirements.txt') + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( - expected_args=['-r', requirements_file, '--dest', mock.ANY], - packages=[ - 'foo-1.2-cp36-cp36m-manylinux1_x86_64.whl' - ], + expected_args=["-r", requirements_file, "--dest", mock.ANY], + packages=["foo-1.2-cp36-cp36m-manylinux1_x86_64.whl"], whl_contents=[ - '{package_name}/placeholder', - '{data_dir}/platlib/bar/placeholder', - '{data_dir}/purelib/baz/placeholder' - ] + "{package_name}/placeholder", + "{data_dir}/platlib/bar/placeholder", + "{data_dir}/purelib/baz/placeholder", + ], ) - site_packages = os.path.join(appdir, '.chalice.', 'site-packages') + site_packages = os.path.join(appdir, ".chalice.", "site-packages") with osutils.tempdir() as scratch_dir: - builder.build_site_packages( - requirements_file, site_packages, scratch_dir) + builder.build_site_packages(requirements_file, site_packages, scratch_dir) installed_packages = os.listdir(site_packages) pip.validate() @@ -496,24 +427,22 @@ def test_can_expand_platlib_and_platlib_and_root(self, tmpdir, assert req in installed_packages def test_can_get_whls_mixed_compat(self, tmpdir, osutils, pip_runner): - reqs = ['foo', 'bar', 'baz'] + reqs = ["foo", "bar", "baz"] pip, runner = pip_runner - appdir, builder = self._make_appdir_and_dependency_builder( - reqs, tmpdir, runner) - requirements_file = os.path.join(appdir, 'requirements.txt') + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( - expected_args=['-r', requirements_file, '--dest', mock.ANY], + expected_args=["-r", requirements_file, "--dest", mock.ANY], packages=[ - 'foo-1.0-cp36-none-any.whl', - 'bar-1.2-cp36-cp36m-manylinux1_x86_64.whl', - 'baz-1.5-cp36-cp36m-linux_x86_64.whl' - ] + "foo-1.0-cp36-none-any.whl", + "bar-1.2-cp36-cp36m-manylinux1_x86_64.whl", + "baz-1.5-cp36-cp36m-linux_x86_64.whl", + ], ) - site_packages = os.path.join(appdir, '.chalice.', 'site-packages') + site_packages = os.path.join(appdir, ".chalice.", "site-packages") with osutils.tempdir() as scratch_dir: - builder.build_site_packages( - requirements_file, site_packages, scratch_dir) + builder.build_site_packages(requirements_file, site_packages, scratch_dir) installed_packages = os.listdir(site_packages) pip.validate() @@ -521,143 +450,125 @@ def test_can_get_whls_mixed_compat(self, tmpdir, osutils, pip_runner): assert req in installed_packages def test_can_get_py27_whls(self, tmpdir, osutils, pip_runner): - reqs = ['foo', 'bar', 'baz'] + reqs = ["foo", "bar", "baz"] pip, runner = pip_runner - appdir, builder = self._make_appdir_and_dependency_builder( - reqs, tmpdir, runner) - requirements_file = os.path.join(appdir, 'requirements.txt') + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( - expected_args=['-r', requirements_file, '--dest', mock.ANY], + expected_args=["-r", requirements_file, "--dest", mock.ANY], packages=[ - 'foo-1.0-cp27-none-any.whl', - 'bar-1.2-cp27-none-manylinux1_x86_64.whl', - 'baz-1.5-cp27-cp27mu-linux_x86_64.whl' - ] + "foo-1.0-cp27-none-any.whl", + "bar-1.2-cp27-none-manylinux1_x86_64.whl", + "baz-1.5-cp27-cp27mu-linux_x86_64.whl", + ], ) - site_packages = os.path.join(appdir, '.chalice.', 'site-packages') + site_packages = os.path.join(appdir, ".chalice.", "site-packages") with osutils.tempdir() as scratch_dir: - builder.build_site_packages( - requirements_file, site_packages, scratch_dir) + builder.build_site_packages(requirements_file, site_packages, scratch_dir) installed_packages = os.listdir(site_packages) pip.validate() for req in reqs: assert req in installed_packages - def test_does_fail_on_invalid_local_package(self, tmpdir, osutils, - pip_runner): - reqs = ['../foo'] + def test_does_fail_on_invalid_local_package(self, tmpdir, osutils, pip_runner): + reqs = ["../foo"] pip, runner = pip_runner - appdir, builder = self._make_appdir_and_dependency_builder( - reqs, tmpdir, runner) - requirements_file = os.path.join(appdir, 'requirements.txt') - pip.set_return_tuple(0, (b"Processing ../foo\n" - b" Link is a directory," - b" ignoring download_dir"), b'') + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") + pip.set_return_tuple(0, (b"Processing ../foo\n" b" Link is a directory," b" ignoring download_dir"), b"") pip.wheels_to_build( - expected_args=['--no-deps', '--wheel-dir', mock.ANY, '../foo'], - wheels_to_build=[ - 'foo-1.2-cp36-cp36m-macosx_10_6_intel.whl' - ] + expected_args=["--no-deps", "--wheel-dir", mock.ANY, "../foo"], + wheels_to_build=["foo-1.2-cp36-cp36m-macosx_10_6_intel.whl"], ) - site_packages = os.path.join(appdir, '.chalice.', 'site-packages') + site_packages = os.path.join(appdir, ".chalice.", "site-packages") with osutils.tempdir() as scratch_dir: with pytest.raises(MissingDependencyError) as e: - builder.build_site_packages( - requirements_file, site_packages, scratch_dir) + builder.build_site_packages(requirements_file, site_packages, scratch_dir) installed_packages = os.listdir(site_packages) missing_packages = list(e.value.missing) pip.validate() assert len(missing_packages) == 1 - assert missing_packages[0].identifier == 'foo==1.2' + assert missing_packages[0].identifier == "foo==1.2" assert len(installed_packages) == 0 - def test_does_fail_on_narrow_py27_unicode(self, tmpdir, osutils, - pip_runner): - reqs = ['baz'] + def test_does_fail_on_narrow_py27_unicode(self, tmpdir, osutils, pip_runner): + reqs = ["baz"] pip, runner = pip_runner - appdir, builder = self._make_appdir_and_dependency_builder( - reqs, tmpdir, runner) - requirements_file = os.path.join(appdir, 'requirements.txt') + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( - expected_args=['-r', requirements_file, '--dest', mock.ANY], - packages=[ - 'baz-1.5-cp27-cp27m-linux_x86_64.whl' - ] + expected_args=["-r", requirements_file, "--dest", mock.ANY], + packages=["baz-1.5-cp27-cp27m-linux_x86_64.whl"], ) - site_packages = os.path.join(appdir, '.chalice.', 'site-packages') + site_packages = os.path.join(appdir, ".chalice.", "site-packages") with osutils.tempdir() as scratch_dir: with pytest.raises(MissingDependencyError) as e: - builder.build_site_packages( - requirements_file, site_packages, scratch_dir) + builder.build_site_packages(requirements_file, site_packages, scratch_dir) installed_packages = os.listdir(site_packages) missing_packages = list(e.value.missing) pip.validate() assert len(missing_packages) == 1 - assert missing_packages[0].identifier == 'baz==1.5' + assert missing_packages[0].identifier == "baz==1.5" assert len(installed_packages) == 0 def test_does_fail_on_python_1_whl(self, tmpdir, osutils, pip_runner): - reqs = ['baz'] + reqs = ["baz"] pip, runner = pip_runner - appdir, builder = self._make_appdir_and_dependency_builder( - reqs, tmpdir, runner) - requirements_file = os.path.join(appdir, 'requirements.txt') + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( - expected_args=['-r', requirements_file, '--dest', mock.ANY], - packages=[ - 'baz-1.5-cp14-cp14m-linux_x86_64.whl' - ] + expected_args=["-r", requirements_file, "--dest", mock.ANY], + packages=["baz-1.5-cp14-cp14m-linux_x86_64.whl"], ) - site_packages = os.path.join(appdir, '.chalice.', 'site-packages') + site_packages = os.path.join(appdir, ".chalice.", "site-packages") with osutils.tempdir() as scratch_dir: with pytest.raises(MissingDependencyError) as e: - builder.build_site_packages( - requirements_file, site_packages, scratch_dir) + builder.build_site_packages(requirements_file, site_packages, scratch_dir) installed_packages = os.listdir(site_packages) missing_packages = list(e.value.missing) pip.validate() assert len(missing_packages) == 1 - assert missing_packages[0].identifier == 'baz==1.5' + assert missing_packages[0].identifier == "baz==1.5" assert len(installed_packages) == 0 def test_can_replace_incompat_whl(self, tmpdir, osutils, pip_runner): - reqs = ['foo', 'bar'] + reqs = ["foo", "bar"] pip, runner = pip_runner - appdir, builder = self._make_appdir_and_dependency_builder( - reqs, tmpdir, runner) - requirements_file = os.path.join(appdir, 'requirements.txt') + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( - expected_args=['-r', requirements_file, '--dest', mock.ANY], - packages=[ - 'foo-1.0-cp36-none-any.whl', - 'bar-1.2-cp36-cp36m-macosx_10_6_intel.whl' - ] + expected_args=["-r", requirements_file, "--dest", mock.ANY], + packages=["foo-1.0-cp36-none-any.whl", "bar-1.2-cp36-cp36m-macosx_10_6_intel.whl"], ) # Once the initial download has 1 incompatible whl file. The second, # more targeted download, finds manylinux1_x86_64 and downloads that. pip.packages_to_download( expected_args=[ - '--only-binary=:all:', '--no-deps', '--platform', - 'manylinux1_x86_64', '--implementation', 'cp', - '--abi', get_lambda_abi(builder.runtime), '--dest', mock.ANY, - 'bar==1.2' + "--only-binary=:all:", + "--no-deps", + "--platform", + "manylinux1_x86_64", + "--implementation", + "cp", + "--abi", + get_lambda_abi(builder.runtime), + "--dest", + mock.ANY, + "bar==1.2", ], - packages=[ - 'bar-1.2-cp36-cp36m-manylinux1_x86_64.whl' - ] + packages=["bar-1.2-cp36-cp36m-manylinux1_x86_64.whl"], ) - site_packages = os.path.join(appdir, '.chalice.', 'site-packages') + site_packages = os.path.join(appdir, ".chalice.", "site-packages") with osutils.tempdir() as scratch_dir: - builder.build_site_packages( - requirements_file, site_packages, scratch_dir) + builder.build_site_packages(requirements_file, site_packages, scratch_dir) installed_packages = os.listdir(site_packages) pip.validate() @@ -665,82 +576,70 @@ def test_can_replace_incompat_whl(self, tmpdir, osutils, pip_runner): assert req in installed_packages def test_whitelist_sqlalchemy(self, tmpdir, osutils, pip_runner): - reqs = ['sqlalchemy==1.1.18'] + reqs = ["sqlalchemy==1.1.18"] pip, runner = pip_runner - appdir, builder = self._make_appdir_and_dependency_builder( - reqs, tmpdir, runner) - requirements_file = os.path.join(appdir, 'requirements.txt') + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( - expected_args=['-r', requirements_file, '--dest', mock.ANY], - packages=[ - 'SQLAlchemy-1.1.18-cp36-cp36m-macosx_10_11_x86_64.whl' - ] + expected_args=["-r", requirements_file, "--dest", mock.ANY], + packages=["SQLAlchemy-1.1.18-cp36-cp36m-macosx_10_11_x86_64.whl"], ) pip.packages_to_download( expected_args=[ - '--only-binary=:all:', '--no-deps', '--platform', - 'manylinux1_x86_64', '--implementation', 'cp', - '--abi', get_lambda_abi(builder.runtime), '--dest', mock.ANY, - 'sqlalchemy==1.1.18' + "--only-binary=:all:", + "--no-deps", + "--platform", + "manylinux1_x86_64", + "--implementation", + "cp", + "--abi", + get_lambda_abi(builder.runtime), + "--dest", + mock.ANY, + "sqlalchemy==1.1.18", ], - packages=[ - 'SQLAlchemy-1.1.18-cp36-cp36m-macosx_10_11_x86_64.whl' - ] + packages=["SQLAlchemy-1.1.18-cp36-cp36m-macosx_10_11_x86_64.whl"], ) - site_packages = os.path.join(appdir, '.chalice.', 'site-packages') + site_packages = os.path.join(appdir, ".chalice.", "site-packages") with osutils.tempdir() as scratch_dir: - builder.build_site_packages( - requirements_file, site_packages, scratch_dir) + builder.build_site_packages(requirements_file, site_packages, scratch_dir) installed_packages = os.listdir(site_packages) pip.validate() - assert installed_packages == ['SQLAlchemy'] + assert installed_packages == ["SQLAlchemy"] def test_can_build_sdist(self, tmpdir, osutils, pip_runner): - reqs = ['foo', 'bar'] + reqs = ["foo", "bar"] pip, runner = pip_runner - appdir, builder = self._make_appdir_and_dependency_builder( - reqs, tmpdir, runner) - requirements_file = os.path.join(appdir, 'requirements.txt') + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( - expected_args=['-r', requirements_file, '--dest', mock.ANY], - packages=[ - 'foo-1.2.zip', - 'bar-1.2-cp36-cp36m-manylinux1_x86_64.whl' - ] + expected_args=["-r", requirements_file, "--dest", mock.ANY], + packages=["foo-1.2.zip", "bar-1.2-cp36-cp36m-manylinux1_x86_64.whl"], ) # Foo is built from and is pure python so it yields a compatible # wheel file. pip.wheels_to_build( - expected_args=['--no-deps', '--wheel-dir', mock.ANY, - PathArgumentEndingWith('foo-1.2.zip')], - wheels_to_build=[ - 'foo-1.2-cp36-none-any.whl' - ] + expected_args=["--no-deps", "--wheel-dir", mock.ANY, PathArgumentEndingWith("foo-1.2.zip")], + wheels_to_build=["foo-1.2-cp36-none-any.whl"], ) - site_packages = os.path.join(appdir, '.chalice.', 'site-packages') + site_packages = os.path.join(appdir, ".chalice.", "site-packages") with osutils.tempdir() as scratch_dir: - builder.build_site_packages( - requirements_file, site_packages, scratch_dir) + builder.build_site_packages(requirements_file, site_packages, scratch_dir) installed_packages = os.listdir(site_packages) pip.validate() for req in reqs: assert req in installed_packages - def test_build_sdist_makes_incompatible_whl(self, tmpdir, osutils, - pip_runner): - reqs = ['foo', 'bar'] + def test_build_sdist_makes_incompatible_whl(self, tmpdir, osutils, pip_runner): + reqs = ["foo", "bar"] pip, runner = pip_runner - appdir, builder = self._make_appdir_and_dependency_builder( - reqs, tmpdir, runner) - requirements_file = os.path.join(appdir, 'requirements.txt') + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( - expected_args=['-r', requirements_file, '--dest', mock.ANY], - packages=[ - 'foo-1.2.zip', - 'bar-1.2-cp36-cp36m-manylinux1_x86_64.whl' - ] + expected_args=["-r", requirements_file, "--dest", mock.ANY], + packages=["foo-1.2.zip", "bar-1.2-cp36-cp36m-manylinux1_x86_64.whl"], ) # foo is compiled since downloading it failed to get any wheels. And # the second download for manylinux1_x86_64 wheels failed as well. @@ -748,50 +647,38 @@ def test_build_sdist_makes_incompatible_whl(self, tmpdir, osutils, # not compatible. In this case currently there is nothing that chalice # can do to install this package. pip.wheels_to_build( - expected_args=['--no-deps', '--wheel-dir', mock.ANY, - PathArgumentEndingWith('foo-1.2.zip')], - wheels_to_build=[ - 'foo-1.2-cp36-cp36m-macosx_10_6_intel.whl' - ] + expected_args=["--no-deps", "--wheel-dir", mock.ANY, PathArgumentEndingWith("foo-1.2.zip")], + wheels_to_build=["foo-1.2-cp36-cp36m-macosx_10_6_intel.whl"], ) - site_packages = os.path.join(appdir, '.chalice.', 'site-packages') + site_packages = os.path.join(appdir, ".chalice.", "site-packages") with osutils.tempdir() as scratch_dir: with pytest.raises(MissingDependencyError) as e: - builder.build_site_packages( - requirements_file, site_packages, scratch_dir) + builder.build_site_packages(requirements_file, site_packages, scratch_dir) installed_packages = os.listdir(site_packages) # bar should succeed and foo should failed. missing_packages = list(e.value.missing) pip.validate() assert len(missing_packages) == 1 - assert missing_packages[0].identifier == 'foo==1.2' - assert installed_packages == ['bar'] + assert missing_packages[0].identifier == "foo==1.2" + assert installed_packages == ["bar"] - def test_can_build_package_with_optional_c_speedups_and_no_wheel( - self, tmpdir, osutils, pip_runner): - reqs = ['foo'] + def test_can_build_package_with_optional_c_speedups_and_no_wheel(self, tmpdir, osutils, pip_runner): + reqs = ["foo"] pip, runner = pip_runner - appdir, builder = self._make_appdir_and_dependency_builder( - reqs, tmpdir, runner) - requirements_file = os.path.join(appdir, 'requirements.txt') + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") # In this scenario we are downloading a package that has no wheel files # at all, and optional c speedups. The initial download will yield an # sdist since there were no wheels. - pip.packages_to_download( - expected_args=['-r', requirements_file, '--dest', mock.ANY], - packages=['foo-1.2.zip'] - ) + pip.packages_to_download(expected_args=["-r", requirements_file, "--dest", mock.ANY], packages=["foo-1.2.zip"]) # Chalice should now try and build this into a wheel file. Since it has # optional c speedups it will build a platform dependent wheel file # which is not compatible with lambda. pip.wheels_to_build( - expected_args=['--no-deps', '--wheel-dir', mock.ANY, - PathArgumentEndingWith('foo-1.2.zip')], - wheels_to_build=[ - 'foo-1.2-cp36-cp36m-macosx_10_6_intel.whl' - ] + expected_args=["--no-deps", "--wheel-dir", mock.ANY, PathArgumentEndingWith("foo-1.2.zip")], + wheels_to_build=["foo-1.2-cp36-cp36m-macosx_10_6_intel.whl"], ) # Now chalice should make a last ditch effort to build the package by @@ -800,154 +687,136 @@ def test_can_build_package_with_optional_c_speedups_and_no_wheel( # c speedups (which in this scenario it did) then it will # successfully fall back to building a pure python wheel file. pip.wheels_to_build( - expected_args=['--no-deps', '--wheel-dir', mock.ANY, - PathArgumentEndingWith('foo-1.2.zip')], + expected_args=["--no-deps", "--wheel-dir", mock.ANY, PathArgumentEndingWith("foo-1.2.zip")], expected_env_vars=pip_no_compile_c_env_vars, expected_shim=pip_no_compile_c_shim, - wheels_to_build=[ - 'foo-1.2-cp36-none-any.whl' - ] + wheels_to_build=["foo-1.2-cp36-none-any.whl"], ) - site_packages = os.path.join(appdir, '.chalice.', 'site-packages') + site_packages = os.path.join(appdir, ".chalice.", "site-packages") with osutils.tempdir() as scratch_dir: - builder.build_site_packages( - requirements_file, site_packages, scratch_dir) + builder.build_site_packages(requirements_file, site_packages, scratch_dir) installed_packages = os.listdir(site_packages) # Now we should have successfully built the foo package. pip.validate() - assert installed_packages == ['foo'] + assert installed_packages == ["foo"] - def test_build_into_existing_dir_with_preinstalled_packages( - self, tmpdir, osutils, pip_runner): + def test_build_into_existing_dir_with_preinstalled_packages(self, tmpdir, osutils, pip_runner): # Same test as above so we should get foo failing and bar succeeding # but in this test we started with a .chalice/site-packages directory # with both foo and bar already installed. It should still fail since # they may be there by happenstance, or from an incompatible version # of python. - reqs = ['foo', 'bar'] + reqs = ["foo", "bar"] pip, runner = pip_runner - appdir, builder = self._make_appdir_and_dependency_builder( - reqs, tmpdir, runner) - requirements_file = os.path.join(appdir, 'requirements.txt') + appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) + requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( - expected_args=['-r', requirements_file, '--dest', mock.ANY], - packages=[ - 'foo-1.2.zip', - 'bar-1.2-cp36-cp36m-manylinux1_x86_64.whl' - ] + expected_args=["-r", requirements_file, "--dest", mock.ANY], + packages=["foo-1.2.zip", "bar-1.2-cp36-cp36m-manylinux1_x86_64.whl"], ) pip.packages_to_download( expected_args=[ - '--only-binary=:all:', '--no-deps', '--platform', - 'manylinux1_x86_64', '--implementation', 'cp', - '--abi', get_lambda_abi(builder.runtime), '--dest', mock.ANY, - 'foo==1.2' + "--only-binary=:all:", + "--no-deps", + "--platform", + "manylinux1_x86_64", + "--implementation", + "cp", + "--abi", + get_lambda_abi(builder.runtime), + "--dest", + mock.ANY, + "foo==1.2", ], - packages=[ - 'foo-1.2-cp36-cp36m-macosx_10_6_intel.whl' - ] + packages=["foo-1.2-cp36-cp36m-macosx_10_6_intel.whl"], ) # Add two fake packages foo and bar that have previously been # installed in the site-packages directory. - site_packages = os.path.join(appdir, '.chalice', 'site-packages') - foo = os.path.join(site_packages, 'foo') + site_packages = os.path.join(appdir, ".chalice", "site-packages") + foo = os.path.join(site_packages, "foo") os.makedirs(foo) - bar = os.path.join(site_packages, 'bar') + bar = os.path.join(site_packages, "bar") os.makedirs(bar) with osutils.tempdir() as scratch_dir: with pytest.raises(MissingDependencyError) as e: - builder.build_site_packages( - requirements_file, site_packages, scratch_dir) + builder.build_site_packages(requirements_file, site_packages, scratch_dir) installed_packages = os.listdir(site_packages) # bar should succeed and foo should failed. missing_packages = list(e.value.missing) pip.validate() assert len(missing_packages) == 1 - assert missing_packages[0].identifier == 'foo==1.2' - assert installed_packages == ['bar'] + assert missing_packages[0].identifier == "foo==1.2" + assert installed_packages == ["bar"] class TestSubprocessPip(object): def test_can_invoke_pip(self): pip = SubprocessPip(python_exe=sys.executable) - rc, out, err = pip.main(['--version']) + rc, out, err = pip.main(["--version"]) # Simple assertion that we can execute pip and it gives us some output # and nothing on stderr. assert rc == 0 - assert err == b'' + assert err == b"" def test_does_error_code_propagate(self): pip = SubprocessPip(python_exe=sys.executable) - rc, _, err = pip.main(['badcommand']) + rc, _, err = pip.main(["badcommand"]) assert rc != 0 # Don't want to depend on a particular error message from pip since it # may change if we pin a differnet version to Chalice at some point. # But there should be a non-empty error message of some kind. - assert err != b'' + assert err != b"" class TestSdistMetadataFetcher(object): - _SETUPTOOLS = 'from setuptools import setup' - _DISTUTILS = 'from distutils.core import setup' + _SETUPTOOLS = "from setuptools import setup" + _DISTUTILS = "from distutils.core import setup" _BOTH = ( - 'try:\n' - ' from setuptools import setup\n' - 'except ImportError:\n' - ' from distutils.core import setuptools\n' + "try:\n" + " from setuptools import setup\n" + "except ImportError:\n" + " from distutils.core import setuptools\n" ) - _SETUP_PY = ( - '%s\n' - 'setup(\n' - ' name="%s",\n' - ' version="%s"\n' - ')\n' - ) - _VALID_TAR_FORMATS = ['tar.gz', 'tar.bz2'] + _SETUP_PY = "%s\n" "setup(\n" ' name="%s",\n' ' version="%s"\n' ")\n" + _VALID_TAR_FORMATS = ["tar.gz", "tar.bz2"] def _write_fake_sdist(self, setup_py, directory, ext): - filename = 'sdist.%s' % ext - path = '%s/%s' % (directory, filename) - if ext == 'zip': - with zipfile.ZipFile(path, 'w', - compression=zipfile.ZIP_DEFLATED) as z: - z.writestr('sdist/setup.py', setup_py) + filename = "sdist.%s" % ext + path = "%s/%s" % (directory, filename) + if ext == "zip": + with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as z: + z.writestr("sdist/setup.py", setup_py) elif ext in self._VALID_TAR_FORMATS: - compression_format = ext.split('.')[1] - with tarfile.open(path, 'w:%s' % compression_format) as tar: - tarinfo = tarfile.TarInfo('sdist/setup.py') + compression_format = ext.split(".")[1] + with tarfile.open(path, "w:%s" % compression_format) as tar: + tarinfo = tarfile.TarInfo("sdist/setup.py") tarinfo.size = len(setup_py) tar.addfile(tarinfo, io.BytesIO(setup_py.encode())) else: - open(path, 'a').close() + open(path, "a").close() filepath = os.path.join(directory, filename) return filepath def test_setup_tar_gz(self, osutils, sdist_reader): - setup_py = self._SETUP_PY % ( - self._SETUPTOOLS, 'foo', '1.0' - ) + setup_py = self._SETUP_PY % (self._SETUPTOOLS, "foo", "1.0") with osutils.tempdir() as tempdir: - filepath = self._write_fake_sdist(setup_py, tempdir, 'tar.gz') - name, version = sdist_reader.get_package_name_and_version( - filepath) - assert name == 'foo' - assert version == '1.0' + filepath = self._write_fake_sdist(setup_py, tempdir, "tar.gz") + name, version = sdist_reader.get_package_name_and_version(filepath) + assert name == "foo" + assert version == "1.0" def test_setup_tar_bz2(self, osutils, sdist_reader): - setup_py = self._SETUP_PY % ( - self._SETUPTOOLS, 'foo', '1.0' - ) + setup_py = self._SETUP_PY % (self._SETUPTOOLS, "foo", "1.0") with osutils.tempdir() as tempdir: - filepath = self._write_fake_sdist(setup_py, tempdir, 'tar.bz2') - name, version = sdist_reader.get_package_name_and_version( - filepath) - assert name == 'foo' - assert version == '1.0' + filepath = self._write_fake_sdist(setup_py, tempdir, "tar.bz2") + name, version = sdist_reader.get_package_name_and_version(filepath) + assert name == "foo" + assert version == "1.0" def test_setup_tar_gz_hyphens_in_name(self, osutils, sdist_reader): # The whole reason we need to use the egg info to get the name and @@ -955,128 +824,98 @@ def test_setup_tar_gz_hyphens_in_name(self, osutils, sdist_reader): # from the filenames themselves. This test puts hyphens in the name # and version which would break a simple ``split("-")`` attempt to get # that information. - setup_py = self._SETUP_PY % ( - self._SETUPTOOLS, 'foo-bar', '1.0-2b' - ) + setup_py = self._SETUP_PY % (self._SETUPTOOLS, "foo-bar", "1.0-2b") with osutils.tempdir() as tempdir: - filepath = self._write_fake_sdist(setup_py, tempdir, 'tar.gz') - name, version = sdist_reader.get_package_name_and_version( - filepath) - assert name == 'foo-bar' - assert version == '1.0-2b' + filepath = self._write_fake_sdist(setup_py, tempdir, "tar.gz") + name, version = sdist_reader.get_package_name_and_version(filepath) + assert name == "foo-bar" + assert version == "1.0-2b" def test_setup_zip(self, osutils, sdist_reader): - setup_py = self._SETUP_PY % ( - self._SETUPTOOLS, 'foo', '1.0' - ) + setup_py = self._SETUP_PY % (self._SETUPTOOLS, "foo", "1.0") with osutils.tempdir() as tempdir: - filepath = self._write_fake_sdist(setup_py, tempdir, 'zip') - name, version = sdist_reader.get_package_name_and_version( - filepath) - assert name == 'foo' - assert version == '1.0' + filepath = self._write_fake_sdist(setup_py, tempdir, "zip") + name, version = sdist_reader.get_package_name_and_version(filepath) + assert name == "foo" + assert version == "1.0" def test_distutil_tar_gz(self, osutils, sdist_reader): - setup_py = self._SETUP_PY % ( - self._DISTUTILS, 'foo', '1.0' - ) + setup_py = self._SETUP_PY % (self._DISTUTILS, "foo", "1.0") with osutils.tempdir() as tempdir: - filepath = self._write_fake_sdist(setup_py, tempdir, 'tar.gz') - name, version = sdist_reader.get_package_name_and_version( - filepath) - assert name == 'foo' - assert version == '1.0' + filepath = self._write_fake_sdist(setup_py, tempdir, "tar.gz") + name, version = sdist_reader.get_package_name_and_version(filepath) + assert name == "foo" + assert version == "1.0" def test_distutil_tar_bz2(self, osutils, sdist_reader): - setup_py = self._SETUP_PY % ( - self._DISTUTILS, 'foo', '1.0' - ) + setup_py = self._SETUP_PY % (self._DISTUTILS, "foo", "1.0") with osutils.tempdir() as tempdir: - filepath = self._write_fake_sdist(setup_py, tempdir, 'tar.bz2') - name, version = sdist_reader.get_package_name_and_version( - filepath) - assert name == 'foo' - assert version == '1.0' + filepath = self._write_fake_sdist(setup_py, tempdir, "tar.bz2") + name, version = sdist_reader.get_package_name_and_version(filepath) + assert name == "foo" + assert version == "1.0" def test_distutil_zip(self, osutils, sdist_reader): - setup_py = self._SETUP_PY % ( - self._DISTUTILS, 'foo', '1.0' - ) + setup_py = self._SETUP_PY % (self._DISTUTILS, "foo", "1.0") with osutils.tempdir() as tempdir: - filepath = self._write_fake_sdist(setup_py, tempdir, 'zip') - name, version = sdist_reader.get_package_name_and_version( - filepath) - assert name == 'foo' - assert version == '1.0' + filepath = self._write_fake_sdist(setup_py, tempdir, "zip") + name, version = sdist_reader.get_package_name_and_version(filepath) + assert name == "foo" + assert version == "1.0" def test_both_tar_gz(self, osutils, sdist_reader): - setup_py = self._SETUP_PY % ( - self._BOTH, 'foo-bar', '1.0-2b' - ) + setup_py = self._SETUP_PY % (self._BOTH, "foo-bar", "1.0-2b") with osutils.tempdir() as tempdir: - filepath = self._write_fake_sdist(setup_py, tempdir, 'tar.gz') - name, version = sdist_reader.get_package_name_and_version( - filepath) - assert name == 'foo-bar' - assert version == '1.0-2b' + filepath = self._write_fake_sdist(setup_py, tempdir, "tar.gz") + name, version = sdist_reader.get_package_name_and_version(filepath) + assert name == "foo-bar" + assert version == "1.0-2b" def test_both_tar_bz2(self, osutils, sdist_reader): - setup_py = self._SETUP_PY % ( - self._BOTH, 'foo-bar', '1.0-2b' - ) + setup_py = self._SETUP_PY % (self._BOTH, "foo-bar", "1.0-2b") with osutils.tempdir() as tempdir: - filepath = self._write_fake_sdist(setup_py, tempdir, 'tar.bz2') - name, version = sdist_reader.get_package_name_and_version( - filepath) - assert name == 'foo-bar' - assert version == '1.0-2b' + filepath = self._write_fake_sdist(setup_py, tempdir, "tar.bz2") + name, version = sdist_reader.get_package_name_and_version(filepath) + assert name == "foo-bar" + assert version == "1.0-2b" def test_both_zip(self, osutils, sdist_reader): - setup_py = self._SETUP_PY % ( - self._BOTH, 'foo', '1.0' - ) + setup_py = self._SETUP_PY % (self._BOTH, "foo", "1.0") with osutils.tempdir() as tempdir: - filepath = self._write_fake_sdist(setup_py, tempdir, 'zip') - name, version = sdist_reader.get_package_name_and_version( - filepath) - assert name == 'foo' - assert version == '1.0' + filepath = self._write_fake_sdist(setup_py, tempdir, "zip") + name, version = sdist_reader.get_package_name_and_version(filepath) + assert name == "foo" + assert version == "1.0" def test_bad_format(self, osutils, sdist_reader): - setup_py = self._SETUP_PY % ( - self._BOTH, 'foo', '1.0' - ) + setup_py = self._SETUP_PY % (self._BOTH, "foo", "1.0") with osutils.tempdir() as tempdir: - filepath = self._write_fake_sdist(setup_py, tempdir, 'tar.gz2') + filepath = self._write_fake_sdist(setup_py, tempdir, "tar.gz2") with pytest.raises(InvalidSourceDistributionNameError): - name, version = sdist_reader.get_package_name_and_version( - filepath) + name, version = sdist_reader.get_package_name_and_version(filepath) class TestPackage(object): - def test_same_pkg_sdist_and_wheel_collide(self, osutils, sdist_builder): with osutils.tempdir() as tempdir: - sdist_builder.write_fake_sdist(tempdir, 'foobar', '1.0') + sdist_builder.write_fake_sdist(tempdir, "foobar", "1.0") pkgs = set() - pkgs.add(Package('', 'foobar-1.0-py3-none-any.whl')) - pkgs.add(Package(tempdir, 'foobar-1.0.zip')) + pkgs.add(Package("", "foobar-1.0-py3-none-any.whl")) + pkgs.add(Package(tempdir, "foobar-1.0.zip")) assert len(pkgs) == 1 - def test_ensure_sdist_name_normalized_for_comparison(self, osutils, - sdist_builder): + def test_ensure_sdist_name_normalized_for_comparison(self, osutils, sdist_builder): with osutils.tempdir() as tempdir: - sdist_builder.write_fake_sdist(tempdir, 'Foobar', '1.0') + sdist_builder.write_fake_sdist(tempdir, "Foobar", "1.0") pkgs = set() - pkgs.add(Package('', 'foobar-1.0-py3-none-any.whl')) - pkgs.add(Package(tempdir, 'Foobar-1.0.zip')) + pkgs.add(Package("", "foobar-1.0-py3-none-any.whl")) + pkgs.add(Package(tempdir, "Foobar-1.0.zip")) assert len(pkgs) == 1 - def test_ensure_wheel_name_normalized_for_comparison(self, osutils, - sdist_builder): + def test_ensure_wheel_name_normalized_for_comparison(self, osutils, sdist_builder): with osutils.tempdir() as tempdir: - sdist_builder.write_fake_sdist(tempdir, 'foobar', '1.0') + sdist_builder.write_fake_sdist(tempdir, "foobar", "1.0") pkgs = set() - pkgs.add(Package('', 'Foobar-1.0-py3-none-any.whl')) - pkgs.add(Package(tempdir, 'foobar-1.0.zip')) + pkgs.add(Package("", "Foobar-1.0-py3-none-any.whl")) + pkgs.add(Package(tempdir, "foobar-1.0.zip")) assert len(pkgs) == 1 diff --git a/tests/functional/workflows/python_pip/test_utils.py b/tests/functional/workflows/python_pip/test_utils.py index d89a46a1d..67075116b 100644 --- a/tests/functional/workflows/python_pip/test_utils.py +++ b/tests/functional/workflows/python_pip/test_utils.py @@ -12,11 +12,10 @@ def osutils(): class TestOSUtils(object): def test_can_read_unicode(self, tmpdir, osutils): - filename = str(tmpdir.join('file.txt')) - checkmark = u'\2713' - with io.open(filename, 'w', encoding='utf-16') as f: + filename = str(tmpdir.join("file.txt")) + checkmark = u"\2713" + with io.open(filename, "w", encoding="utf-16") as f: f.write(checkmark) - content = osutils.get_file_contents(filename, binary=False, - encoding='utf-16') + content = osutils.get_file_contents(filename, binary=False, encoding="utf-16") assert content == checkmark diff --git a/tests/functional/workflows/ruby_bundler/test_ruby_utils.py b/tests/functional/workflows/ruby_bundler/test_ruby_utils.py index 6fe6ed28e..a13eeb41f 100644 --- a/tests/functional/workflows/ruby_bundler/test_ruby_utils.py +++ b/tests/functional/workflows/ruby_bundler/test_ruby_utils.py @@ -9,7 +9,6 @@ class TestOSUtils(TestCase): - def setUp(self): self.osutils = utils.OSUtils() @@ -26,29 +25,26 @@ def test_dirname_returns_directory_for_path(self): self.assertEqual(dirname, os.path.dirname(sys.executable)) def test_abspath_returns_absolute_path(self): - result = self.osutils.abspath('.') + result = self.osutils.abspath(".") self.assertTrue(os.path.isabs(result)) - self.assertEqual(result, os.path.abspath('.')) + self.assertEqual(result, os.path.abspath(".")) def test_joinpath_joins_path_components(self): - result = self.osutils.joinpath('a', 'b', 'c') - self.assertEqual(result, os.path.join('a', 'b', 'c')) + result = self.osutils.joinpath("a", "b", "c") + self.assertEqual(result, os.path.join("a", "b", "c")) def test_popen_runs_a_process_and_returns_outcome(self): - cwd_py = os.path.join(os.path.dirname(__file__), '..', '..', 'testdata', 'cwd.py') - p = self.osutils.popen([sys.executable, cwd_py], - stdout=self.osutils.pipe, - stderr=self.osutils.pipe) + cwd_py = os.path.join(os.path.dirname(__file__), "..", "..", "testdata", "cwd.py") + p = self.osutils.popen([sys.executable, cwd_py], stdout=self.osutils.pipe, stderr=self.osutils.pipe) out, err = p.communicate() self.assertEqual(p.returncode, 0) - self.assertEqual(out.decode('utf8').strip(), os.getcwd()) + self.assertEqual(out.decode("utf8").strip(), os.getcwd()) def test_popen_can_accept_cwd(self): - testdata_dir = os.path.join(os.path.dirname(__file__), '..', '..', 'testdata') - p = self.osutils.popen([sys.executable, 'cwd.py'], - stdout=self.osutils.pipe, - stderr=self.osutils.pipe, - cwd=testdata_dir) + testdata_dir = os.path.join(os.path.dirname(__file__), "..", "..", "testdata") + p = self.osutils.popen( + [sys.executable, "cwd.py"], stdout=self.osutils.pipe, stderr=self.osutils.pipe, cwd=testdata_dir + ) out, err = p.communicate() self.assertEqual(p.returncode, 0) - self.assertEqual(out.decode('utf8').strip(), os.path.abspath(testdata_dir)) + self.assertEqual(out.decode("utf8").strip(), os.path.abspath(testdata_dir)) diff --git a/tests/integration/workflows/dotnet_clipackage/test_dotnet.py b/tests/integration/workflows/dotnet_clipackage/test_dotnet.py index 2e1ec20de..f570b0f33 100644 --- a/tests/integration/workflows/dotnet_clipackage/test_dotnet.py +++ b/tests/integration/workflows/dotnet_clipackage/test_dotnet.py @@ -15,9 +15,7 @@ def setUp(self): self.artifacts_dir = tempfile.mkdtemp() self.scratch_dir = tempfile.mkdtemp() - self.builder = LambdaBuilder(language="dotnet", - dependency_manager="cli-package", - application_framework=None) + self.builder = LambdaBuilder(language="dotnet", dependency_manager="cli-package", application_framework=None) self.runtime = "dotnetcore2.1" @@ -28,17 +26,17 @@ def tearDown(self): def test_with_defaults_file(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "WithDefaultsFile") - self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, - source_dir, - runtime=self.runtime) + self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, source_dir, runtime=self.runtime) - expected_files = {"Amazon.Lambda.Core.dll", - "Amazon.Lambda.Serialization.Json.dll", - "Newtonsoft.Json.dll", - "WithDefaultsFile.deps.json", - "WithDefaultsFile.dll", - "WithDefaultsFile.pdb", - "WithDefaultsFile.runtimeconfig.json"} + expected_files = { + "Amazon.Lambda.Core.dll", + "Amazon.Lambda.Serialization.Json.dll", + "Newtonsoft.Json.dll", + "WithDefaultsFile.deps.json", + "WithDefaultsFile.dll", + "WithDefaultsFile.pdb", + "WithDefaultsFile.runtimeconfig.json", + } output_files = set(os.listdir(self.artifacts_dir)) @@ -47,18 +45,24 @@ def test_with_defaults_file(self): def test_require_parameters(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "RequireParameters") - self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, - source_dir, - runtime=self.runtime, - options={"--framework": "netcoreapp2.1", "--configuration": "Debug"}) - - expected_files = {"Amazon.Lambda.Core.dll", - "Amazon.Lambda.Serialization.Json.dll", - "Newtonsoft.Json.dll", - "RequireParameters.deps.json", - "RequireParameters.dll", - "RequireParameters.pdb", - "RequireParameters.runtimeconfig.json"} + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + source_dir, + runtime=self.runtime, + options={"--framework": "netcoreapp2.1", "--configuration": "Debug"}, + ) + + expected_files = { + "Amazon.Lambda.Core.dll", + "Amazon.Lambda.Serialization.Json.dll", + "Newtonsoft.Json.dll", + "RequireParameters.deps.json", + "RequireParameters.dll", + "RequireParameters.pdb", + "RequireParameters.runtimeconfig.json", + } output_files = set(os.listdir(self.artifacts_dir)) diff --git a/tests/integration/workflows/go_dep/test_go_dep.py b/tests/integration/workflows/go_dep/test_go_dep.py index e79b63ae2..d6b7c6da9 100644 --- a/tests/integration/workflows/go_dep/test_go_dep.py +++ b/tests/integration/workflows/go_dep/test_go_dep.py @@ -19,9 +19,7 @@ def setUp(self): self.no_deps = os.path.join(self.TEST_DATA_FOLDER, "src", "nodeps") - self.builder = LambdaBuilder(language="go", - dependency_manager="dep", - application_framework=None) + self.builder = LambdaBuilder(language="go", dependency_manager="dep", application_framework=None) self.runtime = "go1.x" @@ -32,10 +30,14 @@ def tearDown(self): def test_builds_project_with_no_deps(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "src", "nodeps") - self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, - os.path.join(source_dir, "Gopkg.toml"), - runtime=self.runtime, - options={"artifact_executable_name": "main"}) + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "Gopkg.toml"), + runtime=self.runtime, + options={"artifact_executable_name": "main"}, + ) expected_files = {"main"} output_files = set(os.listdir(self.artifacts_dir)) @@ -46,23 +48,32 @@ def test_builds_project_with_no_gopkg_file(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "src", "no-gopkg") with self.assertRaises(WorkflowFailedError) as ex: - self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, - os.path.join(source_dir, "Gopkg.toml"), - runtime=self.runtime, - options={"artifact_executable_name": "main"}) + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "Gopkg.toml"), + runtime=self.runtime, + options={"artifact_executable_name": "main"}, + ) self.assertEquals( - "GoDepBuilder:DepEnsure - Exec Failed: could not find project Gopkg.toml," + - " use dep init to initiate a manifest", - str(ex.exception)) + "GoDepBuilder:DepEnsure - Exec Failed: could not find project Gopkg.toml," + + " use dep init to initiate a manifest", + str(ex.exception), + ) def test_builds_project_with_remote_deps(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "src", "remote-deps") - self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, - os.path.join(source_dir, "Gopkg.toml"), - runtime=self.runtime, - options={"artifact_executable_name": "main"}) + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "Gopkg.toml"), + runtime=self.runtime, + options={"artifact_executable_name": "main"}, + ) expected_files = {"main"} output_files = set(os.listdir(self.artifacts_dir)) @@ -73,10 +84,14 @@ def test_builds_project_with_failed_remote_deps(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "src", "failed-remote") with self.assertRaises(WorkflowFailedError) as ex: - self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, - os.path.join(source_dir, "Gopkg.toml"), - runtime=self.runtime, - options={"artifact_executable_name": "main"}) + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "Gopkg.toml"), + runtime=self.runtime, + options={"artifact_executable_name": "main"}, + ) # The full message is super long, so part of it is fine. - self.assertNotEqual(str(ex.exception).find('unable to deduce repository and source type for'), -1) + self.assertNotEqual(str(ex.exception).find("unable to deduce repository and source type for"), -1) diff --git a/tests/integration/workflows/go_modules/test_go.py b/tests/integration/workflows/go_modules/test_go.py index a871c7190..fa02363d8 100644 --- a/tests/integration/workflows/go_modules/test_go.py +++ b/tests/integration/workflows/go_modules/test_go.py @@ -18,9 +18,7 @@ class TestGoWorkflow(TestCase): def setUp(self): self.artifacts_dir = tempfile.mkdtemp() self.scratch_dir = tempfile.mkdtemp() - self.builder = LambdaBuilder(language="go", - dependency_manager="modules", - application_framework=None) + self.builder = LambdaBuilder(language="go", dependency_manager="modules", application_framework=None) self.runtime = "go1.x" def tearDown(self): @@ -29,21 +27,29 @@ def tearDown(self): def test_builds_project_without_dependencies(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps") - self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, - os.path.join(source_dir, "go.mod"), - runtime=self.runtime, - options={"artifact_executable_name": "no-deps-main"}) + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "go.mod"), + runtime=self.runtime, + options={"artifact_executable_name": "no-deps-main"}, + ) expected_files = {"no-deps-main"} output_files = set(os.listdir(self.artifacts_dir)) - print(output_files) + print (output_files) self.assertEquals(expected_files, output_files) def test_builds_project_with_dependencies(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "with-deps") - self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, - os.path.join(source_dir, "go.mod"), - runtime=self.runtime, - options={"artifact_executable_name": "with-deps-main"}) + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "go.mod"), + runtime=self.runtime, + options={"artifact_executable_name": "with-deps-main"}, + ) expected_files = {"with-deps-main"} output_files = set(os.listdir(self.artifacts_dir)) self.assertEquals(expected_files, output_files) @@ -51,9 +57,12 @@ def test_builds_project_with_dependencies(self): def test_fails_if_modules_cannot_resolve_dependencies(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "broken-deps") with self.assertRaises(WorkflowFailedError) as ctx: - self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, - os.path.join(source_dir, "go.mod"), - runtime=self.runtime, - options={"artifact_executable_name": "failed"}) - self.assertIn("GoModulesBuilder:Build - Builder Failed: ", - str(ctx.exception)) + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "go.mod"), + runtime=self.runtime, + options={"artifact_executable_name": "failed"}, + ) + self.assertIn("GoModulesBuilder:Build - Builder Failed: ", str(ctx.exception)) diff --git a/tests/integration/workflows/java_gradle/test_java_gradle.py b/tests/integration/workflows/java_gradle/test_java_gradle.py index 8b0fe0271..00e4e5589 100644 --- a/tests/integration/workflows/java_gradle/test_java_gradle.py +++ b/tests/integration/workflows/java_gradle/test_java_gradle.py @@ -16,88 +16,97 @@ class TestJavaGradle(TestCase): def setUp(self): self.artifacts_dir = tempfile.mkdtemp() self.scratch_dir = tempfile.mkdtemp() - self.builder = LambdaBuilder(language='java', dependency_manager='gradle', application_framework=None) - self.runtime = 'java11' + self.builder = LambdaBuilder(language="java", dependency_manager="gradle", application_framework=None) + self.runtime = "java11" def tearDown(self): shutil.rmtree(self.artifacts_dir) shutil.rmtree(self.scratch_dir) def test_build_single_build_with_deps(self): - source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR, 'with-deps') - manifest_path = os.path.join(source_dir, 'build.gradle') + source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR, "with-deps") + manifest_path = os.path.join(source_dir, "build.gradle") self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, manifest_path, runtime=self.runtime) - expected_files = [p('aws', 'lambdabuilders', 'Main.class'), p('lib', 'annotations-2.1.0.jar')] + expected_files = [p("aws", "lambdabuilders", "Main.class"), p("lib", "annotations-2.1.0.jar")] self.assert_artifact_contains_files(expected_files) def test_build_single_build_with_resources(self): - source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR, 'with-resources') - manifest_path = os.path.join(source_dir, 'build.gradle') + source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR, "with-resources") + manifest_path = os.path.join(source_dir, "build.gradle") self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, manifest_path, runtime=self.runtime) - expected_files = [p('aws', 'lambdabuilders', 'Main.class'), p('some_data.txt'), - p('lib', 'annotations-2.1.0.jar')] + expected_files = [ + p("aws", "lambdabuilders", "Main.class"), + p("some_data.txt"), + p("lib", "annotations-2.1.0.jar"), + ] self.assert_artifact_contains_files(expected_files) def test_build_single_build_with_test_deps_test_jars_not_included(self): - source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR, 'with-test-deps') - manifest_path = os.path.join(source_dir, 'build.gradle') + source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR, "with-test-deps") + manifest_path = os.path.join(source_dir, "build.gradle") self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, manifest_path, runtime=self.runtime) - expected_files = [p('aws', 'lambdabuilders', 'Main.class'), p('lib', 'annotations-2.1.0.jar')] + expected_files = [p("aws", "lambdabuilders", "Main.class"), p("lib", "annotations-2.1.0.jar")] self.assert_artifact_contains_files(expected_files) - self.assert_artifact_not_contains_file(p('lib', 's3-2.1.0.jar')) + self.assert_artifact_not_contains_file(p("lib", "s3-2.1.0.jar")) def test_build_single_build_with_deps_gradlew(self): - source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR, 'with-deps-gradlew') - manifest_path = os.path.join(source_dir, 'build.gradle') - self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, manifest_path, runtime=self.runtime, - executable_search_paths=[source_dir]) - expected_files = [p('aws', 'lambdabuilders', 'Main.class'), p('lib', 'annotations-2.1.0.jar')] + source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR, "with-deps-gradlew") + manifest_path = os.path.join(source_dir, "build.gradle") + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + manifest_path, + runtime=self.runtime, + executable_search_paths=[source_dir], + ) + expected_files = [p("aws", "lambdabuilders", "Main.class"), p("lib", "annotations-2.1.0.jar")] self.assert_artifact_contains_files(expected_files) def test_build_multi_build_with_deps_lambda1(self): - parent_dir = os.path.join(self.MULTI_BUILD_TEST_DATA_DIR, 'with-deps') - manifest_path = os.path.join(parent_dir, 'lambda1', 'build.gradle') + parent_dir = os.path.join(self.MULTI_BUILD_TEST_DATA_DIR, "with-deps") + manifest_path = os.path.join(parent_dir, "lambda1", "build.gradle") - lambda1_source = os.path.join(parent_dir, 'lambda1') - self.builder.build(lambda1_source, self.artifacts_dir, self.scratch_dir, manifest_path, - runtime=self.runtime) + lambda1_source = os.path.join(parent_dir, "lambda1") + self.builder.build(lambda1_source, self.artifacts_dir, self.scratch_dir, manifest_path, runtime=self.runtime) - lambda1_expected_files = [p('aws', 'lambdabuilders', 'Lambda1_Main.class'), p('lib', 'annotations-2.1.0.jar')] + lambda1_expected_files = [p("aws", "lambdabuilders", "Lambda1_Main.class"), p("lib", "annotations-2.1.0.jar")] self.assert_artifact_contains_files(lambda1_expected_files) def test_build_multi_build_with_deps_lambda2(self): - parent_dir = os.path.join(self.MULTI_BUILD_TEST_DATA_DIR, 'with-deps') - manifest_path = os.path.join(parent_dir, 'lambda2', 'build.gradle') + parent_dir = os.path.join(self.MULTI_BUILD_TEST_DATA_DIR, "with-deps") + manifest_path = os.path.join(parent_dir, "lambda2", "build.gradle") - lambda2_source = os.path.join(parent_dir, 'lambda2') - self.builder.build(lambda2_source, self.artifacts_dir, self.scratch_dir, manifest_path, - runtime=self.runtime) + lambda2_source = os.path.join(parent_dir, "lambda2") + self.builder.build(lambda2_source, self.artifacts_dir, self.scratch_dir, manifest_path, runtime=self.runtime) - lambda2_expected_files = [p('aws', 'lambdabuilders', 'Lambda2_Main.class'), p('lib', 'annotations-2.1.0.jar')] + lambda2_expected_files = [p("aws", "lambdabuilders", "Lambda2_Main.class"), p("lib", "annotations-2.1.0.jar")] self.assert_artifact_contains_files(lambda2_expected_files) def test_build_multi_build_with_deps_inter_module(self): - parent_dir = os.path.join(self.MULTI_BUILD_TEST_DATA_DIR, 'with-deps-inter-module') - manifest_path = os.path.join(parent_dir, 'lambda1', 'build.gradle') + parent_dir = os.path.join(self.MULTI_BUILD_TEST_DATA_DIR, "with-deps-inter-module") + manifest_path = os.path.join(parent_dir, "lambda1", "build.gradle") - lambda1_source = os.path.join(parent_dir, 'lambda1') - self.builder.build(lambda1_source, self.artifacts_dir, self.scratch_dir, manifest_path, - runtime=self.runtime) + lambda1_source = os.path.join(parent_dir, "lambda1") + self.builder.build(lambda1_source, self.artifacts_dir, self.scratch_dir, manifest_path, runtime=self.runtime) - lambda1_expected_files = [p('aws', 'lambdabuilders', 'Lambda1_Main.class'), p('lib', 'annotations-2.1.0.jar'), - p('lib', 'common.jar')] + lambda1_expected_files = [ + p("aws", "lambdabuilders", "Lambda1_Main.class"), + p("lib", "annotations-2.1.0.jar"), + p("lib", "common.jar"), + ] self.assert_artifact_contains_files(lambda1_expected_files) def test_build_single_build_with_deps_broken(self): - source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR, 'with-deps-broken') - manifest_path = os.path.join(source_dir, 'build.gradle') + source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR, "with-deps-broken") + manifest_path = os.path.join(source_dir, "build.gradle") with self.assertRaises(WorkflowFailedError) as raised: self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, manifest_path, runtime=self.runtime) - self.assertTrue(raised.exception.args[0].startswith('JavaGradleWorkflow:GradleBuild - Gradle Failed')) + self.assertTrue(raised.exception.args[0].startswith("JavaGradleWorkflow:GradleBuild - Gradle Failed")) def assert_artifact_contains_files(self, files): for f in files: diff --git a/tests/integration/workflows/java_maven/test_java_maven.py b/tests/integration/workflows/java_maven/test_java_maven.py index 0a4153eab..a1a77d11d 100644 --- a/tests/integration/workflows/java_maven/test_java_maven.py +++ b/tests/integration/workflows/java_maven/test_java_maven.py @@ -14,38 +14,41 @@ class TestJavaMaven(TestCase): def setUp(self): self.artifacts_dir = tempfile.mkdtemp() self.scratch_dir = tempfile.mkdtemp() - self.builder = LambdaBuilder(language='java', dependency_manager='maven', application_framework=None) - self.runtime = 'java8' + self.builder = LambdaBuilder(language="java", dependency_manager="maven", application_framework=None) + self.runtime = "java8" def tearDown(self): shutil.rmtree(self.artifacts_dir) shutil.rmtree(self.scratch_dir) def test_build_single_build_with_deps_resources_exclude_test_jars(self): - source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR, 'with-deps') - manifest_path = os.path.join(source_dir, 'pom.xml') + source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR, "with-deps") + manifest_path = os.path.join(source_dir, "pom.xml") self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, manifest_path, runtime=self.runtime) - expected_files = [p('aws', 'lambdabuilders', 'Main.class'), p('some_data.txt'), - p('lib', 'annotations-2.1.0.jar')] + expected_files = [ + p("aws", "lambdabuilders", "Main.class"), + p("some_data.txt"), + p("lib", "annotations-2.1.0.jar"), + ] self.assert_artifact_contains_files(expected_files) - self.assert_artifact_not_contains_file(p('lib', 'junit-4.12.jar')) + self.assert_artifact_not_contains_file(p("lib", "junit-4.12.jar")) self.assert_src_dir_not_touched(source_dir) def test_build_single_build_no_deps(self): - source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR, 'no-deps') - manifest_path = os.path.join(source_dir, 'pom.xml') + source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR, "no-deps") + manifest_path = os.path.join(source_dir, "pom.xml") self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, manifest_path, runtime=self.runtime) - expected_files = [p('aws', 'lambdabuilders', 'Main.class'), p('some_data.txt')] + expected_files = [p("aws", "lambdabuilders", "Main.class"), p("some_data.txt")] self.assert_artifact_contains_files(expected_files) - self.assert_artifact_not_contains_file(p('lib')) + self.assert_artifact_not_contains_file(p("lib")) self.assert_src_dir_not_touched(source_dir) def test_build_single_build_with_deps_broken(self): - source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR, 'with-deps-broken') - manifest_path = os.path.join(source_dir, 'pom.xml') + source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR, "with-deps-broken") + manifest_path = os.path.join(source_dir, "pom.xml") with self.assertRaises(WorkflowFailedError) as raised: self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, manifest_path, runtime=self.runtime) - self.assertTrue(raised.exception.args[0].startswith('JavaMavenWorkflow:MavenBuild - Maven Failed')) + self.assertTrue(raised.exception.args[0].startswith("JavaMavenWorkflow:MavenBuild - Maven Failed")) self.assert_src_dir_not_touched(source_dir) def assert_artifact_contains_files(self, files): @@ -59,7 +62,7 @@ def assert_artifact_not_contains_file(self, p): self.assertFalse(os.path.exists(os.path.join(self.artifacts_dir, p))) def assert_src_dir_not_touched(self, source_dir): - self.assertFalse(os.path.exists(os.path.join(source_dir, 'target'))) + self.assertFalse(os.path.exists(os.path.join(source_dir, "target"))) def p(path, *comps): diff --git a/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py b/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py index a8533d5ce..c0ac1d441 100644 --- a/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py +++ b/tests/integration/workflows/nodejs_npm/test_nodejs_npm.py @@ -21,9 +21,7 @@ def setUp(self): self.no_deps = os.path.join(self.TEST_DATA_FOLDER, "no-deps") - self.builder = LambdaBuilder(language="nodejs", - dependency_manager="npm", - application_framework=None) + self.builder = LambdaBuilder(language="nodejs", dependency_manager="npm", application_framework=None) self.runtime = "nodejs8.10" def tearDown(self): @@ -33,9 +31,13 @@ def tearDown(self): def test_builds_project_without_dependencies(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps") - self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, - os.path.join(source_dir, "package.json"), - runtime=self.runtime) + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "package.json"), + runtime=self.runtime, + ) expected_files = {"package.json", "included.js"} output_files = set(os.listdir(self.artifacts_dir)) @@ -44,9 +46,13 @@ def test_builds_project_without_dependencies(self): def test_builds_project_with_remote_dependencies(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "npm-deps") - self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, - os.path.join(source_dir, "package.json"), - runtime=self.runtime) + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "package.json"), + runtime=self.runtime, + ) expected_files = {"package.json", "included.js", "node_modules"} output_files = set(os.listdir(self.artifacts_dir)) @@ -59,9 +65,13 @@ def test_builds_project_with_remote_dependencies(self): def test_builds_project_with_npmrc(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "npmrc") - self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, - os.path.join(source_dir, "package.json"), - runtime=self.runtime) + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "package.json"), + runtime=self.runtime, + ) expected_files = {"package.json", "included.js", "node_modules"} output_files = set(os.listdir(self.artifacts_dir)) @@ -77,9 +87,13 @@ def test_fails_if_npm_cannot_resolve_dependencies(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "broken-deps") with self.assertRaises(WorkflowFailedError) as ctx: - self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, - os.path.join(source_dir, "package.json"), - runtime=self.runtime) + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "package.json"), + runtime=self.runtime, + ) self.assertIn("No matching version found for minimal-request-promise@0.0.0-NON_EXISTENT", str(ctx.exception)) @@ -88,8 +102,12 @@ def test_fails_if_package_json_is_broken(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "broken-package") with self.assertRaises(WorkflowFailedError) as ctx: - self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, - os.path.join(source_dir, "package.json"), - runtime=self.runtime) + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "package.json"), + runtime=self.runtime, + ) self.assertIn("Unexpected end of JSON input", str(ctx.exception)) diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index bfe13e32b..7a665c162 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -1,4 +1,3 @@ - import os import shutil import sys @@ -26,18 +25,15 @@ def setUp(self): self.test_data_files = {"__init__.py", "main.py", "requirements-invalid.txt", "requirements-numpy.txt"} - self.builder = LambdaBuilder(language="python", - dependency_manager="pip", - application_framework=None) + self.builder = LambdaBuilder(language="python", dependency_manager="pip", application_framework=None) self.runtime = "{language}{major}.{minor}".format( - language=self.builder.capability.language, - major=sys.version_info.major, - minor=sys.version_info.minor) + language=self.builder.capability.language, major=sys.version_info.major, minor=sys.version_info.minor + ) self.runtime_mismatch = { - 'python3.6': 'python2.7', - 'python3.7': 'python2.7', - 'python2.7': 'python3.8', - 'python3.8': 'python2.7' + "python3.6": "python2.7", + "python3.7": "python2.7", + "python2.7": "python3.8", + "python3.8": "python2.7", } def tearDown(self): @@ -45,8 +41,9 @@ def tearDown(self): shutil.rmtree(self.scratch_dir) def test_must_build_python_project(self): - self.builder.build(self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid, - runtime=self.runtime) + self.builder.build( + self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid, runtime=self.runtime + ) if self.runtime == "python2.7": expected_files = self.test_data_files.union({"numpy", "numpy-1.15.4.data", "numpy-1.15.4.dist-info"}) @@ -59,38 +56,53 @@ def test_mismatch_runtime_python_project(self): # NOTE : Build still works if other versions of python are accessible on the path. eg: /usr/bin/python2.7 # is still accessible within a python 3 virtualenv. try: - self.builder.build(self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid, - runtime=self.runtime_mismatch[self.runtime]) + self.builder.build( + self.source_dir, + self.artifacts_dir, + self.scratch_dir, + self.manifest_path_valid, + runtime=self.runtime_mismatch[self.runtime], + ) except WorkflowFailedError as ex: self.assertIn("Binary validation failed!", str(ex)) def test_runtime_validate_python_project_fail_open_unsupported_runtime(self): with self.assertRaises(WorkflowFailedError): - self.builder.build(self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid, - runtime="python2.8") + self.builder.build( + self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid, runtime="python2.8" + ) def test_must_fail_to_resolve_dependencies(self): with self.assertRaises(WorkflowFailedError) as ctx: - self.builder.build(self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_invalid, - runtime=self.runtime) + self.builder.build( + self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_invalid, runtime=self.runtime + ) # In Python2 a 'u' is now added to the exception string. To account for this, we see if either one is in the # output - message_in_exception = "Invalid requirement: 'adfasf=1.2.3'" in str(ctx.exception) or \ - "Invalid requirement: u'adfasf=1.2.3'" in str(ctx.exception) + message_in_exception = "Invalid requirement: 'adfasf=1.2.3'" in str( + ctx.exception + ) or "Invalid requirement: u'adfasf=1.2.3'" in str(ctx.exception) self.assertTrue(message_in_exception) def test_must_fail_if_requirements_not_found(self): with self.assertRaises(WorkflowFailedError) as ctx: - self.builder.build(self.source_dir, self.artifacts_dir, self.scratch_dir, - os.path.join("non", "existent", "manifest"), - runtime=self.runtime) - - self.builder.build(self.source_dir, self.artifacts_dir, - self.scratch_dir, - os.path.join("non", "existent", "manifest"), - runtime=self.runtime) + self.builder.build( + self.source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join("non", "existent", "manifest"), + runtime=self.runtime, + ) + + self.builder.build( + self.source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join("non", "existent", "manifest"), + runtime=self.runtime, + ) self.assertIn("Requirements file not found", str(ctx.exception)) diff --git a/tests/integration/workflows/ruby_bundler/test_ruby.py b/tests/integration/workflows/ruby_bundler/test_ruby.py index 5ed47e7f3..3de6eb4c3 100644 --- a/tests/integration/workflows/ruby_bundler/test_ruby.py +++ b/tests/integration/workflows/ruby_bundler/test_ruby.py @@ -19,9 +19,7 @@ def setUp(self): self.artifacts_dir = tempfile.mkdtemp() self.scratch_dir = tempfile.mkdtemp() self.no_deps = os.path.join(self.TEST_DATA_FOLDER, "no-deps") - self.builder = LambdaBuilder(language="ruby", - dependency_manager="bundler", - application_framework=None) + self.builder = LambdaBuilder(language="ruby", dependency_manager="bundler", application_framework=None) self.runtime = "ruby2.5" def tearDown(self): @@ -30,27 +28,27 @@ def tearDown(self): def test_builds_project_without_dependencies(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps") - self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, - os.path.join(source_dir, "Gemfile"), - runtime=self.runtime) + self.builder.build( + source_dir, self.artifacts_dir, self.scratch_dir, os.path.join(source_dir, "Gemfile"), runtime=self.runtime + ) expected_files = {"handler.rb", "Gemfile", "Gemfile.lock", ".bundle", "vendor"} output_files = set(os.listdir(self.artifacts_dir)) self.assertEquals(expected_files, output_files) def test_builds_project_with_dependencies(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "with-deps") - self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, - os.path.join(source_dir, "Gemfile"), - runtime=self.runtime) + self.builder.build( + source_dir, self.artifacts_dir, self.scratch_dir, os.path.join(source_dir, "Gemfile"), runtime=self.runtime + ) expected_files = {"handler.rb", "Gemfile", "Gemfile.lock", ".bundle", "vendor"} output_files = set(os.listdir(self.artifacts_dir)) self.assertEquals(expected_files, output_files) def test_builds_project_and_ignores_excluded_files(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "excluded-files") - self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, - os.path.join(source_dir, "Gemfile"), - runtime=self.runtime) + self.builder.build( + source_dir, self.artifacts_dir, self.scratch_dir, os.path.join(source_dir, "Gemfile"), runtime=self.runtime + ) expected_files = {"handler.rb", "Gemfile", "Gemfile.lock", ".bundle", "vendor"} output_files = set(os.listdir(self.artifacts_dir)) self.assertEquals(expected_files, output_files) @@ -58,8 +56,11 @@ def test_builds_project_and_ignores_excluded_files(self): def test_fails_if_bundler_cannot_resolve_dependencies(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "broken-deps") with self.assertRaises(WorkflowFailedError) as ctx: - self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, - os.path.join(source_dir, "Gemfile"), - runtime=self.runtime) - self.assertIn("RubyBundlerBuilder:RubyBundle - Bundler Failed: ", - str(ctx.exception)) + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(source_dir, "Gemfile"), + runtime=self.runtime, + ) + self.assertIn("RubyBundlerBuilder:RubyBundle - Bundler Failed: ", str(ctx.exception)) diff --git a/tests/unit/test_actions.py b/tests/unit/test_actions.py index d8ef14839..dc1fa38c8 100644 --- a/tests/unit/test_actions.py +++ b/tests/unit/test_actions.py @@ -1,4 +1,3 @@ - from unittest import TestCase from mock import patch, ANY @@ -6,9 +5,7 @@ class TestBaseActionInheritance(TestCase): - def test_must_inherit(self): - class MyAction(BaseAction): NAME = "myname" PURPOSE = Purpose.COPY_SOURCE @@ -19,6 +16,7 @@ class MyAction(BaseAction): def test_must_validate_name_property(self): with self.assertRaises(ValueError): + class MyAction(BaseAction): PURPOSE = Purpose.COPY_SOURCE @@ -32,9 +30,7 @@ class MyAction(BaseAction): class TestBaseAction_repr(TestCase): - def test_must_pretty_print_action_info(self): - class MyAction(BaseAction): NAME = "myname" PURPOSE = Purpose.COPY_SOURCE @@ -46,7 +42,6 @@ class MyAction(BaseAction): class TestCopySourceAction_execute(TestCase): - @patch("aws_lambda_builders.actions.copytree") def test_must_copy(self, copytree_mock): source_dir = "source" diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py index fa4b05603..0551a14f8 100644 --- a/tests/unit/test_builder.py +++ b/tests/unit/test_builder.py @@ -1,4 +1,3 @@ - from unittest import TestCase from mock import patch, call, Mock from parameterized import parameterized, param @@ -17,8 +16,8 @@ def setUp(self): self.lang_framework = "pip" self.app_framework = "chalice" - @patch('aws_lambda_builders.builder.importlib') - @patch('aws_lambda_builders.builder.get_workflow') + @patch("aws_lambda_builders.builder.importlib") + @patch("aws_lambda_builders.builder.get_workflow") def test_must_load_all_default_workflows(self, get_workflow_mock, importlib_mock): # instantiate @@ -30,12 +29,14 @@ def test_must_load_all_default_workflows(self, get_workflow_mock, importlib_mock importlib_mock.import_module.assert_called_once_with(self.DEFAULT_WORKFLOW_MODULE) # then check if we tried to get a workflow for given capability - get_workflow_mock.assert_called_with(Capability(language=self.lang, - dependency_manager=self.lang_framework, - application_framework=self.app_framework)) - - @patch('aws_lambda_builders.builder.importlib') - @patch('aws_lambda_builders.builder.get_workflow') + get_workflow_mock.assert_called_with( + Capability( + language=self.lang, dependency_manager=self.lang_framework, application_framework=self.app_framework + ) + ) + + @patch("aws_lambda_builders.builder.importlib") + @patch("aws_lambda_builders.builder.get_workflow") def test_must_support_loading_custom_workflows(self, get_workflow_mock, importlib_mock): modules = ["a.b.c", "c.d", "e.f", "z.k"] @@ -48,8 +49,8 @@ def test_must_support_loading_custom_workflows(self, get_workflow_mock, importli # Make sure the modules are loaded in same order as passed importlib_mock.import_module.assert_has_calls([call(m) for m in modules], any_order=False) - @patch('aws_lambda_builders.builder.importlib') - @patch('aws_lambda_builders.builder.get_workflow') + @patch("aws_lambda_builders.builder.importlib") + @patch("aws_lambda_builders.builder.get_workflow") def test_must_not_load_any_workflows(self, get_workflow_mock, importlib_mock): modules = [] # Load no modules @@ -67,23 +68,33 @@ def test_with_real_workflow_class(self): # Declare my test workflow. class MyWorkflow(BaseWorkflow): NAME = "MyWorkflow" - CAPABILITY = Capability(language=self.lang, - dependency_manager=self.lang_framework, - application_framework=self.app_framework) - - def __init__(self, - source_dir, - artifacts_dir, - scratch_dir, - manifest_path, - runtime=None, - optimizations=None, - options=None, - executable_search_paths=None, - mode=None): - super(MyWorkflow, self).__init__(source_dir, artifacts_dir, scratch_dir, manifest_path, - runtime=runtime, optimizations=optimizations, options=options, - executable_search_paths=executable_search_paths, mode=mode) + CAPABILITY = Capability( + language=self.lang, dependency_manager=self.lang_framework, application_framework=self.app_framework + ) + + def __init__( + self, + source_dir, + artifacts_dir, + scratch_dir, + manifest_path, + runtime=None, + optimizations=None, + options=None, + executable_search_paths=None, + mode=None, + ): + super(MyWorkflow, self).__init__( + source_dir, + artifacts_dir, + scratch_dir, + manifest_path, + runtime=runtime, + optimizations=optimizations, + options=options, + executable_search_paths=executable_search_paths, + mode=mode, + ) # Don't load any other workflows. The above class declaration will automatically load the workflow into registry builder = LambdaBuilder(self.lang, self.lang_framework, self.app_framework, supported_workflows=[]) @@ -93,7 +104,6 @@ def __init__(self, class TesetLambdaBuilder_build(TestCase): - def tearDown(self): # we don't want test classes lurking around and interfere with other tests DEFAULT_REGISTRY.clear() @@ -103,13 +113,10 @@ def setUp(self): self.lang_framework = "pip" self.app_framework = "chalice" - @parameterized.expand([ - param(True), - param(False) - ]) - @patch('aws_lambda_builders.builder.os') - @patch('aws_lambda_builders.builder.importlib') - @patch('aws_lambda_builders.builder.get_workflow') + @parameterized.expand([param(True), param(False)]) + @patch("aws_lambda_builders.builder.os") + @patch("aws_lambda_builders.builder.importlib") + @patch("aws_lambda_builders.builder.get_workflow") def test_with_mocks(self, scratch_dir_exists, get_workflow_mock, importlib_mock, os_mock): workflow_cls = Mock() workflow_instance = workflow_cls.return_value = Mock() @@ -120,13 +127,29 @@ def test_with_mocks(self, scratch_dir_exists, get_workflow_mock, importlib_mock, builder = LambdaBuilder(self.lang, self.lang_framework, self.app_framework, supported_workflows=[]) - builder.build("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", - runtime="runtime", optimizations="optimizations", options="options", - executable_search_paths="executable_search_paths", mode=None) - - workflow_cls.assert_called_with("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", - runtime="runtime", optimizations="optimizations", options="options", - executable_search_paths="executable_search_paths", mode=None) + builder.build( + "source_dir", + "artifacts_dir", + "scratch_dir", + "manifest_path", + runtime="runtime", + optimizations="optimizations", + options="options", + executable_search_paths="executable_search_paths", + mode=None, + ) + + workflow_cls.assert_called_with( + "source_dir", + "artifacts_dir", + "scratch_dir", + "manifest_path", + runtime="runtime", + optimizations="optimizations", + options="options", + executable_search_paths="executable_search_paths", + mode=None, + ) workflow_instance.run.assert_called_once() os_mock.path.exists.assert_called_once_with("scratch_dir") if scratch_dir_exists: diff --git a/tests/unit/test_path_resolver.py b/tests/unit/test_path_resolver.py index 3295f6a3c..35137028d 100644 --- a/tests/unit/test_path_resolver.py +++ b/tests/unit/test_path_resolver.py @@ -8,7 +8,6 @@ class TestPathResolver(TestCase): - def setUp(self): self.path_resolver = PathResolver(runtime="chitti2.0", binary="chitti") @@ -22,6 +21,6 @@ def test_which_fails(self): self.path_resolver._which() def test_which_success_immediate(self): - with mock.patch.object(self.path_resolver, '_which') as which_mock: + with mock.patch.object(self.path_resolver, "_which") as which_mock: which_mock.return_value = os.getcwd() self.assertEquals(self.path_resolver.exec_paths, os.getcwd()) diff --git a/tests/unit/test_registry.py b/tests/unit/test_registry.py index 86d547861..4acd42c25 100644 --- a/tests/unit/test_registry.py +++ b/tests/unit/test_registry.py @@ -1,4 +1,3 @@ - from unittest import TestCase from mock import Mock, call from parameterized import parameterized @@ -9,7 +8,6 @@ class TestRegistryEndToEnd(TestCase): - def setUp(self): self.registry = Registry() @@ -23,11 +21,13 @@ def test_must_add_item(self): self.registry[capability] = self.workflow_data self.assertEquals(self.workflow_data, self.registry[capability]) - @parameterized.expand([ - (Capability(language=None, dependency_manager="b", application_framework="c"), ), - (Capability(language="a", dependency_manager=None, application_framework="c"), ), - (Capability(language="a", dependency_manager=None, application_framework=None), ), - ]) + @parameterized.expand( + [ + (Capability(language=None, dependency_manager="b", application_framework="c"),), + (Capability(language="a", dependency_manager=None, application_framework="c"),), + (Capability(language="a", dependency_manager=None, application_framework=None),), + ] + ) def test_must_add_item_with_optional_capabilities(self, capability): self.registry[capability] = self.workflow_data @@ -65,7 +65,6 @@ def test_must_clear_entries(self): class TestRegistryLocking(TestCase): - def setUp(self): self.mock_lock = Mock() self.registry = Registry(write_lock=self.mock_lock) @@ -119,7 +118,6 @@ def test_clear_must_lock(self): class TestGetWorkflow(TestCase): - def setUp(self): self.registry = Registry() self.capability = Capability(language="a", dependency_manager="b", application_framework="c") diff --git a/tests/unit/test_validator.py b/tests/unit/test_validator.py index b3f37d8dd..d48abaaa8 100644 --- a/tests/unit/test_validator.py +++ b/tests/unit/test_validator.py @@ -4,7 +4,6 @@ class TestRuntimeValidator(TestCase): - def setUp(self): self.validator = RuntimeValidator(runtime="chitti2.0") diff --git a/tests/unit/test_workflow.py b/tests/unit/test_workflow.py index 08900950a..4c0e1b480 100644 --- a/tests/unit/test_workflow.py +++ b/tests/unit/test_workflow.py @@ -18,13 +18,11 @@ class TestRegisteringWorkflows(TestCase): - CAPABILITY1 = Capability(language="test", - dependency_manager="testframework", - application_framework="appframework") + CAPABILITY1 = Capability(language="test", dependency_manager="testframework", application_framework="appframework") - CAPABILITY2 = Capability(language="test2", - dependency_manager="testframework2", - application_framework="appframework2") + CAPABILITY2 = Capability( + language="test2", dependency_manager="testframework2", application_framework="appframework2" + ) def tearDown(self): DEFAULT_REGISTRY.clear() @@ -41,7 +39,6 @@ class TestWorkflow(BaseWorkflow): self.assertEquals(result_cls, TestWorkflow) def test_must_register_two_workflows(self): - class TestWorkflow1(BaseWorkflow): NAME = "TestWorkflow" CAPABILITY = self.CAPABILITY1 @@ -57,6 +54,7 @@ class TestWorkflow2(BaseWorkflow): def test_must_fail_if_name_not_present(self): with self.assertRaises(ValueError) as ctx: + class TestWorkflow1(BaseWorkflow): CAPABILITY = self.CAPABILITY1 @@ -66,6 +64,7 @@ class TestWorkflow1(BaseWorkflow): def test_must_fail_if_capabilities_not_present(self): with self.assertRaises(ValueError) as ctx: + class TestWorkflow1(BaseWorkflow): NAME = "somename" @@ -75,6 +74,7 @@ class TestWorkflow1(BaseWorkflow): def test_must_fail_if_capabilities_is_wrong_type(self): with self.assertRaises(ValueError) as ctx: + class TestWorkflow1(BaseWorkflow): NAME = "somename" CAPABILITY = "wrong data type" @@ -84,20 +84,24 @@ class TestWorkflow1(BaseWorkflow): class TestBaseWorkflow_init(TestCase): - class MyWorkflow(BaseWorkflow): __TESTING__ = True NAME = "MyWorkflow" - CAPABILITY = Capability(language="test", - dependency_manager="testframework", - application_framework="appframework") + CAPABILITY = Capability( + language="test", dependency_manager="testframework", application_framework="appframework" + ) def test_must_initialize_variables(self): - self.work = self.MyWorkflow("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", - runtime="runtime", - executable_search_paths=[str(sys.executable)], - optimizations={"a": "b"}, - options={"c": "d"}) + self.work = self.MyWorkflow( + "source_dir", + "artifacts_dir", + "scratch_dir", + "manifest_path", + runtime="runtime", + executable_search_paths=[str(sys.executable)], + optimizations={"a": "b"}, + options={"c": "d"}, + ) self.assertEquals(self.work.source_dir, "source_dir") self.assertEquals(self.work.artifacts_dir, "artifacts_dir") @@ -110,20 +114,24 @@ def test_must_initialize_variables(self): class TestBaseWorkflow_is_supported(TestCase): - class MyWorkflow(BaseWorkflow): __TESTING__ = True NAME = "MyWorkflow" - CAPABILITY = Capability(language="test", - dependency_manager="testframework", - application_framework="appframework") + CAPABILITY = Capability( + language="test", dependency_manager="testframework", application_framework="appframework" + ) def setUp(self): - self.work = self.MyWorkflow("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", - runtime="runtime", - executable_search_paths=[], - optimizations={"a": "b"}, - options={"c": "d"}) + self.work = self.MyWorkflow( + "source_dir", + "artifacts_dir", + "scratch_dir", + "manifest_path", + runtime="runtime", + executable_search_paths=[], + optimizations={"a": "b"}, + options={"c": "d"}, + ) def test_must_ignore_manifest_if_not_provided(self): self.work.SUPPORTED_MANIFESTS = [] # No manifest provided @@ -148,20 +156,24 @@ def test_must_fail_if_manifest_not_in_list(self): class TestBaseWorkflow_run(TestCase): - class MyWorkflow(BaseWorkflow): __TESTING__ = True NAME = "MyWorkflow" - CAPABILITY = Capability(language="test", - dependency_manager="testframework", - application_framework="appframework") + CAPABILITY = Capability( + language="test", dependency_manager="testframework", application_framework="appframework" + ) def setUp(self): - self.work = self.MyWorkflow("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", - runtime="runtime", - executable_search_paths=[], - optimizations={"a": "b"}, - options={"c": "d"}) + self.work = self.MyWorkflow( + "source_dir", + "artifacts_dir", + "scratch_dir", + "manifest_path", + runtime="runtime", + executable_search_paths=[], + optimizations={"a": "b"}, + options={"c": "d"}, + ) def test_get_binaries(self): self.assertIsNotNone(self.work.binaries) @@ -177,9 +189,9 @@ def test_must_execute_actions_in_sequence(self): action_mock = Mock() validator_mock = Mock() validator_mock.validate = Mock() - validator_mock.validate.return_value = '/usr/bin/binary' + validator_mock.validate.return_value = "/usr/bin/binary" resolver_mock = Mock() - resolver_mock.exec_paths = ['/usr/bin/binary'] + resolver_mock.exec_paths = ["/usr/bin/binary"] binaries_mock = Mock() binaries_mock.return_value = [] @@ -189,9 +201,9 @@ def test_must_execute_actions_in_sequence(self): self.work.binaries = {"binary": BinaryPath(resolver=resolver_mock, validator=validator_mock, binary="binary")} self.work.run() - self.assertEquals(action_mock.method_calls, [ - call.action1.execute(), call.action2.execute(), call.action3.execute() - ]) + self.assertEquals( + action_mock.method_calls, [call.action1.execute(), call.action2.execute(), call.action3.execute()] + ) self.assertTrue(validator_mock.validate.call_count, 1) def test_must_raise_with_no_actions(self): @@ -230,23 +242,27 @@ def test_supply_executable_path(self): # Run workflow with supplied executable path to search for executables. action_mock = Mock() - self.work = self.MyWorkflow("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", - runtime="runtime", - executable_search_paths=[str(pathlib.Path(os.getcwd()).parent)], - optimizations={"a": "b"}, - options={"c": "d"}) + self.work = self.MyWorkflow( + "source_dir", + "artifacts_dir", + "scratch_dir", + "manifest_path", + runtime="runtime", + executable_search_paths=[str(pathlib.Path(os.getcwd()).parent)], + optimizations={"a": "b"}, + options={"c": "d"}, + ) self.work.actions = [action_mock.action1, action_mock.action2, action_mock.action3] self.work.run() class TestBaseWorkflow_repr(TestCase): - class MyWorkflow(BaseWorkflow): __TESTING__ = True NAME = "MyWorkflow" - CAPABILITY = Capability(language="test", - dependency_manager="testframework", - application_framework="appframework") + CAPABILITY = Capability( + language="test", dependency_manager="testframework", application_framework="appframework" + ) def setUp(self): self.action1 = Mock() @@ -254,16 +270,21 @@ def setUp(self): self.action3 = Mock() self.action1.__repr__ = Mock(return_value="Name=Action1, Purpose=COPY_SOURCE, Description=Copies source code") - self.action2.__repr__ = Mock(return_value="Name=Action2, Purpose=RESOLVE_DEPENDENCIES," - " Description=Resolves dependencies") - self.action3.__repr__ = Mock(return_value="Name=Action3, Purpose=COMPILE_SOURCE, " - "Description=Compiles code") - - self.work = self.MyWorkflow("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", - runtime="runtime", - executable_search_paths=[], - optimizations={"a": "b"}, - options={"c": "d"}) + self.action2.__repr__ = Mock( + return_value="Name=Action2, Purpose=RESOLVE_DEPENDENCIES," " Description=Resolves dependencies" + ) + self.action3.__repr__ = Mock(return_value="Name=Action3, Purpose=COMPILE_SOURCE, " "Description=Compiles code") + + self.work = self.MyWorkflow( + "source_dir", + "artifacts_dir", + "scratch_dir", + "manifest_path", + runtime="runtime", + executable_search_paths=[], + optimizations={"a": "b"}, + options={"c": "d"}, + ) def test_must_pretty_print_workflow_info(self): self.work.actions = [self.action1, self.action2, self.action3] diff --git a/tests/unit/workflows/dotnet_clipackage/test_actions.py b/tests/unit/workflows/dotnet_clipackage/test_actions.py index 17d4edc59..171ff4fe9 100644 --- a/tests/unit/workflows/dotnet_clipackage/test_actions.py +++ b/tests/unit/workflows/dotnet_clipackage/test_actions.py @@ -9,7 +9,6 @@ class TestGlobalToolInstallAction(TestCase): - @patch("aws_lambda_builders.workflows.dotnet_clipackage.dotnetcli.SubprocessDotnetCLI") def setUp(self, MockSubprocessDotnetCLI): self.subprocess_dotnet = MockSubprocessDotnetCLI.return_value @@ -20,32 +19,33 @@ def tearDown(self): def test_global_tool_install(self): action = GlobalToolInstallAction(self.subprocess_dotnet) action.execute() - self.subprocess_dotnet.run.assert_called_once_with(['tool', 'install', '-g', 'Amazon.Lambda.Tools']) + self.subprocess_dotnet.run.assert_called_once_with(["tool", "install", "-g", "Amazon.Lambda.Tools"]) def test_global_tool_update(self): self.subprocess_dotnet.run.side_effect = [DotnetCLIExecutionError(message="Already Installed"), None] action = GlobalToolInstallAction(self.subprocess_dotnet) action.execute() - self.subprocess_dotnet.run.assert_any_call(['tool', 'install', '-g', 'Amazon.Lambda.Tools']) - self.subprocess_dotnet.run.assert_any_call(['tool', 'update', '-g', 'Amazon.Lambda.Tools']) + self.subprocess_dotnet.run.assert_any_call(["tool", "install", "-g", "Amazon.Lambda.Tools"]) + self.subprocess_dotnet.run.assert_any_call(["tool", "update", "-g", "Amazon.Lambda.Tools"]) def test_global_tool_update_failed(self): - self.subprocess_dotnet.run.side_effect = [DotnetCLIExecutionError(message="Already Installed"), - DotnetCLIExecutionError(message="Updated Failed")] + self.subprocess_dotnet.run.side_effect = [ + DotnetCLIExecutionError(message="Already Installed"), + DotnetCLIExecutionError(message="Updated Failed"), + ] action = GlobalToolInstallAction(self.subprocess_dotnet) self.assertRaises(ActionFailedError, action.execute) class TestRunPackageAction(TestCase): - @patch("aws_lambda_builders.workflows.dotnet_clipackage.dotnetcli.SubprocessDotnetCLI") @patch("aws_lambda_builders.workflows.dotnet_clipackage.utils.OSUtils") def setUp(self, MockSubprocessDotnetCLI, MockOSUtils): self.subprocess_dotnet = MockSubprocessDotnetCLI.return_value self.os_utils = MockOSUtils - self.source_dir = os.path.join('/source_dir') - self.artifacts_dir = os.path.join('/artifacts_dir') - self.scratch_dir = os.path.join('/scratch_dir') + self.source_dir = os.path.join("/source_dir") + self.artifacts_dir = os.path.join("/artifacts_dir") + self.scratch_dir = os.path.join("/scratch_dir") def tearDown(self): self.subprocess_dotnet.reset_mock() @@ -54,53 +54,58 @@ def test_build_package(self): mode = "Release" options = {} - action = RunPackageAction(self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, - self.os_utils) + action = RunPackageAction( + self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, self.os_utils + ) action.execute() - zipFilePath = os.path.join('/', 'artifacts_dir', 'source_dir.zip') + zipFilePath = os.path.join("/", "artifacts_dir", "source_dir.zip") - self.subprocess_dotnet.run.assert_called_once_with(['lambda', 'package', '--output-package', zipFilePath], - cwd='/source_dir') + self.subprocess_dotnet.run.assert_called_once_with( + ["lambda", "package", "--output-package", zipFilePath], cwd="/source_dir" + ) def test_build_package_arguments(self): mode = "Release" options = {"--framework": "netcoreapp2.1"} - action = RunPackageAction(self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, - self.os_utils) + action = RunPackageAction( + self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, self.os_utils + ) action.execute() - if platform.system().lower() == 'windows': - zipFilePath = '/artifacts_dir\\source_dir.zip' + if platform.system().lower() == "windows": + zipFilePath = "/artifacts_dir\\source_dir.zip" else: - zipFilePath = '/artifacts_dir/source_dir.zip' + zipFilePath = "/artifacts_dir/source_dir.zip" - self.subprocess_dotnet.run.assert_called_once_with(['lambda', 'package', '--output-package', - zipFilePath, '--framework', 'netcoreapp2.1'], - cwd='/source_dir') + self.subprocess_dotnet.run.assert_called_once_with( + ["lambda", "package", "--output-package", zipFilePath, "--framework", "netcoreapp2.1"], cwd="/source_dir" + ) def test_build_error(self): mode = "Release" self.subprocess_dotnet.run.side_effect = DotnetCLIExecutionError(message="Failed Package") options = {} - action = RunPackageAction(self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, - self.os_utils) + action = RunPackageAction( + self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, self.os_utils + ) self.assertRaises(ActionFailedError, action.execute) def test_debug_configuration_set(self): mode = "Debug" options = None - action = RunPackageAction(self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, - self.os_utils) + action = RunPackageAction( + self.source_dir, self.subprocess_dotnet, self.artifacts_dir, options, mode, self.os_utils + ) - zipFilePath = os.path.join('/', 'artifacts_dir', 'source_dir.zip') + zipFilePath = os.path.join("/", "artifacts_dir", "source_dir.zip") action.execute() self.subprocess_dotnet.run.assert_called_once_with( - ['lambda', 'package', '--output-package', zipFilePath, '--configuration', 'Debug'], - cwd='/source_dir') + ["lambda", "package", "--output-package", zipFilePath, "--configuration", "Debug"], cwd="/source_dir" + ) diff --git a/tests/unit/workflows/dotnet_clipackage/test_dotnetcli.py b/tests/unit/workflows/dotnet_clipackage/test_dotnetcli.py index 45243b7e6..7d014bc63 100644 --- a/tests/unit/workflows/dotnet_clipackage/test_dotnetcli.py +++ b/tests/unit/workflows/dotnet_clipackage/test_dotnetcli.py @@ -5,7 +5,6 @@ class TestSubprocessDotnetCLI(TestCase): - @patch("aws_lambda_builders.workflows.dotnet_clipackage.utils.OSUtils") def setUp(self, MockOSUtils): self.os_utils = MockOSUtils.return_value @@ -16,7 +15,7 @@ def test_dotnetcli_name_windows(self): dotnetCli = SubprocessDotnetCLI(os_utils=self.os_utils) - assert dotnetCli.dotnet_exe == 'dotnet.exe' + assert dotnetCli.dotnet_exe == "dotnet.exe" def test_dotnetcli_name_non_windows(self): self.os_utils.reset_mock() @@ -24,7 +23,7 @@ def test_dotnetcli_name_non_windows(self): dotnetCli = SubprocessDotnetCLI(os_utils=self.os_utils) - assert dotnetCli.dotnet_exe == 'dotnet' + assert dotnetCli.dotnet_exe == "dotnet" def test_invalid_args(self): self.os_utils.reset_mock() diff --git a/tests/unit/workflows/dotnet_clipackage/test_dotnetcli_resolver.py b/tests/unit/workflows/dotnet_clipackage/test_dotnetcli_resolver.py index b824d86f6..4d2b14a3f 100644 --- a/tests/unit/workflows/dotnet_clipackage/test_dotnetcli_resolver.py +++ b/tests/unit/workflows/dotnet_clipackage/test_dotnetcli_resolver.py @@ -5,7 +5,6 @@ class TestDotnetCliResolver(TestCase): - @patch("aws_lambda_builders.workflows.dotnet_clipackage.utils.OSUtils") def setUp(self, MockOSUtils): self.os_utils = MockOSUtils.return_value diff --git a/tests/unit/workflows/dotnet_clipackage/test_workflow.py b/tests/unit/workflows/dotnet_clipackage/test_workflow.py index 6dfc4ccd0..f520c40dd 100644 --- a/tests/unit/workflows/dotnet_clipackage/test_workflow.py +++ b/tests/unit/workflows/dotnet_clipackage/test_workflow.py @@ -5,7 +5,6 @@ class TestDotnetCliPackageWorkflow(TestCase): - def test_actions(self): workflow = DotnetCliPackageWorkflow("source_dir", "artifacts_dir", "scratch_dir", "manifest_path") self.assertEqual(workflow.actions.__len__(), 2) diff --git a/tests/unit/workflows/go_dep/test_actions.py b/tests/unit/workflows/go_dep/test_actions.py index fbefa8c18..33978182a 100644 --- a/tests/unit/workflows/go_dep/test_actions.py +++ b/tests/unit/workflows/go_dep/test_actions.py @@ -49,9 +49,9 @@ def test_runs_go_build(self, SubProcMock): action.execute() - sub_proc_go.run.assert_called_with(["build", "-o", "output", "source"], - cwd="source", - env={"GOOS": "linux", "GOARCH": "amd64"}) + sub_proc_go.run.assert_called_with( + ["build", "-o", "output", "source"], cwd="source", env={"GOOS": "linux", "GOARCH": "amd64"} + ) @patch("aws_lambda_builders.workflows.go_dep.subproc_exec.SubprocessExec") def test_fails_go_build(self, SubProcMock): diff --git a/tests/unit/workflows/go_dep/test_exec.py b/tests/unit/workflows/go_dep/test_exec.py index 13a20f8aa..117505acb 100644 --- a/tests/unit/workflows/go_dep/test_exec.py +++ b/tests/unit/workflows/go_dep/test_exec.py @@ -15,7 +15,6 @@ def communicate(self): class TestSubprocessExec(TestCase): - @patch("aws_lambda_builders.workflows.go_dep.utils.OSUtils") def setUp(self, OSUtilMock): self.osutils = OSUtilMock.return_value @@ -34,14 +33,16 @@ def test_run_executes_bin_on_nixes(self): def test_uses_cwd_if_supplied(self): self.under_test.run(["did", "thing"], cwd="/a/cwd") - self.osutils.popen.assert_called_with(["bin", "did", "thing"], - cwd="/a/cwd", env=None, stderr="PIPE", stdout="PIPE") + self.osutils.popen.assert_called_with( + ["bin", "did", "thing"], cwd="/a/cwd", env=None, stderr="PIPE", stdout="PIPE" + ) def test_uses_env_if_supplied(self): self.under_test.run(["did", "thing"], env={"foo": "bar"}) - self.osutils.popen.assert_called_with(["bin", "did", "thing"], - cwd=None, env={"foo": "bar"}, stderr="PIPE", stdout="PIPE") + self.osutils.popen.assert_called_with( + ["bin", "did", "thing"], cwd=None, env={"foo": "bar"}, stderr="PIPE", stdout="PIPE" + ) def test_returns_popen_out_decoded_if_retcode_is_0(self): self.popen.out = "some encoded text\n\n" diff --git a/tests/unit/workflows/go_dep/test_workflow.py b/tests/unit/workflows/go_dep/test_workflow.py index 000e666c0..d4c14518a 100644 --- a/tests/unit/workflows/go_dep/test_workflow.py +++ b/tests/unit/workflows/go_dep/test_workflow.py @@ -11,11 +11,9 @@ class TestGoDepWorkflow(TestCase): """ def test_workflow_sets_up_workflow(self): - workflow = GoDepWorkflow("source", - "artifacts", - "scratch", - "manifest", - options={"artifact_executable_name": "foo"}) + workflow = GoDepWorkflow( + "source", "artifacts", "scratch", "manifest", options={"artifact_executable_name": "foo"} + ) self.assertEqual(len(workflow.actions), 2) self.assertIsInstance(workflow.actions[0], DepEnsureAction) self.assertIsInstance(workflow.actions[1], GoBuildAction) diff --git a/tests/unit/workflows/go_modules/test_builder.py b/tests/unit/workflows/go_modules/test_builder.py index 9e69db981..daea632c6 100644 --- a/tests/unit/workflows/go_modules/test_builder.py +++ b/tests/unit/workflows/go_modules/test_builder.py @@ -7,7 +7,7 @@ class FakePopen: - def __init__(self, out=b'out', err=b'err', retcode=0): + def __init__(self, out=b"out", err=b"err", retcode=0): self.out = out self.err = err self.returncode = retcode @@ -17,17 +17,13 @@ def communicate(self): class TestGoBuilder(TestCase): - @patch("aws_lambda_builders.workflows.go_modules.utils.OSUtils") def setUp(self, OSUtilMock): self.osutils = OSUtilMock.return_value - self.osutils.pipe = 'PIPE' + self.osutils.pipe = "PIPE" self.popen = FakePopen() self.osutils.popen.side_effect = [self.popen] - self.binaries = { - "go": BinaryPath(resolver=Mock(), validator=Mock(), - binary="go", binary_path="/path/to/go") - } + self.binaries = {"go": BinaryPath(resolver=Mock(), validator=Mock(), binary="go", binary_path="/path/to/go")} self.under_test = GoModulesBuilder(self.osutils, self.binaries) def test_run_executes_bundler_on_nixes(self): @@ -37,19 +33,19 @@ def test_run_executes_bundler_on_nixes(self): self.osutils.popen.assert_called_with( ["/path/to/go", "build", "-o", "output_path", "source_dir"], cwd="source_dir", - env={'GOOS': 'linux', 'GOARCH': 'amd64'}, - stderr='PIPE', - stdout='PIPE', + env={"GOOS": "linux", "GOARCH": "amd64"}, + stderr="PIPE", + stdout="PIPE", ) def test_returns_popen_out_decoded_if_retcode_is_0(self): - self.popen.out = b'some encoded text\n\n' + self.popen.out = b"some encoded text\n\n" result = self.under_test.build("source_dir", "output_path") - self.assertEqual(result, 'some encoded text') + self.assertEqual(result, "some encoded text") def test_raises_BuilderError_with_err_text_if_retcode_is_not_0(self): self.popen.returncode = 1 - self.popen.err = b'some error text\n\n' + self.popen.err = b"some error text\n\n" with self.assertRaises(BuilderError) as raised: self.under_test.build("source_dir", "output_path") self.assertEqual(raised.exception.args[0], "Builder Failed: some error text") diff --git a/tests/unit/workflows/go_modules/test_validator.py b/tests/unit/workflows/go_modules/test_validator.py index 40486c8f7..ffbf90c38 100644 --- a/tests/unit/workflows/go_modules/test_validator.py +++ b/tests/unit/workflows/go_modules/test_validator.py @@ -8,7 +8,6 @@ class MockSubProcess(object): - def __init__(self, returncode, out=b"", err=b""): self.returncode = returncode self.out = out @@ -19,13 +18,10 @@ def communicate(self): class TestGoRuntimeValidator(TestCase): - def setUp(self): self.validator = GoRuntimeValidator(runtime="go1.x") - @parameterized.expand([ - "go1.x", - ]) + @parameterized.expand(["go1.x"]) def test_supported_runtimes(self, runtime): validator = GoRuntimeValidator(runtime=runtime) self.assertTrue(validator.has_runtime()) diff --git a/tests/unit/workflows/go_modules/test_workflow.py b/tests/unit/workflows/go_modules/test_workflow.py index 6a85df8bb..ddab1449d 100644 --- a/tests/unit/workflows/go_modules/test_workflow.py +++ b/tests/unit/workflows/go_modules/test_workflow.py @@ -12,8 +12,12 @@ class TestGoModulesWorkflow(TestCase): def test_workflow_sets_up_builder_actions(self): workflow = GoModulesWorkflow( - "source", "artifacts", "scratch_dir", "manifest", + "source", + "artifacts", + "scratch_dir", + "manifest", runtime="go1.x", - options={"artifact_executable_name": "main"}) + options={"artifact_executable_name": "main"}, + ) self.assertEqual(len(workflow.actions), 1) self.assertIsInstance(workflow.actions[0], GoModulesBuildAction) diff --git a/tests/unit/workflows/java_gradle/test_actions.py b/tests/unit/workflows/java_gradle/test_actions.py index f69a97075..08af9d344 100644 --- a/tests/unit/workflows/java_gradle/test_actions.py +++ b/tests/unit/workflows/java_gradle/test_actions.py @@ -3,73 +3,66 @@ import os from aws_lambda_builders.actions import ActionFailedError -from aws_lambda_builders.workflows.java_gradle.actions import JavaGradleBuildAction, JavaGradleCopyArtifactsAction, \ - GradleExecutionError +from aws_lambda_builders.workflows.java_gradle.actions import ( + JavaGradleBuildAction, + JavaGradleCopyArtifactsAction, + GradleExecutionError, +) class TestJavaGradleBuildAction(TestCase): - @patch("aws_lambda_builders.workflows.java_gradle.utils.OSUtils") @patch("aws_lambda_builders.workflows.java_gradle.gradle.SubprocessGradle") def setUp(self, MockSubprocessGradle, MockOSUtils): self.subprocess_gradle = MockSubprocessGradle.return_value self.os_utils = MockOSUtils.return_value self.os_utils.copy.side_effect = lambda src, dst: dst - self.source_dir = os.path.join('source_dir') - self.manifest_path = os.path.join(self.source_dir, 'manifest') - self.artifacts_dir = os.path.join('artifacts_dir') - self.scratch_dir = os.path.join('scratch_dir') + self.source_dir = os.path.join("source_dir") + self.manifest_path = os.path.join(self.source_dir, "manifest") + self.artifacts_dir = os.path.join("artifacts_dir") + self.scratch_dir = os.path.join("scratch_dir") def test_calls_gradle_build(self): - action = JavaGradleBuildAction(self.source_dir, - self.manifest_path, - self.subprocess_gradle, - self.scratch_dir, - self.os_utils) + action = JavaGradleBuildAction( + self.source_dir, self.manifest_path, self.subprocess_gradle, self.scratch_dir, self.os_utils + ) action.execute() - self.subprocess_gradle.build.assert_called_with(self.source_dir, - self.manifest_path, - os.path.join(self.scratch_dir, - JavaGradleBuildAction.GRADLE_CACHE_DIR_NAME), - os.path.join(self.scratch_dir, - JavaGradleBuildAction.INIT_SCRIPT), - {JavaGradleBuildAction.SCRATCH_DIR_PROPERTY: os.path.abspath( - self.scratch_dir)}) + self.subprocess_gradle.build.assert_called_with( + self.source_dir, + self.manifest_path, + os.path.join(self.scratch_dir, JavaGradleBuildAction.GRADLE_CACHE_DIR_NAME), + os.path.join(self.scratch_dir, JavaGradleBuildAction.INIT_SCRIPT), + {JavaGradleBuildAction.SCRATCH_DIR_PROPERTY: os.path.abspath(self.scratch_dir)}, + ) def test_error_in_init_file_copy_raises_action_error(self): self.os_utils.copy.side_effect = Exception("Copy failed!") - action = JavaGradleBuildAction(self.source_dir, - self.manifest_path, - self.subprocess_gradle, - self.scratch_dir, - self.os_utils) + action = JavaGradleBuildAction( + self.source_dir, self.manifest_path, self.subprocess_gradle, self.scratch_dir, self.os_utils + ) with self.assertRaises(ActionFailedError) as raised: action.execute() self.assertEquals(raised.exception.args[0], "Copy failed!") def test_error_building_project_raises_action_error(self): - self.subprocess_gradle.build.side_effect = GradleExecutionError(message='Build failed!') - action = JavaGradleBuildAction(self.source_dir, - self.manifest_path, - self.subprocess_gradle, - self.scratch_dir, - self.os_utils) + self.subprocess_gradle.build.side_effect = GradleExecutionError(message="Build failed!") + action = JavaGradleBuildAction( + self.source_dir, self.manifest_path, self.subprocess_gradle, self.scratch_dir, self.os_utils + ) with self.assertRaises(ActionFailedError) as raised: action.execute() - self.assertEquals(raised.exception.args[0], 'Gradle Failed: Build failed!') + self.assertEquals(raised.exception.args[0], "Gradle Failed: Build failed!") def test_computes_correct_cache_dir(self): - action = JavaGradleBuildAction(self.source_dir, - self.manifest_path, - self.subprocess_gradle, - self.scratch_dir, - self.os_utils) - self.assertEquals(action.gradle_cache_dir, - os.path.join(self.scratch_dir, JavaGradleBuildAction.GRADLE_CACHE_DIR_NAME)) + action = JavaGradleBuildAction( + self.source_dir, self.manifest_path, self.subprocess_gradle, self.scratch_dir, self.os_utils + ) + self.assertEquals( + action.gradle_cache_dir, os.path.join(self.scratch_dir, JavaGradleBuildAction.GRADLE_CACHE_DIR_NAME) + ) class TestJavaGradleCopyArtifactsAction(TestCase): - @patch("aws_lambda_builders.workflows.java_gradle.utils.OSUtils") def setUp(self, MockOSUtils): self.os_utils = MockOSUtils.return_value @@ -77,27 +70,22 @@ def setUp(self, MockOSUtils): self.source_dir = "source_dir" self.artifacts_dir = "artifacts_dir" self.scratch_dir = "scratch_dir" - self.build_dir = os.path.join(self.scratch_dir, 'build1') + self.build_dir = os.path.join(self.scratch_dir, "build1") def test_copies_artifacts(self): self.os_utils.copytree.side_effect = lambda src, dst: None self.os_utils.copy.side_effect = lambda src, dst: None - action = JavaGradleCopyArtifactsAction(self.source_dir, - self.artifacts_dir, - self.build_dir, - self.os_utils) + action = JavaGradleCopyArtifactsAction(self.source_dir, self.artifacts_dir, self.build_dir, self.os_utils) action.execute() self.os_utils.copytree.assert_called_with( - os.path.join(self.build_dir, 'build', 'distributions', 'lambda-build'), self.artifacts_dir) + os.path.join(self.build_dir, "build", "distributions", "lambda-build"), self.artifacts_dir + ) def test_error_in_artifact_copy_raises_action_error(self): self.os_utils.copytree.side_effect = Exception("scandir failed!") - action = JavaGradleCopyArtifactsAction(self.source_dir, - self.artifacts_dir, - self.build_dir, - self.os_utils) + action = JavaGradleCopyArtifactsAction(self.source_dir, self.artifacts_dir, self.build_dir, self.os_utils) with self.assertRaises(ActionFailedError) as raised: action.execute() self.assertEquals(raised.exception.args[0], "scandir failed!") diff --git a/tests/unit/workflows/java_gradle/test_gradle.py b/tests/unit/workflows/java_gradle/test_gradle.py index b4376cafc..1c46bf966 100644 --- a/tests/unit/workflows/java_gradle/test_gradle.py +++ b/tests/unit/workflows/java_gradle/test_gradle.py @@ -4,12 +4,15 @@ from mock import patch from aws_lambda_builders.binary_path import BinaryPath -from aws_lambda_builders.workflows.java_gradle.gradle import SubprocessGradle, GradleExecutionError, \ - BuildFileNotFoundError +from aws_lambda_builders.workflows.java_gradle.gradle import ( + SubprocessGradle, + GradleExecutionError, + BuildFileNotFoundError, +) class FakePopen: - def __init__(self, out=b'out', err=b'err', retcode=0): + def __init__(self, out=b"out", err=b"err", retcode=0): self.out = out self.err = err self.returncode = retcode @@ -22,72 +25,89 @@ def wait(self): class TestSubprocessGradle(TestCase): - @patch("aws_lambda_builders.workflows.java_gradle.utils.OSUtils") def setUp(self, MockOSUtils): self.os_utils = MockOSUtils.return_value self.os_utils.exists.side_effect = lambda d: True self.popen = FakePopen() self.os_utils.popen.side_effect = [self.popen] - self.gradle_path = '/path/to/gradle' - self.gradle_binary = BinaryPath(None, None, 'gradle', binary_path=self.gradle_path) - self.source_dir = '/foo/bar/baz' - self.manifest_path = '/foo/bar/baz/build.gradle' - self.init_script = '/path/to/init' + self.gradle_path = "/path/to/gradle" + self.gradle_binary = BinaryPath(None, None, "gradle", binary_path=self.gradle_path) + self.source_dir = "/foo/bar/baz" + self.manifest_path = "/foo/bar/baz/build.gradle" + self.init_script = "/path/to/init" def test_no_os_utils_build_init_throws(self): with self.assertRaises(ValueError) as err_assert: SubprocessGradle(gradle_binary=self.gradle_binary) - self.assertEquals(err_assert.exception.args[0], 'Must provide OSUtils') + self.assertEquals(err_assert.exception.args[0], "Must provide OSUtils") def test_no_gradle_exec_init_throws(self): with self.assertRaises(ValueError) as err_assert: SubprocessGradle(None) - self.assertEquals(err_assert.exception.args[0], 'Must provide Gradle BinaryPath') + self.assertEquals(err_assert.exception.args[0], "Must provide Gradle BinaryPath") def test_no_build_file_throws(self): self.os_utils.exists.side_effect = lambda d: False gradle = SubprocessGradle(gradle_binary=self.gradle_binary, os_utils=self.os_utils) with self.assertRaises(BuildFileNotFoundError) as raised: gradle.build(self.source_dir, self.manifest_path) - self.assertEquals(raised.exception.args[0], - 'Gradle Failed: Gradle build file not found: %s' % self.manifest_path) + self.assertEquals( + raised.exception.args[0], "Gradle Failed: Gradle build file not found: %s" % self.manifest_path + ) def test_build_no_init_script(self): gradle = SubprocessGradle(gradle_binary=self.gradle_binary, os_utils=self.os_utils) gradle.build(self.source_dir, self.manifest_path) - self.os_utils.popen.assert_called_with([self.gradle_path, 'build', '--build-file', self.manifest_path], - cwd=self.source_dir, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE) + self.os_utils.popen.assert_called_with( + [self.gradle_path, "build", "--build-file", self.manifest_path], + cwd=self.source_dir, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) def test_gradlew_path_is_dummy_uses_gradle_binary(self): gradle = SubprocessGradle(gradle_binary=self.gradle_binary, os_utils=self.os_utils) gradle.build(self.source_dir, self.manifest_path) - self.os_utils.popen.assert_called_with([self.gradle_path, 'build', '--build-file', self.manifest_path], - cwd=self.source_dir, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE) + self.os_utils.popen.assert_called_with( + [self.gradle_path, "build", "--build-file", self.manifest_path], + cwd=self.source_dir, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) def test_build_with_init_script(self): gradle = SubprocessGradle(gradle_binary=self.gradle_binary, os_utils=self.os_utils) gradle.build(self.source_dir, self.manifest_path, init_script_path=self.init_script) self.os_utils.popen.assert_called_with( - [self.gradle_path, 'build', '--build-file', self.manifest_path, '--init-script', self.init_script], - cwd=self.source_dir, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + [self.gradle_path, "build", "--build-file", self.manifest_path, "--init-script", self.init_script], + cwd=self.source_dir, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) def test_raises_exception_if_retcode_not_0(self): - self.popen = FakePopen(retcode=1, err=b'Some Error Message') + self.popen = FakePopen(retcode=1, err=b"Some Error Message") self.os_utils.popen.side_effect = [self.popen] gradle = SubprocessGradle(gradle_binary=self.gradle_binary, os_utils=self.os_utils) with self.assertRaises(GradleExecutionError) as err: gradle.build(self.source_dir, self.manifest_path) - self.assertEquals(err.exception.args[0], 'Gradle Failed: Some Error Message') + self.assertEquals(err.exception.args[0], "Gradle Failed: Some Error Message") def test_includes_build_properties_in_command(self): gradle = SubprocessGradle(gradle_binary=self.gradle_binary, os_utils=self.os_utils) - gradle.build(self.source_dir, self.manifest_path, init_script_path=self.init_script, properties={'foo': 'bar'}) + gradle.build(self.source_dir, self.manifest_path, init_script_path=self.init_script, properties={"foo": "bar"}) self.os_utils.popen.assert_called_with( - [self.gradle_path, 'build', '--build-file', self.manifest_path, '-Dfoo=bar', '--init-script', - self.init_script], - cwd=self.source_dir, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + [ + self.gradle_path, + "build", + "--build-file", + self.manifest_path, + "-Dfoo=bar", + "--init-script", + self.init_script, + ], + cwd=self.source_dir, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) diff --git a/tests/unit/workflows/java_gradle/test_gradle_validator.py b/tests/unit/workflows/java_gradle/test_gradle_validator.py index 68f62d460..19a7768b6 100644 --- a/tests/unit/workflows/java_gradle/test_gradle_validator.py +++ b/tests/unit/workflows/java_gradle/test_gradle_validator.py @@ -20,43 +20,32 @@ def returncode(self): class TestGradleBinaryValidator(TestCase): - @patch("aws_lambda_builders.workflows.java_gradle.utils.OSUtils") def setUp(self, MockOSUtils): self.mock_os_utils = MockOSUtils.return_value self.mock_log = Mock() - self.gradle_path = '/path/to/gradle' - self.runtime = 'java8' + self.gradle_path = "/path/to/gradle" + self.runtime = "java8" - @parameterized.expand([ - '1.7.0', - '1.8.9', - '11.0.0', - '12 (Fluff)', - '12' - ]) + @parameterized.expand(["1.7.0", "1.8.9", "11.0.0", "12 (Fluff)", "12"]) def test_accepts_any_jvm_mv(self, version): - version_string = ('JVM: %s' % version).encode() + version_string = ("JVM: %s" % version).encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string)] validator = GradleValidator(runtime=self.runtime, os_utils=self.mock_os_utils) self.assertTrue(validator.validate(gradle_path=self.gradle_path)) self.assertEqual(validator.validated_binary_path, self.gradle_path) def test_emits_warning_when_jvm_mv_greater_than_8(self): - version_string = 'JVM: 9.0.0'.encode() + version_string = "JVM: 9.0.0".encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string)] validator = GradleValidator(runtime=self.runtime, os_utils=self.mock_os_utils, log=self.mock_log) self.assertTrue(validator.validate(gradle_path=self.gradle_path)) self.assertEqual(validator.validated_binary_path, self.gradle_path) - self.mock_log.warning.assert_called_with(GradleValidator.MAJOR_VERSION_WARNING, self.gradle_path, '9', '8', '8') + self.mock_log.warning.assert_called_with(GradleValidator.MAJOR_VERSION_WARNING, self.gradle_path, "9", "8", "8") - @parameterized.expand([ - '1.6.0', - '1.7.0', - '1.8.9' - ]) + @parameterized.expand(["1.6.0", "1.7.0", "1.8.9"]) def test_does_not_emit_warning_when_jvm_mv_8_or_less(self, version): - version_string = ('JVM: %s' % version).encode() + version_string = ("JVM: %s" % version).encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string)] validator = GradleValidator(runtime=self.runtime, os_utils=self.mock_os_utils, log=self.mock_log) self.assertTrue(validator.validate(gradle_path=self.gradle_path)) @@ -64,21 +53,21 @@ def test_does_not_emit_warning_when_jvm_mv_8_or_less(self, version): self.mock_log.warning.assert_not_called() def test_emits_warning_when_gradle_excutable_fails(self): - version_string = 'JVM: 9.0.0'.encode() + version_string = "JVM: 9.0.0".encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string, returncode=1)] validator = GradleValidator(runtime=self.runtime, os_utils=self.mock_os_utils, log=self.mock_log) validator.validate(gradle_path=self.gradle_path) self.mock_log.warning.assert_called_with(GradleValidator.VERSION_STRING_WARNING, self.gradle_path) def test_emits_warning_when_version_string_not_found(self): - version_string = 'The Java Version: 9.0.0'.encode() + version_string = "The Java Version: 9.0.0".encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string, returncode=0)] validator = GradleValidator(runtime=self.runtime, os_utils=self.mock_os_utils, log=self.mock_log) validator.validate(gradle_path=self.gradle_path) self.mock_log.warning.assert_called_with(GradleValidator.VERSION_STRING_WARNING, self.gradle_path) def test_no_warning_when_jvm_mv_11_and_java11_runtime(self): - version_string = 'JVM: 11.0.0'.encode() + version_string = "JVM: 11.0.0".encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string)] validator = GradleValidator(runtime="java11", os_utils=self.mock_os_utils, log=self.mock_log) self.assertTrue(validator.validate(gradle_path=self.gradle_path)) diff --git a/tests/unit/workflows/java_gradle/test_gradlew_resolver.py b/tests/unit/workflows/java_gradle/test_gradlew_resolver.py index 10b784c82..900f35a59 100644 --- a/tests/unit/workflows/java_gradle/test_gradlew_resolver.py +++ b/tests/unit/workflows/java_gradle/test_gradlew_resolver.py @@ -6,23 +6,23 @@ class TestGradleResolver(TestCase): - @patch("aws_lambda_builders.workflows.java_gradle.utils.OSUtils") def setUp(self, MockOSUtils): self.mock_os_utils = MockOSUtils.return_value self.mock_os_utils.is_windows.side_effect = [False] def test_gradlew_exists_returns_gradlew(self): - gradlew_path = '/path/to/gradlew' + gradlew_path = "/path/to/gradlew" self.mock_os_utils.which.side_effect = lambda executable, executable_search_paths: [gradlew_path] resolver = GradleResolver(os_utils=self.mock_os_utils) self.assertEquals(resolver.exec_paths, [gradlew_path]) def test_gradlew_not_exists_returns_gradle_on_path(self): - gradle_path = '/path/to/gradle' - self.mock_os_utils.which.side_effect = lambda executable, executable_search_paths: \ - [] if executable == 'gradlew' else [gradle_path] + gradle_path = "/path/to/gradle" + self.mock_os_utils.which.side_effect = ( + lambda executable, executable_search_paths: [] if executable == "gradlew" else [gradle_path] + ) resolver = GradleResolver(os_utils=self.mock_os_utils) self.assertEquals(resolver.exec_paths, [gradle_path]) @@ -32,12 +32,9 @@ def test_throws_value_error_if_no_exec_found(self): resolver = GradleResolver(os_utils=self.mock_os_utils) with self.assertRaises(ValueError) as raised: resolver.exec_paths() - self.assertEquals(raised.exception.args[0], 'No Gradle executable found!') + self.assertEquals(raised.exception.args[0], "No Gradle executable found!") - @parameterized.expand([ - [True, 'gradlew.bat'], - [False, 'gradlew'] - ]) + @parameterized.expand([[True, "gradlew.bat"], [False, "gradlew"]]) def test_uses_correct_gradlew_name(self, is_windows, expected_wrapper_name): self.mock_os_utils.is_windows.side_effect = [is_windows] resolver = GradleResolver(os_utils=self.mock_os_utils) diff --git a/tests/unit/workflows/java_gradle/test_workflow.py b/tests/unit/workflows/java_gradle/test_workflow.py index 1a17da9d5..889f01ce7 100644 --- a/tests/unit/workflows/java_gradle/test_workflow.py +++ b/tests/unit/workflows/java_gradle/test_workflow.py @@ -43,7 +43,7 @@ def test_computes_correct_build_dir(self): workflow = JavaGradleWorkflow("source", "artifacts", "scratch_dir", "manifest") sha1 = hashlib.sha1() - sha1.update(os.path.abspath(workflow.source_dir).encode('utf8')) + sha1.update(os.path.abspath(workflow.source_dir).encode("utf8")) expected_build_dir = os.path.join(workflow.scratch_dir, sha1.hexdigest()) diff --git a/tests/unit/workflows/java_maven/test_actions.py b/tests/unit/workflows/java_maven/test_actions.py index 6e4986c88..dfd47cc30 100644 --- a/tests/unit/workflows/java_maven/test_actions.py +++ b/tests/unit/workflows/java_maven/test_actions.py @@ -3,54 +3,52 @@ import os from aws_lambda_builders.actions import ActionFailedError -from aws_lambda_builders.workflows.java_maven.actions import JavaMavenBuildAction, JavaMavenCopyArtifactsAction, \ - JavaMavenCopyDependencyAction, MavenExecutionError +from aws_lambda_builders.workflows.java_maven.actions import ( + JavaMavenBuildAction, + JavaMavenCopyArtifactsAction, + JavaMavenCopyDependencyAction, + MavenExecutionError, +) class TestJavaMavenBuildAction(TestCase): - @patch("aws_lambda_builders.workflows.java_maven.maven.SubprocessMaven") def setUp(self, MockSubprocessMaven): self.subprocess_maven = MockSubprocessMaven.return_value - self.scratch_dir = os.path.join('scratch_dir') - self.artifacts_dir = os.path.join('artifacts_dir') + self.scratch_dir = os.path.join("scratch_dir") + self.artifacts_dir = os.path.join("artifacts_dir") def test_calls_maven_build(self): - action = JavaMavenBuildAction(self.scratch_dir, - self.subprocess_maven) + action = JavaMavenBuildAction(self.scratch_dir, self.subprocess_maven) action.execute() self.subprocess_maven.build.assert_called_with(self.scratch_dir) def test_error_building_project_raises_action_error(self): - self.subprocess_maven.build.side_effect = MavenExecutionError(message='Build failed!') - action = JavaMavenBuildAction(self.scratch_dir, - self.subprocess_maven) + self.subprocess_maven.build.side_effect = MavenExecutionError(message="Build failed!") + action = JavaMavenBuildAction(self.scratch_dir, self.subprocess_maven) with self.assertRaises(ActionFailedError) as raised: action.execute() - self.assertEquals(raised.exception.args[0], 'Maven Failed: Build failed!') + self.assertEquals(raised.exception.args[0], "Maven Failed: Build failed!") class TestJavaMavenCopyDependencyAction(TestCase): - @patch("aws_lambda_builders.workflows.java_maven.maven.SubprocessMaven") def setUp(self, MockSubprocessMaven): self.subprocess_maven = MockSubprocessMaven.return_value - self.scratch_dir = os.path.join('scratch_dir') - self.artifacts_dir = os.path.join('artifacts_dir') + self.scratch_dir = os.path.join("scratch_dir") + self.artifacts_dir = os.path.join("artifacts_dir") def test_calls_maven_copy_dependency(self): - action = JavaMavenCopyDependencyAction(self.scratch_dir, - self.subprocess_maven) + action = JavaMavenCopyDependencyAction(self.scratch_dir, self.subprocess_maven) action.execute() self.subprocess_maven.copy_dependency.assert_called_with(self.scratch_dir) def test_error_building_project_raises_action_error(self): - self.subprocess_maven.copy_dependency.side_effect = MavenExecutionError(message='Build failed!') - action = JavaMavenCopyDependencyAction(self.scratch_dir, - self.subprocess_maven) + self.subprocess_maven.copy_dependency.side_effect = MavenExecutionError(message="Build failed!") + action = JavaMavenCopyDependencyAction(self.scratch_dir, self.subprocess_maven) with self.assertRaises(ActionFailedError) as raised: action.execute() - self.assertEquals(raised.exception.args[0], 'Maven Failed: Build failed!') + self.assertEquals(raised.exception.args[0], "Maven Failed: Build failed!") class TestJavaMavenCopyArtifactsAction(TestCase): @@ -59,52 +57,49 @@ def setUp(self, MockOSUtils): self.os_utils = MockOSUtils.return_value self.os_utils.copy.side_effect = lambda src, dst: dst self.scratch_dir = "scratch_dir" - self.output_dir = os.path.join(self.scratch_dir, 'target', 'classes') - self.artifacts_dir = os.path.join('artifacts_dir') + self.output_dir = os.path.join(self.scratch_dir, "target", "classes") + self.artifacts_dir = os.path.join("artifacts_dir") def test_copies_artifacts_no_deps(self): self.os_utils.exists.return_value = True self.os_utils.copytree.side_effect = lambda src, dst: None self.os_utils.copy.side_effect = lambda src, dst: None - action = JavaMavenCopyArtifactsAction(self.scratch_dir, - self.artifacts_dir, - self.os_utils) + action = JavaMavenCopyArtifactsAction(self.scratch_dir, self.artifacts_dir, self.os_utils) action.execute() - self.os_utils.copytree.assert_has_calls([ - call(os.path.join(self.scratch_dir, 'target', 'classes'), self.artifacts_dir)]) + self.os_utils.copytree.assert_has_calls( + [call(os.path.join(self.scratch_dir, "target", "classes"), self.artifacts_dir)] + ) def test_copies_artifacts_with_deps(self): self.os_utils.exists.return_value = True self.os_utils.copytree.side_effect = lambda src, dst: None self.os_utils.copy.side_effect = lambda src, dst: None - os.path.join(self.scratch_dir, 'target', 'dependency') + os.path.join(self.scratch_dir, "target", "dependency") - action = JavaMavenCopyArtifactsAction(self.scratch_dir, - self.artifacts_dir, - self.os_utils) + action = JavaMavenCopyArtifactsAction(self.scratch_dir, self.artifacts_dir, self.os_utils) action.execute() - self.os_utils.copytree.assert_has_calls([ - call(os.path.join(self.scratch_dir, 'target', 'classes'), self.artifacts_dir), - call(os.path.join(self.scratch_dir, 'target', 'dependency'), os.path.join(self.artifacts_dir, 'lib'))]) + self.os_utils.copytree.assert_has_calls( + [ + call(os.path.join(self.scratch_dir, "target", "classes"), self.artifacts_dir), + call(os.path.join(self.scratch_dir, "target", "dependency"), os.path.join(self.artifacts_dir, "lib")), + ] + ) def test_error_in_artifact_copy_raises_action_error(self): self.os_utils.exists.return_value = True self.os_utils.copytree.side_effect = Exception("copy failed!") - action = JavaMavenCopyArtifactsAction(self.scratch_dir, - self.artifacts_dir, - self.os_utils) + action = JavaMavenCopyArtifactsAction(self.scratch_dir, self.artifacts_dir, self.os_utils) with self.assertRaises(ActionFailedError) as raised: action.execute() self.assertEquals(raised.exception.args[0], "copy failed!") def test_missing_required_target_class_directory_raises_action_error(self): self.os_utils.exists.return_value = False - action = JavaMavenCopyArtifactsAction(self.scratch_dir, - self.artifacts_dir, - self.os_utils) + action = JavaMavenCopyArtifactsAction(self.scratch_dir, self.artifacts_dir, self.os_utils) with self.assertRaises(ActionFailedError) as raised: action.execute() - self.assertEquals(raised.exception.args[0], "Required target/classes directory was not " - "produced from 'mvn package'") + self.assertEquals( + raised.exception.args[0], "Required target/classes directory was not " "produced from 'mvn package'" + ) diff --git a/tests/unit/workflows/java_maven/test_maven.py b/tests/unit/workflows/java_maven/test_maven.py index cd2181234..f24efad50 100644 --- a/tests/unit/workflows/java_maven/test_maven.py +++ b/tests/unit/workflows/java_maven/test_maven.py @@ -8,7 +8,7 @@ class FakePopen: - def __init__(self, out=b'out', err=b'err', retcode=0): + def __init__(self, out=b"out", err=b"err", retcode=0): self.out = out self.err = err self.returncode = retcode @@ -21,54 +21,56 @@ def wait(self): class TestSubprocessMaven(TestCase): - @patch("aws_lambda_builders.workflows.java_gradle.utils.OSUtils") def setUp(self, MockOSUtils): self.os_utils = MockOSUtils.return_value self.os_utils.exists.side_effect = lambda d: True self.popen = FakePopen() self.os_utils.popen.side_effect = [self.popen] - self.maven_path = '/path/to/mvn' - self.maven_binary = BinaryPath(None, None, 'mvn', binary_path=self.maven_path) - self.source_dir = '/foo/bar/helloworld' - self.module_name = 'helloworld' + self.maven_path = "/path/to/mvn" + self.maven_binary = BinaryPath(None, None, "mvn", binary_path=self.maven_path) + self.source_dir = "/foo/bar/helloworld" + self.module_name = "helloworld" def test_no_os_utils_build_init_throws(self): with self.assertRaises(ValueError) as err_assert: SubprocessMaven(maven_binary=self.maven_binary) - self.assertEquals(err_assert.exception.args[0], 'Must provide OSUtils') + self.assertEquals(err_assert.exception.args[0], "Must provide OSUtils") def test_no_maven_exec_init_throws(self): with self.assertRaises(ValueError) as err_assert: SubprocessMaven(None) - self.assertEquals(err_assert.exception.args[0], 'Must provide Maven BinaryPath') + self.assertEquals(err_assert.exception.args[0], "Must provide Maven BinaryPath") def test_build_project(self): maven = SubprocessMaven(maven_binary=self.maven_binary, os_utils=self.os_utils) maven.build(self.source_dir) self.os_utils.popen.assert_called_with( - [self.maven_path, 'clean', 'install'], - cwd=self.source_dir, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + [self.maven_path, "clean", "install"], cwd=self.source_dir, stderr=subprocess.PIPE, stdout=subprocess.PIPE + ) def test_build_raises_exception_if_retcode_not_0(self): - self.popen = FakePopen(retcode=1, out=b'Some Error Message') + self.popen = FakePopen(retcode=1, out=b"Some Error Message") self.os_utils.popen.side_effect = [self.popen] maven = SubprocessMaven(maven_binary=self.maven_binary, os_utils=self.os_utils) with self.assertRaises(MavenExecutionError) as err: maven.build(self.source_dir) - self.assertEquals(err.exception.args[0], 'Maven Failed: Some Error Message') + self.assertEquals(err.exception.args[0], "Maven Failed: Some Error Message") def test_copy_dependency(self): maven = SubprocessMaven(maven_binary=self.maven_binary, os_utils=self.os_utils) maven.copy_dependency(self.source_dir) self.os_utils.popen.assert_called_with( - [self.maven_path, 'dependency:copy-dependencies', '-DincludeScope=compile'], - cwd=self.source_dir, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + [self.maven_path, "dependency:copy-dependencies", "-DincludeScope=compile"], + cwd=self.source_dir, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) def test_copy_dependency_raises_exception_if_retcode_not_0(self): - self.popen = FakePopen(retcode=1, out=b'Some Error Message') + self.popen = FakePopen(retcode=1, out=b"Some Error Message") self.os_utils.popen.side_effect = [self.popen] maven = SubprocessMaven(maven_binary=self.maven_binary, os_utils=self.os_utils) with self.assertRaises(MavenExecutionError) as err: maven.copy_dependency(self.source_dir) - self.assertEquals(err.exception.args[0], 'Maven Failed: Some Error Message') + self.assertEquals(err.exception.args[0], "Maven Failed: Some Error Message") diff --git a/tests/unit/workflows/java_maven/test_maven_resolver.py b/tests/unit/workflows/java_maven/test_maven_resolver.py index 3e67b7df4..ca78d624a 100644 --- a/tests/unit/workflows/java_maven/test_maven_resolver.py +++ b/tests/unit/workflows/java_maven/test_maven_resolver.py @@ -5,14 +5,13 @@ class TestMavenResolver(TestCase): - @patch("aws_lambda_builders.workflows.java_gradle.utils.OSUtils") def setUp(self, MockOSUtils): self.mock_os_utils = MockOSUtils.return_value self.mock_os_utils.is_windows.side_effect = [False] def test_returns_maven_on_path(self): - maven_path = '/path/to/mvn' + maven_path = "/path/to/mvn" self.mock_os_utils.which.side_effect = lambda executable, executable_search_paths: [maven_path] resolver = MavenResolver(os_utils=self.mock_os_utils) @@ -23,4 +22,4 @@ def test_throws_value_error_if_no_exec_found(self): resolver = MavenResolver(os_utils=self.mock_os_utils) with self.assertRaises(ValueError) as raised: resolver.exec_paths() - self.assertEquals(raised.exception.args[0], 'No Maven executable found!') + self.assertEquals(raised.exception.args[0], "No Maven executable found!") diff --git a/tests/unit/workflows/java_maven/test_maven_validator.py b/tests/unit/workflows/java_maven/test_maven_validator.py index 93dcc8e6c..819e81735 100644 --- a/tests/unit/workflows/java_maven/test_maven_validator.py +++ b/tests/unit/workflows/java_maven/test_maven_validator.py @@ -20,51 +20,40 @@ def returncode(self): class TestMavenBinaryValidator(TestCase): - @patch("aws_lambda_builders.workflows.java_gradle.utils.OSUtils") def setUp(self, MockOSUtils): self.mock_os_utils = MockOSUtils.return_value self.mock_log = Mock() - self.maven_path = '/path/to/maven' - self.runtime = 'java8' + self.maven_path = "/path/to/maven" + self.runtime = "java8" - @parameterized.expand([ - '1.7.0', - '1.8.9', - '11.0.0' - ]) + @parameterized.expand(["1.7.0", "1.8.9", "11.0.0"]) def test_accepts_any_jvm_mv(self, version): - version_string = ('Java version: %s, vendor: Oracle Corporation' % version).encode() + version_string = ("Java version: %s, vendor: Oracle Corporation" % version).encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string)] validator = MavenValidator(self.runtime, os_utils=self.mock_os_utils) self.assertTrue(validator.validate(maven_path=self.maven_path)) self.assertEqual(validator.validated_binary_path, self.maven_path) - @parameterized.expand([ - '12' - ]) + @parameterized.expand(["12"]) def test_accepts_major_version_only_jvm_mv(self, version): - version_string = ('Java version: %s, vendor: Oracle Corporation' % version).encode() + version_string = ("Java version: %s, vendor: Oracle Corporation" % version).encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string)] validator = MavenValidator(self.runtime, os_utils=self.mock_os_utils) self.assertTrue(validator.validate(maven_path=self.maven_path)) self.assertEqual(validator.validated_binary_path, self.maven_path) def test_emits_warning_when_jvm_mv_greater_than_8(self): - version_string = 'Java version: 10.0.1, vendor: Oracle Corporation'.encode() + version_string = "Java version: 10.0.1, vendor: Oracle Corporation".encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string)] validator = MavenValidator(self.runtime, os_utils=self.mock_os_utils, log=self.mock_log) self.assertTrue(validator.validate(maven_path=self.maven_path)) self.assertEqual(validator.validated_binary_path, self.maven_path) - self.mock_log.warning.assert_called_with(MavenValidator.MAJOR_VERSION_WARNING, self.maven_path, '10', '8', '8') + self.mock_log.warning.assert_called_with(MavenValidator.MAJOR_VERSION_WARNING, self.maven_path, "10", "8", "8") - @parameterized.expand([ - '1.6.0', - '1.7.0', - '1.8.9' - ]) + @parameterized.expand(["1.6.0", "1.7.0", "1.8.9"]) def test_does_not_emit_warning_when_jvm_mv_8_or_less(self, version): - version_string = ('Java version: %s, vendor: Oracle Corporation' % version).encode() + version_string = ("Java version: %s, vendor: Oracle Corporation" % version).encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string)] validator = MavenValidator(self.runtime, os_utils=self.mock_os_utils, log=self.mock_log) self.assertTrue(validator.validate(maven_path=self.maven_path)) @@ -72,14 +61,14 @@ def test_does_not_emit_warning_when_jvm_mv_8_or_less(self, version): self.mock_log.warning.assert_not_called() def test_emits_warning_when_maven_excutable_fails(self): - version_string = 'Java version: %s, vendor: Oracle Corporation'.encode() + version_string = "Java version: %s, vendor: Oracle Corporation".encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string, returncode=1)] validator = MavenValidator(self.runtime, os_utils=self.mock_os_utils, log=self.mock_log) validator.validate(maven_path=self.maven_path) self.mock_log.warning.assert_called_with(MavenValidator.VERSION_STRING_WARNING, self.maven_path) def test_emits_warning_when_version_string_not_found(self): - version_string = 'Blah: 9.0.0'.encode() + version_string = "Blah: 9.0.0".encode() self.mock_os_utils.popen.side_effect = [FakePopen(stdout=version_string, returncode=0)] validator = MavenValidator(self.runtime, os_utils=self.mock_os_utils, log=self.mock_log) validator.validate(maven_path=self.maven_path) diff --git a/tests/unit/workflows/java_maven/test_workflow.py b/tests/unit/workflows/java_maven/test_workflow.py index e2cdacc52..7c830b605 100644 --- a/tests/unit/workflows/java_maven/test_workflow.py +++ b/tests/unit/workflows/java_maven/test_workflow.py @@ -1,8 +1,11 @@ from unittest import TestCase from aws_lambda_builders.workflows.java_maven.workflow import JavaMavenWorkflow -from aws_lambda_builders.workflows.java_maven.actions import \ - JavaMavenBuildAction, JavaMavenCopyArtifactsAction, JavaMavenCopyDependencyAction +from aws_lambda_builders.workflows.java_maven.actions import ( + JavaMavenBuildAction, + JavaMavenCopyArtifactsAction, + JavaMavenCopyDependencyAction, +) from aws_lambda_builders.actions import CopySourceAction from aws_lambda_builders.workflows.java_maven.maven_resolver import MavenResolver from aws_lambda_builders.workflows.java_maven.maven_validator import MavenValidator diff --git a/tests/unit/workflows/nodejs_npm/test_actions.py b/tests/unit/workflows/nodejs_npm/test_actions.py index 612ccb189..f456da1e2 100644 --- a/tests/unit/workflows/nodejs_npm/test_actions.py +++ b/tests/unit/workflows/nodejs_npm/test_actions.py @@ -2,34 +2,36 @@ from mock import patch from aws_lambda_builders.actions import ActionFailedError -from aws_lambda_builders.workflows.nodejs_npm.actions import \ - NodejsNpmPackAction, NodejsNpmInstallAction, NodejsNpmrcCopyAction, NodejsNpmrcCleanUpAction +from aws_lambda_builders.workflows.nodejs_npm.actions import ( + NodejsNpmPackAction, + NodejsNpmInstallAction, + NodejsNpmrcCopyAction, + NodejsNpmrcCleanUpAction, +) from aws_lambda_builders.workflows.nodejs_npm.npm import NpmExecutionError class TestNodejsNpmPackAction(TestCase): - @patch("aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils") @patch("aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessNpm") def test_tars_and_unpacks_npm_project(self, OSUtilMock, SubprocessNpmMock): osutils = OSUtilMock.return_value subprocess_npm = SubprocessNpmMock.return_value - action = NodejsNpmPackAction("artifacts", "scratch_dir", - "manifest", - osutils=osutils, - subprocess_npm=subprocess_npm) + action = NodejsNpmPackAction( + "artifacts", "scratch_dir", "manifest", osutils=osutils, subprocess_npm=subprocess_npm + ) osutils.dirname.side_effect = lambda value: "/dir:{}".format(value) osutils.abspath.side_effect = lambda value: "/abs:{}".format(value) osutils.joinpath.side_effect = lambda a, b: "{}/{}".format(a, b) - subprocess_npm.run.return_value = 'package.tar' + subprocess_npm.run.return_value = "package.tar" action.execute() - subprocess_npm.run.assert_called_with(['pack', '-q', 'file:/abs:/dir:manifest'], cwd='scratch_dir') - osutils.extract_tarfile.assert_called_with('scratch_dir/package.tar', 'artifacts') + subprocess_npm.run.assert_called_with(["pack", "-q", "file:/abs:/dir:manifest"], cwd="scratch_dir") + osutils.extract_tarfile.assert_called_with("scratch_dir/package.tar", "artifacts") @patch("aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils") @patch("aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessNpm") @@ -40,9 +42,9 @@ def test_raises_action_failed_when_npm_fails(self, OSUtilMock, SubprocessNpmMock builder_instance = SubprocessNpmMock.return_value builder_instance.run.side_effect = NpmExecutionError(message="boom!") - action = NodejsNpmPackAction("artifacts", "scratch_dir", - "manifest", - osutils=osutils, subprocess_npm=subprocess_npm) + action = NodejsNpmPackAction( + "artifacts", "scratch_dir", "manifest", osutils=osutils, subprocess_npm=subprocess_npm + ) with self.assertRaises(ActionFailedError) as raised: action.execute() @@ -51,19 +53,17 @@ def test_raises_action_failed_when_npm_fails(self, OSUtilMock, SubprocessNpmMock class TestNodejsNpmInstallAction(TestCase): - @patch("aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessNpm") def test_tars_and_unpacks_npm_project(self, SubprocessNpmMock): subprocess_npm = SubprocessNpmMock.return_value - action = NodejsNpmInstallAction("artifacts", - subprocess_npm=subprocess_npm) + action = NodejsNpmInstallAction("artifacts", subprocess_npm=subprocess_npm) action.execute() - expected_args = ['install', '-q', '--no-audit', '--no-save', '--production'] + expected_args = ["install", "-q", "--no-audit", "--no-save", "--production"] - subprocess_npm.run.assert_called_with(expected_args, cwd='artifacts') + subprocess_npm.run.assert_called_with(expected_args, cwd="artifacts") @patch("aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessNpm") def test_raises_action_failed_when_npm_fails(self, SubprocessNpmMock): @@ -72,8 +72,7 @@ def test_raises_action_failed_when_npm_fails(self, SubprocessNpmMock): builder_instance = SubprocessNpmMock.return_value builder_instance.run.side_effect = NpmExecutionError(message="boom!") - action = NodejsNpmInstallAction("artifacts", - subprocess_npm=subprocess_npm) + action = NodejsNpmInstallAction("artifacts", subprocess_npm=subprocess_npm) with self.assertRaises(ActionFailedError) as raised: action.execute() @@ -82,15 +81,12 @@ def test_raises_action_failed_when_npm_fails(self, SubprocessNpmMock): class TestNodejsNpmrcCopyAction(TestCase): - @patch("aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils") def test_copies_npmrc_into_a_project(self, OSUtilMock): osutils = OSUtilMock.return_value osutils.joinpath.side_effect = lambda a, b: "{}/{}".format(a, b) - action = NodejsNpmrcCopyAction("artifacts", - "source", - osutils=osutils) + action = NodejsNpmrcCopyAction("artifacts", "source", osutils=osutils) osutils.file_exists.side_effect = [True] action.execute() @@ -102,9 +98,7 @@ def test_skips_copying_npmrc_into_a_project_if_npmrc_doesnt_exist(self, OSUtilMo osutils = OSUtilMock.return_value osutils.joinpath.side_effect = lambda a, b: "{}/{}".format(a, b) - action = NodejsNpmrcCopyAction("artifacts", - "source", - osutils=osutils) + action = NodejsNpmrcCopyAction("artifacts", "source", osutils=osutils) osutils.file_exists.side_effect = [False] action.execute() @@ -118,24 +112,19 @@ def test_raises_action_failed_when_copying_fails(self, OSUtilMock): osutils.copy_file.side_effect = OSError() - action = NodejsNpmrcCopyAction("artifacts", - "source", - osutils=osutils) + action = NodejsNpmrcCopyAction("artifacts", "source", osutils=osutils) with self.assertRaises(ActionFailedError): action.execute() class TestNodejsNpmrcCleanUpAction(TestCase): - @patch("aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils") def test_removes_npmrc_if_npmrc_exists(self, OSUtilMock): osutils = OSUtilMock.return_value osutils.joinpath.side_effect = lambda a, b: "{}/{}".format(a, b) - action = NodejsNpmrcCleanUpAction( - "artifacts", - osutils=osutils) + action = NodejsNpmrcCleanUpAction("artifacts", osutils=osutils) osutils.file_exists.side_effect = [True] action.execute() @@ -146,9 +135,7 @@ def test_skips_npmrc_removal_if_npmrc_doesnt_exist(self, OSUtilMock): osutils = OSUtilMock.return_value osutils.joinpath.side_effect = lambda a, b: "{}/{}".format(a, b) - action = NodejsNpmrcCleanUpAction( - "artifacts", - osutils=osutils) + action = NodejsNpmrcCleanUpAction("artifacts", osutils=osutils) osutils.file_exists.side_effect = [False] action.execute() diff --git a/tests/unit/workflows/nodejs_npm/test_npm.py b/tests/unit/workflows/nodejs_npm/test_npm.py index 2dc796328..a907e716f 100644 --- a/tests/unit/workflows/nodejs_npm/test_npm.py +++ b/tests/unit/workflows/nodejs_npm/test_npm.py @@ -5,7 +5,7 @@ class FakePopen: - def __init__(self, out=b'out', err=b'err', retcode=0): + def __init__(self, out=b"out", err=b"err", retcode=0): self.out = out self.err = err self.returncode = retcode @@ -15,11 +15,10 @@ def communicate(self): class TestSubprocessNpm(TestCase): - @patch("aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils") def setUp(self, OSUtilMock): self.osutils = OSUtilMock.return_value - self.osutils.pipe = 'PIPE' + self.osutils.pipe = "PIPE" self.popen = FakePopen() self.osutils.popen.side_effect = [self.popen] self.under_test = SubprocessNpm(self.osutils, npm_exe="/a/b/c/npm.exe") @@ -29,49 +28,50 @@ def test_run_executes_npm_on_nixes(self): self.under_test = SubprocessNpm(self.osutils) - self.under_test.run(['pack', '-q']) + self.under_test.run(["pack", "-q"]) - self.osutils.popen.assert_called_with(['npm', 'pack', '-q'], cwd=None, stderr='PIPE', stdout='PIPE') + self.osutils.popen.assert_called_with(["npm", "pack", "-q"], cwd=None, stderr="PIPE", stdout="PIPE") def test_run_executes_npm_cmd_on_windows(self): self.osutils.is_windows.side_effect = [True] self.under_test = SubprocessNpm(self.osutils) - self.under_test.run(['pack', '-q']) + self.under_test.run(["pack", "-q"]) - self.osutils.popen.assert_called_with(['npm.cmd', 'pack', '-q'], cwd=None, stderr='PIPE', stdout='PIPE') + self.osutils.popen.assert_called_with(["npm.cmd", "pack", "-q"], cwd=None, stderr="PIPE", stdout="PIPE") def test_uses_custom_npm_path_if_supplied(self): - self.under_test.run(['pack', '-q']) + self.under_test.run(["pack", "-q"]) - self.osutils.popen.assert_called_with(['/a/b/c/npm.exe', 'pack', '-q'], cwd=None, stderr='PIPE', stdout='PIPE') + self.osutils.popen.assert_called_with(["/a/b/c/npm.exe", "pack", "-q"], cwd=None, stderr="PIPE", stdout="PIPE") def test_uses_cwd_if_supplied(self): - self.under_test.run(['pack', '-q'], cwd='/a/cwd') + self.under_test.run(["pack", "-q"], cwd="/a/cwd") - self.osutils.popen.assert_called_with(['/a/b/c/npm.exe', 'pack', '-q'], - cwd='/a/cwd', stderr='PIPE', stdout='PIPE') + self.osutils.popen.assert_called_with( + ["/a/b/c/npm.exe", "pack", "-q"], cwd="/a/cwd", stderr="PIPE", stdout="PIPE" + ) def test_returns_popen_out_decoded_if_retcode_is_0(self): - self.popen.out = b'some encoded text\n\n' + self.popen.out = b"some encoded text\n\n" - result = self.under_test.run(['pack']) + result = self.under_test.run(["pack"]) - self.assertEqual(result, 'some encoded text') + self.assertEqual(result, "some encoded text") def test_raises_NpmExecutionError_with_err_text_if_retcode_is_not_0(self): self.popen.returncode = 1 - self.popen.err = b'some error text\n\n' + self.popen.err = b"some error text\n\n" with self.assertRaises(NpmExecutionError) as raised: - self.under_test.run(['pack']) + self.under_test.run(["pack"]) self.assertEqual(raised.exception.args[0], "NPM Failed: some error text") def test_raises_ValueError_if_args_not_a_list(self): with self.assertRaises(ValueError) as raised: - self.under_test.run(('pack')) + self.under_test.run(("pack")) self.assertEqual(raised.exception.args[0], "args must be a list") diff --git a/tests/unit/workflows/nodejs_npm/test_workflow.py b/tests/unit/workflows/nodejs_npm/test_workflow.py index c2fe05be3..3dac6d791 100644 --- a/tests/unit/workflows/nodejs_npm/test_workflow.py +++ b/tests/unit/workflows/nodejs_npm/test_workflow.py @@ -2,8 +2,12 @@ from aws_lambda_builders.actions import CopySourceAction from aws_lambda_builders.workflows.nodejs_npm.workflow import NodejsNpmWorkflow -from aws_lambda_builders.workflows.nodejs_npm.actions import \ - NodejsNpmPackAction, NodejsNpmInstallAction, NodejsNpmrcCopyAction, NodejsNpmrcCleanUpAction +from aws_lambda_builders.workflows.nodejs_npm.actions import ( + NodejsNpmPackAction, + NodejsNpmInstallAction, + NodejsNpmrcCopyAction, + NodejsNpmrcCleanUpAction, +) class TestNodejsNpmWorkflow(TestCase): diff --git a/tests/unit/workflows/python_pip/test_actions.py b/tests/unit/workflows/python_pip/test_actions.py index 0f9be9484..e8c98f7a1 100644 --- a/tests/unit/workflows/python_pip/test_actions.py +++ b/tests/unit/workflows/python_pip/test_actions.py @@ -11,34 +11,35 @@ class TestPythonPipBuildAction(TestCase): - @patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipDependencyBuilder") def test_action_must_call_builder(self, PythonPipDependencyBuilderMock): builder_instance = PythonPipDependencyBuilderMock.return_value - action = PythonPipBuildAction("artifacts", "scratch_dir", - "manifest", "runtime", - { - "python": BinaryPath(resolver=Mock(), validator=Mock(), - binary="python", binary_path=sys.executable) - }) + action = PythonPipBuildAction( + "artifacts", + "scratch_dir", + "manifest", + "runtime", + {"python": BinaryPath(resolver=Mock(), validator=Mock(), binary="python", binary_path=sys.executable)}, + ) action.execute() - builder_instance.build_dependencies.assert_called_with(artifacts_dir_path="artifacts", - scratch_dir_path="scratch_dir", - requirements_path="manifest") + builder_instance.build_dependencies.assert_called_with( + artifacts_dir_path="artifacts", scratch_dir_path="scratch_dir", requirements_path="manifest" + ) @patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipDependencyBuilder") def test_must_raise_exception_on_failure(self, PythonPipDependencyBuilderMock): builder_instance = PythonPipDependencyBuilderMock.return_value builder_instance.build_dependencies.side_effect = PackagerError() - action = PythonPipBuildAction("artifacts", "scratch_dir", - "manifest", "runtime", - { - "python": BinaryPath(resolver=Mock(), validator=Mock(), - binary="python", binary_path=sys.executable) - }) + action = PythonPipBuildAction( + "artifacts", + "scratch_dir", + "manifest", + "runtime", + {"python": BinaryPath(resolver=Mock(), validator=Mock(), binary="python", binary_path=sys.executable)}, + ) with self.assertRaises(ActionFailedError): action.execute() diff --git a/tests/unit/workflows/python_pip/test_packager.py b/tests/unit/workflows/python_pip/test_packager.py index 3fa72ff62..39714412b 100644 --- a/tests/unit/workflows/python_pip/test_packager.py +++ b/tests/unit/workflows/python_pip/test_packager.py @@ -14,15 +14,15 @@ from aws_lambda_builders.workflows.python_pip.packager import PipRunner from aws_lambda_builders.workflows.python_pip.packager import SubprocessPip from aws_lambda_builders.workflows.python_pip.packager import get_lambda_abi -from aws_lambda_builders.workflows.python_pip.packager \ - import InvalidSourceDistributionNameError +from aws_lambda_builders.workflows.python_pip.packager import InvalidSourceDistributionNameError from aws_lambda_builders.workflows.python_pip.packager import NoSuchPackageError from aws_lambda_builders.workflows.python_pip.packager import PackageDownloadError from aws_lambda_builders.workflows.python_pip import packager -print(packager) -FakePipCall = namedtuple('FakePipEntry', ['args', 'env_vars', 'shim']) +print (packager) + +FakePipCall = namedtuple("FakePipEntry", ["args", "env_vars", "shim"]) class FakePip(object): @@ -35,7 +35,7 @@ def main(self, args, env_vars=None, shim=None): if self._returns: return self._returns.pop(0) # Return an rc of 0 and an empty stderr and stdout - return 0, b'', b'' + return 0, b"", b"" def add_return(self, return_pair): self._returns.append(return_pair) @@ -49,10 +49,9 @@ def calls(self): def pip_factory(): def create_pip_runner(osutils=None): pip = FakePip() - pip_runner = PipRunner(python_exe=sys.executable, - pip=pip, - osutils=osutils) + pip_runner = PipRunner(python_exe=sys.executable, pip=pip, osutils=osutils) return pip, pip_runner + return create_pip_runner @@ -108,113 +107,106 @@ def test_can_call_dependency_builder(self, osutils): mock_dep_builder = mock.Mock(spec=DependencyBuilder) osutils_mock = mock.Mock(spec=osutils) builder = PythonPipDependencyBuilder( - osutils=osutils_mock, - dependency_builder=mock_dep_builder, - runtime="runtime" - ) - builder.build_dependencies( - 'artifacts/path/', 'scratch_dir/path/', - 'path/to/requirements.txt' + osutils=osutils_mock, dependency_builder=mock_dep_builder, runtime="runtime" ) + builder.build_dependencies("artifacts/path/", "scratch_dir/path/", "path/to/requirements.txt") mock_dep_builder.build_site_packages.assert_called_once_with( - 'path/to/requirements.txt', 'artifacts/path/', 'scratch_dir/path/') - osutils_mock.file_exists.assert_called_once_with( - 'path/to/requirements.txt') + "path/to/requirements.txt", "artifacts/path/", "scratch_dir/path/" + ) + osutils_mock.file_exists.assert_called_once_with("path/to/requirements.txt") class TestPackage(object): def test_can_create_package_with_custom_osutils(self, osutils): - pkg = Package('', 'foobar-1.0-py3-none-any.whl', osutils) + pkg = Package("", "foobar-1.0-py3-none-any.whl", osutils) assert pkg._osutils == osutils def test_wheel_package(self): - filename = 'foobar-1.0-py3-none-any.whl' - pkg = Package('', filename) - assert pkg.dist_type == 'wheel' + filename = "foobar-1.0-py3-none-any.whl" + pkg = Package("", filename) + assert pkg.dist_type == "wheel" assert pkg.filename == filename - assert pkg.identifier == 'foobar==1.0' - assert str(pkg) == 'foobar==1.0(wheel)' + assert pkg.identifier == "foobar==1.0" + assert str(pkg) == "foobar==1.0(wheel)" def test_invalid_package(self): with pytest.raises(InvalidSourceDistributionNameError): - Package('', 'foobar.jpg') + Package("", "foobar.jpg") def test_diff_pkg_sdist_and_whl_do_not_collide(self): pkgs = set() - pkgs.add(Package('', 'foobar-1.0-py3-none-any.whl')) - pkgs.add(Package('', 'badbaz-1.0-py3-none-any.whl')) + pkgs.add(Package("", "foobar-1.0-py3-none-any.whl")) + pkgs.add(Package("", "badbaz-1.0-py3-none-any.whl")) assert len(pkgs) == 2 def test_same_pkg_is_eq(self): - pkg = Package('', 'foobar-1.0-py3-none-any.whl') + pkg = Package("", "foobar-1.0-py3-none-any.whl") assert pkg == pkg def test_pkg_is_eq_to_similar_pkg(self): - pure_pkg = Package('', 'foobar-1.0-py3-none-any.whl') - plat_pkg = Package('', 'foobar-1.0-py3-py36m-manylinux1_x86_64.whl') + pure_pkg = Package("", "foobar-1.0-py3-none-any.whl") + plat_pkg = Package("", "foobar-1.0-py3-py36m-manylinux1_x86_64.whl") assert pure_pkg == plat_pkg def test_pkg_is_not_equal_to_different_type(self): - pkg = Package('', 'foobar-1.0-py3-none-any.whl') + pkg = Package("", "foobar-1.0-py3-none-any.whl") non_package_type = 1 assert not (pkg == non_package_type) def test_pkg_repr(self): - pkg = Package('', 'foobar-1.0-py3-none-any.whl') - assert repr(pkg) == 'foobar==1.0(wheel)' + pkg = Package("", "foobar-1.0-py3-none-any.whl") + assert repr(pkg) == "foobar==1.0(wheel)" def test_wheel_data_dir(self): - pkg = Package('', 'foobar-2.0-py3-none-any.whl') - assert pkg.data_dir == 'foobar-2.0.data' + pkg = Package("", "foobar-2.0-py3-none-any.whl") + assert pkg.data_dir == "foobar-2.0.data" def test_can_read_packages_with_underscore_in_name(self): - pkg = Package('', 'foo_bar-2.0-py3-none-any.whl') - assert pkg.identifier == 'foo-bar==2.0' + pkg = Package("", "foo_bar-2.0-py3-none-any.whl") + assert pkg.identifier == "foo-bar==2.0" def test_can_read_packages_with_period_in_name(self): - pkg = Package('', 'foo.bar-2.0-py3-none-any.whl') - assert pkg.identifier == 'foo-bar==2.0' + pkg = Package("", "foo.bar-2.0-py3-none-any.whl") + assert pkg.identifier == "foo-bar==2.0" class TestPipRunner(object): def test_does_propagate_env_vars(self, pip_factory): - osutils = CustomEnv({'foo': 'bar'}) + osutils = CustomEnv({"foo": "bar"}) pip, runner = pip_factory(osutils) - wheel = 'foobar-1.2-py3-none-any.whl' - directory = 'directory' + wheel = "foobar-1.2-py3-none-any.whl" + directory = "directory" runner.build_wheel(wheel, directory) call = pip.calls[0] - assert 'foo' in call.env_vars - assert call.env_vars['foo'] == 'bar' + assert "foo" in call.env_vars + assert call.env_vars["foo"] == "bar" def test_build_wheel(self, pip_factory): # Test that `pip wheel` is called with the correct params pip, runner = pip_factory() - wheel = 'foobar-1.0-py3-none-any.whl' - directory = 'directory' + wheel = "foobar-1.0-py3-none-any.whl" + directory = "directory" runner.build_wheel(wheel, directory) assert len(pip.calls) == 1 call = pip.calls[0] - assert call.args == ['wheel', '--no-deps', '--wheel-dir', - directory, wheel] + assert call.args == ["wheel", "--no-deps", "--wheel-dir", directory, wheel] for compile_env_var in pip_no_compile_c_env_vars: assert compile_env_var not in call.env_vars - assert call.shim == '' + assert call.shim == "" def test_build_wheel_without_c_extensions(self, pip_factory): # Test that `pip wheel` is called with the correct params when we # call it with compile_c=False. These will differ by platform. pip, runner = pip_factory() - wheel = 'foobar-1.0-py3-none-any.whl' - directory = 'directory' + wheel = "foobar-1.0-py3-none-any.whl" + directory = "directory" runner.build_wheel(wheel, directory, compile_c=False) assert len(pip.calls) == 1 call = pip.calls[0] - assert call.args == ['wheel', '--no-deps', '--wheel-dir', - directory, wheel] + assert call.args == ["wheel", "--no-deps", "--wheel-dir", directory, wheel] for compile_env_var in pip_no_compile_c_env_vars: assert compile_env_var in call.env_vars assert call.shim == pip_no_compile_c_shim @@ -223,12 +215,11 @@ def test_download_all_deps(self, pip_factory): # Make sure that `pip download` is called with the correct arguments # for getting all sdists. pip, runner = pip_factory() - runner.download_all_dependencies('requirements.txt', 'directory') + runner.download_all_dependencies("requirements.txt", "directory") assert len(pip.calls) == 1 call = pip.calls[0] - assert call.args == ['download', '-r', - 'requirements.txt', '--dest', 'directory'] + assert call.args == ["download", "-r", "requirements.txt", "--dest", "directory"] assert call.env_vars is None assert call.shim is None @@ -236,12 +227,21 @@ def test_download_wheels(self, pip_factory): # Make sure that `pip download` is called with the correct arguments # for getting lambda compatible wheels. pip, runner = pip_factory() - packages = ['foo', 'bar', 'baz'] - runner.download_manylinux_wheels(packages, 'directory', "abi") - expected_prefix = ['download', '--only-binary=:all:', '--no-deps', - '--platform', 'manylinux1_x86_64', - '--implementation', 'cp', '--abi', "abi", - '--dest', 'directory'] + packages = ["foo", "bar", "baz"] + runner.download_manylinux_wheels(packages, "directory", "abi") + expected_prefix = [ + "download", + "--only-binary=:all:", + "--no-deps", + "--platform", + "manylinux1_x86_64", + "--implementation", + "cp", + "--abi", + "abi", + "--dest", + "directory", + ] for i, package in enumerate(packages): assert pip.calls[i].args == expected_prefix + [package] assert pip.calls[i].env_vars is None @@ -249,82 +249,75 @@ def test_download_wheels(self, pip_factory): def test_download_wheels_no_wheels(self, pip_factory): pip, runner = pip_factory() - runner.download_manylinux_wheels([], 'directory', "abi") + runner.download_manylinux_wheels([], "directory", "abi") assert len(pip.calls) == 0 def test_does_find_local_directory(self, pip_factory): pip, runner = pip_factory() - pip.add_return((0, - (b"Processing ../local-dir\n" - b" Link is a directory," - b" ignoring download_dir"), - b'')) - runner.download_all_dependencies('requirements.txt', 'directory') + pip.add_return((0, (b"Processing ../local-dir\n" b" Link is a directory," b" ignoring download_dir"), b"")) + runner.download_all_dependencies("requirements.txt", "directory") assert len(pip.calls) == 2 - assert pip.calls[1].args == ['wheel', '--no-deps', '--wheel-dir', - 'directory', '../local-dir'] + assert pip.calls[1].args == ["wheel", "--no-deps", "--wheel-dir", "directory", "../local-dir"] def test_does_find_multiple_local_directories(self, pip_factory): pip, runner = pip_factory() - pip.add_return((0, - (b"Processing ../local-dir-1\n" - b" Link is a directory," - b" ignoring download_dir" - b"\nsome pip output...\n" - b"Processing ../local-dir-2\n" - b" Link is a directory," - b" ignoring download_dir"), - b'')) - runner.download_all_dependencies('requirements.txt', 'directory') + pip.add_return( + ( + 0, + ( + b"Processing ../local-dir-1\n" + b" Link is a directory," + b" ignoring download_dir" + b"\nsome pip output...\n" + b"Processing ../local-dir-2\n" + b" Link is a directory," + b" ignoring download_dir" + ), + b"", + ) + ) + runner.download_all_dependencies("requirements.txt", "directory") assert len(pip.calls) == 3 - assert pip.calls[1].args == ['wheel', '--no-deps', '--wheel-dir', - 'directory', '../local-dir-1'] - assert pip.calls[2].args == ['wheel', '--no-deps', '--wheel-dir', - 'directory', '../local-dir-2'] + assert pip.calls[1].args == ["wheel", "--no-deps", "--wheel-dir", "directory", "../local-dir-1"] + assert pip.calls[2].args == ["wheel", "--no-deps", "--wheel-dir", "directory", "../local-dir-2"] def test_raise_no_such_package_error(self, pip_factory): pip, runner = pip_factory() - pip.add_return((1, b'', - (b'Could not find a version that satisfies the ' - b'requirement BadPackageName '))) + pip.add_return((1, b"", (b"Could not find a version that satisfies the " b"requirement BadPackageName "))) with pytest.raises(NoSuchPackageError) as einfo: - runner.download_all_dependencies('requirements.txt', 'directory') - assert str(einfo.value) == ('Could not satisfy the requirement: ' - 'BadPackageName') + runner.download_all_dependencies("requirements.txt", "directory") + assert str(einfo.value) == ("Could not satisfy the requirement: " "BadPackageName") def test_raise_other_unknown_error_during_downloads(self, pip_factory): pip, runner = pip_factory() - pip.add_return((1, b'', b'SomeNetworkingError: Details here.')) + pip.add_return((1, b"", b"SomeNetworkingError: Details here.")) with pytest.raises(PackageDownloadError) as einfo: - runner.download_all_dependencies('requirements.txt', 'directory') - assert str(einfo.value) == 'SomeNetworkingError: Details here.' + runner.download_all_dependencies("requirements.txt", "directory") + assert str(einfo.value) == "SomeNetworkingError: Details here." def test_inject_unknown_error_if_no_stderr(self, pip_factory): pip, runner = pip_factory() pip.add_return((1, None, None)) with pytest.raises(PackageDownloadError) as einfo: - runner.download_all_dependencies('requirements.txt', 'directory') - assert str(einfo.value) == 'Unknown error' + runner.download_all_dependencies("requirements.txt", "directory") + assert str(einfo.value) == "Unknown error" class TestSubprocessPip(TestCase): def test_does_use_custom_pip_import_string(self): - fake_osutils = FakePopenOSUtils([FakePopen(0, '', '')]) - expected_import_statement = 'foobarbaz' - pip = SubprocessPip(osutils=fake_osutils, - import_string=expected_import_statement, - python_exe=sys.executable) - pip.main(['--version']) + fake_osutils = FakePopenOSUtils([FakePopen(0, "", "")]) + expected_import_statement = "foobarbaz" + pip = SubprocessPip(osutils=fake_osutils, import_string=expected_import_statement, python_exe=sys.executable) + pip.main(["--version"]) pip_execution_string = fake_osutils.popens[0][0][0][2] - import_statement = pip_execution_string.split(';')[1].strip() + import_statement = pip_execution_string.split(";")[1].strip() assert import_statement == expected_import_statement def test_check_pip_runner_string_pip(self): - fake_osutils = FakePopenOSUtils([FakePopen(0, '', '')]) - pip = SubprocessPip(osutils=fake_osutils, - python_exe=sys.executable) - pip.main(['--version']) + fake_osutils = FakePopenOSUtils([FakePopen(0, "", "")]) + pip = SubprocessPip(osutils=fake_osutils, python_exe=sys.executable) + pip.main(["--version"]) pip_runner_string = fake_osutils.popens[0][0][0][2].split(";")[-1:][0] self.assertIn("main", pip_runner_string) diff --git a/tests/unit/workflows/python_pip/test_validator.py b/tests/unit/workflows/python_pip/test_validator.py index bdd4f908e..94682b0b8 100644 --- a/tests/unit/workflows/python_pip/test_validator.py +++ b/tests/unit/workflows/python_pip/test_validator.py @@ -8,7 +8,6 @@ class MockSubProcess(object): - def __init__(self, returncode): self.returncode = returncode @@ -17,40 +16,33 @@ def communicate(self): class TestPythonRuntimeValidator(TestCase): - def setUp(self): - self.validator = PythonRuntimeValidator(runtime='python3.7') - - @parameterized.expand([ - "python2.7", - "python3.6", - "python3.7", - "python3.8", - ]) + self.validator = PythonRuntimeValidator(runtime="python3.7") + + @parameterized.expand(["python2.7", "python3.6", "python3.7", "python3.8"]) def test_supported_runtimes(self, runtime): validator = PythonRuntimeValidator(runtime=runtime) self.assertTrue(validator.has_runtime()) def test_runtime_validate_unsupported_language_fail_open(self): - validator = PythonRuntimeValidator(runtime='python2.6') - validator.validate(runtime_path='/usr/bin/python2.6') + validator = PythonRuntimeValidator(runtime="python2.6") + validator.validate(runtime_path="/usr/bin/python2.6") def test_runtime_validate_supported_version_runtime(self): - with mock.patch('subprocess.Popen') as mock_subprocess: + with mock.patch("subprocess.Popen") as mock_subprocess: mock_subprocess.return_value = MockSubProcess(0) - self.validator.validate(runtime_path='/usr/bin/python3.7') + self.validator.validate(runtime_path="/usr/bin/python3.7") self.assertTrue(mock_subprocess.call_count, 1) def test_runtime_validate_mismatch_version_runtime(self): - with mock.patch('subprocess.Popen') as mock_subprocess: + with mock.patch("subprocess.Popen") as mock_subprocess: mock_subprocess.return_value = MockSubProcess(1) with self.assertRaises(MisMatchRuntimeError): - self.validator.validate(runtime_path='/usr/bin/python3.6') + self.validator.validate(runtime_path="/usr/bin/python3.6") self.assertTrue(mock_subprocess.call_count, 1) def test_python_command(self): - cmd = self.validator._validate_python_cmd(runtime_path='/usr/bin/python3.7') - version_strings = ["sys.version_info.major == 3", - "sys.version_info.minor == 7"] + cmd = self.validator._validate_python_cmd(runtime_path="/usr/bin/python3.7") + version_strings = ["sys.version_info.major == 3", "sys.version_info.minor == 7"] for version_string in version_strings: self.assertTrue(all([part for part in cmd if version_string in part])) diff --git a/tests/unit/workflows/python_pip/test_workflow.py b/tests/unit/workflows/python_pip/test_workflow.py index aec99e28b..b4adbc925 100644 --- a/tests/unit/workflows/python_pip/test_workflow.py +++ b/tests/unit/workflows/python_pip/test_workflow.py @@ -6,7 +6,6 @@ class TestPythonPipWorkflow(TestCase): - def setUp(self): self.workflow = PythonPipWorkflow("source", "artifacts", "scratch_dir", "manifest", runtime="python3.7") diff --git a/tests/unit/workflows/ruby_bundler/test_actions.py b/tests/unit/workflows/ruby_bundler/test_actions.py index 679d70477..c25392d18 100644 --- a/tests/unit/workflows/ruby_bundler/test_actions.py +++ b/tests/unit/workflows/ruby_bundler/test_actions.py @@ -10,18 +10,16 @@ class TestRubyBundlerInstallAction(TestCase): @patch("aws_lambda_builders.workflows.ruby_bundler.bundler.SubprocessBundler") def test_runs_bundle_install(self, SubprocessBundlerMock): subprocess_bundler = SubprocessBundlerMock.return_value - action = RubyBundlerInstallAction("source_dir", - subprocess_bundler=subprocess_bundler) + action = RubyBundlerInstallAction("source_dir", subprocess_bundler=subprocess_bundler) action.execute() - subprocess_bundler.run.assert_called_with(['install', '--without', 'development', 'test'], cwd="source_dir") + subprocess_bundler.run.assert_called_with(["install", "--without", "development", "test"], cwd="source_dir") @patch("aws_lambda_builders.workflows.ruby_bundler.bundler.SubprocessBundler") def test_raises_action_failed_on_failure(self, SubprocessBundlerMock): subprocess_bundler = SubprocessBundlerMock.return_value builder_instance = SubprocessBundlerMock.return_value builder_instance.run.side_effect = BundlerExecutionError(message="Fail") - action = RubyBundlerInstallAction("source_dir", - subprocess_bundler=subprocess_bundler) + action = RubyBundlerInstallAction("source_dir", subprocess_bundler=subprocess_bundler) with self.assertRaises(ActionFailedError) as raised: action.execute() self.assertEqual(raised.exception.args[0], "Bundler Failed: Fail") @@ -31,20 +29,18 @@ class TestRubyBundlerVendorAction(TestCase): @patch("aws_lambda_builders.workflows.ruby_bundler.bundler.SubprocessBundler") def test_runs_bundle_install_deployment(self, SubprocessBundlerMock): subprocess_bundler = SubprocessBundlerMock.return_value - action = RubyBundlerVendorAction("source_dir", - subprocess_bundler=subprocess_bundler) + action = RubyBundlerVendorAction("source_dir", subprocess_bundler=subprocess_bundler) action.execute() - subprocess_bundler.run.assert_called_with([ - 'install', '--deployment', '--without', 'development', 'test' - ], cwd="source_dir") + subprocess_bundler.run.assert_called_with( + ["install", "--deployment", "--without", "development", "test"], cwd="source_dir" + ) @patch("aws_lambda_builders.workflows.ruby_bundler.bundler.SubprocessBundler") def test_raises_action_failed_on_failure(self, SubprocessBundlerMock): subprocess_bundler = SubprocessBundlerMock.return_value builder_instance = SubprocessBundlerMock.return_value builder_instance.run.side_effect = BundlerExecutionError(message="Fail") - action = RubyBundlerVendorAction("source_dir", - subprocess_bundler=subprocess_bundler) + action = RubyBundlerVendorAction("source_dir", subprocess_bundler=subprocess_bundler) with self.assertRaises(ActionFailedError) as raised: action.execute() self.assertEqual(raised.exception.args[0], "Bundler Failed: Fail") diff --git a/tests/unit/workflows/ruby_bundler/test_bundler.py b/tests/unit/workflows/ruby_bundler/test_bundler.py index d812f20cf..7a83af11a 100644 --- a/tests/unit/workflows/ruby_bundler/test_bundler.py +++ b/tests/unit/workflows/ruby_bundler/test_bundler.py @@ -5,7 +5,7 @@ class FakePopen: - def __init__(self, out=b'out', err=b'err', retcode=0): + def __init__(self, out=b"out", err=b"err", retcode=0): self.out = out self.err = err self.returncode = retcode @@ -15,11 +15,10 @@ def communicate(self): class TestSubprocessBundler(TestCase): - @patch("aws_lambda_builders.workflows.ruby_bundler.utils.OSUtils") def setUp(self, OSUtilMock): self.osutils = OSUtilMock.return_value - self.osutils.pipe = 'PIPE' + self.osutils.pipe = "PIPE" self.popen = FakePopen() self.osutils.popen.side_effect = [self.popen] self.under_test = SubprocessBundler(self.osutils, bundler_exe="/a/b/c/bundle") @@ -27,45 +26,46 @@ def setUp(self, OSUtilMock): def test_run_executes_bundler_on_nixes(self): self.osutils.is_windows.side_effect = [False] self.under_test = SubprocessBundler(self.osutils) - self.under_test.run(['install', '--without', 'development', 'test']) - self.osutils.popen.assert_called_with([ - 'bundle', 'install', '--without', 'development', 'test' - ], cwd=None, stderr='PIPE', stdout='PIPE') + self.under_test.run(["install", "--without", "development", "test"]) + self.osutils.popen.assert_called_with( + ["bundle", "install", "--without", "development", "test"], cwd=None, stderr="PIPE", stdout="PIPE" + ) def test_run_executes_bundler_on_windows(self): self.osutils.is_windows.side_effect = [True] self.under_test = SubprocessBundler(self.osutils) - self.under_test.run(['install', '--without', 'development', 'test']) - self.osutils.popen.assert_called_with([ - 'bundler.bat', 'install', '--without', 'development', 'test' - ], cwd=None, stderr='PIPE', stdout='PIPE') + self.under_test.run(["install", "--without", "development", "test"]) + self.osutils.popen.assert_called_with( + ["bundler.bat", "install", "--without", "development", "test"], cwd=None, stderr="PIPE", stdout="PIPE" + ) def test_uses_custom_bundler_path_if_supplied(self): - self.under_test.run(['install', '--without', 'development', 'test']) - self.osutils.popen.assert_called_with([ - '/a/b/c/bundle', 'install', '--without', 'development', 'test' - ], cwd=None, stderr='PIPE', stdout='PIPE') + self.under_test.run(["install", "--without", "development", "test"]) + self.osutils.popen.assert_called_with( + ["/a/b/c/bundle", "install", "--without", "development", "test"], cwd=None, stderr="PIPE", stdout="PIPE" + ) def test_uses_cwd_if_supplied(self): - self.under_test.run(['install', '--without', 'development', 'test'], cwd='/a/cwd') - self.osutils.popen.assert_called_with(['/a/b/c/bundle', 'install', '--without', 'development', 'test'], - cwd='/a/cwd', stderr='PIPE', stdout='PIPE') + self.under_test.run(["install", "--without", "development", "test"], cwd="/a/cwd") + self.osutils.popen.assert_called_with( + ["/a/b/c/bundle", "install", "--without", "development", "test"], cwd="/a/cwd", stderr="PIPE", stdout="PIPE" + ) def test_returns_popen_out_decoded_if_retcode_is_0(self): - self.popen.out = b'some encoded text\n\n' - result = self.under_test.run(['install', '--without', 'development', 'test']) - self.assertEqual(result, 'some encoded text') + self.popen.out = b"some encoded text\n\n" + result = self.under_test.run(["install", "--without", "development", "test"]) + self.assertEqual(result, "some encoded text") def test_raises_BundlerExecutionError_with_err_text_if_retcode_is_not_0(self): self.popen.returncode = 1 - self.popen.err = b'some error text\n\n' + self.popen.err = b"some error text\n\n" with self.assertRaises(BundlerExecutionError) as raised: - self.under_test.run(['install', '--without', 'development', 'test']) + self.under_test.run(["install", "--without", "development", "test"]) self.assertEqual(raised.exception.args[0], "Bundler Failed: some error text") def test_raises_ValueError_if_args_not_a_list(self): with self.assertRaises(ValueError) as raised: - self.under_test.run(('install', '--without', 'development', 'test')) + self.under_test.run(("install", "--without", "development", "test")) self.assertEqual(raised.exception.args[0], "args must be a list") def test_raises_ValueError_if_args_empty(self): From 1e63b823979d7b7c82e4d5316dba356e91b8b343 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Tue, 21 Jan 2020 10:28:39 -0800 Subject: [PATCH 3/7] feat:Add Go Modules Debug Build Flags (#145) --- .../workflows/dotnet_clipackage/actions.py | 2 +- aws_lambda_builders/workflows/go_modules/builder.py | 10 ++++++++-- aws_lambda_builders/workflows/go_modules/workflow.py | 6 ++++-- tests/unit/workflows/go_modules/test_builder.py | 11 +++++++++++ 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/aws_lambda_builders/workflows/dotnet_clipackage/actions.py b/aws_lambda_builders/workflows/dotnet_clipackage/actions.py index accaaad08..dc918fa25 100644 --- a/aws_lambda_builders/workflows/dotnet_clipackage/actions.py +++ b/aws_lambda_builders/workflows/dotnet_clipackage/actions.py @@ -1,5 +1,5 @@ """ -Actions for Ruby dependency resolution with Bundler +Actions for .NET dependency resolution with CLI Package """ import os diff --git a/aws_lambda_builders/workflows/go_modules/builder.py b/aws_lambda_builders/workflows/go_modules/builder.py index 38e220768..9604c6f64 100644 --- a/aws_lambda_builders/workflows/go_modules/builder.py +++ b/aws_lambda_builders/workflows/go_modules/builder.py @@ -3,6 +3,7 @@ """ import logging +from aws_lambda_builders.workflow import BuildMode LOG = logging.getLogger(__name__) @@ -18,7 +19,7 @@ class GoModulesBuilder(object): LANGUAGE = "go" - def __init__(self, osutils, binaries): + def __init__(self, osutils, binaries, mode=BuildMode.RELEASE): """Initialize a GoModulesBuilder. :type osutils: :class:`lambda_builders.utils.OSUtils` @@ -30,6 +31,7 @@ def __init__(self, osutils, binaries): """ self.osutils = osutils self.binaries = binaries + self.mode = mode def build(self, source_dir_path, output_path): """Builds a go project onto an output path. @@ -44,7 +46,11 @@ def build(self, source_dir_path, output_path): env.update(self.osutils.environ) env.update({"GOOS": "linux", "GOARCH": "amd64"}) runtime_path = self.binaries[self.LANGUAGE].binary_path - cmd = [runtime_path, "build", "-o", output_path, source_dir_path] + cmd = [runtime_path, "build"] + if self.mode and self.mode.lower() == BuildMode.DEBUG: + LOG.debug("Debug build requested: Setting configuration to Debug") + cmd += ["-gcflags='all=-N -l'"] + cmd += ["-o", output_path, source_dir_path] p = self.osutils.popen(cmd, cwd=source_dir_path, env=env, stdout=self.osutils.pipe, stderr=self.osutils.pipe) out, err = p.communicate() diff --git a/aws_lambda_builders/workflows/go_modules/workflow.py b/aws_lambda_builders/workflows/go_modules/workflow.py index a80c1d65a..0932e81f5 100644 --- a/aws_lambda_builders/workflows/go_modules/workflow.py +++ b/aws_lambda_builders/workflows/go_modules/workflow.py @@ -15,7 +15,9 @@ class GoModulesWorkflow(BaseWorkflow): CAPABILITY = Capability(language="go", dependency_manager="modules", application_framework=None) - def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=None, osutils=None, **kwargs): + def __init__( + self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=None, osutils=None, mode=None, **kwargs + ): super(GoModulesWorkflow, self).__init__( source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=runtime, **kwargs @@ -29,7 +31,7 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim output_path = osutils.joinpath(artifacts_dir, handler) - builder = GoModulesBuilder(osutils, binaries=self.binaries) + builder = GoModulesBuilder(osutils, binaries=self.binaries, mode=mode) self.actions = [GoModulesBuildAction(source_dir, output_path, builder)] def get_validators(self): diff --git a/tests/unit/workflows/go_modules/test_builder.py b/tests/unit/workflows/go_modules/test_builder.py index daea632c6..5e9b163a1 100644 --- a/tests/unit/workflows/go_modules/test_builder.py +++ b/tests/unit/workflows/go_modules/test_builder.py @@ -49,3 +49,14 @@ def test_raises_BuilderError_with_err_text_if_retcode_is_not_0(self): with self.assertRaises(BuilderError) as raised: self.under_test.build("source_dir", "output_path") self.assertEqual(raised.exception.args[0], "Builder Failed: some error text") + + def test_debug_configuration_set(self): + self.under_test = GoModulesBuilder(self.osutils, self.binaries, "Debug") + self.under_test.build("source_dir", "output_path") + self.osutils.popen.assert_called_with( + ["/path/to/go", "build", "-gcflags='all=-N -l'", "-o", "output_path", "source_dir"], + cwd="source_dir", + env={"GOOS": "linux", "GOARCH": "amd64"}, + stderr="PIPE", + stdout="PIPE", + ) From 8c9d4768e19d5c022331163c890a98a3d28c59e3 Mon Sep 17 00:00:00 2001 From: Jacob Fuss <32497805+jfuss@users.noreply.github.com> Date: Tue, 21 Jan 2020 14:08:35 -0600 Subject: [PATCH 4/7] fix: Regex for getting aws_lambda_builders version (#147) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b10d412ae..9ef976e7c 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ def read_requirements(req="base.txt"): def read_version(): content = read(os.path.join(os.path.dirname(__file__), "aws_lambda_builders", "__init__.py")) - return re.search(r"__version__ = \"([^']+)\"", content).group(1) + return re.search(r"__version__ = \"([^'\"]+)", content).group(1) cmd_name = "lambda-builders" From b911949eac421fc286523988fb83e9b2fdbe0723 Mon Sep 17 00:00:00 2001 From: Sriram Madapusi Vasudevan <3770774+sriram-mv@users.noreply.github.com> Date: Thu, 23 Jan 2020 10:11:17 -0800 Subject: [PATCH 5/7] fix: better error messages on build failures (#146) * fix: better error messages on build failures Why is this change necessary? * Generic "Binary Validation Failed" does not add enough value How does it address the issue? * Showcases which binary has failed and shows which paths were looked at. What side effects does this change have? * None. * fix: explicit error messages on binary validation failure - fail workflow explicitly if binary resolution failed * fix: surface error messages for all binary validation failures * fix: fix tests for unittest imports * fix: make pylint happy * fix: appveyor black version --- .appveyor.yml | 2 +- .gitignore | 4 +- aws_lambda_builders/workflow.py | 39 ++++++++++++------ .../workflows/python_pip/actions.py | 6 ++- .../workflows/python_pip/compat.py | 3 ++ .../workflows/python_pip/exceptions.py | 8 ++++ tests/functional/testdata/cwd.py | 2 +- .../workflows/go_modules/test_go.py | 2 +- .../workflows/python_pip/test_python_pip.py | 2 +- tests/unit/test_workflow.py | 41 ++++++++++++++++++- .../unit/workflows/python_pip/test_actions.py | 16 ++++++++ .../workflows/python_pip/test_packager.py | 2 +- 12 files changed, 105 insertions(+), 22 deletions(-) create mode 100644 aws_lambda_builders/workflows/python_pip/exceptions.py diff --git a/.appveyor.yml b/.appveyor.yml index 68f61232d..e4aed21a9 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -101,7 +101,7 @@ for: - sh: "PATH=/opt/gradle/gradle-5.5/bin:$PATH" # Install black - - sh: "wget -O /tmp/black https://github.com/python/black/releases/download/19.3b0/black" + - sh: "wget -O /tmp/black https://github.com/python/black/releases/download/19.10b0/black" - sh: "chmod +x /tmp/black" - sh: "/tmp/black --version" diff --git a/.gitignore b/.gitignore index ab4e9c47e..c7f990167 100644 --- a/.gitignore +++ b/.gitignore @@ -257,7 +257,7 @@ wheels/ # Installer logs pip-log.txt pip-delete-this-directory.txt - +pip-wheel-metadata/ # Unit test / coverage reports htmlcov/ .tox/ @@ -389,4 +389,4 @@ $RECYCLE.BIN/ tests/integration/workflows/go_dep/data/src/*/vendor/* -# End of https://www.gitignore.io/api/osx,node,macos,linux,python,windows,pycharm,intellij,sublimetext,visualstudiocode \ No newline at end of file +# End of https://www.gitignore.io/api/osx,node,macos,linux,python,windows,pycharm,intellij,sublimetext,visualstudiocode diff --git a/aws_lambda_builders/workflow.py b/aws_lambda_builders/workflow.py index a659f817d..42263085b 100644 --- a/aws_lambda_builders/workflow.py +++ b/aws_lambda_builders/workflow.py @@ -40,25 +40,39 @@ def sanitize(func): @functools.wraps(func) def wrapper(self, *args, **kwargs): - valid_paths = [] + valid_paths = {} + invalid_paths = {} # NOTE: we need to access binaries to get paths and resolvers, before validating. - binaries_copy = self.binaries - for binary, binary_path in binaries_copy.items(): - validator = binary_path.validator - exec_paths = binary_path.resolver.exec_paths if not binary_path.path_provided else binary_path.binary_path + for binary, binary_checker in self.binaries.items(): + invalid_paths[binary] = [] + try: + exec_paths = ( + binary_checker.resolver.exec_paths + if not binary_checker.path_provided + else binary_checker.binary_path + ) + except ValueError as ex: + raise WorkflowFailedError(workflow_name=self.NAME, action_name="Resolver", reason=str(ex)) for executable_path in exec_paths: - valid_path = None try: - valid_path = validator.validate(executable_path) + valid_path = binary_checker.validator.validate(executable_path) + if valid_path: + valid_paths[binary] = valid_path except MisMatchRuntimeError as ex: LOG.debug("Invalid executable for %s at %s", binary, executable_path, exc_info=str(ex)) - if valid_path: - binary_path.binary_path = valid_path - valid_paths.append(valid_path) + invalid_paths[binary].append(executable_path) + if valid_paths.get(binary, None): + binary_checker.binary_path = valid_paths[binary] break - self.binaries = binaries_copy if len(self.binaries) != len(valid_paths): - raise WorkflowFailedError(workflow_name=self.NAME, action_name=None, reason="Binary validation failed!") + validation_failed_binaries = set(self.binaries.keys()).difference(valid_paths.keys()) + messages = [] + for validation_failed_binary in validation_failed_binaries: + message = "Binary validation failed for {0}, searched for {0} in following locations : {1} which did not satisfy constraints for runtime: {2}. Do you have {0} for runtime: {2} on your PATH?".format( + validation_failed_binary, invalid_paths[validation_failed_binary], self.runtime + ) + messages.append(message) + raise WorkflowFailedError(workflow_name=self.NAME, action_name="Validation", reason="\n".join(messages)) func(self, *args, **kwargs) return wrapper @@ -261,6 +275,7 @@ def run(self): raise WorkflowFailedError(workflow_name=self.NAME, action_name=action.NAME, reason=str(ex)) except Exception as ex: + LOG.debug("%s raised unhandled exception", action_info, exc_info=ex) raise WorkflowUnknownError(workflow_name=self.NAME, action_name=action.NAME, reason=str(ex)) diff --git a/aws_lambda_builders/workflows/python_pip/actions.py b/aws_lambda_builders/workflows/python_pip/actions.py index 3c21acf53..f4cd225bd 100644 --- a/aws_lambda_builders/workflows/python_pip/actions.py +++ b/aws_lambda_builders/workflows/python_pip/actions.py @@ -4,6 +4,7 @@ from aws_lambda_builders.actions import BaseAction, Purpose, ActionFailedError from aws_lambda_builders.workflows.python_pip.utils import OSUtils +from .exceptions import MissingPipError from .packager import PythonPipDependencyBuilder, PackagerError, DependencyBuilder, SubprocessPip, PipRunner @@ -24,7 +25,10 @@ def __init__(self, artifacts_dir, scratch_dir, manifest_path, runtime, binaries) def execute(self): os_utils = OSUtils() python_path = self.binaries[self.LANGUAGE].binary_path - pip = SubprocessPip(osutils=os_utils, python_exe=python_path) + try: + pip = SubprocessPip(osutils=os_utils, python_exe=python_path) + except MissingPipError as ex: + raise ActionFailedError(str(ex)) pip_runner = PipRunner(python_exe=python_path, pip=pip) dependency_builder = DependencyBuilder(osutils=os_utils, pip_runner=pip_runner, runtime=self.runtime) diff --git a/aws_lambda_builders/workflows/python_pip/compat.py b/aws_lambda_builders/workflows/python_pip/compat.py index a6c3e0814..51389b7d0 100644 --- a/aws_lambda_builders/workflows/python_pip/compat.py +++ b/aws_lambda_builders/workflows/python_pip/compat.py @@ -1,5 +1,6 @@ import os +from aws_lambda_builders.workflows.python_pip.exceptions import MissingPipError from aws_lambda_builders.workflows.python_pip.utils import OSUtils @@ -8,6 +9,8 @@ def pip_import_string(python_exe): cmd = [python_exe, "-c", "import pip; print(pip.__version__)"] p = os_utils.popen(cmd, stdout=os_utils.pipe, stderr=os_utils.pipe) stdout, stderr = p.communicate() + if not p.returncode == 0: + raise MissingPipError(python_path=python_exe) pip_version = stdout.decode("utf-8").strip() pip_major_version = int(pip_version.split(".")[0]) pip_minor_version = int(pip_version.split(".")[1]) diff --git a/aws_lambda_builders/workflows/python_pip/exceptions.py b/aws_lambda_builders/workflows/python_pip/exceptions.py new file mode 100644 index 000000000..d35915807 --- /dev/null +++ b/aws_lambda_builders/workflows/python_pip/exceptions.py @@ -0,0 +1,8 @@ +""" +Python pip specific workflow exceptions. +""" +from aws_lambda_builders.exceptions import LambdaBuilderError + + +class MissingPipError(LambdaBuilderError): + MESSAGE = "pip executable not found in your python environment at {python_path}" diff --git a/tests/functional/testdata/cwd.py b/tests/functional/testdata/cwd.py index ea0aff47e..ea065561f 100644 --- a/tests/functional/testdata/cwd.py +++ b/tests/functional/testdata/cwd.py @@ -1,3 +1,3 @@ import os -print (os.getcwd()) +print(os.getcwd()) diff --git a/tests/integration/workflows/go_modules/test_go.py b/tests/integration/workflows/go_modules/test_go.py index fa02363d8..d8e1e98ce 100644 --- a/tests/integration/workflows/go_modules/test_go.py +++ b/tests/integration/workflows/go_modules/test_go.py @@ -37,7 +37,7 @@ def test_builds_project_without_dependencies(self): ) expected_files = {"no-deps-main"} output_files = set(os.listdir(self.artifacts_dir)) - print (output_files) + print(output_files) self.assertEquals(expected_files, output_files) def test_builds_project_with_dependencies(self): diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index 7a665c162..eb34293a0 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -64,7 +64,7 @@ def test_mismatch_runtime_python_project(self): runtime=self.runtime_mismatch[self.runtime], ) except WorkflowFailedError as ex: - self.assertIn("Binary validation failed!", str(ex)) + self.assertIn("Binary validation failed", str(ex)) def test_runtime_validate_python_project_fail_open_unsupported_runtime(self): with self.assertRaises(WorkflowFailedError): diff --git a/tests/unit/test_workflow.py b/tests/unit/test_workflow.py index 4c0e1b480..a50f40d1d 100644 --- a/tests/unit/test_workflow.py +++ b/tests/unit/test_workflow.py @@ -1,7 +1,8 @@ import os import sys from unittest import TestCase -from mock import Mock, call + +from mock import Mock, MagicMock, call try: import pathlib @@ -12,7 +13,7 @@ from aws_lambda_builders.validator import RuntimeValidator from aws_lambda_builders.workflow import BaseWorkflow, Capability from aws_lambda_builders.registry import get_workflow, DEFAULT_REGISTRY -from aws_lambda_builders.exceptions import WorkflowFailedError, WorkflowUnknownError +from aws_lambda_builders.exceptions import WorkflowFailedError, WorkflowUnknownError, MisMatchRuntimeError from aws_lambda_builders.actions import ActionFailedError @@ -206,6 +207,42 @@ def test_must_execute_actions_in_sequence(self): ) self.assertTrue(validator_mock.validate.call_count, 1) + def test_must_fail_workflow_binary_resolution_failure(self): + action_mock = Mock() + validator_mock = Mock() + validator_mock.validate = Mock() + validator_mock.validate.return_value = None + resolver_mock = Mock() + resolver_mock.exec_paths = MagicMock(side_effect=ValueError("Binary could not be resolved")) + binaries_mock = Mock() + binaries_mock.return_value = [] + + self.work.get_validators = lambda: validator_mock + self.work.get_resolvers = lambda: resolver_mock + self.work.actions = [action_mock.action1, action_mock.action2, action_mock.action3] + self.work.binaries = {"binary": BinaryPath(resolver=resolver_mock, validator=validator_mock, binary="binary")} + with self.assertRaises(WorkflowFailedError) as ex: + self.work.run() + + def test_must_fail_workflow_binary_validation_failure(self): + action_mock = Mock() + validator_mock = Mock() + validator_mock.validate = Mock() + validator_mock.validate = MagicMock( + side_effect=MisMatchRuntimeError(language="test", required_runtime="test1", runtime_path="/usr/bin/binary") + ) + resolver_mock = Mock() + resolver_mock.exec_paths = ["/usr/bin/binary"] + binaries_mock = Mock() + binaries_mock.return_value = [] + + self.work.get_validators = lambda: validator_mock + self.work.get_resolvers = lambda: resolver_mock + self.work.actions = [action_mock.action1, action_mock.action2, action_mock.action3] + self.work.binaries = {"binary": BinaryPath(resolver=resolver_mock, validator=validator_mock, binary="binary")} + with self.assertRaises(WorkflowFailedError) as ex: + self.work.run() + def test_must_raise_with_no_actions(self): self.work.actions = [] diff --git a/tests/unit/workflows/python_pip/test_actions.py b/tests/unit/workflows/python_pip/test_actions.py index e8c98f7a1..24c1f3d2e 100644 --- a/tests/unit/workflows/python_pip/test_actions.py +++ b/tests/unit/workflows/python_pip/test_actions.py @@ -7,6 +7,7 @@ from aws_lambda_builders.binary_path import BinaryPath from aws_lambda_builders.workflows.python_pip.actions import PythonPipBuildAction +from aws_lambda_builders.workflows.python_pip.exceptions import MissingPipError from aws_lambda_builders.workflows.python_pip.packager import PackagerError @@ -43,3 +44,18 @@ def test_must_raise_exception_on_failure(self, PythonPipDependencyBuilderMock): with self.assertRaises(ActionFailedError): action.execute() + + @patch("aws_lambda_builders.workflows.python_pip.actions.SubprocessPip") + def test_must_raise_exception_on_pip_failure(self, PythonSubProcessPipMock): + PythonSubProcessPipMock.side_effect = MissingPipError(python_path="mockpath") + + action = PythonPipBuildAction( + "artifacts", + "scratch_dir", + "manifest", + "runtime", + {"python": BinaryPath(resolver=Mock(), validator=Mock(), binary="python", binary_path=sys.executable)}, + ) + + with self.assertRaises(ActionFailedError): + action.execute() diff --git a/tests/unit/workflows/python_pip/test_packager.py b/tests/unit/workflows/python_pip/test_packager.py index 39714412b..4aa3ed455 100644 --- a/tests/unit/workflows/python_pip/test_packager.py +++ b/tests/unit/workflows/python_pip/test_packager.py @@ -20,7 +20,7 @@ from aws_lambda_builders.workflows.python_pip import packager -print (packager) +print(packager) FakePipCall = namedtuple("FakePipEntry", ["args", "env_vars", "shim"]) From dd5a990ae095c92a03a4783b0c4079edace1cc0e Mon Sep 17 00:00:00 2001 From: Sriram Madapusi Vasudevan <3770774+sriram-mv@users.noreply.github.com> Date: Thu, 23 Jan 2020 11:10:11 -0800 Subject: [PATCH 6/7] fix: npm output handling (#150) * fix output handling * chore: run black auto formatter Co-authored-by: Alexander Betaev --- aws_lambda_builders/workflows/nodejs_npm/actions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_builders/workflows/nodejs_npm/actions.py b/aws_lambda_builders/workflows/nodejs_npm/actions.py index 5ba7bec3a..3b0bf15fc 100644 --- a/aws_lambda_builders/workflows/nodejs_npm/actions.py +++ b/aws_lambda_builders/workflows/nodejs_npm/actions.py @@ -56,7 +56,7 @@ def execute(self): LOG.debug("NODEJS packaging %s to %s", package_path, self.scratch_dir) - tarfile_name = self.subprocess_npm.run(["pack", "-q", package_path], cwd=self.scratch_dir) + tarfile_name = self.subprocess_npm.run(["pack", "-q", package_path], cwd=self.scratch_dir).splitlines()[-1] LOG.debug("NODEJS packed to %s", tarfile_name) From 7d718d46f23f3dd0d915438f2479601d5576b97c Mon Sep 17 00:00:00 2001 From: Sriram Madapusi Vasudevan <3770774+sriram-mv@users.noreply.github.com> Date: Thu, 23 Jan 2020 11:11:10 -0800 Subject: [PATCH 7/7] chore: version bump (#148) - version bump to 0.7.0 --- aws_lambda_builders/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_builders/__init__.py b/aws_lambda_builders/__init__.py index b5c061292..30bb059c5 100644 --- a/aws_lambda_builders/__init__.py +++ b/aws_lambda_builders/__init__.py @@ -1,5 +1,5 @@ """ AWS Lambda Builder Library """ -__version__ = "0.6.0" +__version__ = "0.7.0" RPC_PROTOCOL_VERSION = "0.3"