diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index a17cf8491..000000000 --- a/.pylintrc +++ /dev/null @@ -1,365 +0,0 @@ -[MASTER] - -# Specify a configuration file. -#rcfile= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Add files or directories to the ignore list. They should be base names, not -# paths. -ignore=compat.py, utils.py - -# Pickle collected data for later comparisons. -persistent=yes - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - -# Use multiple processes to speed up Pylint. -jobs=1 - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist= - -# Allow optimization of some AST trees. This will activate a peephole AST -# optimizer, which will apply various small optimizations. For instance, it can -# be used to obtain the result of joining multiple strings with the addition -# operator. Joining a lot of strings can lead to a maximum recursion error in -# Pylint and this flag can prevent that. It has one side effect, the resulting -# AST will be different than the one from reality. -optimize-ast=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time. See also the "--disable" option for examples. -#enable= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -# R0205,W0107,R1705,R1710,R1719,R1720,R1714 are all disable due to a forced upgrade to support python3.8 -# R1725,W0707 can be removed when we drop Python 2 support -disable=R0201,W0613,I0021,I0020,W1618,W1619,R0902,R0903,W0231,W0611,R0913,W0703,C0330,R0204,I0011,R0904,C0301, R0205,W0107,R1705,R1710,R1719,R1720,R1714,R1725,W0707 - - -[REPORTS] - -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html. You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". -files-output=no - -# Tells whether to display a full report or only the messages -reports=no - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - - -[BASIC] - -# List of builtins function names that should not be used, separated by a comma -bad-functions=apply,reduce - -# Good variable names which should always be accepted, separated by a comma -good-names=e,i,j,k,n,ex,Run,_ - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=yes - -# Regular expression matching correct function names -function-rgx=[a-z_][a-z0-9_]{2,50}$ - -# Naming hint for function names -function-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct variable names -variable-rgx=[a-z_][a-z0-9_]{0,50}$ - -# Naming hint for variable names -variable-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct constant names -const-rgx=(([a-zA-Z_][a-zA-Z0-9_]*)|(__.*__))$ - -# Naming hint for constant names -const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Regular expression matching correct attribute names -attr-rgx=[a-z_][a-z0-9_]{1,50}$ - -# Naming hint for attribute names -attr-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct argument names -argument-rgx=[a-z_][a-z0-9_]{0,50}$ - -# Naming hint for argument names -argument-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct class attribute names -class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Naming hint for class attribute names -class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Regular expression matching correct inline iteration names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Naming hint for inline iteration names -inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ - -# Regular expression matching correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Naming hint for class names -class-name-hint=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression matching correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Naming hint for module names -module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Regular expression matching correct method names -method-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for method names -method-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=.* - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - - -[FORMAT] - -# Maximum number of characters on a single line. -max-line-length=120 - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - -# List of optional constructs for which whitespace checking is disabled -no-space-check=trailing-comma,dict-separator - -# Maximum number of lines in a module -max-module-lines=1000 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX - - -[SIMILARITIES] - -# Minimum lines number of a similarity. -min-similarity-lines=10 - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=yes - - -[SPELLING] - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - -[TYPECHECK] - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis -ignored-modules=six.moves - -# List of classes names for which member attributes should not be checked -# (useful for classes with attributes dynamically set). -ignored-classes=SQLObject - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E0201 when accessed. Python regular -# expressions are accepted. -generated-members=REQUEST,acl_users,aq_parent,objects,DoesNotExist,md5,sha1,sha224,sha256,sha384,sha512 - - -[VARIABLES] - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_|dummy|ignore - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_,_cb - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict,_fields,_replace,_source,_make - - -[DESIGN] - -# Maximum number of arguments for function / method -max-args=5 - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.* - -# Maximum number of locals for function / method body -max-locals=15 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of branch for function / method body -max-branches=12 - -# Maximum number of statements in function / method body -max-statements=30 - -# Maximum number of parents for a class (see R0901). -max-parents=6 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=0 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - - -[IMPORTS] - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub,string,TERMIOS,Bastion,rexec,UserDict - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception diff --git a/Makefile b/Makefile index abaf957ec..7b4b49ec5 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ integ-test: lint: # Liner performs static analysis to catch latent bugs - pylint --rcfile .pylintrc aws_lambda_builders + ruff aws_lambda_builders # Command to run everytime you make changes to verify everything works dev: lint test diff --git a/aws_lambda_builders/__init__.py b/aws_lambda_builders/__init__.py index 3e6cb96eb..ece5e4ff8 100644 --- a/aws_lambda_builders/__init__.py +++ b/aws_lambda_builders/__init__.py @@ -4,5 +4,5 @@ # Changing version will trigger a new release! # Please make the version change as the last step of your development. -__version__ = "1.25.0" +__version__ = "1.26.0" RPC_PROTOCOL_VERSION = "0.3" diff --git a/aws_lambda_builders/builder.py b/aws_lambda_builders/builder.py index 7d5430110..0558589e0 100644 --- a/aws_lambda_builders/builder.py +++ b/aws_lambda_builders/builder.py @@ -6,7 +6,7 @@ import os import logging -from aws_lambda_builders.architecture import X86_64, ARM64 +from aws_lambda_builders.architecture import X86_64 from aws_lambda_builders.registry import get_workflow, DEFAULT_REGISTRY from aws_lambda_builders.workflow import Capability diff --git a/aws_lambda_builders/utils.py b/aws_lambda_builders/utils.py index 201e8f460..a9a59dcd9 100644 --- a/aws_lambda_builders/utils.py +++ b/aws_lambda_builders/utils.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import Union -from aws_lambda_builders.architecture import X86_64, ARM64 +from aws_lambda_builders.architecture import ARM64 LOG = logging.getLogger(__name__) diff --git a/aws_lambda_builders/validator.py b/aws_lambda_builders/validator.py index ce66282a7..05c179c86 100644 --- a/aws_lambda_builders/validator.py +++ b/aws_lambda_builders/validator.py @@ -15,7 +15,6 @@ "nodejs14.x": [ARM64, X86_64], "nodejs16.x": [ARM64, X86_64], "nodejs18.x": [ARM64, X86_64], - "python3.6": [X86_64], "python3.7": [X86_64], "python3.8": [ARM64, X86_64], "python3.9": [ARM64, X86_64], diff --git a/aws_lambda_builders/workflow.py b/aws_lambda_builders/workflow.py index f0dfb9ebc..dcb58eb06 100644 --- a/aws_lambda_builders/workflow.py +++ b/aws_lambda_builders/workflow.py @@ -7,6 +7,7 @@ from collections import namedtuple from enum import Enum +from typing import Optional from aws_lambda_builders.binary_path import BinaryPath from aws_lambda_builders.path_resolver import PathResolver @@ -38,6 +39,12 @@ class BuildMode(object): RELEASE = "release" +class BuildDirectory(Enum): + SCRATCH = "scratch" + ARTIFACTS = "artifacts" + SOURCE = "source" + + class BuildInSourceSupport(Enum): """ Enum to define a workflow's support for building in source. @@ -140,14 +147,13 @@ def __new__(mcs, name, bases, class_dict): if not isinstance(cls.CAPABILITY, Capability): raise ValueError("Workflow '{}' must register valid capabilities".format(cls.NAME)) - # All workflows must define valid default and supported values for build in source - if ( - not isinstance(cls.BUILD_IN_SOURCE_SUPPORT, BuildInSourceSupport) - or cls.BUILD_IN_SOURCE_BY_DEFAULT not in cls.BUILD_IN_SOURCE_SUPPORT.value - ): - raise ValueError( - "Workflow '{}' must define valid default and supported values for build in source".format(cls.NAME) - ) + # All workflows must define supported values for build in source + if not isinstance(cls.BUILD_IN_SOURCE_SUPPORT, BuildInSourceSupport): + raise ValueError("Workflow '{}' must define supported values for build in source".format(cls.NAME)) + + # All workflows must define default build directory + if not isinstance(cls.DEFAULT_BUILD_DIR, BuildDirectory): + raise ValueError("Workflow '{}' must define default build directory".format(cls.NAME)) LOG.debug("Registering workflow '%s' with capability '%s'", cls.NAME, cls.CAPABILITY) DEFAULT_REGISTRY[cls.CAPABILITY] = cls @@ -173,12 +179,12 @@ class BaseWorkflow(object, metaclass=_WorkflowMetaClass): # Optional list of manifests file/folder names supported by this workflow. SUPPORTED_MANIFESTS = [] - # Whether the workflow builds in source by default, each workflow should define this. - # (some workflows build in temporary or artifact directories by default) - BUILD_IN_SOURCE_BY_DEFAULT = None # Support for building in source, each workflow should define this. BUILD_IN_SOURCE_SUPPORT = None + # The directory where the workflow builds/installs by default, each workflow should define this. + DEFAULT_BUILD_DIR = None + def __init__( self, source_dir, @@ -261,21 +267,39 @@ def __init__( self.is_building_layer = is_building_layer self.experimental_flags = experimental_flags if experimental_flags else [] - self.build_in_source = build_in_source + # this represents where the build/install happens, not the final output directory (that's the artifacts_dir) + self.build_dir = self._select_build_dir(build_in_source) + + # Actions are registered by the subclasses as they seem fit + self.actions = [] + self._binaries = {} + + def _select_build_dir(self, build_in_source: Optional[bool]) -> str: + """ + Returns the build directory for the workflow. + """ + + should_build_in_source = build_in_source if build_in_source not in self.BUILD_IN_SOURCE_SUPPORT.value: + # assign default value + should_build_in_source = self.DEFAULT_BUILD_DIR == BuildDirectory.SOURCE + # only show warning if an unsupported value was explicitly passed in if build_in_source is not None: LOG.warning( 'Workflow %s does not support value "%s" for building in source. Using default value "%s".', self.NAME, build_in_source, - self.BUILD_IN_SOURCE_BY_DEFAULT, + should_build_in_source, ) - self.build_in_source = self.BUILD_IN_SOURCE_BY_DEFAULT - # Actions are registered by the subclasses as they seem fit - self.actions = [] - self._binaries = {} + build_directory_mapping = { + BuildDirectory.SCRATCH: self.scratch_dir, + BuildDirectory.ARTIFACTS: self.artifacts_dir, + BuildDirectory.SOURCE: self.source_dir, + } + + return self.source_dir if should_build_in_source else build_directory_mapping.get(self.DEFAULT_BUILD_DIR) def is_supported(self): """ diff --git a/aws_lambda_builders/workflows/custom_make/make.py b/aws_lambda_builders/workflows/custom_make/make.py index 5f097aa04..52223a60e 100644 --- a/aws_lambda_builders/workflows/custom_make/make.py +++ b/aws_lambda_builders/workflows/custom_make/make.py @@ -3,7 +3,6 @@ """ import io import logging -import subprocess import shutil import threading diff --git a/aws_lambda_builders/workflows/custom_make/workflow.py b/aws_lambda_builders/workflows/custom_make/workflow.py index 208e9d469..d5d6dbe23 100644 --- a/aws_lambda_builders/workflows/custom_make/workflow.py +++ b/aws_lambda_builders/workflows/custom_make/workflow.py @@ -2,7 +2,7 @@ ProvidedMakeWorkflow """ from aws_lambda_builders.workflows.custom_make.validator import CustomMakeRuntimeValidator -from aws_lambda_builders.workflow import BaseWorkflow, Capability, BuildInSourceSupport +from aws_lambda_builders.workflow import BaseWorkflow, Capability, BuildInSourceSupport, BuildDirectory from aws_lambda_builders.actions import CopySourceAction from aws_lambda_builders.path_resolver import PathResolver from .actions import CustomMakeAction @@ -23,7 +23,7 @@ class CustomMakeWorkflow(BaseWorkflow): EXCLUDED_FILES = (".aws-sam", ".git") - BUILD_IN_SOURCE_BY_DEFAULT = False + DEFAULT_BUILD_DIR = BuildDirectory.SCRATCH BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.OPTIONALLY_SUPPORTED def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=None, osutils=None, **kwargs): @@ -35,7 +35,6 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim self.os_utils = OSUtils() options = kwargs.get("options") or {} - build_in_source = kwargs.get("build_in_source") build_logical_id = options.get("build_logical_id", None) if not build_logical_id: @@ -47,10 +46,8 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim subprocess_make = SubProcessMake(make_exe=self.binaries["make"].binary_path, osutils=self.os_utils) - # an explicitly definied working directory should take precedence - working_directory = options.get("working_directory") or self._select_working_directory( - source_dir, scratch_dir, build_in_source - ) + # an explicitly defined working directory should take precedence + working_directory = options.get("working_directory") or self.build_dir make_action = CustomMakeAction( artifacts_dir, @@ -63,18 +60,12 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim self.actions = [] - if not self.build_in_source: - # if we're building on scratch_dir, we have to first copy the source there - self.actions.append(CopySourceAction(source_dir, scratch_dir, excludes=self.EXCLUDED_FILES)) + if self.build_dir != source_dir: + # if we're not building in the source directory, we have to first copy the source + self.actions.append(CopySourceAction(source_dir, self.build_dir, excludes=self.EXCLUDED_FILES)) self.actions.append(make_action) - def _select_working_directory(self, source_dir: str, scratch_dir: str, build_in_source: bool): - """ - Returns the directory where the make action should be executed - """ - return source_dir if build_in_source else scratch_dir - def get_resolvers(self): return [PathResolver(runtime="provided", binary="make", executable_search_paths=self.executable_search_paths)] diff --git a/aws_lambda_builders/workflows/dotnet_clipackage/actions.py b/aws_lambda_builders/workflows/dotnet_clipackage/actions.py index 456806c2a..0bbbd1716 100644 --- a/aws_lambda_builders/workflows/dotnet_clipackage/actions.py +++ b/aws_lambda_builders/workflows/dotnet_clipackage/actions.py @@ -45,7 +45,7 @@ def execute(self): LOG.debug("Installing Amazon.Lambda.Tools Global Tool") self.subprocess_dotnet.run(["tool", "install", "-g", "Amazon.Lambda.Tools", "--ignore-failed-sources"]) GlobalToolInstallAction.__tools_installed = True - except DotnetCLIExecutionError as ex: + except DotnetCLIExecutionError: LOG.debug("Error installing probably due to already installed. Attempt to update to latest version.") try: self.subprocess_dotnet.run( diff --git a/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py b/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py index 5986b3656..72c6a76be 100644 --- a/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py +++ b/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py @@ -2,7 +2,6 @@ Wrapper around calls to dotent CLI through a subprocess. """ -import sys import logging import locale diff --git a/aws_lambda_builders/workflows/dotnet_clipackage/utils.py b/aws_lambda_builders/workflows/dotnet_clipackage/utils.py index cb629c777..5a03644ce 100644 --- a/aws_lambda_builders/workflows/dotnet_clipackage/utils.py +++ b/aws_lambda_builders/workflows/dotnet_clipackage/utils.py @@ -72,8 +72,8 @@ def _is_symlink(self, file_info): bool A response regarding whether the ZipInfo defines a symlink or not. """ - - return (file_info.external_attr >> 28) == 0xA + symlink = 0xA + return (file_info.external_attr >> 28) == symlink def _extract(self, file_info, output_dir, zip_ref): """ diff --git a/aws_lambda_builders/workflows/dotnet_clipackage/workflow.py b/aws_lambda_builders/workflows/dotnet_clipackage/workflow.py index f514f31cf..320e7fd95 100644 --- a/aws_lambda_builders/workflows/dotnet_clipackage/workflow.py +++ b/aws_lambda_builders/workflows/dotnet_clipackage/workflow.py @@ -1,7 +1,7 @@ """ .NET Core CLI Package Workflow """ -from aws_lambda_builders.workflow import BaseWorkflow, Capability, BuildInSourceSupport +from aws_lambda_builders.workflow import BaseWorkflow, BuildDirectory, Capability, BuildInSourceSupport from .actions import GlobalToolInstallAction, RunPackageAction from .dotnetcli import SubprocessDotnetCLI @@ -19,7 +19,7 @@ class DotnetCliPackageWorkflow(BaseWorkflow): CAPABILITY = Capability(language="dotnet", dependency_manager="cli-package", application_framework=None) - BUILD_IN_SOURCE_BY_DEFAULT = True + DEFAULT_BUILD_DIR = BuildDirectory.SOURCE BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.EXCLUSIVELY_SUPPORTED def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=None, mode=None, **kwargs): diff --git a/aws_lambda_builders/workflows/go_modules/builder.py b/aws_lambda_builders/workflows/go_modules/builder.py index d2fc048e9..43c8a0e1e 100644 --- a/aws_lambda_builders/workflows/go_modules/builder.py +++ b/aws_lambda_builders/workflows/go_modules/builder.py @@ -5,7 +5,7 @@ from pathlib import Path from aws_lambda_builders.workflow import BuildMode -from aws_lambda_builders.architecture import X86_64, ARM64 +from aws_lambda_builders.architecture import X86_64 from aws_lambda_builders.utils import get_goarch LOG = logging.getLogger(__name__) diff --git a/aws_lambda_builders/workflows/go_modules/workflow.py b/aws_lambda_builders/workflows/go_modules/workflow.py index 72ac03ba9..86f22a318 100644 --- a/aws_lambda_builders/workflows/go_modules/workflow.py +++ b/aws_lambda_builders/workflows/go_modules/workflow.py @@ -1,7 +1,7 @@ """ Go Modules Workflow """ -from aws_lambda_builders.workflow import BaseWorkflow, Capability, BuildInSourceSupport +from aws_lambda_builders.workflow import BaseWorkflow, BuildDirectory, Capability, BuildInSourceSupport from .actions import GoModulesBuildAction from .builder import GoModulesBuilder @@ -15,7 +15,7 @@ class GoModulesWorkflow(BaseWorkflow): CAPABILITY = Capability(language="go", dependency_manager="modules", application_framework=None) - BUILD_IN_SOURCE_BY_DEFAULT = True + DEFAULT_BUILD_DIR = BuildDirectory.SOURCE BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.EXCLUSIVELY_SUPPORTED def __init__( diff --git a/aws_lambda_builders/workflows/java_gradle/gradle_validator.py b/aws_lambda_builders/workflows/java_gradle/gradle_validator.py index 6d839ef3c..d383e74cd 100644 --- a/aws_lambda_builders/workflows/java_gradle/gradle_validator.py +++ b/aws_lambda_builders/workflows/java_gradle/gradle_validator.py @@ -81,7 +81,7 @@ def _get_jvm_string(self, gradle_path): if p.returncode != 0: return None - for l in stdout.splitlines(): - l_dec = l.decode() + for line in stdout.splitlines(): + l_dec = line.decode() if l_dec.startswith("JVM"): return l_dec diff --git a/aws_lambda_builders/workflows/java_gradle/workflow.py b/aws_lambda_builders/workflows/java_gradle/workflow.py index 9a2546dae..d83b54650 100644 --- a/aws_lambda_builders/workflows/java_gradle/workflow.py +++ b/aws_lambda_builders/workflows/java_gradle/workflow.py @@ -4,7 +4,7 @@ import hashlib import os from aws_lambda_builders.actions import CleanUpAction -from aws_lambda_builders.workflow import BaseWorkflow, Capability, BuildInSourceSupport +from aws_lambda_builders.workflow import BaseWorkflow, BuildDirectory, Capability, BuildInSourceSupport from aws_lambda_builders.workflows.java.actions import JavaCopyDependenciesAction, JavaMoveDependenciesAction from aws_lambda_builders.workflows.java.utils import OSUtils @@ -25,7 +25,7 @@ class JavaGradleWorkflow(BaseWorkflow): INIT_FILE = "lambda-build-init.gradle" - BUILD_IN_SOURCE_BY_DEFAULT = False + DEFAULT_BUILD_DIR = BuildDirectory.SCRATCH BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.NOT_SUPPORTED def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, **kwargs): diff --git a/aws_lambda_builders/workflows/java_maven/maven_validator.py b/aws_lambda_builders/workflows/java_maven/maven_validator.py index 995123690..80a9bbe57 100644 --- a/aws_lambda_builders/workflows/java_maven/maven_validator.py +++ b/aws_lambda_builders/workflows/java_maven/maven_validator.py @@ -79,7 +79,7 @@ def _get_jvm_string(self, maven_path): if p.returncode != 0: return None - for l in stdout.splitlines(): - l_dec = l.decode() + for line in stdout.splitlines(): + l_dec = line.decode() if l_dec.startswith("Java version"): return l_dec diff --git a/aws_lambda_builders/workflows/java_maven/workflow.py b/aws_lambda_builders/workflows/java_maven/workflow.py index da80d17ae..164c365c8 100644 --- a/aws_lambda_builders/workflows/java_maven/workflow.py +++ b/aws_lambda_builders/workflows/java_maven/workflow.py @@ -1,7 +1,7 @@ """ Java Maven Workflow """ -from aws_lambda_builders.workflow import BaseWorkflow, Capability, BuildInSourceSupport +from aws_lambda_builders.workflow import BaseWorkflow, BuildDirectory, Capability, BuildInSourceSupport from aws_lambda_builders.actions import CopySourceAction, CleanUpAction from aws_lambda_builders.workflows.java.actions import JavaCopyDependenciesAction, JavaMoveDependenciesAction from aws_lambda_builders.workflows.java.utils import OSUtils @@ -28,7 +28,7 @@ class JavaMavenWorkflow(BaseWorkflow): EXCLUDED_FILES = (".aws-sam", ".git") - BUILD_IN_SOURCE_BY_DEFAULT = False + DEFAULT_BUILD_DIR = BuildDirectory.SCRATCH BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.NOT_SUPPORTED def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, **kwargs): diff --git a/aws_lambda_builders/workflows/nodejs_npm/actions.py b/aws_lambda_builders/workflows/nodejs_npm/actions.py index dd4a8d07e..afadb5572 100644 --- a/aws_lambda_builders/workflows/nodejs_npm/actions.py +++ b/aws_lambda_builders/workflows/nodejs_npm/actions.py @@ -81,11 +81,10 @@ class NodejsNpmInstallAction(BaseAction): DESCRIPTION = "Installing dependencies from NPM" PURPOSE = Purpose.RESOLVE_DEPENDENCIES - def __init__(self, artifacts_dir, subprocess_npm): + def __init__(self, install_dir, subprocess_npm): """ - :type artifacts_dir: str - :param artifacts_dir: an existing (writable) directory with project source files. - Dependencies will be installed in this directory. + :type install_dir: str + :param install_dir: Dependencies will be installed in this directory. :type subprocess_npm: aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessNpm :param subprocess_npm: An instance of the NPM process wrapper @@ -95,7 +94,7 @@ def __init__(self, artifacts_dir, subprocess_npm): """ super(NodejsNpmInstallAction, self).__init__() - self.artifacts_dir = artifacts_dir + self.install_dir = install_dir self.subprocess_npm = subprocess_npm def execute(self): @@ -105,10 +104,10 @@ def execute(self): :raises lambda_builders.actions.ActionFailedError: when NPM execution fails """ try: - LOG.debug("NODEJS installing in: %s", self.artifacts_dir) + LOG.debug("NODEJS installing in: %s", self.install_dir) self.subprocess_npm.run( - ["install", "-q", "--no-audit", "--no-save", "--unsafe-perm", "--production"], cwd=self.artifacts_dir + ["install", "-q", "--no-audit", "--no-save", "--unsafe-perm", "--production"], cwd=self.install_dir ) except NpmExecutionError as ex: @@ -128,18 +127,17 @@ class NodejsNpmCIAction(BaseAction): DESCRIPTION = "Installing dependencies from NPM using the CI method" PURPOSE = Purpose.RESOLVE_DEPENDENCIES - def __init__(self, artifacts_dir, subprocess_npm): + def __init__(self, install_dir, subprocess_npm): """ - :type artifacts_dir: str - :param artifacts_dir: an existing (writable) directory with project source files. - Dependencies will be installed in this directory. + :type install_dir: str + :param install_dir: Dependencies will be installed in this directory. :type subprocess_npm: aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessNpm :param subprocess_npm: An instance of the NPM process wrapper """ super(NodejsNpmCIAction, self).__init__() - self.artifacts_dir = artifacts_dir + self.install_dir = install_dir self.subprocess_npm = subprocess_npm def execute(self): @@ -150,9 +148,9 @@ def execute(self): """ try: - LOG.debug("NODEJS installing ci in: %s", self.artifacts_dir) + LOG.debug("NODEJS installing ci in: %s", self.install_dir) - self.subprocess_npm.run(["ci"], cwd=self.artifacts_dir) + self.subprocess_npm.run(["ci"], cwd=self.install_dir) except NpmExecutionError as ex: raise ActionFailedError(str(ex)) diff --git a/aws_lambda_builders/workflows/nodejs_npm/utils.py b/aws_lambda_builders/workflows/nodejs_npm/utils.py index 529e6fac5..fb5cb904b 100644 --- a/aws_lambda_builders/workflows/nodejs_npm/utils.py +++ b/aws_lambda_builders/workflows/nodejs_npm/utils.py @@ -4,7 +4,6 @@ import os import platform -import tarfile import subprocess import shutil import json diff --git a/aws_lambda_builders/workflows/nodejs_npm/workflow.py b/aws_lambda_builders/workflows/nodejs_npm/workflow.py index 7f71fdf2f..fa2c5f5cb 100644 --- a/aws_lambda_builders/workflows/nodejs_npm/workflow.py +++ b/aws_lambda_builders/workflows/nodejs_npm/workflow.py @@ -5,7 +5,7 @@ import logging from aws_lambda_builders.path_resolver import PathResolver -from aws_lambda_builders.workflow import BaseWorkflow, Capability, BuildInSourceSupport +from aws_lambda_builders.workflow import BaseWorkflow, BuildDirectory, Capability, BuildInSourceSupport from aws_lambda_builders.actions import ( CopySourceAction, CleanUpAction, @@ -42,7 +42,7 @@ class NodejsNpmWorkflow(BaseWorkflow): CONFIG_PROPERTY = "aws_sam" - BUILD_IN_SOURCE_BY_DEFAULT = False + DEFAULT_BUILD_DIR = BuildDirectory.ARTIFACTS BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.NOT_SUPPORTED def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=None, osutils=None, **kwargs): @@ -148,15 +148,15 @@ def get_resolvers(self): return [PathResolver(runtime=self.runtime, binary="npm")] @staticmethod - def get_install_action(source_dir, artifacts_dir, subprocess_npm, osutils, build_options): + def get_install_action(source_dir, install_dir, subprocess_npm, osutils, build_options): """ Get the install action used to install dependencies at artifacts_dir :type source_dir: str :param source_dir: an existing (readable) directory containing source files - :type artifacts_dir: str - :param artifacts_dir: Dependencies will be installed in this directory. + :type install_dir: str + :param install_dir: Dependencies will be installed in this directory. :type osutils: aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils :param osutils: An instance of OS Utilities for file manipulation @@ -181,6 +181,6 @@ def get_install_action(source_dir, artifacts_dir, subprocess_npm, osutils, build npm_ci_option = build_options.get("use_npm_ci", False) if (osutils.file_exists(lockfile_path) or osutils.file_exists(shrinkwrap_path)) and npm_ci_option: - return NodejsNpmCIAction(artifacts_dir, subprocess_npm=subprocess_npm) + return NodejsNpmCIAction(install_dir=install_dir, subprocess_npm=subprocess_npm) - return NodejsNpmInstallAction(artifacts_dir, subprocess_npm=subprocess_npm) + return NodejsNpmInstallAction(install_dir=install_dir, subprocess_npm=subprocess_npm) diff --git a/aws_lambda_builders/workflows/nodejs_npm_esbuild/actions.py b/aws_lambda_builders/workflows/nodejs_npm_esbuild/actions.py index 810b242dc..903ee1fac 100644 --- a/aws_lambda_builders/workflows/nodejs_npm_esbuild/actions.py +++ b/aws_lambda_builders/workflows/nodejs_npm_esbuild/actions.py @@ -12,6 +12,8 @@ LOG = logging.getLogger(__name__) EXTERNAL_KEY = "external" +# minimum esbuild version required to use "--external" +MINIMUM_VERSION_FOR_EXTERNAL = "0.14.13" class EsbuildBundleAction(BaseAction): @@ -27,8 +29,8 @@ class EsbuildBundleAction(BaseAction): def __init__( self, - scratch_dir: str, - artifacts_dir: str, + working_directory: str, + output_directory: str, bundler_config: Dict[str, Any], osutils: OSUtils, subprocess_esbuild: SubprocessEsbuild, @@ -36,31 +38,27 @@ def __init__( skip_deps=False, ): """ - :type scratch_dir: str - :param scratch_dir: an existing (writable) directory for temporary files - - :type artifacts_dir: str - :param artifacts_dir: an existing (writable) directory where to store the output. + Parameters + ---------- + working_directory : str + directory where esbuild is executed + output_directory : str + an existing (writable) directory where to store the output. Note that the actual result will be in the 'package' subdirectory here. - - :type osutils: aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils - :param osutils: An instance of OS Utilities for file manipulation - - :type subprocess_esbuild: aws_lambda_builders.workflows.nodejs_npm_esbuild.esbuild.SubprocessEsbuild - :param subprocess_esbuild: An instance of the Esbuild process wrapper - - :type skip_deps: bool - :param skip_deps: if dependencies should be omitted from bundling - - :type bundler_config: Dict[str,Any] - :param bundler_config: the bundler configuration - - :type manifest: str - :param manifest: path to package.json file contents to read + bundler_config : Dict[str, Any] + the bundle configuration + osutils : OSUtils + An instance of OS Utilities for file manipulation + subprocess_esbuild : SubprocessEsbuild + An instance of the Esbuild process wrapper + manifest : str + path to package.json file contents to read + skip_deps : bool, optional + if dependencies should be omitted from bundling, by default False """ super(EsbuildBundleAction, self).__init__() - self._scratch_dir = scratch_dir - self._artifacts_dir = artifacts_dir + self._working_directory = working_directory + self._output_directory = output_directory self._bundler_config = bundler_config self._osutils = osutils self._subprocess_esbuild = subprocess_esbuild @@ -71,13 +69,21 @@ def execute(self) -> None: """ Runs the action. - :raises lambda_builders.actions.ActionFailedError: when esbuild packaging fails + Raises + ------ + ActionFailedError + when esbuild packaging fails """ esbuild_command = EsbuildCommandBuilder( - self._scratch_dir, self._artifacts_dir, self._bundler_config, self._osutils, self._manifest + self._working_directory, self._output_directory, self._bundler_config, self._osutils, self._manifest ) if self._should_bundle_deps_externally(): + check_minimum_esbuild_version( + minimum_version_required=MINIMUM_VERSION_FOR_EXTERNAL, + working_directory=self._working_directory, + subprocess_esbuild=self._subprocess_esbuild, + ) esbuild_command.build_with_no_dependencies() if EXTERNAL_KEY in self._bundler_config: # Already marking everything as external, @@ -89,7 +95,7 @@ def execute(self) -> None: ) try: - self._subprocess_esbuild.run(args, cwd=self._scratch_dir) + self._subprocess_esbuild.run(args, cwd=self._working_directory) except EsbuildExecutionError as ex: raise ActionFailedError(str(ex)) @@ -103,65 +109,62 @@ def _should_bundle_deps_externally(self) -> bool: return self._skip_deps or "./node_modules/*" in self._bundler_config.get(EXTERNAL_KEY, []) -class EsbuildCheckVersionAction(BaseAction): - """ - A Lambda Builder Action that verifies that esbuild is a version supported by sam accelerate +def check_minimum_esbuild_version( + minimum_version_required: str, working_directory: str, subprocess_esbuild: SubprocessEsbuild +): """ + Checks esbuild version against a minimum version required. - NAME = "EsbuildCheckVersion" - DESCRIPTION = "Checking esbuild version" - PURPOSE = Purpose.COMPILE_SOURCE + Parameters + ---------- + minimum_version_required: str + minimum esbuild version required for check to pass - MIN_VERSION = "0.14.13" + working_directory: str + directory where esbuild is executed - def __init__(self, scratch_dir, subprocess_esbuild): - """ - :type scratch_dir: str - :param scratch_dir: temporary directory where esbuild is executed + subprocess_esbuild: aws_lambda_builders.workflows.nodejs_npm_esbuild.esbuild.SubprocessEsbuild + An instance of the Esbuild process wrapper - :type subprocess_esbuild: aws_lambda_builders.workflows.nodejs_npm_esbuild.esbuild.SubprocessEsbuild - :param subprocess_esbuild: An instance of the Esbuild process wrapper - """ - super().__init__() - self.scratch_dir = scratch_dir - self.subprocess_esbuild = subprocess_esbuild + Raises + ---------- + lambda_builders.actions.ActionFailedError + when esbuild version checking fails + """ + args = ["--version"] - def execute(self): - """ - Runs the action. + try: + version = subprocess_esbuild.run(args, cwd=working_directory) + except EsbuildExecutionError as ex: + raise ActionFailedError(str(ex)) - :raises lambda_builders.actions.ActionFailedError: when esbuild version checking fails - """ - args = ["--version"] + LOG.debug("Found esbuild with version: %s", version) - try: - version = self.subprocess_esbuild.run(args, cwd=self.scratch_dir) - except EsbuildExecutionError as ex: - raise ActionFailedError(str(ex)) + try: + check_version = _get_version_tuple(minimum_version_required) + esbuild_version = _get_version_tuple(version) - LOG.debug("Found esbuild with version: %s", version) + if esbuild_version < check_version: + raise ActionFailedError( + f"Unsupported esbuild version. To use a dependency layer, the esbuild version must be at " + f"least {minimum_version_required}. Version found: {version}" + ) + except (TypeError, ValueError) as ex: + raise ActionFailedError(f"Unable to parse esbuild version: {str(ex)}") - try: - check_version = EsbuildCheckVersionAction._get_version_tuple(self.MIN_VERSION) - esbuild_version = EsbuildCheckVersionAction._get_version_tuple(version) - - if esbuild_version < check_version: - raise ActionFailedError( - f"Unsupported esbuild version. To use a dependency layer, the esbuild version must be at " - f"least {self.MIN_VERSION}. Version found: {version}" - ) - except (TypeError, ValueError) as ex: - raise ActionFailedError(f"Unable to parse esbuild version: {str(ex)}") - - @staticmethod - def _get_version_tuple(version_string): - """ - Get an integer tuple representation of the version for comparison - :type version_string: str - :param version_string: string containing the esbuild version +def _get_version_tuple(version_string: str): + """ + Get an integer tuple representation of the version for comparison - :rtype: tuple - :return: version tuple used for comparison - """ - return tuple(map(int, version_string.split("."))) + Parameters + ---------- + version_string: str + string containing the esbuild version + + Returns + ---------- + tuple + version tuple used for comparison + """ + return tuple(map(int, version_string.split("."))) diff --git a/aws_lambda_builders/workflows/nodejs_npm_esbuild/workflow.py b/aws_lambda_builders/workflows/nodejs_npm_esbuild/workflow.py index 1f6db83d8..aac9908cd 100644 --- a/aws_lambda_builders/workflows/nodejs_npm_esbuild/workflow.py +++ b/aws_lambda_builders/workflows/nodejs_npm_esbuild/workflow.py @@ -5,20 +5,17 @@ import logging import json from pathlib import Path -from typing import List -from aws_lambda_builders.workflow import BaseWorkflow, Capability, BuildInSourceSupport +from aws_lambda_builders.workflow import BaseWorkflow, BuildDirectory, Capability, BuildInSourceSupport from aws_lambda_builders.actions import ( - CopySourceAction, CleanUpAction, + CopySourceAction, MoveDependenciesAction, - BaseAction, LinkSourceAction, ) from aws_lambda_builders.utils import which from .actions import ( EsbuildBundleAction, - EsbuildCheckVersionAction, ) from .esbuild import SubprocessEsbuild, EsbuildExecutionError from ..nodejs_npm import NodejsNpmWorkflow @@ -40,12 +37,12 @@ class NodejsNpmEsbuildWorkflow(BaseWorkflow): CAPABILITY = Capability(language="nodejs", dependency_manager="npm-esbuild", application_framework=None) - EXCLUDED_FILES = (".aws-sam", ".git") + EXCLUDED_FILES = (".aws-sam", ".git", "node_modules") CONFIG_PROPERTY = "aws_sam" - BUILD_IN_SOURCE_BY_DEFAULT = False - BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.NOT_SUPPORTED + DEFAULT_BUILD_DIR = BuildDirectory.SCRATCH + BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.OPTIONALLY_SUPPORTED def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=None, osutils=None, **kwargs): @@ -53,158 +50,90 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=runtime, **kwargs ) - if osutils is None: - osutils = OSUtils() - - subprocess_npm = SubprocessNpm(osutils) - subprocess_esbuild = self._get_esbuild_subprocess(subprocess_npm, scratch_dir, osutils) + self.osutils = osutils or OSUtils() + self.subprocess_npm = SubprocessNpm(self.osutils) + self.subprocess_esbuild = self._get_esbuild_subprocess() bundler_config = self.get_build_properties() - if not osutils.file_exists(manifest_path): - LOG.warning("package.json file not found. Bundling source without dependencies.") + if not self.osutils.file_exists(manifest_path): + LOG.warning("package.json file not found. Bundling source without installing dependencies.") self.actions = [ EsbuildBundleAction( - source_dir, artifacts_dir, bundler_config, osutils, subprocess_esbuild, self.manifest_path + working_directory=self.source_dir, + output_directory=self.artifacts_dir, + bundler_config=bundler_config, + osutils=self.osutils, + subprocess_esbuild=self.subprocess_esbuild, + manifest=self.manifest_path, ) ] return - self.actions = self.actions_with_bundler( - source_dir, scratch_dir, artifacts_dir, bundler_config, osutils, subprocess_npm, subprocess_esbuild - ) - - def actions_with_bundler( - self, source_dir, scratch_dir, artifacts_dir, bundler_config, osutils, subprocess_npm, subprocess_esbuild - ) -> List[BaseAction]: - """ - Generate a list of Nodejs build actions with a bundler - - :type source_dir: str - :param source_dir: an existing (readable) directory containing source files - - :type scratch_dir: str - :param scratch_dir: an existing (writable) directory for temporary files - - :type artifacts_dir: str - :param artifacts_dir: an existing (writable) directory where to store the output. - - :type bundler_config: dict - :param bundler_config: configurations for the bundler action - - :type osutils: aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils - :param osutils: An instance of OS Utilities for file manipulation - - :type subprocess_npm: aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessNpm - :param subprocess_npm: An instance of the NPM process wrapper - - :type subprocess_esbuild: aws_lambda_builders.workflows.nodejs_npm_esbuild.esbuild.SubprocessEsbuild - :param subprocess_esbuild: An instance of the esbuild process wrapper - - :rtype: list - :return: List of build actions to execute - """ - actions: List[BaseAction] = [ - CopySourceAction(source_dir, scratch_dir, excludes=self.EXCLUDED_FILES + tuple(["node_modules"])) - ] - - # Bundle dependencies separately in a dependency layer. We need to check the esbuild - # version here to ensure that it supports skipping dependency bundling - esbuild_no_deps = [ - EsbuildCheckVersionAction(scratch_dir, subprocess_esbuild), - EsbuildBundleAction( - scratch_dir, - artifacts_dir, - bundler_config, - osutils, - subprocess_esbuild, - self.manifest_path, - skip_deps=True, - ), - ] - esbuild_with_deps = EsbuildBundleAction( - scratch_dir, artifacts_dir, bundler_config, osutils, subprocess_esbuild, self.manifest_path - ) - - install_action = NodejsNpmWorkflow.get_install_action( - source_dir, - scratch_dir, - subprocess_npm, - osutils, - self.options, + if not self.download_dependencies and not self.dependencies_dir: + # Invalid workflow, can't have no dependency dir and no installation + raise EsbuildExecutionError( + message="Lambda Builders was unable to find the location of the dependencies since a " + "dependencies directory was not provided and downloading dependencies is disabled." + ) + + # if we're building in the source directory, we don't have to copy the source code + self.actions = ( + [] + if self.build_dir == self.source_dir + else [CopySourceAction(source_dir=self.source_dir, dest_dir=self.build_dir, excludes=self.EXCLUDED_FILES)] ) - if self.download_dependencies and not self.dependencies_dir: - return actions + [install_action, esbuild_with_deps] - - return self._accelerate_workflow_actions( - source_dir, scratch_dir, actions, install_action, esbuild_with_deps, esbuild_no_deps + if self.download_dependencies: + self.actions.append( + NodejsNpmWorkflow.get_install_action( + source_dir=source_dir, + install_dir=self.build_dir, + subprocess_npm=self.subprocess_npm, + osutils=self.osutils, + build_options=self.options, + ) + ) + + bundle_action = EsbuildBundleAction( + working_directory=self.build_dir, + output_directory=self.artifacts_dir, + bundler_config=bundler_config, + osutils=self.osutils, + subprocess_esbuild=self.subprocess_esbuild, + manifest=self.manifest_path, + skip_deps=not self.combine_dependencies, ) - def _accelerate_workflow_actions( - self, source_dir, scratch_dir, actions, install_action, esbuild_with_deps, esbuild_no_deps - ): - """ - Generate a list of Nodejs build actions for incremental build and auto dependency layer - - :type source_dir: str - :param source_dir: an existing (readable) directory containing source files - - :type scratch_dir: str - :param scratch_dir: an existing (writable) directory for temporary files - - :type actions: List[BaseAction] - :param actions: List of existing actions - - :type install_action: BaseAction - :param install_action: Installation action for npm - - :type esbuild_with_deps: BaseAction - :param esbuild_with_deps: Standard esbuild action bundling source with deps - - :type esbuild_no_deps: List[BaseAction] - :param esbuild_no_deps: esbuild action not including dependencies in the bundled artifacts + # If there's no dependencies_dir, just bundle and we're done. + # Same thing if we're building in the source directory (since the dependencies persist in + # the source directory, we don't want to move them or symlink them back to the source) + if not self.dependencies_dir or self.build_dir == self.source_dir: + self.actions.append(bundle_action) + return - :rtype: list - :return: List of build actions to execute - """ if self.download_dependencies: - actions += [install_action, CleanUpAction(self.dependencies_dir)] - if self.combine_dependencies: - # Auto dependency layer disabled, first build - actions += [ - esbuild_with_deps, - MoveDependenciesAction(source_dir, scratch_dir, self.dependencies_dir), - LinkSourceAction(self.dependencies_dir, scratch_dir), - ] - else: - # Auto dependency layer enabled, first build - actions += esbuild_no_deps + [MoveDependenciesAction(source_dir, scratch_dir, self.dependencies_dir)] + # if we downloaded dependencies, bundle and update dependencies_dir + self.actions += [ + bundle_action, + CleanUpAction(self.dependencies_dir), + MoveDependenciesAction(self.source_dir, self.scratch_dir, self.dependencies_dir), + ] else: - if self.dependencies_dir: - actions.append(LinkSourceAction(self.dependencies_dir, scratch_dir)) - - if self.combine_dependencies: - # Auto dependency layer disabled, subsequent builds - actions += [esbuild_with_deps] - else: - # Auto dependency layer enabled, subsequent builds - actions += esbuild_no_deps - else: - # Invalid workflow, can't have no dependency dir and no installation - raise EsbuildExecutionError( - message="Lambda Builders encountered an invalid workflow. A workflow can't " - "include a dependencies directory without installing dependencies." - ) - - return actions + # if we're reusing dependencies, then we need to symlink them before bundling + self.actions += [ + LinkSourceAction(self.dependencies_dir, self.scratch_dir), + bundle_action, + ] def get_build_properties(self): """ Get the aws_sam specific properties from the manifest, if they exist. - :rtype: dict - :return: Dict with aws_sam specific bundler configs + Returns + ------- + dict + aws_sam specific bundler configs """ if self.options and isinstance(self.options, dict): LOG.debug("Lambda Builders found the following esbuild properties:\n%s", json.dumps(self.options)) @@ -217,13 +146,13 @@ def get_resolvers(self): """ return [PathResolver(runtime=self.runtime, binary="npm")] - def _get_esbuild_subprocess(self, subprocess_npm, scratch_dir, osutils) -> SubprocessEsbuild: + def _get_esbuild_subprocess(self) -> SubprocessEsbuild: try: - npm_bin_path_root = subprocess_npm.run(["root"], cwd=scratch_dir) + npm_bin_path_root = self.subprocess_npm.run(["root"], cwd=self.scratch_dir) npm_bin_path = str(Path(npm_bin_path_root, ".bin")) except FileNotFoundError: raise EsbuildExecutionError(message="The esbuild workflow couldn't find npm installed on your system.") executable_search_paths = [npm_bin_path] if self.executable_search_paths is not None: executable_search_paths = executable_search_paths + self.executable_search_paths - return SubprocessEsbuild(osutils, executable_search_paths, which=which) + return SubprocessEsbuild(self.osutils, executable_search_paths, which=which) diff --git a/aws_lambda_builders/workflows/python_pip/compat.py b/aws_lambda_builders/workflows/python_pip/compat.py index 9e83494e4..f29037ef6 100644 --- a/aws_lambda_builders/workflows/python_pip/compat.py +++ b/aws_lambda_builders/workflows/python_pip/compat.py @@ -18,7 +18,8 @@ def pip_import_string(python_exe): # Pip moved its internals to an _internal module in version 10. # In order to be compatible with version 9 which has it at at the # top level we need to figure out the correct import path here. - if pip_major_version == 9: + pip_version_9 = 9 + if pip_major_version == pip_version_9: return "from pip import main" # Pip changed their import structure again in 19.3 # https://github.com/pypa/pip/commit/09fd200 diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index f43b0c274..1b7d23a69 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -82,7 +82,6 @@ def __init__(self, version): def get_lambda_abi(runtime): supported = { - "python3.6": "cp36m", "python3.7": "cp37m", "python3.8": "cp38", "python3.9": "cp39", @@ -100,7 +99,7 @@ def __init__(self, runtime, osutils=None, dependency_builder=None, architecture= :type runtime: str :param runtime: Python version to build dependencies for. This can - either be python3.6, python3.7, python3.8 or python3.9. These are currently the + either be python3.7, python3.8 or python3.9. These are currently the only supported values. :type osutils: :class:`lambda_builders.utils.OSUtils` @@ -199,8 +198,6 @@ class DependencyBuilder(object): # Mapping of abi to glibc version in Lambda runtime. _RUNTIME_GLIBC = { - "cp27mu": (2, 17), - "cp36m": (2, 17), "cp37m": (2, 17), "cp38": (2, 26), "cp39": (2, 26), @@ -410,15 +407,11 @@ def _is_compatible_wheel_filename(self, filename): return True prefix_version = implementation[:3] if prefix_version == "cp3": - # Deploying python 3 function which means we need cp36m abi + # Deploying python 3 function which means we need cp37m abi # We can also accept abi3 which is the CPython 3 Stable ABI and # will work on any version of python 3. if abi == lambda_runtime_abi or abi == "abi3": return True - elif prefix_version == "cp2": - # Deploying to python 2 function which means we need cp27mu abi - if abi == "cp27mu": - return True # Don't know what we have but it didn't pass compatibility tests. return False diff --git a/aws_lambda_builders/workflows/python_pip/utils.py b/aws_lambda_builders/workflows/python_pip/utils.py index 25d77bfc7..e4f4b6a5a 100644 --- a/aws_lambda_builders/workflows/python_pip/utils.py +++ b/aws_lambda_builders/workflows/python_pip/utils.py @@ -8,7 +8,6 @@ import contextlib import tempfile import shutil -import tarfile import subprocess import sys from typing import Optional, List diff --git a/aws_lambda_builders/workflows/python_pip/validator.py b/aws_lambda_builders/workflows/python_pip/validator.py index f3cd35808..7790b04c5 100644 --- a/aws_lambda_builders/workflows/python_pip/validator.py +++ b/aws_lambda_builders/workflows/python_pip/validator.py @@ -31,7 +31,7 @@ def validate(self, runtime_path): Returns ------- str - runtime_path, runtime to check eg: /usr/bin/python3.6 + runtime_path, runtime to check eg: /usr/bin/python3.9 Raises ------ diff --git a/aws_lambda_builders/workflows/python_pip/workflow.py b/aws_lambda_builders/workflows/python_pip/workflow.py index a304226be..58477cef4 100644 --- a/aws_lambda_builders/workflows/python_pip/workflow.py +++ b/aws_lambda_builders/workflows/python_pip/workflow.py @@ -3,7 +3,7 @@ """ import logging -from aws_lambda_builders.workflow import BaseWorkflow, BuildInSourceSupport, Capability +from aws_lambda_builders.workflow import BaseWorkflow, BuildDirectory, BuildInSourceSupport, Capability from aws_lambda_builders.actions import CopySourceAction, CleanUpAction, LinkSourceAction from aws_lambda_builders.workflows.python_pip.validator import PythonRuntimeValidator from aws_lambda_builders.path_resolver import PathResolver @@ -67,7 +67,7 @@ class PythonPipWorkflow(BaseWorkflow): PYTHON_VERSION_THREE = "3" - BUILD_IN_SOURCE_BY_DEFAULT = False + DEFAULT_BUILD_DIR = BuildDirectory.SCRATCH BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.NOT_SUPPORTED def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=None, osutils=None, **kwargs): diff --git a/aws_lambda_builders/workflows/ruby_bundler/utils.py b/aws_lambda_builders/workflows/ruby_bundler/utils.py index ad8e0282b..cf58fca7a 100644 --- a/aws_lambda_builders/workflows/ruby_bundler/utils.py +++ b/aws_lambda_builders/workflows/ruby_bundler/utils.py @@ -4,7 +4,6 @@ import os import platform -import tarfile import subprocess import shutil diff --git a/aws_lambda_builders/workflows/ruby_bundler/workflow.py b/aws_lambda_builders/workflows/ruby_bundler/workflow.py index bc943affa..060664eec 100644 --- a/aws_lambda_builders/workflows/ruby_bundler/workflow.py +++ b/aws_lambda_builders/workflows/ruby_bundler/workflow.py @@ -3,7 +3,7 @@ """ import logging -from aws_lambda_builders.workflow import BaseWorkflow, BuildInSourceSupport, Capability +from aws_lambda_builders.workflow import BaseWorkflow, BuildDirectory, BuildInSourceSupport, Capability from aws_lambda_builders.actions import CopySourceAction, CopyDependenciesAction, CleanUpAction from .actions import RubyBundlerInstallAction, RubyBundlerVendorAction from .utils import OSUtils @@ -25,7 +25,7 @@ class RubyBundlerWorkflow(BaseWorkflow): EXCLUDED_FILES = (".aws-sam", ".git") - BUILD_IN_SOURCE_BY_DEFAULT = False + DEFAULT_BUILD_DIR = BuildDirectory.ARTIFACTS BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.NOT_SUPPORTED def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=None, osutils=None, **kwargs): diff --git a/pyproject.toml b/pyproject.toml index a38f53ff8..ca16b1909 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,20 @@ +[tool.ruff] +line-length = 120 + +select = [ + "E", # Pycodestyle + "F", # Pyflakes + "PL", # pylint +] +ignore = ["PLR0913"] + +[tool.ruff.per-file-ignores] +"__init__.py" = ["F401", "E501"] +"aws_lambda_builders/workflow.py" = ["E501"] + [tool.black] line-length = 120 -target_version = ['py37', 'py36', 'py38'] +target_version = ['py37', 'py38', 'py39'] exclude = ''' ( diff --git a/requirements/dev.txt b/requirements/dev.txt index a73187525..4a87142bc 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -3,7 +3,6 @@ flake8==3.3.0; python_version < '3.8' flake8==3.8.4; python_version >= '3.8' pytest-cov==2.10.1 -pylint~=2.6.0 isort>=4.2.5,<5; python_version < '3.8' # Test requirements @@ -12,10 +11,6 @@ mock==4.0.2 parameterized==0.7.4 pyelftools~=0.27 # Used to verify the generated Go binary architecture in integration tests (utils.py) - -# tempfile backport for < 3.7 -backports.tempfile==1.0; python_version<"3.7" - - # formatter -black==22.3.0 \ No newline at end of file +black==22.3.0 +ruff \ No newline at end of file diff --git a/setup.py b/setup.py index 2d578ab07..c5fe08787 100644 --- a/setup.py +++ b/setup.py @@ -43,8 +43,8 @@ def read_version(): license="Apache License 2.0", packages=find_packages(exclude=["tests.*", "tests"]), keywords="AWS Lambda Functions Building", - # Support 3.6 or greater - python_requires=(">=3.6"), + # Support 3.7 or greater + python_requires=(">=3.7"), entry_points={"console_scripts": ["{}=aws_lambda_builders.__main__:main".format(cmd_name)]}, install_requires=read_requirements("base.txt") + read_requirements("python_pip.txt"), extras_require={"dev": read_requirements("dev.txt")}, @@ -58,7 +58,6 @@ def read_version(): "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Programming Language :: Python", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", diff --git a/tests/functional/testdata/workflows/hello_workflow/write_hello.py b/tests/functional/testdata/workflows/hello_workflow/write_hello.py index 9987b1f5c..295480041 100644 --- a/tests/functional/testdata/workflows/hello_workflow/write_hello.py +++ b/tests/functional/testdata/workflows/hello_workflow/write_hello.py @@ -4,7 +4,7 @@ import os -from aws_lambda_builders.workflow import BaseWorkflow, BuildInSourceSupport, Capability +from aws_lambda_builders.workflow import BaseWorkflow, BuildDirectory, BuildInSourceSupport, Capability from aws_lambda_builders.actions import BaseAction, Purpose @@ -34,7 +34,7 @@ class WriteHelloWorkflow(BaseWorkflow): NAME = "WriteHelloWorkflow" CAPABILITY = Capability(language="python", dependency_manager="test", application_framework="test") - BUILD_IN_SOURCE_BY_DEFAULT = False + DEFAULT_BUILD_DIR = BuildDirectory.SCRATCH BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.OPTIONALLY_SUPPORTED def __init__(self, source_dir, artifacts_dir, *args, **kwargs): diff --git a/tests/functional/workflows/python_pip/test_packager.py b/tests/functional/workflows/python_pip/test_packager.py index 643b08ad3..ed0ae4fb7 100644 --- a/tests/functional/workflows/python_pip/test_packager.py +++ b/tests/functional/workflows/python_pip/test_packager.py @@ -204,7 +204,7 @@ def _write_requirements_txt(self, packages, directory): def _make_appdir_and_dependency_builder(self, reqs, tmpdir, runner, **kwargs): appdir = str(_create_app_structure(tmpdir)) self._write_requirements_txt(reqs, appdir) - builder = DependencyBuilder(OSUtils(), "python3.6", runner, **kwargs) + builder = DependencyBuilder(OSUtils(), "python3.9", runner, **kwargs) return appdir, builder def test_can_build_local_dir_as_whl(self, tmpdir, pip_runner, osutils): @@ -215,7 +215,7 @@ def test_can_build_local_dir_as_whl(self, tmpdir, pip_runner, osutils): pip.set_return_tuple(0, (b"Processing ../foo\n" b" Link is a directory," b" ignoring download_dir"), b"") pip.wheels_to_build( expected_args=["--no-deps", "--wheel-dir", mock.ANY, "../foo"], - wheels_to_build=["foo-1.2-cp36-none-any.whl"], + wheels_to_build=["foo-1.2-cp39-none-any.whl"], ) site_packages = os.path.join(appdir, "site-packages") @@ -233,7 +233,7 @@ def test_can_get_whls_all_manylinux(self, tmpdir, pip_runner, osutils): requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], - packages=["foo-1.2-cp36-cp36m-manylinux1_x86_64.whl", "bar-1.2-cp36-cp36m-manylinux1_x86_64.whl"], + packages=["foo-1.2-cp39-cp39-manylinux1_x86_64.whl", "bar-1.2-cp39-cp39-manylinux1_x86_64.whl"], ) site_packages = os.path.join(appdir, ".chalice.", "site-packages") @@ -276,7 +276,7 @@ def test_can_expand_purelib_whl(self, tmpdir, pip_runner, osutils): requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], - packages=["foo-1.2-cp36-cp36m-manylinux1_x86_64.whl"], + packages=["foo-1.2-cp39-cp39-manylinux1_x86_64.whl"], whl_contents=["foo-1.2.data/purelib/foo/"], ) @@ -296,7 +296,7 @@ def test_can_expand_platlib_whl(self, tmpdir, pip_runner, osutils): requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], - packages=["foo-1.2-cp36-cp36m-manylinux1_x86_64.whl"], + packages=["foo-1.2-cp39-cp39-manylinux1_x86_64.whl"], whl_contents=["foo-1.2.data/platlib/foo/"], ) @@ -318,7 +318,7 @@ def test_can_expand_platlib_and_purelib(self, tmpdir, pip_runner, osutils): requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], - packages=["foo-1.2-cp36-cp36m-manylinux1_x86_64.whl"], + packages=["foo-1.2-cp39-cp39-manylinux1_x86_64.whl"], whl_contents=["foo-1.2.data/platlib/foo/", "foo-1.2.data/purelib/bar/"], ) @@ -340,7 +340,7 @@ def test_does_ignore_data(self, tmpdir, pip_runner, osutils): requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], - packages=["foo-1.2-cp36-cp36m-manylinux1_x86_64.whl"], + packages=["foo-1.2-cp39-cp39-manylinux1_x86_64.whl"], whl_contents=["foo/placeholder", "foo-1.2.data/data/bar/"], ) @@ -363,7 +363,7 @@ def test_does_ignore_include(self, tmpdir, pip_runner, osutils): requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], - packages=["foo-1.2-cp36-cp36m-manylinux1_x86_64.whl"], + packages=["foo-1.2-cp39-cp39-manylinux1_x86_64.whl"], whl_contents=["foo/placeholder", "foo.1.2.data/includes/bar/"], ) @@ -386,7 +386,7 @@ def test_does_ignore_scripts(self, tmpdir, pip_runner, osutils): requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], - packages=["foo-1.2-cp36-cp36m-manylinux1_x86_64.whl"], + packages=["foo-1.2-cp39-cp39-manylinux1_x86_64.whl"], whl_contents=["{package_name}/placeholder", "{data_dir}/scripts/bar/placeholder"], ) @@ -410,7 +410,7 @@ def test_can_expand_platlib_and_platlib_and_root(self, tmpdir, pip_runner, osuti requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], - packages=["foo-1.2-cp36-cp36m-manylinux1_x86_64.whl"], + packages=["foo-1.2-cp39-cp39-manylinux1_x86_64.whl"], whl_contents=[ "{package_name}/placeholder", "{data_dir}/platlib/bar/placeholder", @@ -435,9 +435,9 @@ def test_can_get_whls_mixed_compat(self, tmpdir, osutils, pip_runner): pip.packages_to_download( expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], packages=[ - "foo-1.0-cp36-none-any.whl", - "bar-1.2-cp36-cp36m-manylinux1_x86_64.whl", - "baz-1.5-cp36-cp36m-linux_x86_64.whl", + "foo-1.0-cp39-none-any.whl", + "bar-1.2-cp39-cp39-manylinux1_x86_64.whl", + "baz-1.5-cp39-cp39-linux_x86_64.whl", ], ) @@ -458,7 +458,7 @@ def test_can_support_pep_600_tags(self, tmpdir, osutils, pip_runner): pip.packages_to_download( expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], packages=[ - "foo-1.2-cp36-cp36m-manylinux_2_12_x86_64.whl", + "foo-1.2-cp39-cp39-manylinux_2_12_x86_64.whl", ], ) @@ -479,30 +479,7 @@ def test_can_support_compressed_tags(self, tmpdir, osutils, pip_runner): pip.packages_to_download( expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], packages=[ - "foo-1.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", - ], - ) - - site_packages = os.path.join(appdir, ".chalice.", "site-packages") - with osutils.tempdir() as scratch_dir: - builder.build_site_packages(requirements_file, site_packages, scratch_dir) - installed_packages = os.listdir(site_packages) - - pip.validate() - for req in reqs: - assert req in installed_packages - - def test_can_get_py27_whls(self, tmpdir, osutils, pip_runner): - reqs = ["foo", "bar", "baz"] - pip, runner = pip_runner - appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner) - requirements_file = os.path.join(appdir, "requirements.txt") - pip.packages_to_download( - expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], - packages=[ - "foo-1.0-cp27-none-any.whl", - "bar-1.2-cp27-none-manylinux1_x86_64.whl", - "baz-1.5-cp27-cp27mu-linux_x86_64.whl", + "foo-1.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", ], ) @@ -523,9 +500,9 @@ def test_can_get_arm64_whls(self, tmpdir, osutils, pip_runner): pip.packages_to_download( expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], packages=[ - "foo-1.0-cp36-none-any.whl", - "bar-1.2-cp36-none-manylinux2014_aarch64.whl", - "baz-1.5-cp36-cp36m-manylinux2014_aarch64.whl", + "foo-1.0-cp39-none-any.whl", + "bar-1.2-cp39-none-manylinux2014_aarch64.whl", + "baz-1.5-cp39-cp39-manylinux2014_aarch64.whl", ], ) @@ -613,10 +590,10 @@ def test_does_fail_on_pep_600_tag_with_unsupported_glibc_version(self, tmpdir, o pip.packages_to_download( expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], packages=[ - "foo-1.2-cp36-cp36m-manylinux_2_12_x86_64.whl", - "bar-1.2-cp36-cp36m-manylinux_2_999_x86_64.whl", - "baz-1.2-cp36-cp36m-manylinux_3_12_x86_64.whl", - "qux-1.2-cp36-cp36m-manylinux_3_999_x86_64.whl", + "foo-1.2-cp39-cp39-manylinux_2_12_x86_64.whl", + "bar-1.2-cp39-cp39-manylinux_2_999_x86_64.whl", + "baz-1.2-cp39-cp39-manylinux_3_12_x86_64.whl", + "qux-1.2-cp39-cp39-manylinux_3_999_x86_64.whl", ], ) @@ -642,7 +619,7 @@ def test_can_replace_incompat_whl(self, tmpdir, osutils, pip_runner): requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], - packages=["foo-1.0-cp36-none-any.whl", "bar-1.2-cp36-cp36m-macosx_10_6_intel.whl"], + packages=["foo-1.0-cp39-none-any.whl", "bar-1.2-cp39-cp39-macosx_10_6_intel.whl"], ) # Once the initial download has 1 incompatible whl file. The second, # more targeted download, finds manylinux1_x86_64 and downloads that. @@ -660,7 +637,7 @@ def test_can_replace_incompat_whl(self, tmpdir, osutils, pip_runner): mock.ANY, "bar==1.2", ], - packages=["bar-1.2-cp36-cp36m-manylinux1_x86_64.whl"], + packages=["bar-1.2-cp39-cp39-manylinux1_x86_64.whl"], ) site_packages = os.path.join(appdir, ".chalice.", "site-packages") with osutils.tempdir() as scratch_dir: @@ -711,13 +688,13 @@ def test_can_build_sdist(self, tmpdir, osutils, pip_runner): requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], - packages=["foo-1.2.zip", "bar-1.2-cp36-cp36m-manylinux1_x86_64.whl"], + packages=["foo-1.2.zip", "bar-1.2-cp39-cp39-manylinux1_x86_64.whl"], ) # Foo is built from and is pure python so it yields a compatible # wheel file. pip.wheels_to_build( expected_args=["--no-deps", "--wheel-dir", mock.ANY, PathArgumentEndingWith("foo-1.2.zip")], - wheels_to_build=["foo-1.2-cp36-none-any.whl"], + wheels_to_build=["foo-1.2-cp39-none-any.whl"], ) site_packages = os.path.join(appdir, ".chalice.", "site-packages") with osutils.tempdir() as scratch_dir: @@ -735,7 +712,7 @@ def test_build_sdist_makes_incompatible_whl(self, tmpdir, osutils, pip_runner): requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], - packages=["foo-1.2.zip", "bar-1.2-cp36-cp36m-manylinux1_x86_64.whl"], + packages=["foo-1.2.zip", "bar-1.2-cp39-cp39-manylinux1_x86_64.whl"], ) # foo is compiled since downloading it failed to get any wheels. And # the second download for manylinux1_x86_64 wheels failed as well. @@ -744,7 +721,7 @@ def test_build_sdist_makes_incompatible_whl(self, tmpdir, osutils, pip_runner): # can do to install this package. pip.wheels_to_build( expected_args=["--no-deps", "--wheel-dir", mock.ANY, PathArgumentEndingWith("foo-1.2.zip")], - wheels_to_build=["foo-1.2-cp36-cp36m-macosx_10_6_intel.whl"], + wheels_to_build=["foo-1.2-cp39-cp39-macosx_10_6_intel.whl"], ) site_packages = os.path.join(appdir, ".chalice.", "site-packages") with osutils.tempdir() as scratch_dir: @@ -813,7 +790,7 @@ def test_build_into_existing_dir_with_preinstalled_packages(self, tmpdir, osutil requirements_file = os.path.join(appdir, "requirements.txt") pip.packages_to_download( expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"], - packages=["foo-1.2.zip", "bar-1.2-cp36-cp36m-manylinux1_x86_64.whl"], + packages=["foo-1.2.zip", "bar-1.2-cp39-cp39-manylinux1_x86_64.whl"], ) pip.packages_to_download( expected_args=[ @@ -829,7 +806,7 @@ def test_build_into_existing_dir_with_preinstalled_packages(self, tmpdir, osutil mock.ANY, "foo==1.2", ], - packages=["foo-1.2-cp36-cp36m-macosx_10_6_intel.whl"], + packages=["foo-1.2-cp39-cp39-macosx_10_6_intel.whl"], ) # Add two fake packages foo and bar that have previously been diff --git a/tests/integration/workflows/nodejs_npm_esbuild/test_nodejs_npm_with_esbuild.py b/tests/integration/workflows/nodejs_npm_esbuild/test_nodejs_npm_with_esbuild.py index a9739841b..f7fd167f8 100644 --- a/tests/integration/workflows/nodejs_npm_esbuild/test_nodejs_npm_with_esbuild.py +++ b/tests/integration/workflows/nodejs_npm_esbuild/test_nodejs_npm_with_esbuild.py @@ -20,6 +20,7 @@ class TestNodejsNpmWorkflowWithEsbuild(TestCase): TEST_DATA_FOLDER = os.path.join(os.path.dirname(__file__), "testdata") def setUp(self): + self.source_dir = os.path.join(self.TEST_DATA_FOLDER, "with-deps-esbuild") self.artifacts_dir = tempfile.mkdtemp() self.scratch_dir = tempfile.mkdtemp() self.dependencies_dir = tempfile.mkdtemp() @@ -39,6 +40,11 @@ def tearDown(self): shutil.rmtree(self.artifacts_dir) shutil.rmtree(self.scratch_dir) + # clean up dependencies that were installed in source dir + source_dependencies = os.path.join(self.source_dir, "node_modules") + if os.path.exists(source_dependencies): + shutil.rmtree(source_dependencies) + @parameterized.expand([("nodejs12.x",), ("nodejs14.x",), ("nodejs16.x",), ("nodejs18.x",)]) def test_builds_javascript_project_with_dependencies(self, runtime): source_dir = os.path.join(self.TEST_DATA_FOLDER, "with-deps-esbuild") @@ -265,8 +271,8 @@ def test_builds_project_with_remote_dependencies_without_download_dependencies_w self.assertEqual( str(context.exception), - "Esbuild Failed: Lambda Builders encountered an invalid workflow. A" - " workflow can't include a dependencies directory without installing dependencies.", + "Esbuild Failed: Lambda Builders was unable to find the location of the dependencies since a " + "dependencies directory was not provided and downloading dependencies is disabled.", ) @parameterized.expand([("nodejs12.x",), ("nodejs14.x",), ("nodejs16.x",), ("nodejs18.x",)]) @@ -407,3 +413,29 @@ def test_esbuild_produces_mjs_output_files(self, runtime): expected_files = {"included.mjs", "included.mjs.map"} output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) + + @parameterized.expand([("nodejs12.x",), ("nodejs14.x",), ("nodejs16.x",), ("nodejs18.x",)]) + def test_esbuild_can_build_in_source(self, runtime): + options = {"entry_points": ["included.js"]} + + self.builder.build( + self.source_dir, + self.artifacts_dir, + self.scratch_dir, + os.path.join(self.source_dir, "package.json"), + runtime=runtime, + options=options, + executable_search_paths=[self.binpath], + build_in_source=True, + ) + + # dependencies installed in source folder + self.assertIn("node_modules", os.listdir(self.source_dir)) + + # dependencies not in scratch + self.assertNotIn("node_modules", os.listdir(self.scratch_dir)) + + # bundle is in artifacts + expected_files = {"included.js"} + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEqual(expected_files, output_files) diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index 3e1fe6cb3..705cb676a 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -57,7 +57,6 @@ def setUp(self): language=self.builder.capability.language, major=sys.version_info.major, minor=sys.version_info.minor ) self.runtime_mismatch = { - "python3.6": "python3.7", "python3.7": "python3.8", "python3.8": "python3.9", "python3.9": "python3.7", @@ -93,12 +92,8 @@ def test_must_build_python_project(self): experimental_flags=self.experimental_flags, ) - if self.runtime == "python3.6": - self.check_architecture_in("numpy-1.17.4.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"]) - expected_files = self.test_data_files.union({"numpy", "numpy-1.17.4.dist-info"}) - else: - self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"]) - expected_files = self.test_data_files.union({"numpy", "numpy-1.20.3.dist-info", "numpy.libs"}) + self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"]) + expected_files = self.test_data_files.union({"numpy", "numpy-1.20.3.dist-info", "numpy.libs"}) output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) @@ -118,13 +113,8 @@ def test_must_build_python_project_python3_binary(self): experimental_flags=self.experimental_flags, executable_search_paths=[executable_dir], ) - - if self.runtime == "python3.6": - self.check_architecture_in("numpy-1.17.4.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"]) - expected_files = self.test_data_files.union({"numpy", "numpy-1.17.4.dist-info"}) - else: - self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"]) - expected_files = self.test_data_files.union({"numpy", "numpy-1.20.3.dist-info", "numpy.libs"}) + self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"]) + expected_files = self.test_data_files.union({"numpy", "numpy-1.20.3.dist-info", "numpy.libs"}) output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py index ca99c058c..77bfb73ef 100644 --- a/tests/unit/test_builder.py +++ b/tests/unit/test_builder.py @@ -4,7 +4,7 @@ from parameterized import parameterized, param from aws_lambda_builders.builder import LambdaBuilder -from aws_lambda_builders.workflow import BuildInSourceSupport, Capability, BaseWorkflow +from aws_lambda_builders.workflow import BuildDirectory, BuildInSourceSupport, Capability, BaseWorkflow from aws_lambda_builders.registry import DEFAULT_REGISTRY @@ -71,7 +71,7 @@ class MyWorkflow(BaseWorkflow): CAPABILITY = Capability( language=self.lang, dependency_manager=self.lang_framework, application_framework=self.app_framework ) - BUILD_IN_SOURCE_BY_DEFAULT = False + DEFAULT_BUILD_DIR = BuildDirectory.SCRATCH BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.OPTIONALLY_SUPPORTED def __init__( diff --git a/tests/unit/test_validator.py b/tests/unit/test_validator.py index 8e889c2f3..083774aa6 100644 --- a/tests/unit/test_validator.py +++ b/tests/unit/test_validator.py @@ -22,7 +22,7 @@ def test_validate_with_unsupported_runtime(self): validator.validate("/usr/bin/unknown_runtime") def test_validate_with_runtime_and_incompatible_architecture(self): - runtime_list = ["python3.6", "python3.7"] + runtime_list = ["python3.7"] for runtime in runtime_list: validator = RuntimeValidator(runtime=runtime, architecture="arm64") with self.assertRaises(UnsupportedArchitectureError): diff --git a/tests/unit/test_workflow.py b/tests/unit/test_workflow.py index 523ea6731..878387b6e 100644 --- a/tests/unit/test_workflow.py +++ b/tests/unit/test_workflow.py @@ -12,7 +12,7 @@ from aws_lambda_builders.binary_path import BinaryPath from aws_lambda_builders.validator import RuntimeValidator -from aws_lambda_builders.workflow import BaseWorkflow, BuildInSourceSupport, Capability +from aws_lambda_builders.workflow import BaseWorkflow, BuildDirectory, BuildInSourceSupport, Capability from aws_lambda_builders.registry import get_workflow, DEFAULT_REGISTRY from aws_lambda_builders.exceptions import ( WorkflowFailedError, @@ -41,7 +41,7 @@ def test_must_register_one_workflow(self): class TestWorkflow(BaseWorkflow): NAME = "TestWorkflow" CAPABILITY = self.CAPABILITY1 - BUILD_IN_SOURCE_BY_DEFAULT = False + DEFAULT_BUILD_DIR = BuildDirectory.SCRATCH BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.OPTIONALLY_SUPPORTED result_cls = get_workflow(self.CAPABILITY1) @@ -52,13 +52,13 @@ def test_must_register_two_workflows(self): class TestWorkflow1(BaseWorkflow): NAME = "TestWorkflow" CAPABILITY = self.CAPABILITY1 - BUILD_IN_SOURCE_BY_DEFAULT = False + DEFAULT_BUILD_DIR = BuildDirectory.SCRATCH BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.OPTIONALLY_SUPPORTED class TestWorkflow2(BaseWorkflow): NAME = "TestWorkflow2" CAPABILITY = self.CAPABILITY2 - BUILD_IN_SOURCE_BY_DEFAULT = False + DEFAULT_BUILD_DIR = BuildDirectory.SCRATCH BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.OPTIONALLY_SUPPORTED self.assertEqual(len(DEFAULT_REGISTRY), 2) @@ -71,7 +71,7 @@ def test_must_fail_if_name_not_present(self): class TestWorkflow1(BaseWorkflow): CAPABILITY = self.CAPABILITY1 - BUILD_IN_SOURCE_BY_DEFAULT = False + DEFAULT_BUILD_DIR = BuildDirectory.SCRATCH BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.OPTIONALLY_SUPPORTED self.assertEqual(len(DEFAULT_REGISTRY), 0) @@ -83,7 +83,7 @@ def test_must_fail_if_capabilities_not_present(self): class TestWorkflow1(BaseWorkflow): NAME = "somename" - BUILD_IN_SOURCE_BY_DEFAULT = False + DEFAULT_BUILD_DIR = BuildDirectory.SCRATCH BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.OPTIONALLY_SUPPORTED self.assertEqual(len(DEFAULT_REGISTRY), 0) @@ -96,7 +96,7 @@ def test_must_fail_if_capabilities_is_wrong_type(self): class TestWorkflow1(BaseWorkflow): NAME = "somename" CAPABILITY = "wrong data type" - BUILD_IN_SOURCE_BY_DEFAULT = False + DEFAULT_BUILD_DIR = BuildDirectory.SCRATCH BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.OPTIONALLY_SUPPORTED self.assertEqual(len(DEFAULT_REGISTRY), 0) @@ -104,24 +104,40 @@ class TestWorkflow1(BaseWorkflow): @parameterized.expand( [ - (None, BuildInSourceSupport.NOT_SUPPORTED), # default not defined - (False, None), # support not defined - (False, BuildInSourceSupport.EXCLUSIVELY_SUPPORTED), # default incompatible with support - (True, False), # support not instance of enum + (None,), # support not defined + (False,), # support not instance of enum ] ) - def test_must_fail_if_build_in_source_variables_invalid(self, build_in_source_default, build_in_source_support): + def test_must_fail_if_build_in_source_support_invalid(self, build_in_source_support): with self.assertRaises(ValueError) as ctx: class TestWorkflow1(BaseWorkflow): NAME = "somename" CAPABILITY = self.CAPABILITY1 - BUILD_IN_SOURCE_BY_DEFAULT = build_in_source_default + DEFAULT_BUILD_DIR = BuildDirectory.SCRATCH BUILD_IN_SOURCE_SUPPORT = build_in_source_support self.assertEqual(len(DEFAULT_REGISTRY), 0) + @parameterized.expand( + [ + (None,), # default build dir not defined + ("some_dir",), # default build dir not instance of enum + ] + ) + def test_must_fail_if_default_build_dir_invalid(self, default_build_dir): + + with self.assertRaises(ValueError) as ctx: + + class TestWorkflow1(BaseWorkflow): + NAME = "somename" + CAPABILITY = self.CAPABILITY1 + DEFAULT_BUILD_DIR = default_build_dir + BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.NOT_SUPPORTED + + self.assertEqual(len(DEFAULT_REGISTRY), 0) + class TestBaseWorkflow_init(TestCase): class MyWorkflow(BaseWorkflow): @@ -130,7 +146,7 @@ class MyWorkflow(BaseWorkflow): CAPABILITY = Capability( language="test", dependency_manager="testframework", application_framework="appframework" ) - BUILD_IN_SOURCE_BY_DEFAULT = False + DEFAULT_BUILD_DIR = BuildDirectory.SCRATCH BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.OPTIONALLY_SUPPORTED def test_must_initialize_variables(self): @@ -143,7 +159,6 @@ def test_must_initialize_variables(self): executable_search_paths=[str(sys.executable)], optimizations={"a": "b"}, options={"c": "d"}, - build_in_source=True, ) self.assertEqual(self.work.source_dir, "source_dir") @@ -155,7 +170,6 @@ def test_must_initialize_variables(self): self.assertEqual(self.work.optimizations, {"a": "b"}) self.assertEqual(self.work.options, {"c": "d"}) self.assertEqual(self.work.architecture, "x86_64") - self.assertTrue(self.work.build_in_source) class TestBaseWorkflow_is_supported(TestCase): @@ -165,7 +179,7 @@ class MyWorkflow(BaseWorkflow): CAPABILITY = Capability( language="test", dependency_manager="testframework", application_framework="appframework" ) - BUILD_IN_SOURCE_BY_DEFAULT = False + DEFAULT_BUILD_DIR = BuildDirectory.SCRATCH BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.OPTIONALLY_SUPPORTED def setUp(self): @@ -213,7 +227,7 @@ class MyWorkflow(BaseWorkflow): CAPABILITY = Capability( language="test", dependency_manager="testframework", application_framework="appframework" ) - BUILD_IN_SOURCE_BY_DEFAULT = False + DEFAULT_BUILD_DIR = BuildDirectory.SCRATCH BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.OPTIONALLY_SUPPORTED def setUp(self): @@ -400,7 +414,7 @@ class MyWorkflow(BaseWorkflow): CAPABILITY = Capability( language="test", dependency_manager="testframework", application_framework="appframework" ) - BUILD_IN_SOURCE_BY_DEFAULT = False + DEFAULT_BUILD_DIR = BuildDirectory.SCRATCH BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.OPTIONALLY_SUPPORTED def setUp(self): @@ -440,15 +454,47 @@ def test_must_pretty_print_workflow_info(self): class TestBaseWorkflow_build_in_source(TestCase): - @parameterized.expand([(True,), (False,)]) - def test_must_use_correct_default_value(self, default_value): + def test_must_use_source_directory_if_building_in_source(self): class MyWorkflow(BaseWorkflow): __TESTING__ = True NAME = "MyWorkflow" CAPABILITY = Capability( language="test", dependency_manager="testframework", application_framework="appframework" ) - BUILD_IN_SOURCE_BY_DEFAULT = default_value + DEFAULT_BUILD_DIR = BuildDirectory.SCRATCH + BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.OPTIONALLY_SUPPORTED + + source_dir = "source_dir" + + self.work = MyWorkflow( + source_dir, + "artifacts_dir", + "scratch_dir", + "manifest_path", + runtime="runtime", + executable_search_paths=[str(sys.executable)], + optimizations={"a": "b"}, + options={"c": "d"}, + build_in_source=True, + ) + + self.assertEqual(self.work.build_dir, source_dir) + + @parameterized.expand( + [ + (BuildDirectory.SCRATCH, "scratch_dir"), + (BuildDirectory.SOURCE, "source_dir"), + (BuildDirectory.ARTIFACTS, "artifacts_dir"), + ] + ) + def test_must_use_correct_default_value(self, default_build_dir, expected_build_dir): + class MyWorkflow(BaseWorkflow): + __TESTING__ = True + NAME = "MyWorkflow" + CAPABILITY = Capability( + language="test", dependency_manager="testframework", application_framework="appframework" + ) + DEFAULT_BUILD_DIR = default_build_dir BUILD_IN_SOURCE_SUPPORT = BuildInSourceSupport.OPTIONALLY_SUPPORTED self.work = MyWorkflow( @@ -462,21 +508,32 @@ class MyWorkflow(BaseWorkflow): options={"c": "d"}, ) - self.assertEqual(self.work.build_in_source, default_value) + self.assertEqual(self.work.build_dir, expected_build_dir) @parameterized.expand( [ - (True, False, BuildInSourceSupport.NOT_SUPPORTED), # want to build in source but it's not supported ( - False, True, + BuildInSourceSupport.NOT_SUPPORTED, + BuildDirectory.SCRATCH, + "scratch_dir", + ), # want to build in source but it's not supported + ( + False, BuildInSourceSupport.EXCLUSIVELY_SUPPORTED, + BuildDirectory.SOURCE, + "source_dir", ), # don't want to build in source but workflow requires it - ("unsupported", False, BuildInSourceSupport.OPTIONALLY_SUPPORTED), # unsupported value passed in + ( + "unsupported", + BuildInSourceSupport.OPTIONALLY_SUPPORTED, + BuildDirectory.ARTIFACTS, + "artifacts_dir", + ), # unsupported value passed in ] ) def test_must_use_default_if_unsupported_value_is_provided( - self, build_in_source_value, build_in_source_default, build_in_source_support + self, build_in_source_value, build_in_source_support, default_build_dir, expected_build_dir ): class MyWorkflow(BaseWorkflow): __TESTING__ = True @@ -484,7 +541,7 @@ class MyWorkflow(BaseWorkflow): CAPABILITY = Capability( language="test", dependency_manager="testframework", application_framework="appframework" ) - BUILD_IN_SOURCE_BY_DEFAULT = build_in_source_default + DEFAULT_BUILD_DIR = default_build_dir BUILD_IN_SOURCE_SUPPORT = build_in_source_support self.work = MyWorkflow( @@ -499,4 +556,4 @@ class MyWorkflow(BaseWorkflow): build_in_source=build_in_source_value, ) - self.assertEqual(self.work.build_in_source, build_in_source_default) + self.assertEqual(self.work.build_dir, expected_build_dir) diff --git a/tests/unit/workflows/nodejs_npm_esbuild/test_actions.py b/tests/unit/workflows/nodejs_npm_esbuild/test_actions.py index d08db7d4b..c8dfd722b 100644 --- a/tests/unit/workflows/nodejs_npm_esbuild/test_actions.py +++ b/tests/unit/workflows/nodejs_npm_esbuild/test_actions.py @@ -5,7 +5,11 @@ from parameterized import parameterized from aws_lambda_builders.actions import ActionFailedError -from aws_lambda_builders.workflows.nodejs_npm_esbuild.actions import EsbuildBundleAction, EsbuildCheckVersionAction +from aws_lambda_builders.workflows.nodejs_npm_esbuild.actions import ( + EsbuildBundleAction, + check_minimum_esbuild_version, + MINIMUM_VERSION_FOR_EXTERNAL, +) class TestEsbuildBundleAction(TestCase): @@ -14,6 +18,7 @@ class TestEsbuildBundleAction(TestCase): def setUp(self, OSUtilMock, SubprocessEsbuildMock): self.osutils = OSUtilMock.return_value self.subprocess_esbuild = SubprocessEsbuildMock.return_value + self.subprocess_esbuild.run.return_value = MINIMUM_VERSION_FOR_EXTERNAL self.osutils.joinpath.side_effect = lambda a, b: "{}/{}".format(a, b) self.osutils.file_exists.side_effect = [True, True] @@ -292,15 +297,51 @@ def test_includes_building_with_external_dependencies(self, osutils_mock): cwd="source", ) + @patch("aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils") + @patch("aws_lambda_builders.workflows.nodejs_npm_esbuild.esbuild.SubprocessEsbuild") + def test_building_with_external_dependencies_in_bundler_config_fails_if_esbuild_version_less_than_minimum( + self, osutils_mock, subprocess_esbuild_mock + ): + subprocess_esbuild_mock.run.return_value = "0.14.12" + action = EsbuildBundleAction( + "source", + "artifacts", + {"entry_points": ["x.js", "y.js"], "target": "node14", "external": "./node_modules/*"}, + osutils_mock, + subprocess_esbuild_mock, + "package.json", + ) + with self.assertRaises(ActionFailedError): + action.execute() + + @patch("aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils") + @patch("aws_lambda_builders.workflows.nodejs_npm_esbuild.esbuild.SubprocessEsbuild") + def test_building_with_skip_deps_fails_if_esbuild_version_less_than_minimum( + self, osutils_mock, subprocess_esbuild_mock + ): + subprocess_esbuild_mock.run.return_value = "0.14.12" + action = EsbuildBundleAction( + "source", + "artifacts", + {"entry_points": ["x.js"]}, + osutils_mock, + subprocess_esbuild_mock, + "package.json", + True, + ) + with self.assertRaises(ActionFailedError): + action.execute() + -class TestEsbuildVersionCheckerAction(TestCase): +class TestEsbuildVersionChecker(TestCase): @parameterized.expand(["0.14.0", "0.0.0", "0.14.12"]) def test_outdated_esbuild_versions(self, version): subprocess_esbuild = Mock() subprocess_esbuild.run.return_value = version - action = EsbuildCheckVersionAction("scratch", subprocess_esbuild) with self.assertRaises(ActionFailedError) as content: - action.execute() + check_minimum_esbuild_version( + minimum_version_required="0.14.13", working_directory="scratch", subprocess_esbuild=subprocess_esbuild + ) self.assertEqual( str(content.exception), f"Unsupported esbuild version. To use a dependency layer, the esbuild version " @@ -311,9 +352,10 @@ def test_outdated_esbuild_versions(self, version): def test_invalid_esbuild_versions(self, version): subprocess_esbuild = Mock() subprocess_esbuild.run.return_value = version - action = EsbuildCheckVersionAction("scratch", subprocess_esbuild) with self.assertRaises(ActionFailedError) as content: - action.execute() + check_minimum_esbuild_version( + minimum_version_required="0.14.13", working_directory="scratch", subprocess_esbuild=subprocess_esbuild + ) self.assertEqual( str(content.exception), "Unable to parse esbuild version: invalid literal for int() with base 10: 'a'" ) @@ -322,8 +364,9 @@ def test_invalid_esbuild_versions(self, version): def test_valid_esbuild_versions(self, version): subprocess_esbuild = Mock() subprocess_esbuild.run.return_value = version - action = EsbuildCheckVersionAction("scratch", subprocess_esbuild) try: - action.execute() + check_minimum_esbuild_version( + minimum_version_required="0.14.13", working_directory="scratch", subprocess_esbuild=subprocess_esbuild + ) except ActionFailedError: self.fail("Encountered an unexpected exception.") diff --git a/tests/unit/workflows/nodejs_npm_esbuild/test_workflow.py b/tests/unit/workflows/nodejs_npm_esbuild/test_workflow.py index 91d2ee500..70ccb5a73 100644 --- a/tests/unit/workflows/nodejs_npm_esbuild/test_workflow.py +++ b/tests/unit/workflows/nodejs_npm_esbuild/test_workflow.py @@ -13,8 +13,9 @@ from aws_lambda_builders.architecture import ARM64 from aws_lambda_builders.workflows.nodejs_npm.actions import NodejsNpmInstallAction, NodejsNpmCIAction from aws_lambda_builders.workflows.nodejs_npm_esbuild import NodejsNpmEsbuildWorkflow -from aws_lambda_builders.workflows.nodejs_npm_esbuild.actions import EsbuildBundleAction, EsbuildCheckVersionAction +from aws_lambda_builders.workflows.nodejs_npm_esbuild.actions import EsbuildBundleAction from aws_lambda_builders.workflows.nodejs_npm_esbuild.esbuild import SubprocessEsbuild +from aws_lambda_builders.workflows.nodejs_npm_esbuild.exceptions import EsbuildExecutionError class FakePopen: @@ -244,11 +245,10 @@ def test_workflow_sets_up_esbuild_actions_without_download_dependencies_with_dep experimental_flags=[], ) - self.assertEqual(len(workflow.actions), 4) + self.assertEqual(len(workflow.actions), 3) self.assertIsInstance(workflow.actions[0], CopySourceAction) self.assertIsInstance(workflow.actions[1], LinkSourceAction) - self.assertIsInstance(workflow.actions[2], EsbuildCheckVersionAction) - self.assertIsInstance(workflow.actions[3], EsbuildBundleAction) + self.assertIsInstance(workflow.actions[2], EsbuildBundleAction) def test_workflow_sets_up_esbuild_actions_with_download_dependencies_and_dependencies_dir(self): @@ -265,14 +265,13 @@ def test_workflow_sets_up_esbuild_actions_with_download_dependencies_and_depende experimental_flags=[], ) - self.assertEqual(len(workflow.actions), 6) + self.assertEqual(len(workflow.actions), 5) self.assertIsInstance(workflow.actions[0], CopySourceAction) self.assertIsInstance(workflow.actions[1], NodejsNpmInstallAction) - self.assertIsInstance(workflow.actions[2], CleanUpAction) - self.assertIsInstance(workflow.actions[3], EsbuildBundleAction) + self.assertIsInstance(workflow.actions[2], EsbuildBundleAction) + self.assertIsInstance(workflow.actions[3], CleanUpAction) self.assertIsInstance(workflow.actions[4], MoveDependenciesAction) - self.assertIsInstance(workflow.actions[5], LinkSourceAction) def test_workflow_sets_up_esbuild_actions_with_download_dependencies_and_dependencies_dir_no_combine_deps(self): workflow = NodejsNpmEsbuildWorkflow( @@ -287,14 +286,13 @@ def test_workflow_sets_up_esbuild_actions_with_download_dependencies_and_depende experimental_flags=[], ) - self.assertEqual(len(workflow.actions), 6) + self.assertEqual(len(workflow.actions), 5) self.assertIsInstance(workflow.actions[0], CopySourceAction) self.assertIsInstance(workflow.actions[1], NodejsNpmInstallAction) - self.assertIsInstance(workflow.actions[2], CleanUpAction) - self.assertIsInstance(workflow.actions[3], EsbuildCheckVersionAction) - self.assertIsInstance(workflow.actions[4], EsbuildBundleAction) - self.assertIsInstance(workflow.actions[5], MoveDependenciesAction) + self.assertIsInstance(workflow.actions[2], EsbuildBundleAction) + self.assertIsInstance(workflow.actions[3], CleanUpAction) + self.assertIsInstance(workflow.actions[4], MoveDependenciesAction) @patch("aws_lambda_builders.workflows.nodejs_npm_esbuild.workflow.NodejsNpmWorkflow") def test_workflow_uses_production_npm_version(self, get_workflow_mock): @@ -314,4 +312,51 @@ def test_workflow_uses_production_npm_version(self, get_workflow_mock): self.assertIsInstance(workflow.actions[0], CopySourceAction) self.assertIsInstance(workflow.actions[2], EsbuildBundleAction) - get_workflow_mock.get_install_action.assert_called_with("source", "scratch_dir", ANY, ANY, None) + get_workflow_mock.get_install_action.assert_called_with( + source_dir="source", install_dir="scratch_dir", subprocess_npm=ANY, osutils=ANY, build_options=None + ) + + @patch("aws_lambda_builders.workflows.nodejs_npm_esbuild.workflow.SubprocessNpm") + @patch("aws_lambda_builders.workflows.nodejs_npm_esbuild.workflow.OSUtils") + def test_manifest_not_found(self, osutils_mock, subprocess_npm_mock): + osutils_mock.file_exists.return_value = False + + workflow = NodejsNpmEsbuildWorkflow( + "source", + "artifacts", + "scratch_dir", + "manifest", + osutils=osutils_mock, + ) + + self.assertEqual(len(workflow.actions), 1) + self.assertIsInstance(workflow.actions[0], EsbuildBundleAction) + + def test_no_download_dependencies_and_no_dependencies_dir_fails(self): + with self.assertRaises(EsbuildExecutionError): + NodejsNpmEsbuildWorkflow( + "source", + "artifacts", + "scratch_dir", + "manifest", + osutils=self.osutils, + download_dependencies=False, + ) + + def test_build_in_source(self): + source_dir = "source" + workflow = NodejsNpmEsbuildWorkflow( + source_dir=source_dir, + artifacts_dir="artifacts", + scratch_dir="scratch_dir", + manifest_path="manifest", + osutils=self.osutils, + build_in_source=True, + ) + + self.assertEqual(len(workflow.actions), 2) + + self.assertIsInstance(workflow.actions[0], NodejsNpmInstallAction) + self.assertEqual(workflow.actions[0].install_dir, source_dir) + self.assertIsInstance(workflow.actions[1], EsbuildBundleAction) + self.assertEqual(workflow.actions[1]._working_directory, source_dir) diff --git a/tests/unit/workflows/python_pip/test_packager.py b/tests/unit/workflows/python_pip/test_packager.py index 7eaeb30b1..f27686af8 100644 --- a/tests/unit/workflows/python_pip/test_packager.py +++ b/tests/unit/workflows/python_pip/test_packager.py @@ -90,9 +90,6 @@ def popen(self, *args, **kwargs): class TestGetLambdaAbi(object): - def test_get_lambda_abi_python36(self): - assert "cp36m" == get_lambda_abi("python3.6") - def test_get_lambda_abi_python37(self): assert "cp37m" == get_lambda_abi("python3.7") @@ -157,7 +154,7 @@ def test_same_pkg_is_eq(self): def test_pkg_is_eq_to_similar_pkg(self): pure_pkg = Package("", "foobar-1.0-py3-none-any.whl") - plat_pkg = Package("", "foobar-1.0-py3-py36m-manylinux1_x86_64.whl") + plat_pkg = Package("", "foobar-1.0-py3-py39-manylinux1_x86_64.whl") assert pure_pkg == plat_pkg def test_pkg_is_not_equal_to_different_type(self): diff --git a/tests/unit/workflows/python_pip/test_validator.py b/tests/unit/workflows/python_pip/test_validator.py index 6d5abfcb7..cf9f7ac23 100644 --- a/tests/unit/workflows/python_pip/test_validator.py +++ b/tests/unit/workflows/python_pip/test_validator.py @@ -35,7 +35,7 @@ def test_runtime_validate_mismatch_version_runtime(self): with mock.patch("subprocess.Popen") as mock_subprocess: mock_subprocess.return_value = MockSubProcess(1) with self.assertRaises(MisMatchRuntimeError): - self.validator.validate(runtime_path="/usr/bin/python3.6") + self.validator.validate(runtime_path="/usr/bin/python3.9") self.assertTrue(mock_subprocess.call_count, 1) def test_python_command(self): @@ -46,7 +46,6 @@ def test_python_command(self): @parameterized.expand( [ - ("python3.6", "arm64"), ("python3.7", "arm64"), ] )