From ae2e3e6a3d4ce82ac0d3c9faad713c57e3190d6f Mon Sep 17 00:00:00 2001 From: matt-lumer <4656758+matt-lumer@users.noreply.github.com> Date: Tue, 9 Jul 2019 11:27:11 -0400 Subject: [PATCH 1/9] Ignore .git files This causes issues on Windows machines due to file handles not being released --- aws_lambda_builders/workflows/java_maven/workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_builders/workflows/java_maven/workflow.py b/aws_lambda_builders/workflows/java_maven/workflow.py index 9960fb938..f26b02d12 100644 --- a/aws_lambda_builders/workflows/java_maven/workflow.py +++ b/aws_lambda_builders/workflows/java_maven/workflow.py @@ -20,7 +20,7 @@ class JavaMavenWorkflow(BaseWorkflow): dependency_manager="maven", application_framework=None) - EXCLUDED_FILES = (".aws-sam") + EXCLUDED_FILES = (".aws-sam", ".git") def __init__(self, source_dir, From 432481c3f7392effe9321bf29d3465a118a0da3b Mon Sep 17 00:00:00 2001 From: matt-lumer <4656758+matt-lumer@users.noreply.github.com> Date: Tue, 3 Sep 2019 13:28:43 -0400 Subject: [PATCH 2/9] unit test for excluded files --- tests/unit/workflows/java_maven/test_workflow.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/unit/workflows/java_maven/test_workflow.py b/tests/unit/workflows/java_maven/test_workflow.py index b4f89fbfc..6897c075d 100644 --- a/tests/unit/workflows/java_maven/test_workflow.py +++ b/tests/unit/workflows/java_maven/test_workflow.py @@ -20,7 +20,7 @@ def test_workflow_sets_up_maven_actions(self): self.assertEqual(len(workflow.actions), 4) self.assertIsInstance(workflow.actions[0], CopySourceAction) - + self.assertIsInstance(workflow.actions[1], JavaMavenBuildAction) self.assertIsInstance(workflow.actions[2], JavaMavenCopyDependencyAction) @@ -42,3 +42,12 @@ def test_workflow_sets_up_validators(self): self.assertEqual(len(validators), 1) self.assertIsInstance(validators[0], MavenValidator) + + def test_workflow_excluded_files(self): + workflow = JavaMavenWorkflow("source", "artifacts", "scratch_dir", "manifest") + + self.assertIsInstance(workflow.actions[0], CopySourceAction) + + self.assertEqual(".aws_sam", workflow.actions[0].excludes[0]) + + self.assertEqual(".git", workflow.actions[0].excludes[0]) From d3d3b15b3cad839579908c414b5adb1d0268275a Mon Sep 17 00:00:00 2001 From: matt-lumer <4656758+matt-lumer@users.noreply.github.com> Date: Tue, 3 Sep 2019 13:35:17 -0400 Subject: [PATCH 3/9] fixed .aws-sam name --- tests/unit/workflows/java_maven/test_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/workflows/java_maven/test_workflow.py b/tests/unit/workflows/java_maven/test_workflow.py index 6897c075d..08d28321e 100644 --- a/tests/unit/workflows/java_maven/test_workflow.py +++ b/tests/unit/workflows/java_maven/test_workflow.py @@ -48,6 +48,6 @@ def test_workflow_excluded_files(self): self.assertIsInstance(workflow.actions[0], CopySourceAction) - self.assertEqual(".aws_sam", workflow.actions[0].excludes[0]) + self.assertEqual(".aws-sam", workflow.actions[0].excludes[0]) self.assertEqual(".git", workflow.actions[0].excludes[0]) From 4d4ae8f5a0abb738ad9fae3cf4b3bbb9103a9383 Mon Sep 17 00:00:00 2001 From: matt-lumer <4656758+matt-lumer@users.noreply.github.com> Date: Tue, 3 Sep 2019 13:41:45 -0400 Subject: [PATCH 4/9] white spaces --- tests/unit/workflows/java_maven/test_workflow.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/workflows/java_maven/test_workflow.py b/tests/unit/workflows/java_maven/test_workflow.py index 08d28321e..b6de98fc8 100644 --- a/tests/unit/workflows/java_maven/test_workflow.py +++ b/tests/unit/workflows/java_maven/test_workflow.py @@ -20,7 +20,7 @@ def test_workflow_sets_up_maven_actions(self): self.assertEqual(len(workflow.actions), 4) self.assertIsInstance(workflow.actions[0], CopySourceAction) - + self.assertIsInstance(workflow.actions[1], JavaMavenBuildAction) self.assertIsInstance(workflow.actions[2], JavaMavenCopyDependencyAction) @@ -47,7 +47,7 @@ def test_workflow_excluded_files(self): workflow = JavaMavenWorkflow("source", "artifacts", "scratch_dir", "manifest") self.assertIsInstance(workflow.actions[0], CopySourceAction) - + self.assertEqual(".aws-sam", workflow.actions[0].excludes[0]) - + self.assertEqual(".git", workflow.actions[0].excludes[0]) From 5fa54d780a0597a8d46ca9e2845d108f959cb07f Mon Sep 17 00:00:00 2001 From: matt-lumer <4656758+matt-lumer@users.noreply.github.com> Date: Tue, 3 Sep 2019 13:47:44 -0400 Subject: [PATCH 5/9] typo --- tests/unit/workflows/java_maven/test_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/workflows/java_maven/test_workflow.py b/tests/unit/workflows/java_maven/test_workflow.py index b6de98fc8..e2cdacc52 100644 --- a/tests/unit/workflows/java_maven/test_workflow.py +++ b/tests/unit/workflows/java_maven/test_workflow.py @@ -50,4 +50,4 @@ def test_workflow_excluded_files(self): self.assertEqual(".aws-sam", workflow.actions[0].excludes[0]) - self.assertEqual(".git", workflow.actions[0].excludes[0]) + self.assertEqual(".git", workflow.actions[0].excludes[1]) From ffdb37e6451c8f9143a5609df061929bb11700da Mon Sep 17 00:00:00 2001 From: khamaileon Date: Mon, 9 Sep 2019 22:49:07 +0200 Subject: [PATCH 6/9] fix: Add missing comma in python pip workflow (#128) --- aws_lambda_builders/workflows/python_pip/workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_builders/workflows/python_pip/workflow.py b/aws_lambda_builders/workflows/python_pip/workflow.py index a3225ce66..5d5cee0c7 100644 --- a/aws_lambda_builders/workflows/python_pip/workflow.py +++ b/aws_lambda_builders/workflows/python_pip/workflow.py @@ -39,7 +39,7 @@ class PythonPipWorkflow(BaseWorkflow): ".python-version", # mypy, Pyre - ".mypy_cache", ".dmypy.json", ".pyre" + ".mypy_cache", ".dmypy.json", ".pyre", # environments ".env", ".venv", "venv", "venv.bak", "env.bak", "ENV", From fc2508d68c053ca501490f015371d4fab4988d88 Mon Sep 17 00:00:00 2001 From: Sriram Madapusi Vasudevan <3770774+TheSriram@users.noreply.github.com> Date: Thu, 17 Oct 2019 07:13:44 -0700 Subject: [PATCH 7/9] chore: bump version to 0.5.0 (#133) --- aws_lambda_builders/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_builders/__init__.py b/aws_lambda_builders/__init__.py index a0adf692b..0b9662f4c 100644 --- a/aws_lambda_builders/__init__.py +++ b/aws_lambda_builders/__init__.py @@ -1,5 +1,5 @@ """ AWS Lambda Builder Library """ -__version__ = '0.4.0' +__version__ = '0.5.0' RPC_PROTOCOL_VERSION = "0.3" From 9133418ce1bec51fe9ee8d9abdce483c826339e2 Mon Sep 17 00:00:00 2001 From: Sriram Madapusi Vasudevan Date: Wed, 16 Oct 2019 15:55:27 -0700 Subject: [PATCH 8/9] fix: change the pip runner string based on a pip version check - if pip version is less than 19.3, choose to run main directly as its a function, whereas with pip versions starting with 19.3, execute the function `main` inside the pip.main module --- .../workflows/python_pip/compat.py | 17 +++++++++++++++++ .../workflows/python_pip/packager.py | 5 +++-- .../unit/workflows/python_pip/test_packager.py | 17 ++++++++++++++--- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/aws_lambda_builders/workflows/python_pip/compat.py b/aws_lambda_builders/workflows/python_pip/compat.py index ff35cbabe..912856d95 100644 --- a/aws_lambda_builders/workflows/python_pip/compat.py +++ b/aws_lambda_builders/workflows/python_pip/compat.py @@ -21,6 +21,23 @@ def pip_import_string(python_exe): return 'from pip._internal import main' +def pip_runner_string(python_exe): + os_utils = OSUtils() + cmd = [ + python_exe, + "-c", + "import pip; assert (int(pip.__version__.split('.')[0]) <= 19 and int(pip.__version__.split('.')[1]) < 3)" + ] + p = os_utils.popen(cmd, stdout=os_utils.pipe, stderr=os_utils.pipe) + p.communicate() + # Pip changed main to be a module instead of a function from 19.3 + # and added a separate main function within the main module. + if p.returncode == 0: + return 'import sys; %s; sys.exit(main(%s))' + else: + return 'import sys; %s; sys.exit(main.main(%s))' + + if os.name == 'nt': # windows # This is the actual patch used on windows to prevent distutils from diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index a89ae626e..0bdc3f731 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -9,7 +9,7 @@ from email.parser import FeedParser -from .compat import pip_import_string +from .compat import pip_import_string, pip_runner_string from .compat import pip_no_compile_c_env_vars from .compat import pip_no_compile_c_shim from .utils import OSUtils @@ -553,6 +553,7 @@ def __init__(self, osutils=None, python_exe=None, import_string=None): self.python_exe = python_exe if import_string is None: import_string = pip_import_string(python_exe=self.python_exe) + self.run_pip_string = pip_runner_string(python_exe=self.python_exe) self._import_string = import_string def main(self, args, env_vars=None, shim=None): @@ -561,7 +562,7 @@ def main(self, args, env_vars=None, shim=None): if shim is None: shim = '' run_pip = ( - 'import sys; %s; sys.exit(main(%s))' + self.run_pip_string ) % (self._import_string, args) exec_string = '%s%s' % (shim, run_pip) invoke_pip = [self.python_exe, '-c', exec_string] diff --git a/tests/unit/workflows/python_pip/test_packager.py b/tests/unit/workflows/python_pip/test_packager.py index 937f52aad..452d23987 100644 --- a/tests/unit/workflows/python_pip/test_packager.py +++ b/tests/unit/workflows/python_pip/test_packager.py @@ -1,5 +1,6 @@ import sys from collections import namedtuple +from unittest import TestCase import mock import pytest @@ -70,7 +71,7 @@ def osutils(): class FakePopen(object): def __init__(self, rc, out, err): - self.returncode = 0 + self.returncode = rc self._out = out self._err = err @@ -303,14 +304,24 @@ def test_inject_unknown_error_if_no_stderr(self, pip_factory): assert str(einfo.value) == 'Unknown error' -class TestSubprocessPip(object): +class TestSubprocessPip(TestCase): def test_does_use_custom_pip_import_string(self): fake_osutils = FakePopenOSUtils([FakePopen(0, '', '')]) expected_import_statement = 'foobarbaz' pip = SubprocessPip(osutils=fake_osutils, - import_string=expected_import_statement) + import_string=expected_import_statement, + python_exe=sys.executable) pip.main(['--version']) pip_execution_string = fake_osutils.popens[0][0][0][2] import_statement = pip_execution_string.split(';')[1].strip() assert import_statement == expected_import_statement + + def test_check_pip_runner_string_pip(self): + fake_osutils = FakePopenOSUtils([FakePopen(0, '', '')]) + pip = SubprocessPip(osutils=fake_osutils, + python_exe=sys.executable) + pip.main(['--version']) + + pip_runner_string = fake_osutils.popens[0][0][0][2].split(";")[-1:][0] + self.assertIn("main", pip_runner_string) From c1f09ba50edc4cc0c49779cbc9c472587165fecd Mon Sep 17 00:00:00 2001 From: Sriram Madapusi Vasudevan Date: Thu, 17 Oct 2019 12:33:50 -0700 Subject: [PATCH 9/9] fix: adopt same methodology as chalice for determining pip import string --- .../workflows/python_pip/compat.py | 33 +++++++------------ .../workflows/python_pip/packager.py | 5 ++- 2 files changed, 14 insertions(+), 24 deletions(-) diff --git a/aws_lambda_builders/workflows/python_pip/compat.py b/aws_lambda_builders/workflows/python_pip/compat.py index 912856d95..2eb885f20 100644 --- a/aws_lambda_builders/workflows/python_pip/compat.py +++ b/aws_lambda_builders/workflows/python_pip/compat.py @@ -8,36 +8,27 @@ def pip_import_string(python_exe): cmd = [ python_exe, "-c", - "import pip; assert int(pip.__version__.split('.')[0]) <= 9" + "import pip; print(pip.__version__)" ] - p = os_utils.popen(cmd,stdout=os_utils.pipe, stderr=os_utils.pipe) - p.communicate() + p = os_utils.popen(cmd, stdout=os_utils.pipe, stderr=os_utils.pipe) + stdout, stderr = p.communicate() + pip_version = stdout.decode('utf-8').strip() + pip_major_version = int(pip_version.split('.')[0]) + pip_minor_version = int(pip_version.split('.')[1]) + # Pip moved its internals to an _internal module in version 10. # In order to be compatible with version 9 which has it at at the # top level we need to figure out the correct import path here. - if p.returncode == 0: + if pip_major_version == 9: return 'from pip import main' + # Pip changed their import structure again in 19.3 + # https://github.com/pypa/pip/commit/09fd200 + elif pip_major_version >= 19 and pip_minor_version >= 3: + return 'from pip._internal.main import main' else: return 'from pip._internal import main' -def pip_runner_string(python_exe): - os_utils = OSUtils() - cmd = [ - python_exe, - "-c", - "import pip; assert (int(pip.__version__.split('.')[0]) <= 19 and int(pip.__version__.split('.')[1]) < 3)" - ] - p = os_utils.popen(cmd, stdout=os_utils.pipe, stderr=os_utils.pipe) - p.communicate() - # Pip changed main to be a module instead of a function from 19.3 - # and added a separate main function within the main module. - if p.returncode == 0: - return 'import sys; %s; sys.exit(main(%s))' - else: - return 'import sys; %s; sys.exit(main.main(%s))' - - if os.name == 'nt': # windows # This is the actual patch used on windows to prevent distutils from diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index 0bdc3f731..a89ae626e 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -9,7 +9,7 @@ from email.parser import FeedParser -from .compat import pip_import_string, pip_runner_string +from .compat import pip_import_string from .compat import pip_no_compile_c_env_vars from .compat import pip_no_compile_c_shim from .utils import OSUtils @@ -553,7 +553,6 @@ def __init__(self, osutils=None, python_exe=None, import_string=None): self.python_exe = python_exe if import_string is None: import_string = pip_import_string(python_exe=self.python_exe) - self.run_pip_string = pip_runner_string(python_exe=self.python_exe) self._import_string = import_string def main(self, args, env_vars=None, shim=None): @@ -562,7 +561,7 @@ def main(self, args, env_vars=None, shim=None): if shim is None: shim = '' run_pip = ( - self.run_pip_string + 'import sys; %s; sys.exit(main(%s))' ) % (self._import_string, args) exec_string = '%s%s' % (shim, run_pip) invoke_pip = [self.python_exe, '-c', exec_string]