Merge pull request #86 from awslabs/develop

Release: v0.1.0

.appveyor.yml

@@ -2,6 +2,9 @@ version: 1.0.{build}
 image: Visual Studio 2017
 
 environment:
+  GOPATH: c:\gopath
+  GOVERSION: 1.11
+  GRADLE_OPTS: -Dorg.gradle.daemon=false
 
   matrix:
 
@@ -16,12 +19,26 @@ install:
 # To run Nodejs workflow integ tests
 - ps: Install-Product node 8.10
 
-- "set PATH=%PYTHON%\\Scripts;%PYTHON%\\bin;%PATH%"
+- "set PATH=%PYTHON%;%PYTHON%\\Scripts;%PYTHON%\\bin;%PATH%"
 - "%PYTHON%\\python.exe -m pip install -r requirements/dev.txt"
 - "%PYTHON%\\python.exe -m pip install -e ."
 - "set PATH=C:\\Ruby25-x64\\bin;%PATH%"
-- "gem install bundler --no-ri --no-rdoc"
+- "gem --version"
+- "gem install bundler -v 1.17.3"
 - "bundler --version"
+- "echo %PATH%"
+
+# setup go
+- rmdir c:\go /s /q
+- "choco install golang"
+- "choco install bzr"
+- "choco install dep"
+- setx PATH "C:\go\bin;C:\gopath\bin;C:\Program Files (x86)\Bazaar\;C:\Program Files\Mercurial;%PATH%;"
+- "go version"
+- "go env"
+
+# setup Gradle
+- "choco install gradle"
 
 test_script:
 - "%PYTHON%\\python.exe -m pytest --cov aws_lambda_builders --cov-report term-missing tests/unit tests/functional"

.gitignore

@@ -383,4 +383,6 @@ $RECYCLE.BIN/
 
 /Dockerfile
 
-# End of https://www.gitignore.io/api/osx,node,macos,linux,python,windows,pycharm,intellij,sublimetext,visualstudiocode
+tests/integration/workflows/go_dep/data/src/*/vendor/*
+
+# End of https://www.gitignore.io/api/osx,node,macos,linux,python,windows,pycharm,intellij,sublimetext,visualstudiocode

.pylintrc

@@ -9,7 +9,7 @@
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.
-ignore=compat.py
+ignore=compat.py, utils.py
 
 # Pickle collected data for later comparisons.
 persistent=yes
@@ -360,4 +360,4 @@ int-import-graph=
 
 # Exceptions that will emit a warning when being caught. Defaults to
 # "Exception"
-overgeneral-exceptions=Exception
+overgeneral-exceptions=Exception

.travis.yml

@@ -15,6 +15,15 @@ install:
   # To run Nodejs workflow integ tests
   - nvm install 8.10.0
   - nvm use 8.10.0
+  # To run Ruby workflow integ tests
+  - rvm install ruby-2.5.3
+  - rvm use ruby-2.5.3
+
+  # Go workflow integ tests require Go 1.11+
+  - eval "$(gimme 1.11.2)"
+  - go version
+
+  - go get -u github.com/golang/dep/cmd/dep
 
   # Install the code requirements
   - make init

DESIGN.md

@@ -39,7 +39,7 @@ If your build action implementation requires 3rd party libraries, here is how yo
 
 Each build action has its own design document. 
 
-* [python-pip](./lambda_builders/actions/python_pip/DESIGN.md)
+* [python-pip](./aws_lambda_builders/workflows/python_pip/DESIGN.md)
 
 
 ### Builders Library

NOTICE

@@ -1,2 +1,6 @@
 AWS Lambda Builders
 Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+The function "which" at aws_lambda_builders/utils.py was copied from https://github.com/python/cpython/blob/3.7/Lib/shutil.py
+SPDX-License-Identifier: Python-2.0
+Copyright 2019 by the Python Software Foundation

aws_lambda_builders/__init__.py

@@ -1,5 +1,5 @@
 """
 AWS Lambda Builder Library
 """
-__version__ = '0.0.5'
-RPC_PROTOCOL_VERSION = "0.1"
+__version__ = '0.1.0'
+RPC_PROTOCOL_VERSION = "0.2"

aws_lambda_builders/__main__.py

@@ -10,10 +10,11 @@
 import json
 import os
 import logging
+import re
 
 from aws_lambda_builders.builder import LambdaBuilder
 from aws_lambda_builders.exceptions import WorkflowNotFoundError, WorkflowUnknownError, WorkflowFailedError
-
+from aws_lambda_builders import RPC_PROTOCOL_VERSION as lambda_builders_protocol_version
 
 log_level = int(os.environ.get("LAMBDA_BUILDERS_LOG_LEVEL", logging.INFO))
 
@@ -24,6 +25,8 @@
 
 LOG = logging.getLogger(__name__)
 
+VERSION_REGEX = re.compile("^([0-9])+.([0-9]+)$")
+
 
 def _success_response(request_id, artifacts_dir):
     return json.dumps({
@@ -46,6 +49,31 @@ def _error_response(request_id, http_status_code, message):
     })
 
 
+def _parse_version(version_string):
+
+    if VERSION_REGEX.match(version_string):
+        return float(version_string)
+    else:
+        ex = "Protocol Version does not match : {}".format(VERSION_REGEX.pattern)
+        LOG.debug(ex)
+        raise ValueError(ex)
+
+
+def version_compatibility_check(version):
+    # The following check is between current protocol version vs version of the protocol
+    # with which aws-lambda-builders is called.
+    # Example:
+    # 0.2 < 0.2 comparison will fail, don't throw a value Error saying incompatible version.
+    # 0.2 < 0.3 comparison will pass, throwing a ValueError
+    # 0.2 < 0.1 comparison will fail, don't throw a value Error saying incompatible version
+
+    if _parse_version(lambda_builders_protocol_version) < version:
+        ex = "Incompatible Protocol Version : {}, " \
+             "Current Protocol Version: {}".format(version, lambda_builders_protocol_version)
+        LOG.error(ex)
+        raise ValueError(ex)
+
+
 def _write_response(response, exit_code):
     sys.stdout.write(response)
     sys.stdout.flush()  # Make sure it is written
@@ -77,11 +105,20 @@ def main():  # pylint: disable=too-many-statements
         response = _error_response(request_id, -32601, "Method unavailable")
         return _write_response(response, 1)
 
+    try:
+        protocol_version = _parse_version(params.get("__protocol_version"))
+        version_compatibility_check(protocol_version)
+
+    except ValueError:
+        response = _error_response(request_id, 505, "Unsupported Protocol Version")
+        return _write_response(response, 1)
+
     capabilities = params["capability"]
     supported_workflows = params.get("supported_workflows")
 
     exit_code = 0
     response = None
+
     try:
         builder = LambdaBuilder(language=capabilities["language"],
                                 dependency_manager=capabilities["dependency_manager"],
@@ -93,6 +130,7 @@ def main():  # pylint: disable=too-many-statements
                       params["artifacts_dir"],
                       params["scratch_dir"],
                       params["manifest_path"],
+                      executable_search_paths=params.get('executable_search_paths', None),
                       runtime=params["runtime"],
                       optimizations=params["optimizations"],
                       options=params["options"])

aws_lambda_builders/binary_path.py

@@ -0,0 +1,21 @@
+"""
+Class containing resolved path of binary given a validator and a resolver and the name of the binary.
+"""
+
+
+class BinaryPath(object):
+
+    def __init__(self, resolver, validator, binary, binary_path=None):
+        self.resolver = resolver
+        self.validator = validator
+        self.binary = binary
+        self._binary_path = binary_path
+        self.path_provided = True if self._binary_path else False
+
+    @property
+    def binary_path(self):
+        return self._binary_path
+
+    @binary_path.setter
+    def binary_path(self, binary_path):
+        self._binary_path = binary_path

aws_lambda_builders/builder.py

@@ -7,7 +7,6 @@
 import logging
 
 from aws_lambda_builders.registry import get_workflow, DEFAULT_REGISTRY
-from aws_lambda_builders.validate import RuntimeValidator
 from aws_lambda_builders.workflow import Capability
 
 LOG = logging.getLogger(__name__)
@@ -57,7 +56,7 @@ def __init__(self, language, dependency_manager, application_framework, supporte
         LOG.debug("Found workflow '%s' to support capabilities '%s'", self.selected_workflow_cls.NAME, self.capability)
 
     def build(self, source_dir, artifacts_dir, scratch_dir, manifest_path,
-              runtime=None, optimizations=None, options=None):
+              runtime=None, optimizations=None, options=None, executable_search_paths=None):
         """
         Actually build the code by running workflows
 
@@ -90,9 +89,11 @@ def build(self, source_dir, artifacts_dir, scratch_dir, manifest_path,
         :type options: dict
         :param options:
             Optional dictionary of options ot pass to build action. **Not supported**.
+
+        :type executable_search_paths: list
+        :param executable_search_paths:
+            Additional list of paths to search for executables required by the workflow.
         """
-        if runtime:
-            self._validate_runtime(runtime)
 
         if not os.path.exists(scratch_dir):
             os.makedirs(scratch_dir)
@@ -103,20 +104,10 @@ def build(self, source_dir, artifacts_dir, scratch_dir, manifest_path,
                                               manifest_path,
                                               runtime=runtime,
                                               optimizations=optimizations,
-                                              options=options)
+                                              options=options,
+                                              executable_search_paths=executable_search_paths)
 
         return workflow.run()
 
-    def _validate_runtime(self, runtime):
-        """
-        validate runtime and local runtime version to make sure they match
-
-        :type runtime: str
-        :param runtime:
-            String matching a lambda runtime eg: python3.6
-        """
-        RuntimeValidator.validate_runtime(required_language=self.capability.language,
-                                          required_runtime=runtime)
-
     def _clear_workflows(self):
         DEFAULT_REGISTRY.clear()

aws_lambda_builders/exceptions.py

@@ -18,7 +18,7 @@ class UnsupportedManifestError(LambdaBuilderError):
 class MisMatchRuntimeError(LambdaBuilderError):
     MESSAGE = "{language} executable found in your path does not " \
               "match runtime. " \
-              "\n Expected version: {required_runtime}, Found version: {found_runtime}. " \
+              "\n Expected version: {required_runtime}, Found version: {runtime_path}. " \
               "\n Possibly related: https://github.com/awslabs/aws-lambda-builders/issues/30"
 
 

aws_lambda_builders/path_resolver.py

@@ -0,0 +1,29 @@
+"""
+Basic Path Resolver that looks for the executable by runtime first, before proceeding to 'language' in PATH.
+"""
+
+from aws_lambda_builders.utils import which
+
+
+class PathResolver(object):
+
+    def __init__(self, binary, runtime, executable_search_paths=None):
+        self.binary = binary
+        self.runtime = runtime
+        self.executables = [self.runtime, self.binary]
+        self.executable_search_paths = executable_search_paths
+
+    def _which(self):
+        exec_paths = []
+        for executable in [executable for executable in self.executables if executable is not None]:
+            paths = which(executable, executable_search_paths=self.executable_search_paths)
+            exec_paths.extend(paths)
+
+        if not exec_paths:
+            raise ValueError("Path resolution for runtime: {} of binary: "
+                             "{} was not successful".format(self.runtime, self.binary))
+        return exec_paths
+
+    @property
+    def exec_paths(self):
+        return self._which()