chore: merge from develop branch, and apply the linking refactor (#5701)

* fix(invoke): Write in UTF-8 string instead of bytes (#5642) * Revert "fix: Revert UTF-8 fixes #5485 and #5427 (#5512)" This reverts commit 36f8bf9. * Enforce utf8 on stdout/stderr/logfile --------- Co-authored-by: Jacob Fuss <jfuss@users.noreply.github.com> * chore(deps): bump cryptography from 41.0.2 to 41.0.3 in /requirements (#5675) Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.2 to 41.0.3. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](pyca/cryptography@41.0.2...41.0.3) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * fix(test): Force reading file with utf8 in tests for windows (#5679) Co-authored-by: Jacob Fuss <jfuss@users.noreply.github.com> * fix(test): Increase max execution time for timeout tests (#5680) Co-authored-by: Jacob Fuss <jfuss@users.noreply.github.com> * chore: auto update reproducible requirements when there is a dependency change (#5677) * chore: add reproducible gha * format * formatting v2 * format v3 * use win make target * update windows job * update win paths * why windows why * remove if stmt for now * test with direct paths * update windows paths again * bring back activate * add dummy win file and push changes * update base.txt for testing * Update reproducibles * run jobs sequentially * print changed flag * check changes again * run git status before moving forward * refresh index before checking any changed files * Update reproducibles * commit or skip * Update reproducibles * rerun build & pyinstaller jobs once update reproducible finishes * add more details to commit message * nuke all reproducibles to trigger the CI * Update reproducibles: update-reproducible-linux-reqs * Update reproducibles: update-reproducible-mac-reqs * Update reproducibles: update-reproducible-win-reqs * change lb version for testing * update permissions and limit only for aws/aws-sam-cli * Update reproducibles: update-reproducible-linux-reqs * Update reproducibles: update-reproducible-mac-reqs * Update reproducibles: update-reproducible-win-reqs * update other jobs as well * run without file filter * Update reproducibles: update-reproducible-linux-reqs * Update reproducibles: update-reproducible-mac-reqs * Update reproducibles: update-reproducible-win-reqs * put the file filter back --------- Co-authored-by: GitHub Action <action@github.com> * fix: Repair failing integration test (#5698) * Fix failing integration test * Be more specific with error message --------- Co-authored-by: Leonardo Gama <leogama@amazon.com> * fix: link the API gateway resource parent to either rest api or another gateway resource (#5697) * chore: merge from develop branch, and apply the linking refactor * fix merge mistake in the integration test cases --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: Jacob Fuss <32497805+jfuss@users.noreply.github.com> Co-authored-by: Jacob Fuss <jfuss@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Mehmet Nuri Deveci <5735811+mndeveci@users.noreply.github.com> Co-authored-by: GitHub Action <action@github.com> Co-authored-by: Leonardo Gama <51037424+Leo10Gama@users.noreply.github.com> Co-authored-by: Leonardo Gama <leogama@amazon.com>
aws · Aug 4, 2023 · 2b857ac · 2b857ac
1 parent 0df8cfb
commit 2b857ac
Show file tree

Hide file tree

Showing 46 changed files with 2,626 additions and 1,044 deletions.
diff --git a/.github/workflows/automated-updates-to-sam-cli.yml b/.github/workflows/automated-updates-to-sam-cli.yml
@@ -71,12 +71,6 @@ jobs:
         with:
           repository: aws/aws-sam-cli
           path: aws-sam-cli
-
-      - uses: actions/setup-python@v4 # used for make update-reproducible-reqs below
-        with:
-          python-version: |
-            3.8
-            3.11
 
       - name: Update aws-sam-translator & commit
         run: |
@@ -90,7 +84,7 @@ jobs:
           SAM_T_PRE_VERSION=$(grep "aws-sam-translator=" requirements/base.txt)
           echo "SAM-T pre version is $SAM_T_PRE_VERSION"
           git reset --hard develop
-          sed -i "s/$SAM_T_PRE_VERSION/aws-sam-translator==$SAM_T_CUR_VERSION/g" requirements/base.txt; make update-reproducible-reqs
+          sed -i "s/$SAM_T_PRE_VERSION/aws-sam-translator==$SAM_T_CUR_VERSION/g" requirements/base.txt
           cp -r ../serverless-application-model/tests/translator/input ./tests/functional/commands/validate/lib/models
           git status
           git diff --quiet && exit 0 # exit if there is no change
@@ -129,12 +123,6 @@ jobs:
           repository: aws/aws-sam-cli
           path: aws-sam-cli
 
-      - uses: actions/setup-python@v4 # used for make update-reproducible-reqs below
-        with:
-          python-version: |
-            3.8
-            3.11
-
       - name: Upgrade aws_lambda_builders & commit
         run: |
           git config --global user.email "action@github.com"
@@ -147,7 +135,7 @@ jobs:
           BUILDERS_PRE_VERSION=$(grep "aws_lambda_builders=" requirements/base.txt)
           echo "Lambda Builders pre version is $BUILDERS_PRE_VERSION"
           git reset --hard develop
-          sed -i "s/$BUILDERS_PRE_VERSION/aws_lambda_builders==$BUILDERS_CUR_VERSION/g" requirements/base.txt; make update-reproducible-reqs
+          sed -i "s/$BUILDERS_PRE_VERSION/aws_lambda_builders==$BUILDERS_CUR_VERSION/g" requirements/base.txt
           git status
           git diff --quiet && exit 0 # exit if there is no change
           echo "is_new_lambda_builders=1" >> $GITHUB_ENV # set env variable for next step run decision

diff --git a/.github/workflows/update-reproducibles.yml b/.github/workflows/update-reproducibles.yml
@@ -0,0 +1,42 @@
+name: Update reproducible requirements
+on:
+  pull_request:
+    branches: [develop]
+    paths:
+      - requirements/base.txt # run this GHA only if requirements file is changed
+
+jobs:
+  update-reqs:
+    permissions:
+      pull-requests: write
+      contents: write
+    if: github.repository_owner == 'aws'
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-latest
+            python: 3.11
+            target: update-reproducible-linux-reqs
+          - os: macos-latest
+            python: 3.8
+            target: update-reproducible-mac-reqs
+          - os: windows-latest
+            python: 3.8
+            target: update-reproducible-win-reqs
+      max-parallel: 1
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+      - run: make ${{ matrix.target }}
+      - name: Push changes
+        run: |
+          git config --global user.email "action@github.com"
+          git config --global user.name "GitHub Action"
+          git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}
+          git commit -am "Update reproducibles: ${{ matrix.target }}" || echo "nothing to commit"
+          git push
diff --git a/Makefile b/Makefile
@@ -70,4 +70,13 @@ update-reproducible-mac-reqs:
 	venv-update-reproducible-mac/bin/pip install -r requirements/base.txt
 	venv-update-reproducible-mac/bin/pip-compile --generate-hashes --allow-unsafe -o requirements/reproducible-mac.txt
 
+# note that this should be run on a windows environment with python3.8 as default interpreter
+update-reproducible-win-reqs:
+	python -m venv venv-update-reproducible-win
+	.\venv-update-reproducible-win\Scripts\activate
+	pip install --upgrade pip-tools pip
+	pip install -r requirements\base.txt
+	pip-compile --generate-hashes --allow-unsafe -o requirements\reproducible-win.txt
+
+
 update-reproducible-reqs: update-reproducible-linux-reqs update-reproducible-mac-reqs
diff --git a/requirements/reproducible-linux.txt b/requirements/reproducible-linux.txt
diff --git a/requirements/reproducible-mac.txt b/requirements/reproducible-mac.txt
diff --git a/requirements/reproducible-win.txt b/requirements/reproducible-win.txt
diff --git a/samcli/commands/local/cli_common/invoke_context.py b/samcli/commands/local/cli_common/invoke_context.py
@@ -7,7 +7,7 @@
 import os
 from enum import Enum
 from pathlib import Path
-from typing import IO, Any, Dict, List, Optional, Tuple, Type, cast
+from typing import Any, Dict, List, Optional, TextIO, Tuple, Type, cast
 
 from samcli.commands._utils.template import TemplateFailedParsingException, TemplateNotFoundException
 from samcli.commands.exceptions import ContainersInitializationException
@@ -196,7 +196,7 @@ def __init__(
         self._stacks: List[Stack] = None  # type: ignore
         self._env_vars_value: Optional[Dict] = None
         self._container_env_vars_value: Optional[Dict] = None
-        self._log_file_handle: Optional[IO] = None
+        self._log_file_handle: Optional[TextIO] = None
         self._debug_context: Optional[DebugContext] = None
         self._layers_downloader: Optional[LayerDownloader] = None
         self._container_manager: Optional[ContainerManager] = None
@@ -490,7 +490,7 @@ def _get_env_vars_value(filename: Optional[str]) -> Optional[Dict]:
             ) from ex
 
     @staticmethod
-    def _setup_log_file(log_file: Optional[str]) -> Optional[IO]:
+    def _setup_log_file(log_file: Optional[str]) -> Optional[TextIO]:
         """
         Open a log file if necessary and return the file handle. This will create a file if it does not exist
 
@@ -500,7 +500,7 @@ def _setup_log_file(log_file: Optional[str]) -> Optional[IO]:
         if not log_file:
             return None
 
-        return open(log_file, "wb")
+        return open(log_file, "w", encoding="utf8")
 
     @staticmethod
     def _get_debug_context(

diff --git a/samcli/commands/remote/remote_invoke_context.py b/samcli/commands/remote/remote_invoke_context.py
@@ -242,7 +242,7 @@ class DefaultRemoteInvokeResponseConsumer(RemoteInvokeConsumer[RemoteInvokeRespo
     _stream_writer: StreamWriter
 
     def consume(self, remote_invoke_response: RemoteInvokeResponse) -> None:
-        self._stream_writer.write(cast(str, remote_invoke_response.response).encode())
+        self._stream_writer.write_bytes(cast(str, remote_invoke_response.response).encode())
 
 
 @dataclass
@@ -254,4 +254,4 @@ class DefaultRemoteInvokeLogConsumer(RemoteInvokeConsumer[RemoteInvokeLogOutput]
     _stream_writer: StreamWriter
 
     def consume(self, remote_invoke_response: RemoteInvokeLogOutput) -> None:
-        self._stream_writer.write(remote_invoke_response.log_output.encode())
+        self._stream_writer.write_bytes(remote_invoke_response.log_output.encode())
diff --git a/samcli/hook_packages/terraform/hooks/prepare/constants.py b/samcli/hook_packages/terraform/hooks/prepare/constants.py
@@ -13,3 +13,18 @@
     CFN_AWS_LAMBDA_LAYER_VERSION: "Content",
 }
 COMPILED_REGULAR_EXPRESSION = re.compile(r"\[[^\[\]]*\]")
+REMOTE_DUMMY_VALUE = "<<REMOTE DUMMY VALUE - RAISE ERROR IF IT IS STILL THERE>>"
+TF_AWS_LAMBDA_FUNCTION = "aws_lambda_function"
+TF_AWS_LAMBDA_LAYER_VERSION = "aws_lambda_layer_version"
+TF_AWS_API_GATEWAY_RESOURCE = "aws_api_gateway_resource"
+TF_AWS_API_GATEWAY_REST_API = "aws_api_gateway_rest_api"
+TF_AWS_API_GATEWAY_STAGE = "aws_api_gateway_stage"
+TF_AWS_API_GATEWAY_METHOD = "aws_api_gateway_method"
+TF_AWS_API_GATEWAY_INTEGRATION = "aws_api_gateway_integration"
+TF_AWS_API_GATEWAY_AUTHORIZER = "aws_api_gateway_authorizer"
+TF_AWS_API_GATEWAY_INTEGRATION_RESPONSE = "aws_api_gateway_method_response"
+TF_AWS_API_GATEWAY_V2_API = "aws_apigatewayv2_api"
+TF_AWS_API_GATEWAY_V2_ROUTE = "aws_apigatewayv2_route"
+TF_AWS_API_GATEWAY_V2_STAGE = "aws_apigatewayv2_stage"
+TF_AWS_API_GATEWAY_V2_INTEGRATION = "aws_apigatewayv2_integration"
+TF_AWS_API_GATEWAY_V2_AUTHORIZER = "aws_apigatewayv2_authorizer"
diff --git a/samcli/hook_packages/terraform/hooks/prepare/exceptions.py b/samcli/hook_packages/terraform/hooks/prepare/exceptions.py
@@ -19,6 +19,13 @@ def __init__(self, message):
         UserException.__init__(self, msg)
 
 
+class UnexpectedDestinationResource(InvalidResourceLinkingException):
+    """
+    Exception that will be thrown while doing terraform linking logic in case if the found destination resource is not
+    of the expected type, or the field used for linking is not the expected field.
+    """
+
+
 class ApplyLimitationException(UserException):
     def __init__(self, message):
         fmt = "{message}{line_sep}{line_sep}{apply_work_around}"
@@ -85,6 +92,20 @@ class GatewayResourceToGatewayRestApiLocalVariablesLinkingLimitationException(Lo
     """
 
 
+class OneGatewayResourceToParentResourceLinkingLimitationException(OneResourceLinkingLimitationException):
+    """
+    Exception specific for Gateway Resource linking to more than one Parent Resources which is either Rest API or
+    Gateway Resource
+    """
+
+
+class GatewayResourceToParentResourceLocalVariablesLinkingLimitationException(LocalVariablesLinkingLimitationException):
+    """
+    Exception specific for Gateway Resource linking to Parent Resources which is either Rest API or Gateway Resource
+    using locals.
+    """
+
+
 class OneRestApiToApiGatewayMethodLinkingLimitationException(OneResourceLinkingLimitationException):
     """
     Exception specific for Gateway Method linking to more than Rest API

diff --git a/samcli/hook_packages/terraform/hooks/prepare/property_builder.py b/samcli/hook_packages/terraform/hooks/prepare/property_builder.py
@@ -6,6 +6,23 @@
 from json.decoder import JSONDecodeError
 from typing import Any, Dict, Optional
 
+from samcli.hook_packages.terraform.hooks.prepare.constants import (
+    REMOTE_DUMMY_VALUE,
+    TF_AWS_API_GATEWAY_AUTHORIZER,
+    TF_AWS_API_GATEWAY_INTEGRATION,
+    TF_AWS_API_GATEWAY_INTEGRATION_RESPONSE,
+    TF_AWS_API_GATEWAY_METHOD,
+    TF_AWS_API_GATEWAY_RESOURCE,
+    TF_AWS_API_GATEWAY_REST_API,
+    TF_AWS_API_GATEWAY_STAGE,
+    TF_AWS_API_GATEWAY_V2_API,
+    TF_AWS_API_GATEWAY_V2_AUTHORIZER,
+    TF_AWS_API_GATEWAY_V2_INTEGRATION,
+    TF_AWS_API_GATEWAY_V2_ROUTE,
+    TF_AWS_API_GATEWAY_V2_STAGE,
+    TF_AWS_LAMBDA_FUNCTION,
+    TF_AWS_LAMBDA_LAYER_VERSION,
+)
 from samcli.hook_packages.terraform.hooks.prepare.resource_linking import _resolve_resource_attribute
 from samcli.hook_packages.terraform.hooks.prepare.resources.internal import (
     INTERNAL_API_GATEWAY_INTEGRATION,
@@ -34,24 +51,6 @@
 
 LOG = logging.getLogger(__name__)
 
-REMOTE_DUMMY_VALUE = "<<REMOTE DUMMY VALUE - RAISE ERROR IF IT IS STILL THERE>>"
-TF_AWS_LAMBDA_FUNCTION = "aws_lambda_function"
-TF_AWS_LAMBDA_LAYER_VERSION = "aws_lambda_layer_version"
-
-TF_AWS_API_GATEWAY_RESOURCE = "aws_api_gateway_resource"
-TF_AWS_API_GATEWAY_REST_API = "aws_api_gateway_rest_api"
-TF_AWS_API_GATEWAY_STAGE = "aws_api_gateway_stage"
-TF_AWS_API_GATEWAY_METHOD = "aws_api_gateway_method"
-TF_AWS_API_GATEWAY_INTEGRATION = "aws_api_gateway_integration"
-TF_AWS_API_GATEWAY_AUTHORIZER = "aws_api_gateway_authorizer"
-TF_AWS_API_GATEWAY_INTEGRATION_RESPONSE = "aws_api_gateway_method_response"
-
-TF_AWS_API_GATEWAY_V2_API = "aws_apigatewayv2_api"
-TF_AWS_API_GATEWAY_V2_ROUTE = "aws_apigatewayv2_route"
-TF_AWS_API_GATEWAY_V2_STAGE = "aws_apigatewayv2_stage"
-TF_AWS_API_GATEWAY_V2_INTEGRATION = "aws_apigatewayv2_integration"
-TF_AWS_API_GATEWAY_V2_AUTHORIZER = "aws_apigatewayv2_authorizer"
-
 
 def _build_code_property(tf_properties: dict, resource: TFResource) -> Any:
     """