Merge remote-tracking branch 'origin/main' into react-template

.github/workflows/aws-replicator.yml

@@ -20,10 +20,10 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Set up Python 3.10
+      - name: Set up Python 3.11
         uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
 
       - name: Set up Terraform CLI
         uses: hashicorp/setup-terraform@v2
@@ -32,8 +32,10 @@ jobs:
         env:
           LOCALSTACK_API_KEY: ${{ secrets.LOCALSTACK_API_KEY }}
         run: |
+          set -e
           docker pull localstack/localstack-pro &
           docker pull public.ecr.aws/lambda/python:3.8 &
+
           # install latest CLI packages (dev releases)
           pip install --upgrade --pre localstack localstack-ext
 
@@ -48,7 +50,11 @@ jobs:
           localstack extensions init
           (
             cd aws-replicator
-            make install && make build && make enable
+            make install
+            . .venv/bin/activate
+            pip install --upgrade --pre localstack localstack-ext
+            make build
+            make enable
           )
 
           # install awslocal/tflocal command lines

README.md

@@ -74,6 +74,7 @@ You can install the respective extension by calling `localstack install <Install
 | [MailHog](https://github.com/localstack/localstack-extensions/tree/main/mailhog) | localstack-extension-mailhog | 0.1.0 | Stable |
 | [Miniflare](https://github.com/localstack/localstack-extensions/tree/main/miniflare) | localstack-extension-miniflare | 0.1.0 | Experimental |
 | [Stripe](https://github.com/localstack/localstack-extensions/tree/main/stripe) | localstack-extension-stripe | 0.1.0 | Stable |
+| [Terraform Init](https://github.com/localstack/localstack-extensions/tree/main/terraform-init) | localstack-extension-terraform-init | 0.2.0 | Experimental |
 
 
 ## Developing Extensions

aws-replicator/Makefile

@@ -5,6 +5,9 @@ VENV_RUN = . $(VENV_ACTIVATE)
 TEST_PATH ?= tests
 PIP_CMD ?= pip
 
+usage:                    ## Show this help
+	@grep -Fh "##" $(MAKEFILE_LIST) | grep -Fv fgrep | sed -e 's/:.*##\s*/##/g' | awk -F'##' '{ printf "%-25s %s\n", $$1, $$2 }'
+
 venv: $(VENV_ACTIVATE)
 
 $(VENV_ACTIVATE): setup.py setup.cfg
@@ -40,13 +43,13 @@ build:   ## Build the extension
 	cp -r setup.py setup.cfg README.md aws_replicator build/
 	(cd build && python setup.py sdist)
 
-enable: $(wildcard ./build/dist/localstack-extension-aws-replicator-*.tar.gz)  ## Enable the extension in LocalStack
+enable: $(wildcard ./build/dist/localstack_extension_aws_replicator-*.tar.gz)  ## Enable the extension in LocalStack
 	$(VENV_RUN); \
 		pip uninstall --yes localstack-extension-aws-replicator; \
-		localstack extensions -v install file://./$?
+		localstack extensions -v install file://$?
 
 publish: clean-dist venv dist
-	$(VENV_RUN); pip install --upgrade twine; twine upload dist/*
+	$(VENV_RUN); cd build; pip install --upgrade twine; twine upload dist/*
 
 clean-dist: clean
 	rm -rf dist/

aws-replicator/README.md

@@ -152,6 +152,10 @@ localstack extensions install "git+https://github.com/localstack/localstack-exte
 
 ## Change Log
 
+* `0.1.18`: Update environment check to use SDK Docker client and enable starting the proxy from within Docker (e.g., from the LS main container as part of an init script)
+* `0.1.17`: Add basic support for ARN-based pattern-matching for `secretsmanager` resources
+* `0.1.16`: Update imports for localstack >=3.6 compatibility
+* `0.1.15`: Move localstack dependency installation to extra since it's provided at runtime
 * `0.1.14`: Install missing dependencies into proxy container for localstack >=3.4 compatibility
 * `0.1.13`: Add compatibility with localstack >=3.4; add http2-server; migrate to localstack auth login
 * `0.1.12`: Modify aws credentials text field type to password  

aws-replicator/aws_replicator/client/auth_proxy.py

@@ -12,9 +12,7 @@
 import requests
 from botocore.awsrequest import AWSPreparedRequest
 from botocore.model import OperationModel
-from localstack import config
 from localstack import config as localstack_config
-from localstack.aws.protocol.parser import create_parser
 from localstack.aws.spec import load_service
 from localstack.config import external_service_url
 from localstack.constants import AWS_REGION_US_EAST_1, DOCKER_IMAGE_NAME_PRO
@@ -29,7 +27,6 @@
 from localstack.utils.net import get_docker_host_from_container, get_free_tcp_port
 from localstack.utils.serving import Server
 from localstack.utils.strings import short_uid, to_bytes, to_str, truncate
-from localstack_ext.bootstrap.licensingv2 import ENV_LOCALSTACK_API_KEY, ENV_LOCALSTACK_AUTH_TOKEN
 from requests import Response
 
 from aws_replicator import config as repl_config
@@ -39,9 +36,21 @@
 
 from .http2_server import run_server
 
+try:
+    from localstack.pro.core.bootstrap.licensingv2 import (
+        ENV_LOCALSTACK_API_KEY,
+        ENV_LOCALSTACK_AUTH_TOKEN,
+    )
+except ImportError:
+    # TODO remove once we don't need compatibility with <3.6 anymore
+    from localstack_ext.bootstrap.licensingv2 import (
+        ENV_LOCALSTACK_API_KEY,
+        ENV_LOCALSTACK_AUTH_TOKEN,
+    )
+
 LOG = logging.getLogger(__name__)
 LOG.setLevel(logging.INFO)
-if config.DEBUG:
+if localstack_config.DEBUG:
     LOG.setLevel(logging.DEBUG)
 
 # TODO make configurable
@@ -158,6 +167,8 @@ def register_in_instance(self):
     def _parse_aws_request(
         self, request: Request, service_name: str, region_name: str, client
     ) -> Tuple[OperationModel, AWSPreparedRequest, Dict]:
+        from localstack.aws.protocol.parser import create_parser
+
         parser = create_parser(load_service(service_name))
         operation_model, parsed_request = parser.parse(request)
         request_context = {
@@ -359,10 +370,14 @@ def start_aws_auth_proxy_in_container(
         target_host = get_docker_host_from_container()
     env_vars["LOCALSTACK_HOST"] = target_host
 
+    # Use the Docker SDK command either if quiet mode is enabled, or if we're executing
+    # in Docker itself (e.g., within the LocalStack main container, as part of an init script)
+    use_docker_sdk_command = quiet or localstack_config.is_in_docker
+
     try:
         print("Proxy container is ready.")
         command = f"{venv_activate}; localstack aws proxy -c {CONTAINER_CONFIG_FILE} -p {port} --host 0.0.0.0 > {CONTAINER_LOG_FILE} 2>&1"
-        if quiet:
+        if use_docker_sdk_command:
             DOCKER_CLIENT.exec_in_container(
                 container_name, command=["bash", "-c", command], env_vars=env_vars, interactive=True
             )

aws-replicator/aws_replicator/client/cli.py

@@ -6,12 +6,19 @@
 from localstack.cli import LocalstackCli, LocalstackCliPlugin, console
 from localstack.logging.setup import setup_logging
 from localstack.utils.files import load_file
-from localstack_ext.bootstrap.auth import get_auth_headers
-from localstack_ext.cli.aws import aws
-from localstack_ext.config import is_api_key_configured
 
 from aws_replicator.shared.models import ProxyConfig, ProxyServiceConfig
 
+try:
+    from localstack.pro.core.bootstrap.auth import get_auth_headers
+    from localstack.pro.core.cli.aws import aws
+    from localstack.pro.core.config import is_api_key_configured
+except ImportError:
+    # TODO remove once we don't need compatibility with <3.6 anymore
+    from localstack_ext.bootstrap.auth import get_auth_headers
+    from localstack_ext.cli.aws import aws
+    from localstack_ext.config import is_api_key_configured
+
 
 class AwsReplicatorPlugin(LocalstackCliPlugin):
     name = "aws-replicator"
@@ -67,10 +74,7 @@ def _is_logged_in() -> bool:
     required=False,
 )
 def cmd_aws_proxy(services: str, config: str, container: bool, port: int, host: str):
-    from aws_replicator.client.auth_proxy import (
-        start_aws_auth_proxy,
-        start_aws_auth_proxy_in_container,
-    )
+    from aws_replicator.client.auth_proxy import start_aws_auth_proxy_in_container
 
     config_json: ProxyConfig = {"services": {}}
     if config:
@@ -84,6 +88,10 @@ def cmd_aws_proxy(services: str, config: str, container: bool, port: int, host:
     try:
         if container:
             return start_aws_auth_proxy_in_container(config_json)
+
+        # note: deferring the import here, to avoid import errors in CLI context
+        from aws_replicator.client.auth_proxy import start_aws_auth_proxy
+
         proxy = start_aws_auth_proxy(config_json, port=port)
         proxy.join()
     except Exception as e:

aws-replicator/aws_replicator/client/service_states.py

@@ -1,14 +1,17 @@
 import logging
-from typing import Dict, Type
+from typing import Dict, Optional, Type
 
 import boto3
 from botocore.client import BaseClient
 from localstack.services.cloudformation.models.s3 import S3Bucket
+from localstack.services.cloudformation.service_models import GenericBaseModel
 from localstack.utils.aws import aws_stack
+from localstack.utils.objects import get_all_subclasses
 from localstack.utils.threads import parallelize
 
 from aws_replicator.client.utils import post_request_to_instance
-from aws_replicator.shared.models import ExtendedResourceStateReplicator, ReplicateStateRequest
+from aws_replicator.shared.models import ReplicateStateRequest
+from aws_replicator.shared.utils import get_resource_type
 
 LOG = logging.getLogger(__name__)
 
@@ -23,6 +26,37 @@ def wrapper(wrapping_clazz):
     return wrapper
 
 
+# TODO: remove / adjust to use latest upstream CFn models!
+class ExtendedResourceStateReplicator(GenericBaseModel):
+    """Extended resource models, used to replicate (inject) additional state into a resource instance"""
+
+    def add_extended_state_external(self, remote_client: BaseClient = None):
+        """Called in the context of external CLI execution to fetch/replicate resource details from a remote account"""
+
+    def add_extended_state_internal(self, state: Dict):
+        """Called in the context of the internal LocalStack instance to inject the state into a resource"""
+
+    @classmethod
+    def get_resource_instance(cls, resource: Dict) -> Optional["ExtendedResourceStateReplicator"]:
+        resource_type = get_resource_type(resource)
+        resource_class = cls.find_resource_classes().get(resource_type)
+        if resource_class:
+            return resource_class(resource)
+
+    @classmethod
+    def get_resource_class(
+        cls, resource_type: str
+    ) -> Optional[Type["ExtendedResourceStateReplicator"]]:
+        return cls.find_resource_classes().get(resource_type)
+
+    @classmethod
+    def find_resource_classes(cls) -> Dict[str, "ExtendedResourceStateReplicator"]:
+        return {
+            inst.cloudformation_type(): inst
+            for inst in get_all_subclasses(ExtendedResourceStateReplicator)
+        }
+
+
 # resource-specific replications
 
 

aws-replicator/aws_replicator/server/aws_request_forwarder.py

@@ -9,7 +9,7 @@
 from localstack.constants import APPLICATION_JSON, LOCALHOST, LOCALHOST_HOSTNAME
 from localstack.http import Response
 from localstack.utils.aws import arns
-from localstack.utils.aws.arns import sqs_queue_arn
+from localstack.utils.aws.arns import secretsmanager_secret_arn, sqs_queue_arn
 from localstack.utils.aws.aws_stack import get_valid_regions
 from localstack.utils.aws.request_context import mock_aws_request_headers
 from localstack.utils.collections import ensure_list
@@ -118,6 +118,12 @@ def _request_matches_resource(
                 if re.match(resource_name_pattern, candidate):
                     return True
             return False
+        if service_name == "secretsmanager":
+            secret_id = context.service_request.get("SecretId") or ""
+            secret_arn = secretsmanager_secret_arn(
+                secret_id, account_id=context.account_id, region_name=context.region
+            )
+            return bool(re.match(resource_name_pattern, secret_arn))
         # TODO: add more resource patterns
         return True
 

aws-replicator/aws_replicator/server/resource_replicator.py

@@ -66,9 +66,15 @@ def _get_cf_model_class(self, resource: Dict) -> Optional[Type]:
 
     def _load_resource_models(self):
         if not hasattr(template_deployer, "_ls_patch_applied"):
-            from localstack_ext.services.cloudformation.cloudformation_extended import (
-                patch_cloudformation,
-            )
+            try:
+                from localstack.pro.core.services.cloudformation.cloudformation_extended import (
+                    patch_cloudformation,
+                )
+            except ImportError:
+                # TODO remove once we don't need compatibility with <3.6 anymore
+                from localstack_ext.services.cloudformation.cloudformation_extended import (
+                    patch_cloudformation,
+                )
 
             patch_cloudformation()
             template_deployer._ls_patch_applied = True

aws-replicator/aws_replicator/shared/models.py

@@ -1,46 +1,10 @@
 import logging
 from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Optional, Type, TypedDict, Union
-
-from botocore.client import BaseClient
-from localstack.services.cloudformation.service_models import GenericBaseModel
-from localstack.utils.objects import get_all_subclasses
-
-from aws_replicator.shared.utils import get_resource_type
+from typing import Any, Dict, List, Optional, TypedDict, Union
 
 LOG = logging.getLogger(__name__)
 
 
-class ExtendedResourceStateReplicator(GenericBaseModel):
-    """Extended resource models, used to replicate (inject) additional state into a resource instance"""
-
-    def add_extended_state_external(self, remote_client: BaseClient = None):
-        """Called in the context of external CLI execution to fetch/replicate resource details from a remote account"""
-
-    def add_extended_state_internal(self, state: Dict):
-        """Called in the context of the internal LocalStack instance to inject the state into a resource"""
-
-    @classmethod
-    def get_resource_instance(cls, resource: Dict) -> Optional["ExtendedResourceStateReplicator"]:
-        resource_type = get_resource_type(resource)
-        resource_class = cls.find_resource_classes().get(resource_type)
-        if resource_class:
-            return resource_class(resource)
-
-    @classmethod
-    def get_resource_class(
-        cls, resource_type: str
-    ) -> Optional[Type["ExtendedResourceStateReplicator"]]:
-        return cls.find_resource_classes().get(resource_type)
-
-    @classmethod
-    def find_resource_classes(cls) -> Dict[str, "ExtendedResourceStateReplicator"]:
-        return {
-            inst.cloudformation_type(): inst
-            for inst in get_all_subclasses(ExtendedResourceStateReplicator)
-        }
-
-
 class ReplicateStateRequest(TypedDict):
     """
     Represents a request sent from the CLI to the extension request

aws-replicator/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = localstack-extension-aws-replicator
-version = 0.1.14
+version = 0.1.18
 summary = LocalStack Extension: AWS replicator
 description = Replicate AWS resources into your LocalStack instance
 long_description = file: README.md
@@ -18,9 +18,7 @@ install_requires =
     # TODO: currently requires a version pin, see note in auth_proxy.py
     botocore>=1.29.151
     flask
-    localstack
     localstack-client
-    localstack-ext
     xmltodict
     # TODO: refactor the use of http2_server
     hypercorn
@@ -37,6 +35,8 @@ install_requires =
 [options.extras_require]
 test =
     apispec
+    localstack-core
+    localstack-ext
     openapi-spec-validator
     pyproject-flake8
     pytest

diagnosis-viewer/Makefile

@@ -8,7 +8,6 @@ venv: $(VENV_ACTIVATE)
 $(VENV_ACTIVATE): setup.py setup.cfg
 	test -d .venv || $(VENV_BIN) .venv
 	$(VENV_RUN); pip install --upgrade pip setuptools plux wheel
-	$(VENV_RUN); pip install -e .
 	touch $(VENV_DIR)/bin/activate
 
 clean:
@@ -18,7 +17,7 @@ clean:
 	rm -rf *.egg-info/
 
 install: venv
-	$(VENV_RUN); python setup.py develop
+	$(VENV_RUN); python -m pip install -e .[dev]
 
 dist: venv
 	$(VENV_RUN); python setup.py sdist bdist_wheel

diagnosis-viewer/setup.cfg

@@ -13,9 +13,12 @@ long_description_content_type = text/markdown; charset=UTF-8
 zip_safe = False
 packages = find:
 install_requires =
-    localstack>=1.4
     diapretty
 
+[options.extras_require]
+dev =
+    localstack-core>=1.4
+
 [options.entry_points]
 localstack.extensions =
     diagnosis-viewer = diagnosis_viewer.extension:DiagnosisViewerExtension