diff --git a/.github/workflows/aws-replicator.yml b/.github/workflows/aws-replicator.yml index 49bd888..b3aaefb 100644 --- a/.github/workflows/aws-replicator.yml +++ b/.github/workflows/aws-replicator.yml @@ -20,10 +20,10 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - name: Set up Terraform CLI uses: hashicorp/setup-terraform@v2 @@ -32,8 +32,10 @@ jobs: env: LOCALSTACK_API_KEY: ${{ secrets.LOCALSTACK_API_KEY }} run: | + set -e docker pull localstack/localstack-pro & docker pull public.ecr.aws/lambda/python:3.8 & + # install latest CLI packages (dev releases) pip install --upgrade --pre localstack localstack-ext @@ -48,7 +50,11 @@ jobs: localstack extensions init ( cd aws-replicator - make install && make build && make enable + make install + . .venv/bin/activate + pip install --upgrade --pre localstack localstack-ext + make build + make enable ) # install awslocal/tflocal command lines diff --git a/README.md b/README.md index f70523d..bc8eafc 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,7 @@ You can install the respective extension by calling `localstack install =3.6 compatibility +* `0.1.15`: Move localstack dependency installation to extra since it's provided at runtime * `0.1.14`: Install missing dependencies into proxy container for localstack >=3.4 compatibility * `0.1.13`: Add compatibility with localstack >=3.4; add http2-server; migrate to localstack auth login * `0.1.12`: Modify aws credentials text field type to password diff --git a/aws-replicator/aws_replicator/client/auth_proxy.py b/aws-replicator/aws_replicator/client/auth_proxy.py index 966abe2..e79fa2f 100644 --- a/aws-replicator/aws_replicator/client/auth_proxy.py +++ b/aws-replicator/aws_replicator/client/auth_proxy.py @@ -12,9 +12,7 @@ import requests from botocore.awsrequest import AWSPreparedRequest from botocore.model import OperationModel -from localstack import config from localstack import config as localstack_config -from localstack.aws.protocol.parser import create_parser from localstack.aws.spec import load_service from localstack.config import external_service_url from localstack.constants import AWS_REGION_US_EAST_1, DOCKER_IMAGE_NAME_PRO @@ -29,7 +27,6 @@ from localstack.utils.net import get_docker_host_from_container, get_free_tcp_port from localstack.utils.serving import Server from localstack.utils.strings import short_uid, to_bytes, to_str, truncate -from localstack_ext.bootstrap.licensingv2 import ENV_LOCALSTACK_API_KEY, ENV_LOCALSTACK_AUTH_TOKEN from requests import Response from aws_replicator import config as repl_config @@ -39,9 +36,21 @@ from .http2_server import run_server +try: + from localstack.pro.core.bootstrap.licensingv2 import ( + ENV_LOCALSTACK_API_KEY, + ENV_LOCALSTACK_AUTH_TOKEN, + ) +except ImportError: + # TODO remove once we don't need compatibility with <3.6 anymore + from localstack_ext.bootstrap.licensingv2 import ( + ENV_LOCALSTACK_API_KEY, + ENV_LOCALSTACK_AUTH_TOKEN, + ) + LOG = logging.getLogger(__name__) LOG.setLevel(logging.INFO) -if config.DEBUG: +if localstack_config.DEBUG: LOG.setLevel(logging.DEBUG) # TODO make configurable @@ -158,6 +167,8 @@ def register_in_instance(self): def _parse_aws_request( self, request: Request, service_name: str, region_name: str, client ) -> Tuple[OperationModel, AWSPreparedRequest, Dict]: + from localstack.aws.protocol.parser import create_parser + parser = create_parser(load_service(service_name)) operation_model, parsed_request = parser.parse(request) request_context = { @@ -359,10 +370,14 @@ def start_aws_auth_proxy_in_container( target_host = get_docker_host_from_container() env_vars["LOCALSTACK_HOST"] = target_host + # Use the Docker SDK command either if quiet mode is enabled, or if we're executing + # in Docker itself (e.g., within the LocalStack main container, as part of an init script) + use_docker_sdk_command = quiet or localstack_config.is_in_docker + try: print("Proxy container is ready.") command = f"{venv_activate}; localstack aws proxy -c {CONTAINER_CONFIG_FILE} -p {port} --host 0.0.0.0 > {CONTAINER_LOG_FILE} 2>&1" - if quiet: + if use_docker_sdk_command: DOCKER_CLIENT.exec_in_container( container_name, command=["bash", "-c", command], env_vars=env_vars, interactive=True ) diff --git a/aws-replicator/aws_replicator/client/cli.py b/aws-replicator/aws_replicator/client/cli.py index 8c90918..d199095 100644 --- a/aws-replicator/aws_replicator/client/cli.py +++ b/aws-replicator/aws_replicator/client/cli.py @@ -6,12 +6,19 @@ from localstack.cli import LocalstackCli, LocalstackCliPlugin, console from localstack.logging.setup import setup_logging from localstack.utils.files import load_file -from localstack_ext.bootstrap.auth import get_auth_headers -from localstack_ext.cli.aws import aws -from localstack_ext.config import is_api_key_configured from aws_replicator.shared.models import ProxyConfig, ProxyServiceConfig +try: + from localstack.pro.core.bootstrap.auth import get_auth_headers + from localstack.pro.core.cli.aws import aws + from localstack.pro.core.config import is_api_key_configured +except ImportError: + # TODO remove once we don't need compatibility with <3.6 anymore + from localstack_ext.bootstrap.auth import get_auth_headers + from localstack_ext.cli.aws import aws + from localstack_ext.config import is_api_key_configured + class AwsReplicatorPlugin(LocalstackCliPlugin): name = "aws-replicator" @@ -67,10 +74,7 @@ def _is_logged_in() -> bool: required=False, ) def cmd_aws_proxy(services: str, config: str, container: bool, port: int, host: str): - from aws_replicator.client.auth_proxy import ( - start_aws_auth_proxy, - start_aws_auth_proxy_in_container, - ) + from aws_replicator.client.auth_proxy import start_aws_auth_proxy_in_container config_json: ProxyConfig = {"services": {}} if config: @@ -84,6 +88,10 @@ def cmd_aws_proxy(services: str, config: str, container: bool, port: int, host: try: if container: return start_aws_auth_proxy_in_container(config_json) + + # note: deferring the import here, to avoid import errors in CLI context + from aws_replicator.client.auth_proxy import start_aws_auth_proxy + proxy = start_aws_auth_proxy(config_json, port=port) proxy.join() except Exception as e: diff --git a/aws-replicator/aws_replicator/client/service_states.py b/aws-replicator/aws_replicator/client/service_states.py index 1cad6d4..148339b 100644 --- a/aws-replicator/aws_replicator/client/service_states.py +++ b/aws-replicator/aws_replicator/client/service_states.py @@ -1,14 +1,17 @@ import logging -from typing import Dict, Type +from typing import Dict, Optional, Type import boto3 from botocore.client import BaseClient from localstack.services.cloudformation.models.s3 import S3Bucket +from localstack.services.cloudformation.service_models import GenericBaseModel from localstack.utils.aws import aws_stack +from localstack.utils.objects import get_all_subclasses from localstack.utils.threads import parallelize from aws_replicator.client.utils import post_request_to_instance -from aws_replicator.shared.models import ExtendedResourceStateReplicator, ReplicateStateRequest +from aws_replicator.shared.models import ReplicateStateRequest +from aws_replicator.shared.utils import get_resource_type LOG = logging.getLogger(__name__) @@ -23,6 +26,37 @@ def wrapper(wrapping_clazz): return wrapper +# TODO: remove / adjust to use latest upstream CFn models! +class ExtendedResourceStateReplicator(GenericBaseModel): + """Extended resource models, used to replicate (inject) additional state into a resource instance""" + + def add_extended_state_external(self, remote_client: BaseClient = None): + """Called in the context of external CLI execution to fetch/replicate resource details from a remote account""" + + def add_extended_state_internal(self, state: Dict): + """Called in the context of the internal LocalStack instance to inject the state into a resource""" + + @classmethod + def get_resource_instance(cls, resource: Dict) -> Optional["ExtendedResourceStateReplicator"]: + resource_type = get_resource_type(resource) + resource_class = cls.find_resource_classes().get(resource_type) + if resource_class: + return resource_class(resource) + + @classmethod + def get_resource_class( + cls, resource_type: str + ) -> Optional[Type["ExtendedResourceStateReplicator"]]: + return cls.find_resource_classes().get(resource_type) + + @classmethod + def find_resource_classes(cls) -> Dict[str, "ExtendedResourceStateReplicator"]: + return { + inst.cloudformation_type(): inst + for inst in get_all_subclasses(ExtendedResourceStateReplicator) + } + + # resource-specific replications diff --git a/aws-replicator/aws_replicator/server/aws_request_forwarder.py b/aws-replicator/aws_replicator/server/aws_request_forwarder.py index c2e6ba1..a9d1021 100644 --- a/aws-replicator/aws_replicator/server/aws_request_forwarder.py +++ b/aws-replicator/aws_replicator/server/aws_request_forwarder.py @@ -9,7 +9,7 @@ from localstack.constants import APPLICATION_JSON, LOCALHOST, LOCALHOST_HOSTNAME from localstack.http import Response from localstack.utils.aws import arns -from localstack.utils.aws.arns import sqs_queue_arn +from localstack.utils.aws.arns import secretsmanager_secret_arn, sqs_queue_arn from localstack.utils.aws.aws_stack import get_valid_regions from localstack.utils.aws.request_context import mock_aws_request_headers from localstack.utils.collections import ensure_list @@ -118,6 +118,12 @@ def _request_matches_resource( if re.match(resource_name_pattern, candidate): return True return False + if service_name == "secretsmanager": + secret_id = context.service_request.get("SecretId") or "" + secret_arn = secretsmanager_secret_arn( + secret_id, account_id=context.account_id, region_name=context.region + ) + return bool(re.match(resource_name_pattern, secret_arn)) # TODO: add more resource patterns return True diff --git a/aws-replicator/aws_replicator/server/resource_replicator.py b/aws-replicator/aws_replicator/server/resource_replicator.py index 7c7c2ba..63687cb 100644 --- a/aws-replicator/aws_replicator/server/resource_replicator.py +++ b/aws-replicator/aws_replicator/server/resource_replicator.py @@ -66,9 +66,15 @@ def _get_cf_model_class(self, resource: Dict) -> Optional[Type]: def _load_resource_models(self): if not hasattr(template_deployer, "_ls_patch_applied"): - from localstack_ext.services.cloudformation.cloudformation_extended import ( - patch_cloudformation, - ) + try: + from localstack.pro.core.services.cloudformation.cloudformation_extended import ( + patch_cloudformation, + ) + except ImportError: + # TODO remove once we don't need compatibility with <3.6 anymore + from localstack_ext.services.cloudformation.cloudformation_extended import ( + patch_cloudformation, + ) patch_cloudformation() template_deployer._ls_patch_applied = True diff --git a/aws-replicator/aws_replicator/shared/models.py b/aws-replicator/aws_replicator/shared/models.py index 2d64a6d..5644212 100644 --- a/aws-replicator/aws_replicator/shared/models.py +++ b/aws-replicator/aws_replicator/shared/models.py @@ -1,46 +1,10 @@ import logging from abc import ABC, abstractmethod -from typing import Any, Dict, List, Optional, Type, TypedDict, Union - -from botocore.client import BaseClient -from localstack.services.cloudformation.service_models import GenericBaseModel -from localstack.utils.objects import get_all_subclasses - -from aws_replicator.shared.utils import get_resource_type +from typing import Any, Dict, List, Optional, TypedDict, Union LOG = logging.getLogger(__name__) -class ExtendedResourceStateReplicator(GenericBaseModel): - """Extended resource models, used to replicate (inject) additional state into a resource instance""" - - def add_extended_state_external(self, remote_client: BaseClient = None): - """Called in the context of external CLI execution to fetch/replicate resource details from a remote account""" - - def add_extended_state_internal(self, state: Dict): - """Called in the context of the internal LocalStack instance to inject the state into a resource""" - - @classmethod - def get_resource_instance(cls, resource: Dict) -> Optional["ExtendedResourceStateReplicator"]: - resource_type = get_resource_type(resource) - resource_class = cls.find_resource_classes().get(resource_type) - if resource_class: - return resource_class(resource) - - @classmethod - def get_resource_class( - cls, resource_type: str - ) -> Optional[Type["ExtendedResourceStateReplicator"]]: - return cls.find_resource_classes().get(resource_type) - - @classmethod - def find_resource_classes(cls) -> Dict[str, "ExtendedResourceStateReplicator"]: - return { - inst.cloudformation_type(): inst - for inst in get_all_subclasses(ExtendedResourceStateReplicator) - } - - class ReplicateStateRequest(TypedDict): """ Represents a request sent from the CLI to the extension request diff --git a/aws-replicator/setup.cfg b/aws-replicator/setup.cfg index f9a75d8..fd4d9ee 100644 --- a/aws-replicator/setup.cfg +++ b/aws-replicator/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = localstack-extension-aws-replicator -version = 0.1.14 +version = 0.1.18 summary = LocalStack Extension: AWS replicator description = Replicate AWS resources into your LocalStack instance long_description = file: README.md @@ -18,9 +18,7 @@ install_requires = # TODO: currently requires a version pin, see note in auth_proxy.py botocore>=1.29.151 flask - localstack localstack-client - localstack-ext xmltodict # TODO: refactor the use of http2_server hypercorn @@ -37,6 +35,8 @@ install_requires = [options.extras_require] test = apispec + localstack-core + localstack-ext openapi-spec-validator pyproject-flake8 pytest diff --git a/diagnosis-viewer/Makefile b/diagnosis-viewer/Makefile index c86b76f..30feecb 100644 --- a/diagnosis-viewer/Makefile +++ b/diagnosis-viewer/Makefile @@ -8,7 +8,6 @@ venv: $(VENV_ACTIVATE) $(VENV_ACTIVATE): setup.py setup.cfg test -d .venv || $(VENV_BIN) .venv $(VENV_RUN); pip install --upgrade pip setuptools plux wheel - $(VENV_RUN); pip install -e . touch $(VENV_DIR)/bin/activate clean: @@ -18,7 +17,7 @@ clean: rm -rf *.egg-info/ install: venv - $(VENV_RUN); python setup.py develop + $(VENV_RUN); python -m pip install -e .[dev] dist: venv $(VENV_RUN); python setup.py sdist bdist_wheel diff --git a/diagnosis-viewer/setup.cfg b/diagnosis-viewer/setup.cfg index 55a2437..b27ee38 100644 --- a/diagnosis-viewer/setup.cfg +++ b/diagnosis-viewer/setup.cfg @@ -13,9 +13,12 @@ long_description_content_type = text/markdown; charset=UTF-8 zip_safe = False packages = find: install_requires = - localstack>=1.4 diapretty +[options.extras_require] +dev = + localstack-core>=1.4 + [options.entry_points] localstack.extensions = diagnosis-viewer = diagnosis_viewer.extension:DiagnosisViewerExtension diff --git a/hello-world/Makefile b/hello-world/Makefile index 084fa9b..1cc5f9a 100644 --- a/hello-world/Makefile +++ b/hello-world/Makefile @@ -9,7 +9,6 @@ $(VENV_ACTIVATE): setup.py setup.cfg test -d .venv || $(VENV_BIN) .venv $(VENV_RUN); pip install --upgrade pip setuptools plux wheel $(VENV_RUN); pip install --upgrade black isort pyproject-flake8 flake8-black flake8-isort - $(VENV_RUN); pip install -e . touch $(VENV_DIR)/bin/activate clean: @@ -25,7 +24,7 @@ format: ## Run black and isort code formatter $(VENV_RUN); python -m isort helloworld; python -m black helloworld install: venv - $(VENV_RUN); python setup.py develop + $(VENV_RUN); python -m pip install -e .[dev] dist: venv $(VENV_RUN); python setup.py sdist bdist_wheel diff --git a/hello-world/setup.cfg b/hello-world/setup.cfg index 2da1782..3e91181 100644 --- a/hello-world/setup.cfg +++ b/hello-world/setup.cfg @@ -12,8 +12,10 @@ author_email = thomas@localstack.cloud [options] zip_safe = False packages = find: -install_requires = - localstack>=1.0 + +[options.extras_require] +dev = + localstack-core>=1.0 [options.entry_points] localstack.extensions = diff --git a/httpbin/Makefile b/httpbin/Makefile index 480979a..ac6aefe 100644 --- a/httpbin/Makefile +++ b/httpbin/Makefile @@ -9,7 +9,6 @@ $(VENV_ACTIVATE): setup.py setup.cfg test -d .venv || $(VENV_BIN) .venv $(VENV_RUN); pip install --upgrade pip setuptools plux wheel $(VENV_RUN); pip install --upgrade black isort pyproject-flake8 flake8-black flake8-isort - $(VENV_RUN); pip install -e . touch $(VENV_DIR)/bin/activate clean: @@ -25,7 +24,7 @@ format: venv $(VENV_RUN); python -m isort .; python -m black . install: venv - $(VENV_RUN); python setup.py develop + $(VENV_RUN); python -m pip install -e .[dev] dist: venv $(VENV_RUN); python setup.py sdist bdist_wheel diff --git a/httpbin/setup.cfg b/httpbin/setup.cfg index f79e303..79c0647 100644 --- a/httpbin/setup.cfg +++ b/httpbin/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = localstack-extension-httpbin -version = 0.1.1 +version = 0.1.2 url = https://github.com/localstack/localstack-extensions/tree/main/httpbin author = LocalStack author_email = info@localstack.cloud @@ -13,7 +13,6 @@ long_description_content_type = text/markdown; charset=UTF-8 zip_safe = False packages = find: install_requires = - localstack>=2.2 # requirements for vendored httpbin Flask MarkupSafe @@ -24,6 +23,10 @@ install_requires = gevent flasgger +[options.extras_require] +dev = + localstack-core>=2.2 + [options.entry_points] localstack.extensions = httpbin = localstack_httpbin.extension:HttpbinExtension diff --git a/mailhog/Makefile b/mailhog/Makefile index 5c13051..e636a5f 100644 --- a/mailhog/Makefile +++ b/mailhog/Makefile @@ -9,7 +9,6 @@ $(VENV_ACTIVATE): setup.py setup.cfg test -d .venv || $(VENV_BIN) .venv $(VENV_RUN); pip install --upgrade pip setuptools plux wheel $(VENV_RUN); pip install --upgrade black isort pyproject-flake8 flake8-black flake8-isort - $(VENV_RUN); pip install -e . touch $(VENV_DIR)/bin/activate clean: @@ -25,7 +24,7 @@ format: ## Run black and isort code formatter $(VENV_RUN); python -m isort mailhog; python -m black mailhog install: venv - $(VENV_RUN); python setup.py develop + $(VENV_RUN); python -m pip install -e .[dev] dist: venv $(VENV_RUN); python setup.py sdist bdist_wheel diff --git a/mailhog/mailhog/extension.py b/mailhog/mailhog/extension.py index 696833c..2347086 100644 --- a/mailhog/mailhog/extension.py +++ b/mailhog/mailhog/extension.py @@ -4,7 +4,12 @@ from localstack import config, constants from localstack.extensions.api import Extension, http -from localstack_ext import config as config_ext + +try: + from localstack.pro.core import config as config_pro +except ImportError: + # TODO remove once we don't need compatibility with <3.6 anymore + from localstack_ext import config as config_pro if TYPE_CHECKING: # conditional import for type checking during development. the actual import is deferred to plugin loading @@ -57,10 +62,10 @@ def on_platform_start(self): LOG.info("starting mailhog server") self.server.start() - if not config_ext.SMTP_HOST: - config_ext.SMTP_HOST = f"localhost:{self.server.smtp_port}" - os.environ["SMTP_HOST"] = config_ext.SMTP_HOST - LOG.info("configuring SMTP host to internal mailhog smtp: %s", config_ext.SMTP_HOST) + if not config_pro.SMTP_HOST: + config_pro.SMTP_HOST = f"localhost:{self.server.smtp_port}" + os.environ["SMTP_HOST"] = config_pro.SMTP_HOST + LOG.info("configuring SMTP host to internal mailhog smtp: %s", config_pro.SMTP_HOST) def on_platform_ready(self): # FIXME: reconcile with LOCALSTACK_HOST. the URL should be reachable from the host (the idea is diff --git a/mailhog/mailhog/package.py b/mailhog/mailhog/package.py index 960a1f2..2da60c7 100644 --- a/mailhog/mailhog/package.py +++ b/mailhog/mailhog/package.py @@ -1,6 +1,7 @@ """ Package for mailhog that downloads the mailhog binary from https://github.com/mailhog/MailHog. """ + import os from functools import lru_cache diff --git a/mailhog/mailhog/server.py b/mailhog/mailhog/server.py index c81d423..b1f799b 100644 --- a/mailhog/mailhog/server.py +++ b/mailhog/mailhog/server.py @@ -1,6 +1,7 @@ """ Tools to run the mailhog service. """ + import logging import os diff --git a/mailhog/setup.cfg b/mailhog/setup.cfg index 0a5ee07..369f45d 100644 --- a/mailhog/setup.cfg +++ b/mailhog/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = localstack-extension-mailhog -version = 0.1.0 +version = 0.1.2 url = https://github.com/localstack/localstack-extensions/tree/main/mailhog author = LocalStack author_email = info@localstack.cloud @@ -12,8 +12,10 @@ long_description_content_type = text/markdown; charset=UTF-8 [options] zip_safe = False packages = find: -install_requires = - localstack>=2.2 + +[options.extras_require] +dev = + localstack-core>=2.2 [options.entry_points] localstack.extensions = diff --git a/miniflare/Makefile b/miniflare/Makefile index 8490197..218eb31 100644 --- a/miniflare/Makefile +++ b/miniflare/Makefile @@ -9,7 +9,6 @@ $(VENV_ACTIVATE): setup.py setup.cfg test -d .venv || $(VENV_BIN) .venv $(VENV_RUN); pip install --upgrade pip setuptools plux wheel $(VENV_RUN); pip install --upgrade black isort pyproject-flake8 flake8-black flake8-isort - $(VENV_RUN); pip install -e . touch $(VENV_DIR)/bin/activate clean: @@ -25,7 +24,7 @@ format: ## Run black and isort code formatter $(VENV_RUN); python -m isort .; python -m black . install: venv - $(VENV_RUN); python setup.py develop + $(VENV_RUN); python -m pip install -e .[dev] dist: venv $(VENV_RUN); python setup.py sdist bdist_wheel diff --git a/miniflare/setup.cfg b/miniflare/setup.cfg index c8aaaed..d668b3e 100644 --- a/miniflare/setup.cfg +++ b/miniflare/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = localstack-extension-miniflare -version = 0.1.1 +version = 0.1.2 summary = LocalStack Extension: Miniflare description = This extension makes Miniflare (dev environment for Cloudflare workers) available directly in LocalStack long_description = file: README.md @@ -12,8 +12,10 @@ author_email = waldemar@localstack.cloud [options] zip_safe = False packages = find: -install_requires = - localstack>=1.0.0 + +[options.extras_require] +dev = + localstack-core>=1.0.0 [options.entry_points] localstack.extensions = diff --git a/openai/Makefile b/openai/Makefile index d63477e..e299dc6 100644 --- a/openai/Makefile +++ b/openai/Makefile @@ -9,7 +9,6 @@ $(VENV_ACTIVATE): setup.py setup.cfg test -d .venv || $(VENV_BIN) .venv $(VENV_RUN); pip install --upgrade pip setuptools plux wheel $(VENV_RUN); pip install --upgrade black isort pyproject-flake8 flake8-black flake8-isort - $(VENV_RUN); pip install -e . touch $(VENV_DIR)/bin/activate clean: @@ -25,7 +24,7 @@ format: ## Run black and isort code formatter $(VENV_RUN); python -m isort .; python -m black . install: venv - $(VENV_RUN); python setup.py develop + $(VENV_RUN); python -m pip install -e .[dev] dist: venv $(VENV_RUN); python setup.py sdist bdist_wheel diff --git a/openai/setup.cfg b/openai/setup.cfg index 643dc48..681e4e6 100644 --- a/openai/setup.cfg +++ b/openai/setup.cfg @@ -24,7 +24,6 @@ zip_safe = False packages = find: install_requires = faker>=8.12.1 - localstack>=3.1 plux>=1.3 rolo>=0.3 test_requires = @@ -33,6 +32,7 @@ test_requires = [options.extras_require] dev = + localstack-core>=3.1 openai>=0.10.2,<1.0 pytest>=6.2.4 black==22.3.0 diff --git a/stripe/Makefile b/stripe/Makefile index 9b7963c..75b92e4 100644 --- a/stripe/Makefile +++ b/stripe/Makefile @@ -32,7 +32,7 @@ dist: venv $(VENV_ACTIVATE); python setup.py sdist bdist_wheel install: venv - $(VENV_ACTIVATE); python setup.py install + $(VENV_ACTIVATE); python -m pip install -e .[dev] upload: venv dist $(VENV_ACTIVATE); pip install --upgrade twine; twine upload dist/* diff --git a/stripe/setup.cfg b/stripe/setup.cfg index 1480f97..4de70aa 100644 --- a/stripe/setup.cfg +++ b/stripe/setup.cfg @@ -29,13 +29,13 @@ setup_requires = install_requires = stevedore>=3.4 plux>=1.3 - localstack>=1.0 localstack-localstripe>=1.13.8 test_requires = pytest>=6.2.4 [options.extras_require] dev = + localstack-core>=1.0 pytest>=6.2.4 black==22.3.0 isort==5.10.1 diff --git a/template/README.md b/template/README.md index 0213a57..3ed9bbb 100644 --- a/template/README.md +++ b/template/README.md @@ -1,6 +1,9 @@ Extension Template ================== +> [!NOTE] +> This template is used for localstack CLI versions <= 3.6.0. For later version see https://github.com/localstack/localstack-extensions/tree/main/templates + This is a [cookiecutter](https://github.com/cookiecutter/cookiecutter) template that is used when you invoke. ```console diff --git a/template/{{cookiecutter.project_slug}}/Makefile b/template/{{cookiecutter.project_slug}}/Makefile index ecc8d87..efd455a 100644 --- a/template/{{cookiecutter.project_slug}}/Makefile +++ b/template/{{cookiecutter.project_slug}}/Makefile @@ -8,7 +8,6 @@ venv: $(VENV_ACTIVATE) $(VENV_ACTIVATE): setup.py setup.cfg test -d .venv || $(VENV_BIN) .venv $(VENV_RUN); pip install --upgrade pip setuptools plux - $(VENV_RUN); pip install -e . touch $(VENV_DIR)/bin/activate clean: @@ -18,7 +17,7 @@ clean: rm -rf *.egg-info/ install: venv - $(VENV_RUN); python setup.py develop + $(VENV_RUN); python -m pip install -e .[dev] dist: venv $(VENV_RUN); python -m build diff --git a/templates/basic/{{cookiecutter.project_slug}}/Makefile b/templates/basic/{{cookiecutter.project_slug}}/Makefile index ecc8d87..7059d14 100644 --- a/templates/basic/{{cookiecutter.project_slug}}/Makefile +++ b/templates/basic/{{cookiecutter.project_slug}}/Makefile @@ -5,10 +5,10 @@ VENV_RUN = . $(VENV_ACTIVATE) venv: $(VENV_ACTIVATE) -$(VENV_ACTIVATE): setup.py setup.cfg +$(VENV_ACTIVATE): pyproject.toml test -d .venv || $(VENV_BIN) .venv $(VENV_RUN); pip install --upgrade pip setuptools plux - $(VENV_RUN); pip install -e . + $(VENV_RUN); pip install -e .[dev] touch $(VENV_DIR)/bin/activate clean: @@ -18,7 +18,7 @@ clean: rm -rf *.egg-info/ install: venv - $(VENV_RUN); python setup.py develop + $(VENV_RUN); python -m plux entrypoints dist: venv $(VENV_RUN); python -m build diff --git a/templates/basic/{{cookiecutter.project_slug}}/pyproject.toml b/templates/basic/{{cookiecutter.project_slug}}/pyproject.toml index dacb4fd..af0a134 100644 --- a/templates/basic/{{cookiecutter.project_slug}}/pyproject.toml +++ b/templates/basic/{{cookiecutter.project_slug}}/pyproject.toml @@ -14,6 +14,8 @@ authors = [ ] keywords = ["localstack", "localstack-extension", "extension"] classifiers = [] +dependencies = [ +] [project.urls] Homepage = "https://github.com/{{ cookiecutter.github_username }}/{{ cookiecutter.project_slug }}" diff --git a/templates/basic/{{cookiecutter.project_slug}}/setup.cfg b/templates/basic/{{cookiecutter.project_slug}}/setup.cfg deleted file mode 100644 index 32c9e6c..0000000 --- a/templates/basic/{{cookiecutter.project_slug}}/setup.cfg +++ /dev/null @@ -1,22 +0,0 @@ -[metadata] -name = {{ cookiecutter.project_slug }} -version = {{ cookiecutter.version }} -summary = LocalStack Extension: {{ cookiecutter.project_name }} -url = https://github.com/{{ cookiecutter.github_username }}/{{ cookiecutter.project_slug }} -author = {{ cookiecutter.full_name }} -author_email = {{ cookiecutter.email }} -description = {{ cookiecutter.project_short_description }} -long_description = file: README.md -long_description_content_type = text/markdown; charset=UTF-8 - -[options] -zip_safe = False -packages = find: - -[options.extras_require] -test= - localstack-core>=2.2 - -[options.entry_points] -localstack.extensions = - {{ cookiecutter.project_slug }} = {{ cookiecutter.module_name }}.extension:{{ cookiecutter.class_name }} diff --git a/terraform-init/Makefile b/terraform-init/Makefile new file mode 100644 index 0000000..4ade53f --- /dev/null +++ b/terraform-init/Makefile @@ -0,0 +1,35 @@ +VENV_BIN = python3 -m venv +VENV_DIR ?= .venv +VENV_ACTIVATE = $(VENV_DIR)/bin/activate +VENV_RUN = . $(VENV_ACTIVATE) + +venv: $(VENV_ACTIVATE) + +$(VENV_ACTIVATE): setup.py setup.cfg + test -d .venv || $(VENV_BIN) .venv + $(VENV_RUN); pip install --upgrade pip setuptools plux build + $(VENV_RUN); pip install --upgrade black isort + touch $(VENV_DIR)/bin/activate + +clean: + rm -rf .venv/ + rm -rf build/ + rm -rf .eggs/ + rm -rf *.egg-info/ + +install: venv + $(VENV_RUN); python -m pip install -e .[dev] + +format: venv + $(VENV_RUN); python -m isort .; python -m black . + +dist: venv + $(VENV_RUN); python -m build + +publish: clean-dist venv dist + $(VENV_RUN); pip install --upgrade twine; twine upload dist/* + +clean-dist: clean + rm -rf dist/ + +.PHONY: clean clean-dist dist install publish diff --git a/terraform-init/README.md b/terraform-init/README.md new file mode 100644 index 0000000..d122683 --- /dev/null +++ b/terraform-init/README.md @@ -0,0 +1,110 @@ +Use Terraform files in LocalStack init hooks +============================================ + +LocalStack Extension for using Terraform files in [init hooks](https://docs.localstack.cloud/references/init-hooks/). + +> [!WARNING] +> This extension is experimental and subject to change. + +> [!NOTE] +> The extension is designed for simple self-contained terraform files, not complex projects or modules. +> If you have larger projects, then we recommend running them from the host. + +## Usage + +* Start localstack with `EXTENSION_AUTO_INSTALL="localstack-extension-terraform-init"` +* Mount a `main.tf` file into `/etc/localstack/init/ready.d` + +When LocalStack starts up, it will install the extension, which in turn install `terraform` and `tflocal` into the container. +If one of the init stage directories contain a `main.tf`, the extension will run `tflocal init` and `tflocal apply` on that directory. + +> [!NOTE] +> Terraform state files will be created in your host directory if you mounted an entire folder into `/etc/localstack/init/ready.d`. +> These files are created from within the container using the container user, so you may need `sudo` to remove the files from your host. +> If you only mount the `main.tf` file, not an entire directory, localstack will have to download the AWS terraform provider every time during `tflocal init`. +> +### Example + +Example `main.tf`: +```hcl +resource "aws_s3_bucket" "example" { + bucket = "my-tf-test-bucket" + + tags = { + Name = "My bucket" + Environment = "Dev" + } +} +``` + +Start LocalStack Pro with mounted `main.tf`: + +```console +localstack start \ + -e EXTENSION_AUTO_INSTALL="localstack-extension-terraform-init" \ + -v ./main.tf:/etc/localstack/init/ready.d/main.tf +``` + +Or, if you use a docker-compose file: + +```yaml +services: + localstack: + container_name: "localstack-main" + image: localstack/localstack-pro # required for Pro + ports: + - "127.0.0.1:4566:4566" # LocalStack Gateway + environment: + # Activate LocalStack Pro: https://docs.localstack.cloud/getting-started/auth-token/ + - LOCALSTACK_AUTH_TOKEN=${LOCALSTACK_AUTH_TOKEN:?} + - EXTENSION_AUTO_LOAD=localstack-extension-terraform-init" + volumes: + # you could also place your main.tf in `./ready.d` and set "./ready.d:/etc/localstack/init/ready.d" + - "./main.tf:/etc/localstack/init/ready.d/main.tf" + - "./volume:/var/lib/localstack" + - "/var/run/docker.sock:/var/run/docker.sock" +``` + +In a new terminal window, you can wait for localstack to complete and then print the created s3 buckets. + +```console +localstack wait && awslocal s3 ls +``` + +The logs should show something like: + +``` +2024-06-26T20:36:19.946 INFO --- [ady_monitor)] l.extension : Applying terraform project from file /etc/localstack/init/ready.d/main.tf +2024-06-26T20:36:19.946 DEBUG --- [ady_monitor)] localstack.utils.run : Executing command: ['tflocal', '-chdir=/etc/localstack/init/ready.d', 'init', '-input=false'] +2024-06-26T20:36:26.864 DEBUG --- [ady_monitor)] localstack.utils.run : Executing command: ['tflocal', '-chdir=/etc/localstack/init/ready.d', 'apply', '-auto-approve'] +``` + +## Install local development version + +To install the extension into localstack in developer mode, you will need Python 3.10, and create a virtual environment in the extensions project. + +In the newly generated project, simply run + +```bash +make install +``` + +Then, to enable the extension for LocalStack, run + +```bash +localstack extensions dev enable . +``` + +You can then start LocalStack with `EXTENSION_DEV_MODE=1` to load all enabled extensions: + +```bash +EXTENSION_DEV_MODE=1 localstack start +``` + +## Install from GitHub repository + +To distribute your extension, simply upload it to your github account. Your extension can then be installed via: + +```bash +localstack extensions install "git+https://github.com/localstack/localstack-extensions/#egg=localstack-extension-terraform-init&subdirectory=terraform-init" +``` diff --git a/terraform-init/localstack_terraform_init/__init__.py b/terraform-init/localstack_terraform_init/__init__.py new file mode 100644 index 0000000..c311b1a --- /dev/null +++ b/terraform-init/localstack_terraform_init/__init__.py @@ -0,0 +1 @@ +name = "localstack_terraform_init" diff --git a/terraform-init/localstack_terraform_init/extension.py b/terraform-init/localstack_terraform_init/extension.py new file mode 100644 index 0000000..b9ef81e --- /dev/null +++ b/terraform-init/localstack_terraform_init/extension.py @@ -0,0 +1,79 @@ +import logging +import os +from typing import List + +from localstack import config +from localstack.extensions.api import Extension +from localstack.packages import InstallTarget, Package, PackageInstaller +from localstack.packages.core import PythonPackageInstaller +from localstack.packages.terraform import terraform_package +from localstack.runtime.init import ScriptRunner +from localstack.utils.run import run + +LOG = logging.getLogger(__name__) + + +class TflocalInitExtension(Extension): + # the extension itself is just used for discoverability + name = "localstack-terraform-init" + + def on_extension_load(self): + logging.getLogger("localstack_terraform_init").setLevel( + logging.DEBUG if config.DEBUG else logging.INFO + ) + + +class TflocalPackage(Package): + def __init__(self, default_version: str = "0.18.2"): + super().__init__(name="terraform_local", default_version=default_version) + + def _get_installer(self, version: str) -> PackageInstaller: + return TflocalPackageInstaller(version) + + def get_versions(self) -> List[str]: + return [self.default_version] + + +class TflocalPackageInstaller(PythonPackageInstaller): + def __init__(self, version: str): + super().__init__("terraform_local", version) + + +tflocal_package = TflocalPackage() + + +class TflocalScriptRunner(ScriptRunner): + name = "tflocal" + + def load(self, *args, **kwargs): + terraform_package.install() + tflocal_package.install() + + def should_run(self, script_file: str) -> bool: + if os.path.basename(script_file) == "main.tf": + return True + return False + + def run(self, path: str) -> None: + # create path to find ``terraform`` and ``tflocal`` binaries + # TODO: better way to define path + tf_path = terraform_package.get_installed_dir() + install_dir = tflocal_package.get_installer()._get_install_dir( + InstallTarget.VAR_LIBS + ) + tflocal_path = f"{install_dir}/bin" + env_path = f"{tflocal_path}:{tf_path}:{os.getenv('PATH')}" + + LOG.info("Applying terraform project from file %s", path) + # run tflocal + workdir = os.path.dirname(path) + LOG.debug("Initializing terraform provider in %s", workdir) + run( + ["tflocal", f"-chdir={workdir}", "init", "-input=false"], + env_vars={"PATH": env_path}, + ) + LOG.debug("Applying terraform file %s", path) + run( + ["tflocal", f"-chdir={workdir}", "apply", "-auto-approve"], + env_vars={"PATH": env_path}, + ) diff --git a/terraform-init/setup.cfg b/terraform-init/setup.cfg new file mode 100644 index 0000000..10bb9e3 --- /dev/null +++ b/terraform-init/setup.cfg @@ -0,0 +1,26 @@ +[metadata] +name = localstack-extension-terraform-init +version = 0.2.1 +summary = LocalStack Extension: LocalStack Terraform Init +url = https://github.com/localstack/localstack-extensions/tree/main/terraform-init +author = Thomas Rausch +author_email = thomas@localstack.cloud +description = LocalStack Extension for using Terraform files in init hooks +long_description = file: README.md +long_description_content_type = text/markdown; charset=UTF-8 + +[options] +zip_safe = False +packages = find: +install_requires = + plux + +[options.extras_require] +dev = + localstack-core>=3.4 + +[options.entry_points] +localstack.extensions = + localstack-terraform-init = localstack_terraform_init.extension:TflocalInitExtension +localstack.init.runner= + tflocal = localstack_terraform_init.extension:TflocalScriptRunner diff --git a/terraform-init/setup.py b/terraform-init/setup.py new file mode 100644 index 0000000..c823345 --- /dev/null +++ b/terraform-init/setup.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python +from setuptools import setup + +setup()