feat: Support list of env files

.gitignore

@@ -115,7 +115,7 @@ web_modules/
 
 # dotenv environment variables file
 .env
-.env.test
+.env.*
 
 # parcel-bundler cache (https://parceljs.org/)
 .cache
@@ -385,4 +385,8 @@ archive/
 _build/
 
 # Pytest Setup
-/module/
+/module/
+
+# Testing
+seedfarmer.yaml
+tmp-metadata

CHANGELOG.md

@@ -8,6 +8,8 @@ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a Ch
 
 ### New
 
+- support list of env files using `--env-file`
+
 ### Changes
 - added `--update-seedkit` support to `apply` 
   - SeedFarmer will no longer try to update the seedkit on every request

docs/source/manifests.md

@@ -364,6 +364,8 @@ In this example, the `glue-db-suffix` parameter will be exposed to the CodeBuild
 ### Environment Variables
 `SeedFarmer` supports using [Dotenv](https://github.com/theskumar/python-dotenv) for dynamic replacement.  When a file named `.env` is placed at the projecr root (where `seedfarmer.yaml` resides), any value in a manifest with a key of `envVariable` will be matched and replaced with the corresponding environment variable.  You can pass in overriding `.env` files by using the `--env-file` on CLI command invocation.
 
+`SeedFarmer` also supports passing multiple `.env`, by using `--env-file` multiple times. For example: `seedfarmer apply --env-file .env.shared --env-file .env.secret`. If the same value is present in multiple `.env` files, subsequent files will override the value from the previous one. In the aforementioned example, values from `.env.secret` will override values from `.env.shared`.
+
 ```yaml
 name: opensearch
 path: modules/core/opensearch/

seedfarmer/__main__.py

@@ -14,16 +14,15 @@
 
 
 import logging
-import os
-from typing import Optional
+from typing import List, Optional
 
 import click
-from dotenv import load_dotenv
 
 import seedfarmer
 from seedfarmer import DEBUG_LOGGING_FORMAT, commands, config, enable_debug
 from seedfarmer.cli_groups import bootstrap, init, list, metadata, projectpolicy, remove, store
 from seedfarmer.output_utils import print_bolded
+from seedfarmer.utils import load_dotenv_files
 
 _logger: logging.Logger = logging.getLogger(__name__)
 
@@ -65,8 +64,10 @@ def version() -> None:
 )
 @click.option(
     "--env-file",
-    default=".env",
+    "env_files",
+    default=[".env"],
     help="A relative path to the .env file to load environment variables from",
+    multiple=True,
     required=False,
 )
 @click.option(
@@ -116,7 +117,7 @@ def apply(
     profile: Optional[str],
     region: Optional[str],
     qualifier: Optional[str],
-    env_file: str,
+    env_files: List[str],
     debug: bool,
     dry_run: bool,
     show_manifest: bool,
@@ -129,7 +130,7 @@ def apply(
         enable_debug(format=DEBUG_LOGGING_FORMAT)
 
     # Load environment variables from .env file if it exists
-    load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True)
+    load_dotenv_files(config.OPS_ROOT, env_files)
 
     _logger.info("Apply request with manifest %s", spec)
     if dry_run:
@@ -186,8 +187,10 @@ def apply(
 )
 @click.option(
     "--env-file",
-    default=".env",
+    "env_files",
+    default=[".env"],
     help="A relative path to the .env file to load environment variables from",
+    multiple=True,
     required=False,
 )
 @click.option(
@@ -217,7 +220,7 @@ def destroy(
     profile: Optional[str],
     region: Optional[str],
     qualifier: Optional[str],
-    env_file: str,
+    env_files: List[str],
     debug: bool,
     enable_session_timeout: bool,
     session_timeout_interval: int,
@@ -227,7 +230,7 @@ def destroy(
         enable_debug(format=DEBUG_LOGGING_FORMAT)
 
     # Load environment variables from .env file if it exists
-    load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True)
+    load_dotenv_files(config.OPS_ROOT, env_files)
 
     # MUST use seedfarmer.yaml so we can initialize codeseeder configs
     project = config.PROJECT

seedfarmer/cli_groups/_list_group.py

@@ -14,12 +14,10 @@
 
 import json
 import logging
-import os
 import sys
-from typing import Optional
+from typing import List, Optional
 
 import click
-from dotenv import load_dotenv
 
 import seedfarmer.mgmt.build_info as bi
 import seedfarmer.mgmt.deploy_utils as du
@@ -33,6 +31,7 @@
     print_manifest_inventory,
 )
 from seedfarmer.services.session_manager import SessionManager
+from seedfarmer.utils import load_dotenv_files
 
 _logger: logging.Logger = logging.getLogger(__name__)
 
@@ -110,8 +109,10 @@ def list() -> None:
 )
 @click.option(
     "--env-file",
-    default=".env",
+    "env_files",
+    default=[".env"],
     help="A relative path to the .env file to load environment variables from",
+    multiple=True,
     required=False,
 )
 @click.option(
@@ -128,7 +129,7 @@ def list_dependencies(
     profile: Optional[str],
     region: Optional[str],
     qualifier: Optional[str],
-    env_file: str,
+    env_files: List[str],
     debug: bool,
 ) -> None:
     if debug:
@@ -137,7 +138,8 @@ def list_dependencies(
 
     if project is None:
         project = _load_project()
-    load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True)
+
+    load_dotenv_files(config.OPS_ROOT, env_files=env_files)
 
     SessionManager().get_or_create(project_name=project, profile=profile, region_name=region, qualifier=qualifier)
     dep_manifest = du.generate_deployed_manifest(deployment_name=deployment, skip_deploy_spec=True)
@@ -203,8 +205,10 @@ def list_dependencies(
 )
 @click.option(
     "--env-file",
-    default=".env",
+    "env_files",
+    default=[".env"],
     help="A relative path to the .env file to load environment variables from",
+    multiple=True,
     required=False,
 )
 @click.option(
@@ -221,7 +225,7 @@ def list_deployspec(
     profile: Optional[str],
     region: Optional[str],
     qualifier: Optional[str],
-    env_file: str,
+    env_files: List[str],
     debug: bool,
 ) -> None:
     if debug:
@@ -230,7 +234,8 @@ def list_deployspec(
 
     if project is None:
         project = _load_project()
-    load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True)
+
+    load_dotenv_files(config.OPS_ROOT, env_files=env_files)
 
     session = SessionManager().get_or_create(
         project_name=project, profile=profile, region_name=region, qualifier=qualifier
@@ -310,8 +315,10 @@ def list_deployspec(
 )
 @click.option(
     "--env-file",
-    default=".env",
+    "env_files",
+    default=[".env"],
     help="A relative path to the .env file to load environment variables from",
+    multiple=True,
     required=False,
 )
 @click.option(
@@ -328,7 +335,7 @@ def list_module_metadata(
     profile: Optional[str],
     region: Optional[str],
     qualifier: Optional[str],
-    env_file: str,
+    env_files: List[str],
     export_local_env: bool,
     debug: bool,
 ) -> None:
@@ -338,7 +345,8 @@ def list_module_metadata(
 
     if project is None:
         project = _load_project()
-    load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True)
+
+    load_dotenv_files(config.OPS_ROOT, env_files=env_files)
 
     session = SessionManager().get_or_create(
         project_name=project, profile=profile, region_name=region, qualifier=qualifier
@@ -407,8 +415,10 @@ def list_module_metadata(
 )
 @click.option(
     "--env-file",
-    default=".env",
+    "env_files",
+    default=[".env"],
     help="A relative path to the .env file to load environment variables from",
+    multiple=True,
     required=False,
 )
 @click.option(
@@ -423,7 +433,7 @@ def list_all_module_metadata(
     profile: Optional[str],
     region: Optional[str],
     qualifier: Optional[str],
-    env_file: str,
+    env_files: List[str],
     debug: bool,
 ) -> None:
     if debug:
@@ -432,7 +442,8 @@ def list_all_module_metadata(
 
     if project is None:
         project = _load_project()
-    load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True)
+
+    load_dotenv_files(config.OPS_ROOT, env_files=env_files)
 
     session = SessionManager().get_or_create(
         project_name=project, profile=profile, region_name=region, qualifier=qualifier
@@ -500,8 +511,10 @@ def list_all_module_metadata(
 )
 @click.option(
     "--env-file",
-    default=".env",
+    "env_files",
+    default=[".env"],
     help="A relative path to the .env file to load environment variables from",
+    multiple=True,
     required=False,
 )
 @click.option(
@@ -516,7 +529,7 @@ def list_modules(
     profile: Optional[str],
     region: Optional[str],
     qualifier: Optional[str],
-    env_file: str,
+    env_files: List[str],
     debug: bool,
 ) -> None:
     if debug:
@@ -525,7 +538,9 @@ def list_modules(
 
     if project is None:
         project = _load_project()
-    load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True)
+
+    load_dotenv_files(config.OPS_ROOT, env_files=env_files)
+
     SessionManager().get_or_create(project_name=project, profile=profile, region_name=region, qualifier=qualifier)
 
     dep_manifest = du.generate_deployed_manifest(deployment_name=deployment, skip_deploy_spec=True)
@@ -647,8 +662,10 @@ def list_deployments(
 )
 @click.option(
     "--env-file",
-    default=".env",
+    "env_files",
+    default=[".env"],
     help="A relative path to the .env file to load environment variables from",
+    multiple=True,
     required=False,
 )
 @click.option(
@@ -666,7 +683,7 @@ def list_build_env_params(
     profile: Optional[str],
     region: Optional[str],
     qualifier: Optional[str],
-    env_file: str,
+    env_files: List[str],
     export_local_env: str,
     debug: bool,
 ) -> None:
@@ -678,7 +695,8 @@ def list_build_env_params(
 
     if project is None:
         project = _load_project()
-    load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True)
+
+    load_dotenv_files(config.OPS_ROOT, env_files=env_files)
 
     session = SessionManager().get_or_create(
         project_name=project, profile=profile, region_name=region, qualifier=qualifier

seedfarmer/utils.py

@@ -14,11 +14,13 @@
 
 import hashlib
 import logging
-from typing import Optional
+import os
+from typing import List, Optional
 
 import humps
 import yaml
 from boto3 import Session
+from dotenv import dotenv_values, load_dotenv
 
 from seedfarmer.services._service_utils import get_region, get_sts_identity_info
 
@@ -158,3 +160,26 @@ def get_deployment_role_arn(
 
 def valid_qualifier(qualifer: str) -> bool:
     return True if ((len(qualifer) <= 6) and qualifer.isalnum()) else False
+
+
+def load_dotenv_files(root_path: str, env_files: List[str]) -> None:
+    """
+    Load the environment variables from the .env files
+
+    Parameters
+    ----------
+    root_path : str
+        The path to the root of the project
+    env_files : List[str]
+        The list of the .env files to load
+    """
+    loaded_values = {}
+
+    for env_file in env_files:
+        _logger.info("Loading environment variables from %s", env_file)
+        dotenv_path = os.path.join(root_path, env_file)
+
+        load_dotenv(dotenv_path=dotenv_path, verbose=True, override=True)
+        loaded_values.update(dotenv_values(dotenv_path, verbose=True))
+
+    _logger.debug("Loaded environment variables: %s", loaded_values)