feat: support terraform nested directories project structure (#5745)

* feat: support terraform project structure that does not have everything in root module directory

* apply PR comments

samcli/commands/_utils/custom_options/hook_name_option.py

@@ -87,9 +87,19 @@ def _call_prepare_hook(self, iac_hook_wrapper, opts):
             aws_region = opts.get("region")
             skip_prepare_infra = opts.get("skip_prepare_infra", False)
             plan_file = opts.get("terraform_plan_file")
+            project_root_dir = opts.get(
+                "terraform_project_root_path",
+            )
 
             metadata_file = iac_hook_wrapper.prepare(
-                output_dir_path, iac_project_path, debug, aws_profile, aws_region, skip_prepare_infra, plan_file
+                output_dir_path,
+                iac_project_path,
+                debug,
+                aws_profile,
+                aws_region,
+                skip_prepare_infra,
+                plan_file,
+                project_root_dir,
             )
 
             LOG.info("Prepare hook completed and metadata file generated at: %s", metadata_file)
@@ -111,6 +121,23 @@ def _validate_build_command_parameters(command_name, opts):
     if command_name == "build" and opts.get("use_container") and not opts.get("build_image"):
         raise click.UsageError("Missing required parameter --build-image.")
 
+    # validate that terraform-project-root-path is a parent path of the current directory, or it is a relative path
+    project_root_dir = opts.get("terraform_project_root_path")
+    if (
+        command_name == "build"
+        and project_root_dir
+        and os.path.isabs(project_root_dir)
+        and not os.getcwd().startswith(project_root_dir)
+    ):
+        LOG.debug(
+            f"the provided path {project_root_dir} as terraform project path is not a parent of the current directory "
+            f"{os.getcwd()}"
+        )
+        raise click.UsageError(
+            f"{project_root_dir} is not a valid value for Terraform Project Root Path. It should be a parent of the "
+            f"current directory that contains the root module of the terraform project."
+        )
+
 
 def _check_experimental_flag(hook_name, command_name, opts, default_map):
     # check beta-feature

samcli/commands/_utils/options.py

@@ -823,6 +823,41 @@ def terraform_plan_file_option(f):
     return terraform_plan_file_click_option()(f)
 
 
+def terraform_project_root_path_callback(ctx, param, provided_value):
+    """
+    Callback for --terraform-project-root-path to check if --hook-name is also specified
+
+    Parameters
+    ----------
+    ctx: click.core.Context
+        Click context
+    param: click.Option
+        Parameter properties
+    provided_value: bool
+        True if option was provided
+    """
+    is_option_provided = provided_value or ctx.default_map.get("terraform_project_root_path")
+    is_hook_provided = ctx.params.get("hook_name") or ctx.default_map.get("hook_name")
+
+    if is_option_provided and not is_hook_provided:
+        raise click.BadOptionUsage(option_name=param.name, ctx=ctx, message="Missing option --hook-name")
+
+
+def terraform_project_root_path_click_option():
+    return click.option(
+        "--terraform-project-root-path",
+        type=click.Path(),
+        required=False,
+        callback=terraform_project_root_path_callback,
+        help="Used for passing the Terraform project root directory path. Current directory will be used as a default "
+        "value, if this parameter is not provided.",
+    )
+
+
+def terraform_project_root_path_option(f):
+    return terraform_project_root_path_click_option()(f)
+
+
 def build_image_click_option(cls):
     return click.option(
         "--build-image",

samcli/commands/build/command.py

@@ -23,6 +23,7 @@
     build_image_option,
     hook_name_click_option,
     terraform_plan_file_option,
+    terraform_project_root_path_option,
 )
 from samcli.commands._utils.option_value_processor import process_env_var, process_image_options
 from samcli.cli.main import pass_context, common_options as cli_framework_options, aws_creds_options, print_cmdline_args
@@ -72,6 +73,7 @@
 )
 @configuration_option(provider=ConfigProvider(section="parameters"))
 @terraform_plan_file_option
+@terraform_project_root_path_option
 @hook_name_click_option(
     force_prepare=True,
     invalid_coexist_options=["t", "template-file", "template", "parameter-overrides"],
@@ -157,7 +159,8 @@ def cli(
     hook_name: Optional[str],
     skip_prepare_infra: bool,
     mount_with,
-    terraform_plan_file,
+    terraform_plan_file: Optional[str],
+    terraform_project_root_path: Optional[str],
 ) -> None:
     """
     `sam build` command entry point

samcli/commands/build/core/options.py

@@ -37,7 +37,7 @@
 
 TEMPLATE_OPTIONS: List[str] = ["parameter_overrides"]
 
-TERRAFORM_HOOK_OPTIONS: List[str] = ["terraform_plan_file"]
+TERRAFORM_HOOK_OPTIONS: List[str] = ["terraform_plan_file", "terraform_project_root_path"]
 
 ALL_OPTIONS: List[str] = (
     REQUIRED_OPTIONS

samcli/hook_packages/terraform/hooks/prepare/enrich.py

@@ -50,6 +50,7 @@ def enrich_resources_and_generate_makefile(
     output_directory_path: str,
     terraform_application_dir: str,
     lambda_resources_to_code_map: Dict,
+    project_root_dir: str,
 ) -> None:
     """
     Use the sam metadata resources to enrich the mapped resources and to create a Makefile with a rule for
@@ -64,9 +65,11 @@ def enrich_resources_and_generate_makefile(
     output_directory_path: str
         the output directory path to write the generated metadata and makefile
     terraform_application_dir: str
-        the terraform project root directory
+        the terraform configuration root module directory.
     lambda_resources_to_code_map: Dict
         The map between lambda resources code path, and lambda resources logical ids
+    project_root_dir: str
+        the project root directory where terraform configurations, src code, and other modules exist
     """
 
     python_command_name = _get_python_command_name()
@@ -102,6 +105,7 @@ def enrich_resources_and_generate_makefile(
                 logical_id,
                 terraform_application_dir,
                 output_directory_path,
+                project_root_dir,
             )
 
             # get makefile rule for resource
@@ -121,6 +125,7 @@ def _enrich_zip_lambda_function(
     cfn_lambda_function_logical_id: str,
     terraform_application_dir: str,
     output_directory_path: str,
+    project_root_dir: str,
 ):
     """
     Use the sam metadata resources to enrich the zip lambda function.
@@ -136,7 +141,9 @@ def _enrich_zip_lambda_function(
     output_directory_path: str
         the output directory path to write the generated metadata and makefile
     terraform_application_dir: str
-        the terraform project root directory
+        the terraform configuration root module directory.
+    project_root_dir: str
+        the project root directory where terraform configurations, src code, and other modules exist
     """
     sam_metadata_resource_address = sam_metadata_resource.get("address")
     if not sam_metadata_resource_address:
@@ -168,6 +175,7 @@ def _enrich_zip_lambda_function(
         output_directory_path,
         terraform_application_dir,
         CFN_CODE_PROPERTIES[CFN_AWS_LAMBDA_FUNCTION],
+        project_root_dir,
     )
 
 
@@ -177,6 +185,7 @@ def _enrich_image_lambda_function(
     cfn_lambda_function_logical_id: str,
     terraform_application_dir: str,
     output_directory_path: str,
+    project_root_dir: str,
 ):
     """
     Use the sam metadata resources to enrich the image lambda function.
@@ -192,7 +201,9 @@ def _enrich_image_lambda_function(
     output_directory_path: str
         the output directory path to write the generated metadata and makefile
     terraform_application_dir: str
-        the terraform project root directory
+        the terraform configuration root module directory.
+    project_root_dir: str
+        the project root directory where terraform configurations, src code, and other modules exist
     """
     sam_metadata_resource_address = sam_metadata_resource.get("address")
     if not sam_metadata_resource_address:
@@ -266,6 +277,7 @@ def _enrich_lambda_layer(
     cfn_lambda_layer_logical_id: str,
     terraform_application_dir: str,
     output_directory_path: str,
+    project_root_dir: str,
 ) -> None:
     """
     Use the sam metadata resources to enrich the lambda layer.
@@ -281,7 +293,9 @@ def _enrich_lambda_layer(
     output_directory_path: str
        the output directory path to write the generated metadata and makefile
     terraform_application_dir: str
-       the terraform project root directory
+       the terraform configuration root module directory.
+    project_root_dir: str
+        the project root directory where terraform configurations, src code, and other modules exist
     """
     sam_metadata_resource_address = sam_metadata_resource.get("address")
     if not sam_metadata_resource_address:
@@ -312,6 +326,7 @@ def _enrich_lambda_layer(
         output_directory_path,
         terraform_application_dir,
         CFN_CODE_PROPERTIES[CFN_AWS_LAMBDA_LAYER_VERSION],
+        project_root_dir,
     )
 
 
@@ -577,6 +592,7 @@ def _set_zip_metadata_resources(
     output_directory_path: str,
     terraform_application_dir: str,
     code_property: str,
+    project_root_dir: str,
 ) -> None:
     """
     Update the CloudFormation resource metadata with the enrichment properties from the TF resource
@@ -590,9 +606,11 @@ def _set_zip_metadata_resources(
     output_directory_path: str
         The directory where to find the Makefile the path to be copied into the temp dir.
     terraform_application_dir: str
-        The working directory from which to run the Makefile.
+        the terraform configuration root module directory from which to run the Makefile.
     code_property:
         The property in the configuration used to denote the code e.g. "Code" or "Content"
+    project_root_dir: str
+        the project root directory where terraform configurations, src code, and other modules exist
     """
     resource_properties = resource.get("Properties", {})
     resource_properties[code_property] = cfn_source_code_path
@@ -602,9 +620,7 @@ def _set_zip_metadata_resources(
     resource["Metadata"]["BuildMethod"] = "makefile"
     resource["Metadata"]["ContextPath"] = output_directory_path
     resource["Metadata"]["WorkingDirectory"] = terraform_application_dir
-    # currently we set the terraform project root directory that contains all the terraform artifacts as the project
-    # directory till we work on the custom hook properties, and add a property for this value.
-    resource["Metadata"]["ProjectRootDirectory"] = terraform_application_dir
+    resource["Metadata"]["ProjectRootDirectory"] = project_root_dir
 
 
 def _validate_referenced_resource_matches_sam_metadata_type(

samcli/hook_packages/terraform/hooks/prepare/hook.py

@@ -42,13 +42,20 @@ def prepare(params: dict) -> dict:
     output_dir_path = params.get("OutputDirPath")
 
     terraform_application_dir = params.get("IACProjectPath", os.getcwd())
+    project_root_dir = params.get("ProjectRootDir", terraform_application_dir)
+
     if not output_dir_path:
         raise PrepareHookException("OutputDirPath was not supplied")
 
-    LOG.debug("Normalize the project root directory path %s", terraform_application_dir)
+    LOG.debug("Normalize the terraform application root module directory path %s", terraform_application_dir)
     if not os.path.isabs(terraform_application_dir):
         terraform_application_dir = os.path.normpath(os.path.join(os.getcwd(), terraform_application_dir))
-        LOG.debug("The normalized project root directory path %s", terraform_application_dir)
+        LOG.debug("The normalized terraform application root module directory path %s", terraform_application_dir)
+
+    LOG.debug("Normalize the project root directory path %s", project_root_dir)
+    if not os.path.isabs(project_root_dir):
+        project_root_dir = os.path.normpath(os.path.join(os.getcwd(), project_root_dir))
+        LOG.debug("The normalized project root directory path %s", project_root_dir)
 
     LOG.debug("Normalize the OutputDirPath %s", output_dir_path)
     if not os.path.isabs(output_dir_path):
@@ -75,7 +82,7 @@ def prepare(params: dict) -> dict:
 
             # convert terraform to cloudformation
             LOG.info("Generating metadata file")
-            cfn_dict = translate_to_cfn(tf_json, output_dir_path, terraform_application_dir)
+            cfn_dict = translate_to_cfn(tf_json, output_dir_path, terraform_application_dir, project_root_dir)
 
             if cfn_dict.get("Resources"):
                 _update_resources_paths(cfn_dict.get("Resources"), terraform_application_dir)  # type: ignore

samcli/hook_packages/terraform/hooks/prepare/translate.py

@@ -148,7 +148,9 @@ def _check_unresolvable_values(root_module: dict, root_tf_module: TFModule) -> N
                     return
 
 
-def translate_to_cfn(tf_json: dict, output_directory_path: str, terraform_application_dir: str) -> dict:
+def translate_to_cfn(
+    tf_json: dict, output_directory_path: str, terraform_application_dir: str, project_root_dir: str
+) -> dict:
     """
     Translates the json output of a terraform show into CloudFormation
 
@@ -159,7 +161,9 @@ def translate_to_cfn(tf_json: dict, output_directory_path: str, terraform_applic
     output_directory_path: str
         the string path to write the metadata file and makefile
     terraform_application_dir: str
-        the terraform project root directory
+        the terraform configuration root module directory.
+    project_root_dir: str
+        the project root directory where terraform configurations, src code, and other modules exist
 
     Returns
     -------
@@ -331,6 +335,7 @@ def translate_to_cfn(tf_json: dict, output_directory_path: str, terraform_applic
             output_directory_path,
             terraform_application_dir,
             lambda_resources_to_code_map,
+            project_root_dir,
         )
     else:
         LOG.debug("There is no sam metadata resources, no enrichment or Makefile is required")

samcli/lib/hook/hook_wrapper.py

@@ -52,6 +52,7 @@ def prepare(
         aws_region: Optional[str] = None,
         skip_prepare_infra: bool = False,
         plan_file: Optional[str] = None,
+        project_root_dir: Optional[str] = None,
     ) -> str:
         """
         Run the prepare hook to generate the IaC Metadata file.
@@ -69,10 +70,11 @@ def prepare(
         aws_region: str
             AWS region to use. Default is None (use default region)
         skip_prepare_infra: bool
-            Flag to skip skip prepare hook if we already have the metadata file. Default is False.
+            Flag to skip prepare hook if we already have the metadata file. Default is False.
         plan_file: Optional[str]
             Provided plan file to use instead of generating one from the hook
-
+        project_root_dir: Optional[str]
+            The Project root directory that contains the application directory, src code, and other modules
         Returns
         -------
         str
@@ -91,6 +93,8 @@ def prepare(
             params["Region"] = aws_region
         if plan_file:
             params["PlanFile"] = plan_file
+        if project_root_dir:
+            params["ProjectRootDir"] = project_root_dir
 
         output = self._execute("prepare", params)