From 811bdc36993cf0b09684f39179c8a3d7748331fe Mon Sep 17 00:00:00 2001 From: David Szabo Date: Mon, 7 Oct 2024 15:57:12 +0200 Subject: [PATCH 1/4] CB-27430 Change the data lake created by Express Onboarding to light duty --- aws/quickstart.sh | 13 +++++++++++++ azure/quickstart.sh | 13 +++++++++++++ gcp/quickstart.sh | 13 +++++++++++++ 3 files changed, 39 insertions(+) diff --git a/aws/quickstart.sh b/aws/quickstart.sh index 223ba06..2895863 100644 --- a/aws/quickstart.sh +++ b/aws/quickstart.sh @@ -24,6 +24,7 @@ export TF_VAR_env_tags='{"deploy_tool": "express-tf", "env_prefix": "'"$2"'"}' export TF_VAR_create_vpc_endpoints="false" export TF_VAR_environment_async_creation="true" export TF_VAR_datalake_async_creation="true" +export TF_VAR_datalake_scale="LIGHT_DUTY" # Save TF variables to file output_file="variables.sh" @@ -38,10 +39,22 @@ export TF_VAR_create_vpc_endpoints="${TF_VAR_create_vpc_endpoints}" export TF_VAR_environment_async_creation="${TF_VAR_environment_async_creation}" export TF_VAR_datalake_async_creation="${TF_VAR_datalake_async_creation}" export TF_VAR_ingress_extra_cidrs_and_ports="${TF_VAR_ingress_extra_cidrs_and_ports}" +export TF_VAR_datalake_scale="${TF_VAR_datalake_scale}" +EOF + +destroy_file="destroy.sh" + +cat < $destroy_file +cd cdp-tf-quickstarts/aws + +source variables.sh + +terraform destroy -auto-approve EOF # Make the file executable chmod +x $output_file +chmod +x $destroy_file # Install Terraform curl -fsSL https://releases.hashicorp.com/terraform/1.7.1/terraform_1.7.1_linux_amd64.zip -o terraform.zip diff --git a/azure/quickstart.sh b/azure/quickstart.sh index 1f74c5b..ca58e03 100644 --- a/azure/quickstart.sh +++ b/azure/quickstart.sh @@ -23,6 +23,7 @@ export TF_VAR_ingress_extra_cidrs_and_ports="${6:-"{ cidrs = [\"0.0.0.0/0\"], po export TF_VAR_env_tags='{"deploy_tool": "express-tf", "env_prefix": "'"$2"'"}' export TF_VAR_environment_async_creation="true" export TF_VAR_datalake_async_creation="true" +export TF_VAR_datalake_scale="LIGHT_DUTY" # Save TF variables to file output_file="variables.sh" @@ -36,10 +37,22 @@ export TF_VAR_env_tags="${TF_VAR_env_tags}" export TF_VAR_environment_async_creation="${TF_VAR_environment_async_creation}" export TF_VAR_datalake_async_creation="${TF_VAR_datalake_async_creation}" export TF_VAR_ingress_extra_cidrs_and_ports="${TF_VAR_ingress_extra_cidrs_and_ports}" +export TF_VAR_datalake_scale="${TF_VAR_datalake_scale}" +EOF + +destroy_file="destroy.sh" + +cat < $destroy_file +cd cdp-tf-quickstarts/azure + +source variables.sh + +terraform destroy -auto-approve EOF # Make the file executable chmod +x $output_file +chmod +x $destroy_file # Checkout CDP Quickstart Repository git clone --branch v0.7.2 https://github.com/cloudera-labs/cdp-tf-quickstarts.git diff --git a/gcp/quickstart.sh b/gcp/quickstart.sh index f0c6f94..6d377a3 100644 --- a/gcp/quickstart.sh +++ b/gcp/quickstart.sh @@ -24,6 +24,7 @@ export TF_VAR_ingress_extra_cidrs_and_ports="${7:-"{ cidrs = [\"0.0.0.0/0\"], po export TF_VAR_env_tags='{"deploy_tool": "express-tf", "env_prefix": "'"$2"'"}' export TF_VAR_environment_async_creation="true" export TF_VAR_datalake_async_creation="true" +export TF_VAR_datalake_scale="LIGHT_DUTY" # Save TF variables to file output_file="variables.sh" @@ -38,10 +39,22 @@ export TF_VAR_env_tags="${TF_VAR_env_tags}" export TF_VAR_environment_async_creation="${TF_VAR_environment_async_creation}" export TF_VAR_datalake_async_creation="${TF_VAR_datalake_async_creation}" export TF_VAR_ingress_extra_cidrs_and_ports="${TF_VAR_ingress_extra_cidrs_and_ports}" +export TF_VAR_datalake_scale="${TF_VAR_datalake_scale}" +EOF + +destroy_file="destroy.sh" + +cat < $destroy_file +cd cdp-tf-quickstarts/gcp + +source variables.sh + +terraform destroy -auto-approve EOF # Make the file executable chmod +x $output_file +chmod +x $destroy_file # Checkout CDP Quickstart Repository git clone --branch v0.7.2 https://github.com/cloudera-labs/cdp-tf-quickstarts.git From a114ba33c5a15c989fbb478ff64e2a4cd9edb498 Mon Sep 17 00:00:00 2001 From: David Szabo Date: Mon, 7 Oct 2024 15:58:27 +0200 Subject: [PATCH 2/4] CB-27431 Move AWS Express Onboarding to CDP quickstart branch 0.8.2 --- aws/quickstart.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/quickstart.sh b/aws/quickstart.sh index 2895863..5b40aa5 100644 --- a/aws/quickstart.sh +++ b/aws/quickstart.sh @@ -62,7 +62,7 @@ unzip -o terraform.zip -d ${HOME} rm terraform.zip # Checkout CDP Quickstart Repository -git clone --branch v0.7.2 https://github.com/cloudera-labs/cdp-tf-quickstarts.git +git clone --branch v0.8.2 https://github.com/cloudera-labs/cdp-tf-quickstarts.git cd cdp-tf-quickstarts/aws # Install CDP CLI and Log In From 13a6e3b054b08486083c9146fa4fbb4d11cd5e2b Mon Sep 17 00:00:00 2001 From: David Szabo Date: Mon, 7 Oct 2024 16:19:27 +0200 Subject: [PATCH 3/4] CB-27432 azurerm requires subscription ID to be provided explicitly --- aws/quickstart.sh | 17 ++++++++++------- azure/main.tf | 2 +- azure/quickstart.sh | 21 +++++++++++++-------- azure/variables.tf | 8 ++++++++ gcp/quickstart.sh | 19 +++++++++++-------- 5 files changed, 43 insertions(+), 24 deletions(-) diff --git a/aws/quickstart.sh b/aws/quickstart.sh index 5b40aa5..e1adfca 100644 --- a/aws/quickstart.sh +++ b/aws/quickstart.sh @@ -19,7 +19,7 @@ export TF_VAR_env_prefix="${2:-""}" export ACCOUNT_ID="${3:-""}" export CDP_REGION="${4:-"us-west-1"}" export TF_VAR_deployment_template="${5:-"semi-private"}" -export TF_VAR_ingress_extra_cidrs_and_ports="${6:-"{ cidrs = [\"0.0.0.0/0\"], ports = [443, 22] }"}" +export TF_VAR_ingress_extra_cidrs_and_ports=${6:-"{ cidrs = [\"0.0.0.0/0\"], ports = [443, 22] }"} export TF_VAR_env_tags='{"deploy_tool": "express-tf", "env_prefix": "'"$2"'"}' export TF_VAR_create_vpc_endpoints="false" export TF_VAR_environment_async_creation="true" @@ -34,11 +34,11 @@ export TF_VAR_aws_region="${TF_VAR_aws_region}" export TF_VAR_env_prefix="${TF_VAR_env_prefix}" export ACCOUNT_ID="${ACCOUNT_ID}" export TF_VAR_deployment_template="${TF_VAR_deployment_template}" -export TF_VAR_env_tags="${TF_VAR_env_tags}" +export TF_VAR_env_tags='${TF_VAR_env_tags}' export TF_VAR_create_vpc_endpoints="${TF_VAR_create_vpc_endpoints}" export TF_VAR_environment_async_creation="${TF_VAR_environment_async_creation}" export TF_VAR_datalake_async_creation="${TF_VAR_datalake_async_creation}" -export TF_VAR_ingress_extra_cidrs_and_ports="${TF_VAR_ingress_extra_cidrs_and_ports}" +export TF_VAR_ingress_extra_cidrs_and_ports='${TF_VAR_ingress_extra_cidrs_and_ports}' export TF_VAR_datalake_scale="${TF_VAR_datalake_scale}" EOF @@ -47,9 +47,9 @@ destroy_file="destroy.sh" cat < $destroy_file cd cdp-tf-quickstarts/aws -source variables.sh +source ${HOME}/variables.sh -terraform destroy -auto-approve +${HOME}/terraform destroy -auto-approve EOF # Make the file executable @@ -69,8 +69,11 @@ cd cdp-tf-quickstarts/aws pip install cdpcli config_file="${HOME}/.cdp/config" -region_config="cdp_region = ${CDP_REGION}" -echo "$region_config" >> "$config_file" +mkdir -p "$(dirname "$config_file")" +cat < $config_file +[default] +cdp_region = ${CDP_REGION} +EOF cdp login --account-id "${ACCOUNT_ID}" --use-device-code diff --git a/azure/main.tf b/azure/main.tf index 31867cd..239591b 100644 --- a/azure/main.tf +++ b/azure/main.tf @@ -92,7 +92,7 @@ module "cdp_deploy" { datalake_async_creation = var.datalake_async_creation # From pre-reqs module output - azure_subscription_id = module.cdp_azure_prereqs.azure_subscription_id + azure_subscription_id = var.azure_subscription_id == null ? module.cdp_azure_prereqs.azure_subscription_id : var.azure_subscription_id azure_tenant_id = module.cdp_azure_prereqs.azure_tenant_id azure_resource_group_name = module.cdp_azure_prereqs.azure_resource_group_name diff --git a/azure/quickstart.sh b/azure/quickstart.sh index ca58e03..8b59413 100644 --- a/azure/quickstart.sh +++ b/azure/quickstart.sh @@ -19,11 +19,12 @@ export TF_VAR_env_prefix="${2:-""}" export ACCOUNT_ID="${3:-""}" export CDP_REGION="${4:-"us-west-1"}" export TF_VAR_deployment_template="${5:-"semi-private"}" -export TF_VAR_ingress_extra_cidrs_and_ports="${6:-"{ cidrs = [\"0.0.0.0/0\"], ports = [443, 22] }"}" +export TF_VAR_ingress_extra_cidrs_and_ports=${6:-"{ cidrs = [\"0.0.0.0/0\"], ports = [443, 22] }"} export TF_VAR_env_tags='{"deploy_tool": "express-tf", "env_prefix": "'"$2"'"}' export TF_VAR_environment_async_creation="true" export TF_VAR_datalake_async_creation="true" export TF_VAR_datalake_scale="LIGHT_DUTY" +export TF_VAR_azure_subscription_id="$(az account show --query id --output tsv)" # Save TF variables to file output_file="variables.sh" @@ -33,11 +34,12 @@ export TF_VAR_azure_region="${TF_VAR_azure_region}" export TF_VAR_env_prefix="${TF_VAR_env_prefix}" export ACCOUNT_ID="${ACCOUNT_ID}" export TF_VAR_deployment_template="${TF_VAR_deployment_template}" -export TF_VAR_env_tags="${TF_VAR_env_tags}" +export TF_VAR_env_tags='${TF_VAR_env_tags}' export TF_VAR_environment_async_creation="${TF_VAR_environment_async_creation}" export TF_VAR_datalake_async_creation="${TF_VAR_datalake_async_creation}" -export TF_VAR_ingress_extra_cidrs_and_ports="${TF_VAR_ingress_extra_cidrs_and_ports}" +export TF_VAR_ingress_extra_cidrs_and_ports='${TF_VAR_ingress_extra_cidrs_and_ports}' export TF_VAR_datalake_scale="${TF_VAR_datalake_scale}" +export TF_VAR_azure_subscription_id="${TF_VAR_azure_subscription_id}" EOF destroy_file="destroy.sh" @@ -45,9 +47,9 @@ destroy_file="destroy.sh" cat < $destroy_file cd cdp-tf-quickstarts/azure -source variables.sh +source ${HOME}/variables.sh -terraform destroy -auto-approve +${HOME}/terraform destroy -auto-approve EOF # Make the file executable @@ -55,15 +57,18 @@ chmod +x $output_file chmod +x $destroy_file # Checkout CDP Quickstart Repository -git clone --branch v0.7.2 https://github.com/cloudera-labs/cdp-tf-quickstarts.git +git clone --branch v0.8.2 https://github.com/cloudera-labs/cdp-tf-quickstarts.git cd cdp-tf-quickstarts/azure # Install CDP CLI and Log In pip install cdpcli config_file="${HOME}/.cdp/config" -region_config="cdp_region = ${CDP_REGION}" -echo "$region_config" >> "$config_file" +mkdir -p "$(dirname "$config_file")" +cat < $config_file +[default] +cdp_region = ${CDP_REGION} +EOF cdp login --account-id "${ACCOUNT_ID}" --use-device-code diff --git a/azure/variables.tf b/azure/variables.tf index 79de355..6fd9f92 100644 --- a/azure/variables.tf +++ b/azure/variables.tf @@ -105,6 +105,14 @@ variable "datalake_recipes" { default = null } +variable "azure_subscription_id" { + type = string + + description = "The subscription where the environment should be deployed to" + + default = null +} + # ------- Network Resources ------- variable "ingress_extra_cidrs_and_ports" { type = object({ diff --git a/gcp/quickstart.sh b/gcp/quickstart.sh index 6d377a3..c1b9949 100644 --- a/gcp/quickstart.sh +++ b/gcp/quickstart.sh @@ -20,7 +20,7 @@ export TF_VAR_env_prefix="${3:-""}" export ACCOUNT_ID="${4:-""}" export CDP_REGION="${5:-"us-west-1"}" export TF_VAR_deployment_template="${6:-"semi-private"}" -export TF_VAR_ingress_extra_cidrs_and_ports="${7:-"{ cidrs = [\"0.0.0.0/0\"], ports = [443, 22] }"}" +export TF_VAR_ingress_extra_cidrs_and_ports=${6:-"{ cidrs = [\"0.0.0.0/0\"], ports = [443, 22] }"} export TF_VAR_env_tags='{"deploy_tool": "express-tf", "env_prefix": "'"$2"'"}' export TF_VAR_environment_async_creation="true" export TF_VAR_datalake_async_creation="true" @@ -32,13 +32,13 @@ output_file="variables.sh" cat < $output_file export TF_VAR_gcp_project="${TF_VAR_gcp_project}" export TF_VAR_gcp_region="${TF_VAR_gcp_region}" -export TF_VAR_env_prefix="${TF_VAR_env_prefix}" +export TF_VAR_env_prefix='${TF_VAR_env_prefix}' export ACCOUNT_ID="${ACCOUNT_ID}" export TF_VAR_deployment_template="${TF_VAR_deployment_template}" export TF_VAR_env_tags="${TF_VAR_env_tags}" export TF_VAR_environment_async_creation="${TF_VAR_environment_async_creation}" export TF_VAR_datalake_async_creation="${TF_VAR_datalake_async_creation}" -export TF_VAR_ingress_extra_cidrs_and_ports="${TF_VAR_ingress_extra_cidrs_and_ports}" +export TF_VAR_ingress_extra_cidrs_and_ports='${TF_VAR_ingress_extra_cidrs_and_ports}' export TF_VAR_datalake_scale="${TF_VAR_datalake_scale}" EOF @@ -47,9 +47,9 @@ destroy_file="destroy.sh" cat < $destroy_file cd cdp-tf-quickstarts/gcp -source variables.sh +source ${HOME}/variables.sh -terraform destroy -auto-approve +${HOME}/terraform destroy -auto-approve EOF # Make the file executable @@ -57,15 +57,18 @@ chmod +x $output_file chmod +x $destroy_file # Checkout CDP Quickstart Repository -git clone --branch v0.7.2 https://github.com/cloudera-labs/cdp-tf-quickstarts.git +git clone --branch v0.8.2 https://github.com/cloudera-labs/cdp-tf-quickstarts.git cd cdp-tf-quickstarts/gcp # Install CDP CLI and Log In pip install cdpcli config_file="${HOME}/.cdp/config" -region_config="cdp_region = ${CDP_REGION}" -echo "$region_config" >> "$config_file" +mkdir -p "$(dirname "$config_file")" +cat < $config_file +[default] +cdp_region = ${CDP_REGION} +EOF cdp login --account-id "${ACCOUNT_ID}" --use-device-code From 2be1528d19db754811c05620b70a6378a0d871bd Mon Sep 17 00:00:00 2001 From: David Szabo Date: Wed, 9 Oct 2024 13:15:12 +0200 Subject: [PATCH 4/4] CB-27431 Move AWS Express Onboarding to CDP quickstart branch 0.8.3 --- aws/quickstart.sh | 2 +- azure/quickstart.sh | 2 +- gcp/quickstart.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/aws/quickstart.sh b/aws/quickstart.sh index e1adfca..e3c68ee 100644 --- a/aws/quickstart.sh +++ b/aws/quickstart.sh @@ -62,7 +62,7 @@ unzip -o terraform.zip -d ${HOME} rm terraform.zip # Checkout CDP Quickstart Repository -git clone --branch v0.8.2 https://github.com/cloudera-labs/cdp-tf-quickstarts.git +git clone --branch v0.8.3 https://github.com/cloudera-labs/cdp-tf-quickstarts.git cd cdp-tf-quickstarts/aws # Install CDP CLI and Log In diff --git a/azure/quickstart.sh b/azure/quickstart.sh index 8b59413..62dd33c 100644 --- a/azure/quickstart.sh +++ b/azure/quickstart.sh @@ -57,7 +57,7 @@ chmod +x $output_file chmod +x $destroy_file # Checkout CDP Quickstart Repository -git clone --branch v0.8.2 https://github.com/cloudera-labs/cdp-tf-quickstarts.git +git clone --branch v0.8.3 https://github.com/cloudera-labs/cdp-tf-quickstarts.git cd cdp-tf-quickstarts/azure # Install CDP CLI and Log In diff --git a/gcp/quickstart.sh b/gcp/quickstart.sh index c1b9949..efb5839 100644 --- a/gcp/quickstart.sh +++ b/gcp/quickstart.sh @@ -57,7 +57,7 @@ chmod +x $output_file chmod +x $destroy_file # Checkout CDP Quickstart Repository -git clone --branch v0.8.2 https://github.com/cloudera-labs/cdp-tf-quickstarts.git +git clone --branch v0.8.3 https://github.com/cloudera-labs/cdp-tf-quickstarts.git cd cdp-tf-quickstarts/gcp # Install CDP CLI and Log In