From 0a37a851c64424380e9a8ba5526d42fa021c7bb1 Mon Sep 17 00:00:00 2001
From: D10S0VSkY-OSS <79284025+D10S0VSkY-OSS@users.noreply.github.com>
Date: Wed, 29 Jun 2022 04:32:42 +0200
Subject: [PATCH] feat: Add tfvars files parameter for deploy stack
---
.github/workflows/sld-api-docker-image.yml | 4 +-
.../workflows/sld-dashboard-docker-image.yml | 4 +-
sld-api-backend/api_v1/endpoints/deploy.py | 16 +-
sld-api-backend/api_v1/endpoints/plan.py | 11 +-
sld-api-backend/core/providers/terraform.py | 43 ++++-
sld-api-backend/crud/deploys.py | 3 +
sld-api-backend/db/models.py | 1 +
sld-api-backend/helpers/push_task.py | 21 ++-
sld-api-backend/schemas/schemas.py | 3 +
sld-api-backend/tasks/celery_worker.py | 158 +++++++++++++-----
.../app/base/templates/accounts/login.html | 4 +-
sld-dashboard/app/home/forms.py | 6 +
sld-dashboard/app/home/routes.py | 25 ++-
.../app/home/templates/deploy-edit.html | 9 +-
.../app/home/templates/deploy-plan.html | 4 +-
.../app/home/templates/stacks-deploy.html | 3 +
16 files changed, 233 insertions(+), 82 deletions(-)
diff --git a/.github/workflows/sld-api-docker-image.yml b/.github/workflows/sld-api-docker-image.yml
index 7fef4859..6d6b195a 100644
--- a/.github/workflows/sld-api-docker-image.yml
+++ b/.github/workflows/sld-api-docker-image.yml
@@ -26,10 +26,10 @@ jobs:
- name: Build the Docker image with tag
working-directory: ./sld-api-backend
- run: docker build . --file Dockerfile --tag ${{ secrets.DOCKER_USERNAME }}/sld-api:2.2.0
+ run: docker build . --file Dockerfile --tag ${{ secrets.DOCKER_USERNAME }}/sld-api:2.3.0
- name: Docker Push with tag
- run: docker push ${{ secrets.DOCKER_USERNAME }}/sld-api:2.2.0
+ run: docker push ${{ secrets.DOCKER_USERNAME }}/sld-api:2.3.0
- name: Build the Docker image
working-directory: ./sld-api-backend
diff --git a/.github/workflows/sld-dashboard-docker-image.yml b/.github/workflows/sld-dashboard-docker-image.yml
index 958c111c..dc43bd82 100644
--- a/.github/workflows/sld-dashboard-docker-image.yml
+++ b/.github/workflows/sld-dashboard-docker-image.yml
@@ -26,10 +26,10 @@ jobs:
- name: Build the Docker image with tags
working-directory: ./sld-dashboard
- run: docker build . --file Dockerfile --tag ${{ secrets.DOCKER_USERNAME }}/sld-dashboard:2.2.0
+ run: docker build . --file Dockerfile --tag ${{ secrets.DOCKER_USERNAME }}/sld-dashboard:2.3.0
- name: Docker Push with tags
- run: docker push ${{ secrets.DOCKER_USERNAME }}/sld-dashboard:2.2.0
+ run: docker push ${{ secrets.DOCKER_USERNAME }}/sld-dashboard:2.3.0
- name: Build the Docker image
working-directory: ./sld-dashboard
diff --git a/sld-api-backend/api_v1/endpoints/deploy.py b/sld-api-backend/api_v1/endpoints/deploy.py
index ed3aad46..d0c90312 100644
--- a/sld-api-backend/api_v1/endpoints/deploy.py
+++ b/sld-api-backend/api_v1/endpoints/deploy.py
@@ -66,7 +66,8 @@ async def deploy_infra_by_stack_name(
branch,
tf_ver,
deploy.variables,
- secreto)
+ secreto,
+ deploy.tfvar_file)
# Push deploy task data
db_deploy = crud_deploys.create_new_deploy(
db=db,
@@ -149,7 +150,8 @@ async def update_deploy_by_id(
branch,
tf_ver,
deploy_update.variables,
- secreto)
+ secreto,
+ deploy_update.tfvar_file)
# Push deploy task data
crud_deploys.update_deploy(
db=db,
@@ -157,6 +159,7 @@ async def update_deploy_by_id(
task_id=pipeline_deploy,
action="Update",
user_id=current_user.id,
+ tfvar_file=deploy_update.tfvar_file,
variables=deploy_update.variables,
start_time=deploy_update.start_time,
destroy_time=deploy_update.destroy_time,
@@ -202,6 +205,7 @@ async def destroy_infra(
start_time = deploy_data.start_time
destroy_time = deploy_data.destroy_time
variables = deploy_data.variables
+ tfvar_file = deploy_data.tfvar_file
name = deploy_data.name
# Get credentials by providers supported
secreto = tokens.check_prefix(
@@ -227,7 +231,8 @@ async def destroy_infra(
branch,
tf_ver,
variables,
- secreto
+ secreto,
+ tfvar_file
)
# Push deploy task data
crud_deploys.update_deploy(
@@ -238,6 +243,7 @@ async def destroy_infra(
user_id=current_user.id,
start_time=start_time,
destroy_time=destroy_time,
+ tfvar_file=tfvar_file,
variables=variables,
username=current_user.username)
# Push task data
@@ -312,6 +318,7 @@ async def delete_infra_by_id(
stack_name = deploy_data.stack_name
environment = deploy_data.environment
name = deploy_data.name
+ tfvar_file = deploy_data.tfvar_file
variables = deploy_data.variables
# Get credentials by providers supported
secreto = tokens.check_prefix(
@@ -341,7 +348,8 @@ async def delete_infra_by_id(
branch,
tf_ver,
variables,
- secreto
+ secreto,
+ tfvar_file
)
# Push task data
db_task = crud_tasks.create_task(
diff --git a/sld-api-backend/api_v1/endpoints/plan.py b/sld-api-backend/api_v1/endpoints/plan.py
index 974ec4d6..0c685d09 100644
--- a/sld-api-backend/api_v1/endpoints/plan.py
+++ b/sld-api-backend/api_v1/endpoints/plan.py
@@ -58,7 +58,8 @@ async def plan_infra_by_stack_name(
branch,
tf_ver,
deploy.variables,
- secreto)
+ secreto,
+ deploy.tfvar_file)
# Push deploy task data
db_deploy = crud_deploys.create_new_deploy(
db=db,
@@ -132,7 +133,9 @@ async def update_plan_by_id(
branch,
tf_ver,
deploy_update.variables,
- secreto)
+ secreto,
+ deploy_update.tfvar_file
+ )
# Push deploy task data
if "Plan" in deploy_data.action:
crud_deploys.update_deploy(
@@ -141,6 +144,7 @@ async def update_plan_by_id(
task_id=pipeline_plan,
action="Plan",
user_id=current_user.id,
+ tfvar_file=deploy_update.tfvar_file,
variables=deploy_update.variables,
start_time=deploy_update.start_time,
destroy_time=deploy_update.destroy_time,
@@ -202,7 +206,8 @@ async def get_plan_by_id_deploy(
branch,
tf_ver,
deploy_data.variables,
- secreto)
+ secreto,
+ deploy_data.tfvar_file)
return {"task": pipeline_plan}
except Exception as err:
raise HTTPException(
diff --git a/sld-api-backend/core/providers/terraform.py b/sld-api-backend/core/providers/terraform.py
index 08b037f6..f50b7987 100644
--- a/sld-api-backend/core/providers/terraform.py
+++ b/sld-api-backend/core/providers/terraform.py
@@ -4,6 +4,8 @@
import subprocess
import jmespath
from os import path
+from os.path import isfile, join
+from os import listdir
import hcl
import shutil
@@ -72,16 +74,18 @@ def git_clone(
module='git',
module_args=f'repo={git_repo} dest={stack_name}/{environment}/{squad}/{name} version={branch} force=yes',
)
+ path_tfvars = f'/tmp/{stack_name}/{environment}/{squad}/{name}'
+ tfvars_files = [f for f in listdir(path_tfvars) if f.endswith('.tfvars') and isfile(join(path_tfvars, f))]
logs = [i for i in runner_response.events]
git_stdout = jmespath.search(
'[*].event_data.res.stdout_lines', logs)
git_stderr = jmespath.search('[*].event_data.res.msg', logs)
rc = runner_response.rc
if rc != 0:
- return {"command": "git", "rc": rc, "stdout": git_stderr}
- return {"command": "git", "rc": rc, "stdout": git_stdout}
+ return {"command": "git", "rc": rc, "tfvars": tfvars_files, "stdout": git_stderr}
+ return {"command": "git", "rc": rc, "tfvars": tfvars_files, "stdout": git_stdout}
except Exception as err:
- return {"command": "git", "rc": 1, "stdout": err}
+ return {"command": "git", "rc": 1, "tfvars": tfvars_files, "stdout": err}
@staticmethod
def tfstate_render(
@@ -155,10 +159,13 @@ def plan_execute(
squad: str,
name: str,
version: str,
+ variables_file: str = "",
**secreto: dict) -> dict:
try:
secret(stack_name, environment, squad, name, secreto)
+ deploy_state=f'{environment}_{stack_name}_{squad}_{name}'
# Execute task
+ variables_files = f'{stack_name}.tfvars.json' if variables_file == "" or variables_file == None else variables_file
runner_response = ansible_runner.run(
private_data_dir=f'/tmp/{stack_name}/{environment}/{squad}/{name}',
host_pattern='localhost',
@@ -166,7 +173,7 @@ def plan_execute(
module_args=f'binary_path=/tmp/{version}/terraform '
f'force_init=True project_path=/tmp/{stack_name}/{environment}/{squad}/{name} '
f'plan_file=/tmp/{stack_name}/{environment}/{squad}/{name}/{stack_name}.tfplan '
- f'variables_files={stack_name}.tfvars.json state=planned',
+ f'variables_files={variables_files} state=planned',
)
unsecret(stack_name, environment, squad, name, secreto)
@@ -184,6 +191,8 @@ def plan_execute(
"squad": squad,
"environment": environment,
"rc": rc,
+ "tfvars_files": variables_file,
+ "remote_state": f'http://remote-state:8080/terraform_state/{deploy_state}',
"stdout": plan_stderr
}
return {
@@ -192,6 +201,8 @@ def plan_execute(
"squad": squad,
"environment": environment,
"rc": rc,
+ "tfvars_files": variables_file,
+ "remote_state": f'http://remote-state:8080/terraform_state/{deploy_state}',
"stdout": plan_stdout
}
except Exception as err:
@@ -201,6 +212,8 @@ def plan_execute(
"squad": squad,
"environment": environment,
"rc": 1,
+ "tfvars_files": variables_file,
+ "remote_state": f'http://remote-state:8080/terraform_state/{deploy_state}',
"stdout": f'{err}'
}
@@ -211,17 +224,20 @@ def apply_execute(
squad: str,
name: str,
version: str,
+ variables_file: str = "",
**secreto: dict) -> dict:
try:
secret(stack_name, environment, squad, name, secreto)
+ deploy_state=f'{environment}_{stack_name}_{squad}_{name}'
# Execute task
+ variables_files = f'{stack_name}.tfvars.json' if variables_file == "" or variables_file == None else variables_file
runner_response = ansible_runner.run(
private_data_dir=f'/tmp/{stack_name}/{environment}/{squad}/{name}',
host_pattern='localhost',
module='terraform',
module_args=f'binary_path=/tmp/{version}/terraform lock=True force_init=True project_path=/tmp/{stack_name}/{environment}/{squad}/{name} '
f'plan_file=/tmp/{stack_name}/{environment}/{squad}/{name}/{stack_name}.tfplan state=present '
- f'variables_files={stack_name}.tfvars.json',
+ f'variables_files={variables_files}',
)
unsecret(stack_name, environment, squad, name, secreto)
# Capture events
@@ -239,6 +255,8 @@ def apply_execute(
"squad": squad,
"environment": environment,
"rc": rc,
+ "tfvars_files": variables_file,
+ "remote_state": f'http://remote-state:8080/terraform_state/{deploy_state}',
"stdout": apply_stderr
}
return {
@@ -247,6 +265,8 @@ def apply_execute(
"squad": squad,
"environment": environment,
"rc": rc,
+ "tfvars_files": variables_file,
+ "remote_state": f'http://remote-state:8080/terraform_state/{deploy_state}',
"stdout": apply_stdout
}
except Exception as err:
@@ -256,6 +276,8 @@ def apply_execute(
"squad": squad,
"environment": environment,
"rc": 1,
+ "tfvars_files": variables_file,
+ "remote_state": f'http://remote-state:8080/terraform_state/{deploy_state}',
"stdout": f'{err}'
}
@@ -266,16 +288,19 @@ def destroy_execute(
squad: str,
name: str,
version: str,
+ variables_file: str = "",
**secreto: dict) -> dict:
try:
secret(stack_name, environment, squad, name, secreto)
+ deploy_state=f'{environment}_{stack_name}_{squad}_{name}'
# Execute task
+ variables_files = f'{stack_name}.tfvars.json' if variables_file == "" or variables_file == None else variables_file
runner_response = ansible_runner.run(
private_data_dir=f'/tmp/{stack_name}/{environment}/{squad}/{name}',
host_pattern='localhost',
module='terraform',
module_args=f'binary_path=/tmp/{version}/terraform force_init=True project_path=/tmp/{stack_name}/{environment}/{squad}/{name} '
- f'variables_files={stack_name}.tfvars.json state=absent',
+ f'variables_files={variables_files} state=absent',
)
unsecret(stack_name, environment, squad, name, secreto)
# Capture events
@@ -292,6 +317,8 @@ def destroy_execute(
"squad": squad,
"environment": environment,
"rc": rc,
+ "tfvars_files": variables_file,
+ "remote_state": f'http://remote-state:8080/terraform_state/{deploy_state}',
"stdout": destroy_stderr
}
return {
@@ -300,6 +327,8 @@ def destroy_execute(
"squad": squad,
"environment": environment,
"rc": rc,
+ "tfvars_files": variables_file,
+ "remote_state": f'http://remote-state:8080/terraform_state/{deploy_state}',
"stdout": destroy_stdout
}
except Exception as err:
@@ -309,6 +338,8 @@ def destroy_execute(
"squad": squad,
"environment": environment,
"rc": 1,
+ "tfvars_files": variables_file,
+ "remote_state": f'http://remote-state:8080/terraform_state/{deploy_state}',
"stdout": f'{err}'
}
diff --git a/sld-api-backend/crud/deploys.py b/sld-api-backend/crud/deploys.py
index 25a00ed7..f506c3cf 100644
--- a/sld-api-backend/crud/deploys.py
+++ b/sld-api-backend/crud/deploys.py
@@ -20,6 +20,7 @@ def create_new_deploy(
name=deploy.name,
stack_name=deploy.stack_name,
environment=deploy.environment,
+ tfvar_file=deploy.tfvar_file,
variables=deploy.variables,
action=action,
username=username,
@@ -48,6 +49,7 @@ def update_deploy(
task_id: str,
start_time: str,
destroy_time: str,
+ tfvar_file: str,
variables: dict):
db_deploy = db.query(models.Deploy).filter(
models.Deploy.id == deploy_id).first()
@@ -56,6 +58,7 @@ def update_deploy(
db_deploy.task_id = task_id
db_deploy.username = username
db_deploy.user_id = user_id
+ db_deploy.tfvar_file = tfvar_file
db_deploy.variables = variables
db_deploy.updated_at = datetime.datetime.now()
check_None = ["string"]
diff --git a/sld-api-backend/db/models.py b/sld-api-backend/db/models.py
index 581363fa..3add19c7 100644
--- a/sld-api-backend/db/models.py
+++ b/sld-api-backend/db/models.py
@@ -90,6 +90,7 @@ class Deploy(Base):
squad = Column(String(50), nullable=False)
variables = Column(JSON)
environment = Column(String(50))
+ tfvar_file = Column(String(50))
__table_args__ = (UniqueConstraint(
'squad', 'environment', 'name', 'stack_name'),)
diff --git a/sld-api-backend/helpers/push_task.py b/sld-api-backend/helpers/push_task.py
index 6cb7651c..5143e3ff 100644
--- a/sld-api-backend/helpers/push_task.py
+++ b/sld-api-backend/helpers/push_task.py
@@ -18,7 +18,9 @@ def async_deploy(
branch: str,
tf_ver: str,
variables: dict,
- secreto: str):
+ secreto: str,
+ variables_file: str = ""
+ ):
pipeline_deploy_result = pipeline_deploy.s(
git_repo=git_repo,
@@ -29,7 +31,8 @@ def async_deploy(
branch=branch,
version=tf_ver,
kwargs=variables,
- secreto=secreto
+ secreto=secreto,
+ variables_file=variables_file
).apply_async(queue=squad,
retry=True,
retry_policy={
@@ -50,7 +53,9 @@ def async_destroy(
branch: str,
tf_ver: str,
variables: dict,
- secreto: str):
+ secreto: str,
+ variables_file: str = ""
+ ):
pipeline_destroy_result = pipeline_destroy.s(
git_repo=git_repo,
@@ -61,7 +66,8 @@ def async_destroy(
branch=branch,
version=tf_ver,
kwargs=variables,
- secreto=secreto
+ secreto=secreto,
+ variables_file=variables_file
).apply_async(
queue=squad,
retry=True,
@@ -83,7 +89,9 @@ def async_plan(
branch: str,
tf_ver: str,
variables: dict,
- secreto: str):
+ secreto: str,
+ variables_file: str = ""
+ ):
pipeline_plan_result = pipeline_plan.s(
git_repo=git_repo,
@@ -94,7 +102,8 @@ def async_plan(
branch=branch,
version=tf_ver,
kwargs=variables,
- secreto=secreto
+ secreto=secreto,
+ variables_file=variables_file
).apply_async(queue=squad,
retry=True,
retry_policy={
diff --git a/sld-api-backend/schemas/schemas.py b/sld-api-backend/schemas/schemas.py
index 3280ca87..0ef6192e 100644
--- a/sld-api-backend/schemas/schemas.py
+++ b/sld-api-backend/schemas/schemas.py
@@ -151,6 +151,7 @@ class DeployCreate(BaseModel):
environment: constr(strip_whitespace=True)
start_time: Optional[constr(strip_whitespace=True)] = Field(None, example="30 7 * * 0-4")
destroy_time: Optional[constr(strip_whitespace=True)] = Field(None, example="30 18 * * 0-4")
+ tfvar_file: Optional[constr(strip_whitespace=True)] = Field("", example="terraform.tfvars")
variables: dict
@@ -165,6 +166,7 @@ class DeployDeleteMaster(BaseModel):
class DeployUpdate(BaseModel):
start_time: constr(strip_whitespace=True)
destroy_time: constr(strip_whitespace=True)
+ tfvar_file: Optional[constr(strip_whitespace=True)] = Field("", example="terraform.tfvars")
variables: dict
@@ -184,6 +186,7 @@ class PlanCreate(BaseModel):
environment: constr(strip_whitespace=True)
start_time: Optional[constr(strip_whitespace=True)] = Field(None, example="30 7 * * 0-4")
destroy_time: Optional[constr(strip_whitespace=True)] = Field(None, example="30 18 * * 0-4")
+ tfvar_file: Optional[constr(strip_whitespace=True)] = Field("", example="terraform.tfvars")
variables: dict
diff --git a/sld-api-backend/tasks/celery_worker.py b/sld-api-backend/tasks/celery_worker.py
index 346a9965..f21b5ce0 100644
--- a/sld-api-backend/tasks/celery_worker.py
+++ b/sld-api-backend/tasks/celery_worker.py
@@ -10,11 +10,23 @@
r = redis.Redis(host=settings.BACKEND_SERVER, port=6379, db=2,
- charset="utf-8", decode_responses=True)
+ charset="utf-8", decode_responses=True)
@celery_app.task(bind=True, acks_late=True, time_limit=settings.DEPLOY_TMOUT, name='pipeline Deploy')
-def pipeline_deploy(self, git_repo, name, stack_name, environment, squad, branch, version, kwargs, secreto):
+def pipeline_deploy(
+ self,
+ git_repo: str,
+ name: str,
+ stack_name: str,
+ environment: str,
+ squad: str,
+ branch: str,
+ version: str,
+ kwargs: any,
+ secreto: str,
+ variables_file: str = ""
+ ):
filter_kwargs = {key: value for (
key, value) in kwargs.items() if "pass" not in key}
try:
@@ -22,7 +34,7 @@ def pipeline_deploy(self, git_repo, name, stack_name, environment, squad, branch
r.expire(f"{name}-{squad}-{environment}", settings.TASK_LOCKED_EXPIRED)
# Git clone repo
result = tf.git_clone(git_repo, name, stack_name,
- environment, squad, branch)
+ environment, squad, branch)
self.update_state(state='PULLING', meta={'done': "1 of 6"})
if result['rc'] != 0:
raise Exception(result)
@@ -47,7 +59,7 @@ def pipeline_deploy(self, git_repo, name, stack_name, environment, squad, branch
# Plan execute
self.update_state(state='PLANNING', meta={'done': "5 of 6"})
result = tf.plan_execute(stack_name, environment,
- squad, name, version, data=secreto)
+ squad, name, version, variables_file, data=secreto)
# Delete artifactory to avoid duplicating the runner logs
dir_path = f"/tmp/{ stack_name }/{environment}/{squad}/{name}/artifacts"
tf.delete_local_folder(dir_path)
@@ -56,23 +68,23 @@ def pipeline_deploy(self, git_repo, name, stack_name, environment, squad, branch
# Apply execute
self.update_state(state='APPLYING', meta={'done': "6 of 6"})
result = tf.apply_execute(
- stack_name, environment, squad, name, version, data=secreto)
+ stack_name, environment, squad, name, version, variables_file, data=secreto)
if result['rc'] != 0:
raise Exception(result)
return result
except Exception as err:
if not settings.ROLLBACK:
self.retry(countdown=settings.TASK_RETRY_INTERVAL,
- exc=err, max_retries=settings.TASK_MAX_RETRY)
+ exc=err, max_retries=settings.TASK_MAX_RETRY)
self.update_state(state=states.FAILURE, meta={'exc': result})
raise Ignore()
self.update_state(state="ROLLBACK", meta={'done': "1 of 1"})
destroy_eresult = tf.destroy_execute(
- stack_name, environment, squad, name, version, data=secreto)
+ stack_name, environment, squad, name, version,variables_file, data=secreto)
self.update_state(state=states.FAILURE, meta={
'exc_type': type(err).__name__,
'exc_message': traceback.format_exc().split('\n')
- })
+ })
raise Ignore()
finally:
dir_path = f"/tmp/{ stack_name }/{environment}/{squad}/{name}"
@@ -82,7 +94,19 @@ def pipeline_deploy(self, git_repo, name, stack_name, environment, squad, branch
@celery_app.task(bind=True, acks_late=True, name='pipeline Destroy')
-def pipeline_destroy(self, git_repo, name, stack_name, environment, squad, branch, version, kwargs, secreto):
+def pipeline_destroy(
+ self,
+ git_repo: str,
+ name: str,
+ stack_name: str,
+ environment: str,
+ squad: str,
+ branch: str,
+ version: str,
+ kwargs: any,
+ secreto: str,
+ variables_file: str = ""
+ ):
filter_kwargs = {key: value for (
key, value) in kwargs.items() if "pass" not in key}
try:
@@ -90,7 +114,7 @@ def pipeline_destroy(self, git_repo, name, stack_name, environment, squad, branc
r.expire(f"{name}-{squad}-{environment}", settings.TASK_LOCKED_EXPIRED)
# Git clone repo
result = tf.git_clone(git_repo, name, stack_name,
- environment, squad, branch)
+ environment, squad, branch)
self.update_state(state='PULLING', meta={'done': "1 of 6"})
if result['rc'] != 0:
raise Exception(result)
@@ -115,7 +139,7 @@ def pipeline_destroy(self, git_repo, name, stack_name, environment, squad, branc
self.update_state(state='DESTROYING', meta={'done': "6 of 6"})
result = tf.destroy_execute(
- stack_name, environment, squad, name, version, data=secreto)
+ stack_name, environment, squad, name, version, variables_file, data=secreto)
if result['rc'] != 0:
raise Exception(result)
return result
@@ -130,16 +154,27 @@ def pipeline_destroy(self, git_repo, name, stack_name, environment, squad, branc
@celery_app.task(bind=True, acks_late=True, name='pipeline Plan')
-def pipeline_plan(self, git_repo, name, stack_name, environment, squad, branch, version, kwargs, secreto):
+def pipeline_plan(
+ self,
+ git_repo: str,
+ name: str,
+ stack_name: str,
+ environment: str,
+ squad: str,
+ branch: str,
+ version: str,
+ kwargs: any,
+ secreto: str,
+ variables_file: str = ""
+ ):
filter_kwargs = {key: value for (
key, value) in kwargs.items() if "pass" not in key}
try:
self.update_state(state='GIT', meta={'done': "1 of 5"})
result = tf.git_clone(git_repo, name, stack_name,
- environment, squad, branch)
+ environment, squad, branch)
if result['rc'] != 0:
raise Exception(result)
-
self.update_state(state='BINARY', meta={'done': "2 of 5"})
result = tf.binary_download(stack_name, environment, squad, version)
dir_path = f"/tmp/{ stack_name }/{environment}/{squad}/artifacts"
@@ -159,7 +194,7 @@ def pipeline_plan(self, git_repo, name, stack_name, environment, squad, branch,
self.update_state(state='PLAN', meta={'done': "5 of 5"})
result = tf.plan_execute(stack_name, environment,
- squad, name, version, data=secreto)
+ squad, name, version, variables_file, data=secreto)
dir_path = f"/tmp/{ stack_name }/{environment}/{squad}/{name}/artifacts"
tf.delete_local_folder(dir_path)
if result['rc'] != 0:
@@ -175,25 +210,34 @@ def pipeline_plan(self, git_repo, name, stack_name, environment, squad, branch,
@celery_app.task(bind=True,
- acks_late=True,
- time_limit=settings.GIT_TMOUT,
- name='pipeline git pull')
-def pipeline_git_pull(self, git_repo, name, stack_name, environment, squad, branch):
+ acks_late=True,
+ time_limit=settings.GIT_TMOUT,
+ name='pipeline git pull')
+def pipeline_git_pull(
+ self,
+ git_repo: str,
+ name: str,
+ stack_name: str,
+ environment: str,
+ squad: str,
+ branch: str
+ ):
try:
- result = tf.git_clone(git_repo, name, stack_name,
- environment, squad, branch)
- if result['rc'] != 0:
+ git_result = tf.git_clone(git_repo, name, stack_name,
+ environment, squad, branch)
+ if git_result['rc'] != 0:
raise Exception(result.get('stdout'))
self.update_state(state='GET_VARS_AS_JSON', meta={'done': "2 of 2"})
result = tf.get_vars_json(
- environment=environment, stack_name=stack_name, squad=squad, name=name)
+ environment=environment, stack_name=stack_name, squad=squad, name=name)
if result['rc'] != 0:
raise Exception(result)
+ result["tfvars"] = git_result["tfvars"]
return result
except Exception as err:
self.retry(countdown=settings.GIT_TMOUT,
- exc=err, max_retries=settings.TASK_MAX_RETRY)
+ exc=err, max_retries=settings.TASK_MAX_RETRY)
self.update_state(state=states.FAILURE, meta={'exc': result})
raise Ignore()
finally:
@@ -202,26 +246,40 @@ def pipeline_git_pull(self, git_repo, name, stack_name, environment, squad, bran
@celery_app.task(bind=True,
- acks_late=True,
- time_limit=settings.GIT_TMOUT,
- name='download git repo')
-def git(self, git_repo, name, stack_name, environment, squad, branch):
+ acks_late=True,
+ time_limit=settings.GIT_TMOUT,
+ name='download git repo')
+def git(
+ self,
+ git_repo: str,
+ name: str,
+ stack_name: str,
+ environment: str,
+ squad: str,
+ branch: str
+ ):
try:
result = tf.git_clone(git_repo, name, stack_name,
- environment, squad, branch)
+ environment, squad, branch)
except Exception as err:
self.retry(countdown=settings.GIT_TMOUT,
- exc=err, max_retries=settings.TASK_MAX_RETRY)
+ exc=err, max_retries=settings.TASK_MAX_RETRY)
self.update_state(state=states.FAILURE, meta={'exc': result})
raise Ignore()
return stack_name, environment, squad, name, result
@celery_app.task(bind=True, acks_late=True, time_limit=settings.WORKER_TMOUT, max_retries=1, name='terraform output')
-def output(self, stack_name, environment, squad, name):
+def output(
+ self,
+ stack_name: str,
+ environment: str,
+ squad: str,
+ name: str
+ ):
try:
output_result = tf.output_execute(
- stack_name, environment, squad, name)
+ stack_name, environment, squad, name)
return output_result
except Exception as err:
return {"stdout": err}
@@ -231,19 +289,31 @@ def output(self, stack_name, environment, squad, name):
@celery_app.task(bind=True, acks_late=True, time_limit=settings.WORKER_TMOUT, max_retries=1, name='terraform unlock')
-def unlock(self, stack_name, environment, squad, name):
+def unlock(
+ self,
+ stack_name: str,
+ environment: str,
+ squad: str,
+ name: str
+ ):
try:
unlock_result = tf.unlock_execute(
- stack_name, environment, squad, name)
+ stack_name, environment, squad, name)
return unlock_result
except Exception as err:
return {"stdout": err}
@celery_app.task(bind=True, acks_late=True, time_limit=settings.WORKER_TMOUT, name='terraform show')
-def show(self, stack_name, environment, squad, name):
+def show(
+ self,
+ stack_name: str,
+ environment: str,
+ squad: str,
+ name: str
+ ):
show_result = tf.show_execute(
- stack_name, environment, squad, name)
+ stack_name, environment, squad, name)
return show_result
@@ -256,7 +326,7 @@ def schedules_list(self):
@celery_app.task(bind=True, acks_late=True, time_limit=settings.WORKER_TMOUT, name='schedule get')
-def schedule_get(self, deploy_name):
+def schedule_get(self, deploy_name: str):
try:
return request_url(verb='GET', uri=f'schedule/{deploy_name}').get('json')
except Exception as err:
@@ -264,7 +334,7 @@ def schedule_get(self, deploy_name):
@celery_app.task(bind=True, acks_late=True, time_limit=settings.WORKER_TMOUT, name='schedule remove')
-def schedule_delete(self, deploy_name):
+def schedule_delete(self, deploy_name: str):
try:
return request_url(verb='DELETE', uri=f'schedule/{deploy_name}').get('json')
except Exception as err:
@@ -272,7 +342,7 @@ def schedule_delete(self, deploy_name):
@celery_app.task(bind=True, acks_late=True, time_limit=settings.WORKER_TMOUT, name='schedule add')
-def schedule_add(self, deploy_name):
+def schedule_add(self, deploy_name: str):
try:
return request_url(verb='POST', uri=f'schedule/{deploy_name}').get('json')
except Exception as err:
@@ -280,7 +350,7 @@ def schedule_add(self, deploy_name):
@celery_app.task(bind=True, acks_late=True, time_limit=settings.WORKER_TMOUT, max_retries=1, name='Update schedule')
-def schedule_update(self, deploy_name):
+def schedule_update(self, deploy_name: str):
try:
request_url(verb='DELETE', uri=f'schedule/{deploy_name}')
return request_url(verb='POST', uri=f'schedule/{deploy_name}').get('json')
@@ -291,24 +361,24 @@ def schedule_update(self, deploy_name):
@celery_app.task(bind=True, acks_late=True, name='delete local module stack ')
-def delete_local_stack(self, environment, squad, args):
+def delete_local_stack(self, environment: str, squad: str, args: any):
result = tf.delete_local_repo(environment, squad, args)
return result
@celery_app.task(bind=True, acks_late=True, name='get variables list')
-def get_variable_list(self, environment, stack_name, squad, name):
+def get_variable_list(self, environment: str, stack_name: str, squad: str, name: str):
result = tf.get_vars_list(environment, stack_name, squad, name)
return result
@celery_app.task(bind=True, acks_late=True, name='get variables json')
-def get_variable_json(self, environment, stack_name, squad, name):
+def get_variable_json(self, environment: str, stack_name: str, squad: str, name: str):
result = tf.get_vars_json(environment, stack_name, squad, name)
return result
@celery_app.task(bind=True, acks_late=True, name='get variables from tfvars')
-def get_tfvars(self, environment, stack_name, squad, name):
+def get_tfvars(self, environment: str, stack_name: str, squad: str, name: str):
result = tf.get_vars_tfvars(environment, stack_name, squad, name)
return result
diff --git a/sld-dashboard/app/base/templates/accounts/login.html b/sld-dashboard/app/base/templates/accounts/login.html
index bd39a612..deda68ce 100644
--- a/sld-dashboard/app/base/templates/accounts/login.html
+++ b/sld-dashboard/app/base/templates/accounts/login.html
@@ -27,7 +27,7 @@
{% if msg %}
- {{ msg | safe }}
+
{{ msg | safe }}
{% else %}
Add your credentials
{% endif %}
@@ -77,4 +77,4 @@
{% endblock content %}
-{% block javascripts %}{% endblock javascripts %}
\ No newline at end of file
+{% block javascripts %}{% endblock javascripts %}
diff --git a/sld-dashboard/app/home/forms.py b/sld-dashboard/app/home/forms.py
index 9364455c..a0b92095 100644
--- a/sld-dashboard/app/home/forms.py
+++ b/sld-dashboard/app/home/forms.py
@@ -32,6 +32,9 @@ class StackForm(FlaskForm):
validators.DataRequired(message='The description is required.')
], render_kw={'rows': 5})
squad_access_edit = StringField('Squad Access (* share with everyone or pass a list: squad1, squad2)', render_kw={'rows': 1})
+ tfvar_file = StringField('tfvar_file', [
+ validators.length(min=5, max=30, message='Tfvars file name.'),
+ ])
description_edit = StringField('Description', render_kw={'rows': 1})
@@ -58,6 +61,9 @@ class DeployForm(FlaskForm):
destroy_time = StringField('Destroy Time', [
validators.length(min=3, max=30, message='Time out of reange.'),
])
+ tfvar_file = StringField('tfvar_file', [
+ validators.length(min=5, max=30, message='Tfvars file name.'),
+ ])
environment = StringField('Environment', [
validators.length(min=2, max=250, message='Branch out of reange.'),
validators.DataRequired(message='Environment requerid.')
diff --git a/sld-dashboard/app/home/routes.py b/sld-dashboard/app/home/routes.py
index defbd5c7..925f2f09 100644
--- a/sld-dashboard/app/home/routes.py
+++ b/sld-dashboard/app/home/routes.py
@@ -162,6 +162,7 @@ def relaunch_deploy(deploy_id):
data = {
"start_time": content['start_time'],
"destroy_time": content['destroy_time'],
+ "tfvar_file" : content['tfvar_file'],
"variables": content['variables']
}
response = request_url(
@@ -204,12 +205,13 @@ def edit_deploy(deploy_id):
response = request_url(verb='GET', uri=f'{endpoint}', headers={
"Authorization": f"Bearer {token}"})
deploy = response.get('json')
+ tfvar_file = deploy.get('tfvar_file')
# When user push data with POST verb
if request.method == 'POST':
# List for exclude in vars
form_vars = ["csrf_token", 'button', 'start_time',
- 'destroy_time', 'sld_key', 'sld_value']
+ 'destroy_time', 'sld_key', 'sld_value', 'tfvar_file']
# Clean exclude data vars
data_raw = {key: value for key,
value in request.form.items() if key not in form_vars}
@@ -223,6 +225,7 @@ def edit_deploy(deploy_id):
data = {
"start_time": form.start_time.data,
"destroy_time": form.destroy_time.data,
+ "tfvar_file" : form.tfvar_file.data,
"variables": ast.literal_eval(variables)
}
if not "deploy" in request.form.get('button'):
@@ -278,7 +281,7 @@ def get_plan(deploy_id):
# When user push data with POST verb
if request.method == 'POST':
# List for exclude in vars
- form_vars = ["csrf_token", 'button', 'start_time', 'destroy_time']
+ form_vars = ["csrf_token", 'button', 'start_time', 'destroy_time', 'tfvar_file']
# Clean exclude data vars
data_raw = {key: value for key,
value in request.form.items() if key not in form_vars}
@@ -290,6 +293,7 @@ def get_plan(deploy_id):
"environment": deploy['environment'],
"start_time": form.start_time.data,
"destroy_time": form.destroy_time.data,
+ "tfvar_file" : form.tfvar_file.data,
"variables": ast.literal_eval(variables)
}
# Deploy
@@ -342,7 +346,7 @@ def edit_schedule(deploy_id):
# When user push data with POST verb
if request.method == 'POST':
# List for exclude in vars
- form_vars = ["csrf_token", 'button', 'start_time', 'destroy_time']
+ form_vars = ["csrf_token", 'button', 'start_time', 'destroy_time', 'tfvar_file']
data = {
"start_time": form.start_time.data,
"destroy_time": form.destroy_time.data,
@@ -555,10 +559,12 @@ def deploy_stack(stack_id):
if request.method == 'POST':
# Define list for exclude vars in variables data
form_vars = ["csrf_token", 'environment', 'deploy_name',
- 'button', 'start_time', 'destroy_time', 'squad']
- data_raw = {key: value for key,
- value in request.form.items() if key not in form_vars}
- variables = json.dumps(convert_to_dict(data_raw))
+ 'button', 'start_time', 'destroy_time', 'squad', 'tfvar_file']
+ variables = {}
+ if request.form.get('tfvar_file') == "":
+ data_raw = {key: value for key,
+ value in request.form.items() if key not in form_vars}
+ variables = ast.literal_eval(json.dumps(convert_to_dict(data_raw)))
data = {
"name": form.deploy_name.data,
"stack_name": stack['json']['stack_name'],
@@ -566,7 +572,8 @@ def deploy_stack(stack_id):
"destroy_time": form.destroy_time.data,
"squad": request.form.get('squad'),
"environment": request.form.get('environment'),
- "variables": ast.literal_eval(variables)
+ "tfvar_file": request.form.get('tfvar_file'),
+ "variables": variables
}
endpoint = f'plan'
if not "plan" in request.form.get('button'):
@@ -582,7 +589,7 @@ def deploy_stack(stack_id):
json=data
)
if response.get('status_code') == 202:
- flash(f"Deploying stack {form.stack_name.data}")
+ flash(f"Deploying stack {stack['json']['stack_name']}")
else:
flash(response['json']['detail'], 'error')
return redirect(url_for('home_blueprint.route_template', template="deploys-list"))
diff --git a/sld-dashboard/app/home/templates/deploy-edit.html b/sld-dashboard/app/home/templates/deploy-edit.html
index c616771e..2319c172 100644
--- a/sld-dashboard/app/home/templates/deploy-edit.html
+++ b/sld-dashboard/app/home/templates/deploy-edit.html
@@ -49,6 +49,11 @@ {{deploy.name}}
{{ render_field(form.destroy_time, autocomplete="off", class='form-control',
value=deploy.destroy_time) }}
+
+ {{ render_field(form.tfvar_file, autocomplete="off", class='form-control',
+ value=deploy.tfvar_file) }}
+
+ {% if deploy.tfvar_file == "" %}
{% for key, value in data_json.items()|sort %}
@@ -64,7 +69,6 @@
{{key}}
{% endif %}
{% endfor %}
-
@@ -78,13 +82,14 @@
{{key}}
+ {% endif %}
{% if deploy.action == "Plan"%}
{% else %}
-
diff --git a/sld-dashboard/app/home/templates/deploy-plan.html b/sld-dashboard/app/home/templates/deploy-plan.html
index 6961c6fd..915cefd3 100644
--- a/sld-dashboard/app/home/templates/deploy-plan.html
+++ b/sld-dashboard/app/home/templates/deploy-plan.html
@@ -58,7 +58,7 @@ {{key}}
{% endfor %}
@@ -79,4 +79,4 @@ {{key}}
{% endblock content %}
-{% block javascripts %}{% endblock javascripts %}
\ No newline at end of file
+{% block javascripts %}{% endblock javascripts %}
diff --git a/sld-dashboard/app/home/templates/stacks-deploy.html b/sld-dashboard/app/home/templates/stacks-deploy.html
index 403b6f87..6609a497 100644
--- a/sld-dashboard/app/home/templates/stacks-deploy.html
+++ b/sld-dashboard/app/home/templates/stacks-deploy.html
@@ -145,6 +145,9 @@ Environment
{{ render_field(form.destroy_time, autocomplete="off", class='form-control', value='30 18 * * 0-4') }}
+
+ {{ render_field(form.tfvar_file, autocomplete="off", class='form-control', value='') }}
+
{% if not sort_form %}
{% for key, value in data_json.items()%}