From 68d977b6c21ccf875a473c46be92539f521b6720 Mon Sep 17 00:00:00 2001 From: D10S0VSkY-OSS <79284025+D10S0VSkY-OSS@users.noreply.github.com> Date: Mon, 4 Dec 2023 20:35:59 +0100 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=94=A7refactor:=20Set=20all=20outputs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/deploy/api/container/deploy/get.py | 49 ++++++++++++------- .../src/worker/providers/hashicorp/actions.py | 1 - sld-dashboard/app/home/forms.py | 2 +- .../app/home/templates/stacks-list.html | 14 +++--- 4 files changed, 40 insertions(+), 26 deletions(-) diff --git a/sld-api-backend/src/deploy/api/container/deploy/get.py b/sld-api-backend/src/deploy/api/container/deploy/get.py index bb9ccc0f..cdaee40c 100644 --- a/sld-api-backend/src/deploy/api/container/deploy/get.py +++ b/sld-api-backend/src/deploy/api/container/deploy/get.py @@ -1,4 +1,7 @@ -from fastapi import Depends, HTTPException +import logging +import requests +import jmespath +from fastapi import Depends, HTTPException, Query from sqlalchemy.orm import Session from src.deploy.infrastructure import repositories as crud_deploys @@ -7,6 +10,7 @@ from src.shared.security import deps from src.users.domain.entities import users as schemas_users from src.users.infrastructure import repositories as crud_users +from config.api import settings async def unlock_deploy( @@ -92,24 +96,35 @@ async def get_show( raise HTTPException(status_code=400, detail=f"{err}") +async def check_task_is_dict( + task_id: str, deploy_data): + if isinstance(task_id.info, dict): + return task_id.info.get("stdout", []) + else: + logging.error(f"Task {deploy_data.id} is not a dict") + raise HTTPException( + status_code=404, detail=f"Not enough output in {deploy_data.name}" + ) + async def get_output( deploy_id: int, db: Session = Depends(deps.get_db), current_user: schemas_users.User = Depends(deps.get_current_active_user), ): - # Get info from deploy data - deploy_data = deploy(db, deploy_id=deploy_id) - squad = deploy_data.squad - if not crud_users.is_master(db, current_user): - if not check_squad_user(current_user.squad, [squad]): - raise HTTPException( - status_code=403, detail=f"Not enough permissions in {squad}" - ) - try: - stack_name = deploy_data.stack_name - environment = deploy_data.environment - name = deploy_data.name - # Get credentials by providers supported - return {"task": async_output(stack_name, squad, environment, name)} - except Exception as err: - raise HTTPException(status_code=400, detail=f"{err}") + if crud_users.is_master(db, current_user): + deploy_data = deploy(db, deploy_id=deploy_id) + else: + # Get squad from current user + squad = current_user.squad + deploy_data = deploy_squad(db, deploy_id=deploy_id, squad=squad) + get_path = f"{deploy_data.stack_name}-{deploy_data.squad}-{deploy_data.environment}-{deploy_data.name}" + response = requests.get( + f"{settings.REMOTE_STATE}/terraform_state/{get_path}" + ) + json_data = response.json() + result = json_data.get("outputs") + if not result: + raise HTTPException( + status_code=404, detail=f"Not enough output in {deploy_data.name}" + ) + return result \ No newline at end of file diff --git a/sld-api-backend/src/worker/providers/hashicorp/actions.py b/sld-api-backend/src/worker/providers/hashicorp/actions.py index 4718d9d9..1ddad3a5 100644 --- a/sld-api-backend/src/worker/providers/hashicorp/actions.py +++ b/sld-api-backend/src/worker/providers/hashicorp/actions.py @@ -129,7 +129,6 @@ def unlock_execute(self): def show_execute(self): try: get_path = f"{self.stack_name}-{self.squad}-{self.environment}-{self.name}" - print(get_path) response = requests.get( f"{settings.REMOTE_STATE}/terraform_state/{get_path}" ) diff --git a/sld-dashboard/app/home/forms.py b/sld-dashboard/app/home/forms.py index 37ee60c1..15604c1c 100644 --- a/sld-dashboard/app/home/forms.py +++ b/sld-dashboard/app/home/forms.py @@ -38,7 +38,7 @@ class StackForm(FlaskForm): ) iac_type = SelectField( "IaC Type", - choices=[('terraform', 'Terraform'), ('openTofu', 'openTofu')], + choices=[('', 'Select an IaC Type'), ('terraform', 'Terraform'), ('openTofu', 'openTofu')], validators=[validators.DataRequired()], coerce=lambda x: 'tofu' if x == 'openTofu' else x ) diff --git a/sld-dashboard/app/home/templates/stacks-list.html b/sld-dashboard/app/home/templates/stacks-list.html index 798b685c..d736438a 100644 --- a/sld-dashboard/app/home/templates/stacks-list.html +++ b/sld-dashboard/app/home/templates/stacks-list.html @@ -76,13 +76,13 @@

All Stacks

{% if "yoda" in current_user.role or "darth_vader" in current_user.role %} {{ stack.git_repo }} {{ stack.branch }} - - {% if stack.iac_type == 'terraform' %} - Terraform Icon - {% elif stack.iac_type == 'tofu' %} - Tofu Icon - {% endif %} - + + {% if stack.iac_type == 'terraform' %} + Terraform Icon + {% elif stack.iac_type == 'tofu' %} + Tofu Icon + {% endif %} + {{ stack.tf_version }} {{ stack.squad_access }} {% endif %} From 082b0fca6f609034458aecea4745814223112b2f Mon Sep 17 00:00:00 2001 From: d10s <79284025+D10S0VSkY-OSS@users.noreply.github.com> Date: Mon, 4 Dec 2023 23:47:29 +0100 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=94=A7refactor:=20outputs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/deploy/api/container/deploy/get.py | 102 ++++++++++-------- sld-api-backend/src/deploy/api/v1/deploy.py | 8 +- .../src/shared/helpers/push_task.py | 24 ----- .../src/worker/domain/services/provider.py | 20 +--- .../src/worker/providers/hashicorp/actions.py | 46 +------- .../src/worker/tasks/terraform_worker.py | 43 +------- sld-dashboard/app/home/routes.py | 27 ++++- 7 files changed, 94 insertions(+), 176 deletions(-) diff --git a/sld-api-backend/src/deploy/api/container/deploy/get.py b/sld-api-backend/src/deploy/api/container/deploy/get.py index cdaee40c..4deb4efc 100644 --- a/sld-api-backend/src/deploy/api/container/deploy/get.py +++ b/sld-api-backend/src/deploy/api/container/deploy/get.py @@ -1,39 +1,16 @@ -import logging import requests -import jmespath -from fastapi import Depends, HTTPException, Query +from fastapi import Depends, HTTPException from sqlalchemy.orm import Session from src.deploy.infrastructure import repositories as crud_deploys from src.shared.helpers.get_data import check_squad_user, deploy, deploy_squad -from src.shared.helpers.push_task import async_output, async_show, async_unlock from src.shared.security import deps from src.users.domain.entities import users as schemas_users from src.users.infrastructure import repositories as crud_users from config.api import settings -async def unlock_deploy( - deploy_id: int, - db: Session = Depends(deps.get_db), - current_user: schemas_users.User = Depends(deps.get_current_active_user), -): - # Get info from deploy data - deploy_data = deploy(db, deploy_id=deploy_id) - squad = deploy_data.squad - if not crud_users.is_master(db, current_user): - if not check_squad_user(current_user.squad, [squad]): - raise HTTPException( - status_code=403, detail=f"Not enough permissions in {squad}" - ) - try: - stack_name = deploy_data.stack_name - environment = deploy_data.environment - name = deploy_data.name - # Get credentials by providers supported - return {"task": async_unlock(stack_name, squad, environment, name)} - except Exception as err: - raise HTTPException(status_code=400, detail=f"{err}") + async def get_all_deploys( @@ -70,7 +47,6 @@ async def get_deploy_by_id( raise HTTPException(status_code=404, detail="Deploy id Not Found") return result except Exception as err: - print(err) raise HTTPException(status_code=404, detail=f"{err}") @@ -82,30 +58,22 @@ async def get_show( # Get info from deploy data if crud_users.is_master(db, current_user): deploy_data = deploy(db, deploy_id=deploy_id) - squad = deploy_data.squad else: # Get squad from current user squad = current_user.squad deploy_data = deploy_squad(db, deploy_id=deploy_id, squad=squad) - stack_name = deploy_data.stack_name - environment = deploy_data.environment - name = deploy_data.name - try: - return {"task": async_show(stack_name, squad, environment, name)} - except Exception as err: - raise HTTPException(status_code=400, detail=f"{err}") - - -async def check_task_is_dict( - task_id: str, deploy_data): - if isinstance(task_id.info, dict): - return task_id.info.get("stdout", []) - else: - logging.error(f"Task {deploy_data.id} is not a dict") + get_path = f"{deploy_data.stack_name}-{deploy_data.squad}-{deploy_data.environment}-{deploy_data.name}" + response = requests.get( + f"{settings.REMOTE_STATE}/terraform_state/{get_path}" + ) + result = response.json() + if not result: raise HTTPException( status_code=404, detail=f"Not enough output in {deploy_data.name}" ) - + return result + + async def get_output( deploy_id: int, db: Session = Depends(deps.get_db), @@ -127,4 +95,50 @@ async def get_output( raise HTTPException( status_code=404, detail=f"Not enough output in {deploy_data.name}" ) - return result \ No newline at end of file + return result + + +async def unlock_deploy( + deploy_id: int, + db: Session = Depends(deps.get_db), + current_user: schemas_users.User = Depends(deps.get_current_active_user), +): + # Get info from deploy data + deploy_data = deploy(db, deploy_id=deploy_id) + squad = deploy_data.squad + if not crud_users.is_master(db, current_user): + if not check_squad_user(current_user.squad, [squad]): + raise HTTPException( + status_code=403, detail=f"Not enough permissions in {squad}" + ) + try: + get_path = f"{deploy_data.stack_name}-{deploy_data.squad}-{deploy_data.environment}-{deploy_data.name}" + response = requests.delete( + f"{settings.REMOTE_STATE}/terraform_lock/{get_path}", json={} + ) + return response.json() + except Exception as err: + raise err + + +async def lock_deploy( + deploy_id: int, + db: Session = Depends(deps.get_db), + current_user: schemas_users.User = Depends(deps.get_current_active_user), +): + # Get info from deploy data + deploy_data = deploy(db, deploy_id=deploy_id) + squad = deploy_data.squad + if not crud_users.is_master(db, current_user): + if not check_squad_user(current_user.squad, [squad]): + raise HTTPException( + status_code=403, detail=f"Not enough permissions in {squad}" + ) + try: + get_path = f"{deploy_data.stack_name}-{deploy_data.squad}-{deploy_data.environment}-{deploy_data.name}" + response = requests.put( + f"{settings.REMOTE_STATE}/terraform_lock/{get_path}", json={} + ) + return response.json() + except Exception as err: + raise err diff --git a/sld-api-backend/src/deploy/api/v1/deploy.py b/sld-api-backend/src/deploy/api/v1/deploy.py index ad0fd312..767abd16 100644 --- a/sld-api-backend/src/deploy/api/v1/deploy.py +++ b/sld-api-backend/src/deploy/api/v1/deploy.py @@ -57,12 +57,18 @@ async def get_output( return get_output -@router.put("/unlock/{deploy_id}", status_code=200) +@router.delete("/unlock/{deploy_id}", status_code=200) async def unlock_deploy( unlock_deploy: schemas_deploy.DeployBase = Depends(get.unlock_deploy), ): return unlock_deploy +@router.put("/lock/{deploy_id}", status_code=200) +async def unlock_deploy( + lock_deploy: schemas_deploy.DeployBase = Depends(get.lock_deploy), +): + return lock_deploy + @router.get("/show/{deploy_id}", status_code=202) async def get_show( diff --git a/sld-api-backend/src/shared/helpers/push_task.py b/sld-api-backend/src/shared/helpers/push_task.py index 9d1e537d..bcd2ccfa 100644 --- a/sld-api-backend/src/shared/helpers/push_task.py +++ b/sld-api-backend/src/shared/helpers/push_task.py @@ -5,7 +5,6 @@ from src.worker.domain.entities.worker import DeployParams, DownloadGitRepoParams from src.worker.tasks.terraform_worker import ( - output, pipeline_deploy, pipeline_destroy, pipeline_git_pull, @@ -15,8 +14,6 @@ schedule_get, schedule_update, schedules_list, - show, - unlock, ) @@ -59,20 +56,6 @@ def async_plan(plan_params: DeployParams): return pipeline_deploy_result.task_id -def async_output(stack_name: str, environment: str, squad: str, name: str): - output_result = output.s(stack_name, environment, squad, name).apply_async( - queue="squad" - ) - return output_result.task_id - - -def async_unlock(stack_name: str, squad: str, environment: str, name: str): - unlock_result = unlock.s(stack_name, squad, environment, name).apply_async( - queue="squad" - ) - return unlock_result.task_id - - def async_schedule_delete(deploy_name: str, squad: str): deploy_schedule_delete_result = schedule_delete.s(deploy_name).apply_async( queue="squad" @@ -100,13 +83,6 @@ def async_schedule_update(deploy_name: str): return schedule_update_result.task_id -def async_show(stack_name: str, environment: str, squad: str, name: str): - show_result = show.s(stack_name, environment, squad, name).apply_async( - queue="squad" - ) - return show_result.task_id - - def sync_git( stack_name: str, git_repo: str, diff --git a/sld-api-backend/src/worker/domain/services/provider.py b/sld-api-backend/src/worker/domain/services/provider.py index b5385a06..2e3354fa 100644 --- a/sld-api-backend/src/worker/domain/services/provider.py +++ b/sld-api-backend/src/worker/domain/services/provider.py @@ -1,5 +1,5 @@ # DI terraform provider -from src.worker.providers.hashicorp.actions import Actions, SimpleActions +from src.worker.providers.hashicorp.actions import Actions from src.worker.providers.hashicorp.artifact import Artifact from src.worker.providers.hashicorp.download import BinaryDownload from src.worker.providers.hashicorp.templates import Backend, GetVars, Tfvars @@ -123,21 +123,3 @@ def destroy(params: DeployParams, action: Actions = Actions) -> dict: params.task_id, ) return config_action.execute_terraform_command("destroy") - - def output( - stack_name: str, squad: str, environment: str, name: str, action=SimpleActions - ) -> dict: - config_action = action(stack_name, squad, environment, name) - return config_action.output_execute() - - def unlock( - stack_name: str, squad: str, environment: str, name: str, action=SimpleActions - ) -> dict: - config_action = action(stack_name, squad, environment, name) - return config_action.unlock_execute() - - def show( - stack_name: str, squad: str, environment: str, name: str, action=SimpleActions - ) -> dict: - config_action = action(stack_name, squad, environment, name) - return config_action.show_execute() diff --git a/sld-api-backend/src/worker/providers/hashicorp/actions.py b/sld-api-backend/src/worker/providers/hashicorp/actions.py index 1ddad3a5..dde645c1 100644 --- a/sld-api-backend/src/worker/providers/hashicorp/actions.py +++ b/sld-api-backend/src/worker/providers/hashicorp/actions.py @@ -91,48 +91,4 @@ def execute_terraform_command(self, action: str) -> dict: "project_path": f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}", "remote_state": f"http://remote-state:8080/terraform_state/{deploy_state}", "stdout": output, - } - - -@dataclass -class SimpleActions: - stack_name: str - squad: str - environment: str - name: str - - def output_execute(self): - try: - get_path = f"{self.stack_name}-{self.squad}-{self.environment}-{self.name}" - print(get_path) - response = requests.get( - f"{settings.REMOTE_STATE}/terraform_state/{get_path}" - ) - json_data = response.json() - result = json_data.get("outputs") - if not result: - result = jmespath.search("modules[*].outputs", json_data) - return result - except Exception as err: - return {"command": "output", "rc": 1, "stdout": err} - - def unlock_execute(self): - try: - get_path = f"{self.stack_name}-{self.squad}-{self.environment}-{self.name}" - response = requests.delete( - f"{settings.REMOTE_STATE}/terraform_lock/{get_path}", json={} - ) - return response.json() - except Exception as err: - return {"command": "unlock", "rc": 1, "stdout": err} - - def show_execute(self): - try: - get_path = f"{self.stack_name}-{self.squad}-{self.environment}-{self.name}" - response = requests.get( - f"{settings.REMOTE_STATE}/terraform_state/{get_path}" - ) - json_data = response.json() - return json_data - except Exception as err: - return {"command": "show", "rc": 1, "stdout": err} + } \ No newline at end of file diff --git a/sld-api-backend/src/worker/tasks/terraform_worker.py b/sld-api-backend/src/worker/tasks/terraform_worker.py index a827bfaf..c3bca249 100644 --- a/sld-api-backend/src/worker/tasks/terraform_worker.py +++ b/sld-api-backend/src/worker/tasks/terraform_worker.py @@ -136,6 +136,7 @@ def pipeline_plan( if not settings.DEBUG: Utils.delete_local_folder(dir_path) + @celery_app.task( bind=True, acks_late=True, time_limit=settings.GIT_TMOUT, name="pipeline git pull" ) @@ -166,48 +167,6 @@ def pipeline_git_pull( Utils.delete_local_folder(dir_path) - -@celery_app.task( - bind=True, - acks_late=True, - time_limit=settings.WORKER_TMOUT, - max_retries=1, - name="terraform output", -) -def output(self, stack_name: str, squad: str, environment: str, name: str): - try: - output_result = ProviderActions.output(stack_name, squad, environment, name) - return output_result - except Exception as err: - return {"stdout": err} - finally: - dir_path = f"/tmp/{ stack_name }/{environment}/{squad}/{name}" - Utils.delete_local_folder(dir_path) - - -@celery_app.task( - bind=True, - acks_late=True, - time_limit=settings.WORKER_TMOUT, - max_retries=1, - name="terraform unlock", -) -def unlock(self, stack_name: str, squad: str, environment: str, name: str): - try: - unlock_result = ProviderActions.unlock(stack_name, squad, environment, name) - return unlock_result - except Exception as err: - return {"stdout": err} - - -@celery_app.task( - bind=True, acks_late=True, time_limit=settings.WORKER_TMOUT, name="terraform show" -) -def show(self, stack_name: str, squad: str, environment: str, name: str): - show_result = ProviderActions.show(stack_name, squad, environment, name) - return show_result - - @celery_app.task( bind=True, acks_late=True, time_limit=settings.WORKER_TMOUT, name="schedules list" ) diff --git a/sld-dashboard/app/home/routes.py b/sld-dashboard/app/home/routes.py index e50564fb..e99dd2d0 100644 --- a/sld-dashboard/app/home/routes.py +++ b/sld-dashboard/app/home/routes.py @@ -247,6 +247,31 @@ def destroy_deploy_console(deploy_id): @blueprint.route("/deploys/unlock/") @login_required def unlock_deploy(deploy_id): + try: + token = decrypt(r.get(current_user.id)) + # Check if token no expired + check_unauthorized_token(token) + endpoint = f"deploy/unlock/{deploy_id}" + response = request_url( + verb="DELETE", uri=f"{endpoint}", headers={"Authorization": f"Bearer {token}"} + ) + if response.get("status_code") == 200: + flash("Unlock deploy") + else: + flash(response["json"]["detail"], "error") + return redirect( + url_for("home_blueprint.route_template", template="deploys-list") + ) + except TemplateNotFound: + return render_template("page-404.html"), 404 + except TypeError: + return redirect(url_for("base_blueprint.logout")) + except Exception: + return render_template("page-500.html"), 500 + +@blueprint.route("/deploys/lock/") +@login_required +def lock_deploy(deploy_id): try: token = decrypt(r.get(current_user.id)) # Check if token no expired @@ -256,7 +281,7 @@ def unlock_deploy(deploy_id): verb="PUT", uri=f"{endpoint}", headers={"Authorization": f"Bearer {token}"} ) if response.get("status_code") == 200: - flash(f"Unlock deploy") + flash("lock deploy") else: flash(response["json"]["detail"], "error") return redirect(