🔧refactor: Migrate to python 3.11 (#168)

* 🔧refactor: Migrate to python 3.11 * 🔧refactor: outputs * 🔧refactor: outputs symbol colors * 🔧refactor: outputs symbol colors update in place * 🔧refactor: deploy set pagination and search * 🔧refactor: add stack pagination and search * 🔧refactor: add pagination and search component js * 🔧refactor: add pagination and search component js for users * 🐛fix: edit deploy * 🐛fix: edit users * 🐛fix: add destroy for plan UI * 🔧refactor: stacks forms * 🔧refactor: stacks forms variables * 🔧refactor: Dockerfiles for use asdf and python3.11 * 🔧refactor: github action bump version
D10S0VSkY-OSS · Nov 11, 2023 · df1936b · df1936b
1 parent 4a62a93
commit df1936b
Show file tree

Hide file tree

Showing 38 changed files with 683 additions and 488 deletions.
diff --git a/.github/workflows/sld-api-docker-image.yml b/.github/workflows/sld-api-docker-image.yml
@@ -24,11 +24,11 @@ jobs:
 
     - name: Build the Docker image with tag
       working-directory: ./sld-api-backend
-      run: docker build . --file Dockerfile --tag ${{ secrets.DOCKER_USERNAME }}/sld-api:2.17.0
+      run: docker build . --file Dockerfile --tag ${{ secrets.DOCKER_USERNAME }}/sld-api:2.18.0
 
     - name: Docker Push with tag
       #if: github.event.pull_request.merged == true
-      run: docker push ${{ secrets.DOCKER_USERNAME }}/sld-api:2.17.0
+      run: docker push ${{ secrets.DOCKER_USERNAME }}/sld-api:2.18.0
 
     - name: Build the Docker image
       working-directory: ./sld-api-backend

diff --git a/.github/workflows/sld-dashboard-docker-image.yml b/.github/workflows/sld-dashboard-docker-image.yml
@@ -24,11 +24,11 @@ jobs:
 
     - name: Build the Docker image with tags
       working-directory: ./sld-dashboard
-      run: docker build . --file Dockerfile --tag ${{ secrets.DOCKER_USERNAME }}/sld-dashboard:2.15.0
+      run: docker build . --file Dockerfile --tag ${{ secrets.DOCKER_USERNAME }}/sld-dashboard:2.16.0
 
     - name: Docker Push with tags
       #if: github.event.pull_request.merged == true
-      run: docker push ${{ secrets.DOCKER_USERNAME }}/sld-dashboard:2.15.0
+      run: docker push ${{ secrets.DOCKER_USERNAME }}/sld-dashboard:2.16.0
 
     - name: Build the Docker image
       working-directory: ./sld-dashboard

diff --git a/.github/workflows/sld-remote-state-docker-image.yml b/.github/workflows/sld-remote-state-docker-image.yml
@@ -24,11 +24,11 @@ jobs:
 
     - name: Build the Docker image with tags
       working-directory: ./sld-remote-state
-      run: docker build . --file Dockerfile --tag ${{ secrets.DOCKER_USERNAME }}/sld-remote-state:2.10.0
+      run: docker build . --file Dockerfile --tag ${{ secrets.DOCKER_USERNAME }}/sld-remote-state:2.11.0
 
     - name: Docker Push with tags
       #if: github.event.pull_request.merged == true
-      run: docker push ${{ secrets.DOCKER_USERNAME }}/sld-remote-state:2.10.0
+      run: docker push ${{ secrets.DOCKER_USERNAME }}/sld-remote-state:2.11.0
 
     - name: Build the Docker image
       working-directory: ./sld-remote-state

diff --git a/.github/workflows/sld-schedule-docker-image.yml b/.github/workflows/sld-schedule-docker-image.yml
@@ -24,11 +24,11 @@ jobs:
 
     - name: Build the Docker image with tags
       working-directory: ./sld-schedule
-      run: docker build . --file Dockerfile --tag ${{ secrets.DOCKER_USERNAME }}/sld-schedule:2.6.0
+      run: docker build . --file Dockerfile --tag ${{ secrets.DOCKER_USERNAME }}/sld-schedule:2.7.0
 
     - name: Docker Push with tags
       #if: github.event.pull_request.merged == true
-      run: docker push ${{ secrets.DOCKER_USERNAME }}/sld-schedule:2.6.0
+      run: docker push ${{ secrets.DOCKER_USERNAME }}/sld-schedule:2.7.0
 
     - name: Build the Docker image
       working-directory: ./sld-schedule

diff --git a/sld-api-backend/.tool-versions b/sld-api-backend/.tool-versions
@@ -0,0 +1 @@
+python 3.11.6
diff --git a/sld-api-backend/Dockerfile b/sld-api-backend/Dockerfile
@@ -1,32 +1,54 @@
 FROM ubuntu:22.04
-MAINTAINER D10S0VSkY
 
+# Metadata
+MAINTAINER D10S0VSkY
 ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
 ENV TZ=Europe/Madrid
 
+# Set up working directory
 WORKDIR /app
 ADD ./requirements.txt /app/requirements.txt
 
+# Create a user and group
 RUN groupadd --gid 10000 sld && \
-    useradd  --uid 10000 --gid sld --shell /bin/bash --create-home sld
+    useradd --uid 10000 --gid sld --shell /bin/bash --create-home sld
 
+# Set timezone
 RUN echo $TZ > /etc/timezone  && ln -snf /usr/share/zoneinfo/$TZ /etc/localtime
 
-
+# Install dependencies including build tools
 RUN export DEBIAN_FRONTEND=noninteractive && \
-apt-get update && \
-apt-get upgrade -yq && \
-apt-get -yq install \
-python3.10 pip default-libmysqlclient-dev zip git tzdata && \
-pip install --no-cache-dir -r requirements.txt
+    apt-get update && \
+    apt-get -yq install curl git zip tzdata build-essential libssl-dev libffi-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev pkg-config libmysqlclient-dev
 
-RUN apt-get clean autoclean && \
-apt-get autoremove -y && \
-rm -rf /var/lib/{apt,dpkg,cache,log}/
 
+# Install asdf, Python plugin, and Python version
+RUN git clone https://github.com/asdf-vm/asdf.git ~/.asdf --branch v0.10.0 && \
+    echo '. $HOME/.asdf/asdf.sh' >> ~/.bashrc && \
+    echo '. $HOME/.asdf/completions/asdf.bash' >> ~/.bashrc
+
+SHELL ["/bin/bash", "-c"]
+
+RUN . $HOME/.asdf/asdf.sh && \
+    asdf plugin add python && \
+    asdf install python 3.11.6 && \
+    asdf global python 3.11.6
+
+
+# Install Python packages
+RUN . $HOME/.asdf/asdf.sh && \
+    python -m pip install --upgrade pip setuptools && \
+    python -m pip install --no-cache-dir -r requirements.txt
+
+# Clean up
+RUN apt-get clean autoclean && \
+    apt-get autoremove -y && \
+    rm -rf /var/lib/{apt,dpkg,cache,log}
 
+# Add the rest of the application
 ADD . /app/
 RUN chown -R sld /app
 
+# Switch to user
 USER sld
diff --git a/sld-api-backend/config/api.py b/sld-api-backend/config/api.py
@@ -1,8 +1,7 @@
 import os
 from typing import List
 
-from pydantic import BaseSettings
-
+from pydantic_settings import BaseSettings
 
 class Settings(BaseSettings):
     # Schedle config
@@ -23,7 +22,7 @@ class Settings(BaseSettings):
         "SLD_SECRET_KEY",
         "09d25e094faa6ca2556c818166b7a9563b93f7099f6f0f4caa6cf63b88e8d3e7",
     )
-    ALGORITHM = "HS256"
+    ALGORITHM: str = "HS256"
     # 60 minutes * 24 hours * 8 days = 8 days
     ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 8
     SECRET_VAULT: bytes = os.getenv(
@@ -36,11 +35,11 @@ class Settings(BaseSettings):
     WORKER_TMOUT: int = os.getenv("SLD_WORKER_TMOUT", 300)
     ENV: str = os.getenv("SLD_ENV", "dev")
     DEBUG: bool = os.getenv("SLD_DEBUG", False)
-    BACKEND_USER = os.getenv("BACKEND_USER", "")
-    BACKEND_PASSWD = os.getenv("BACKEND_PASSWD", "")
-    BACKEND_SERVER = os.getenv("BACKEND_SERVER", "redis")
+    BACKEND_USER: str = os.getenv("BACKEND_USER", "")
+    BACKEND_PASSWD: str = os.getenv("BACKEND_PASSWD", "")
+    BACKEND_SERVER: str = os.getenv("BACKEND_SERVER", "redis")
     # init user
-    INIT_USER = {
+    INIT_USER: dict = {
         "username": os.getenv("SLD_INIT_USER_NAME", "admin"),
         "fullname": os.getenv("SLD_INIT_USER_FULLNAME", "Master of the universe user"),
         "email": os.getenv("SLD_INIT_USER_email", "admin@example.com"),
@@ -50,7 +49,7 @@ class Settings(BaseSettings):
     AWS_SHARED_CONFIG_FILE: str = f"{AWS_CONGIG_DEFAULT_FOLDER}/config"
     TASK_MAX_RETRY: int = os.getenv("SLD_TASK_MAX_RETRY", 1)
     TASK_RETRY_INTERVAL: int = os.getenv("SLD_TASK_RETRY_INTERVAL", 20)
-    TASK_LOCKED_EXPIRED = os.getenv("SLD_TASK_LOCKED_EXPIRED", 300)
+    TASK_LOCKED_EXPIRED: int = os.getenv("SLD_TASK_LOCKED_EXPIRED", 300)
     TASK_ROUTE: bool = os.getenv("SLD_TASK_ROUTE", False)
     TERRAFORM_BIN_REPO: str = os.getenv(
         "SLD_TERRAFORM_BIN_REPO", "https://releases.hashicorp.com/terraform"

diff --git a/sld-api-backend/config/celery_config.py b/sld-api-backend/config/celery_config.py
@@ -4,13 +4,13 @@
 
 celery_app = None
 # Rabbit broker config
-BROKER_USER = os.getenv("BROKER_USER", "admin")
-BROKER_PASSWD = os.getenv("BROKER_PASSWD", "admin")
-BROKER_SERVER = os.getenv("BROKER_SERVER", "rabbit")  # use rabbit or redis
+BROKER_USER = os.getenv("BROKER_USER", "")
+BROKER_PASSWD = os.getenv("BROKER_PASSWD", "")
+BROKER_SERVER = os.getenv("BROKER_SERVER", "redis")  # use rabbit or redis
 BROKER_SERVER_PORT = os.getenv(
-    "BROKER_SERVER_PORT", "5672"
+    "BROKER_SERVER_PORT", "6379"
 )  # use por 6379 for redis or 5672 for RabbitMQ
-BROKER_TYPE = os.getenv("BROKER_TYPE", "amqp")  # use amqp for RabbitMQ or redis
+BROKER_TYPE = os.getenv("BROKER_TYPE", "redis")  # use amqp for RabbitMQ or redis
 # Redus backend config
 BACKEND_TYPE = os.getenv("BACKEND_TYPE", "redis")
 BACKEND_USER = os.getenv("BACKEND_USER", "")

diff --git a/sld-api-backend/requirements.txt b/sld-api-backend/requirements.txt
@@ -1,68 +1,73 @@
 aioredis==2.0.1
 amqp==5.1.1
-anyio==3.6.2
-async-timeout==4.0.2
-attrs==22.2.0
+annotated-types==0.6.0
+anyio==3.7.1
+async-timeout==4.0.3
+attrs==23.1.0
 bcrypt==4.0.1
-billiard==3.6.4.0
-celery==5.2.7
-certifi==2022.12.7
-cffi==1.15.1
-charset-normalizer==3.0.1
-click==8.1.3
+billiard==4.1.0
+celery==5.3.4
+certifi==2023.7.22
+cffi==1.16.0
+charset-normalizer==3.3.0
+click==8.1.7
 click-didyoumean==0.3.0
 click-plugins==1.1.1
-click-repl==0.2.0
-croniter==1.3.8
-cryptography==39.0.1
+click-repl==0.3.0
+croniter==2.0.1
+cryptography==41.0.4
 dependency-injector==4.41.0
-Deprecated==1.2.13
-dnspython==2.3.0
+Deprecated==1.2.14
+dnspython==2.4.2
 ecdsa==0.18.0
-email-validator==1.3.1
-exceptiongroup==1.1.0
-fastapi==0.91.0
+email-validator==2.0.0.post2
+exceptiongroup==1.1.3
+fastapi==0.104.1
 fastapi-limiter==0.1.5
 fastapi-versioning==0.10.0
-greenlet==2.0.2
+greenlet==3.0.0
 h11==0.14.0
 idna==3.4
 iniconfig==2.0.0
 Jinja2==3.1.2
 jmespath==1.0.1
-kombu==5.2.4
-MarkupSafe==2.1.2
-mysqlclient==2.1.1
-packaging==23.0
+kombu==5.3.2
+MarkupSafe==2.1.3
+mysqlclient==2.2.0
+packaging==23.2
 passlib==1.7.4
 password-strength==0.0.3.post2
-pluggy==1.0.0
-prompt-toolkit==3.0.36
+pluggy==1.3.0
+prompt-toolkit==3.0.39
 py==1.11.0
-pyasn1==0.4.8
+pyasn1==0.5.0
 pycparser==2.21
-pydantic==1.10.4
-pyhcl==0.4.4
-PyJWT==2.6.0
-PyMySQL==1.0.2
-pyparsing==3.0.9
-pytest==7.2.1
+pydantic==2.4.2
+pydantic-settings==2.0.3
+pydantic_core==2.10.1
+pyhcl==0.4.5
+PyJWT==2.8.0
+PyMySQL==1.1.0
+pyparsing==3.1.1
+pytest==7.4.2
 python-dateutil==2.8.2
+python-dotenv==1.0.0
 python-jose==3.3.0
-python-multipart==0.0.5
-python-usernames==0.3.1
-pytz==2022.7.1
-redis==4.5.1
-requests==2.28.2
+python-multipart==0.0.6
+python-usernames==1.0.0
+pytz==2023.3.post1
+redis==4.6.0
+requests==2.31.0
 rsa==4.9
 six==1.16.0
 sniffio==1.3.0
-SQLAlchemy==2.0.3
-starlette==0.24.0
+SQLAlchemy==2.0.22
+starlette==0.27.0
 tomli==2.0.1
-typing_extensions==4.4.0
-urllib3==1.26.14
-uvicorn==0.20.0
+typing_extensions==4.8.0
+tzdata==2023.3
+urllib3==2.0.7
+uvicorn==0.23.2
 vine==5.0.0
-wcwidth==0.2.6
-wrapt==1.14.1
+wcwidth==0.2.8
+wrapt==1.15.0
diff --git a/sld-api-backend/src/users/application/validator.py b/sld-api-backend/src/users/application/validator.py
@@ -2,7 +2,7 @@
 
 from dependency_injector import containers, providers
 from password_strength import PasswordPolicy
-from usernames import is_safe_username
+from python_usernames import is_safe_username
 
 
 class PasswordValidator:

diff --git a/sld-api-backend/src/worker/providers/hashicorp/actions.py b/sld-api-backend/src/worker/providers/hashicorp/actions.py
@@ -80,7 +80,7 @@ def plan_execute(self) -> dict:
                     "tfvars_files": self.variables_file,
                     "remote_state": f"http://remote-state:8080/terraform_state/{deploy_state}",
                     "project_path": f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}",
-                    "stdout": [result.stderr.split("\n")],
+                    "stdout": result.stderr.split("\n"),
                 }
             return {
                 "command": "plan",
@@ -92,7 +92,7 @@ def plan_execute(self) -> dict:
                 "tfvars_files": self.variables_file,
                 "remote_state": f"http://remote-state:8080/terraform_state/{deploy_state}",
                 "project_path": f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}",
-                "stdout": [result.stdout.split("\n")],
+                "stdout": result.stdout.split("\n"),
             }
         except Exception:
             return {
@@ -105,7 +105,7 @@ def plan_execute(self) -> dict:
                 "tfvars_files": self.variables_file,
                 "remote_state": f"http://remote-state:8080/terraform_state/{deploy_state}",
                 "project_path": f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}",
-                "stdout": [result.stderr.split("\n")],
+                "stdout": result.stderr.split("\n"),
             }
 
     def apply_execute(self) -> dict:
@@ -156,7 +156,7 @@ def apply_execute(self) -> dict:
                     "tfvars_files": self.variables_file,
                     "remote_state": f"http://remote-state:8080/terraform_state/{deploy_state}",
                     "self.project_path": f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}",
-                    "stdout": [result.stderr.split("\n")],
+                    "stdout": result.stderr.split("\n"),
                 }
             return {
                 "command": "apply",
@@ -168,7 +168,7 @@ def apply_execute(self) -> dict:
                 "tfvars_files": self.variables_file,
                 "self.project_path": f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}",
                 "remote_state": f"http://remote-state:8080/terraform_state/{deploy_state}",
-                "stdout": [result.stdout.split("\n")],
+                "stdout": result.stdout.split("\n"),
             }
         except Exception:
             return {
@@ -235,7 +235,7 @@ def destroy_execute(self) -> dict:
                     "tfvars_files": self.variables_file,
                     "remote_state": f"http://remote-state:8080/terraform_state/{deploy_state}",
                     "self.project_path": f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}",
-                    "stdout": [result.stderr.split("\n")],
+                    "stdout": result.stderr.split("\n"),
                 }
             return {
                 "command": "destroy",
@@ -247,7 +247,7 @@ def destroy_execute(self) -> dict:
                 "tfvars_files": self.variables_file,
                 "self.project_path": f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}",
                 "remote_state": f"http://remote-state:8080/terraform_state/{deploy_state}",
-                "stdout": [result.stdout.split("\n")],
+                "stdout": result.stdout.split("\n"),
             }
         except Exception:
             return {
@@ -260,7 +260,6 @@ def destroy_execute(self) -> dict:
                 "tfvars_files": self.variables_file,
                 "self.project_path": f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}",
                 "remote_state": f"http://remote-state:8080/terraform_state/{deploy_state}",
-                # "stdout": [result.stderr.split("\n")],
                 "stdout": "ko",
             }
 

diff --git a/sld-api-backend/src/worker/providers/hashicorp/templates.py b/sld-api-backend/src/worker/providers/hashicorp/templates.py
@@ -41,7 +41,7 @@ def save(self) -> dict:
                 tf_state.write(provider_backend)
             return {"command": "tfserver", "rc": 0, "stdout": data}
         except Exception as err:
-            return {"command": "tfserver", "rc": 1, "stderr": err}
+            return {"command": "tfserver", "rc": 1, "stdout": str(err)}
 
 
 @dataclass