Merge pull request #138 from Tauffer-Consulting/chore/update-airflow-…

…pydantic

Chore/update airflow pydantic

Dockerfile-airflow-domino.dev

@@ -1,4 +1,4 @@
-FROM apache/airflow:2.6.3-python3.9
+FROM apache/airflow:2.7.2-python3.9
 
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONUNBUFFERED=1

Dockerfile-airflow-domino.prod

@@ -1,4 +1,4 @@
-FROM apache/airflow:2.6.3-python3.9
+FROM apache/airflow:2.7.2-python3.9
 
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONUNBUFFERED=1

docker-compose-dev.yaml

@@ -4,7 +4,7 @@
 version: '3.8'
 x-airflow-common:
   &airflow-common
-  image: apache/airflow:2.6.3-python3.9
+  image: apache/airflow:2.7.2-python3.9
   # build: .
   environment:
     &airflow-common-env

pyproject.toml

@@ -18,11 +18,11 @@ classifiers = [
 
 ]
 dependencies = [
-    "pydantic==1.10.11",
+    "pydantic==2.4.2",
     "tomli==2.0.1",
     "tomli-w==1.0.0",
     "PyYAML==6.0.1",
-    "jsonschema==4.17.3",
+    "jsonschema==4.18.0",
     "click==8.1.3",
     "rich==12.6.0",
     "colorama==0.4.6",
@@ -71,7 +71,7 @@ cli = [
     "bottle==0.12.25"
 ]
 airflow = [
-    "apache-airflow==2.6.3",
+    "apache-airflow==2.7.2",
     "apache-airflow-providers-cncf-kubernetes==5.0.0",
     "apache-airflow-providers-docker==3.6.0",
 ]
@@ -83,7 +83,7 @@ full = [
     "PyGithub==1.55",
     "docker>=6.0.1",
     "kubernetes==23.6.0",
-    "apache-airflow==2.6.3",
+    "apache-airflow==2.7.2",
     "apache-airflow-providers-cncf-kubernetes==5.0.0",
     "apache-airflow-providers-docker==3.6.0",
 ]

rest/core/settings.py

@@ -1,30 +1,17 @@
-from pydantic import BaseSettings, validators
-from typing import Optional
+from pydantic_settings import BaseSettings
+from typing import Union
 import os
 from database.models.enums import RepositorySource
 
 
 
-def empty_to_none(v: str) -> Optional[str]:
-    if v == '':
-        return None
-    return v
-
-
-class EmptyStrToNone(str):
-    @classmethod
-    def __get_validators__(cls):
-        yield validators.str_validator
-        yield empty_to_none
-
-
 class Settings(BaseSettings):
     # General app config
-    VERSION = "0.1.0"
-    APP_TITLE = "Domino REST api"
+    VERSION: str = "0.1.0"
+    APP_TITLE: str = "Domino REST api"
 
     # Database config
-    DB_URL = 'postgresql://{user}:{password}@{host}:{port}/{name}'.format(
+    DB_URL: str = 'postgresql://{user}:{password}@{host}:{port}/{name}'.format(
         user=os.environ.get("DOMINO_DB_USER", "postgres"),
         password=os.environ.get("DOMINO_DB_PASSWORD", "postgres"),
         host=os.environ.get("DOMINO_DB_HOST", "localhost"),
@@ -33,75 +20,76 @@ class Settings(BaseSettings):
     )
 
     # Auth config
-    AUTH_SECRET_KEY = os.environ.get('AUTH_SECRET_KEY', "SECRET")
-    AUTH_ALGORITHM = os.environ.get('AUTH_ALGORITHM', "HS256")
-    AUTH_ACCESS_TOKEN_EXPIRE_MINUTES = 600
+    AUTH_SECRET_KEY: str = os.environ.get('AUTH_SECRET_KEY', "SECRET")
+    AUTH_ALGORITHM: str = os.environ.get('AUTH_ALGORITHM', "HS256")
+    AUTH_ACCESS_TOKEN_EXPIRE_MINUTES: int = 600
 
     # Secrets config
-    SECRETS_SECRET_KEY = os.environ.get('SECRETS_SECRET_KEY', b'j1DsRJ-ehxU_3PbXW0c_-U4nTOx3knRB4zzWguMVaio=')
-    GITHUB_TOKEN_SECRET_KEY = os.environ.get('GITHUB_TOKEN_SECRET_KEY', b'j1DsRJ-ehxU_3PbXW0c_-U4nTOx3knRB4zzWguMVaio=')
+    SECRETS_SECRET_KEY: str = os.environ.get('SECRETS_SECRET_KEY', b'j1DsRJ-ehxU_3PbXW0c_-U4nTOx3knRB4zzWguMVaio=')
+    GITHUB_TOKEN_SECRET_KEY: str = os.environ.get('GITHUB_TOKEN_SECRET_KEY', b'j1DsRJ-ehxU_3PbXW0c_-U4nTOx3knRB4zzWguMVaio=')
 
     # Used by github rest client
-    DOMINO_GITHUB_ACCESS_TOKEN_WORKFLOWS = os.environ.get('DOMINO_GITHUB_ACCESS_TOKEN_WORKFLOWS', '')
+    DOMINO_GITHUB_ACCESS_TOKEN_WORKFLOWS: str = os.environ.get('DOMINO_GITHUB_ACCESS_TOKEN_WORKFLOWS', '')
 
     # Workflows storage
-    DOMINO_GITHUB_WORKFLOWS_REPOSITORY = os.environ.get('DOMINO_GITHUB_WORKFLOWS_REPOSITORY', "Tauffer-Consulting/domino_workflows_dev")
-    DOMINO_LOCAL_WORKFLOWS_REPOSITORY = '/opt/airflow/dags'
+    DOMINO_GITHUB_WORKFLOWS_REPOSITORY: str = os.environ.get('DOMINO_GITHUB_WORKFLOWS_REPOSITORY', "Tauffer-Consulting/domino_workflows_dev")
+    DOMINO_LOCAL_WORKFLOWS_REPOSITORY: str = '/opt/airflow/dags'
 
     # Default domino pieces repository
-    DOMINO_DEFAULT_PIECES_REPOSITORY = os.environ.get('DOMINO_DEFAULT_PIECES_REPOSITORY', "Tauffer-Consulting/default_domino_pieces")
-    DOMINO_DEFAULT_PIECES_REPOSITORY_VERSION = os.environ.get('DOMINO_DEFAULT_PIECES_REPOSITORY_VERSION', "0.4.3")
-    DOMINO_DEFAULT_PIECES_REPOSITORY_SOURCE = os.environ.get('DOMINO_DEFAULT_PIECES_REPOSITORY_SOURCE', "github")
-    DOMINO_DEFAULT_PIECES_REPOSITORY_TOKEN: EmptyStrToNone = os.environ.get('DOMINO_DEFAULT_PIECES_REPOSITORY_TOKEN', "")
+    DOMINO_DEFAULT_PIECES_REPOSITORY: str = os.environ.get('DOMINO_DEFAULT_PIECES_REPOSITORY', "Tauffer-Consulting/default_domino_pieces")
+    DOMINO_DEFAULT_PIECES_REPOSITORY_VERSION: str = os.environ.get('DOMINO_DEFAULT_PIECES_REPOSITORY_VERSION', "0.4.3")
+    DOMINO_DEFAULT_PIECES_REPOSITORY_SOURCE: str = os.environ.get('DOMINO_DEFAULT_PIECES_REPOSITORY_SOURCE', "github")
+    DOMINO_DEFAULT_PIECES_REPOSITORY_TOKEN: Union[str, None] = os.environ.get('DOMINO_DEFAULT_PIECES_REPOSITORY_TOKEN', '')
+
     DOMINO_DEFAULT_PIECES_REPOSITORY_URL: str = os.environ.get('DOMINO_DEFAULT_PIECES_REPOSITORY_URL', 'https://github.com/Tauffer-Consulting/default_domino_pieces')
 
     # Default DB mock data
-    RUN_CREATE_MOCK_DATA = False
-    ADMIN_CREDENTIALS = {
+    RUN_CREATE_MOCK_DATA: bool = False
+    ADMIN_CREDENTIALS: dict = {
         "email": "admin@email.com",
         "password": "admin"
     }
-    AIRFLOW_ADMIN_CREDENTIALS = {
+    AIRFLOW_ADMIN_CREDENTIALS: dict = {
         "username": os.environ.get('AIRFLOW_ADMIN_USERNAME', "admin"),
         "password": os.environ.get('AIRFLOW_ADMIN_PASSWORD', "admin")
     }
-    AIRFLOW_WEBSERVER_HOST = os.environ.get('AIRFLOW_WEBSERVER_HOST', "http://airflow-webserver:8080/")
+    AIRFLOW_WEBSERVER_HOST: str = os.environ.get('AIRFLOW_WEBSERVER_HOST', "http://airflow-webserver:8080/")
 
     # Default repositories
-    DEFAULT_STORAGE_REPOSITORY = dict(
+    DEFAULT_STORAGE_REPOSITORY: dict = dict(
         name="default_storage_repository",
         path="default_storage_repository",
         source=getattr(RepositorySource, 'default').value,
         version="0.0.1",
         url="domino-default/default_storage_repository"
     )
 
-    DEPLOY_MODE = os.environ.get('DOMINO_DEPLOY_MODE', 'local-k8s')
+    DEPLOY_MODE: str = os.environ.get('DOMINO_DEPLOY_MODE', 'local-k8s')
 
-    CONDITIONAL_ENDPOINTS_ENABLED = False if DEPLOY_MODE == 'local-compose' else True
+    CONDITIONAL_ENDPOINTS_ENABLED: bool = False if DEPLOY_MODE == 'local-compose' else True
 
 class LocalK8sSettings(Settings):
-    SERVER_HOST = "0.0.0.0"
-    DEBUG = True
-    PORT = 8000
-    RELOAD = True
-    CORS = {
+    SERVER_HOST: str = "0.0.0.0"
+    DEBUG: bool = True
+    PORT: int = 8000
+    RELOAD: bool = True
+    CORS: dict = {
         "origins": [
             "*",
         ],
         "allow_credentials": True,
         "allow_methods": ["*"],
         "allow_headers": ["*"],
     }
-    ROOT_PATH = '/api'
+    ROOT_PATH: str = '/api'
 
 
 class LocalComposeSettings(Settings):
-    SERVER_HOST = "0.0.0.0"
-    DEBUG = True
-    PORT = 8000
-    RELOAD = True
-    CORS = {
+    SERVER_HOST: str = "0.0.0.0"
+    DEBUG: bool = True
+    PORT: int = 8000
+    RELOAD: bool = True
+    CORS: dict = {
         "origins": [
             "*",
         ],
@@ -110,15 +98,15 @@ class LocalComposeSettings(Settings):
         "allow_headers": ["*"],
     }
 
-    ROOT_PATH = '/'
+    ROOT_PATH: str = '/'
 
 
 class ProdSettings(Settings):
-    SERVER_HOST = "0.0.0.0"
-    DEBUG = False
-    PORT = 8000
-    RELOAD = False
-    CORS = {
+    SERVER_HOST: str = "0.0.0.0"
+    DEBUG: bool = False
+    PORT: int = 8000
+    RELOAD: bool = False
+    CORS: dict = {
         "origins": [
             "*",
         ],
@@ -128,7 +116,7 @@ class ProdSettings(Settings):
     }
 
     # ROOT_PATH is based in proxy config. Must be the same as the path to the api in the proxy
-    ROOT_PATH = '/api'
+    ROOT_PATH: str = '/api'
 
 
 def get_settings():

rest/main.py

@@ -2,7 +2,6 @@
 from dotenv import find_dotenv, load_dotenv
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
-
 from routers.auth_router import router as auth_router
 from routers.user_router import router as user_router
 from routers.workflow_router import router as workflow_router

rest/requirements-test.txt

@@ -1,4 +1,4 @@
-fastapi==0.88.0
+fastapi==0.104.1
 pytest==7.2.0
 httpx==0.23.1
 requests==2.28.1

rest/requirements.txt

@@ -1,10 +1,10 @@
 alembic==1.12.0
 bcrypt==3.2.2
-fastapi==0.103.0
-fastapi-utils==0.2.1
+fastapi==0.104.1
 Jinja2==2.11.3
 psycopg2-binary==2.9.3
-pydantic[email]<2.0.0
+pydantic[email]==2.4.2
+pydantic-settings==2.0.3
 MarkupSafe==2.0.1
 PyGithub==1.55
 PyJWT==2.4.0
@@ -16,7 +16,6 @@ sqlalchemy-redshift==0.8.9
 SQLAlchemy-Utils==0.38.2
 uvicorn==0.18.1
 uvicorn[standard]==0.18.1
-python-dotenv==0.20.0
 passlib==1.7.4
 tomli==2.0.1
 tomli_w==1.0.0

rest/routers/piece_repository_router.py

@@ -14,7 +14,7 @@
 from database.models.enums import RepositorySource
 from schemas.exceptions.base import BaseException, ConflictException, ForbiddenException, ResourceNotFoundException, UnauthorizedException
 from schemas.errors.base import ConflictError, ForbiddenError, ResourceNotFoundError, SomethingWrongError, UnauthorizedError
-from typing import List
+from typing import List, Optional
 
 router = APIRouter(prefix="/pieces-repositories")
 auth_service = AuthService()
@@ -124,8 +124,8 @@ def get_piece_repository_release_data(
 )
 def get_pieces_repositories(
     workspace_id: int,
-    page: int = 0,
-    page_size: int = 100,
+    page: Optional[int] = 0,
+    page_size: Optional[int] = 100,
     filters: ListRepositoryFilters = Depends(),
 ) -> GetWorkspaceRepositoriesResponse:
     """Get pieces repositories for workspace"""

rest/schemas/context/auth_context.py

@@ -4,7 +4,7 @@
 class WorkspaceAuthorizerData(BaseModel):
     id: int
     name: str
-    github_access_token: Optional[str]
+    github_access_token: Optional[str] = None
     user_permission: str
 
 

rest/schemas/errors/base.py

@@ -2,30 +2,30 @@
 
 
 class BadRequestError(BaseModel):
-    status_code = 400
+    status_code: int = 400
     message: str = Field(default="Bad request data")
 
 
 class SomethingWrongError(BaseModel):
-    status_code = 500
+    status_code: int = 500
     message: str = Field(default="Something went wrong")
 
 
 class UnauthorizedError(BaseModel):
-    status_code = 401
+    status_code: int = 401
     message: str = Field(default="Unauthorized")
 
 
 class ForbiddenError(BaseModel):
-    status_code = 403
+    status_code: int = 403
     message: str = Field(default="Forbidden")
 
 
 class ConflictError(BaseModel):
-    status_code = 409
+    status_code: int = 409
     message: str = Field(default="Conflict")
 
 
 class ResourceNotFoundError(BaseModel):
-    status_code = 404
+    status_code: int = 404
     message: str = Field(default="Resource not found") 

rest/schemas/requests/piece.py

@@ -2,4 +2,4 @@
 from typing import Optional
 
 class ListPiecesFilters(BaseModel):
-    name__like: Optional[str]
+    name__like: Optional[str] = None

rest/schemas/requests/piece_repository.py

@@ -10,19 +10,19 @@ class Config:
         use_enum_values = True
 
 class ListRepositoryFilters(BaseModel):
-    name__like: Optional[str]
-    path__like: Optional[str]
-    version: Optional[str]
-    url: Optional[str]
-    workspace_id: Optional[int]
+    name__like: Optional[str] = None
+    path__like: Optional[str] = None
+    version: Optional[str] = None
+    url: Optional[str] = None
+    workspace_id: Optional[int] = None
     source: Optional[RepositorySource] = Field(description="Source of the repository.", default=RepositorySource.github.value)
 
 class CreateRepositoryRequest(BaseModel):
     workspace_id: int = Field(description='Workspace id to create repository')
     source: RepositorySourceRequestEnum = Field(description="Source of the repository", default=RepositorySource.github.value)
     path: str = Field(..., description="Path to the repository.")
-    version: str = Field(regex=r'((^\d+\.\d+\.\d+$))', description="Version of the repository.")
+    version: str = Field(pattern=r'((^\d+\.\d+\.\d+$))', description="Version of the repository.")
     url: str = Field(..., description="Url of the repository.")
 
 class PatchRepositoryRequest(BaseModel):
-    version: str = Field(regex=r'((^\d+\.\d+\.\d+$))', description="Version of the repository.")
+    version: str = Field(pattern=r'((^\d+\.\d+\.\d+$))', description="Version of the repository.")