Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dev #139

Merged
merged 33 commits into from
Nov 3, 2023
Merged

Dev #139

Changes from 7 commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
a1afd3b
fix complex array upstream options
vinicvaz Oct 31, 2023
11c9163
start updating airflow and pydantic dependencies
vinicvaz Nov 1, 2023
824cafb
migrating rest to pydantic 2
vinicvaz Nov 1, 2023
b6b882f
update tests dependencies
vinicvaz Nov 1, 2023
eace869
fix token
vinicvaz Nov 1, 2023
e279bd2
run tst
vinicvaz Nov 1, 2023
c640c27
workflow
vinicvaz Nov 1, 2023
0bf99f4
Merge pull request #138 from Tauffer-Consulting/chore/update-airflow-…
vinicvaz Nov 1, 2023
9d35edc
Merge branch 'dev' of https://github.com/Tauffer-Consulting/domino in…
vinicvaz Nov 1, 2023
2190fe4
Merge pull request #137 from Tauffer-Consulting/fix/upstream-options-…
vinicvaz Nov 1, 2023
6957e58
remove piece id dependency from create workflow tasks
vinicvaz Nov 1, 2023
4df7b60
remove piece id from data models
vinicvaz Nov 1, 2023
6b3e4f1
Merge pull request #141 from Tauffer-Consulting/fix/importable-create…
vinicvaz Nov 1, 2023
d74aa5a
add menu item to import button
vinicvaz Oct 31, 2023
c8cdc2c
files and modal
vinicvaz Oct 31, 2023
8d5fcb6
ref: use repositories as global context
nathan-vm Oct 31, 2023
1b301fb
feat: create import from examples
nathan-vm Nov 1, 2023
b460851
chore: cards styles
nathan-vm Nov 1, 2023
2bc3e5a
add source_image to frontend payload on create workflow
vinicvaz Nov 2, 2023
099e396
using source_image in create workflow and update data models pydantic 2
vinicvaz Nov 2, 2023
5911557
update tests
vinicvaz Nov 2, 2023
3e51dab
Merge pull request #144 from Tauffer-Consulting/fix/workflows
vinicvaz Nov 2, 2023
e00e144
Merge branch 'dev' of https://github.com/Tauffer-Consulting/domino in…
vinicvaz Nov 2, 2023
67fa028
fix: impossible fetch pieces
nathan-vm Nov 2, 2023
40259b2
fix: impossible fetch pieces
nathan-vm Nov 2, 2023
f101cac
Merge branch 'feat/workflows-gallery' of https://github.com/Tauffer-C…
vinicvaz Nov 2, 2023
f2e2504
chore: use 3 examples per line in examples gallery
nathan-vm Nov 2, 2023
30ca970
Merge branch 'feat/workflows-gallery' of https://github.com/Tauffer-C…
vinicvaz Nov 3, 2023
96c001d
Merge pull request #142 from Tauffer-Consulting/feat/workflows-gallery
vinicvaz Nov 3, 2023
a2409e2
fix: frontend build
nathan-vm Nov 3, 2023
0b51991
update pydantic extra args in core
vinicvaz Nov 3, 2023
77d54b4
Merge branch 'dev' of https://github.com/Tauffer-Consulting/domino in…
vinicvaz Nov 3, 2023
8b03889
update version
vinicvaz Nov 3, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Dockerfile-airflow-domino.dev
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM apache/airflow:2.6.3-python3.9
FROM apache/airflow:2.7.2-python3.9

ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONUNBUFFERED=1
2 changes: 1 addition & 1 deletion Dockerfile-airflow-domino.prod
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM apache/airflow:2.6.3-python3.9
FROM apache/airflow:2.7.2-python3.9

ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONUNBUFFERED=1
2 changes: 1 addition & 1 deletion docker-compose-dev.yaml
Original file line number Diff line number Diff line change
@@ -4,7 +4,7 @@
version: '3.8'
x-airflow-common:
&airflow-common
image: apache/airflow:2.6.3-python3.9
image: apache/airflow:2.7.2-python3.9
# build: .
environment:
&airflow-common-env
8 changes: 4 additions & 4 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -18,11 +18,11 @@ classifiers = [

]
dependencies = [
"pydantic==1.10.11",
"pydantic==2.4.2",
"tomli==2.0.1",
"tomli-w==1.0.0",
"PyYAML==6.0.1",
"jsonschema==4.17.3",
"jsonschema==4.18.0",
"click==8.1.3",
"rich==12.6.0",
"colorama==0.4.6",
@@ -71,7 +71,7 @@ cli = [
"bottle==0.12.25"
]
airflow = [
"apache-airflow==2.6.3",
"apache-airflow==2.7.2",
"apache-airflow-providers-cncf-kubernetes==5.0.0",
"apache-airflow-providers-docker==3.6.0",
]
@@ -83,7 +83,7 @@ full = [
"PyGithub==1.55",
"docker>=6.0.1",
"kubernetes==23.6.0",
"apache-airflow==2.6.3",
"apache-airflow==2.7.2",
"apache-airflow-providers-cncf-kubernetes==5.0.0",
"apache-airflow-providers-docker==3.6.0",
]
98 changes: 43 additions & 55 deletions rest/core/settings.py
Original file line number Diff line number Diff line change
@@ -1,30 +1,17 @@
from pydantic import BaseSettings, validators
from typing import Optional
from pydantic_settings import BaseSettings
from typing import Union
import os
from database.models.enums import RepositorySource



def empty_to_none(v: str) -> Optional[str]:
if v == '':
return None
return v


class EmptyStrToNone(str):
@classmethod
def __get_validators__(cls):
yield validators.str_validator
yield empty_to_none


class Settings(BaseSettings):
# General app config
VERSION = "0.1.0"
APP_TITLE = "Domino REST api"
VERSION: str = "0.1.0"
APP_TITLE: str = "Domino REST api"

# Database config
DB_URL = 'postgresql://{user}:{password}@{host}:{port}/{name}'.format(
DB_URL: str = 'postgresql://{user}:{password}@{host}:{port}/{name}'.format(
user=os.environ.get("DOMINO_DB_USER", "postgres"),
password=os.environ.get("DOMINO_DB_PASSWORD", "postgres"),
host=os.environ.get("DOMINO_DB_HOST", "localhost"),
@@ -33,75 +20,76 @@ class Settings(BaseSettings):
)

# Auth config
AUTH_SECRET_KEY = os.environ.get('AUTH_SECRET_KEY', "SECRET")
AUTH_ALGORITHM = os.environ.get('AUTH_ALGORITHM', "HS256")
AUTH_ACCESS_TOKEN_EXPIRE_MINUTES = 600
AUTH_SECRET_KEY: str = os.environ.get('AUTH_SECRET_KEY', "SECRET")
AUTH_ALGORITHM: str = os.environ.get('AUTH_ALGORITHM', "HS256")
AUTH_ACCESS_TOKEN_EXPIRE_MINUTES: int = 600

# Secrets config
SECRETS_SECRET_KEY = os.environ.get('SECRETS_SECRET_KEY', b'j1DsRJ-ehxU_3PbXW0c_-U4nTOx3knRB4zzWguMVaio=')
GITHUB_TOKEN_SECRET_KEY = os.environ.get('GITHUB_TOKEN_SECRET_KEY', b'j1DsRJ-ehxU_3PbXW0c_-U4nTOx3knRB4zzWguMVaio=')
SECRETS_SECRET_KEY: str = os.environ.get('SECRETS_SECRET_KEY', b'j1DsRJ-ehxU_3PbXW0c_-U4nTOx3knRB4zzWguMVaio=')
GITHUB_TOKEN_SECRET_KEY: str = os.environ.get('GITHUB_TOKEN_SECRET_KEY', b'j1DsRJ-ehxU_3PbXW0c_-U4nTOx3knRB4zzWguMVaio=')

# Used by github rest client
DOMINO_GITHUB_ACCESS_TOKEN_WORKFLOWS = os.environ.get('DOMINO_GITHUB_ACCESS_TOKEN_WORKFLOWS', '')
DOMINO_GITHUB_ACCESS_TOKEN_WORKFLOWS: str = os.environ.get('DOMINO_GITHUB_ACCESS_TOKEN_WORKFLOWS', '')

# Workflows storage
DOMINO_GITHUB_WORKFLOWS_REPOSITORY = os.environ.get('DOMINO_GITHUB_WORKFLOWS_REPOSITORY', "Tauffer-Consulting/domino_workflows_dev")
DOMINO_LOCAL_WORKFLOWS_REPOSITORY = '/opt/airflow/dags'
DOMINO_GITHUB_WORKFLOWS_REPOSITORY: str = os.environ.get('DOMINO_GITHUB_WORKFLOWS_REPOSITORY', "Tauffer-Consulting/domino_workflows_dev")
DOMINO_LOCAL_WORKFLOWS_REPOSITORY: str = '/opt/airflow/dags'

# Default domino pieces repository
DOMINO_DEFAULT_PIECES_REPOSITORY = os.environ.get('DOMINO_DEFAULT_PIECES_REPOSITORY', "Tauffer-Consulting/default_domino_pieces")
DOMINO_DEFAULT_PIECES_REPOSITORY_VERSION = os.environ.get('DOMINO_DEFAULT_PIECES_REPOSITORY_VERSION', "0.4.3")
DOMINO_DEFAULT_PIECES_REPOSITORY_SOURCE = os.environ.get('DOMINO_DEFAULT_PIECES_REPOSITORY_SOURCE', "github")
DOMINO_DEFAULT_PIECES_REPOSITORY_TOKEN: EmptyStrToNone = os.environ.get('DOMINO_DEFAULT_PIECES_REPOSITORY_TOKEN', "")
DOMINO_DEFAULT_PIECES_REPOSITORY: str = os.environ.get('DOMINO_DEFAULT_PIECES_REPOSITORY', "Tauffer-Consulting/default_domino_pieces")
DOMINO_DEFAULT_PIECES_REPOSITORY_VERSION: str = os.environ.get('DOMINO_DEFAULT_PIECES_REPOSITORY_VERSION', "0.4.3")
DOMINO_DEFAULT_PIECES_REPOSITORY_SOURCE: str = os.environ.get('DOMINO_DEFAULT_PIECES_REPOSITORY_SOURCE', "github")
DOMINO_DEFAULT_PIECES_REPOSITORY_TOKEN: Union[str, None] = os.environ.get('DOMINO_DEFAULT_PIECES_REPOSITORY_TOKEN', '')

DOMINO_DEFAULT_PIECES_REPOSITORY_URL: str = os.environ.get('DOMINO_DEFAULT_PIECES_REPOSITORY_URL', 'https://github.com/Tauffer-Consulting/default_domino_pieces')

# Default DB mock data
RUN_CREATE_MOCK_DATA = False
ADMIN_CREDENTIALS = {
RUN_CREATE_MOCK_DATA: bool = False
ADMIN_CREDENTIALS: dict = {
"email": "admin@email.com",
"password": "admin"
}
AIRFLOW_ADMIN_CREDENTIALS = {
AIRFLOW_ADMIN_CREDENTIALS: dict = {
"username": os.environ.get('AIRFLOW_ADMIN_USERNAME', "admin"),
"password": os.environ.get('AIRFLOW_ADMIN_PASSWORD', "admin")
}
AIRFLOW_WEBSERVER_HOST = os.environ.get('AIRFLOW_WEBSERVER_HOST', "http://airflow-webserver:8080/")
AIRFLOW_WEBSERVER_HOST: str = os.environ.get('AIRFLOW_WEBSERVER_HOST', "http://airflow-webserver:8080/")

# Default repositories
DEFAULT_STORAGE_REPOSITORY = dict(
DEFAULT_STORAGE_REPOSITORY: dict = dict(
name="default_storage_repository",
path="default_storage_repository",
source=getattr(RepositorySource, 'default').value,
version="0.0.1",
url="domino-default/default_storage_repository"
)

DEPLOY_MODE = os.environ.get('DOMINO_DEPLOY_MODE', 'local-k8s')
DEPLOY_MODE: str = os.environ.get('DOMINO_DEPLOY_MODE', 'local-k8s')

CONDITIONAL_ENDPOINTS_ENABLED = False if DEPLOY_MODE == 'local-compose' else True
CONDITIONAL_ENDPOINTS_ENABLED: bool = False if DEPLOY_MODE == 'local-compose' else True

class LocalK8sSettings(Settings):
SERVER_HOST = "0.0.0.0"
DEBUG = True
PORT = 8000
RELOAD = True
CORS = {
SERVER_HOST: str = "0.0.0.0"
DEBUG: bool = True
PORT: int = 8000
RELOAD: bool = True
CORS: dict = {
"origins": [
"*",
],
"allow_credentials": True,
"allow_methods": ["*"],
"allow_headers": ["*"],
}
ROOT_PATH = '/api'
ROOT_PATH: str = '/api'


class LocalComposeSettings(Settings):
SERVER_HOST = "0.0.0.0"
DEBUG = True
PORT = 8000
RELOAD = True
CORS = {
SERVER_HOST: str = "0.0.0.0"
DEBUG: bool = True
PORT: int = 8000
RELOAD: bool = True
CORS: dict = {
"origins": [
"*",
],
@@ -110,15 +98,15 @@ class LocalComposeSettings(Settings):
"allow_headers": ["*"],
}

ROOT_PATH = '/'
ROOT_PATH: str = '/'


class ProdSettings(Settings):
SERVER_HOST = "0.0.0.0"
DEBUG = False
PORT = 8000
RELOAD = False
CORS = {
SERVER_HOST: str = "0.0.0.0"
DEBUG: bool = False
PORT: int = 8000
RELOAD: bool = False
CORS: dict = {
"origins": [
"*",
],
@@ -128,7 +116,7 @@ class ProdSettings(Settings):
}

# ROOT_PATH is based in proxy config. Must be the same as the path to the api in the proxy
ROOT_PATH = '/api'
ROOT_PATH: str = '/api'


def get_settings():
1 change: 0 additions & 1 deletion rest/main.py
Original file line number Diff line number Diff line change
@@ -2,7 +2,6 @@
from dotenv import find_dotenv, load_dotenv
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware

from routers.auth_router import router as auth_router
from routers.user_router import router as user_router
from routers.workflow_router import router as workflow_router
2 changes: 1 addition & 1 deletion rest/requirements-test.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
fastapi==0.88.0
fastapi==0.104.1
pytest==7.2.0
httpx==0.23.1
requests==2.28.1
7 changes: 3 additions & 4 deletions rest/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
alembic==1.12.0
bcrypt==3.2.2
fastapi==0.103.0
fastapi-utils==0.2.1
fastapi==0.104.1
Jinja2==2.11.3
psycopg2-binary==2.9.3
pydantic[email]<2.0.0
pydantic[email]==2.4.2
pydantic-settings==2.0.3
MarkupSafe==2.0.1
PyGithub==1.55
PyJWT==2.4.0
@@ -16,7 +16,6 @@ sqlalchemy-redshift==0.8.9
SQLAlchemy-Utils==0.38.2
uvicorn==0.18.1
uvicorn[standard]==0.18.1
python-dotenv==0.20.0
passlib==1.7.4
tomli==2.0.1
tomli_w==1.0.0
6 changes: 3 additions & 3 deletions rest/routers/piece_repository_router.py
Original file line number Diff line number Diff line change
@@ -14,7 +14,7 @@
from database.models.enums import RepositorySource
from schemas.exceptions.base import BaseException, ConflictException, ForbiddenException, ResourceNotFoundException, UnauthorizedException
from schemas.errors.base import ConflictError, ForbiddenError, ResourceNotFoundError, SomethingWrongError, UnauthorizedError
from typing import List
from typing import List, Optional

router = APIRouter(prefix="/pieces-repositories")
auth_service = AuthService()
@@ -124,8 +124,8 @@ def get_piece_repository_release_data(
)
def get_pieces_repositories(
workspace_id: int,
page: int = 0,
page_size: int = 100,
page: Optional[int] = 0,
page_size: Optional[int] = 100,
filters: ListRepositoryFilters = Depends(),
) -> GetWorkspaceRepositoriesResponse:
"""Get pieces repositories for workspace"""
2 changes: 1 addition & 1 deletion rest/schemas/context/auth_context.py
Original file line number Diff line number Diff line change
@@ -4,7 +4,7 @@
class WorkspaceAuthorizerData(BaseModel):
id: int
name: str
github_access_token: Optional[str]
github_access_token: Optional[str] = None
user_permission: str


12 changes: 6 additions & 6 deletions rest/schemas/errors/base.py
Original file line number Diff line number Diff line change
@@ -2,30 +2,30 @@


class BadRequestError(BaseModel):
status_code = 400
status_code: int = 400
message: str = Field(default="Bad request data")


class SomethingWrongError(BaseModel):
status_code = 500
status_code: int = 500
message: str = Field(default="Something went wrong")


class UnauthorizedError(BaseModel):
status_code = 401
status_code: int = 401
message: str = Field(default="Unauthorized")


class ForbiddenError(BaseModel):
status_code = 403
status_code: int = 403
message: str = Field(default="Forbidden")


class ConflictError(BaseModel):
status_code = 409
status_code: int = 409
message: str = Field(default="Conflict")


class ResourceNotFoundError(BaseModel):
status_code = 404
status_code: int = 404
message: str = Field(default="Resource not found")
2 changes: 1 addition & 1 deletion rest/schemas/requests/piece.py
Original file line number Diff line number Diff line change
@@ -2,4 +2,4 @@
from typing import Optional

class ListPiecesFilters(BaseModel):
name__like: Optional[str]
name__like: Optional[str] = None
14 changes: 7 additions & 7 deletions rest/schemas/requests/piece_repository.py
Original file line number Diff line number Diff line change
@@ -10,19 +10,19 @@ class Config:
use_enum_values = True

class ListRepositoryFilters(BaseModel):
name__like: Optional[str]
path__like: Optional[str]
version: Optional[str]
url: Optional[str]
workspace_id: Optional[int]
name__like: Optional[str] = None
path__like: Optional[str] = None
version: Optional[str] = None
url: Optional[str] = None
workspace_id: Optional[int] = None
source: Optional[RepositorySource] = Field(description="Source of the repository.", default=RepositorySource.github.value)

class CreateRepositoryRequest(BaseModel):
workspace_id: int = Field(description='Workspace id to create repository')
source: RepositorySourceRequestEnum = Field(description="Source of the repository", default=RepositorySource.github.value)
path: str = Field(..., description="Path to the repository.")
version: str = Field(regex=r'((^\d+\.\d+\.\d+$))', description="Version of the repository.")
version: str = Field(pattern=r'((^\d+\.\d+\.\d+$))', description="Version of the repository.")
url: str = Field(..., description="Url of the repository.")

class PatchRepositoryRequest(BaseModel):
version: str = Field(regex=r'((^\d+\.\d+\.\d+$))', description="Version of the repository.")
version: str = Field(pattern=r'((^\d+\.\d+\.\d+$))', description="Version of the repository.")
Loading
Loading