Merge pull request #210 from Tauffer-Consulting/k8s-configs

Add the necessary settings to work with istio

CHANGELOG.md

@@ -1,3 +1,29 @@
+# v0.8.4
+
+### Features
+- [x] Add basic config in helm chart and CLI for Istio.
+- [x] Add worker route to get pieces repository.
+
+
+### Fixes
+- [x] Add classic authorization to default route for get pieces repositories.
+
+
+# v0.8.3
+
+### Features
+- [x] Allow download results [Issue #174](https://github.com/Tauffer-Consulting/domino/issues/174)
+- [x] Highlight edges on running pieces [Issue #166](https://github.com/Tauffer-Consulting/domino/issues/166)
+- [x] Generalize options of languages for codeeditor [Issue #170](https://github.com/Tauffer-Consulting/domino/issues/170)
+- [x] New formats for display results [Issue #25](https://github.com/Tauffer-Consulting/domino/issues/25)
+- [x] New page for display results and export as PDF[Issue #208](https://github.com/Tauffer-Consulting/domino/issues/208)
+
+### Fixes
+- [x] Add `API_URL` to frontend env entrypoint
+- [x] Update helm with new `API_URL` var
+- [x] Update base compose fies with `API_URL`
+- [x] Remove `API_ENV` from frontend entrypoint
+
 # v0.8.2
 
 ### Features
@@ -47,4 +73,4 @@
 
 ### Fixes
 * Fixes for the migration to Pydantic 2. [PR #152](https://github.com/Tauffer-Consulting/domino/pull/152)
-* Remove old docs files. [PR #156](https://github.com/Tauffer-Consulting/domino/pull/156)
+* Remove old docs files. [PR #156](https://github.com/Tauffer-Consulting/domino/pull/156)

helm/domino/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: domino
 description: A Helm chart for Domino
 type: application
-version: 0.1.9
-appVersion: 0.1.9
+version: 0.1.10
+appVersion: 0.1.10
 home: https://github.com/Tauffer-Consulting/domino
 sources:
   - https://github.com/Tauffer-Consulting/domino

helm/domino/templates/jobs/domino-migrations.yml

@@ -8,6 +8,9 @@ spec:
   parallelism: 1
   backoffLimit: 4
   template:
+    metadata:
+      annotations:
+        sidecar.istio.io/inject: "false"
     spec:
       restartPolicy: OnFailure
       containers:

rest/routers/piece_repository_router.py

@@ -120,8 +120,7 @@ def get_piece_repository_release_data(
         status.HTTP_500_INTERNAL_SERVER_ERROR: {'model': SomethingWrongError},
         status.HTTP_403_FORBIDDEN: {'model': ForbiddenError},
     },
-    # TODO - I commented this to make it easier to test, but we should solve the auth service
-    # dependencies=[Depends(auth_service.workspace_access_authorizer)]
+    dependencies=[Depends(auth_service.workspace_access_authorizer)]
 )
 def get_pieces_repositories(
     workspace_id: int,
@@ -142,6 +141,39 @@ def get_pieces_repositories(
         raise HTTPException(status_code=e.status_code, detail=e.message)
 
 
+@router.get(
+    path="/worker",
+    status_code=status.HTTP_200_OK,
+    responses={
+        status.HTTP_200_OK: {'model': GetWorkspaceRepositoriesResponse},
+        status.HTTP_500_INTERNAL_SERVER_ERROR: {'model': SomethingWrongError},
+        status.HTTP_403_FORBIDDEN: {'model': ForbiddenError},
+    },
+)
+def get_pieces_repositories_worker(
+    workspace_id: int,
+    page: Optional[int] = 0,
+    page_size: Optional[int] = 100,
+    filters: ListRepositoryFilters = Depends(),
+) -> GetWorkspaceRepositoriesResponse:
+    """
+    Get pieces repositories for workspace.
+    This endpoint is used by the worker to get the repositories to be processed.
+    Is the same endpoint as the one above, but without the auth service.
+    The authorization is done by our service mesh Authorization Policy.
+    """
+    try:
+        response = piece_repository_service.get_pieces_repositories(
+            workspace_id=workspace_id,
+            page=page,
+            page_size=page_size,
+            filters=filters
+        )
+        return response
+    except (BaseException, ForbiddenException) as e:
+        raise HTTPException(status_code=e.status_code, detail=e.message)
+
+
 @router.delete(
     path="/{piece_repository_id}",
     status_code=status.HTTP_204_NO_CONTENT,

rest/routers/secret_router.py

@@ -70,7 +70,7 @@ def update_repository_secret(
         raise HTTPException(status_code=e.status_code, detail=e.message)
 
 @router.get(
-    path='/{piece_name}',
+    path='/{piece_name}/secrets-values', # using sufix /secrets-values only because istio does not support wildcards in paths
     status_code=200,
     responses={
         status.HTTP_200_OK: {'model': List[GetSecretsByPieceResponse]},
@@ -79,13 +79,15 @@ def update_repository_secret(
     },
     include_in_schema=False
 )
-#@auth_service.authorize_repository_workspace_access # TODO authorize only worker
 def get_piece_secrets(
     piece_repository_id: int,
-    piece_name: str, # TODO check what is better to use. query or path ?
-    #auth_context: AuthorizationContextData = Depends(auth_service.auth_wrapper)
+    piece_name: str,
 ) -> List[GetSecretsByPieceResponse]:
-    """Get secrets for a specific Piece from an piece repository, in a specific workspace"""
+    """
+    Get secrets values for a specific Piece from an piece repository, in a specific workspace
+    This endpoint is not using authorization service because it is used by airflow to get secrets values
+    In production this endpoint should be blocked from external access using security strategies like authorization policies.
+    """
     try:
         response = secret_service.get_piece_secrets(
             piece_repository_id=piece_repository_id,

src/domino/VERSION

@@ -1 +1 @@
-0.8.3
+0.8.4

src/domino/cli/utils/platform.py

@@ -397,6 +397,22 @@ def create_platform(install_airflow: bool = True, use_gpu: bool = False) -> None
                 "sshKeySecret": "airflow-ssh-secret"
             },
         },
+        "migrateDatabaseJob": {
+            "jobAnnotations": {
+                "sidecar.istio.io/inject": "false"
+            },
+            "annotations": {
+                "sidecar.istio.io/inject": "false"
+            },
+        },
+        "createUserJob": {
+            "jobAnnotations": {
+                "sidecar.istio.io/inject": "false"
+            },
+            "annotations": {
+                "sidecar.istio.io/inject": "false"
+            },
+        },
         **workers,
         **scheduler,
     }
@@ -417,7 +433,7 @@ def create_platform(install_airflow: bool = True, use_gpu: bool = False) -> None
                 "-f", str(fp.name),
                 "airflow",
                 "apache-airflow/airflow",
-                "--version", " 1.9.0",
+                "--version", " 1.11.0",
             ]
             subprocess.run(commands)
 

src/domino/client/domino_backend_client.py

@@ -31,30 +31,22 @@ def health_check(self) -> requests.Response:
         return response
 
     def get_piece_secrets(self, piece_repository_id: int, piece_name: str) -> requests.Response:
-        resource = f"/pieces-repositories/{piece_repository_id}/secrets/{piece_name}"
-        response = self.request(
-            method='get',
-            resource=resource
-        )
-        return response
-
-    def get_piece_repository(self, piece_repository_id: int) -> requests.Response:
-        resource = f"/pieces-repositories/{piece_repository_id}"
+        resource = f"/pieces-repositories/{piece_repository_id}/secrets/{piece_name}/secrets-values"
         response = self.request(
             method='get',
             resource=resource
         )
         return response
 
     def get_piece_repositories_from_workspace_id(self, params: dict) -> requests.Response:
-        resource = "/pieces-repositories"
+        resource = "/pieces-repositories/worker"
         response = self.request(
             method='get',
             resource=resource,
             params=params
         )
         return response
-    
+
     def check_create_airflow_connection(self, conn_id: str, conn_type: str):
         """
         This should check if a specific Airflow connection exists and create it if it doesn't.

src/domino/task.py

@@ -103,6 +103,7 @@ def _set_operator(self) -> BaseOperator:
             # - https://www.astronomer.io/guides/templating/
             # - good example: https://github.com/apache/airflow/blob/main/tests/system/providers/cncf/kubernetes/example_kubernetes.py
             # - commands HAVE to go in a list object: https://stackoverflow.com/a/55149915/11483674
+
             return DominoKubernetesPodOperator(
                 dag_id=self.dag_id,
                 task_id=self.task_id,
@@ -115,11 +116,12 @@ def _set_operator(self) -> BaseOperator:
                 workflow_shared_storage=self.workflow_shared_storage,
                 container_resources=self.container_resources,
                 # ----------------- Kubernetes -----------------
-                namespace='default',  # TODO - separate namespace by User or Workspace?
+                namespace='default',
                 image=self.piece.get("source_image"),
                 image_pull_policy='IfNotPresent',
                 name=f"airflow-worker-pod-{self.task_id}",
                 startup_timeout_seconds=600,
+                annotations={"sidecar.istio.io/inject": "false"}, # TODO - remove this when istio is working with airflow k8s pod
                 # cmds=["/bin/bash"],
                 # arguments=["-c", "sleep 120;"],
                 cmds=["domino"],