diff --git a/README.md b/README.md new file mode 100644 index 0000000..5020651 --- /dev/null +++ b/README.md @@ -0,0 +1,16 @@ +# Airflow X ArgoCD + +## Prérequis + +- Avoir un cluster kubernetes +- Installer helm + +## Installation de ArgoCD + +Pour installer ArgoCD il suffit de lancer les commandes suivantes : + +``` bash +kubectl create namespace argocd +helm repo add argo https://argoproj.github.io/argo-helm +helm install my-argo-cd argo/argo-cd --version 7.6.8 -n argocd +``` diff --git a/airflow/application.yaml b/airflow/application.yaml new file mode 100644 index 0000000..12b4531 --- /dev/null +++ b/airflow/application.yaml @@ -0,0 +1,61 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: airflow + namespace: argocd +spec: + project: default + source: + repoURL: https://github.com/airflow-helm/charts.git # Airflow Helm chart repository + targetRevision: main + path: charts/airflow # The path where the Helm chart is located in the repo + helm: + values: | + airflow: + executor: "KubernetesExecutor" + + config: + AIRFLOW__CORE__LOAD_EXAMPLES: "False" + AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "True" + + flower: + enabled: false + + redis: + enabled: false + + postgresql: + enabled: false + + workers: + enabled: false + + pgbouncer: + enabled: false + + dags: + persistence: + enabled: false + + web: + defaultUser: + enabled: true + username: "admin" + password: "admin" + + externalDatabase: + type: postgres + host: postgresql-ha-postgresql + port: 5432 + database: postgres + user: postgres + passwordSecret: "postgresql-ha-postgresql" + passwordSecretKey: "password" + + destination: + server: https://kubernetes.default.svc + namespace: airflow + syncPolicy: + automated: + prune: true + selfHeal: true \ No newline at end of file diff --git a/dags/pipeline.py b/dags/pipeline.py new file mode 100644 index 0000000..1e8f940 --- /dev/null +++ b/dags/pipeline.py @@ -0,0 +1,51 @@ +from airflow import DAG +from airflow.operators.python import PythonOperator, BranchPythonOperator +from airflow.operators.bash import BashOperator +from datetime import datetime +from random import randint + +def _choose_best_model(ti): + accuracies = ti.xcom_pull(task_ids=[ + 'training_model_A', + 'training_model_B', + 'training_model_C' + ]) + if max(accuracies) > 8: + return 'is_accurate' + return 'is_inaccurate' + +def _training_model(model): + print(model) + return randint(1, 10) + +with DAG("my_dag", + start_date=datetime(2023, 1 ,1), + schedule_interval='@daily', + catchup=False): + + training_model_tasks = [ + PythonOperator( + task_id=f"training_model_{model_id}", + python_callable=_training_model, + op_kwargs={ + "model": model_id + } + ) for model_id in ['A', 'B', 'C'] + ] + + choose_best_model = BranchPythonOperator( + task_id="choose_best_model", + python_callable=_choose_best_model + ) + + accurate = BashOperator( + task_id="is_accurate", + bash_command="echo 'accurate'" + ) + + inaccurate = BashOperator( + task_id="is_inaccurate", + bash_command=" echo 'inaccurate'" + ) + + training_model_tasks >> choose_best_model >> [is_accurate, is_inaccurate] \ No newline at end of file diff --git a/infrastructure/application.yaml b/infrastructure/application.yaml new file mode 100644 index 0000000..4e00b3b --- /dev/null +++ b/infrastructure/application.yaml @@ -0,0 +1,28 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: postgresql-ha + namespace: argocd # Or the namespace where Argo CD is installed +spec: + project: default + + source: + repoURL: 'https://charts.bitnami.com/bitnami' # Example: Bitnami Helm chart repository + targetRevision: 14.2.34 + chart: postgresql-ha + helm: + values: | + postgresql: + replicaCount: 3 + replication: + enabled: true + readReplicas: 1 + + destination: + server: https://kubernetes.default.svc # The Kubernetes API server address + namespace: airflow # Namespace where PostgreSQL will be deployed + + syncPolicy: + automated: + prune: true + selfHeal: true