From 33bfbc48da4a1c72762b211613a3b26bfff3f8f9 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Wed, 4 Oct 2023 11:24:40 +0200 Subject: [PATCH] Removed AirflowDB references in docs and RBAC config (#331) --- deploy/helm/airflow-operator/templates/roles.yaml | 9 --------- docs/modules/airflow/pages/index.adoc | 14 +++----------- 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/deploy/helm/airflow-operator/templates/roles.yaml b/deploy/helm/airflow-operator/templates/roles.yaml index 72ca5fbb..f8844bd8 100644 --- a/deploy/helm/airflow-operator/templates/roles.yaml +++ b/deploy/helm/airflow-operator/templates/roles.yaml @@ -73,8 +73,6 @@ rules: - {{ include "operator.name" . }}.stackable.tech resources: - {{ include "operator.name" . }}clusters - - airflowdbs - - airflowdbs/status verbs: - get - list @@ -86,13 +84,6 @@ rules: - {{ include "operator.name" . }}clusters/status verbs: - patch - # The operator creates the airflowdb resource for the cluster automatically - - apiGroups: - - {{ include "operator.name" . }}.stackable.tech - resources: - - airflowdbs - verbs: - - create - apiGroups: - authentication.stackable.tech resources: diff --git a/docs/modules/airflow/pages/index.adoc b/docs/modules/airflow/pages/index.adoc index 7f9f1e6b..28837fa6 100644 --- a/docs/modules/airflow/pages/index.adoc +++ b/docs/modules/airflow/pages/index.adoc @@ -15,16 +15,9 @@ Get started using Airflow with the Stackable Operator by following the xref:gett guides you through installing the Operator alongside a PostgreSQL database and Redis instance, connecting to your Airflow instance and running your first workflow. -== Resources - -The Operator manages three {k8s-crs}[custom resources]: The _AirflowCluster_ and _AirflowDB_. It creates a number of -different Kubernetes resources based on the custom resources. - === Custom resources -The AirflowCluster is the main resource for the configuration of the Airflow instance. The resource defines three xref:concepts:roles-and-role-groups.adoc[roles]: `webserver`, `worker` and `scheduler` (the `worker` role is embedded within `spec.celeryExecutors`: this is described in the next section). The various configuration options are explained in the xref:usage-guide/index.adoc[]. It helps you tune your cluster to your needs by configuring xref:usage-guide/storage-resources.adoc[resource usage], xref:usage-guide/security.adoc[security], xref:usage-guide/logging.adoc[logging] and more. - -When an AirflowCluster is first deployed, an AirflowDB resource is created. The AirflowDB resource is a wrapper resource for the metadata SQL database that is used by Airflow to store information on users and permissions as well as workflows, task instances and their execution. The resource contains some configuration but also keeps track of whether the database has been initialized or not. It is not deleted automatically if a AirflowCluster is deleted, and so can be reused. +The AirflowCluster is the resource for the configuration of the Airflow instance. The resource defines three xref:concepts:roles-and-role-groups.adoc[roles]: `webserver`, `worker` and `scheduler` (the `worker` role is embedded within `spec.celeryExecutors`: this is described in the next section). The various configuration options are explained in the xref:usage-guide/index.adoc[]. It helps you tune your cluster to your needs by configuring xref:usage-guide/storage-resources.adoc[resource usage], xref:usage-guide/security.adoc[security], xref:usage-guide/logging.adoc[logging] and more. === Executors @@ -69,8 +62,7 @@ Based on the custom resources you define, the Operator creates ConfigMaps, State image::airflow_overview.drawio.svg[A diagram depicting the Kubernetes resources created by the operator] -The diagram above depicts all the Kubernetes resources created by the operator, and how they relate to each other. The -Job created for the AirflowDB is not shown. +The diagram above depicts all the Kubernetes resources created by the operator, and how they relate to each other. For every xref:concepts:roles-and-role-groups.adoc#_role_groups[role group] you define, the Operator creates a StatefulSet with the amount of replicas defined in the RoleGroup. Every Pod in the StatefulSet has two containers: the @@ -78,7 +70,7 @@ main container running Airflow and a sidecar container gathering metrics for xre Operator creates a Service per role group as well as a single service for the whole `webserver` role called `-webserver`. -ConfigMaps are created, one per RoleGroup and also one for the AirflowDB. Both ConfigMaps contains two files: `log_config.py` and `webserver_config.py` which contain logging and general Airflow configuration respectively. +Additionally, a ConfigMap is created for each RoleGroup. These ConfigMaps contain two files, `log_config.py` and `webserver_config.py`, which contain logging and general Airflow configuration respectively. == Required external components