Improved Armada Airflow Operator (#3672)

* Syncing out Armada Airflow Operator (#164)

* changes

* Fixing tests

* Formatting

* Making integration tests work

* Formatting files

* Formatting, ensuring Armada trigger can cancel

* Fixing docs and adding back examples

* Fixing docs

* Addressing comments, supporting more python versions

* Updating docs

* Removing airflow operator build script from github workflow check

---------

Co-authored-by: Mustafa Ilyas <Mustafa.Ilyas@gresearch.co.uk>

.github/workflows/airflow-operator.yml

@@ -16,7 +16,6 @@ on:
  - 'internal/jobservice/*'
  - 'pkg/api/*.proto'
  - 'pkg/api/jobservice/*.proto'
- - 'scripts/build-airflow-operator.sh'
  - 'scripts/build-python-client.sh'
  - 'third_party/airflow/**'
  - './magefiles/tests.go'
@@ -37,7 +36,6 @@ on:
  - 'internal/jobservice/*'
  - 'pkg/api/*.proto'
  - 'pkg/api/jobservice/*.proto'
- - 'scripts/build-airflow-operator.sh'
  - 'scripts/build-python-client.sh'
  - 'third_party/airflow/**'
 

build/airflow-operator/Dockerfile

@@ -1,5 +1,5 @@
 ARG PLATFORM=x86_64
-ARG BASE_IMAGE=python:3.8.18-bookworm
+ARG BASE_IMAGE=python:3.10.14-bookworm
 FROM --platform=$PLATFORM ${BASE_IMAGE}
 
 RUN mkdir /proto

developer/env/docker/server.env

@@ -1,3 +1,3 @@
 ARMADA_QUEUECACHEREFRESHPERIOD="1s"
 ARMADA_CORSALLOWEDORIGINS="http://localhost:3000,http://localhost:10000,http://example.com:10000"
-
+ARMADA_QUERYAPI_POSTGRES_CONNECTION_HOST=postgres

docker-compose.yaml

@@ -61,6 +61,7 @@ services:
  depends_on:
  - lookoutv2-migration
  - eventingester
+ - lookoutingesterv2
  working_dir: /app
  env_file:
  - developer/env/docker/server.env