From 93e1322336067d0e17f01bc6a018b8683ca71c18 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Tue, 6 Aug 2024 06:44:27 +0200 Subject: [PATCH] FIX - Ingestion Airflow Image constraints (#17296) * FIX - Ingestion constraints * FIX - Ingestion constraints * FIX - Ingestion constraints --- ingestion/Dockerfile | 2 +- ingestion/Dockerfile.ci | 6 +++--- ingestion/Makefile | 10 +++++----- ingestion/constraints.txt | 1 - openmetadata-airflow-apis/constraints.txt | 1 - openmetadata-airflow-apis/pyproject.toml | 4 ++-- 6 files changed, 11 insertions(+), 13 deletions(-) delete mode 100644 ingestion/constraints.txt delete mode 100644 openmetadata-airflow-apis/constraints.txt diff --git a/ingestion/Dockerfile b/ingestion/Dockerfile index be5f17ba734f..3f7f7c90c312 100644 --- a/ingestion/Dockerfile +++ b/ingestion/Dockerfile @@ -70,7 +70,7 @@ ENV PIP_NO_CACHE_DIR=1 ENV PIP_QUIET=1 ARG RI_VERSION="1.5.0.0rc1" RUN pip install --upgrade pip -RUN pip install "openmetadata-managed-apis~=${RI_VERSION}" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt" +RUN pip install "openmetadata-managed-apis~=${RI_VERSION}" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.9.1/constraints-3.10.txt" RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]~=${RI_VERSION}" # Temporary workaround for https://github.com/open-metadata/OpenMetadata/issues/9593 diff --git a/ingestion/Dockerfile.ci b/ingestion/Dockerfile.ci index bc49bdf9e77b..23736c7e3249 100644 --- a/ingestion/Dockerfile.ci +++ b/ingestion/Dockerfile.ci @@ -65,7 +65,7 @@ COPY --chown=airflow:0 openmetadata-airflow-apis /home/airflow/openmetadata-airf COPY --chown=airflow:0 ingestion/examples/airflow/dags /opt/airflow/dags USER airflow -ARG AIRFLOW_CONSTRAINTS_LOCATION="https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt" +ARG AIRFLOW_CONSTRAINTS_LOCATION="https://raw.githubusercontent.com/apache/airflow/constraints-2.9.1/constraints-3.10.txt" # Disable pip cache dir # https://pip.pypa.io/en/stable/topics/caching/#avoiding-caching @@ -76,13 +76,13 @@ ENV PIP_QUIET=1 RUN pip install --upgrade pip WORKDIR /home/airflow/openmetadata-airflow-apis -RUN PIP_CONSTRAINT=/home/airflow/openmetadata-airflow-apis/constraints.txt pip install "." +RUN pip install "." --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" WORKDIR /home/airflow/ingestion # Argument to provide for Ingestion Dependencies to install. Defaults to all ARG INGESTION_DEPENDENCY="all" -RUN PIP_CONSTRAINT=/home/airflow/ingestion/constraints.txt pip install ".[${INGESTION_DEPENDENCY}]" +RUN pip install ".[${INGESTION_DEPENDENCY}]" # Temporary workaround for https://github.com/open-metadata/OpenMetadata/issues/9593 RUN echo "Image built for $(uname -m)" diff --git a/ingestion/Makefile b/ingestion/Makefile index 915bdc5d1fea..4698a13700bd 100644 --- a/ingestion/Makefile +++ b/ingestion/Makefile @@ -11,23 +11,23 @@ endif .PHONY: install install: ## Install the ingestion module to the current environment - PIP_CONSTRAINT=$(INGESTION_DIR)/constraints.txt python -m pip install $(INGESTION_DIR)/ + python -m pip install $(INGESTION_DIR)/ .PHONY: install_dev install_dev: ## Install the ingestion module with dev dependencies - PIP_CONSTRAINT=$(INGESTION_DIR)/constraints.txt python -m pip install "$(INGESTION_DIR)[dev]/" + python -m pip install "$(INGESTION_DIR)[dev]/" .PHONY: install_test install_test: ## Install the ingestion module with test dependencies - PIP_CONSTRAINT=$(INGESTION_DIR)/constraints.txt python -m pip install "$(INGESTION_DIR)[test]/" + python -m pip install "$(INGESTION_DIR)[test]/" .PHONY: install_all install_all: ## Install the ingestion module with all dependencies - PIP_CONSTRAINT=$(INGESTION_DIR)/constraints.txt python -m pip install "$(INGESTION_DIR)[all]/" + python -m pip install "$(INGESTION_DIR)[all]/" .PHONY: install_apis install_apis: ## Install the REST APIs module to the current environment - PIP_CONSTRAINT=$(ROOT_DIR)/openmetadata-airflow-apis/constraints.txt python -m pip install $(ROOT_DIR)/openmetadata-airflow-apis/ setuptools~=70.3.0 + python -m pip install $(ROOT_DIR)/openmetadata-airflow-apis/ setuptools~=70.3.0 .PHONY: lint lint: ## Run pylint on the Python sources to analyze the codebase diff --git a/ingestion/constraints.txt b/ingestion/constraints.txt deleted file mode 100644 index 0685d33561b5..000000000000 --- a/ingestion/constraints.txt +++ /dev/null @@ -1 +0,0 @@ -setuptools<72 \ No newline at end of file diff --git a/openmetadata-airflow-apis/constraints.txt b/openmetadata-airflow-apis/constraints.txt deleted file mode 100644 index 0685d33561b5..000000000000 --- a/openmetadata-airflow-apis/constraints.txt +++ /dev/null @@ -1 +0,0 @@ -setuptools<72 \ No newline at end of file diff --git a/openmetadata-airflow-apis/pyproject.toml b/openmetadata-airflow-apis/pyproject.toml index c7bed887fa3f..f84aa07b9a26 100644 --- a/openmetadata-airflow-apis/pyproject.toml +++ b/openmetadata-airflow-apis/pyproject.toml @@ -15,9 +15,9 @@ license = {file = "LICENSE"} description = "Airflow REST APIs to create and manage DAGS" requires-python = ">=3.8" dependencies = [ - "pendulum~=2.1.2", + "pendulum~=3.0", "apache-airflow>=2.2.2", - "Flask>=1.1.4", + "Flask==2.2.5", "Flask-Admin==1.6.0", ]