From 7e08113b43361cdeaac304833e5279870c92081c Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Fri, 17 Nov 2023 12:58:19 +0100 Subject: [PATCH 01/12] test --- ingestion/{setup.py => _setup.py} | 0 ingestion/pyproject.toml | 17 +++++++++++++++++ 2 files changed, 17 insertions(+) rename ingestion/{setup.py => _setup.py} (100%) create mode 100644 ingestion/pyproject.toml diff --git a/ingestion/setup.py b/ingestion/_setup.py similarity index 100% rename from ingestion/setup.py rename to ingestion/_setup.py diff --git a/ingestion/pyproject.toml b/ingestion/pyproject.toml new file mode 100644 index 000000000000..1a5193ace939 --- /dev/null +++ b/ingestion/pyproject.toml @@ -0,0 +1,17 @@ +[build-system] +requires = ['hatchling', 'hatch-fancy-pypi-readme>=22.5.0'] +build-backend = 'hatchling.build' + +[versions] +airflow = "apache-airflow==2.6.3" +avro = "avro~=1.11" + +[project] +name = "openmetadata-ingestion" +version = "1.3.0.0.dev0" +url = "https://open-metadata.org/" +author = "OpenMetadata Committers" +license = "Apache License 2.0" +description = "Ingestion Framework for OpenMetadata" +requires-python = ">=3.8" +dependencies = [versions.avro] \ No newline at end of file From d802294631917494e46b53eab5291ada11eea6ef Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Sun, 19 Nov 2023 16:42:58 +0100 Subject: [PATCH 02/12] Use pyproject.toml --- .../workflows/openmetadata-airflow-apis.yml | 2 +- .../workflows/py-ingestion-core-publish.yml | 49 -------- .github/workflows/py-ingestion-publish.yml | 2 +- Makefile | 34 ------ ingestion/.coveragerc | 11 -- ingestion/Makefile | 2 +- ingestion/pyproject.toml | 112 ++++++++++++++++-- ingestion/setup.cfg | 52 -------- ingestion/{_setup.py => setup.py} | 41 +------ 9 files changed, 108 insertions(+), 197 deletions(-) delete mode 100644 .github/workflows/py-ingestion-core-publish.yml delete mode 100644 ingestion/.coveragerc delete mode 100644 ingestion/setup.cfg rename ingestion/{_setup.py => setup.py} (87%) diff --git a/.github/workflows/openmetadata-airflow-apis.yml b/.github/workflows/openmetadata-airflow-apis.yml index 13f887d984fc..f87348ce8bb3 100644 --- a/.github/workflows/openmetadata-airflow-apis.yml +++ b/.github/workflows/openmetadata-airflow-apis.yml @@ -34,6 +34,6 @@ jobs: run: | make install_dev install_apis cd openmetadata-airflow-apis; \ - python setup.py build sdist bdist_wheel; \ + python -m build; \ twine check dist/*; \ twine upload dist/* --verbose diff --git a/.github/workflows/py-ingestion-core-publish.yml b/.github/workflows/py-ingestion-core-publish.yml deleted file mode 100644 index 9f840e5161f8..000000000000 --- a/.github/workflows/py-ingestion-core-publish.yml +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2021 Collate -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# When a new push happens to the main branch that -# modifies the JSON Schemas, we are going to -# generate the updated pydantic models and -# publish a new version of openmetadata-core - -name: Publish openmetadata-ingestion-core packages - -on: - workflow_dispatch: - -concurrency: - group: py-ingestion-core-publish-${{ github.head_ref || github.run_id }} - cancel-in-progress: true -jobs: - py-core-build-and-push: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.9 - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - name: Install Ubuntu related dependencies - run: | - sudo apt-get update && sudo apt-get install -y libsasl2-dev unixodbc-dev python3-venv - - name: Install, Generate and Publish Test PyPi packages - env: - TWINE_USERNAME: ${{ secrets.TWINE_USERNAME_TEST }} - TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD_TEST }} - run: | - python3 -m venv env - source env/bin/activate - sudo make install_antlr_cli - make install_dev generate install - cd ingestion-core; \ - python setup.py build sdist bdist_wheel; \ - twine check dist/*; \ - twine upload dist/* --verbose diff --git a/.github/workflows/py-ingestion-publish.yml b/.github/workflows/py-ingestion-publish.yml index dd4567460881..7838e25c20b8 100644 --- a/.github/workflows/py-ingestion-publish.yml +++ b/.github/workflows/py-ingestion-publish.yml @@ -37,6 +37,6 @@ jobs: sudo make install_antlr_cli make install_dev generate install cd ingestion; \ - python setup.py build sdist bdist_wheel; \ + python -m build; \ twine check dist/*; \ twine upload dist/* --verbose diff --git a/Makefile b/Makefile index 3bf028852878..b9fca04865ff 100644 --- a/Makefile +++ b/Makefile @@ -24,40 +24,6 @@ yarn_install_cache: ## Use Yarn to install UI dependencies yarn_start_dev_ui: ## Run the UI locally with Yarn cd openmetadata-ui/src/main/resources/ui && yarn start -## Ingestion Core -.PHONY: core_install_dev -core_install_dev: ## Prepare a venv for the ingestion-core module - cd ingestion-core; \ - rm -rf venv; \ - python3 -m venv venv; \ - . venv/bin/activate; \ - python3 -m pip install ".[dev]" - -.PHONY: core_clean -core_clean: ## Clean the ingestion-core generated files - rm -rf ingestion-core/src/metadata/generated - rm -rf ingestion-core/build - rm -rf ingestion-core/dist - -.PHONY: core_generate -core_generate: ## Generate the pydantic models from the JSON Schemas to the ingestion-core module - $(MAKE) core_install_dev - mkdir -p ingestion-core/src/metadata/generated; \ - . ingestion-core/venv/bin/activate; \ - datamodel-codegen --input openmetadata-spec/src/main/resources/json/schema --input-file-type jsonschema --output ingestion-core/src/metadata/generated/schema - $(MAKE) core_py_antlr - -.PHONY: core_bump_version_dev -core_bump_version_dev: ## Bump a `dev` version to the ingestion-core module. To be used when schemas are updated - $(MAKE) core_install_dev - cd ingestion-core; \ - . venv/bin/activate; \ - python -m incremental.update metadata --dev - -.PHONY: core_py_antlr -core_py_antlr: ## Generate the Python core code for parsing FQNs under ingestion-core - antlr4 -Dlanguage=Python3 -o ingestion-core/src/metadata/generated/antlr ${PWD}/openmetadata-spec/src/main/antlr4/org/openmetadata/schema/*.g4 - .PHONY: py_antlr py_antlr: ## Generate the Python code for parsing FQNs antlr4 -Dlanguage=Python3 -o ingestion/src/metadata/generated/antlr ${PWD}/openmetadata-spec/src/main/antlr4/org/openmetadata/schema/*.g4 diff --git a/ingestion/.coveragerc b/ingestion/.coveragerc deleted file mode 100644 index 0d6a5c99fdf2..000000000000 --- a/ingestion/.coveragerc +++ /dev/null @@ -1,11 +0,0 @@ -[run] -source = env/lib/python3.9/site-packages/metadata -relative_files = True -branch = True -[report] -omit = - *__init__* - */generated/* - tests/* - ingestion/src/* - */src/metadata/ingestion/source/database/sample_* \ No newline at end of file diff --git a/ingestion/Makefile b/ingestion/Makefile index 6d57e471f3b8..ccc7a028a61b 100644 --- a/ingestion/Makefile +++ b/ingestion/Makefile @@ -31,7 +31,7 @@ install_apis: ## Install the REST APIs module to the current environment .PHONY: lint lint: ## Run pylint on the Python sources to analyze the codebase - PYTHONPATH="${PYTHONPATH}:$(INGESTION_DIR)/plugins" find $(PY_SOURCE) -path $(PY_SOURCE)/metadata/generated -prune -false -o -type f -name "*.py" | xargs pylint + PYTHONPATH="${PYTHONPATH}:$(INGESTION_DIR)/plugins" find $(PY_SOURCE) -path $(PY_SOURCE)/metadata/generated -prune -false -o -type f -name "*.py" | xargs pylint --rcfile=$(INGESTION_DIR)/pyproject.toml .PHONY: precommit_install precommit_install: ## Install the project's precommit hooks from .pre-commit-config.yaml diff --git a/ingestion/pyproject.toml b/ingestion/pyproject.toml index 1a5193ace939..ece41e4e188f 100644 --- a/ingestion/pyproject.toml +++ b/ingestion/pyproject.toml @@ -1,17 +1,109 @@ [build-system] -requires = ['hatchling', 'hatch-fancy-pypi-readme>=22.5.0'] -build-backend = 'hatchling.build' - -[versions] -airflow = "apache-airflow==2.6.3" -avro = "avro~=1.11" +requires = ["setuptools"] +build-backend = "setuptools.build_meta" +# We will keep handling dependencies in setup.py +# since it helps us organize and isolate version management [project] name = "openmetadata-ingestion" version = "1.3.0.0.dev0" -url = "https://open-metadata.org/" -author = "OpenMetadata Committers" -license = "Apache License 2.0" +dynamic = ["readme"] +authors = [ + {name = "OpenMetadata Committers"} +] +license = {file = "LICENSE"} description = "Ingestion Framework for OpenMetadata" requires-python = ">=3.8" -dependencies = [versions.avro] \ No newline at end of file + +[project.urls] +Homepage = "https://open-metadata.org/" +Documentation = "https://docs.open-metadata.org/" +Source = "https://github.com/open-metadata/OpenMetadata" + +[tool.setuptools.dynamic] +readme = {file = ["README.md"]} + +[tool.setuptools.packages.find] +where = ["./src"] +exclude = ["tests*"] +namespaces = true + +[tool.setuptools.package-data] +"metadata.examples" = ["workflows/*.yaml"] + +[project.scripts] +metadata-cli = "metadata.cmd:metadata" + +[project.entry-points.apache_airflow_provider] +provider_info = "airflow_provider_openmetadata:get_provider_config" + +[tool.coverage.run] +source = "env/lib/python3.9/site-packages/metadata" +relative_files = true +branch = true + +[tool.coverage.report] +omit = [ + "*__init__*", + "*/generated/*", + "tests/*", + "ingestion/src/*", + "*/src/metadata/ingestion/source/database/sample_*" +] + +[tool.mypy] +mypy_path = "src" +plugins = [ + "sqlalchemy.ext.mypy.plugin", + "pydantic.mypy" +] +ignore_missing_imports = true +namespace_packages = true +strict_optional = true +check_untyped_defs = true +# eventually we'd like to enable these +disallow_untyped_defs = false +disallow_incomplete_defs = false + +[tool.isort] +profile = "black" +indent = " " + +[tool.pytest.ini_options] +markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')" +] + +[tool.pylint.BASIC] +# W1203: logging-fstring-interpolation - f-string brings better readability and unifies style +# W1202: logging-format-interpolation - lazy formatting in logging functions +# R0903: too-few-public-methods - False negatives in pydantic classes +# W0707: raise-missing-from - Tends to be a false positive as exception are closely encapsulated +# R0901: too-many-ancestors - We are already inheriting from SQA classes with a bunch of ancestors +# W0703: broad-except - We are dealing with many different source systems, but we want to make sure workflows run until the end +# W0511: fixme - These are internal notes and guides +# W1518: method-cache-max-size-none - allow us to use LRU Cache with maxsize `None` to speed up certain calls +disable = "W1203,W1202,R0903,W0707,R0901,W1201,W0703,W0511,W1518" + +docstring-min-length = 20 +max-args = 7 +max-attributes = 12 + +# usual typevar naming +good-names = "T,C,fn,db,df,i" +module-rgx = "(([a-z_][a-z0-9_]*)|([a-zA-Z0-9]+))$" + +[tool.pylint.MASTER] +fail-under = 6.0 +init-hook = "from pylint.config import find_default_config_files; import os, sys; sys.path.append(os.path.dirname(next(find_default_config_files())))" +extension-pkg-allow-list = "pydantic" +load-plugins = "OpenMetadata.ingestion.plugins.print_checker" +max-public-methods = 25 + +[tool.pylint."MESSAGES CONTROL"] +disable = "no-name-in-module,import-error,duplicate-code" +enable = "useless-suppression" + +[tool.pylint.FORMAT] +# We all have big monitors now +max-line-length = 120 diff --git a/ingestion/setup.cfg b/ingestion/setup.cfg deleted file mode 100644 index 34ec910e7ce6..000000000000 --- a/ingestion/setup.cfg +++ /dev/null @@ -1,52 +0,0 @@ -[flake8] -# We ignore the line length issues here, since black will take care of them. -max-line-length = 150 -max-complexity = 15 -ignore = - # Ignore: 1 blank line required before class docstring. - D203, - W503 -exclude = - .git, - __pycache__ -per-file-ignores = - # imported but unused - __init__.py: F401 -[metadata] -license_files = LICENSE -[mypy] -mypy_path = src -plugins = - sqlalchemy.ext.mypy.plugin, - pydantic.mypy -ignore_missing_imports = yes -namespace_packages = true -strict_optional = yes -check_untyped_defs = yes -# eventually we'd like to enable these -disallow_untyped_defs = no -disallow_incomplete_defs = no - -[isort] -profile = black -indent=' ' -sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER - -[tool:pytest] -markers = - slow: marks tests as slow (deselect with '-m "not slow"') -testpaths = - tests/unit - -[options] -packages = find: -package_dir = - =src - -[options.packages.find] -where = src -include = * - -[options.package_data] -metadata.examples = - workflows/*.yaml diff --git a/ingestion/_setup.py b/ingestion/setup.py similarity index 87% rename from ingestion/_setup.py rename to ingestion/setup.py index c7aab12cdb14..e76c1e47aac8 100644 --- a/ingestion/_setup.py +++ b/ingestion/setup.py @@ -13,17 +13,9 @@ Python Dependencies """ -import os from typing import Dict, Set -from setuptools import find_namespace_packages, setup - - -def get_long_description(): - root = os.path.dirname(__file__) - with open(os.path.join(root, "README.md"), encoding="UTF-8") as file: - description = file.read() - return description +from setuptools import setup # Add here versions required for multiple plugins @@ -123,7 +115,6 @@ def get_long_description(): "sqlalchemy>=1.4.0,<2", "collate-sqllineage>=1.0.4", "tabulate==0.9.0", - "typing-compat~=0.1.0", # compatibility requirements for 3.7 "typing_extensions<=4.5.0", # We need to have this fixed due to a yanked release 4.6.0 "typing-inspect", "wheel~=0.38.4", @@ -261,12 +252,13 @@ def get_long_description(): dev = { "black==22.3.0", "datamodel-code-generator==0.22.0", - "docker", "isort", "pre-commit", "pycln", "pylint~=3.0.0", + # For publishing "twine", + "build" } test = { @@ -306,34 +298,7 @@ def get_long_description(): "pytest-base-url", } -build_options = {"includes": ["_cffi_backend"]} setup( - name="openmetadata-ingestion", - version="1.3.0.0.dev0", - url="https://open-metadata.org/", - author="OpenMetadata Committers", - license="Apache License 2.0", - description="Ingestion Framework for OpenMetadata", - long_description=get_long_description(), - long_description_content_type="text/markdown", - python_requires=">=3.8", - options={"build_exe": build_options}, - package_dir={"": "src"}, - package_data={"metadata.examples": ["workflows/*.yaml"]}, - zip_safe=False, - dependency_links=[], - project_urls={ - "Documentation": "https://docs.open-metadata.org/", - "Source": "https://github.com/open-metadata/OpenMetadata", - }, - packages=find_namespace_packages(where="./src", exclude=["tests*"]), - namespace_package=["metadata"], - entry_points={ - "console_scripts": ["metadata = metadata.cmd:metadata"], - "apache_airflow_provider": [ - "provider_info = airflow_provider_openmetadata:get_provider_config" - ], - }, install_requires=list(base_requirements), extras_require={ "base": list(base_requirements), From c7f80591bb48f5af123fec48ef09d0e1d50efb2c Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Sun, 19 Nov 2023 16:45:55 +0100 Subject: [PATCH 03/12] Fix pylint --- ingestion/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingestion/pyproject.toml b/ingestion/pyproject.toml index ece41e4e188f..cff5f0733589 100644 --- a/ingestion/pyproject.toml +++ b/ingestion/pyproject.toml @@ -97,7 +97,7 @@ module-rgx = "(([a-z_][a-z0-9_]*)|([a-zA-Z0-9]+))$" fail-under = 6.0 init-hook = "from pylint.config import find_default_config_files; import os, sys; sys.path.append(os.path.dirname(next(find_default_config_files())))" extension-pkg-allow-list = "pydantic" -load-plugins = "OpenMetadata.ingestion.plugins.print_checker" +load-plugins = "ingestion.plugins.print_checker" max-public-methods = 25 [tool.pylint."MESSAGES CONTROL"] From 209cf25f632bbeb23cad072666effa6eca08e776 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Sun, 19 Nov 2023 17:23:20 +0100 Subject: [PATCH 04/12] makefile --- ingestion/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ingestion/Makefile b/ingestion/Makefile index ccc7a028a61b..9215c632fa89 100644 --- a/ingestion/Makefile +++ b/ingestion/Makefile @@ -41,15 +41,15 @@ precommit_install: ## Install the project's precommit hooks from .pre-commit-co .PHONY: py_format py_format: ## Run black and isort to format the Python codebase - pycln $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --extend-exclude $(PY_SOURCE)/metadata/generated --all + pycln $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --config $(INGESTION_DIR)/pyproject.toml isort $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --skip $(PY_SOURCE)/metadata/generated --skip $(INGESTION_DIR)/env --skip $(INGESTION_DIR)/build --skip $(ROOT_DIR)/openmetadata-airflow-apis/build --profile black --multi-line 3 - black $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --extend-exclude $(PY_SOURCE)/metadata/generated + black $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --config $(INGESTION_DIR)/pyproject.toml .PHONY: py_format_check py_format_check: ## Check if Python sources are correctly formatted - pycln $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --diff --extend-exclude $(PY_SOURCE)/metadata/generated --all + pycln $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --diff isort --check-only $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --skip $(PY_SOURCE)/metadata/generated --skip $(INGESTION_DIR)/build --skip $(ROOT_DIR)/openmetadata-airflow-apis/build --profile black --multi-line 3 - black --check --diff $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --extend-exclude $(PY_SOURCE)/metadata/generated + black --check --diff $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ PYTHONPATH="${PYTHONPATH}:$(INGESTION_DIR)/plugins" pylint --fail-under=10 $(PY_SOURCE)/metadata --ignore-paths $(PY_SOURCE)/metadata/generated || (echo "PyLint error code $$?"; exit 1) .PHONY: unit_ingestion From b99502380f8749928df50c1096574ab751da73a0 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Sun, 19 Nov 2023 17:24:21 +0100 Subject: [PATCH 05/12] makefile --- ingestion/pyproject.toml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ingestion/pyproject.toml b/ingestion/pyproject.toml index cff5f0733589..f5e7d571b87f 100644 --- a/ingestion/pyproject.toml +++ b/ingestion/pyproject.toml @@ -107,3 +107,11 @@ enable = "useless-suppression" [tool.pylint.FORMAT] # We all have big monitors now max-line-length = 120 + +[tool.black] +line-length = 120 +extend-exclude = "src/metadata/generated" + +[tool.pycln] +all = true +extend-exclude = "src/metadata/generated" From ee406af808dedee5376e45fb8bf3cc87b2f8e497 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Sun, 19 Nov 2023 17:31:43 +0100 Subject: [PATCH 06/12] Fix pylint --- ingestion/Makefile | 4 ++-- ingestion/pyproject.toml | 11 ++++++----- ingestion/setup.py | 3 +-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ingestion/Makefile b/ingestion/Makefile index 9215c632fa89..da0da0b9963b 100644 --- a/ingestion/Makefile +++ b/ingestion/Makefile @@ -42,13 +42,13 @@ precommit_install: ## Install the project's precommit hooks from .pre-commit-co .PHONY: py_format py_format: ## Run black and isort to format the Python codebase pycln $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --config $(INGESTION_DIR)/pyproject.toml - isort $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --skip $(PY_SOURCE)/metadata/generated --skip $(INGESTION_DIR)/env --skip $(INGESTION_DIR)/build --skip $(ROOT_DIR)/openmetadata-airflow-apis/build --profile black --multi-line 3 + isort $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --settings-file $(INGESTION_DIR)/pyproject.toml black $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --config $(INGESTION_DIR)/pyproject.toml .PHONY: py_format_check py_format_check: ## Check if Python sources are correctly formatted pycln $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --diff - isort --check-only $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --skip $(PY_SOURCE)/metadata/generated --skip $(INGESTION_DIR)/build --skip $(ROOT_DIR)/openmetadata-airflow-apis/build --profile black --multi-line 3 + isort --check-only $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --cr $(INGESTION_DIR)/pyproject.toml black --check --diff $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ PYTHONPATH="${PYTHONPATH}:$(INGESTION_DIR)/plugins" pylint --fail-under=10 $(PY_SOURCE)/metadata --ignore-paths $(PY_SOURCE)/metadata/generated || (echo "PyLint error code $$?"; exit 1) diff --git a/ingestion/pyproject.toml b/ingestion/pyproject.toml index f5e7d571b87f..da8708bb9805 100644 --- a/ingestion/pyproject.toml +++ b/ingestion/pyproject.toml @@ -65,10 +65,6 @@ check_untyped_defs = true disallow_untyped_defs = false disallow_incomplete_defs = false -[tool.isort] -profile = "black" -indent = " " - [tool.pytest.ini_options] markers = [ "slow: marks tests as slow (deselect with '-m \"not slow\"')" @@ -109,9 +105,14 @@ enable = "useless-suppression" max-line-length = 120 [tool.black] -line-length = 120 extend-exclude = "src/metadata/generated" [tool.pycln] all = true extend-exclude = "src/metadata/generated" + +[tool.isort] +skip = "src/metadata/generated,env,build,../openmetadata-airflow-apis/build" +profile = "black" +indent = " " +multi_line_output = 3 diff --git a/ingestion/setup.py b/ingestion/setup.py index e76c1e47aac8..3a8f6541328d 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -17,7 +17,6 @@ from setuptools import setup - # Add here versions required for multiple plugins VERSIONS = { "airflow": "apache-airflow==2.6.3", @@ -258,7 +257,7 @@ "pylint~=3.0.0", # For publishing "twine", - "build" + "build", } test = { From e774f2dcf8106933669985214a11bf7f9cb8c40c Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Sun, 19 Nov 2023 17:33:06 +0100 Subject: [PATCH 07/12] isort --- ingestion/pyproject.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ingestion/pyproject.toml b/ingestion/pyproject.toml index da8708bb9805..9d3f1965b061 100644 --- a/ingestion/pyproject.toml +++ b/ingestion/pyproject.toml @@ -112,7 +112,11 @@ all = true extend-exclude = "src/metadata/generated" [tool.isort] -skip = "src/metadata/generated,env,build,../openmetadata-airflow-apis/build" +skip = [ + "src/metadata/generated", + "build", + "../openmetadata-airflow-apis/build" +] profile = "black" indent = " " multi_line_output = 3 From 6caeb1edba7d70792d4bf4d064556e50c2c4f7b3 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Sun, 19 Nov 2023 17:40:40 +0100 Subject: [PATCH 08/12] pyproject --- ingestion/Makefile | 22 +++++++++++----------- ingestion/pyproject.toml | 5 ++++- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/ingestion/Makefile b/ingestion/Makefile index da0da0b9963b..b9ebbf583168 100644 --- a/ingestion/Makefile +++ b/ingestion/Makefile @@ -47,26 +47,26 @@ py_format: ## Run black and isort to format the Python codebase .PHONY: py_format_check py_format_check: ## Check if Python sources are correctly formatted - pycln $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --diff - isort --check-only $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --cr $(INGESTION_DIR)/pyproject.toml - black --check --diff $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ - PYTHONPATH="${PYTHONPATH}:$(INGESTION_DIR)/plugins" pylint --fail-under=10 $(PY_SOURCE)/metadata --ignore-paths $(PY_SOURCE)/metadata/generated || (echo "PyLint error code $$?"; exit 1) + pycln $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --diff --config $(INGESTION_DIR)/pyproject.toml + isort --check-only $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --settings-file $(INGESTION_DIR)/pyproject.toml + black --check --diff $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --config $(INGESTION_DIR)/pyproject.toml + PYTHONPATH="${PYTHONPATH}:$(INGESTION_DIR)/plugins" pylint --rcfile=$(INGESTION_DIR)/pyproject.toml --fail-under=10 $(PY_SOURCE)/metadata --ignore-paths $(PY_SOURCE)/metadata/generated || (echo "PyLint error code $$?"; exit 1) .PHONY: unit_ingestion unit_ingestion: ## Run Python unit tests - coverage run --rcfile $(INGESTION_DIR)/.coveragerc -a --branch -m pytest -c $(INGESTION_DIR)/setup.cfg --junitxml=$(INGESTION_DIR)/junit/test-results-unit.xml --ignore=$(INGESTION_DIR)/tests/unit/source $(INGESTION_DIR)/tests/unit + coverage run --rcfile $(INGESTION_DIR)/pyproject.toml -a --branch -m pytest -c $(INGESTION_DIR)/pyproject.toml --junitxml=$(INGESTION_DIR)/junit/test-results-unit.xml --ignore=$(INGESTION_DIR)/tests/unit/source $(INGESTION_DIR)/tests/unit ## Ingestion tests & QA .PHONY: run_ometa_integration_tests run_ometa_integration_tests: ## Run Python integration tests - coverage run --rcfile $(INGESTION_DIR)/.coveragerc -a --branch -m pytest -c $(INGESTION_DIR)/setup.cfg --junitxml=$(INGESTION_DIR)/junit/test-results-integration.xml $(INGESTION_DIR)/tests/integration/ometa $(INGESTION_DIR)/tests/integration/orm_profiler $(INGESTION_DIR)/tests/integration/test_suite $(INGESTION_DIR)/tests/integration/data_insight $(INGESTION_DIR)/tests/integration/lineage + coverage run --rcfile $(INGESTION_DIR)/pyproject.toml -a --branch -m pytest -c $(INGESTION_DIR)/pyproject.toml --junitxml=$(INGESTION_DIR)/junit/test-results-integration.xml $(INGESTION_DIR)/tests/integration/ometa $(INGESTION_DIR)/tests/integration/orm_profiler $(INGESTION_DIR)/tests/integration/test_suite $(INGESTION_DIR)/tests/integration/data_insight $(INGESTION_DIR)/tests/integration/lineage .PHONY: run_python_tests run_python_tests: ## Run all Python tests with coverage coverage erase $(MAKE) unit_ingestion $(MAKE) run_ometa_integration_tests - coverage report --rcfile $(INGESTION_DIR)/.coveragerc || true + coverage report --rcfile $(INGESTION_DIR)/pyproject.toml || true .PHONY: sonar_ingestion sonar_ingestion: ## Run the Sonar analysis based on the tests results and push it to SonarCloud @@ -82,14 +82,14 @@ sonar_ingestion: ## Run the Sonar analysis based on the tests results and push .PHONY: run_apis_tests run_apis_tests: ## Run the openmetadata airflow apis tests coverage erase - coverage run --rcfile $(ROOT_DIR)/openmetadata-airflow-apis/.coveragerc -a --branch -m pytest --junitxml=$(ROOT_DIR)/openmetadata-airflow-apis/junit/test-results.xml $(ROOT_DIR)/openmetadata-airflow-apis/tests - coverage report --rcfile $(ROOT_DIR)/openmetadata-airflow-apis/.coveragerc + coverage run --rcfile $(ROOT_DIR)/openmetadata-airflow-apis/pyproject.toml -a --branch -m pytest -c $(INGESTION_DIR)/pyproject.toml --junitxml=$(ROOT_DIR)/openmetadata-airflow-apis/junit/test-results.xml $(ROOT_DIR)/openmetadata-airflow-apis/tests + coverage report --rcfile $(ROOT_DIR)/openmetadata-airflow-apis/pyproject.toml .PHONY: coverage_apis coverage_apis: ## Run the python tests on openmetadata-airflow-apis $(MAKE) run_apis_tests - coverage xml --rcfile $(ROOT_DIR)/openmetadata-airflow-apis/.coveragerc -o $(ROOT_DIR)/openmetadata-airflow-apis/coverage.xml + coverage xml --rcfile $(ROOT_DIR)/openmetadata-airflow-apis/pyproject.toml -o $(ROOT_DIR)/openmetadata-airflow-apis/coverage.xml sed -e "s/$(shell python -c "import site; import os; from pathlib import Path; print(os.path.relpath(site.getsitepackages()[0], str(Path.cwd())).replace('/','\/'))")\///g" $(ROOT_DIR)/openmetadata-airflow-apis/coverage.xml >> $(ROOT_DIR)/openmetadata-airflow-apis/ci-coverage.xml @@ -97,5 +97,5 @@ coverage_apis: ## Run the python tests on openmetadata-airflow-apis .PHONY: coverage coverage: ## Run all Python tests and generate the coverage XML report $(MAKE) run_python_tests - coverage xml --rcfile $(INGESTION_DIR)/.coveragerc -o $(INGESTION_DIR)/coverage.xml || true + coverage xml --rcfile $(INGESTION_DIR)/pyproject.toml -o $(INGESTION_DIR)/coverage.xml || true sed -e "s/$(shell python -c "import site; import os; from pathlib import Path; print(os.path.relpath(site.getsitepackages()[0], str(Path.cwd())).replace('/','\/'))")/src/g" $(INGESTION_DIR)/coverage.xml >> $(INGESTION_DIR)/ci-coverage.xml diff --git a/ingestion/pyproject.toml b/ingestion/pyproject.toml index 9d3f1965b061..3ad4d21e437e 100644 --- a/ingestion/pyproject.toml +++ b/ingestion/pyproject.toml @@ -38,7 +38,9 @@ metadata-cli = "metadata.cmd:metadata" provider_info = "airflow_provider_openmetadata:get_provider_config" [tool.coverage.run] -source = "env/lib/python3.9/site-packages/metadata" +source = [ + "env/lib/python3.9/site-packages/metadata" +] relative_files = true branch = true @@ -115,6 +117,7 @@ extend-exclude = "src/metadata/generated" skip = [ "src/metadata/generated", "build", + "env", "../openmetadata-airflow-apis/build" ] profile = "black" From 862400283fd0485ac7265fa12a2fd3e662f90884 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Sun, 19 Nov 2023 17:51:06 +0100 Subject: [PATCH 09/12] Airflow apis pyproject --- ingestion/Makefile | 3 - openmetadata-airflow-apis/.coveragerc | 10 -- openmetadata-airflow-apis/LICENSE | 201 +++++++++++++++++++++++ openmetadata-airflow-apis/pyproject.toml | 65 ++++++++ openmetadata-airflow-apis/setup.py | 96 ----------- 5 files changed, 266 insertions(+), 109 deletions(-) delete mode 100644 openmetadata-airflow-apis/.coveragerc create mode 100644 openmetadata-airflow-apis/LICENSE create mode 100644 openmetadata-airflow-apis/pyproject.toml delete mode 100644 openmetadata-airflow-apis/setup.py diff --git a/ingestion/Makefile b/ingestion/Makefile index b9ebbf583168..72e7f34470cc 100644 --- a/ingestion/Makefile +++ b/ingestion/Makefile @@ -85,15 +85,12 @@ run_apis_tests: ## Run the openmetadata airflow apis tests coverage run --rcfile $(ROOT_DIR)/openmetadata-airflow-apis/pyproject.toml -a --branch -m pytest -c $(INGESTION_DIR)/pyproject.toml --junitxml=$(ROOT_DIR)/openmetadata-airflow-apis/junit/test-results.xml $(ROOT_DIR)/openmetadata-airflow-apis/tests coverage report --rcfile $(ROOT_DIR)/openmetadata-airflow-apis/pyproject.toml - .PHONY: coverage_apis coverage_apis: ## Run the python tests on openmetadata-airflow-apis $(MAKE) run_apis_tests coverage xml --rcfile $(ROOT_DIR)/openmetadata-airflow-apis/pyproject.toml -o $(ROOT_DIR)/openmetadata-airflow-apis/coverage.xml sed -e "s/$(shell python -c "import site; import os; from pathlib import Path; print(os.path.relpath(site.getsitepackages()[0], str(Path.cwd())).replace('/','\/'))")\///g" $(ROOT_DIR)/openmetadata-airflow-apis/coverage.xml >> $(ROOT_DIR)/openmetadata-airflow-apis/ci-coverage.xml - - .PHONY: coverage coverage: ## Run all Python tests and generate the coverage XML report $(MAKE) run_python_tests diff --git a/openmetadata-airflow-apis/.coveragerc b/openmetadata-airflow-apis/.coveragerc deleted file mode 100644 index 64a3eb0c3c5a..000000000000 --- a/openmetadata-airflow-apis/.coveragerc +++ /dev/null @@ -1,10 +0,0 @@ -[run] -source = env/lib/python3.9/site-packages/openmetadata_managed_apis -relative_files = True -branch = True -[report] -omit = - *__init__* - tests/* - views/** - plugin.py diff --git a/openmetadata-airflow-apis/LICENSE b/openmetadata-airflow-apis/LICENSE new file mode 100644 index 000000000000..f49a4e16e68b --- /dev/null +++ b/openmetadata-airflow-apis/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/openmetadata-airflow-apis/pyproject.toml b/openmetadata-airflow-apis/pyproject.toml new file mode 100644 index 000000000000..8e5eafcdc556 --- /dev/null +++ b/openmetadata-airflow-apis/pyproject.toml @@ -0,0 +1,65 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +# We will keep handling dependencies in setup.py +# since it helps us organize and isolate version management +[project] +name = "openmetadata_managed_apis" +version = "1.3.0.0.dev0" +dynamic = ["readme"] +authors = [ + {name = "OpenMetadata Committers"} +] +license = {file = "LICENSE"} +description = "Airflow REST APIs to create and manage DAGS" +requires-python = ">=3.8" +dependencies = [ + "pendulum~=2.1.2", + "apache-airflow>=2.2.2", + "Flask>=1.1.4", + "Flask-Admin==1.6.0", +] + +[project.optional-dependencies] +dev = [ + "black==22.3.0", + "pytest", + "pylint", + "pytest-cov", + "isort", + "pycln", +] + +[project.urls] +Homepage = "https://open-metadata.org/" +Documentation = "https://docs.open-metadata.org/" +Source = "https://github.com/open-metadata/OpenMetadata" + +[tool.setuptools.dynamic] +readme = {file = ["README.md"]} + +[tool.setuptools.packages.find] +include = ["openmetadata_managed_apis.*", "openmetadata_managed_apis"] + +[tool.setuptools.package-data] +"openmetadata_managed_apis" = ["views/templates/rest_api/index.html", "resources/dag_runner.j2"] + +[project.entry-points."airflow.plugins"] +openmetadata_managed_apis = "openmetadata_managed_apis.plugin:RestApiPlugin" + + +[tool.coverage.run] +source = [ + "env/lib/python3.9/site-packages/openmetadata_managed_apis" +] +relative_files = true +branch = true + +[tool.coverage.report] +omit = [ + "*__init__*", + "tests/*", + "views/**", + "plugin.py" +] \ No newline at end of file diff --git a/openmetadata-airflow-apis/setup.py b/openmetadata-airflow-apis/setup.py deleted file mode 100644 index 33a3925c94ea..000000000000 --- a/openmetadata-airflow-apis/setup.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright 2021 Collate -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import glob -import os -import pathlib -from itertools import chain - -from setuptools import find_packages, setup - -PLUGIN_NAME = "openmetadata_managed_apis" -PLUGIN_ROOT_FILE = "plugin" -PLUGIN_ENTRY_POINT = "RestApiPlugin" -HOME = pathlib.Path(__file__).parent -README = (HOME / "README.md").read_text() -CONFIG = HOME / PLUGIN_NAME / "config.py" - - -def get_package_data(): - """ - Add templates and html files to the built - package for easier deployments - """ - extensions = ["html", "j2"] - - files = list( - chain( - *[ - glob.glob(f"{PLUGIN_NAME}/**/*.%s" % x, recursive=True) - for x in extensions - ] - ) - ) - return [x.split(os.sep, 1)[1] for x in files] - - -def get_long_description(): - root = os.path.dirname(__file__) - with open(os.path.join(root, "README.md")) as f: - description = f.read() - return description - - -base_requirements = { - "pendulum~=2.1.2", - "apache-airflow>=2.2.2", - "Flask>=1.1.4", - "Flask-Admin==1.6.0", -} - -dev_requirements = { - "black==22.3.0", - "pytest", - "pylint", - "pytest-cov", - "isort", - "pycln", -} - -setup( - name=PLUGIN_NAME, - packages=find_packages(include=[f"{PLUGIN_NAME}.*", PLUGIN_NAME]), - include_package_data=True, - package_data={PLUGIN_NAME: get_package_data()}, - version="1.3.0.0.dev0", - url="https://open-metadata.org/", - author="OpenMetadata Committers", - license="Apache License 2.0", - description="Airflow REST APIs to create and manage DAGS", - long_description=get_long_description(), - long_description_content_type="text/markdown", - entry_points={ - "airflow.plugins": [ - f"{PLUGIN_NAME} = {PLUGIN_NAME}.{PLUGIN_ROOT_FILE}:{PLUGIN_ENTRY_POINT}" - ] - }, - python_requires=">=3.8", - zip_safe=False, - dependency_links=[], - project_urls={ - "Documentation": "https://docs.open-metadata.org/", - "Source": "https://github.com/open-metadata/OpenMetadata", - }, - install_requires=list(base_requirements), - extras_require={ - "base": list(base_requirements), - "dev": list(dev_requirements), - }, -) From e0f79c1e9572dcaa295b003f098e44a6ec6924ec Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Sun, 19 Nov 2023 17:53:50 +0100 Subject: [PATCH 10/12] Remove ingestion core --- ingestion-core/README.md | 31 -------------- ingestion-core/setup.py | 56 ------------------------- ingestion-core/src/metadata/_version.py | 11 ----- 3 files changed, 98 deletions(-) delete mode 100644 ingestion-core/README.md delete mode 100644 ingestion-core/setup.py delete mode 100644 ingestion-core/src/metadata/_version.py diff --git a/ingestion-core/README.md b/ingestion-core/README.md deleted file mode 100644 index 9080a188d0c1..000000000000 --- a/ingestion-core/README.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -This guide will help you set up OpenMetadata Core Models ---- - -![Python version 3.8+](https://img.shields.io/badge/python-3.8%2B-blue) - -These models are `pydantic` models automatically generated from the -central JSON Schemas that define our APIs and Entities. - -**Prerequisites** - -- Python >= 3.8.x - -### Docs - -Please refer to the documentation here https://docs.open-metadata.org/openmetadata/connectors - -### Contribution - -In order to contribute to this package: - -```bash -cd ingestion-core -python -m virtualenv venv -source venv/bin/activate -python -m pip install ".[dev]" -``` - -> OBS: During development we might need to treat this in a different - virtual environment if we are yet to update the reference to the core - package in `openmetadata-ingestion`. diff --git a/ingestion-core/setup.py b/ingestion-core/setup.py deleted file mode 100644 index c8e5dab39be7..000000000000 --- a/ingestion-core/setup.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2021 Collate -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os - -from setuptools import find_namespace_packages, setup - - -def get_long_description(): - root = os.path.dirname(__file__) - with open(os.path.join(root, "README.md")) as f: - description = f.read() - return description - - -dev = { - "datamodel-code-generator==0.13.0", - "black==21.12b0", - "incremental", - "twine", - "twisted", - "wheel", - "click", - "pydantic[email]==1.9.0", -} - -setup( - name="openmetadata-ingestion-core", - url="https://open-metadata.org/", - author="OpenMetadata Committers", - license="Apache License 2.0", - description="These are the generated Python classes from JSON Schema", - long_description=get_long_description(), - long_description_content_type="text/markdown", - python_requires=">=3.7", - package_dir={"": "src"}, - zip_safe=False, - use_incremental=True, - setup_requires=["incremental"], - install_requires=["incremental"], - project_urls={ - "Documentation": "https://docs.open-metadata.org/", - "Source": "https://github.com/open-metadata/OpenMetadata", - }, - packages=find_namespace_packages(where="./src", exclude=["tests*"]), - extras_require={ - "dev": list(dev), - }, -) diff --git a/ingestion-core/src/metadata/_version.py b/ingestion-core/src/metadata/_version.py deleted file mode 100644 index 140659dbc491..000000000000 --- a/ingestion-core/src/metadata/_version.py +++ /dev/null @@ -1,11 +0,0 @@ -""" -Provides metadata version information. -""" - -# This file is auto-generated! Do not edit! -# Use `python -m incremental.update metadata` to change this file. - -from incremental import Version - -__version__ = Version("metadata", 0, 12, 0, dev=20) -__all__ = ["__version__"] From 0060e3456602090388acd3c22770992781f2e04f Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Sun, 19 Nov 2023 19:43:44 +0100 Subject: [PATCH 11/12] isort --- ingestion/pyproject.toml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ingestion/pyproject.toml b/ingestion/pyproject.toml index 3ad4d21e437e..9b6923a16abf 100644 --- a/ingestion/pyproject.toml +++ b/ingestion/pyproject.toml @@ -114,11 +114,11 @@ all = true extend-exclude = "src/metadata/generated" [tool.isort] -skip = [ - "src/metadata/generated", - "build", - "env", - "../openmetadata-airflow-apis/build" +skip_glob = [ + "src/metadata/generated/*", + "build/*", + "env/*", + "../openmetadata-airflow-apis/build/*" ] profile = "black" indent = " " From 8beb7ac195431fa93c1a2e168099d1a9692d262e Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 20 Nov 2023 09:11:00 +0100 Subject: [PATCH 12/12] Fix makefile help --- Makefile | 14 +++++++------- ingestion/Makefile | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index b9fca04865ff..b345cfdee9e4 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ include ingestion/Makefile .PHONY: help help: - @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[35m%-30s\033[0m %s\n", $$1, $$2}' + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":"}; {printf "\033[35m%-35s\033[0m %s\n", $$2, $$3}' .PHONY: install_e2e_tests install_e2e_tests: ## Install the ingestion module with e2e test dependencies (playwright) @@ -137,7 +137,7 @@ generate-schema-docs: ## Generates markdown files for documenting the JSON Sche #Upgrade release automation scripts below .PHONY: update_all -update_all: ## To update all the release related files run make update_all RELEASE_VERSION=2.2.2 PY_RELEASE_VERSION=2.2.2.2 +update_all: ## To update all the release related files run make update_all RELEASE_VERSION=2.2.2 PY_RELEASE_VERSION=2.2.2.2 @echo "The release version is: $(RELEASE_VERSION)" ; \ echo "The python metadata release version: $(PY_RELEASE_VERSION)" ; \ $(MAKE) update_maven ; \ @@ -150,7 +150,7 @@ update_all: ## To update all the release related files run make update_all RELEA #make update_all RELEASE_VERSION=2.2.2 PY_RELEASE_VERSION=2.2.2.2 .PHONY: update_maven -update_maven: ## To update the common and pom.xml maven version +update_maven: ## To update the common and pom.xml maven version @echo "Updating Maven projects to version $(RELEASE_VERSION)..."; \ mvn versions:set -DnewVersion=$(RELEASE_VERSION) #remove comment and use the below section when want to use this sub module "update_maven" independently to update github actions @@ -158,7 +158,7 @@ update_maven: ## To update the common and pom.xml maven version .PHONY: update_github_action_paths -update_github_action_paths: ## To update the github action ci docker files +update_github_action_paths: ## To update the github action ci docker files @echo "Updating docker github action release version to $(RELEASE_VERSION)... "; \ file_paths="docker/docker-compose-quickstart/Dockerfile \ .github/workflows/docker-openmetadata-db.yml \ @@ -178,7 +178,7 @@ update_github_action_paths: ## To update the github action ci docker files #make update_github_action_paths RELEASE_VERSION=2.2.2 .PHONY: update_python_release_paths -update_python_release_paths: ## To update the setup.py files +update_python_release_paths: ## To update the setup.py files file_paths="ingestion/setup.py \ openmetadata-airflow-apis/setup.py"; \ echo "Updating Python setup file versions to $(PY_RELEASE_VERSION)... "; \ @@ -189,7 +189,7 @@ update_python_release_paths: ## To update the setup.py files #make update_python_release_paths PY_RELEASE_VERSION=2.2.2.2 .PHONY: update_dockerfile_version -update_dockerfile_version: ## To update the dockerfiles version +update_dockerfile_version: ## To update the dockerfiles version @file_paths="docker/docker-compose-ingestion/docker-compose-ingestion.yml \ docker/docker-compose-openmetadata/docker-compose-openmetadata.yml \ docker/docker-compose-quickstart/docker-compose-postgres.yml \ @@ -202,7 +202,7 @@ update_dockerfile_version: ## To update the dockerfiles version #make update_dockerfile_version RELEASE_VERSION=2.2.2 .PHONY: update_ingestion_dockerfile_version -update_ingestion_dockerfile_version: ## To update the ingestion dockerfiles version +update_ingestion_dockerfile_version: ## To update the ingestion dockerfiles version @file_paths="ingestion/Dockerfile \ ingestion/operators/docker/Dockerfile"; \ echo "Updating ingestion dockerfile release version to $(PY_RELEASE_VERSION)... "; \ diff --git a/ingestion/Makefile b/ingestion/Makefile index 72e7f34470cc..41f181406229 100644 --- a/ingestion/Makefile +++ b/ingestion/Makefile @@ -30,7 +30,7 @@ install_apis: ## Install the REST APIs module to the current environment python -m pip install $(ROOT_DIR)/openmetadata-airflow-apis/ .PHONY: lint -lint: ## Run pylint on the Python sources to analyze the codebase +lint: ## Run pylint on the Python sources to analyze the codebase PYTHONPATH="${PYTHONPATH}:$(INGESTION_DIR)/plugins" find $(PY_SOURCE) -path $(PY_SOURCE)/metadata/generated -prune -false -o -type f -name "*.py" | xargs pylint --rcfile=$(INGESTION_DIR)/pyproject.toml .PHONY: precommit_install