Backup restore patch

.github/workflows/cypress-integration-tests-mysql.yml

@@ -15,6 +15,7 @@
 name: MySQL Cypress Integration Tests
 
 on:
+ workflow_dispatch:
  push:
  branches:
  - main 

.github/workflows/cypress-integration-tests-postgresql.yml

@@ -15,6 +15,7 @@
 name: PostgreSQL Cypress Integration Tests
 
 on:
+ workflow_dispatch:
  push:
  branches:
  - main 

.github/workflows/docker-openmetadata-db.yml

@@ -31,7 +31,7 @@ jobs:
  steps:
  - name: Check trigger type
  if: ${{ env.input == '' }}
- run: echo "input=1.2.0" >> $GITHUB_ENV
+ run: echo "input=1.1.5" >> $GITHUB_ENV
 
  - name: Check out the Repo
  uses: actions/checkout@v3

.github/workflows/docker-openmetadata-ingestion-base.yml

@@ -31,7 +31,7 @@ jobs:
  steps:
  - name: Check trigger type
  if: ${{ env.input == '' }}
- run: echo "input=1.2.0" >> $GITHUB_ENV
+ run: echo "input=1.1.5" >> $GITHUB_ENV
 
  - name: Check out the Repo
  uses: actions/checkout@v3

.github/workflows/docker-openmetadata-ingestion.yml

@@ -31,7 +31,7 @@ jobs:
  steps:
  - name: Check trigger type
  if: ${{ env.input == '' }}
- run: echo "input=1.2.0" >> $GITHUB_ENV
+ run: echo "input=1.1.5" >> $GITHUB_ENV
 
  - name: Check out the Repo
  uses: actions/checkout@v3

.github/workflows/docker-openmetadata-postgres.yml

@@ -31,7 +31,7 @@ jobs:
  steps:
  - name: Check trigger type
  if: ${{ env.input == '' }}
- run: echo "input=1.2.0" >> $GITHUB_ENV
+ run: echo "input=1.1.5" >> $GITHUB_ENV
 
  - name: Check out the Repo
  uses: actions/checkout@v3

.github/workflows/docker-openmetadata-server.yml

@@ -63,7 +63,7 @@ jobs:
  steps:
  - name: Check trigger type
  id: check_trigger
- run: echo "DOCKER_RELEASE_TAG=1.2.0" >> $GITHUB_OUTPUT
+ run: echo "DOCKER_RELEASE_TAG=1.1.5" >> $GITHUB_OUTPUT
 
  - name: Download application from Artifiact
  uses: actions/download-artifact@v2
@@ -128,7 +128,7 @@ jobs:
  - name: Check trigger type
  id: check_trigger
  if: ${{ env.DOCKER_RELEASE_TAG == '' }} 
- run: echo "DOCKER_RELEASE_TAG=1.2.0" >> $GITHUB_ENV
+ run: echo "DOCKER_RELEASE_TAG=1.1.5" >> $GITHUB_ENV
 
  - name: Check out the Repo
  uses: actions/checkout@v3

.github/workflows/maven-build.yml

@@ -12,6 +12,7 @@
 name: Maven MySQL Tests CI
 
 on:
+ workflow_dispatch:
  push:
  branches:
  - main
@@ -116,7 +117,7 @@ jobs:
  - name: Build with Maven
  env:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- if: ${{ github.event_name == 'push' }}
+ if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
  run: mvn -Dsonar.login=${{ secrets.SONAR_TOKEN }} clean test
 
  - name: Clean Up

.github/workflows/openmetadata-airflow-apis.yml

@@ -34,6 +34,6 @@ jobs:
  run: |
  make install_dev install_apis
  cd openmetadata-airflow-apis; \
- python setup.py install sdist bdist_wheel; \
+ python setup.py build sdist bdist_wheel; \
  twine check dist/*; \
  twine upload dist/* --verbose

Makefile

@@ -246,4 +246,83 @@ generate-schema-docs: ## Generates markdown files for documenting the JSON Sche
  @echo "Generating Schema docs"
  python -m pip install "jsonschema2md"
  python scripts/generate_docs_schemas.py
-
+
+#Upgrade release automation scripts below 
+.PHONY: update_all
+update_all: ## To update all the release related files run make update_all RELEASE_VERSION=2.2.2 PY_RELEASE_VERSION=2.2.2.2
+ @echo "The release version is: $(RELEASE_VERSION)" ; \
+ echo "The python metadata release version: $(PY_RELEASE_VERSION)" ; \
+ $(MAKE) update_maven ; \
+ $(MAKE) update_github_action_paths ; \
+ $(MAKE) update_python_release_paths ; \
+ $(MAKE) update_dockerfile_version ; \
+ $(MAKE) update_ingestion_dockerfile_version ; \
+
+#remove comment and use the below section when want to use this sub module "update_all" independently to update github actions
+#make update_all RELEASE_VERSION=2.2.2 PY_RELEASE_VERSION=2.2.2.2
+
+.PHONY: update_maven
+update_maven: ## To update the common and pom.xml maven version
+ @echo "Updating Maven projects to version $(RELEASE_VERSION)..."; \
+ mvn versions:set -DnewVersion=$(RELEASE_VERSION)
+#remove comment and use the below section when want to use this sub module "update_maven" independently to update github actions
+#make update_maven RELEASE_VERSION=2.2.2
+
+
+.PHONY: update_github_action_paths
+update_github_action_paths: ## To update the github action ci docker files
+ @echo "Updating docker github action release version to $(RELEASE_VERSION)... "; \
+ file_paths="docker/docker-compose-quickstart/Dockerfile \
+ .github/workflows/docker-openmetadata-db.yml \
+ .github/workflows/docker-openmetadata-ingestion-base.yml \
+ .github/workflows/docker-openmetadata-ingestion.yml \
+ .github/workflows/docker-openmetadata-postgres.yml \
+ .github/workflows/docker-openmetadata-server.yml"; \
+ for file_path in $$file_paths; do \
+ python3 scripts/update_version.py 1 $$file_path -s $(RELEASE_VERSION) ; \
+ done; \
+ file_paths1="docker/docker-compose-quickstart/Dockerfile"; \
+ for file_path in $$file_paths1; do \
+ python3 scripts/update_version.py 4 $$file_path -s $(RELEASE_VERSION) ; \
+ done
+
+#remove comment and use the below section when want to use this sub module "update_github_action_paths" independently to update github actions
+#make update_github_action_paths RELEASE_VERSION=2.2.2
+
+.PHONY: update_python_release_paths
+update_python_release_paths: ## To update the setup.py files
+ file_paths="ingestion/setup.py \
+ openmetadata-airflow-apis/setup.py"; \
+ echo "Updating Python setup file versions to $(PY_RELEASE_VERSION)... "; \
+ for file_path in $$file_paths; do \
+ python3 scripts/update_version.py 2 $$file_path -s $(PY_RELEASE_VERSION) ; \
+ done
+# Commented section for independent usage of the module update_python_release_paths independently to update github actions
+#make update_python_release_paths PY_RELEASE_VERSION=2.2.2.2
+
+.PHONY: update_dockerfile_version
+update_dockerfile_version: ## To update the dockerfiles version
+ @file_paths="docker/docker-compose-ingestion/docker-compose-ingestion-postgres.yml \
+ docker/docker-compose-ingestion/docker-compose-ingestion.yml \
+ docker/docker-compose-openmetadata/docker-compose-openmetadata.yml \
+ docker/docker-compose-quickstart/docker-compose-postgres.yml \
+ docker/docker-compose-quickstart/docker-compose.yml"; \
+ echo "Updating docker github action release version to $(RELEASE_VERSION)... "; \
+ for file_path in $$file_paths; do \
+ python3 scripts/update_version.py 3 $$file_path -s $(RELEASE_VERSION) ; \
+ done
+#remove comment and use the below section when want to use this sub module "update_dockerfile_version" independently to update github actions
+#make update_dockerfile_version RELEASE_VERSION=2.2.2
+
+.PHONY: update_ingestion_dockerfile_version
+update_ingestion_dockerfile_version: ## To update the ingestion dockerfiles version
+ @file_paths="ingestion/Dockerfile \
+ ingestion/operators/docker/Dockerfile"; \
+ echo "Updating ingestion dockerfile release version to $(PY_RELEASE_VERSION)... "; \
+ for file_path in $$file_paths; do \
+ python3 scripts/update_version.py 4 $$file_path -s $(PY_RELEASE_VERSION) ; \
+ done
+#remove comment and use the below section when want to use this sub module "update_ingestion_dockerfile_version" independently to update github actions
+#make update_ingestion_dockerfile_version PY_RELEASE_VERSION=2.2.2.2
+
+#Upgrade release automation scripts above

bootstrap/sql/migrations/native/1.1.2/mysql/schemaChanges.sql

@@ -20,4 +20,4 @@ SET json = JSON_INSERT(
  '$.connection.config.authType.password',
  JSON_EXTRACT(json, '$.connection.config.password'))
 where serviceType = 'Trino'
- AND JSON_EXTRACT(json, '$.connection.config.password') IS NOT NULL;
+ AND JSON_EXTRACT(json, '$.connection.config.password') IS NOT NULL;

bootstrap/sql/migrations/native/1.1.5/mysql/postDataMigrationSQLScript.sql

@@ -0,0 +1,50 @@
+START TRANSACTION;
+-- We'll rank all the runs (timestamps) for every day, and delete all the data but the most recent one.
+DELETE FROM report_data_time_series WHERE JSON_EXTRACT(json, '$.id') IN (
+ select ids FROM (
+ SELECT 
+ (json ->> '$.id') AS ids,
+ DENSE_RANK() OVER(PARTITION BY `date` ORDER BY `timestamp` DESC) as denseRank
+ FROM (
+ SELECT
+ *
+ FROM report_data_time_series rdts
+ WHERE json ->> '$.reportDataType' = 'WebAnalyticEntityViewReportData'
+ ) duplicates
+ ORDER BY `date` DESC, `timestamp` DESC
+ ) as dense_ranked
+ WHERE denseRank != 1
+);
+
+DELETE FROM report_data_time_series WHERE JSON_EXTRACT(json, '$.id') IN (
+ select ids FROM (
+ SELECT 
+ (json ->> '$.id') AS ids,
+ DENSE_RANK() OVER(PARTITION BY `date` ORDER BY `timestamp` DESC) as denseRank
+ FROM (
+ SELECT
+ *
+ FROM report_data_time_series rdts
+ WHERE json ->> '$.reportDataType' = 'EntityReportData'
+ ) duplicates
+ ORDER BY `date` DESC, `timestamp` DESC
+ ) as dense_ranked
+ WHERE denseRank != 1
+);
+
+DELETE FROM report_data_time_series WHERE JSON_EXTRACT(json, '$.id') IN (
+ select ids FROM (
+ SELECT 
+ (json ->> '$.id') AS ids,
+ DENSE_RANK() OVER(PARTITION BY `date` ORDER BY `timestamp` DESC) as denseRank
+ FROM (
+ SELECT
+ *
+ FROM report_data_time_series rdts
+ WHERE json ->> '$.reportDataType' = 'WebAnalyticUserActivityReportData'
+ ) duplicates
+ ORDER BY `date` DESC, `timestamp` DESC
+ ) as dense_ranked
+ WHERE denseRank != 1
+);
+COMMIT;

bootstrap/sql/migrations/native/1.1.3/mysql/schemaChanges.sql → bootstrap/sql/migrations/native/1.1.5/mysql/schemaChanges.sql

@@ -1,9 +1,83 @@
+-- Update table and column profile timestamps to be in milliseconds
+UPDATE entity_extension_time_series
+ SET json = JSON_INSERT(
+ JSON_REMOVE(json, '$.timestamp'),
+ '$.timestamp',
+ JSON_EXTRACT(json, '$.timestamp') * 1000
+ )
+WHERE
+ extension in ('table.tableProfile', 'table.columnProfile', 'testCase.testCaseResult');
+;
+
+START TRANSACTION;
+-- Create report data time series table and move data from entity_extension_time_series
+CREATE TABLE IF NOT EXISTS report_data_time_series (
+ entityFQNHash VARCHAR(768) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
+ extension VARCHAR(256) NOT NULL,
+ jsonSchema VARCHAR(256) NOT NULL,
+ json JSON NOT NULL,
+ timestamp BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.timestamp') NOT NULL,
+ date DATE GENERATED ALWAYS AS (FROM_UNIXTIME((json ->> '$.timestamp') DIV 1000)) NOT NULL,
+ INDEX report_data_time_series_point_ts (timestamp),
+ INDEX report_data_time_series_date (date)
+);
+
+INSERT INTO report_data_time_series (entityFQNHash,extension,jsonSchema,json)
+SELECT entityFQNHash, extension, jsonSchema, json
+FROM entity_extension_time_series WHERE extension = 'reportData.reportDataResult';
+
+DELETE FROM entity_extension_time_series
+WHERE extension = 'reportData.reportDataResult';
+COMMIT;
+
+START TRANSACTION;
+-- Create profiler data time series table and move data from entity_extension_time_series
+CREATE TABLE IF NOT EXISTS profiler_data_time_series (
+ entityFQNHash VARCHAR(768) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
+ extension VARCHAR(256) NOT NULL,
+ jsonSchema VARCHAR(256) NOT NULL,
+ json JSON NOT NULL,
+ operation VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.operation') NULL,
+ timestamp BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.timestamp') NOT NULL,
+ UNIQUE profiler_data_time_series_unique_hash_extension_ts (entityFQNHash, extension, operation, timestamp),
+ INDEX profiler_data_time_series_combined_id_ts (extension, timestamp)
+);
+
+INSERT INTO profiler_data_time_series (entityFQNHash,extension,jsonSchema,json)
+SELECT entityFQNHash, extension, jsonSchema, json
+FROM entity_extension_time_series
+WHERE extension IN ('table.columnProfile', 'table.tableProfile', 'table.systemProfile');
+
+DELETE FROM entity_extension_time_series
+WHERE extension IN ('table.columnProfile', 'table.tableProfile', 'table.systemProfile');
+COMMIT;
+
+START TRANSACTION;
+-- Create data quality data time series table and move data from entity_extension_time_series
+CREATE TABLE IF NOT EXISTS data_quality_data_time_series (
+ entityFQNHash VARCHAR(768) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
+ extension VARCHAR(256) NOT NULL,
+ jsonSchema VARCHAR(256) NOT NULL,
+ json JSON NOT NULL,
+ timestamp BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.timestamp') NOT NULL,
+ UNIQUE data_quality_data_time_series_unique_hash_extension_ts (entityFQNHash, extension, timestamp),
+ INDEX data_quality_data_time_series_combined_id_ts (extension, timestamp)
+);
+
+INSERT INTO data_quality_data_time_series (entityFQNHash,extension,jsonSchema,json)
+SELECT entityFQNHash, extension, jsonSchema, json
+FROM entity_extension_time_series
+WHERE extension = 'testCase.testCaseResult';
+
+DELETE FROM entity_extension_time_series
+WHERE extension = 'testCase.testCaseResult';
+COMMIT;
+
 ALTER TABLE automations_workflow MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin,MODIFY COLUMN workflowType VARCHAR(256) COLLATE ascii_bin, MODIFY COLUMN status VARCHAR(256) COLLATE ascii_bin;
-ALTER TABLE entity_extension MODIFY COLUMN extension VARCHAR(256) COLLATE ascii_bin;
 ALTER TABLE entity_extension_time_series MODIFY COLUMN entityFQNHash VARCHAR(768) COLLATE ascii_bin, MODIFY COLUMN jsonSchema VARCHAR(50) COLLATE ascii_bin, MODIFY COLUMN extension VARCHAR(100) COLLATE ascii_bin,
  ADD CONSTRAINT entity_extension_time_series_constraint UNIQUE (entityFQNHash, extension, timestamp);
 ALTER TABLE field_relationship MODIFY COLUMN fromFQNHash VARCHAR(768) COLLATE ascii_bin, MODIFY COLUMN toFQNHash VARCHAR(768) COLLATE ascii_bin;
-ALTER TABLE thread_entity MODIFY COLUMN entityLink VARCHAR(3072) GENERATED ALWAYS AS (json ->> '$.about') NOT NULL, MODIFY COLUMN createdBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.createdBy') STORED NOT NULL COLLATE ascii_bin;
+ALTER TABLE thread_entity MODIFY COLUMN entityLink VARCHAR(3072) GENERATED ALWAYS AS (json ->> '$.about') NOT NULL;
 ALTER TABLE event_subscription_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;
 ALTER TABLE ingestion_pipeline_entity MODIFY COLUMN fqnHash VARCHAR(768) COLLATE ascii_bin;
 ALTER TABLE bot_entity MODIFY COLUMN nameHash VARCHAR(256) COLLATE ascii_bin;

bootstrap/sql/migrations/native/1.1.5/postgres/postDataMigrationSQLScript.sql

@@ -0,0 +1,53 @@
+BEGIN;
+-- We'll rank all the runs (timestamps) for every day, and delete all the data but the most recent one.
+DELETE FROM report_data_time_series WHERE (json ->> 'id') IN (
+ select ids FROM (
+ SELECT 
+ (json ->> 'id') AS ids,
+ DENSE_RANK() OVER(PARTITION BY date ORDER BY timestamp DESC) as denseRank
+ FROM (
+ SELECT
+ *,
+ DATE(TO_TIMESTAMP((json ->> 'timestamp')::bigint/1000)) as date
+ FROM report_data_time_series rdts
+ WHERE json ->> 'reportDataType' = 'WebAnalyticEntityViewReportData'
+ ) duplicates
+ ORDER BY date DESC, timestamp DESC
+ ) as dense_ranked
+ WHERE denseRank != 1
+);
+
+DELETE FROM report_data_time_series WHERE (json ->> 'id') IN (
+ select ids FROM (
+ SELECT 
+ (json ->> 'id') AS ids,
+ DENSE_RANK() OVER(PARTITION BY date ORDER BY timestamp DESC) as denseRank
+ FROM (
+ SELECT
+ *,
+ DATE(TO_TIMESTAMP((json ->> 'timestamp')::bigint/1000)) as date
+ FROM report_data_time_series rdts
+ WHERE json ->> 'reportDataType' = 'EntityReportData'
+ ) duplicates
+ ORDER BY date DESC, timestamp DESC
+ ) as dense_ranked
+ WHERE denseRank != 1
+);
+
+DELETE FROM report_data_time_series WHERE (json ->> 'id') IN (
+ select ids FROM (
+ SELECT 
+ (json ->> 'id') AS ids,
+ DENSE_RANK() OVER(PARTITION BY date ORDER BY timestamp DESC) as denseRank
+ FROM (
+ SELECT
+ *,
+ DATE(TO_TIMESTAMP((json ->> 'timestamp')::bigint/1000)) as date
+ FROM report_data_time_series rdts
+ WHERE json ->> 'reportDataType' = 'WebAnalyticUserActivityReportData'
+ ) duplicates
+ ORDER BY date DESC, timestamp DESC
+ ) as dense_ranked
+ WHERE denseRank != 1
+);
+COMMIT;