Merge branch 'main' into add-tox

open-metadata · Sep 19, 2024 · e440cbf · e440cbf
2 parents 3a4b4b6 + 760b8eb
commit e440cbf
Show file tree

Hide file tree

Showing 62 changed files with 2,872 additions and 1,438 deletions.
diff --git a/.github/workflows/playwright-mysql-e2e-skip.yml b/.github/workflows/playwright-mysql-e2e-skip.yml
@@ -21,7 +21,6 @@ on:
       - opened
       - synchronize
       - reopened
-      - ready_for_review
     paths:
       - openmetadata-docs/**
       - .github/**

diff --git a/.github/workflows/playwright-mysql-e2e.yml b/.github/workflows/playwright-mysql-e2e.yml
@@ -21,7 +21,6 @@ on:
       - opened
       - synchronize
       - reopened
-      - ready_for_review
     paths-ignore:
       - openmetadata-docs/**
       - .github/**

diff --git a/.github/workflows/playwright-postgresql-e2e-skip.yml b/.github/workflows/playwright-postgresql-e2e-skip.yml
@@ -21,7 +21,6 @@ on:
       - opened
       - synchronize
       - reopened
-      - ready_for_review
     paths:
       - openmetadata-docs/**
       - .github/**

diff --git a/.github/workflows/playwright-postgresql-e2e.yml b/.github/workflows/playwright-postgresql-e2e.yml
@@ -21,7 +21,6 @@ on:
       - opened
       - synchronize
       - reopened
-      - ready_for_review
     paths-ignore:
       - openmetadata-docs/**
       - .github/**

diff --git a/bootstrap/sql/migrations/native/1.6.0/mysql/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.6.0/mysql/postDataMigrationSQLScript.sql
@@ -13,4 +13,18 @@ ADD COLUMN id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
 ADD CONSTRAINT UNIQUE (id);
 
 -- Create index on id column
-CREATE INDEX data_quality_data_time_series_id_index ON data_quality_data_time_series (id);
+CREATE INDEX data_quality_data_time_series_id_index ON data_quality_data_time_series (id);
+
+-- Remove VIRTUAL status column from test_case table and remove
+-- testCaseResult state from testCase; fetch from search repo.
+ALTER TABLE test_case DROP COLUMN status;
+UPDATE test_case SET json = JSON_SET(json, '$.testCaseStatus', JSON_EXTRACT(json, '$.testCaseResult.testCaseStatus'));
+ALTER TABLE test_case ADD COLUMN status VARCHAR(56) GENERATED ALWAYS AS (JSON_UNQUOTE(JSON_EXTRACT(json, '$.testCaseStatus'))) STORED;
+
+
+-- Remove test case result states
+UPDATE test_suite
+SET json = JSON_REMOVE(json, '$.testCaseResultSummary');
+
+UPDATE test_case
+SET json = JSON_REMOVE(json, '$.testCaseResult');
diff --git a/bootstrap/sql/migrations/native/1.6.0/postgres/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.6.0/postgres/postDataMigrationSQLScript.sql
@@ -7,6 +7,7 @@ SET json = jsonb_set(
 FROM test_case tc
 WHERE dqdts.entityfqnHash = tc.fqnHash;
 
+
 -- Add id column to data_quality_data_time_series table
 -- after we have added the id values to the records
 ALTER TABLE data_quality_data_time_series
@@ -15,3 +16,17 @@ ADD CONSTRAINT id_unique UNIQUE (id);
 
 -- Create index on id column
 CREATE INDEX IF NOT EXISTS  data_quality_data_time_series_id_index  ON data_quality_data_time_series (id);
+
+-- Remove VIRTUAL status column from test_case table and remove
+-- testCaseResult state from testCase; fetch from search repo.
+ALTER TABLE test_case DROP COLUMN status;
+UPDATE test_case SET json = jsonb_set(json, '{testCaseStatus}', json->'testCaseResult'->'testCaseStatus');
+ALTER TABLE test_case ADD COLUMN status VARCHAR(56) GENERATED ALWAYS AS (json ->> 'testCaseStatus') STORED NULL;
+
+
+-- Remove test case result states
+UPDATE test_suite
+SET json = json - 'testCaseResultSummary';
+
+UPDATE test_case
+SET json = json - 'testCaseResult';
diff --git a/ingestion/setup.py b/ingestion/setup.py
@@ -214,7 +214,10 @@
         "s3fs[boto3]",
         *COMMONS["datalake"],
     },
-    "deltalake": {"delta-spark<=2.3.0", "deltalake~=0.17"},
+    "deltalake": {
+        "delta-spark<=2.3.0",
+        "deltalake~=0.17,<0.20",
+    },  # TODO: remove pinning to under 0.20 after https://github.com/open-metadata/OpenMetadata/issues/17909
     "deltalake-storage": {"deltalake~=0.17"},
     "deltalake-spark": {"delta-spark<=2.3.0"},
     "domo": {VERSIONS["pydomo"]},

diff --git a/ingestion/tests/integration/data_quality/test_data_diff.py b/ingestion/tests/integration/data_quality/test_data_diff.py
@@ -1,4 +1,5 @@
 import sys
+from datetime import datetime
 
 import pytest
 from pydantic import BaseModel
@@ -67,6 +68,7 @@ def __init__(self, *args, **kwargs):
                 ),
                 "POSTGRES_SERVICE.dvdrental.public.customer",
                 TestCaseResult(
+                    timestamp=int(datetime.now().timestamp() * 1000),
                     testCaseStatus=TestCaseStatus.Success,
                     failedRows=0,
                     passedRows=599,
@@ -85,6 +87,7 @@ def __init__(self, *args, **kwargs):
                 ),
                 "POSTGRES_SERVICE.dvdrental.public.changed_customer",
                 TestCaseResult(
+                    timestamp=int(datetime.now().timestamp() * 1000),
                     testCaseStatus=TestCaseStatus.Failed,
                     failedRows=321,
                     passedRows=278,
@@ -99,6 +102,7 @@ def __init__(self, *args, **kwargs):
                 ),
                 "POSTGRES_SERVICE.dvdrental.public.changed_customer",
                 TestCaseResult(
+                    timestamp=int(datetime.now().timestamp() * 1000),
                     testCaseStatus=TestCaseStatus.Failed,
                     failedRows=321,
                     passedRows=278,
@@ -114,6 +118,7 @@ def __init__(self, *args, **kwargs):
                 ),
                 "POSTGRES_SERVICE.dvdrental.public.changed_customer",
                 TestCaseResult(
+                    timestamp=int(datetime.now().timestamp() * 1000),
                     testCaseStatus=TestCaseStatus.Success,
                     failedRows=321,
                 ),
@@ -128,6 +133,7 @@ def __init__(self, *args, **kwargs):
                 ),
                 "POSTGRES_SERVICE.dvdrental.public.changed_customer",
                 TestCaseResult(
+                    timestamp=int(datetime.now().timestamp() * 1000),
                     testCaseStatus=TestCaseStatus.Failed,
                     failedRows=321,
                 ),
@@ -146,6 +152,7 @@ def __init__(self, *args, **kwargs):
                 ),
                 "POSTGRES_SERVICE.dvdrental.public.changed_customer",
                 TestCaseResult(
+                    timestamp=int(datetime.now().timestamp() * 1000),
                     testCaseStatus=TestCaseStatus.Success,
                 ),
             ),
@@ -158,6 +165,7 @@ def __init__(self, *args, **kwargs):
                 ),
                 "POSTGRES_SERVICE.dvdrental.public.customer_without_first_name",
                 TestCaseResult(
+                    timestamp=int(datetime.now().timestamp() * 1000),
                     testCaseStatus=TestCaseStatus.Failed,
                     testResultValue=[
                         TestResultValue(name="removedColumns", value="1"),
@@ -179,6 +187,7 @@ def __init__(self, *args, **kwargs):
                 ),
                 "POSTGRES_SERVICE.dvdrental.public.customer_without_first_name",
                 TestCaseResult(
+                    timestamp=int(datetime.now().timestamp() * 1000),
                     testCaseStatus=TestCaseStatus.Success,
                 ),
             ),
@@ -208,6 +217,7 @@ def __init__(self, *args, **kwargs):
                 ),
                 "MYSQL_SERVICE.default.test.customer",
                 TestCaseResult(
+                    timestamp=int(datetime.now().timestamp() * 1000),
                     testCaseStatus=TestCaseStatus.Success,
                 ),
             ),
@@ -220,6 +230,7 @@ def __init__(self, *args, **kwargs):
                 ),
                 "MYSQL_SERVICE.default.test.changed_customer",
                 TestCaseResult(
+                    timestamp=int(datetime.now().timestamp() * 1000),
                     testCaseStatus=TestCaseStatus.Failed,
                 ),
             ),
@@ -290,6 +301,9 @@ def test_happy_paths(
         fields=["*"],
     )
     assert "ERROR: Unexpected error" not in test_case_entity.testCaseResult.result
+    parameters.expected.timestamp = (
+        test_case_entity.testCaseResult.timestamp
+    )  # timestamp is not deterministic
     assert_equal_pydantic_objects(parameters.expected, test_case_entity.testCaseResult)
 
 
@@ -313,6 +327,7 @@ def test_happy_paths(
                 ],
             ),
             TestCaseResult(
+                timestamp=int(datetime.now().timestamp() * 1000),
                 testCaseStatus=TestCaseStatus.Aborted,
                 result="Unsupported dialect in param table2.serviceUrl: mongodb",
             ),
@@ -331,6 +346,7 @@ def test_happy_paths(
                 ],
             ),
             TestCaseResult(
+                timestamp=int(datetime.now().timestamp() * 1000),
                 testCaseStatus=TestCaseStatus.Failed,
                 result="Tables have 1 different columns:"
                 "\n  Changed columns:"
@@ -405,6 +421,9 @@ def test_error_paths(
     test_case_entity: TestCase = metadata.get_or_create_test_case(
         f"{table1.fullyQualifiedName.root}.{parameters.name}"
     )
+    expected.timestamp = (
+        test_case_entity.testCaseResult.timestamp
+    )  # timestamp is not deterministic
     assert_equal_pydantic_objects(expected, test_case_entity.testCaseResult)
 
 

diff --git a/ingestion/tests/integration/mysql/test_data_quality.py b/ingestion/tests/integration/mysql/test_data_quality.py
@@ -70,6 +70,7 @@ class TestColumnParameter:
                 ],
             ),
             expected_result=TestCaseResult(
+                timestamp=int(datetime.now().timestamp() * 1000),
                 testCaseStatus=TestCaseStatus.Failed,
             ),
         ),
@@ -86,6 +87,7 @@ class TestColumnParameter:
                 ],
             ),
             expected_result=TestCaseResult(
+                timestamp=int(datetime.now().timestamp() * 1000),
                 testCaseStatus=TestCaseStatus.Failed,
             ),
         ),
@@ -101,6 +103,7 @@ class TestColumnParameter:
                 ],
             ),
             expected_result=TestCaseResult(
+                timestamp=int(datetime.now().timestamp() * 1000),
                 testCaseStatus=TestCaseStatus.Failed,
             ),
         ),
@@ -116,6 +119,7 @@ class TestColumnParameter:
                 ],
             ),
             expected_result=TestCaseResult(
+                timestamp=int(datetime.now().timestamp() * 1000),
                 testCaseStatus=TestCaseStatus.Success,
             ),
         ),
@@ -131,6 +135,7 @@ class TestColumnParameter:
                 ],
             ),
             expected_result=TestCaseResult(
+                timestamp=int(datetime.now().timestamp() * 1000),
                 testCaseStatus=TestCaseStatus.Success,
             ),
         ),
@@ -144,6 +149,7 @@ class TestColumnParameter:
                 parameterValues=[],
             ),
             expected_result=TestCaseResult(
+                timestamp=int(datetime.now().timestamp() * 1000),
                 testCaseStatus=TestCaseStatus.Success,
             ),
         ),
@@ -162,6 +168,7 @@ class TestColumnParameter:
                 ],
             ),
             expected_result=TestCaseResult(
+                timestamp=int(datetime.now().timestamp() * 1000),
                 testCaseStatus=TestCaseStatus.Success,
             ),
         ),
@@ -180,6 +187,7 @@ class TestColumnParameter:
                 ],
             ),
             expected_result=TestCaseResult(
+                timestamp=int(datetime.now().timestamp() * 1000),
                 testCaseStatus=TestCaseStatus.Failed,
             ),
         ),
@@ -216,6 +224,9 @@ def test_column_test_cases(
         nullable=False,
     )
     cleanup_fqns(TestCase, test_case.fullyQualifiedName.root)
+    parameters.expected_result.timestamp = (
+        test_case.testCaseResult.timestamp
+    )  # timestamp is not deterministic
     assert_equal_pydantic_objects(
         parameters.expected_result,
         test_case.testCaseResult,

diff --git a/ingestion/tests/integration/sources/database/delta_lake/test_deltalake_storage.py b/ingestion/tests/integration/sources/database/delta_lake/test_deltalake_storage.py
@@ -136,6 +136,7 @@ def ingest(metadata, service, create_data):
 
     ingestion = MetadataWorkflow.create(workflow_config)
     ingestion.execute()
+    ingestion.raise_from_status()
     return
 
 

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java
@@ -124,6 +124,8 @@ public final class Entity {
 
   public static final String FIELD_DISABLED = "disabled";
 
+  public static final String FIELD_TEST_SUITES = "testSuites";
+
   //
   // Service entities
   //
@@ -215,7 +217,7 @@ public final class Entity {
   // Time series entities
   public static final String ENTITY_REPORT_DATA = "entityReportData";
   public static final String TEST_CASE_RESOLUTION_STATUS = "testCaseResolutionStatus";
-  public static final String TEST_CASE_RESULTS = "testCaseResult";
+  public static final String TEST_CASE_RESULT = "testCaseResult";
   public static final String WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA =
       "webAnalyticEntityViewReportData";
   public static final String WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA =