Minor: Fix e2e count mismatch (#14352)

* Minor: Fix e2e count mismatch * fix dbt test * fix pyformat * use > instead of ==
open-metadata · Dec 12, 2023 · f53fcf9 · f53fcf9
1 parent 2337e23
commit f53fcf9
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 6 deletions.
diff --git a/ingestion/tests/cli_e2e/database/bigquery_multiple_project/bigquery_multiple_project.yaml b/ingestion/tests/cli_e2e/database/bigquery_multiple_project/bigquery_multiple_project.yaml
@@ -21,6 +21,10 @@ source:
   sourceConfig:
     config:
       type: DatabaseMetadata
+      schemaFilterPattern:
+        includes:
+        - do_not_touch
+        - exclude_me
 sink:
   type: metadata-rest
   config: {}

diff --git a/ingestion/tests/cli_e2e/test_cli_bigquery_multiple_project.py b/ingestion/tests/cli_e2e/test_cli_bigquery_multiple_project.py
@@ -14,6 +14,8 @@
 """
 from typing import List
 
+from metadata.ingestion.api.status import Status
+
 from .base.e2e_types import E2EType
 from .common.test_cli_db import CliCommonDB
 from .common_e2e_sqa_mixins import SQACommonMethods
@@ -102,15 +104,15 @@ def expected_filtered_schema_excludes() -> int:
 
     @staticmethod
     def expected_filtered_table_includes() -> int:
-        return 9
+        return 10
 
     @staticmethod
     def expected_filtered_table_excludes() -> int:
-        return 10
+        return 11
 
     @staticmethod
     def expected_filtered_mix() -> int:
-        return 9
+        return 10
 
     @staticmethod
     def delete_queries() -> List[str]:
@@ -128,6 +130,17 @@ def update_queries() -> List[str]:
             """,
         ]
 
+    def assert_for_vanilla_ingestion(
+        self, source_status: Status, sink_status: Status
+    ) -> None:
+        self.assertTrue(len(source_status.failures) == 0)
+        self.assertTrue(len(source_status.warnings) == 0)
+        self.assertTrue(len(source_status.filtered) >= 9)
+        self.assertTrue(len(source_status.records) >= self.expected_tables())
+        self.assertTrue(len(sink_status.failures) == 0)
+        self.assertTrue(len(sink_status.warnings) == 0)
+        self.assertTrue(len(sink_status.records) > self.expected_tables())
+
     def test_create_table_with_profiler(self) -> None:
         # delete table in case it exists
         self.delete_table_and_view()

diff --git a/ingestion/tests/cli_e2e/test_cli_dbt_redshift.py b/ingestion/tests/cli_e2e/test_cli_dbt_redshift.py
@@ -67,7 +67,7 @@ def assert_for_vanilla_ingestion(
     ) -> None:
         self.assertTrue(len(source_status.failures) == 0)
         self.assertTrue(len(source_status.warnings) == 0)
-        self.assertTrue(len(source_status.filtered) == 9)
+        self.assertTrue(len(source_status.filtered) >= 10)
         self.assertTrue(len(source_status.records) >= self.expected_tables())
         self.assertTrue(len(sink_status.failures) == 0)
         self.assertTrue(len(sink_status.warnings) == 0)

diff --git a/ingestion/tests/cli_e2e/test_cli_mysql.py b/ingestion/tests/cli_e2e/test_cli_mysql.py
@@ -117,12 +117,12 @@ def expected_filtered_schema_excludes() -> int:
 
     @staticmethod
     def expected_filtered_table_includes() -> int:
-        return 63
+        return 64
 
     @staticmethod
     def expected_filtered_table_excludes() -> int:
         return 4
 
     @staticmethod
     def expected_filtered_mix() -> int:
-        return 63
+        return 64