diff --git a/metadata-ingestion/tests/integration/dbt/test_dbt.py b/metadata-ingestion/tests/integration/dbt/test_dbt.py index a84538ad2eaa6..2676daeeca45a 100644 --- a/metadata-ingestion/tests/integration/dbt/test_dbt.py +++ b/metadata-ingestion/tests/integration/dbt/test_dbt.py @@ -48,10 +48,10 @@ class DbtTestConfig: sink_config_modifiers: Dict[str, Any] = dataclasses.field(default_factory=dict) def set_paths( - self, - dbt_metadata_uri_prefix: PathLike, - test_resources_dir: PathLike, - tmp_path: PathLike, + self, + dbt_metadata_uri_prefix: PathLike, + test_resources_dir: PathLike, + tmp_path: PathLike, ) -> None: manifest_path = f"{dbt_metadata_uri_prefix}/{self.manifest_file}" catalog_path = f"{dbt_metadata_uri_prefix}/{self.catalog_file}" @@ -236,12 +236,12 @@ def set_paths( @pytest.mark.integration @freeze_time(FROZEN_TIME) def test_dbt_ingest( - dbt_test_config, - test_resources_dir, - pytestconfig, - tmp_path, - mock_time, - requests_mock, + dbt_test_config, + test_resources_dir, + pytestconfig, + tmp_path, + mock_time, + requests_mock, ): config: DbtTestConfig = dbt_test_config test_resources_dir = pytestconfig.rootpath / "tests/integration/dbt" @@ -284,20 +284,20 @@ def test_dbt_ingest( "config_dict, is_success", [ ( - { - "manifest_path": "dbt_manifest.json", - "catalog_path": "dbt_catalog.json", - "target_platform": "postgres", - }, - True, + { + "manifest_path": "dbt_manifest.json", + "catalog_path": "dbt_catalog.json", + "target_platform": "postgres", + }, + True, ), ( - { - "manifest_path": "dbt_manifest.json", - "catalog_path": "dbt_catalog-this-file-does-not-exist.json", - "target_platform": "postgres", - }, - False, + { + "manifest_path": "dbt_manifest.json", + "catalog_path": "dbt_catalog-this-file-does-not-exist.json", + "target_platform": "postgres", + }, + False, ), ], ) @@ -339,9 +339,11 @@ def test_dbt_tests(test_resources_dir, pytestconfig, tmp_path, mock_time, **kwar (test_resources_dir / "jaffle_shop_catalog.json").resolve() ), target_platform="postgres", - run_results_paths=[str( - (test_resources_dir / "jaffle_shop_test_results.json").resolve() - )], + run_results_paths=[ + str( + (test_resources_dir / "jaffle_shop_test_results.json").resolve() + ) + ], ), ), sink=DynamicTypedConfig(type="file", config={"filename": str(output_file)}), @@ -386,8 +388,8 @@ def test_dbt_tests(test_resources_dir, pytestconfig, tmp_path, mock_time, **kwar ) def test_resolve_trino_modified_type(data_type, expected_data_type): assert ( - resolve_trino_modified_type(data_type) - == TRINO_SQL_TYPES_MAP[expected_data_type] + resolve_trino_modified_type(data_type) + == TRINO_SQL_TYPES_MAP[expected_data_type] ) @@ -416,15 +418,15 @@ def test_resolve_trino_modified_type(data_type, expected_data_type): ) def test_resolve_athena_modified_type(data_type, expected_data_type): assert ( - resolve_athena_modified_type(data_type) - == ATHENA_SQL_TYPES_MAP[expected_data_type] + resolve_athena_modified_type(data_type) + == ATHENA_SQL_TYPES_MAP[expected_data_type] ) @pytest.mark.integration @freeze_time(FROZEN_TIME) def test_dbt_tests_only_assertions( - test_resources_dir, pytestconfig, tmp_path, mock_time, **kwargs + test_resources_dir, pytestconfig, tmp_path, mock_time, **kwargs ): # Run the metadata ingestion pipeline. output_file = tmp_path / "test_only_assertions.json" @@ -442,9 +444,11 @@ def test_dbt_tests_only_assertions( (test_resources_dir / "jaffle_shop_catalog.json").resolve() ), target_platform="postgres", - run_results_paths=[str( - (test_resources_dir / "jaffle_shop_test_results.json").resolve() - )], + run_results_paths=[ + str( + (test_resources_dir / "jaffle_shop_test_results.json").resolve() + ) + ], entities_enabled=DBTEntitiesEnabled( test_results=EmitDirective.ONLY ), @@ -458,24 +462,24 @@ def test_dbt_tests_only_assertions( # Verify the output. # No datasets were emitted, and more than 20 events were emitted assert ( - mce_helpers.assert_entity_urn_not_like( - entity_type="dataset", - regex_pattern="urn:li:dataset:\\(urn:li:dataPlatform:dbt", - file=output_file, - ) - > 20 + mce_helpers.assert_entity_urn_not_like( + entity_type="dataset", + regex_pattern="urn:li:dataset:\\(urn:li:dataPlatform:dbt", + file=output_file, + ) + > 20 ) number_of_valid_assertions_in_test_results = 23 assert ( - mce_helpers.assert_entity_urn_like( - entity_type="assertion", regex_pattern="urn:li:assertion:", file=output_file - ) - == number_of_valid_assertions_in_test_results + mce_helpers.assert_entity_urn_like( + entity_type="assertion", regex_pattern="urn:li:assertion:", file=output_file + ) + == number_of_valid_assertions_in_test_results ) # no assertionInfo should be emitted with pytest.raises( - AssertionError, match="Failed to find aspect_name assertionInfo for urns" + AssertionError, match="Failed to find aspect_name assertionInfo for urns" ): mce_helpers.assert_for_each_entity( entity_type="assertion", @@ -486,21 +490,21 @@ def test_dbt_tests_only_assertions( # all assertions must have an assertionRunEvent emitted (except for one assertion) assert ( - mce_helpers.assert_for_each_entity( - entity_type="assertion", - aspect_name="assertionRunEvent", - aspect_field_matcher={}, - file=output_file, - exception_urns=["urn:li:assertion:2ff754df689ea951ed2e12cbe356708f"], - ) - == number_of_valid_assertions_in_test_results + mce_helpers.assert_for_each_entity( + entity_type="assertion", + aspect_name="assertionRunEvent", + aspect_field_matcher={}, + file=output_file, + exception_urns=["urn:li:assertion:2ff754df689ea951ed2e12cbe356708f"], + ) + == number_of_valid_assertions_in_test_results ) @pytest.mark.integration @freeze_time(FROZEN_TIME) def test_dbt_only_test_definitions_and_results( - test_resources_dir, pytestconfig, tmp_path, mock_time, **kwargs + test_resources_dir, pytestconfig, tmp_path, mock_time, **kwargs ): # Run the metadata ingestion pipeline. output_file = tmp_path / "test_only_definitions_and_assertions.json" @@ -518,9 +522,11 @@ def test_dbt_only_test_definitions_and_results( (test_resources_dir / "jaffle_shop_catalog.json").resolve() ), target_platform="postgres", - run_results_paths=[str( - (test_resources_dir / "jaffle_shop_test_results.json").resolve() - )], + run_results_paths=[ + str( + (test_resources_dir / "jaffle_shop_test_results.json").resolve() + ) + ], entities_enabled=DBTEntitiesEnabled( sources=EmitDirective.NO, seeds=EmitDirective.NO, @@ -535,38 +541,38 @@ def test_dbt_only_test_definitions_and_results( pipeline.raise_from_status() # Verify the output. No datasets were emitted assert ( - mce_helpers.assert_entity_urn_not_like( - entity_type="dataset", - regex_pattern="urn:li:dataset:\\(urn:li:dataPlatform:dbt", - file=output_file, - ) - > 20 + mce_helpers.assert_entity_urn_not_like( + entity_type="dataset", + regex_pattern="urn:li:dataset:\\(urn:li:dataPlatform:dbt", + file=output_file, + ) + > 20 ) number_of_assertions = 24 assert ( - mce_helpers.assert_entity_urn_like( - entity_type="assertion", regex_pattern="urn:li:assertion:", file=output_file - ) - == number_of_assertions + mce_helpers.assert_entity_urn_like( + entity_type="assertion", regex_pattern="urn:li:assertion:", file=output_file + ) + == number_of_assertions ) # all assertions must have an assertionInfo emitted assert ( - mce_helpers.assert_for_each_entity( - entity_type="assertion", - aspect_name="assertionInfo", - aspect_field_matcher={}, - file=output_file, - ) - == number_of_assertions + mce_helpers.assert_for_each_entity( + entity_type="assertion", + aspect_name="assertionInfo", + aspect_field_matcher={}, + file=output_file, + ) + == number_of_assertions ) # all assertions must have an assertionRunEvent emitted (except for one assertion) assert ( - mce_helpers.assert_for_each_entity( - entity_type="assertion", - aspect_name="assertionRunEvent", - aspect_field_matcher={}, - file=output_file, - exception_urns=["urn:li:assertion:2ff754df689ea951ed2e12cbe356708f"], - ) - == number_of_assertions - 1 + mce_helpers.assert_for_each_entity( + entity_type="assertion", + aspect_name="assertionRunEvent", + aspect_field_matcher={}, + file=output_file, + exception_urns=["urn:li:assertion:2ff754df689ea951ed2e12cbe356708f"], + ) + == number_of_assertions - 1 )