Skip to content

Commit

Permalink
Update test_dbt.py
Browse files Browse the repository at this point in the history
  • Loading branch information
@acrylJonny
acrylJonny committed Aug 9, 2024
1 parent c16d8e3 commit 9fc8a6c
Showing 1 changed file with 87 additions and 81 deletions.
168 changes: 87 additions & 81 deletions metadata-ingestion/tests/integration/dbt/test_dbt.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,10 @@ class DbtTestConfig:
sink_config_modifiers: Dict[str, Any] = dataclasses.field(default_factory=dict)

def set_paths(
self,
dbt_metadata_uri_prefix: PathLike,
test_resources_dir: PathLike,
tmp_path: PathLike,
self,
dbt_metadata_uri_prefix: PathLike,
test_resources_dir: PathLike,
tmp_path: PathLike,
) -> None:
manifest_path = f"{dbt_metadata_uri_prefix}/{self.manifest_file}"
catalog_path = f"{dbt_metadata_uri_prefix}/{self.catalog_file}"
Expand Down Expand Up @@ -236,12 +236,12 @@ def set_paths(
@pytest.mark.integration
@freeze_time(FROZEN_TIME)
def test_dbt_ingest(
dbt_test_config,
test_resources_dir,
pytestconfig,
tmp_path,
mock_time,
requests_mock,
dbt_test_config,
test_resources_dir,
pytestconfig,
tmp_path,
mock_time,
requests_mock,
):
config: DbtTestConfig = dbt_test_config
test_resources_dir = pytestconfig.rootpath / "tests/integration/dbt"
Expand Down Expand Up @@ -284,20 +284,20 @@ def test_dbt_ingest(
"config_dict, is_success",
[
(
{
"manifest_path": "dbt_manifest.json",
"catalog_path": "dbt_catalog.json",
"target_platform": "postgres",
},
True,
{
"manifest_path": "dbt_manifest.json",
"catalog_path": "dbt_catalog.json",
"target_platform": "postgres",
},
True,
),
(
{
"manifest_path": "dbt_manifest.json",
"catalog_path": "dbt_catalog-this-file-does-not-exist.json",
"target_platform": "postgres",
},
False,
{
"manifest_path": "dbt_manifest.json",
"catalog_path": "dbt_catalog-this-file-does-not-exist.json",
"target_platform": "postgres",
},
False,
),
],
)
Expand Down Expand Up @@ -339,9 +339,11 @@ def test_dbt_tests(test_resources_dir, pytestconfig, tmp_path, mock_time, **kwar
(test_resources_dir / "jaffle_shop_catalog.json").resolve()
),
target_platform="postgres",
run_results_paths=[str(
(test_resources_dir / "jaffle_shop_test_results.json").resolve()
)],
run_results_paths=[
str(
(test_resources_dir / "jaffle_shop_test_results.json").resolve()
)
],
),
),
sink=DynamicTypedConfig(type="file", config={"filename": str(output_file)}),
Expand Down Expand Up @@ -386,8 +388,8 @@ def test_dbt_tests(test_resources_dir, pytestconfig, tmp_path, mock_time, **kwar
)
def test_resolve_trino_modified_type(data_type, expected_data_type):
assert (
resolve_trino_modified_type(data_type)
== TRINO_SQL_TYPES_MAP[expected_data_type]
resolve_trino_modified_type(data_type)
== TRINO_SQL_TYPES_MAP[expected_data_type]
)


Expand Down Expand Up @@ -416,15 +418,15 @@ def test_resolve_trino_modified_type(data_type, expected_data_type):
)
def test_resolve_athena_modified_type(data_type, expected_data_type):
assert (
resolve_athena_modified_type(data_type)
== ATHENA_SQL_TYPES_MAP[expected_data_type]
resolve_athena_modified_type(data_type)
== ATHENA_SQL_TYPES_MAP[expected_data_type]
)


@pytest.mark.integration
@freeze_time(FROZEN_TIME)
def test_dbt_tests_only_assertions(
test_resources_dir, pytestconfig, tmp_path, mock_time, **kwargs
test_resources_dir, pytestconfig, tmp_path, mock_time, **kwargs
):
# Run the metadata ingestion pipeline.
output_file = tmp_path / "test_only_assertions.json"
Expand All @@ -442,9 +444,11 @@ def test_dbt_tests_only_assertions(
(test_resources_dir / "jaffle_shop_catalog.json").resolve()
),
target_platform="postgres",
run_results_paths=[str(
(test_resources_dir / "jaffle_shop_test_results.json").resolve()
)],
run_results_paths=[
str(
(test_resources_dir / "jaffle_shop_test_results.json").resolve()
)
],
entities_enabled=DBTEntitiesEnabled(
test_results=EmitDirective.ONLY
),
Expand All @@ -458,24 +462,24 @@ def test_dbt_tests_only_assertions(
# Verify the output.
# No datasets were emitted, and more than 20 events were emitted
assert (
mce_helpers.assert_entity_urn_not_like(
entity_type="dataset",
regex_pattern="urn:li:dataset:\\(urn:li:dataPlatform:dbt",
file=output_file,
)
> 20
mce_helpers.assert_entity_urn_not_like(
entity_type="dataset",
regex_pattern="urn:li:dataset:\\(urn:li:dataPlatform:dbt",
file=output_file,
)
> 20
)
number_of_valid_assertions_in_test_results = 23
assert (
mce_helpers.assert_entity_urn_like(
entity_type="assertion", regex_pattern="urn:li:assertion:", file=output_file
)
== number_of_valid_assertions_in_test_results
mce_helpers.assert_entity_urn_like(
entity_type="assertion", regex_pattern="urn:li:assertion:", file=output_file
)
== number_of_valid_assertions_in_test_results
)

# no assertionInfo should be emitted
with pytest.raises(
AssertionError, match="Failed to find aspect_name assertionInfo for urns"
AssertionError, match="Failed to find aspect_name assertionInfo for urns"
):
mce_helpers.assert_for_each_entity(
entity_type="assertion",
Expand All @@ -486,21 +490,21 @@ def test_dbt_tests_only_assertions(

# all assertions must have an assertionRunEvent emitted (except for one assertion)
assert (
mce_helpers.assert_for_each_entity(
entity_type="assertion",
aspect_name="assertionRunEvent",
aspect_field_matcher={},
file=output_file,
exception_urns=["urn:li:assertion:2ff754df689ea951ed2e12cbe356708f"],
)
== number_of_valid_assertions_in_test_results
mce_helpers.assert_for_each_entity(
entity_type="assertion",
aspect_name="assertionRunEvent",
aspect_field_matcher={},
file=output_file,
exception_urns=["urn:li:assertion:2ff754df689ea951ed2e12cbe356708f"],
)
== number_of_valid_assertions_in_test_results
)


@pytest.mark.integration
@freeze_time(FROZEN_TIME)
def test_dbt_only_test_definitions_and_results(
test_resources_dir, pytestconfig, tmp_path, mock_time, **kwargs
test_resources_dir, pytestconfig, tmp_path, mock_time, **kwargs
):
# Run the metadata ingestion pipeline.
output_file = tmp_path / "test_only_definitions_and_assertions.json"
Expand All @@ -518,9 +522,11 @@ def test_dbt_only_test_definitions_and_results(
(test_resources_dir / "jaffle_shop_catalog.json").resolve()
),
target_platform="postgres",
run_results_paths=[str(
(test_resources_dir / "jaffle_shop_test_results.json").resolve()
)],
run_results_paths=[
str(
(test_resources_dir / "jaffle_shop_test_results.json").resolve()
)
],
entities_enabled=DBTEntitiesEnabled(
sources=EmitDirective.NO,
seeds=EmitDirective.NO,
Expand All @@ -535,38 +541,38 @@ def test_dbt_only_test_definitions_and_results(
pipeline.raise_from_status()
# Verify the output. No datasets were emitted
assert (
mce_helpers.assert_entity_urn_not_like(
entity_type="dataset",
regex_pattern="urn:li:dataset:\\(urn:li:dataPlatform:dbt",
file=output_file,
)
> 20
mce_helpers.assert_entity_urn_not_like(
entity_type="dataset",
regex_pattern="urn:li:dataset:\\(urn:li:dataPlatform:dbt",
file=output_file,
)
> 20
)
number_of_assertions = 24
assert (
mce_helpers.assert_entity_urn_like(
entity_type="assertion", regex_pattern="urn:li:assertion:", file=output_file
)
== number_of_assertions
mce_helpers.assert_entity_urn_like(
entity_type="assertion", regex_pattern="urn:li:assertion:", file=output_file
)
== number_of_assertions
)
# all assertions must have an assertionInfo emitted
assert (
mce_helpers.assert_for_each_entity(
entity_type="assertion",
aspect_name="assertionInfo",
aspect_field_matcher={},
file=output_file,
)
== number_of_assertions
mce_helpers.assert_for_each_entity(
entity_type="assertion",
aspect_name="assertionInfo",
aspect_field_matcher={},
file=output_file,
)
== number_of_assertions
)
# all assertions must have an assertionRunEvent emitted (except for one assertion)
assert (
mce_helpers.assert_for_each_entity(
entity_type="assertion",
aspect_name="assertionRunEvent",
aspect_field_matcher={},
file=output_file,
exception_urns=["urn:li:assertion:2ff754df689ea951ed2e12cbe356708f"],
)
== number_of_assertions - 1
mce_helpers.assert_for_each_entity(
entity_type="assertion",
aspect_name="assertionRunEvent",
aspect_field_matcher={},
file=output_file,
exception_urns=["urn:li:assertion:2ff754df689ea951ed2e12cbe356708f"],
)
== number_of_assertions - 1
)

0 comments on commit 9fc8a6c

Please sign in to comment.