diff --git a/ingestion/setup.py b/ingestion/setup.py index d1ee38c9de01..e7c81a6f3b4a 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -258,6 +258,7 @@ VERSIONS["lkml"], "gitpython~=3.1.34", VERSIONS["giturlparse"], + "python-liquid", }, "mlflow": {"mlflow-skinny>=2.3.0"}, "mongo": {VERSIONS["mongo"], VERSIONS["pandas"], VERSIONS["numpy"]}, diff --git a/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py index 89bb367a53f0..4cf1858227f9 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py @@ -28,6 +28,7 @@ import giturlparse import lkml +from liquid import Template from looker_sdk.sdk.api40.methods import Looker40SDK from looker_sdk.sdk.api40.models import Dashboard as LookerDashboard from looker_sdk.sdk.api40.models import ( @@ -450,11 +451,11 @@ def yield_bulk_datamodel( view.name, "Data model (View) filtered out." ) continue - + view_name = view.from_ if view.from_ else view.name yield from self._process_view( - view_name=ViewName(view.name), explore=model + view_name=ViewName(view_name), explore=model ) - if len(model.joins) == 0 and model.sql_table_name: + if model.view_name: yield from self._process_view( view_name=ViewName(model.view_name), explore=model ) @@ -570,7 +571,8 @@ def add_view_lineage( db_service_names = self.get_db_service_names() if view.sql_table_name: - source_table_name = self._clean_table_name(view.sql_table_name) + sql_table_name = self._render_table_name(view.sql_table_name) + source_table_name = self._clean_table_name(sql_table_name) # View to the source is only there if we are informing the dbServiceNames for db_service_name in db_service_names or []: @@ -726,6 +728,33 @@ def _clean_table_name(table_name: str) -> str: return table_name.lower().split(" as ")[0].strip() + @staticmethod + def _render_table_name(table_name: str) -> str: + """ + sql_table_names might contain Liquid templates + when defining an explore. e.g,: + sql_table_name: + {% if openmetadata %} + event + {% elsif event.created_week._in_query %} + event_by_week + {% else %} + event + {% endif %} ;; + we should render the template and give the option + to render a specific value during metadata ingestion + using the "openmetadata" context argument + :param table_name: table name with possible templating + :return: rendered table name + """ + try: + context = {"openmetadata": True} + template = Template(table_name) + sql_table_name = template.render(context) + except Exception: + sql_table_name = table_name + return sql_table_name + @staticmethod def get_dashboard_sources(dashboard_details: LookerDashboard) -> Set[str]: """ diff --git a/ingestion/tests/unit/topology/dashboard/test_looker.py b/ingestion/tests/unit/topology/dashboard/test_looker.py index ea195f1c6ece..b5a2b2b0ab46 100644 --- a/ingestion/tests/unit/topology/dashboard/test_looker.py +++ b/ingestion/tests/unit/topology/dashboard/test_looker.py @@ -300,6 +300,53 @@ def test_clean_table_name(self): self.assertEqual(self.looker._clean_table_name("TABLE AS ALIAS"), "table") + def test_render_table_name(self): + """ + Check that table is rendered correctly if "openmetadata" or default condition apply, or no templating is present + """ + tagged_table_name_template = """ + {%- if openmetadata -%} + `BQ-project.dataset.sample_data` + {%- elsif prod -%} + `BQ-project.dataset.sample_data` + {%- elsif dev -%} + `BQ-project.{{_user_attributes['dbt_dev_schema']}}.sample_data` + {%- endif -%} + """ + default_table_name_template = """ + {%- if prod -%} + `BQ-project.dataset.sample_data` + {%- elsif dev -%} + `BQ-project.{{_user_attributes['dbt_dev_schema']}}.sample_data` + {%- else -%} + `BQ-project.dataset.sample_data` + {%- endif -%} + """ + untagged_table_name_template = """ + {%- if prod -%} + `BQ-project.dataset.sample_data` + {%- elsif dev -%} + `BQ-project.{{_user_attributes['dbt_dev_schema']}}.sample_data` + {%- endif -%} + """ + table_name_plain = "`BQ-project.dataset.sample_data`" + self.assertEqual( + self.looker._render_table_name(tagged_table_name_template), + "`BQ-project.dataset.sample_data`", + ) + self.assertEqual( + self.looker._render_table_name(default_table_name_template), + "`BQ-project.dataset.sample_data`", + ) + self.assertNotEqual( + self.looker._render_table_name(untagged_table_name_template), + "`BQ-project.dataset.sample_data`", + ) + self.assertEqual( + self.looker._render_table_name(table_name_plain), + "`BQ-project.dataset.sample_data`", + ) + def test_get_dashboard_sources(self): """ Check how we are building the sources