From f7ae2112907d1096cb69fdccaf9e068a135cc9b4 Mon Sep 17 00:00:00 2001 From: Sam McCarty Date: Wed, 18 Sep 2024 11:25:20 -0400 Subject: [PATCH 1/4] Looker parsing improvements for liquid templating and view/model aliasing --- .../ingestion/source/dashboard/looker/metadata.py | 14 ++++++++++---- ingestion/src/metadata/utils/fqn.py | 2 +- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py index 89bb367a53f0..c6cac46a545f 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py @@ -28,6 +28,7 @@ import giturlparse import lkml +from liquid import Template from looker_sdk.sdk.api40.methods import Looker40SDK from looker_sdk.sdk.api40.models import Dashboard as LookerDashboard from looker_sdk.sdk.api40.models import ( @@ -450,11 +451,11 @@ def yield_bulk_datamodel( view.name, "Data model (View) filtered out." ) continue - + view_name = view.from_ if view.from_ else view.name yield from self._process_view( - view_name=ViewName(view.name), explore=model + view_name=ViewName(view_name), explore=model ) - if len(model.joins) == 0 and model.sql_table_name: + if model.view_name: yield from self._process_view( view_name=ViewName(model.view_name), explore=model ) @@ -570,7 +571,12 @@ def add_view_lineage( db_service_names = self.get_db_service_names() if view.sql_table_name: - source_table_name = self._clean_table_name(view.sql_table_name) + try: + template = Template(view.sql_table_name) + sql_table_name = template.render() + except Exception: + sql_table_name = view.sql_table_name + source_table_name = self._clean_table_name(sql_table_name) # View to the source is only there if we are informing the dbServiceNames for db_service_name in db_service_names or []: diff --git a/ingestion/src/metadata/utils/fqn.py b/ingestion/src/metadata/utils/fqn.py index 05fa11e08bd3..5dff52edd144 100644 --- a/ingestion/src/metadata/utils/fqn.py +++ b/ingestion/src/metadata/utils/fqn.py @@ -554,7 +554,7 @@ def split_table_name(table_name: str) -> Dict[str, Optional[str]]: """ # Revisit: Check the antlr grammer for issue when string has double quotes # Issue Link: https://github.com/open-metadata/OpenMetadata/issues/8874 - details: List[str] = split(table_name.replace('"', "")) + details: List[str] = split(table_name.replace('"', "").replace("`", "")) # Pad None to the left until size of list is 3 full_details: List[Optional[str]] = ([None] * (3 - len(details))) + details From 682be9f100c4f68c3bc1ef1410911b1d6a01ed06 Mon Sep 17 00:00:00 2001 From: Sam McCarty Date: Wed, 18 Sep 2024 12:19:42 -0400 Subject: [PATCH 2/4] add python-liquid dependency to looker plugin requirements --- ingestion/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ingestion/setup.py b/ingestion/setup.py index e0d99d733612..f3d9e1dad6ed 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -258,6 +258,7 @@ VERSIONS["lkml"], "gitpython~=3.1.34", VERSIONS["giturlparse"], + "python-liquid", }, "mlflow": {"mlflow-skinny>=2.3.0"}, "mongo": {VERSIONS["mongo"], VERSIONS["pandas"], VERSIONS["numpy"]}, From afcc12d17b54be70e11e19012803de4485981167 Mon Sep 17 00:00:00 2001 From: Sam McCarty Date: Thu, 19 Sep 2024 11:31:58 -0400 Subject: [PATCH 3/4] move to static method with 'openmetadata' context and add rendering tests --- .../source/dashboard/looker/metadata.py | 33 +++++++++++-- .../unit/topology/dashboard/test_looker.py | 47 +++++++++++++++++++ 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py index c6cac46a545f..4cf1858227f9 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py @@ -571,11 +571,7 @@ def add_view_lineage( db_service_names = self.get_db_service_names() if view.sql_table_name: - try: - template = Template(view.sql_table_name) - sql_table_name = template.render() - except Exception: - sql_table_name = view.sql_table_name + sql_table_name = self._render_table_name(view.sql_table_name) source_table_name = self._clean_table_name(sql_table_name) # View to the source is only there if we are informing the dbServiceNames @@ -732,6 +728,33 @@ def _clean_table_name(table_name: str) -> str: return table_name.lower().split(" as ")[0].strip() + @staticmethod + def _render_table_name(table_name: str) -> str: + """ + sql_table_names might contain Liquid templates + when defining an explore. e.g,: + sql_table_name: + {% if openmetadata %} + event + {% elsif event.created_week._in_query %} + event_by_week + {% else %} + event + {% endif %} ;; + we should render the template and give the option + to render a specific value during metadata ingestion + using the "openmetadata" context argument + :param table_name: table name with possible templating + :return: rendered table name + """ + try: + context = {"openmetadata": True} + template = Template(table_name) + sql_table_name = template.render(context) + except Exception: + sql_table_name = table_name + return sql_table_name + @staticmethod def get_dashboard_sources(dashboard_details: LookerDashboard) -> Set[str]: """ diff --git a/ingestion/tests/unit/topology/dashboard/test_looker.py b/ingestion/tests/unit/topology/dashboard/test_looker.py index ea195f1c6ece..b5a2b2b0ab46 100644 --- a/ingestion/tests/unit/topology/dashboard/test_looker.py +++ b/ingestion/tests/unit/topology/dashboard/test_looker.py @@ -300,6 +300,53 @@ def test_clean_table_name(self): self.assertEqual(self.looker._clean_table_name("TABLE AS ALIAS"), "table") + def test_render_table_name(self): + """ + Check that table is rendered correctly if "openmetadata" or default condition apply, or no templating is present + """ + tagged_table_name_template = """ + {%- if openmetadata -%} + `BQ-project.dataset.sample_data` + {%- elsif prod -%} + `BQ-project.dataset.sample_data` + {%- elsif dev -%} + `BQ-project.{{_user_attributes['dbt_dev_schema']}}.sample_data` + {%- endif -%} + """ + default_table_name_template = """ + {%- if prod -%} + `BQ-project.dataset.sample_data` + {%- elsif dev -%} + `BQ-project.{{_user_attributes['dbt_dev_schema']}}.sample_data` + {%- else -%} + `BQ-project.dataset.sample_data` + {%- endif -%} + """ + untagged_table_name_template = """ + {%- if prod -%} + `BQ-project.dataset.sample_data` + {%- elsif dev -%} + `BQ-project.{{_user_attributes['dbt_dev_schema']}}.sample_data` + {%- endif -%} + """ + table_name_plain = "`BQ-project.dataset.sample_data`" + self.assertEqual( + self.looker._render_table_name(tagged_table_name_template), + "`BQ-project.dataset.sample_data`", + ) + self.assertEqual( + self.looker._render_table_name(default_table_name_template), + "`BQ-project.dataset.sample_data`", + ) + self.assertNotEqual( + self.looker._render_table_name(untagged_table_name_template), + "`BQ-project.dataset.sample_data`", + ) + self.assertEqual( + self.looker._render_table_name(table_name_plain), + "`BQ-project.dataset.sample_data`", + ) + def test_get_dashboard_sources(self): """ Check how we are building the sources From d3511ce6e009b1c46c46f4c02162c3fc58659305 Mon Sep 17 00:00:00 2001 From: Sam McCarty Date: Thu, 19 Sep 2024 15:29:26 -0400 Subject: [PATCH 4/4] remove backtick stripping --- ingestion/src/metadata/utils/fqn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingestion/src/metadata/utils/fqn.py b/ingestion/src/metadata/utils/fqn.py index 5dff52edd144..05fa11e08bd3 100644 --- a/ingestion/src/metadata/utils/fqn.py +++ b/ingestion/src/metadata/utils/fqn.py @@ -554,7 +554,7 @@ def split_table_name(table_name: str) -> Dict[str, Optional[str]]: """ # Revisit: Check the antlr grammer for issue when string has double quotes # Issue Link: https://github.com/open-metadata/OpenMetadata/issues/8874 - details: List[str] = split(table_name.replace('"', "").replace("`", "")) + details: List[str] = split(table_name.replace('"', "")) # Pad None to the left until size of list is 3 full_details: List[Optional[str]] = ([None] * (3 - len(details))) + details