From 6342464ea0b8f0394a535f3bd8ae387ad21b7ba7 Mon Sep 17 00:00:00 2001 From: Sam McCarty Date: Thu, 19 Sep 2024 11:31:58 -0400 Subject: [PATCH] move to static method with 'openmetadata' context and add rendering tests --- .../source/dashboard/looker/metadata.py | 33 +++++++++++-- .../unit/topology/dashboard/test_looker.py | 47 +++++++++++++++++++ 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py index c6cac46a545f..4cf1858227f9 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py @@ -571,11 +571,7 @@ def add_view_lineage( db_service_names = self.get_db_service_names() if view.sql_table_name: - try: - template = Template(view.sql_table_name) - sql_table_name = template.render() - except Exception: - sql_table_name = view.sql_table_name + sql_table_name = self._render_table_name(view.sql_table_name) source_table_name = self._clean_table_name(sql_table_name) # View to the source is only there if we are informing the dbServiceNames @@ -732,6 +728,33 @@ def _clean_table_name(table_name: str) -> str: return table_name.lower().split(" as ")[0].strip() + @staticmethod + def _render_table_name(table_name: str) -> str: + """ + sql_table_names might contain Liquid templates + when defining an explore. e.g,: + sql_table_name: + {% if openmetadata %} + event + {% elsif event.created_week._in_query %} + event_by_week + {% else %} + event + {% endif %} ;; + we should render the template and give the option + to render a specific value during metadata ingestion + using the "openmetadata" context argument + :param table_name: table name with possible templating + :return: rendered table name + """ + try: + context = {"openmetadata": True} + template = Template(table_name) + sql_table_name = template.render(context) + except Exception: + sql_table_name = table_name + return sql_table_name + @staticmethod def get_dashboard_sources(dashboard_details: LookerDashboard) -> Set[str]: """ diff --git a/ingestion/tests/unit/topology/dashboard/test_looker.py b/ingestion/tests/unit/topology/dashboard/test_looker.py index ea195f1c6ece..b5a2b2b0ab46 100644 --- a/ingestion/tests/unit/topology/dashboard/test_looker.py +++ b/ingestion/tests/unit/topology/dashboard/test_looker.py @@ -300,6 +300,53 @@ def test_clean_table_name(self): self.assertEqual(self.looker._clean_table_name("TABLE AS ALIAS"), "table") + def test_render_table_name(self): + """ + Check that table is rendered correctly if "openmetadata" or default condition apply, or no templating is present + """ + tagged_table_name_template = """ + {%- if openmetadata -%} + `BQ-project.dataset.sample_data` + {%- elsif prod -%} + `BQ-project.dataset.sample_data` + {%- elsif dev -%} + `BQ-project.{{_user_attributes['dbt_dev_schema']}}.sample_data` + {%- endif -%} + """ + default_table_name_template = """ + {%- if prod -%} + `BQ-project.dataset.sample_data` + {%- elsif dev -%} + `BQ-project.{{_user_attributes['dbt_dev_schema']}}.sample_data` + {%- else -%} + `BQ-project.dataset.sample_data` + {%- endif -%} + """ + untagged_table_name_template = """ + {%- if prod -%} + `BQ-project.dataset.sample_data` + {%- elsif dev -%} + `BQ-project.{{_user_attributes['dbt_dev_schema']}}.sample_data` + {%- endif -%} + """ + table_name_plain = "`BQ-project.dataset.sample_data`" + self.assertEqual( + self.looker._render_table_name(tagged_table_name_template), + "`BQ-project.dataset.sample_data`", + ) + self.assertEqual( + self.looker._render_table_name(default_table_name_template), + "`BQ-project.dataset.sample_data`", + ) + self.assertNotEqual( + self.looker._render_table_name(untagged_table_name_template), + "`BQ-project.dataset.sample_data`", + ) + self.assertEqual( + self.looker._render_table_name(table_name_plain), + "`BQ-project.dataset.sample_data`", + ) + def test_get_dashboard_sources(self): """ Check how we are building the sources