Looker parsing improvements for liquid templating and view/model aliasing

ingestion/setup.py

@@ -258,6 +258,7 @@
  VERSIONS["lkml"],
  "gitpython~=3.1.34",
  VERSIONS["giturlparse"],
+ "python-liquid",
  },
  "mlflow": {"mlflow-skinny>=2.3.0"},
  "mongo": {VERSIONS["mongo"], VERSIONS["pandas"], VERSIONS["numpy"]},

ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py

@@ -28,6 +28,7 @@
 
 import giturlparse
 import lkml
+from liquid import Template
 from looker_sdk.sdk.api40.methods import Looker40SDK
 from looker_sdk.sdk.api40.models import Dashboard as LookerDashboard
 from looker_sdk.sdk.api40.models import (
@@ -450,11 +451,11 @@ def yield_bulk_datamodel(
  view.name, "Data model (View) filtered out."
  )
  continue
-
+ view_name = view.from_ if view.from_ else view.name
  yield from self._process_view(
- view_name=ViewName(view.name), explore=model
+ view_name=ViewName(view_name), explore=model
  )
- if len(model.joins) == 0 and model.sql_table_name:
+ if model.view_name:
  yield from self._process_view(
  view_name=ViewName(model.view_name), explore=model
  )
@@ -570,7 +571,8 @@ def add_view_lineage(
  db_service_names = self.get_db_service_names()
 
  if view.sql_table_name:
- source_table_name = self._clean_table_name(view.sql_table_name)
+ sql_table_name = self._render_table_name(view.sql_table_name)
+ source_table_name = self._clean_table_name(sql_table_name)
 
  # View to the source is only there if we are informing the dbServiceNames
  for db_service_name in db_service_names or []:
@@ -726,6 +728,33 @@ def _clean_table_name(table_name: str) -> str:
 
  return table_name.lower().split(" as ")[0].strip()
 
+ @staticmethod
+ def _render_table_name(table_name: str) -> str:
+ """
+ sql_table_names might contain Liquid templates
+ when defining an explore. e.g,:
+ sql_table_name:
+ {% if openmetadata %}
+ event
+ {% elsif event.created_week._in_query %}
+ event_by_week
+ {% else %}
+ event
+ {% endif %} ;;
+ we should render the template and give the option
+ to render a specific value during metadata ingestion
+ using the "openmetadata" context argument
+ :param table_name: table name with possible templating
+ :return: rendered table name
+ """
+ try:
+ context = {"openmetadata": True}
+ template = Template(table_name)
+ sql_table_name = template.render(context)
+ except Exception:
+ sql_table_name = table_name
+ return sql_table_name
+
  @staticmethod
  def get_dashboard_sources(dashboard_details: LookerDashboard) -> Set[str]:
  """

ingestion/tests/unit/topology/dashboard/test_looker.py

@@ -300,6 +300,53 @@ def test_clean_table_name(self):
 
  self.assertEqual(self.looker._clean_table_name("TABLE AS ALIAS"), "table")
 
+ def test_render_table_name(self):
+ """
+ Check that table is rendered correctly if "openmetadata" or default condition apply, or no templating is present
+ """
+ tagged_table_name_template = """
+ {%- if openmetadata -%}
+ `BQ-project.dataset.sample_data`
+ {%- elsif prod -%}
+ `BQ-project.dataset.sample_data`
+ {%- elsif dev -%}
+ `BQ-project.{{_user_attributes['dbt_dev_schema']}}.sample_data`
+ {%- endif -%}
+ """
+ default_table_name_template = """
+ {%- if prod -%}
+ `BQ-project.dataset.sample_data`
+ {%- elsif dev -%}
+ `BQ-project.{{_user_attributes['dbt_dev_schema']}}.sample_data`
+ {%- else -%}
+ `BQ-project.dataset.sample_data`
+ {%- endif -%}
+ """
+ untagged_table_name_template = """
+ {%- if prod -%}
+ `BQ-project.dataset.sample_data`
+ {%- elsif dev -%}
+ `BQ-project.{{_user_attributes['dbt_dev_schema']}}.sample_data`
+ {%- endif -%}
+ """
+ table_name_plain = "`BQ-project.dataset.sample_data`"
+ self.assertEqual(
+ self.looker._render_table_name(tagged_table_name_template),
+ "`BQ-project.dataset.sample_data`",
+ )
+ self.assertEqual(
+ self.looker._render_table_name(default_table_name_template),
+ "`BQ-project.dataset.sample_data`",
+ )
+ self.assertNotEqual(
+ self.looker._render_table_name(untagged_table_name_template),
+ "`BQ-project.dataset.sample_data`",
+ )
+ self.assertEqual(
+ self.looker._render_table_name(table_name_plain),
+ "`BQ-project.dataset.sample_data`",
+ )
+
  def test_get_dashboard_sources(self):
  """
  Check how we are building the sources