Skip to content

Commit

Permalink
Don't apply unnecessary DATE_TRUNC to base time spine column
Browse files Browse the repository at this point in the history
This will enable more efficient queries. We require this grain to be accurate.
  • Loading branch information
courtneyholcomb committed Nov 23, 2024
1 parent 9633e66 commit 74ba4b7
Show file tree
Hide file tree
Showing 23 changed files with 60 additions and 56 deletions.
10 changes: 4 additions & 6 deletions metricflow/dataset/convert_semantic_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -528,13 +528,11 @@ def build_time_spine_source_data_set(self, time_spine_source: TimeSpineSource) -
time_granularity=ExpandedTimeGranularity.from_time_granularity(base_granularity),
)
time_dimension_instances.append(base_time_dimension_instance)
base_dimension_select_expr = SemanticModelToDataSetConverter._make_element_sql_expr(
table_alias=from_source_alias, element_name=base_column_name
base_dimension_select_expr = SqlColumnReferenceExpression.from_table_and_column_names(
table_alias=from_source_alias, column_name=base_column_name
)
base_select_column = self._build_column_for_standard_time_granularity(
time_granularity=base_granularity,
expr=base_dimension_select_expr,
column_alias=base_time_dimension_instance.associated_column.column_name,
base_select_column = SqlSelectColumn(
expr=base_dimension_select_expr, column_alias=base_time_dimension_instance.associated_column.column_name
)
select_columns.append(base_select_column)
new_base_instances, new_base_columns = self._build_time_dimension_instances_and_columns(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -413,7 +413,7 @@ FROM (
FROM (
-- Read From Time Spine 'mf_time_spine'
SELECT
DATE_TRUNC('day', time_spine_src_28006.ds) AS ds__day
time_spine_src_28006.ds AS ds__day
, DATE_TRUNC('week', time_spine_src_28006.ds) AS ds__week
, DATE_TRUNC('month', time_spine_src_28006.ds) AS ds__month
, DATE_TRUNC('quarter', time_spine_src_28006.ds) AS ds__quarter
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,14 @@ SELECT
subq_12.martian_day AS user__bio_added_ts__martian_day
, subq_11.martian_day AS metric_time__martian_day
, DATE_TRUNC('month', users_ds_source_src_28000.bio_added_ts) AS user__bio_added_ts__month
, DATE_TRUNC('day', time_spine_src_28006.ds) AS metric_time__day
, time_spine_src_28006.ds AS metric_time__day
FROM ***************************.dim_users users_ds_source_src_28000
CROSS JOIN
***************************.mf_time_spine time_spine_src_28006
LEFT OUTER JOIN
***************************.mf_time_spine subq_11
ON
DATE_TRUNC('day', time_spine_src_28006.ds) = subq_11.ds
time_spine_src_28006.ds = subq_11.ds
LEFT OUTER JOIN
***************************.mf_time_spine subq_12
ON
Expand All @@ -26,4 +26,4 @@ GROUP BY
subq_12.martian_day
, subq_11.martian_day
, DATE_TRUNC('month', users_ds_source_src_28000.bio_added_ts)
, DATE_TRUNC('day', time_spine_src_28006.ds)
, time_spine_src_28006.ds
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ FROM (
FROM (
-- Read From Time Spine 'mf_time_spine'
SELECT
DATE_TRUNC('day', time_spine_src_28006.ds) AS ds__day
time_spine_src_28006.ds AS ds__day
, DATE_TRUNC('week', time_spine_src_28006.ds) AS ds__week
, DATE_TRUNC('month', time_spine_src_28006.ds) AS ds__month
, DATE_TRUNC('quarter', time_spine_src_28006.ds) AS ds__quarter
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,6 @@ FROM ***************************.mf_time_spine time_spine_src_28006
LEFT OUTER JOIN
***************************.mf_time_spine subq_4
ON
DATE_TRUNC('day', time_spine_src_28006.ds) = subq_4.ds
time_spine_src_28006.ds = subq_4.ds
GROUP BY
subq_4.martian_day
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ FROM (
FROM (
-- Read From Time Spine 'mf_time_spine_millisecond'
SELECT
DATE_TRUNC('millisecond', time_spine_src_28002.ts) AS ts__millisecond
time_spine_src_28002.ts AS ts__millisecond
, DATE_TRUNC('second', time_spine_src_28002.ts) AS ts__second
, DATE_TRUNC('minute', time_spine_src_28002.ts) AS ts__minute
, DATE_TRUNC('hour', time_spine_src_28002.ts) AS ts__hour
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ sql_engine: DuckDB
-- Metric Time Dimension 'ts'
-- Pass Only Elements: ['metric_time__millisecond',]
SELECT
DATE_TRUNC('millisecond', ts) AS metric_time__millisecond
ts AS metric_time__millisecond
FROM ***************************.mf_time_spine_millisecond time_spine_src_28002
GROUP BY
DATE_TRUNC('millisecond', ts)
ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ FROM (
FROM (
-- Read From Time Spine 'mf_time_spine_second'
SELECT
DATE_TRUNC('second', time_spine_src_28003.ts) AS ts__second
time_spine_src_28003.ts AS ts__second
, DATE_TRUNC('minute', time_spine_src_28003.ts) AS ts__minute
, DATE_TRUNC('hour', time_spine_src_28003.ts) AS ts__hour
, DATE_TRUNC('day', time_spine_src_28003.ts) AS ts__day
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ sql_engine: DuckDB
-- Constrain Time Range to [2020-01-01T00:00:02, 2020-01-01T00:00:08]
-- Pass Only Elements: ['metric_time__second',]
SELECT
DATE_TRUNC('second', ts) AS metric_time__second
ts AS metric_time__second
FROM ***************************.mf_time_spine_second time_spine_src_28003
WHERE DATE_TRUNC('second', ts) BETWEEN '2020-01-01 00:00:02' AND '2020-01-01 00:00:08'
WHERE ts BETWEEN '2020-01-01 00:00:02' AND '2020-01-01 00:00:08'
GROUP BY
DATE_TRUNC('second', ts)
ts
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ FROM (
FROM (
-- Read From Time Spine 'mf_time_spine'
SELECT
DATE_TRUNC('day', time_spine_src_28006.ds) AS ds__day
time_spine_src_28006.ds AS ds__day
, DATE_TRUNC('week', time_spine_src_28006.ds) AS ds__week
, DATE_TRUNC('month', time_spine_src_28006.ds) AS ds__month
, DATE_TRUNC('quarter', time_spine_src_28006.ds) AS ds__quarter
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ sql_engine: DuckDB
-- Constrain Time Range to [2020-01-01T00:00:00, 2020-01-03T00:00:00]
-- Pass Only Elements: ['user__home_state_latest', 'listing__is_lux_latest', 'metric_time__day']
SELECT
DATE_TRUNC('day', time_spine_src_28006.ds) AS metric_time__day
time_spine_src_28006.ds AS metric_time__day
, listings_latest_src_28000.is_lux AS listing__is_lux_latest
, users_latest_src_28000.home_state_latest AS user__home_state_latest
FROM ***************************.dim_listings_latest listings_latest_src_28000
Expand All @@ -16,8 +16,8 @@ FULL OUTER JOIN
***************************.dim_users_latest users_latest_src_28000
ON
listings_latest_src_28000.user_id = users_latest_src_28000.user_id
WHERE DATE_TRUNC('day', time_spine_src_28006.ds) BETWEEN '2020-01-01' AND '2020-01-03'
WHERE time_spine_src_28006.ds BETWEEN '2020-01-01' AND '2020-01-03'
GROUP BY
DATE_TRUNC('day', time_spine_src_28006.ds)
time_spine_src_28006.ds
, listings_latest_src_28000.is_lux
, users_latest_src_28000.home_state_latest
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ FROM (
FROM (
-- Read From Time Spine 'mf_time_spine'
SELECT
DATE_TRUNC('day', time_spine_src_28006.ds) AS ds__day
time_spine_src_28006.ds AS ds__day
, DATE_TRUNC('week', time_spine_src_28006.ds) AS ds__week
, DATE_TRUNC('month', time_spine_src_28006.ds) AS ds__month
, DATE_TRUNC('quarter', time_spine_src_28006.ds) AS ds__quarter
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ sql_engine: DuckDB
-- Metric Time Dimension 'ds'
-- Pass Only Elements: ['metric_time__day',]
SELECT
DATE_TRUNC('day', ds) AS metric_time__day
ds AS metric_time__day
FROM ***************************.mf_time_spine time_spine_src_28006
GROUP BY
DATE_TRUNC('day', ds)
ds
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ FROM (
FROM (
-- Read From Time Spine 'mf_time_spine'
SELECT
DATE_TRUNC('day', time_spine_src_28006.ds) AS ds__day
time_spine_src_28006.ds AS ds__day
, DATE_TRUNC('week', time_spine_src_28006.ds) AS ds__week
, DATE_TRUNC('month', time_spine_src_28006.ds) AS ds__month
, DATE_TRUNC('quarter', time_spine_src_28006.ds) AS ds__quarter
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ FROM (
FROM (
-- Read From Time Spine 'mf_time_spine'
SELECT
DATE_TRUNC('day', time_spine_src_28006.ds) AS ds__day
time_spine_src_28006.ds AS ds__day
, DATE_TRUNC('week', time_spine_src_28006.ds) AS ds__week
, DATE_TRUNC('month', time_spine_src_28006.ds) AS ds__month
, DATE_TRUNC('quarter', time_spine_src_28006.ds) AS ds__quarter
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ sql_engine: DuckDB
-- Join Standard Outputs
-- Pass Only Elements: ['user__home_state_latest', 'listing__is_lux_latest', 'metric_time__day']
SELECT
DATE_TRUNC('day', time_spine_src_28006.ds) AS metric_time__day
time_spine_src_28006.ds AS metric_time__day
, listings_latest_src_28000.is_lux AS listing__is_lux_latest
, users_latest_src_28000.home_state_latest AS user__home_state_latest
FROM ***************************.dim_listings_latest listings_latest_src_28000
Expand All @@ -16,6 +16,6 @@ FULL OUTER JOIN
ON
listings_latest_src_28000.user_id = users_latest_src_28000.user_id
GROUP BY
DATE_TRUNC('day', time_spine_src_28006.ds)
time_spine_src_28006.ds
, listings_latest_src_28000.is_lux
, users_latest_src_28000.home_state_latest
Original file line number Diff line number Diff line change
Expand Up @@ -803,22 +803,25 @@ test_filename: test_metric_time_without_metrics.py
<SqlSelectStatementNode>
<!-- description = "Read From Time Spine 'mf_time_spine'" -->
<!-- node_id = NodeId(id_str='ss_1') -->
<!-- col0 = -->
<!-- SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28293), column_alias='ds__day') -->
<!-- col0 = -->
<!-- SqlSelectColumn( -->
<!-- expr=SqlColumnReferenceExpression(node_id=cr_28130), -->
<!-- column_alias='ds__day', -->
<!-- ) -->
<!-- col1 = -->
<!-- SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28294), column_alias='ds__week') -->
<!-- SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28287), column_alias='ds__week') -->
<!-- col2 = -->
<!-- SqlSelectColumn( -->
<!-- expr=SqlDateTruncExpression(node_id=dt_28295), -->
<!-- expr=SqlDateTruncExpression(node_id=dt_28288), -->
<!-- column_alias='ds__month', -->
<!-- ) -->
<!-- col3 = -->
<!-- SqlSelectColumn( -->
<!-- expr=SqlDateTruncExpression(node_id=dt_28296), -->
<!-- expr=SqlDateTruncExpression(node_id=dt_28289), -->
<!-- column_alias='ds__quarter', -->
<!-- ) -->
<!-- col4 = -->
<!-- SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28297), column_alias='ds__year') -->
<!-- SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28290), column_alias='ds__year') -->
<!-- col5 = -->
<!-- SqlSelectColumn( -->
<!-- expr=SqlExtractExpression(node_id=ex_28300), -->
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,11 +84,11 @@ docstring:
<SqlSelectStatementNode>
<!-- description = "Read From Time Spine 'mf_time_spine'" -->
<!-- node_id = NodeId(id_str='ss_0') -->
<!-- col0 = SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28293), column_alias='ds__day') -->
<!-- col1 = SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28294), column_alias='ds__week') -->
<!-- col2 = SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28295), column_alias='ds__month') -->
<!-- col3 = SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28296), column_alias='ds__quarter') -->
<!-- col4 = SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28297), column_alias='ds__year') -->
<!-- col0 = SqlSelectColumn(expr=SqlColumnReferenceExpression(node_id=cr_28130), column_alias='ds__day') -->
<!-- col1 = SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28287), column_alias='ds__week') -->
<!-- col2 = SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28288), column_alias='ds__month') -->
<!-- col3 = SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28289), column_alias='ds__quarter') -->
<!-- col4 = SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28290), column_alias='ds__year') -->
<!-- col5 = -->
<!-- SqlSelectColumn(expr=SqlExtractExpression(node_id=ex_28300), column_alias='ds__extract_year') -->
<!-- col6 = -->
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,11 +83,11 @@ test_filename: test_metric_time_without_metrics.py
<SqlSelectStatementNode>
<!-- description = "Read From Time Spine 'mf_time_spine'" -->
<!-- node_id = NodeId(id_str='ss_0') -->
<!-- col0 = SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28293), column_alias='ds__day') -->
<!-- col1 = SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28294), column_alias='ds__week') -->
<!-- col2 = SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28295), column_alias='ds__month') -->
<!-- col3 = SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28296), column_alias='ds__quarter') -->
<!-- col4 = SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28297), column_alias='ds__year') -->
<!-- col0 = SqlSelectColumn(expr=SqlColumnReferenceExpression(node_id=cr_28130), column_alias='ds__day') -->
<!-- col1 = SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28287), column_alias='ds__week') -->
<!-- col2 = SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28288), column_alias='ds__month') -->
<!-- col3 = SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28289), column_alias='ds__quarter') -->
<!-- col4 = SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28290), column_alias='ds__year') -->
<!-- col5 = -->
<!-- SqlSelectColumn(expr=SqlExtractExpression(node_id=ex_28300), column_alias='ds__extract_year') -->
<!-- col6 = -->
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -552,16 +552,19 @@ test_filename: test_metric_time_without_metrics.py
<SqlSelectStatementNode>
<!-- description = "Read From Time Spine 'mf_time_spine'" -->
<!-- node_id = NodeId(id_str='ss_1') -->
<!-- col0 = -->
<!-- SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28293), column_alias='ds__day') -->
<!-- col0 = -->
<!-- SqlSelectColumn( -->
<!-- expr=SqlColumnReferenceExpression(node_id=cr_28130), -->
<!-- column_alias='ds__day', -->
<!-- ) -->
<!-- col1 = -->
<!-- SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28294), column_alias='ds__week') -->
<!-- SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28287), column_alias='ds__week') -->
<!-- col2 = -->
<!-- SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28295), column_alias='ds__month') -->
<!-- SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28288), column_alias='ds__month') -->
<!-- col3 = -->
<!-- SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28296), column_alias='ds__quarter') -->
<!-- SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28289), column_alias='ds__quarter') -->
<!-- col4 = -->
<!-- SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28297), column_alias='ds__year') -->
<!-- SqlSelectColumn(expr=SqlDateTruncExpression(node_id=dt_28290), column_alias='ds__year') -->
<!-- col5 = -->
<!-- SqlSelectColumn( -->
<!-- expr=SqlExtractExpression(node_id=ex_28300), -->
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ FROM (
FROM (
-- Read From Time Spine 'mf_time_spine'
SELECT
DATE_TRUNC('day', time_spine_src_28006.ds) AS ds__day
time_spine_src_28006.ds AS ds__day
, DATE_TRUNC('week', time_spine_src_28006.ds) AS ds__week
, DATE_TRUNC('month', time_spine_src_28006.ds) AS ds__month
, DATE_TRUNC('quarter', time_spine_src_28006.ds) AS ds__quarter
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ FROM (
-- Metric Time Dimension 'ds'
-- Pass Only Elements: ['metric_time__day',]
SELECT
DATE_TRUNC('day', ds) AS metric_time__day
ds AS metric_time__day
FROM ***************************.mf_time_spine time_spine_src_28006
GROUP BY
DATE_TRUNC('day', ds)
ds
) subq_5
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ FROM (
FROM (
-- Read From Time Spine 'mf_time_spine'
SELECT
DATE_TRUNC('day', time_spine_src_28006.ds) AS ds__day
time_spine_src_28006.ds AS ds__day
, DATE_TRUNC('week', time_spine_src_28006.ds) AS ds__week
, DATE_TRUNC('month', time_spine_src_28006.ds) AS ds__month
, DATE_TRUNC('quarter', time_spine_src_28006.ds) AS ds__quarter
Expand Down

0 comments on commit 74ba4b7

Please sign in to comment.