Allow configuration of snapshot column names (#10608)

.changes/unreleased/Features-20240903-154133.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Allow configuring snapshot column names
+time: 2024-09-03T15:41:33.167097-04:00
+custom:
+ Author: gshank
+ Issue: "10185"

core/dbt/artifacts/resources/v1/snapshot.py

@@ -1,10 +1,18 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Dict, List, Literal, Optional, Union
 
 from dbt.artifacts.resources.types import NodeType
 from dbt.artifacts.resources.v1.components import CompiledResource, DeferRelation
 from dbt.artifacts.resources.v1.config import NodeConfig
-from dbt_common.dataclass_schema import ValidationError
+from dbt_common.dataclass_schema import ValidationError, dbtClassMixin
+
+
+@dataclass
+class SnapshotMetaColumnNames(dbtClassMixin):
+ dbt_valid_to: Optional[str] = None
+ dbt_valid_from: Optional[str] = None
+ dbt_scd_id: Optional[str] = None
+ dbt_updated_at: Optional[str] = None
 
 
 @dataclass
@@ -17,6 +25,18 @@ class SnapshotConfig(NodeConfig):
  updated_at: Optional[str] = None
  # Not using Optional because of serialization issues with a Union of str and List[str]
  check_cols: Union[str, List[str], None] = None
+ snapshot_meta_column_names: SnapshotMetaColumnNames = field(
+ default_factory=SnapshotMetaColumnNames
+ )
+
+ @property
+ def snapshot_table_column_names(self):
+ return {
+ "dbt_valid_from": self.snapshot_meta_column_names.dbt_valid_from or "dbt_valid_from",
+ "dbt_valid_to": self.snapshot_meta_column_names.dbt_valid_to or "dbt_valid_to",
+ "dbt_scd_id": self.snapshot_meta_column_names.dbt_scd_id or "dbt_scd_id",
+ "dbt_updated_at": self.snapshot_meta_column_names.dbt_updated_at or "dbt_updated_at",
+ }
 
  def final_validate(self):
  if not self.strategy or not self.unique_key:

core/dbt/context/context_config.py

@@ -8,7 +8,7 @@
 from dbt.contracts.graph.model_config import get_config_for
 from dbt.node_types import NodeType
 from dbt.utils import fqn_search
-from dbt_common.contracts.config.base import BaseConfig, _listify
+from dbt_common.contracts.config.base import BaseConfig, merge_config_dicts
 from dbt_common.exceptions import DbtInternalError
 
 
@@ -293,55 +293,7 @@ def __init__(
 
  def add_config_call(self, opts: Dict[str, Any]) -> None:
  dct = self._config_call_dict
- self._add_config_call(dct, opts)
-
- @classmethod
- def _add_config_call(cls, config_call_dict, opts: Dict[str, Any]) -> None:
- # config_call_dict is already encountered configs, opts is new
- # This mirrors code in _merge_field_value in model_config.py which is similar but
- # operates on config objects.
- for k, v in opts.items():
- # MergeBehavior for post-hook and pre-hook is to collect all
- # values, instead of overwriting
- if k in BaseConfig.mergebehavior["append"]:
- if not isinstance(v, list):
- v = [v]
- if k in config_call_dict: # should always be a list here
- config_call_dict[k].extend(v)
- else:
- config_call_dict[k] = v
-
- elif k in BaseConfig.mergebehavior["update"]:
- if not isinstance(v, dict):
- raise DbtInternalError(f"expected dict, got {v}")
- if k in config_call_dict and isinstance(config_call_dict[k], dict):
- config_call_dict[k].update(v)
- else:
- config_call_dict[k] = v
- elif k in BaseConfig.mergebehavior["dict_key_append"]:
- if not isinstance(v, dict):
- raise DbtInternalError(f"expected dict, got {v}")
- if k in config_call_dict: # should always be a dict
- for key, value in v.items():
- extend = False
- # This might start with a +, to indicate we should extend the list
- # instead of just clobbering it
- if key.startswith("+"):
- extend = True
- if key in config_call_dict[k] and extend:
- # extend the list
- config_call_dict[k][key].extend(_listify(value))
- else:
- # clobber the list
- config_call_dict[k][key] = _listify(value)
- else:
- # This is always a dictionary
- config_call_dict[k] = v
- # listify everything
- for key, value in config_call_dict[k].items():
- config_call_dict[k][key] = _listify(value)
- else:
- config_call_dict[k] = v
+ merge_config_dicts(dct, opts)
 
  def build_config_dict(
  self,

core/dbt/parser/models.py

@@ -22,6 +22,7 @@
 from dbt.node_types import ModelLanguage, NodeType
 from dbt.parser.base import SimpleSQLParser
 from dbt.parser.search import FileBlock
+from dbt_common.contracts.config.base import merge_config_dicts
 from dbt_common.dataclass_schema import ValidationError
 from dbt_common.exceptions.macros import UndefinedMacroError
 from dbt_extractor import ExtractionError, py_extract_from_source # type: ignore
@@ -467,7 +468,7 @@ def _get_config_call_dict(static_parser_result: Dict[str, Any]) -> Dict[str, Any
  config_call_dict: Dict[str, Any] = {}
 
  for c in static_parser_result["configs"]:
- ContextConfig._add_config_call(config_call_dict, {c[0]: c[1]})
+ merge_config_dicts(config_call_dict, {c[0]: c[1]})
 
  return config_call_dict
 

core/setup.py

@@ -71,8 +71,8 @@
  "dbt-extractor>=0.5.0,<=0.6",
  "dbt-semantic-interfaces>=0.7.1,<0.8",
  # Minor versions for these are expected to be backwards-compatible
- "dbt-common>=1.6.0,<2.0",
- "dbt-adapters>=1.6.0,<2.0",
+ "dbt-common>=1.9.0,<2.0",
+ "dbt-adapters>=1.7.0,<2.0",
  # ----
  # Expect compatibility with all new versions of these packages, so lower bounds only.
  "packaging>20.9",

schemas/dbt/manifest/v12.json

@@ -6610,6 +6610,57 @@
  }
  ],
  "default": null
+ },
+ "snapshot_meta_column_names": {
+ "type": "object",
+ "title": "SnapshotMetaColumnNames",
+ "properties": {
+ "dbt_valid_to": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "default": null
+ },
+ "dbt_valid_from": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "default": null
+ },
+ "dbt_scd_id": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "default": null
+ },
+ "dbt_updated_at": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "default": null
+ }
+ },
+ "additionalProperties": false
  }
  },
  "additionalProperties": true
@@ -16336,6 +16387,57 @@
  }
  ],
  "default": null
+ },
+ "snapshot_meta_column_names": {
+ "type": "object",
+ "title": "SnapshotMetaColumnNames",
+ "properties": {
+ "dbt_valid_to": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "default": null
+ },
+ "dbt_valid_from": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "default": null
+ },
+ "dbt_scd_id": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "default": null
+ },
+ "dbt_updated_at": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "default": null
+ }
+ },
+ "additionalProperties": false
  }
  },
  "additionalProperties": true

tests/functional/artifacts/expected_manifest.py

@@ -103,6 +103,12 @@ def get_rendered_snapshot_config(**updates):
  "post-hook": [],
  "column_types": {},
  "quoting": {},
+ "snapshot_meta_column_names": {
+ "dbt_valid_to": None,
+ "dbt_valid_from": None,
+ "dbt_updated_at": None,
+ "dbt_scd_id": None,
+ },
  "tags": [],
  "persist_docs": {},
  "full_refresh": None,

tests/functional/list/test_list.py

@@ -63,6 +63,12 @@ def expect_snapshot_output(self, happy_path_project): # noqa: F811
  "persist_docs": {},
  "target_database": happy_path_project.database,
  "target_schema": happy_path_project.test_schema,
+ "snapshot_meta_column_names": {
+ "dbt_scd_id": None,
+ "dbt_updated_at": None,
+ "dbt_valid_from": None,
+ "dbt_valid_to": None,
+ },
  "unique_key": "id",
  "strategy": "timestamp",
  "updated_at": "updated_at",

tests/functional/snapshots/data/seed_cn.sql

@@ -0,0 +1,82 @@
+create table {database}.{schema}.seed (
+ id INTEGER,
+ first_name VARCHAR(50),
+ last_name VARCHAR(50),
+ email VARCHAR(50),
+ gender VARCHAR(50),
+ ip_address VARCHAR(20),
+ updated_at TIMESTAMP WITHOUT TIME ZONE
+);
+
+create table {database}.{schema}.snapshot_expected (
+ id INTEGER,
+ first_name VARCHAR(50),
+ last_name VARCHAR(50),
+ email VARCHAR(50),
+ gender VARCHAR(50),
+ ip_address VARCHAR(20),
+
+ -- snapshotting fields
+ updated_at TIMESTAMP WITHOUT TIME ZONE,
+ test_valid_from TIMESTAMP WITHOUT TIME ZONE,
+ test_valid_to TIMESTAMP WITHOUT TIME ZONE,
+ test_scd_id TEXT,
+ test_updated_at TIMESTAMP WITHOUT TIME ZONE
+);
+
+
+-- seed inserts
+-- use the same email for two users to verify that duplicated check_cols values
+-- are handled appropriately
+insert into {database}.{schema}.seed (id, first_name, last_name, email, gender, ip_address, updated_at) values
+(1, 'Judith', 'Kennedy', '(not provided)', 'Female', '54.60.24.128', '2015-12-24 12:19:28'),
+(2, 'Arthur', 'Kelly', '(not provided)', 'Male', '62.56.24.215', '2015-10-28 16:22:15'),
+(3, 'Rachel', 'Moreno', 'rmoreno2@msu.edu', 'Female', '31.222.249.23', '2016-04-05 02:05:30'),
+(4, 'Ralph', 'Turner', 'rturner3@hp.com', 'Male', '157.83.76.114', '2016-08-08 00:06:51'),
+(5, 'Laura', 'Gonzales', 'lgonzales4@howstuffworks.com', 'Female', '30.54.105.168', '2016-09-01 08:25:38'),
+(6, 'Katherine', 'Lopez', 'klopez5@yahoo.co.jp', 'Female', '169.138.46.89', '2016-08-30 18:52:11'),
+(7, 'Jeremy', 'Hamilton', 'jhamilton6@mozilla.org', 'Male', '231.189.13.133', '2016-07-17 02:09:46'),
+(8, 'Heather', 'Rose', 'hrose7@goodreads.com', 'Female', '87.165.201.65', '2015-12-29 22:03:56'),
+(9, 'Gregory', 'Kelly', 'gkelly8@trellian.com', 'Male', '154.209.99.7', '2016-03-24 21:18:16'),
+(10, 'Rachel', 'Lopez', 'rlopez9@themeforest.net', 'Female', '237.165.82.71', '2016-08-20 15:44:49'),
+(11, 'Donna', 'Welch', 'dwelcha@shutterfly.com', 'Female', '103.33.110.138', '2016-02-27 01:41:48'),
+(12, 'Russell', 'Lawrence', 'rlawrenceb@qq.com', 'Male', '189.115.73.4', '2016-06-11 03:07:09'),
+(13, 'Michelle', 'Montgomery', 'mmontgomeryc@scientificamerican.com', 'Female', '243.220.95.82', '2016-06-18 16:27:19'),
+(14, 'Walter', 'Castillo', 'wcastillod@pagesperso-orange.fr', 'Male', '71.159.238.196', '2016-10-06 01:55:44'),
+(15, 'Robin', 'Mills', 'rmillse@vkontakte.ru', 'Female', '172.190.5.50', '2016-10-31 11:41:21'),
+(16, 'Raymond', 'Holmes', 'rholmesf@usgs.gov', 'Male', '148.153.166.95', '2016-10-03 08:16:38'),
+(17, 'Gary', 'Bishop', 'gbishopg@plala.or.jp', 'Male', '161.108.182.13', '2016-08-29 19:35:20'),
+(18, 'Anna', 'Riley', 'arileyh@nasa.gov', 'Female', '253.31.108.22', '2015-12-11 04:34:27'),
+(19, 'Sarah', 'Knight', 'sknighti@foxnews.com', 'Female', '222.220.3.177', '2016-09-26 00:49:06'),
+(20, 'Phyllis', 'Fox', null, 'Female', '163.191.232.95', '2016-08-21 10:35:19');
+
+
+-- populate snapshot table
+insert into {database}.{schema}.snapshot_expected (
+ id,
+ first_name,
+ last_name,
+ email,
+ gender,
+ ip_address,
+ updated_at,
+ test_valid_from,
+ test_valid_to,
+ test_updated_at,
+ test_scd_id
+)
+
+select
+ id,
+ first_name,
+ last_name,
+ email,
+ gender,
+ ip_address,
+ updated_at,
+ -- fields added by snapshotting
+ updated_at as test_valid_from,
+ null::timestamp as test_valid_to,
+ updated_at as test_updated_at,
+ md5(id || '-' || first_name || '|' || updated_at::text) as test_scd_id
+from {database}.{schema}.seed;