diff --git a/.changes/unreleased/Fixes-20240926-101220.yaml b/.changes/unreleased/Fixes-20240926-101220.yaml new file mode 100644 index 00000000000..677ee458048 --- /dev/null +++ b/.changes/unreleased/Fixes-20240926-101220.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Attempt to skip saved query processing when no semantic manifest changes +time: 2024-09-26T10:12:20.193453-04:00 +custom: + Author: gshank + Issue: "10563" diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index 63cab649573..97dc5ea93c9 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -1696,6 +1696,7 @@ class ParsedSingularTestPatch(ParsedPatch): TestNode = Union[SingularTestNode, GenericTestNode] +SemanticManifestNode = Union[SavedQuery, SemanticModel, Metric] RESOURCE_CLASS_TO_NODE_CLASS: Dict[Type[BaseResource], Type[BaseNode]] = { node_class.resource_class(): node_class diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 5536bf39270..7ffd00febc5 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -57,6 +57,7 @@ ResultNode, SavedQuery, SeedNode, + SemanticManifestNode, SemanticModel, SourceDefinition, ) @@ -1141,6 +1142,23 @@ def process_metrics(self, config: RuntimeConfig): def process_saved_queries(self, config: RuntimeConfig): """Processes SavedQuery nodes to populate their `depends_on`.""" + # Note: This will also capture various nodes which have been re-parsed + # because they refer to some other changed node, so there will be + # false positives. Ideally we would compare actual changes. + semantic_manifest_changed = False + semantic_manifest_nodes: chain[SemanticManifestNode] = chain( + self.manifest.saved_queries.values(), + self.manifest.semantic_models.values(), + self.manifest.metrics.values(), + ) + for node in semantic_manifest_nodes: + # Check if this node has been modified in this parsing run + if node.created_at > self.started_at: + semantic_manifest_changed = True + break # as soon as we run into one changed node we can stop + if semantic_manifest_changed is False: + return + current_project = config.project_name for saved_query in self.manifest.saved_queries.values(): # TODO: