From 6a857acd255fc01afa433ec57b013f143971eaff Mon Sep 17 00:00:00 2001 From: Hendrik van Antwerpen Date: Mon, 9 Oct 2023 17:04:54 +0200 Subject: [PATCH 1/6] Add support for definiens_node --- tree-sitter-stack-graphs/src/lib.rs | 38 +++++++++++++--------- tree-sitter-stack-graphs/tests/it/nodes.rs | 27 +++++++++++++++ 2 files changed, 49 insertions(+), 16 deletions(-) diff --git a/tree-sitter-stack-graphs/src/lib.rs b/tree-sitter-stack-graphs/src/lib.rs index e2377f853..31b679a5c 100644 --- a/tree-sitter-stack-graphs/src/lib.rs +++ b/tree-sitter-stack-graphs/src/lib.rs @@ -364,6 +364,7 @@ static SCOPE_TYPE: &'static str = "scope"; // Node attribute names static DEBUG_ATTR_PREFIX: &'static str = "debug_"; +static DEFINIENS_NODE: &'static str = "definiens_node"; static EMPTY_SOURCE_SPAN_ATTR: &'static str = "empty_source_span"; static IS_DEFINITION_ATTR: &'static str = "is_definition"; static IS_ENDPOINT_ATTR: &'static str = "is_endpoint"; @@ -885,7 +886,7 @@ impl<'a> Builder<'a> { NodeType::PushSymbol => self.load_push_symbol(node_ref)?, NodeType::Scope => self.load_scope(node_ref)?, }; - self.load_span(node_ref, handle)?; + self.load_source_info(node_ref, handle)?; self.load_node_debug_info(node_ref, handle)?; } @@ -1091,28 +1092,33 @@ impl<'a> Builder<'a> { } } - fn load_span( + fn load_source_info( &mut self, node_ref: GraphNodeRef, node_handle: Handle, ) -> Result<(), BuildError> { let node = &self.graph[node_ref]; - let source_node = match node.attributes.get(SOURCE_NODE_ATTR) { - Some(source_node) => &self.graph[source_node.as_syntax_node_ref()?], - None => return Ok(()), + if let Some(source_node) = node.attributes.get(SOURCE_NODE_ATTR) { + let source_node = &self.graph[source_node.as_syntax_node_ref()?]; + let mut span = self.span_calculator.for_node(source_node); + if match node.attributes.get(EMPTY_SOURCE_SPAN_ATTR) { + Some(empty_source_span) => empty_source_span.as_boolean()?, + None => false, + } { + span.end = span.start.clone(); + } + let containing_line = &self.source[span.start.containing_line.clone()]; + let containing_line = self.stack_graph.add_string(containing_line); + let source_info = self.stack_graph.source_info_mut(node_handle); + source_info.span = span; + source_info.containing_line = ControlledOption::some(containing_line); }; - let mut span = self.span_calculator.for_node(source_node); - if match node.attributes.get(EMPTY_SOURCE_SPAN_ATTR) { - Some(empty_source_span) => empty_source_span.as_boolean()?, - None => false, - } { - span.end = span.start.clone(); + if let Some(definiens_node) = node.attributes.get(DEFINIENS_NODE) { + let definiens_node = &self.graph[definiens_node.as_syntax_node_ref()?]; + let span = self.span_calculator.for_node(definiens_node); + let source_info = self.stack_graph.source_info_mut(node_handle); + source_info.definiens_span = span; } - let containing_line = &self.source[span.start.containing_line.clone()]; - let containing_line = self.stack_graph.add_string(containing_line); - let source_info = self.stack_graph.source_info_mut(node_handle); - source_info.span = span; - source_info.containing_line = ControlledOption::some(containing_line); Ok(()) } diff --git a/tree-sitter-stack-graphs/tests/it/nodes.rs b/tree-sitter-stack-graphs/tests/it/nodes.rs index 2befc6a95..5306929d4 100644 --- a/tree-sitter-stack-graphs/tests/it/nodes.rs +++ b/tree-sitter-stack-graphs/tests/it/nodes.rs @@ -284,3 +284,30 @@ fn can_calculate_spans() { let trimmed_line = &python[source_info.span.start.trimmed_line.clone()]; assert_eq!(trimmed_line, "a"); } + +#[test] +fn can_set_definiens() { + let tsg = r#" + (function_definition body:(_)@body) { + node result + attr (result) definiens_node = @body + } + "#; + let python = r#" + def foo(): + pass + "#; + + let (graph, file) = build_stack_graph(python, tsg).unwrap(); + let node_handle = graph.nodes_for_file(file).next().unwrap(); + let source_info = graph.source_info(node_handle).unwrap(); + + let actual_span = format!( + "{}:{}-{}:{}", + source_info.definiens_span.start.line, + source_info.definiens_span.start.column.utf8_offset, + source_info.definiens_span.end.line, + source_info.definiens_span.end.column.utf8_offset, + ); + assert_eq!("2:8-2:12", actual_span) +} From 76e28700129cfc025584598f6650c86e1887e9b4 Mon Sep 17 00:00:00 2001 From: Hendrik van Antwerpen Date: Mon, 9 Oct 2023 17:06:43 +0200 Subject: [PATCH 2/6] Add syntax_type support --- tree-sitter-stack-graphs/src/lib.rs | 7 ++++++ tree-sitter-stack-graphs/tests/it/nodes.rs | 25 ++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/tree-sitter-stack-graphs/src/lib.rs b/tree-sitter-stack-graphs/src/lib.rs index 31b679a5c..672463bdc 100644 --- a/tree-sitter-stack-graphs/src/lib.rs +++ b/tree-sitter-stack-graphs/src/lib.rs @@ -373,6 +373,7 @@ static IS_REFERENCE_ATTR: &'static str = "is_reference"; static SCOPE_ATTR: &'static str = "scope"; static SOURCE_NODE_ATTR: &'static str = "source_node"; static SYMBOL_ATTR: &'static str = "symbol"; +static SYNTAX_TYPE: &'static str = "syntax_type"; static TYPE_ATTR: &'static str = "type"; // Expected attributes per node type @@ -1119,6 +1120,12 @@ impl<'a> Builder<'a> { let source_info = self.stack_graph.source_info_mut(node_handle); source_info.definiens_span = span; } + if let Some(syntax_type) = node.attributes.get(SYNTAX_TYPE) { + let syntax_type = syntax_type.as_str()?; + let interned_string = self.stack_graph.add_string(syntax_type); + let source_info = self.stack_graph.source_info_mut(node_handle); + source_info.syntax_type = interned_string.into(); + } Ok(()) } diff --git a/tree-sitter-stack-graphs/tests/it/nodes.rs b/tree-sitter-stack-graphs/tests/it/nodes.rs index 5306929d4..d200ed05d 100644 --- a/tree-sitter-stack-graphs/tests/it/nodes.rs +++ b/tree-sitter-stack-graphs/tests/it/nodes.rs @@ -311,3 +311,28 @@ fn can_set_definiens() { ); assert_eq!("2:8-2:12", actual_span) } + +#[test] +fn can_set_syntax_type() { + let tsg = r#" + (function_definition) { + node result + attr (result) syntax_type = "function" + } + "#; + let python = r#" + def foo(): + pass + "#; + + let (graph, file) = build_stack_graph(python, tsg).unwrap(); + let node_handle = graph.nodes_for_file(file).next().unwrap(); + let source_info = graph.source_info(node_handle).unwrap(); + + let syntax_type = source_info + .syntax_type + .into_option() + .map(|s| &graph[s]) + .unwrap_or("MISSING"); + assert_eq!("function", syntax_type) +} From 1c39888ab2ab1231e270aebd1b227791f9f7f518 Mon Sep 17 00:00:00 2001 From: Hendrik van Antwerpen Date: Mon, 9 Oct 2023 17:09:49 +0200 Subject: [PATCH 3/6] Document syntax_type attribute --- tree-sitter-stack-graphs/src/lib.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tree-sitter-stack-graphs/src/lib.rs b/tree-sitter-stack-graphs/src/lib.rs index 672463bdc..9a1ba8996 100644 --- a/tree-sitter-stack-graphs/src/lib.rs +++ b/tree-sitter-stack-graphs/src/lib.rs @@ -160,6 +160,19 @@ //! } //! ``` //! +//! ### Annotating nodes with syntax type information +//! +//! You can annotate any stack graph node with information about its syntax type. To do this, add a `syntax_type` +//! attribute, whose value is a string indicating the syntax type. +//! +//! ``` skip +//! (function_definition name: (identifier) @id) @func { +//! node def +//! ; ... +//! attr (def) syntax_type = "function" +//! } +//! ``` +//! //! ### Connecting stack graph nodes with edges //! //! To connect two stack graph nodes, use the `edge` statement to add an edge between them: From 0d4f95a968a6189c8834c1efe1c7fb2ff6054f16 Mon Sep 17 00:00:00 2001 From: Hendrik van Antwerpen Date: Mon, 9 Oct 2023 17:29:47 +0200 Subject: [PATCH 4/6] Only load definions for definitions --- tree-sitter-stack-graphs/src/lib.rs | 42 +++++++++++++++------- tree-sitter-stack-graphs/tests/it/nodes.rs | 3 +- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/tree-sitter-stack-graphs/src/lib.rs b/tree-sitter-stack-graphs/src/lib.rs index 9a1ba8996..564b32ba2 100644 --- a/tree-sitter-stack-graphs/src/lib.rs +++ b/tree-sitter-stack-graphs/src/lib.rs @@ -391,9 +391,9 @@ static TYPE_ATTR: &'static str = "type"; // Expected attributes per node type static POP_SCOPED_SYMBOL_ATTRS: Lazy> = - Lazy::new(|| HashSet::from([TYPE_ATTR, SYMBOL_ATTR, IS_DEFINITION_ATTR])); + Lazy::new(|| HashSet::from([TYPE_ATTR, SYMBOL_ATTR, IS_DEFINITION_ATTR, DEFINIENS_NODE])); static POP_SYMBOL_ATTRS: Lazy> = - Lazy::new(|| HashSet::from([TYPE_ATTR, SYMBOL_ATTR, IS_DEFINITION_ATTR])); + Lazy::new(|| HashSet::from([TYPE_ATTR, SYMBOL_ATTR, IS_DEFINITION_ATTR, DEFINIENS_NODE])); static PUSH_SCOPED_SYMBOL_ATTRS: Lazy> = Lazy::new(|| HashSet::from([TYPE_ATTR, SYMBOL_ATTR, SCOPE_ATTR, IS_REFERENCE_ATTR])); static PUSH_SYMBOL_ATTRS: Lazy> = @@ -1019,10 +1019,14 @@ impl<'a> Builder<'a> { let id = self.node_id_for_graph_node(node_ref); let is_definition = self.load_flag(node, IS_DEFINITION_ATTR)?; self.verify_attributes(node, POP_SCOPED_SYMBOL_TYPE, &POP_SCOPED_SYMBOL_ATTRS); - Ok(self + let node_handle = self .stack_graph .add_pop_scoped_symbol_node(id, symbol, is_definition) - .unwrap()) + .unwrap(); + if is_definition { + self.load_definiens_info(node_ref, node_handle)?; + } + Ok(node_handle) } fn load_pop_symbol(&mut self, node_ref: GraphNodeRef) -> Result, BuildError> { @@ -1035,10 +1039,14 @@ impl<'a> Builder<'a> { let id = self.node_id_for_graph_node(node_ref); let is_definition = self.load_flag(node, IS_DEFINITION_ATTR)?; self.verify_attributes(node, POP_SYMBOL_TYPE, &POP_SYMBOL_ATTRS); - Ok(self + let node_handle = self .stack_graph .add_pop_symbol_node(id, symbol, is_definition) - .unwrap()) + .unwrap(); + if is_definition { + self.load_definiens_info(node_ref, node_handle)?; + } + Ok(node_handle) } fn load_push_scoped_symbol( @@ -1127,12 +1135,6 @@ impl<'a> Builder<'a> { source_info.span = span; source_info.containing_line = ControlledOption::some(containing_line); }; - if let Some(definiens_node) = node.attributes.get(DEFINIENS_NODE) { - let definiens_node = &self.graph[definiens_node.as_syntax_node_ref()?]; - let span = self.span_calculator.for_node(definiens_node); - let source_info = self.stack_graph.source_info_mut(node_handle); - source_info.definiens_span = span; - } if let Some(syntax_type) = node.attributes.get(SYNTAX_TYPE) { let syntax_type = syntax_type.as_str()?; let interned_string = self.stack_graph.add_string(syntax_type); @@ -1142,6 +1144,22 @@ impl<'a> Builder<'a> { Ok(()) } + fn load_definiens_info( + &mut self, + node_ref: GraphNodeRef, + node_handle: Handle, + ) -> Result<(), BuildError> { + let node = &self.graph[node_ref]; + let definiens_node = match node.attributes.get(DEFINIENS_NODE) { + Some(definiens_node) => &self.graph[definiens_node.as_syntax_node_ref()?], + None => return Ok(()), + }; + let span = self.span_calculator.for_node(definiens_node); + let source_info = self.stack_graph.source_info_mut(node_handle); + source_info.definiens_span = span; + Ok(()) + } + fn load_node_debug_info( &mut self, node_ref: GraphNodeRef, diff --git a/tree-sitter-stack-graphs/tests/it/nodes.rs b/tree-sitter-stack-graphs/tests/it/nodes.rs index d200ed05d..ccb3e220e 100644 --- a/tree-sitter-stack-graphs/tests/it/nodes.rs +++ b/tree-sitter-stack-graphs/tests/it/nodes.rs @@ -288,8 +288,9 @@ fn can_calculate_spans() { #[test] fn can_set_definiens() { let tsg = r#" - (function_definition body:(_)@body) { + (function_definition name:(_)@name body:(_)@body) { node result + attr (result) type = "pop_symbol", symbol = (source-text @name), source_node = @name, is_definition attr (result) definiens_node = @body } "#; From 31fbee70cdb9a12cd43d0257634a37a785e937a5 Mon Sep 17 00:00:00 2001 From: Hendrik van Antwerpen Date: Mon, 9 Oct 2023 17:29:56 +0200 Subject: [PATCH 5/6] Document definiens_node --- tree-sitter-stack-graphs/src/lib.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tree-sitter-stack-graphs/src/lib.rs b/tree-sitter-stack-graphs/src/lib.rs index 564b32ba2..bca687b70 100644 --- a/tree-sitter-stack-graphs/src/lib.rs +++ b/tree-sitter-stack-graphs/src/lib.rs @@ -173,6 +173,20 @@ //! } //! ``` //! +//! ### Annotating definitions with definiens information +//! +//! You cannot annotate definitions with a definiens, which is the thing the definition covers. For example, for +//! a function definition, the definiens would be the function body. To do this, add a `definiens_node` attribute, +//! whose value is a syntax node that spans the definiens. +//! +//! ``` skip +//! (function_definition name: (identifier) @id body: (_) @body) @func { +//! node def +//! ; ... +//! attr (def) definiens_node = @body +//! } +//! ``` +//! //! ### Connecting stack graph nodes with edges //! //! To connect two stack graph nodes, use the `edge` statement to add an edge between them: From edc71106e3a10edc00434f9e3c88169fe02c35e6 Mon Sep 17 00:00:00 2001 From: Hendrik van Antwerpen Date: Mon, 9 Oct 2023 17:39:58 +0200 Subject: [PATCH 6/6] Allow null values for definiens_node --- tree-sitter-stack-graphs/src/lib.rs | 3 +++ tree-sitter-stack-graphs/tests/it/nodes.rs | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/tree-sitter-stack-graphs/src/lib.rs b/tree-sitter-stack-graphs/src/lib.rs index bca687b70..e8bdd7492 100644 --- a/tree-sitter-stack-graphs/src/lib.rs +++ b/tree-sitter-stack-graphs/src/lib.rs @@ -187,6 +187,8 @@ //! } //! ``` //! +//! Definiens are optional and setting them to `#null` explicitly is allowed. +//! //! ### Connecting stack graph nodes with edges //! //! To connect two stack graph nodes, use the `edge` statement to add an edge between them: @@ -1165,6 +1167,7 @@ impl<'a> Builder<'a> { ) -> Result<(), BuildError> { let node = &self.graph[node_ref]; let definiens_node = match node.attributes.get(DEFINIENS_NODE) { + Some(Value::Null) => return Ok(()), Some(definiens_node) => &self.graph[definiens_node.as_syntax_node_ref()?], None => return Ok(()), }; diff --git a/tree-sitter-stack-graphs/tests/it/nodes.rs b/tree-sitter-stack-graphs/tests/it/nodes.rs index ccb3e220e..c36a08744 100644 --- a/tree-sitter-stack-graphs/tests/it/nodes.rs +++ b/tree-sitter-stack-graphs/tests/it/nodes.rs @@ -313,6 +313,26 @@ fn can_set_definiens() { assert_eq!("2:8-2:12", actual_span) } +#[test] +fn can_set_null_definiens() { + let tsg = r#" + (function_definition name:(_)@name) { + node result + attr (result) type = "pop_symbol", symbol = (source-text @name), source_node = @name, is_definition + attr (result) definiens_node = #null + } + "#; + let python = r#" + def foo(): + pass + "#; + + let (graph, file) = build_stack_graph(python, tsg).unwrap(); + let node_handle = graph.nodes_for_file(file).next().unwrap(); + let source_info = graph.source_info(node_handle).unwrap(); + assert_eq!(lsp_positions::Span::default(), source_info.definiens_span) +} + #[test] fn can_set_syntax_type() { let tsg = r#"