Support for trailing commas & skip Comments with concrete syntax (#596)

experimental/requirements.txt

@@ -1,4 +1,4 @@
-tree-sitter
+tree-sitter==0.20.2
 tree-sitter-languages
 attrs
 openai
@@ -8,4 +8,4 @@ pytest
 flask
 flask-socketio
 comby
-eventlet
+eventlet

src/models/concrete_syntax.rs

@@ -17,6 +17,7 @@ use regex::Regex;
 
 use std::collections::HashMap;
 use tree_sitter::{Node, TreeCursor};
+use tree_sitter_traversal::Cursor;
 
 use crate::models::capture_group_patterns::ConcreteSyntax;
 use crate::models::matches::Match;
@@ -128,7 +129,13 @@ pub(crate) fn get_matches_for_node(
  );
  }
 
- let node = cursor.node();
+ let mut node = cursor.node();
+
+ // Skip comment nodes always
+ while node.kind().contains("comment") && cursor.goto_next_sibling() {
+ node = cursor.node();
+ }
+
  // In case the template starts with :[var_name], we try match
  if let Some(caps) = RE_VAR.captures(match_template) {
  let var_name = &caps["var_name"];
@@ -148,6 +155,15 @@ pub(crate) fn get_matches_for_node(
  let current_node_code = current_node.utf8_text(source_code).unwrap();
  find_next_sibling(&mut tmp_cursor);
 
+ // Support for trailing commas
+ // This skips trailing commas as we are parsing through the match template
+ // Skips the comma node if the template doesn't contain it.
+ let next_node = tmp_cursor.node();
+ let next_node_text = next_node.utf8_text(source_code).unwrap();
+ if next_node_text == "," && !meta_advanced.0.starts_with(',') {
+ find_next_sibling(&mut tmp_cursor); // Skip comma
+ }
+
  if let (mut recursive_matches, true) =
  get_matches_for_node(&mut tmp_cursor, source_code, &meta_advanced)
  {

src/models/unit_tests/concrete_syntax_test.rs

@@ -13,12 +13,14 @@
 
 use crate::models::capture_group_patterns::ConcreteSyntax;
 use crate::models::concrete_syntax::get_all_matches_for_concrete_syntax;
+use crate::models::default_configs::GO;
 use crate::models::{default_configs::JAVA, language::PiranhaLanguage};
 
 fn run_test(
  code: &str, pattern: &str, expected_matches: usize, expected_vars: Vec<Vec<(&str, &str)>>,
+ language: &str,
 ) {
- let java = PiranhaLanguage::from(JAVA);
+ let java = PiranhaLanguage::from(language);
  let mut parser = java.parser();
  let tree = parser.parse(code.as_bytes(), None).unwrap();
  let meta = ConcreteSyntax(String::from(pattern));
@@ -49,6 +51,7 @@ fn test_single_match() {
  "public int :[name] = :[value];",
  1,
  vec![vec![("name", "a"), ("value", "10")]],
+ JAVA,
  );
 }
 
@@ -62,6 +65,7 @@ fn test_multiple_match() {
  vec![("name", "a"), ("value", "10")],
  vec![("name", "b"), ("value", "20")],
  ],
+ JAVA,
  );
 }
 
@@ -72,5 +76,19 @@ fn test_no_match() {
  "public String :[name] = :[value];",
  0,
  vec![],
+ JAVA,
+ );
+}
+
+#[test]
+fn test_trailing_comma() {
+ run_test(
+ "a.foo(x, // something about the first argument
+ y, // something about the second argumet
+ );",
+ ":[var].foo(:[arg1], :[arg2])",
+ 2,
+ vec![vec![("var", "a"), ("arg1", "x"), ("arg2", "y")]],
+ GO,
  );
 }