diff --git a/Cargo.lock b/Cargo.lock index aa5f50f..d6a2323 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -20,6 +20,8 @@ dependencies = [ "tracing", "tracing-subscriber", "tree-sitter", + "tree-sitter-python", + "tree-sitter-rust", "unicode-width", ] @@ -770,6 +772,26 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8ddffe35a0e5eeeadf13ff7350af564c6e73993a24db62caee1822b185c2600" +[[package]] +name = "tree-sitter-python" +version = "0.23.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d065aaa27f3aaceaf60c1f0e0ac09e1cb9eb8ed28e7bcdaa52129cffc7f4b04" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-rust" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4d64d449ca63e683c562c7743946a646671ca23947b9c925c0cfbe65051a4af" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "unicode-ident" version = "1.0.13" diff --git a/Cargo.toml b/Cargo.toml index e0f717f..0880717 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,3 +54,5 @@ toml = "0.8.19" [dev-dependencies] simple_test_case = "1.2.0" criterion = "0.5" +tree-sitter-python = "0.23.6" +tree-sitter-rust = "0.23.2" diff --git a/data/config.toml b/data/config.toml index e763b5e..3cbc283 100644 --- a/data/config.toml +++ b/data/config.toml @@ -72,6 +72,7 @@ function = { fg = "#957FB8" } keyword = { fg = "#Bf616A" } module = { fg = "#2D4F67" } number = { fg = "#D27E99" } +operator = { fg = "#E6C384" } punctuation = { fg = "#9CABCA" } string = { fg = "#61DCA5" } type = { fg = "#7E9CD8" } diff --git a/data/tree-sitter/queries/dart/highlights.scm b/data/tree-sitter/queries/dart/highlights.scm index 4090824..a5fa9d4 100644 --- a/data/tree-sitter/queries/dart/highlights.scm +++ b/data/tree-sitter/queries/dart/highlights.scm @@ -210,4 +210,3 @@ "?." "?" ] @punctuation.delimiter - diff --git a/data/tree-sitter/queries/proto/highlights.scm b/data/tree-sitter/queries/proto/highlights.scm index 1bbb113..86d9aa7 100644 --- a/data/tree-sitter/queries/proto/highlights.scm +++ b/data/tree-sitter/queries/proto/highlights.scm @@ -57,6 +57,9 @@ (comment) @comment +((comment) @comment.documentation + (#match? @comment.documentation "^/[*][*][^*].*[*]/$")) + [ "(" ")" diff --git a/data/tree-sitter/queries/rust/highlights.scm b/data/tree-sitter/queries/rust/highlights.scm index 54c30f0..e1135f9 100644 --- a/data/tree-sitter/queries/rust/highlights.scm +++ b/data/tree-sitter/queries/rust/highlights.scm @@ -1,3 +1,5 @@ +(shebang) @keyword.directive + (function_item (identifier) @function) (function_signature_item (identifier) @function) (macro_invocation @@ -6,6 +8,10 @@ (identifier) @variable +; Assume all-caps names are constants +((identifier) @constant + (#match? @constant "^[A-Z][A-Z%d_]*$")) + (const_item name: (identifier) @constant) (type_identifier) @type @@ -18,10 +24,19 @@ name: (identifier) @module) (self) @keyword.builtin -"_" @character.special +[ + (line_comment) + (block_comment) + (outer_doc_comment_marker) + (inner_doc_comment_marker) +] @comment @spell + +(line_comment + (doc_comment)) @comment.documentation + +(block_comment + (doc_comment)) @comment.documentation -(line_comment) @comment -(block_comment) @comment (boolean_literal) @boolean (integer_literal) @number @@ -32,6 +47,14 @@ (string_literal) @string (raw_string_literal) @string +(use_wildcard + "*" @character.special) + +(remaining_field_pattern + ".." @character.special) + +"_" @character.special + ; Keywords [ "use" @@ -111,30 +134,6 @@ (closure_parameters "|" @punctuation.bracket) -(type_arguments - [ - "<" - ">" - ] @punctuation.bracket) - -(type_parameters - [ - "<" - ">" - ] @punctuation.bracket) - -(bracketed_type - [ - "<" - ">" - ] @punctuation.bracket) - -(for_lifetimes - [ - "<" - ">" - ] @punctuation.bracket) - [ "," "." @@ -185,17 +184,40 @@ "||" ] @operator -(use_wildcard - "*" @character.special) +(type_arguments + [ + "<" + ">" + ] @punctuation.bracket) -(remaining_field_pattern - ".." @character.special) +(type_parameters + [ + "<" + ">" + ] @punctuation.bracket) + +(bracketed_type + [ + "<" + ">" + ] @punctuation.bracket) + +(for_lifetimes + [ + "<" + ">" + ] @punctuation.bracket) + + +(attribute_item + "#" @punctuation.special) + +(inner_attribute_item + [ + "!" + "#" + ] @punctuation.special) -; (attribute_item -; "#" @punctuation.special) -; (inner_attribute_item -; [ -; "!" -; "#" -; ] @punctuation.special) +((identifier) @constant.builtin + (#any-of? @constant.builtin "Some" "None" "Ok" "Err")) diff --git a/data/tree-sitter/queries/yaml/highlights.scm b/data/tree-sitter/queries/yaml/highlights.scm index 0930626..3843248 100644 --- a/data/tree-sitter/queries/yaml/highlights.scm +++ b/data/tree-sitter/queries/yaml/highlights.scm @@ -1,3 +1,17 @@ +(boolean_scalar) @boolean +(null_scalar) @constant.builtin +(double_quote_scalar) @string +(single_quote_scalar) @string + +((block_scalar) @string + (#set! priority 99)) + +(string_scalar) @string +(escape_sequence) @string.escape +(integer_scalar) @number +(float_scalar) @number +(comment) @comment + [ (anchor_name) (alias_name) @@ -59,18 +73,3 @@ "---" "..." ] @punctuation.special - -(boolean_scalar) @boolean -(null_scalar) @constant.builtin -(double_quote_scalar) @string -(single_quote_scalar) @string - -((block_scalar) @string - (#set! priority 99)) - -(string_scalar) @string -(escape_sequence) @string.escape -(integer_scalar) @number -(float_scalar) @number -(comment) @comment @spell - diff --git a/docs/tree-sitter-queries.md b/docs/tree-sitter-queries.md new file mode 100644 index 0000000..2acf877 --- /dev/null +++ b/docs/tree-sitter-queries.md @@ -0,0 +1,60 @@ +# Writing tree-sitter queries for ad + +### Overview + +`ad` currently supports a subset of the tree-sitter based syntax highlighting you may be familiar +with from other text editors. The configuration options in your `config.toml` under the `tree_sitter` +key will direct `ad` to search for scheme highlights files under a given directory for the languages +that you configure. These query files can, for the most part, be copied from other editors but there +are a couple of caveats that you should keep in mind. + + 1. Support for custom predicates (such as neovim's `#lua-match?` and `#contains?` is not provided. + Queries using unsupported predicates will be rejected, resulting in tree-sitter not being enabled + for the language in question. + 2. ad follows the same precedence approach as neovim for overlapping queries, with the _last_ match + being used in place of previous matches. When writing your `highlights.scm` file you should place + more specific rules towards the end of the file and more general rules towards the start. + 3. ad does not currently provide any support for indentation, injections, locals or folds. So the + resulting highlights you obtain may not exactly match what you are used to from another editor if + you are copying over existing queries. + + +### Getting started + +ad provides some support for tree-sitter parsers that I have been able to test and verify as part of my +local setup. For the most part they are adapted from the ones found in [nvim-treesitter](https://github.com/nvim-treesitter/nvim-treesitter/tree/master/queries) +with the the modigications outlined below. If you are testing your own queries it is often better to start +with a subset of the `highlights.scm` file from another editor and incrementally add in more queries +while verifying that the results in ad look the way that you expect. + +Beyond that simple advice, the tree-sitter documentation on writing queries is the best place to start +if you are looking to configure your own custom queries: + https://tree-sitter.github.io/tree-sitter/using-parsers/queries/index.html + + +### Troubleshooting + +- **Where are my highlights that I configured?** + - First make sure that you don't have any typos in your config (we all do it) and that the highlighting + provided in the ad repo is working correctly. + - If you are sure that you have things set up correctly then try running `:view-logs` to see if there + are any warnings about the queries that you are using. You should see a log line asserting that the + tree-sitter parser has been initialised, and if not there should be an error message explaining why + setup failed. + +- **What is this warning about unsupported predicates?** + - A common feature of tree-sitter queries is the use of [predicates](https://tree-sitter.github.io/tree-sitter/using-parsers/queries/3-predicates-and-directives.html) + to conditionally select nodes in the tree based on their content as well as their type. Tree-sitter + supports providing and running custom predicates but it is up to the program running the query to + provide the implementation of how custom predicates are applied. `neovim` and other editors tend to + make extensive use of custom predicates that are not supported inside of ad. If a query you are + using contains any unsupported predicates it will be rejected with warning logs listing the predicates + that were found. + - Typically, you can replace `#lua-match?` with the tree-sitter built-in `#match?` predicate and have + it produce the expected result. + - You can also replace `#contains?` with `#match? .. ".*$contained_string.*"` + +- **Why are comments not rendering with the correct styling?** + - Comment queries from neovim frequently have a secondary `@spell` tag associated with them that ad + will end up using as the tag for the match (in place of, for example, `@comment`). If you remove + the `@spell` tag from your query you should see your comments receive the correct highlighting. diff --git a/src/buffer/internal.rs b/src/buffer/internal.rs index 19b93f4..fcde7c5 100644 --- a/src/buffer/internal.rs +++ b/src/buffer/internal.rs @@ -414,7 +414,7 @@ impl GapBuffer { } /// Convert a byte index to a character index - pub fn byte_to_char(&self, byte_idx: usize) -> usize { + pub fn raw_byte_to_char(&self, byte_idx: usize) -> usize { self.chars_in_raw_range(0, byte_idx) } @@ -739,7 +739,7 @@ impl GapBuffer { } #[inline] - fn byte_to_raw_byte(&self, byte: usize) -> usize { + pub fn byte_to_raw_byte(&self, byte: usize) -> usize { if byte > self.gap_start { byte + self.gap() } else { diff --git a/src/buffer/mod.rs b/src/buffer/mod.rs index 8858016..f53be55 100644 --- a/src/buffer/mod.rs +++ b/src/buffer/mod.rs @@ -410,6 +410,10 @@ impl Buffer { s } + pub(crate) fn pretty_print_ts_tree(&self) -> Option { + self.ts_state.as_ref().map(|ts| ts.pretty_print_tree()) + } + pub(crate) fn string_lines(&self) -> Vec { self.txt .iter_lines() diff --git a/src/editor/actions.rs b/src/editor/actions.rs index 5550ec3..04d0116 100644 --- a/src/editor/actions.rs +++ b/src/editor/actions.rs @@ -118,6 +118,7 @@ pub enum Action { ShellRun { cmd: String }, ShellSend { cmd: String }, ShowHelp, + TsShowTree, Undo, UpdateConfig { input: String }, ViewLogs, @@ -495,6 +496,13 @@ where .open_virtual("+logs", self.log_buffer.content(), false) } + pub(super) fn show_active_ts_tree(&mut self) { + match self.layout.active_buffer().pretty_print_ts_tree() { + Some(s) => self.layout.open_virtual("+ts-tree", s, false), + None => self.set_status_message("no tree-sitter tree for current buffer"), + } + } + pub(super) fn show_help(&mut self) { self.layout.open_virtual("+help", gen_help_docs(), false) } diff --git a/src/editor/commands.rs b/src/editor/commands.rs index 850aa40..0be532a 100644 --- a/src/editor/commands.rs +++ b/src/editor/commands.rs @@ -138,6 +138,8 @@ fn parse_command(input: &str, active_buffer_id: usize, cwd: &Path) -> Result Ok(Single(TsShowTree)), + "view-logs" => Ok(Single(ViewLogs)), "w" | "write" => { diff --git a/src/editor/mod.rs b/src/editor/mod.rs index ceea600..6134ac1 100644 --- a/src/editor/mod.rs +++ b/src/editor/mod.rs @@ -492,6 +492,7 @@ where ShellReplace { cmd } => self.replace_dot_with_shell_cmd(&cmd), ShellRun { cmd } => self.run_shell_cmd(&cmd), ShowHelp => self.show_help(), + TsShowTree => self.show_active_ts_tree(), UpdateConfig { input } => self.update_config(&input), ViewLogs => self.view_logs(), Yank => self.set_clipboard(self.layout.active_buffer().dot_contents()), diff --git a/src/lsp/capabilities.rs b/src/lsp/capabilities.rs index 01ed6c5..1abfd67 100644 --- a/src/lsp/capabilities.rs +++ b/src/lsp/capabilities.rs @@ -86,7 +86,7 @@ impl PositionEncoding { Self::Utf8 => { let line_start = b.txt.line_to_char(pos.line as usize); let byte_idx = b.txt.char_to_byte(line_start + pos.character as usize); - let col = b.txt.byte_to_char(byte_idx); + let col = b.txt.raw_byte_to_char(byte_idx); (pos.line as usize, col) } diff --git a/src/ts.rs b/src/ts.rs index 23b7317..e05e43a 100644 --- a/src/ts.rs +++ b/src/ts.rs @@ -26,19 +26,22 @@ use crate::{ use libloading::{Library, Symbol}; use std::{ cmp::{max, min, Ord, Ordering, PartialOrd}, + collections::HashSet, fmt, fs, - iter::Peekable, + iter::{repeat_n, Peekable}, ops::{Deref, DerefMut}, path::Path, slice, }; use streaming_iterator::StreamingIterator; +use tracing::{error, info}; use tree_sitter::{self as ts, ffi::TSLanguage}; pub const TK_DEFAULT: &str = "default"; pub const TK_DOT: &str = "dot"; pub const TK_LOAD: &str = "load"; pub const TK_EXEC: &str = "exec"; +pub const SUPPORTED_PREDICATES: [&str; 0] = []; /// Buffer level tree-sitter state for parsing and highlighting #[derive(Debug)] @@ -61,15 +64,34 @@ impl TsState { Err(e) => return Err(format!("unable to read tree-sitter query file: {e}")), }; - let mut p = Parser::try_new(so_dir, lang)?; + let p = Parser::try_new(so_dir, lang)?; + + Self::try_new_explicit(p, &query, gb) + } + + #[cfg(test)] + fn try_new_from_language( + lang_name: &str, + lang: ts::Language, + query: &str, + gb: &GapBuffer, + ) -> Result { + let p = Parser::try_new_from_language(lang_name, lang)?; + + Self::try_new_explicit(p, query, gb) + } + + fn try_new_explicit(mut p: Parser, query: &str, gb: &GapBuffer) -> Result { let tree = p.parse_with( &mut |byte_offset, _| gb.maximal_slice_from_offset(byte_offset), None, ); + match tree { Some(tree) => { - let mut t = p.new_tokenizer(&query)?; + let mut t = p.new_tokenizer(query)?; t.update(tree.root_node(), gb); + info!("TS loaded for {}", p.lang_name); Ok(Self { p, t, tree }) } None => Err("failed to parse file".to_owned()), @@ -127,6 +149,41 @@ impl TsState { self.t .iter_tokenized_lines_from(line, gb, dot_range, load_exec_range) } + + pub fn pretty_print_tree(&self) -> String { + let sexp = self.tree.root_node().to_sexp(); + let mut buf = String::with_capacity(sexp.len()); // better starting point than default + let mut has_field = false; + let mut indent = 0; + + for s in sexp.split([' ', ')']) { + if s.is_empty() { + indent -= 1; + buf.push(')'); + } else if s.starts_with('(') { + if has_field { + has_field = false; + } else { + if indent > 0 { + buf.push('\n'); + buf.extend(repeat_n(' ', indent * 2)); + } + indent += 1; + } + + buf.push_str(s); // "(node_name" + } else if s.ends_with(':') { + buf.push('\n'); + buf.extend(repeat_n(' ', indent * 2)); + buf.push_str(s); // "field:" + buf.push(' '); + has_field = true; + indent += 1; + } + } + + buf + } } // Required for us to be able to pass GapBuffers to the tree-sitter API @@ -139,8 +196,8 @@ impl<'a> ts::TextProvider<&'a [u8]> for &'a GapBuffer { end_byte, .. } = node.range(); - let char_from = self.byte_to_char(start_byte); - let char_to = self.byte_to_char(end_byte); + let char_from = self.raw_byte_to_char(self.byte_to_raw_byte(start_byte)); + let char_to = self.raw_byte_to_char(self.byte_to_raw_byte(end_byte)); self.slice(char_from, char_to).slice_iter() } @@ -151,7 +208,9 @@ pub struct Parser { lang_name: String, inner: ts::Parser, lang: ts::Language, - _lib: Library, // Need to prevent drop while the parser is in use + // Need to prevent drop while the parser is in use + // Stored as an Option to allow for crate-based parsers that are not backed by a .so file + _lib: Option, } impl Deref for Parser { @@ -204,15 +263,53 @@ impl Parser { lang_name: lang_name.to_owned(), inner, lang, - _lib: lib, + _lib: Some(lib), }) } } + /// Construct a new tokenizer directly from a ts::Language provided by a crate + #[cfg(test)] + fn try_new_from_language(lang_name: &str, lang: ts::Language) -> Result { + let mut inner = ts::Parser::new(); + inner.set_language(&lang).map_err(|e| e.to_string())?; + + Ok(Self { + lang_name: lang_name.to_owned(), + inner, + lang, + _lib: None, + }) + } + pub fn new_tokenizer(&self, query: &str) -> Result { let q = ts::Query::new(&self.lang, query).map_err(|e| format!("{e:?}"))?; let cur = ts::QueryCursor::new(); + // If a query has been copied from another text editor then there is a chance that + // it makes use of custom predicates that we don't know how to handle. The highlights + // as a whole won't behave as the user expects in this instance so we error out the + // setup of syntax-highlighting as a whole in this case and log an error + let mut unsupported_predicates = HashSet::new(); + for i in 0..q.pattern_count() { + for p in q.general_predicates(i) { + if !SUPPORTED_PREDICATES.contains(&p.operator.as_ref()) { + unsupported_predicates.insert(p.operator.clone()); + } + } + } + + if !unsupported_predicates.is_empty() { + error!("Unsupported custom tree-sitter predicates found: {unsupported_predicates:?}"); + info!("Supported custom tree-sitter predicates: {SUPPORTED_PREDICATES:?}"); + info!("Please modify the highlights.scm file to remove the unsupported predicates"); + + return Err(format!( + "{} highlights query contained unsupported custom predicates", + self.lang_name + )); + } + Ok(Tokenizer { q, cur, @@ -236,36 +333,29 @@ impl fmt::Debug for Tokenizer { } impl Tokenizer { - // Compound queries such as the example below can result in duplicate nodes being returned - // from the caputures iterator in both the init and update methods. As such, we need to sort - // and dedupe the list of resulting syntax ranges in order to correctly ensure that we have - // no overlapping or duplicated tokens emitted. - // - // (macro_invocation - // macro: (identifier) @function.macro - // "!" @function.macro) - pub fn update(&mut self, root: ts::Node<'_>, gb: &GapBuffer) { // This is a streaming-iterator not an interator, hence the odd while-let that follows let mut it = self.cur.captures(&self.q, root, gb); // FIXME: this is really inefficient. Ideally we should be able to apply a diff here self.ranges.clear(); - while let Some((m, _)) = it.next() { - for cap_idx in 0..self.q.capture_names().len() { - for node in m.nodes_for_capture_index(cap_idx as u32) { - let r = ByteRange::from(node.range()); - if let Some(prev) = self.ranges.last() { - if r.from < prev.r.to && prev.r.from < r.to { - continue; - } - } - self.ranges.push(SyntaxRange { - r, - cap_idx: Some(cap_idx), - }); + while let Some((m, idx)) = it.next() { + let cap = m.captures[*idx]; + let r = ByteRange::from(cap.node.range()); + if let Some(prev) = self.ranges.last_mut() { + if r == prev.r { + // prefering the the last capture found so that precedence ordering + // in query files matches Neovim & the treesitter-cli + prev.cap_idx = Some(cap.index as usize); + continue; + } else if r.from < prev.r.to && prev.r.from < r.to { + continue; } } + self.ranges.push(SyntaxRange { + r, + cap_idx: Some(cap.index as usize), + }); } self.ranges.sort_unstable(); @@ -290,6 +380,19 @@ impl Tokenizer { &self.ranges, ) } + + #[cfg(test)] + fn range_tokens(&self) -> Vec> { + let names = self.q.capture_names(); + + self.ranges + .iter() + .map(|sr| RangeToken { + tag: sr.cap_idx.map(|i| names[i]).unwrap_or(TK_DEFAULT), + r: sr.r, + }) + .collect() + } } /// Byte offsets within a Buffer @@ -1100,49 +1203,128 @@ mod tests { assert_eq!(held, expected); } + fn rt(tag: &str, from: usize, to: usize) -> RangeToken<'_> { + RangeToken { + tag, + r: ByteRange { from, to }, + } + } + #[test] - #[ignore = "this test requires installed parsers and queries"] fn char_delete_correctly_update_state() { + // minimal query for the fn keyword and parens + let query = r#" +"fn" @keyword + +[ "(" ")" "{" "}" ] @punctuation"#; + let s = "fn main() {}"; let mut b = Buffer::new_unnamed(0, s); + let gb = &b.txt; b.ts_state = Some( - TsState::try_new( - "rust", - "/home/sminez/.local/share/nvim/lazy/nvim-treesitter/parser", - "data/tree-sitter/queries", - &b.txt, - ) - .unwrap(), + TsState::try_new_from_language("rust", tree_sitter_rust::LANGUAGE.into(), query, gb) + .unwrap(), ); - let ranges = b.ts_state.as_ref().unwrap().t.ranges.clone(); - let sr = |idx, from, to| SyntaxRange { - cap_idx: Some(idx), - r: ByteRange { from, to }, - }; - assert_eq!(b.str_contents(), "fn main() {}\n"); assert_eq!( - ranges, + b.ts_state.as_ref().unwrap().t.range_tokens(), vec![ - sr(5, 0, 2), // fn - sr(14, 7, 8), // ( - sr(14, 8, 9), // ) - sr(14, 10, 11), // { - sr(14, 11, 12), // } + rt("keyword", 0, 2), // fn + rt("punctuation", 7, 8), // ( + rt("punctuation", 8, 9), // ) + rt("punctuation", 10, 11), // { + rt("punctuation", 11, 12), // } ] ); b.dot = Dot::Cur { c: Cur { idx: 9 } }; b.handle_action(Action::Delete, Source::Fsys); b.ts_state.as_mut().unwrap().update(&b.txt); - let ranges = b.ts_state.as_ref().unwrap().t.ranges.clone(); + let ranges = b.ts_state.as_ref().unwrap().t.range_tokens(); assert_eq!(b.str_contents(), "fn main(){}\n"); assert_eq!(ranges.len(), 5); // these two should have moved left one character - assert_eq!(ranges[3], sr(14, 9, 10), "opening curly"); - assert_eq!(ranges[4], sr(14, 10, 11), "closing curly"); + assert_eq!(ranges[3], rt("punctuation", 9, 10), "opening curly"); + assert_eq!(ranges[4], rt("punctuation", 10, 11), "closing curly"); + } + + #[test] + fn overlapping_tokens_prefer_previous_matches() { + // Minimal query extracted from the full query in gh#88 that resulted in + // overlapping tokens being produced + let query = r#" +(identifier) @variable + +(import_statement + name: (dotted_name + (identifier) @module)) + +(import_statement + name: (aliased_import + name: (dotted_name + (identifier) @module) + alias: (identifier) @module)) + +(import_from_statement + module_name: (dotted_name + (identifier) @module))"#; + + let s = "import builtins as _builtins"; + let b = Buffer::new_unnamed(0, s); + let gb = &b.txt; + let ts = TsState::try_new_from_language( + "python", + tree_sitter_python::LANGUAGE.into(), + query, + gb, + ) + .unwrap(); + + assert_eq!( + ts.t.range_tokens(), + vec![ + rt("module", 7, 15), // builtins + rt("module", 19, 28) // _builtins + ] + ); + } + + #[test] + fn built_in_predicates_work() { + let query = r#" +(identifier) @variable + +; Assume all-caps names are constants +((identifier) @constant + (#match? @constant "^[A-Z][A-Z%d_]*$")) + +((identifier) @constant.builtin + (#any-of? @constant.builtin "Some" "None" "Ok" "Err")) + +[ "(" ")" "{" "}" ] @punctuation"#; + + let s = "Ok(Some(42)) foo BAR"; + let b = Buffer::new_unnamed(0, s); + let gb = &b.txt; + let ts = + TsState::try_new_from_language("rust", tree_sitter_rust::LANGUAGE.into(), query, gb) + .unwrap(); + + assert_eq!( + ts.t.range_tokens(), + vec![ + rt("constant.builtin", 0, 2), // Ok + rt("punctuation", 2, 3), // ( + rt("constant.builtin", 3, 7), // Some + rt("punctuation", 7, 8), // ( + rt("punctuation", 10, 11), // ) + rt("punctuation", 11, 12), // ) + rt("variable", 13, 16), // foo + rt("constant", 17, 20), // BAR + ] + ); } }