Skip to content

Commit 895a048

Browse files
kofukkofuk
authored
Port Minecraft lexers from Pygments (#992)
Ported lexers for mcfuntion, snbt from Pygments using `pygments2chroma_xml.py` script. While doing so, I encountered lack of `LiteralNumberByte` in TokenType, so I've added the type and regenerated tokentype_enumer.go.
1 parent 763c2ea commit 895a048

File tree

8 files changed

+492
-327
lines changed

8 files changed

+492
-327
lines changed

‎README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,13 +45,13 @@ translators for Pygments lexers and styles.
4545
| J | J, Java, JavaScript, JSON, Julia, Jungle |
4646
| K | Kotlin |
4747
| L | Lighttpd configuration file, LLVM, Lua |
48-
| M | Makefile, Mako, markdown, Mason, Materialize SQL dialect, Mathematica, Matlab, mcfunction, Meson, Metal, MiniZinc, MLIR, Modula-2, MonkeyC, MorrowindScript, Myghty, MySQL |
48+
| M | Makefile, Mako, markdown, Mason, Materialize SQL dialect, Mathematica, Matlab, MCFunction, Meson, Metal, MiniZinc, MLIR, Modula-2, MonkeyC, MorrowindScript, Myghty, MySQL |
4949
| N | NASM, Natural, Newspeak, Nginx configuration file, Nim, Nix |
5050
| O | Objective-C, OCaml, Octave, Odin, OnesEnterprise, OpenEdge ABL, OpenSCAD, Org Mode |
5151
| P | PacmanConf, Perl, PHP, PHTML, Pig, PkgConfig, PL/pgSQL, plaintext, Plutus Core, Pony, PostgreSQL SQL dialect, PostScript, POVRay, PowerQuery, PowerShell, Prolog, PromQL, Promela, properties, Protocol Buffer, PRQL, PSL, Puppet, Python, Python 2 |
5252
| Q | QBasic, QML |
5353
| R | R, Racket, Ragel, Raku, react, ReasonML, reg, Rego, reStructuredText, Rexx, RPMSpec, Ruby, Rust |
54-
| S | SAS, Sass, Scala, Scheme, Scilab, SCSS, Sed, Sieve, Smali, Smalltalk, Smarty, Snobol, Solidity, SourcePawn, SPARQL, SQL, SquidConf, Standard ML, stas, Stylus, Svelte, Swift, SYSTEMD, systemverilog |
54+
| S | SAS, Sass, Scala, Scheme, Scilab, SCSS, Sed, Sieve, Smali, Smalltalk, Smarty, SNBT, Snobol, Solidity, SourcePawn, SPARQL, SQL, SquidConf, Standard ML, stas, Stylus, Svelte, Swift, SYSTEMD, systemverilog |
5555
| T | TableGen, Tal, TASM, Tcl, Tcsh, Termcap, Terminfo, Terraform, TeX, Thrift, TOML, TradingView, Transact-SQL, Turing, Turtle, Twig, TypeScript, TypoScript, TypoScriptCssData, TypoScriptHtmlData |
5656
| V | V, V shell, Vala, VB.net, verilog, VHDL, VHS, VimL, vue |
5757
| W | WDTE, WebGPU Shading Language, Whiley |

‎lexers/embedded/mcfunction.xml‎

Lines changed: 128 additions & 172 deletions
Original file line numberDiff line numberDiff line change
@@ -1,182 +1,138 @@
1+
12
<lexer>
23
<config>
3-
<name>mcfunction</name>
4+
<name>MCFunction</name>
45
<alias>mcfunction</alias>
6+
<alias>mcf</alias>
57
<filename>*.mcfunction</filename>
6-
<dot_all>true</dot_all>
7-
<not_multiline>true</not_multiline>
8+
<mime_type>text/mcfunction</mime_type>
89
</config>
910
<rules>
10-
<state name="nbtobjectvalue">
11-
<rule pattern="(&#34;(\\\\|\\&#34;|[^&#34;])*&#34;|[a-zA-Z0-9_]+)">
12-
<token type="NameTag"/>
13-
<push state="nbtobjectattribute"/>
14-
</rule>
15-
<rule pattern="\}">
16-
<token type="Punctuation"/>
17-
<pop depth="1"/>
18-
</rule>
19-
</state>
20-
<state name="nbtarrayvalue">
21-
<rule>
22-
<include state="nbtvalue"/>
23-
</rule>
24-
<rule pattern=",">
25-
<token type="Punctuation"/>
26-
</rule>
27-
<rule pattern="\]">
28-
<token type="Punctuation"/>
29-
<pop depth="1"/>
30-
</rule>
31-
</state>
32-
<state name="nbtvalue">
33-
<rule>
34-
<include state="simplevalue"/>
35-
</rule>
36-
<rule pattern="\{">
37-
<token type="Punctuation"/>
38-
<push state="nbtobjectvalue"/>
39-
</rule>
40-
<rule pattern="\[">
41-
<token type="Punctuation"/>
42-
<push state="nbtarrayvalue"/>
43-
</rule>
44-
</state>
45-
<state name="argumentvalue">
46-
<rule>
47-
<include state="simplevalue"/>
48-
</rule>
49-
<rule pattern=",">
50-
<token type="Punctuation"/>
51-
<pop depth="1"/>
52-
</rule>
53-
<rule pattern="[}\]]">
54-
<token type="Punctuation"/>
55-
<pop depth="2"/>
56-
</rule>
57-
</state>
58-
<state name="argumentlist">
59-
<rule pattern="(nbt)(={)">
60-
<bygroups>
61-
<token type="NameAttribute"/>
62-
<token type="Punctuation"/>
63-
</bygroups>
64-
<push state="nbtobjectvalue"/>
65-
</rule>
66-
<rule pattern="([A-Za-z0-9/_!]+)(={)">
67-
<bygroups>
68-
<token type="NameAttribute"/>
69-
<token type="Punctuation"/>
70-
</bygroups>
71-
<push state="argumentlist"/>
72-
</rule>
73-
<rule pattern="([A-Za-z0-9/_!]+)(=)">
74-
<bygroups>
75-
<token type="NameAttribute"/>
76-
<token type="Punctuation"/>
77-
</bygroups>
78-
<push state="argumentvalue"/>
79-
</rule>
80-
<rule>
81-
<include state="simplevalue"/>
82-
</rule>
83-
<rule pattern=",">
84-
<token type="Punctuation"/>
85-
</rule>
86-
<rule pattern="[}\]]">
87-
<token type="Punctuation"/>
88-
<pop depth="1"/>
89-
</rule>
90-
</state>
9111
<state name="root">
92-
<rule pattern="#.*?\n">
93-
<token type="CommentSingle"/>
94-
</rule>
95-
<rule pattern="/?(geteduclientinfo|clearspawnpoint|defaultgamemode|transferserver|toggledownfall|immutableworld|detectredstone|setidletimeout|playanimation|classroommode|spreadplayers|testforblocks|setmaxplayers|setworldspawn|testforblock|worldbuilder|createagent|worldborder|camerashake|advancement|raytracefog|locatebiome|tickingarea|replaceitem|attributes|spawnpoint|difficulty|experience|scoreboard|whitelist|structure|playsound|stopsound|forceload|spectate|gamerule|function|schedule|wsserver|teleport|position|save-off|particle|setblock|datapack|mobevent|transfer|gamemode|save-all|bossbar|enchant|trigger|collect|execute|weather|teammsg|tpagent|banlist|dropall|publish|tellraw|testfor|save-on|destroy|ability|locate|summon|remove|effect|reload|ban-ip|recipe|pardon|detect|music|clear|clone|event|mixer|debug|title|ride|stop|list|turn|data|team|kick|loot|tell|help|give|flog|fill|move|time|seed|kill|save|item|deop|code|tag|ban|msg|say|tp|me|op|xp|w|place)\b">
96-
<token type="KeywordReserved"/>
97-
</rule>
98-
<rule pattern="(@p|@r|@a|@e|@s|@c|@v)">
99-
<token type="KeywordConstant"/>
100-
</rule>
101-
<rule pattern="\[">
102-
<token type="Punctuation"/>
103-
<push state="argumentlist"/>
104-
</rule>
105-
<rule pattern="{">
106-
<token type="Punctuation"/>
107-
<push state="nbtobjectvalue"/>
108-
</rule>
109-
<rule pattern="~">
110-
<token type="NameBuiltin"/>
111-
</rule>
112-
<rule pattern="([a-zA-Z_]+:)?[a-zA-Z_]+\b">
113-
<token type="Text"/>
114-
</rule>
115-
<rule pattern="([a-z]+)(\.)([0-9]+)\b">
116-
<bygroups>
117-
<token type="Text"/>
118-
<token type="Punctuation"/>
119-
<token type="LiteralNumber"/>
120-
</bygroups>
121-
</rule>
122-
<rule pattern="([&lt;&gt;=]|&lt;=|&gt;=)">
123-
<token type="Punctuation"/>
124-
</rule>
125-
<rule>
126-
<include state="simplevalue"/>
127-
</rule>
128-
<rule pattern="\s+">
129-
<token type="TextWhitespace"/>
130-
</rule>
131-
</state>
132-
<state name="simplevalue">
133-
<rule pattern="(true|false)">
134-
<token type="KeywordConstant"/>
135-
</rule>
136-
<rule pattern="[01]b">
137-
<token type="LiteralNumber"/>
138-
</rule>
139-
<rule pattern="-?(0|[1-9]\d*)(\.\d+[eE](\+|-)?\d+|[eE](\+|-)?\d+|\.\d+)">
140-
<token type="LiteralNumberFloat"/>
141-
</rule>
142-
<rule pattern="(-?\d+)(\.\.)(-?\d+)">
143-
<bygroups>
144-
<token type="LiteralNumberInteger"/>
145-
<token type="Punctuation"/>
146-
<token type="LiteralNumberInteger"/>
147-
</bygroups>
148-
</rule>
149-
<rule pattern="-?(0|[1-9]\d*)">
150-
<token type="LiteralNumberInteger"/>
151-
</rule>
152-
<rule pattern="&#34;(\\\\|\\&#34;|[^&#34;])*&#34;">
153-
<token type="LiteralStringDouble"/>
154-
</rule>
155-
<rule pattern="&#39;[^&#39;]+&#39;">
156-
<token type="LiteralStringSingle"/>
157-
</rule>
158-
<rule pattern="([!#]?)(\w+)">
159-
<bygroups>
160-
<token type="Punctuation"/>
161-
<token type="Text"/>
162-
</bygroups>
163-
</rule>
164-
</state>
165-
<state name="nbtobjectattribute">
166-
<rule>
167-
<include state="nbtvalue"/>
168-
</rule>
169-
<rule pattern=":">
170-
<token type="Punctuation"/>
171-
</rule>
172-
<rule pattern=",">
173-
<token type="Punctuation"/>
174-
<pop depth="1"/>
175-
</rule>
176-
<rule pattern="\}">
177-
<token type="Punctuation"/>
178-
<pop depth="2"/>
179-
</rule>
12+
<rule><include state="names"/></rule>
13+
<rule><include state="comments"/></rule>
14+
<rule><include state="literals"/></rule>
15+
<rule><include state="whitespace"/></rule>
16+
<rule><include state="property"/></rule>
17+
<rule><include state="operators"/></rule>
18+
<rule><include state="selectors"/></rule>
19+
</state>
20+
<state name="names">
21+
<rule pattern="^(\s*)([a-z_]+)"><bygroups><token type="TextWhitespace"/><token type="NameBuiltin"/></bygroups></rule>
22+
<rule pattern="(?&lt;=run)\s+[a-z_]+"><token type="NameBuiltin"/></rule>
23+
<rule pattern="\b[0-9a-fA-F]+(?:-[0-9a-fA-F]+){4}\b"><token type="NameVariable"/></rule>
24+
<rule><include state="resource-name"/></rule>
25+
<rule pattern="[A-Za-z_][\w.#%$]+"><token type="KeywordConstant"/></rule>
26+
<rule pattern="[#%$][\w.#%$]+"><token type="NameVariableMagic"/></rule>
27+
</state>
28+
<state name="resource-name">
29+
<rule pattern="#?[a-z_][a-z_.-]*:[a-z0-9_./-]+"><token type="NameFunction"/></rule>
30+
<rule pattern="#?[a-z0-9_\.\-]+\/[a-z0-9_\.\-\/]+"><token type="NameFunction"/></rule>
31+
</state>
32+
<state name="whitespace">
33+
<rule pattern="\s+"><token type="TextWhitespace"/></rule>
34+
</state>
35+
<state name="comments">
36+
<rule pattern="^\s*(#[&gt;!])"><token type="CommentMultiline"/><push state="comments.block" state="comments.block.emphasized"/></rule>
37+
<rule pattern="#.*$"><token type="CommentSingle"/></rule>
38+
</state>
39+
<state name="comments.block">
40+
<rule pattern="^\s*#[&gt;!]"><token type="CommentMultiline"/><push state="comments.block.emphasized"/></rule>
41+
<rule pattern="^\s*#"><token type="CommentMultiline"/><push state="comments.block.normal"/></rule>
42+
<rule><pop depth="1"/></rule>
43+
</state>
44+
<state name="comments.block.normal">
45+
<rule><include state="comments.block.special"/></rule>
46+
<rule pattern="\S+"><token type="CommentMultiline"/></rule>
47+
<rule pattern="\n"><token type="Text"/><pop depth="1"/></rule>
48+
<rule><include state="whitespace"/></rule>
49+
</state>
50+
<state name="comments.block.emphasized">
51+
<rule><include state="comments.block.special"/></rule>
52+
<rule pattern="\S+"><token type="LiteralStringDoc"/></rule>
53+
<rule pattern="\n"><token type="Text"/><pop depth="1"/></rule>
54+
<rule><include state="whitespace"/></rule>
55+
</state>
56+
<state name="comments.block.special">
57+
<rule pattern="@\S+"><token type="NameDecorator"/></rule>
58+
<rule><include state="resource-name"/></rule>
59+
<rule pattern="[#%$][\w.#%$]+"><token type="NameVariableMagic"/></rule>
60+
</state>
61+
<state name="operators">
62+
<rule pattern="[\-~%^?!+*&lt;&gt;\\/|&amp;=.]"><token type="Operator"/></rule>
63+
</state>
64+
<state name="literals">
65+
<rule pattern="\.\."><token type="Literal"/></rule>
66+
<rule pattern="(true|false)"><token type="KeywordPseudo"/></rule>
67+
<rule pattern="[A-Za-z_]+"><token type="NameVariableClass"/></rule>
68+
<rule pattern="[0-7]b"><token type="LiteralNumberByte"/></rule>
69+
<rule pattern="[+-]?\d*\.?\d+([eE]?[+-]?\d+)?[df]?\b"><token type="LiteralNumberFloat"/></rule>
70+
<rule pattern="[+-]?\d+\b"><token type="LiteralNumberInteger"/></rule>
71+
<rule pattern="&quot;"><token type="LiteralStringDouble"/><push state="literals.string-double"/></rule>
72+
<rule pattern="&#x27;"><token type="LiteralStringSingle"/><push state="literals.string-single"/></rule>
73+
</state>
74+
<state name="literals.string-double">
75+
<rule pattern="\\."><token type="LiteralStringEscape"/></rule>
76+
<rule pattern="[^\\&quot;\n]+"><token type="LiteralStringDouble"/></rule>
77+
<rule pattern="&quot;"><token type="LiteralStringDouble"/><pop depth="1"/></rule>
78+
</state>
79+
<state name="literals.string-single">
80+
<rule pattern="\\."><token type="LiteralStringEscape"/></rule>
81+
<rule pattern="[^\\&#x27;\n]+"><token type="LiteralStringSingle"/></rule>
82+
<rule pattern="&#x27;"><token type="LiteralStringSingle"/><pop depth="1"/></rule>
83+
</state>
84+
<state name="selectors">
85+
<rule pattern="@[a-z]"><token type="NameVariable"/></rule>
86+
</state>
87+
<state name="property">
88+
<rule pattern="\{"><token type="Punctuation"/><push state="property.curly" state="property.key"/></rule>
89+
<rule pattern="\["><token type="Punctuation"/><push state="property.square" state="property.key"/></rule>
90+
</state>
91+
<state name="property.curly">
92+
<rule><include state="whitespace"/></rule>
93+
<rule><include state="property"/></rule>
94+
<rule pattern="\}"><token type="Punctuation"/><pop depth="1"/></rule>
95+
</state>
96+
<state name="property.square">
97+
<rule><include state="whitespace"/></rule>
98+
<rule><include state="property"/></rule>
99+
<rule pattern="\]"><token type="Punctuation"/><pop depth="1"/></rule>
100+
<rule pattern=","><token type="Punctuation"/></rule>
101+
</state>
102+
<state name="property.key">
103+
<rule><include state="whitespace"/></rule>
104+
<rule pattern="#?[a-z_][a-z_\.\-]*\:[a-z0-9_\.\-/]+(?=\s*\=)"><token type="NameAttribute"/><push state="property.delimiter"/></rule>
105+
<rule pattern="#?[a-z_][a-z0-9_\.\-/]+"><token type="NameAttribute"/><push state="property.delimiter"/></rule>
106+
<rule pattern="[A-Za-z_\-\+]+"><token type="NameAttribute"/><push state="property.delimiter"/></rule>
107+
<rule pattern="&quot;"><token type="NameAttribute"/><push state="property.delimiter"/></rule>
108+
<rule pattern="&#x27;"><token type="NameAttribute"/><push state="property.delimiter"/></rule>
109+
<rule pattern="-?\d+"><token type="LiteralNumberInteger"/><push state="property.delimiter"/></rule>
110+
<rule><pop depth="1"/></rule>
111+
</state>
112+
<state name="property.key.string-double">
113+
<rule pattern="\\."><token type="LiteralStringEscape"/></rule>
114+
<rule pattern="[^\\&quot;\n]+"><token type="NameAttribute"/></rule>
115+
<rule pattern="&quot;"><token type="NameAttribute"/><pop depth="1"/></rule>
116+
</state>
117+
<state name="property.key.string-single">
118+
<rule pattern="\\."><token type="LiteralStringEscape"/></rule>
119+
<rule pattern="[^\\&#x27;\n]+"><token type="NameAttribute"/></rule>
120+
<rule pattern="&#x27;"><token type="NameAttribute"/><pop depth="1"/></rule>
121+
</state>
122+
<state name="property.delimiter">
123+
<rule><include state="whitespace"/></rule>
124+
<rule pattern="[:=]!?"><token type="Punctuation"/><push state="property.value"/></rule>
125+
<rule pattern=","><token type="Punctuation"/></rule>
126+
<rule><pop depth="1"/></rule>
127+
</state>
128+
<state name="property.value">
129+
<rule><include state="whitespace"/></rule>
130+
<rule pattern="#?[a-z_][a-z_\.\-]*\:[a-z0-9_\.\-/]+"><token type="NameTag"/></rule>
131+
<rule pattern="#?[a-z_][a-z0-9_\.\-/]+"><token type="NameTag"/></rule>
132+
<rule><include state="literals"/></rule>
133+
<rule><include state="property"/></rule>
134+
<rule><pop depth="1"/></rule>
180135
</state>
181136
</rules>
182137
</lexer>
138+

‎lexers/embedded/snbt.xml‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
2+
<lexer>
3+
<config>
4+
<name>SNBT</name>
5+
<alias>snbt</alias>
6+
<filename>*.snbt</filename>
7+
<mime_type>text/snbt</mime_type>
8+
</config>
9+
<rules>
10+
<state name="root">
11+
<rule pattern="\{"><token type="Punctuation"/><push state="compound"/></rule>
12+
<rule pattern="[^\{]+"><token type="Text"/></rule>
13+
</state>
14+
<state name="whitespace">
15+
<rule pattern="\s+"><token type="TextWhitespace"/></rule>
16+
</state>
17+
<state name="operators">
18+
<rule pattern="[,:;]"><token type="Punctuation"/></rule>
19+
</state>
20+
<state name="literals">
21+
<rule pattern="(true|false)"><token type="KeywordConstant"/></rule>
22+
<rule pattern="-?\d+[eE]-?\d+"><token type="LiteralNumberFloat"/></rule>
23+
<rule pattern="-?\d*\.\d+[fFdD]?"><token type="LiteralNumberFloat"/></rule>
24+
<rule pattern="-?\d+[bBsSlLfFdD]?"><token type="LiteralNumberInteger"/></rule>
25+
<rule pattern="&quot;"><token type="LiteralStringDouble"/><push state="literals.string_double"/></rule>
26+
<rule pattern="&#x27;"><token type="LiteralStringSingle"/><push state="literals.string_single"/></rule>
27+
</state>
28+
<state name="literals.string_double">
29+
<rule pattern="\\."><token type="LiteralStringEscape"/></rule>
30+
<rule pattern="[^\\&quot;\n]+"><token type="LiteralStringDouble"/></rule>
31+
<rule pattern="&quot;"><token type="LiteralStringDouble"/><pop depth="1"/></rule>
32+
</state>
33+
<state name="literals.string_single">
34+
<rule pattern="\\."><token type="LiteralStringEscape"/></rule>
35+
<rule pattern="[^\\&#x27;\n]+"><token type="LiteralStringSingle"/></rule>
36+
<rule pattern="&#x27;"><token type="LiteralStringSingle"/><pop depth="1"/></rule>
37+
</state>
38+
<state name="compound">
39+
<rule pattern="[A-Z_a-z]+"><token type="NameAttribute"/></rule>
40+
<rule><include state="operators"/></rule>
41+
<rule><include state="whitespace"/></rule>
42+
<rule><include state="literals"/></rule>
43+
<rule pattern="\{"><token type="Punctuation"/><push/></rule>
44+
<rule pattern="\["><token type="Punctuation"/><push state="list"/></rule>
45+
<rule pattern="\}"><token type="Punctuation"/><pop depth="1"/></rule>
46+
</state>
47+
<state name="list">
48+
<rule pattern="[A-Z_a-z]+"><token type="NameAttribute"/></rule>
49+
<rule><include state="literals"/></rule>
50+
<rule><include state="operators"/></rule>
51+
<rule><include state="whitespace"/></rule>
52+
<rule pattern="\["><token type="Punctuation"/><push/></rule>
53+
<rule pattern="\{"><token type="Punctuation"/><push state="compound"/></rule>
54+
<rule pattern="\]"><token type="Punctuation"/><pop depth="1"/></rule>
55+
</state>
56+
</rules>
57+
</lexer>
58+

0 commit comments

Comments
 (0)