From 3ae9d472905c18936b056dfc897477eaf2415d3d Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Wed, 3 Sep 2014 04:54:57 +0200 Subject: [PATCH 01/16] Built-in functions are now only matched by ident Essentially reverts #8, but this is in scope of a bigger change. Also clearly separate built-in functions and built-in types because some used to be duplicated. --- PythonImproved.YAML-tmLanguage | 31 ++---------- PythonImproved.tmLanguage | 88 ++-------------------------------- 2 files changed, 8 insertions(+), 111 deletions(-) diff --git a/PythonImproved.YAML-tmLanguage b/PythonImproved.YAML-tmLanguage index 474aef1..f5824a9 100644 --- a/PythonImproved.YAML-tmLanguage +++ b/PythonImproved.YAML-tmLanguage @@ -290,8 +290,6 @@ patterns: - include: '#keyword_arguments' - include: $self -- include: '#builtin_functions' - - include: '#builtin_types' - include: '#builtin_exceptions' @@ -486,35 +484,12 @@ repository: match: \b((Arithmetic|Assertion|Attribute|Buffer|EOF|Environment|FloatingPoint|IO|Import|Indentation|Index|Key|Lookup|Memory|Name|NotImplemented|OS|Overflow|Reference|Runtime|Standard|Syntax|System|Tab|Type|UnboundLocal|Unicode(Encode|Decode|Translate)?|Value|VMS|Windows|ZeroDivision|([.a-zA-Z0-9_]+))Error|((Pending)?Deprecation|Runtime|Syntax|User|Future|Import|Unicode|Bytes)?Warning|SystemExit|StopIteration|NotImplemented|KeyboardInterrupt|GeneratorExit|([.a-zA-Z0-9_]+)?Exception)\b builtin_functions: - patterns: - - name: meta.function-call.python - begin: (? - - include - #builtin_functions - include #builtin_types @@ -1412,89 +1408,15 @@ builtin_functions - patterns - - - begin - (?<!\.)(__import__|ascii|abs|all|any|apply|bin|bool|buffer|bytearray|bytes|callable|chr|classmethod|cmp|coerce|compile|complex|copyright|credits|del|delattr|dict|dir|divmod|enumerate|eval|exec|execfile|exit|file|filter|float|format|frozenset|getattr|globals|hasattr|hash|help|hex|id|input|int|intern|isinstance|issubclass|iter|len|license|list|locals|long|map|max|memoryview|min|next|object|oct|open|ord|pow|print|property|quit|range|raw_input|reduce|reload|repr|reversed|round|set|setattr|slice|sorted|staticmethod|str|sum|super|tuple|type|unicode|unichr|vars|xrange|zip)\s*(?=\() - beginCaptures - - 1 - - name - support.function.builtin.python - - - end - (\)) - endCaptures - - 1 - - name - punctuation.definition.arguments.end.python - - - name - meta.function-call.python - patterns - - - begin - (?=[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*\s*\() - end - (?=\s*\() - patterns - - - include - #dotted_name - - - - - begin - (\() - beginCaptures - - 1 - - name - punctuation.definition.arguments.begin.python - - - contentName - meta.function-call.arguments.python - end - (?=\)) - patterns - - - include - #keyword_arguments - - - include - $self - - - - - - - comment - These are statements in Py2, but don't really fit into keyword.control.flow with the rest - match - (?<!\.)\b(print|del)\b - name - support.function.builtin.python - - + match + (?<!\.)\b(__import__|abs|all|any|apply|ascii|bin|buffer|callable|chr|cmp|coerce|compile|copyright|credits|delattr|dir|divmod|enumerate|eval|exec|execfile|exit|filter|format|getattr|globals|hasattr|hash|help|hex|id|input|intern|isinstance|issubclass|iter|len|license|locals|long|map|max|memoryview|min|next|oct|open|ord|pow|print|property|quit|range|raw_input|reduce|reload|repr|reversed|round|setattr|sorted|sum|unichr|vars|xrange|zip)\b + name + support.function.builtin.python builtin_types match - (?<!\.)\b(ascii|basestring|bin|bool|buffer|bytearray|bytes|classmethod|complex|dict|file|float|frozenset|hex|int|list|long|object|oct|property|reversed|set|slice|staticmethod|str|super|tuple|type|unicode|xrange)\b + (?<!\.)\b(basestring|bool|bytearray|bytes|classmethod|complex|dict|file|float|frozenset|int|list|object|property|set|slice|staticmethod|str|super|tuple|type|unicode)\b name support.type.python From 6bd2fc20c4fb5cdb4e50ca9b2b3df5bc2541ca61 Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Wed, 3 Sep 2014 05:04:54 +0200 Subject: [PATCH 02/16] Properly match ident patterns inside function call The previous attempt just didn't work because of two reasons: 1. The "dotted_name" was already consumed in function-call's "begin" match. 2. All "special" identifiers were essentially matched before the function call which meant that the actual call would never be matched since the function to be called (the identifier) was already consumed. Simplify #dotted_name a bit in the process. --- PythonImproved.YAML-tmLanguage | 48 ++++++--------- PythonImproved.tmLanguage | 104 +++++++-------------------------- 2 files changed, 40 insertions(+), 112 deletions(-) diff --git a/PythonImproved.YAML-tmLanguage b/PythonImproved.YAML-tmLanguage index f5824a9..85afb97 100644 --- a/PythonImproved.YAML-tmLanguage +++ b/PythonImproved.YAML-tmLanguage @@ -290,32 +290,16 @@ patterns: - include: '#keyword_arguments' - include: $self -- include: '#builtin_types' - -- include: '#builtin_exceptions' - - include: '#docstrings' -- include: '#magic_function_names' - -- include: '#magic_variable_names' - -- include: '#language_variables' - -- include: '#generic_object_names' +# - include: '#generic_object_names' - name: meta.function-call.python - begin: (?:\.)?([a-zA-Z_][a-zA-Z_0-9]*)\s*(?=\() - beginCaptures: - '1': {name: meta.function-call.generic.python} + begin: (?=\b[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_])*\s*\() end: (\)) endCaptures: '1': {name: punctuation.definition.arguments.end.python} patterns: - - begin: (?=[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*\s*\() - end: (?=\s*\() - patterns: - - include: '#dotted_name' - contentName: meta.function-call.arguments.python begin: (\() beginCaptures: @@ -324,6 +308,10 @@ patterns: patterns: - include: '#keyword_arguments' - include: $self + - begin: \G + end: (?=\() + patterns: + - include: '#dotted_name' - name: meta.item-access.python begin: (?=[A-Za-z_][A-Za-z0-9_]*(?:\.[a-zA-Z_][a-zA-Z_0-9]*)*\s*\[) @@ -535,17 +523,18 @@ repository: - include: '#string_quoted_single' dotted_name: - begin: (?=[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*) + begin: (?=\b[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*(?![A-Za-z0-9_\.])) end: (?![A-Za-z0-9_\.]) + contentName: meta.dotted-name.python patterns: - - begin: (\.)(?=[A-Za-z_][A-Za-z0-9_]*) + - begin: (\.) end: (?![A-Za-z0-9_]) patterns: - include: '#magic_function_names' - include: '#magic_variable_names' - include: '#illegal_names' - include: '#generic_names' - - begin: (? - - include - #builtin_types - - - include - #builtin_exceptions - include #docstrings - - include - #magic_function_names - - - include - #magic_variable_names - - - include - #language_variables - - - include - #generic_object_names - begin - (?:\.)?([a-zA-Z_][a-zA-Z_0-9]*)\s*(?=\() - beginCaptures - - 1 - - name - meta.function-call.generic.python - - + (?=\b[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_])*\s*\() end (\)) endCaptures @@ -879,19 +847,6 @@ meta.function-call.python patterns - - begin - (?=[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*\s*\() - end - (?=\s*\() - patterns - - - include - #dotted_name - - - begin (\() @@ -919,6 +874,19 @@ + + begin + \G + end + (?=\() + patterns + + + include + #dotted_name + + + @@ -1037,10 +1005,6 @@ include #line_continuation - - include - #language_variables - match \b(None|True|False|Ellipsis|NotImplemented)\b @@ -1546,14 +1510,16 @@ dotted_name begin - (?=[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*) + (?=\b[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*(?![A-Za-z0-9_\.])) + contentName + meta.dotted-name.python end (?![A-Za-z0-9_\.]) patterns begin - (\.)(?=[A-Za-z_][A-Za-z0-9_]*) + (\.) end (?![A-Za-z0-9_]) patterns @@ -1578,7 +1544,7 @@ begin - (?<!\.)(?=[A-Za-z_][A-Za-z0-9_]*) + (?=[A-Za-z_]) end (?![A-Za-z0-9_]) patterns @@ -1747,40 +1713,12 @@ match (\\U[0-9A-Fa-f]{8})|(\\u[0-9A-Fa-f]{4})|(\\N\{[a-zA-Z0-9 ]+\}) - function_name - - patterns - - - include - #magic_function_names - - - include - #magic_variable_names - - - include - #builtin_exceptions - - - include - #builtin_functions - - - include - #builtin_types - - - include - #generic_names - - - generic_names match [A-Za-z_][A-Za-z0-9_]* + name + meta.generic-name.python generic_object_names From 828ab4310263083984635c8c7c0f808bfe65ae69 Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Wed, 3 Sep 2014 05:08:54 +0200 Subject: [PATCH 03/16] Re-add print and del statement matching The print statement can now only be matched if it's not used in a function previously since otherwise it would've been consumed. --- PythonImproved.YAML-tmLanguage | 12 ++++++++---- PythonImproved.tmLanguage | 12 +++++++++++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/PythonImproved.YAML-tmLanguage b/PythonImproved.YAML-tmLanguage index 85afb97..a49d422 100644 --- a/PythonImproved.YAML-tmLanguage +++ b/PythonImproved.YAML-tmLanguage @@ -101,7 +101,7 @@ patterns: - comment: keywords that haven't fit into other groups (yet). name: keyword.other.python - match: \b(assert)\b + match: \b(assert|del)\b - name: invalid.deprecated.operator.python match: <> @@ -313,6 +313,10 @@ patterns: patterns: - include: '#dotted_name' +- comment: Py2 print statement that should only be matched after function calls + name: keyword.other.print.python + match: \b(print)(?!\s*(?!\.)) + - name: meta.item-access.python begin: (?=[A-Za-z_][A-Za-z0-9_]*(?:\.[a-zA-Z_][a-zA-Z_0-9]*)*\s*\[) end: (\]) @@ -352,8 +356,6 @@ patterns: - include: '#line_continuation' -- include: '#language_variables' - - name: constant.language.python match: \b(None|True|False|Ellipsis|NotImplemented)\b @@ -361,7 +363,8 @@ patterns: - include: '#string_quoted_double' -- include: '#dotted_name' +- comment: Covers every (remaining) identifier + include: '#dotted_name' - begin: (\() end: (\)) @@ -1212,6 +1215,7 @@ repository: patterns: - include: '#string_quoted_double' - include: '#string_quoted_single' + author: Matt Morrison @MattDMo mattdmo@pigimal.com bundleUUID: 11B0273F-0284-4483-B17B-4B8D0A9294CC firstLineMatch: ^#!/.*\bpython[0-9.-]*\b diff --git a/PythonImproved.tmLanguage b/PythonImproved.tmLanguage index 5512447..b8d7cc0 100644 --- a/PythonImproved.tmLanguage +++ b/PythonImproved.tmLanguage @@ -239,7 +239,7 @@ comment keywords that haven't fit into other groups (yet). match - \b(assert)\b + \b(assert|del)\b name keyword.other.python @@ -889,6 +889,14 @@ + + comment + Py2 print statement that should only be matched after function calls + match + \b(print)(?!\s*(?!\.)) + name + keyword.other.print.python + begin (?=[A-Za-z_][A-Za-z0-9_]*(?:\.[a-zA-Z_][a-zA-Z_0-9]*)*\s*\[) @@ -1020,6 +1028,8 @@ #string_quoted_double + comment + Covers every (remaining) identifier include #dotted_name From b847ef4106b7549f1ec40343259ed21eb1b32f10 Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Wed, 3 Sep 2014 05:33:34 +0200 Subject: [PATCH 04/16] Spaces are allowed in dotted idents --- PythonImproved.YAML-tmLanguage | 12 +++++++----- PythonImproved.tmLanguage | 18 +++++++++++++----- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/PythonImproved.YAML-tmLanguage b/PythonImproved.YAML-tmLanguage index a49d422..7abf960 100644 --- a/PythonImproved.YAML-tmLanguage +++ b/PythonImproved.YAML-tmLanguage @@ -295,7 +295,7 @@ patterns: # - include: '#generic_object_names' - name: meta.function-call.python - begin: (?=\b[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_])*\s*\() + begin: (?=\b[A-Za-z_][A-Za-z0-9_]*\s*(\.\s*[A-Za-z_][A-Za-z0-9_])*\s*\() end: (\)) endCaptures: '1': {name: punctuation.definition.arguments.end.python} @@ -315,7 +315,7 @@ patterns: - comment: Py2 print statement that should only be matched after function calls name: keyword.other.print.python - match: \b(print)(?!\s*(?!\.)) + match: \b(print)(?=\s)(?!\s*\.) # Otherwise this would qualify as a dotted name, which is matched later - name: meta.item-access.python begin: (?=[A-Za-z_][A-Za-z0-9_]*(?:\.[a-zA-Z_][a-zA-Z_0-9]*)*\s*\[) @@ -526,11 +526,13 @@ repository: - include: '#string_quoted_single' dotted_name: - begin: (?=\b[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*(?![A-Za-z0-9_\.])) - end: (?![A-Za-z0-9_\.]) + begin: (?=\b[A-Za-z_][A-Za-z0-9_]*\s*(?:\.\s*[A-Za-z_][A-Za-z0-9_]*)*) + end: (?![A-Za-z0-9_\.\s])|(?=(? begin - (?=\b[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_])*\s*\() + (?=\b[A-Za-z_][A-Za-z0-9_]*\s*(\.\s*[A-Za-z_][A-Za-z0-9_])*\s*\() end (\)) endCaptures @@ -893,7 +893,7 @@ comment Py2 print statement that should only be matched after function calls match - \b(print)(?!\s*(?!\.)) + \b(print)(?=\s)(?!\s*\.) name keyword.other.print.python @@ -1520,16 +1520,24 @@ dotted_name begin - (?=\b[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*(?![A-Za-z0-9_\.])) + (?=\b[A-Za-z_][A-Za-z0-9_]*\s*(?:\.\s*[A-Za-z_][A-Za-z0-9_]*)*) contentName meta.dotted-name.python end - (?![A-Za-z0-9_\.]) + (?![A-Za-z0-9_\.\s])|(?=(?<!\.)\s+[^.]) patterns begin - (\.) + \. + beginCaptures + + 0 + + name + meta.dot.python + + end (?![A-Za-z0-9_]) patterns From cf5b3497ac48a45d829689dc8c0aa74253cbbf0a Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Wed, 3 Sep 2014 06:30:04 +0200 Subject: [PATCH 05/16] Forgot about decorators and item access --- PythonImproved.YAML-tmLanguage | 35 +++++++++++----------- PythonImproved.tmLanguage | 54 ++++++++++++++++------------------ 2 files changed, 43 insertions(+), 46 deletions(-) diff --git a/PythonImproved.YAML-tmLanguage b/PythonImproved.YAML-tmLanguage index 7abf960..cd41314 100644 --- a/PythonImproved.YAML-tmLanguage +++ b/PythonImproved.YAML-tmLanguage @@ -244,17 +244,16 @@ patterns: '2': {name: punctuation.separator.parameters.python} - comment: a decorator may be a function call which returns a decorator. - name: meta.function.decorator.python - begin: ^\s*(?=@\s*[A-Za-z_][A-Za-z0-9_]*(?:\.[a-zA-Z_][a-zA-Z_0-9]*)*\s*\() + name: meta.function.decorator-call.python entity.name.function.decorator.python + begin: ^\s*(?=@\s*[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[A-Za-z_][A-Za-z_0-9]*)*\s*\() end: (\)) endCaptures: '1': {name: punctuation.definition.arguments.end.python} patterns: - - contentName: entity.name.function.decorator.python - begin: (?=(@)\s*[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*\s*\() + - begin: '@' beginCaptures: - '1': {name: punctuation.definition.decorator.python} - end: (?=\s*\() + '0': {name: punctuation.definition.decorator.python} + end: (?=\() patterns: - include: '#dotted_name' - contentName: meta.function.decorator.arguments.python @@ -268,13 +267,13 @@ patterns: - name: meta.function.decorator.python contentName: entity.name.function.decorator.python - begin: ^\s*(?=@\s*[A-Za-z_][A-Za-z0-9_]*(?:\.[a-zA-Z_][a-zA-Z_0-9]*)*) - end: (?=\s|$\n?|#) + begin: ^\s*(?=@\s*[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[a-zA-Z_][a-zA-Z_0-9]*)*) + end: (?=\s*(#|$)) patterns: - - begin: (?=(@)\s*[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*) + - begin: '@' beginCaptures: - '1': {name: punctuation.definition.decorator.python} - end: (?=\s|$\n?|#) + '0': {name: punctuation.definition.decorator.python} + end: (?=\s*(#|$)) patterns: - include: '#dotted_name' @@ -295,7 +294,7 @@ patterns: # - include: '#generic_object_names' - name: meta.function-call.python - begin: (?=\b[A-Za-z_][A-Za-z0-9_]*\s*(\.\s*[A-Za-z_][A-Za-z0-9_])*\s*\() + begin: (?=\b[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[A-Za-z_][A-Za-z0-9_]*)*\s*\() end: (\)) endCaptures: '1': {name: punctuation.definition.arguments.end.python} @@ -318,15 +317,11 @@ patterns: match: \b(print)(?=\s)(?!\s*\.) # Otherwise this would qualify as a dotted name, which is matched later - name: meta.item-access.python - begin: (?=[A-Za-z_][A-Za-z0-9_]*(?:\.[a-zA-Z_][a-zA-Z_0-9]*)*\s*\[) + begin: (?=[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[a-zA-Z_][a-zA-Z_0-9]*)*\s*\[) end: (\]) endCaptures: '1': {name: punctuation.definition.arguments.end.python} patterns: - - begin: (?=[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*\s*\[) - end: (?=\s*\[) - patterns: - - include: '#dotted_name' - contentName: meta.item-access.arguments.python begin: (\[) beginCaptures: @@ -334,6 +329,10 @@ patterns: end: (?=\]) patterns: - include: $self + - begin: \G + end: (?=\[) + patterns: + - include: '#dotted_name' - name: meta.item-access.python contentName: meta.item-access.arguments.python @@ -526,7 +525,7 @@ repository: - include: '#string_quoted_single' dotted_name: - begin: (?=\b[A-Za-z_][A-Za-z0-9_]*\s*(?:\.\s*[A-Za-z_][A-Za-z0-9_]*)*) + begin: (?=\b[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[A-Za-z_][A-Za-z0-9_]*)*) end: (?![A-Za-z0-9_\.\s])|(?=(? begin - ^\s*(?=@\s*[A-Za-z_][A-Za-z0-9_]*(?:\.[a-zA-Z_][a-zA-Z_0-9]*)*\s*\() + ^\s*(?=@\s*[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[A-Za-z_][A-Za-z_0-9]*)*\s*\() comment a decorator may be a function call which returns a decorator. end @@ -700,24 +700,22 @@ name - meta.function.decorator.python + meta.function.decorator-call.python entity.name.function.decorator.python patterns begin - (?=(@)\s*[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*\s*\() + @ beginCaptures - 1 + 0 name punctuation.definition.decorator.python - contentName - entity.name.function.decorator.python end - (?=\s*\() + (?=\() patterns @@ -757,28 +755,28 @@ begin - ^\s*(?=@\s*[A-Za-z_][A-Za-z0-9_]*(?:\.[a-zA-Z_][a-zA-Z_0-9]*)*) + ^\s*(?=@\s*[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[a-zA-Z_][a-zA-Z_0-9]*)*) contentName entity.name.function.decorator.python end - (?=\s|$\n?|#) + (?=\s*(#|$)) name meta.function.decorator.python patterns begin - (?=(@)\s*[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*) + @ beginCaptures - 1 + 0 name punctuation.definition.decorator.python end - (?=\s|$\n?|#) + (?=\s*(#|$)) patterns @@ -832,7 +830,7 @@ begin - (?=\b[A-Za-z_][A-Za-z0-9_]*\s*(\.\s*[A-Za-z_][A-Za-z0-9_])*\s*\() + (?=\b[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[A-Za-z_][A-Za-z0-9_]*)*\s*\() end (\)) endCaptures @@ -899,7 +897,7 @@ begin - (?=[A-Za-z_][A-Za-z0-9_]*(?:\.[a-zA-Z_][a-zA-Z_0-9]*)*\s*\[) + (?=[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[a-zA-Z_][a-zA-Z_0-9]*)*\s*\[) end (\]) endCaptures @@ -914,19 +912,6 @@ meta.item-access.python patterns - - begin - (?=[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*\s*\[) - end - (?=\s*\[) - patterns - - - include - #dotted_name - - - begin (\[) @@ -950,6 +935,19 @@ + + begin + \G + end + (?=\[) + patterns + + + include + #dotted_name + + + @@ -1520,7 +1518,7 @@ dotted_name begin - (?=\b[A-Za-z_][A-Za-z0-9_]*\s*(?:\.\s*[A-Za-z_][A-Za-z0-9_]*)*) + (?=\b[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[A-Za-z_][A-Za-z0-9_]*)*) contentName meta.dotted-name.python end From 3ea8cb855137dc8d92140d4c86adf7b92e93d949 Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Thu, 4 Sep 2014 16:02:36 +0200 Subject: [PATCH 06/16] Fix indent of decorators being highlighted Although it looked okay-ish with background color it was not intended. --- PythonImproved.YAML-tmLanguage | 15 ++++++++------- PythonImproved.tmLanguage | 28 +++++++++++++++------------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/PythonImproved.YAML-tmLanguage b/PythonImproved.YAML-tmLanguage index cd41314..a676ae5 100644 --- a/PythonImproved.YAML-tmLanguage +++ b/PythonImproved.YAML-tmLanguage @@ -244,11 +244,10 @@ patterns: '2': {name: punctuation.separator.parameters.python} - comment: a decorator may be a function call which returns a decorator. - name: meta.function.decorator-call.python entity.name.function.decorator.python + name: meta.function.decorator-call.python + contentName: entity.name.function.decorator.python begin: ^\s*(?=@\s*[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[A-Za-z_][A-Za-z_0-9]*)*\s*\() - end: (\)) - endCaptures: - '1': {name: punctuation.definition.arguments.end.python} + end: (?<=\)) patterns: - begin: '@' beginCaptures: @@ -257,10 +256,12 @@ patterns: patterns: - include: '#dotted_name' - contentName: meta.function.decorator.arguments.python - begin: (\() + begin: \( beginCaptures: - '1': {name: punctuation.definition.arguments.begin.python} - end: (?=\)) + '0': {name: punctuation.definition.arguments.begin.python} + end: \) + endCaptures: + '0': {name: punctuation.definition.arguments.end.python} patterns: - include: '#keyword_arguments' - include: $self diff --git a/PythonImproved.tmLanguage b/PythonImproved.tmLanguage index eeecb7c..bd69a4d 100644 --- a/PythonImproved.tmLanguage +++ b/PythonImproved.tmLanguage @@ -689,18 +689,12 @@ ^\s*(?=@\s*[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[A-Za-z_][A-Za-z_0-9]*)*\s*\() comment a decorator may be a function call which returns a decorator. + contentName + entity.name.function.decorator.python end - (\)) - endCaptures - - 1 - - name - punctuation.definition.arguments.end.python - - + (?<=\)) name - meta.function.decorator-call.python entity.name.function.decorator.python + meta.function.decorator-call.python patterns @@ -726,10 +720,10 @@ begin - (\() + \( beginCaptures - 1 + 0 name punctuation.definition.arguments.begin.python @@ -738,7 +732,15 @@ contentName meta.function.decorator.arguments.python end - (?=\)) + \) + endCaptures + + 0 + + name + punctuation.definition.arguments.end.python + + patterns From 42bb6537826731056b9b62300c9a3d595193faf2 Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Thu, 4 Sep 2014 16:09:27 +0200 Subject: [PATCH 07/16] dotted_name should not include linebreaks Otherwise, color schemes defining a background color for `entitiy.name.function` would make decorators span the remainder of the line. --- PythonImproved.YAML-tmLanguage | 2 +- PythonImproved.tmLanguage | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/PythonImproved.YAML-tmLanguage b/PythonImproved.YAML-tmLanguage index a676ae5..7db36dc 100644 --- a/PythonImproved.YAML-tmLanguage +++ b/PythonImproved.YAML-tmLanguage @@ -527,7 +527,7 @@ repository: dotted_name: begin: (?=\b[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[A-Za-z_][A-Za-z0-9_]*)*) - end: (?![A-Za-z0-9_\.\s])|(?=(?contentName meta.dotted-name.python end - (?![A-Za-z0-9_\.\s])|(?=(?<!\.)\s+[^.]) + (?![A-Za-z0-9_\.\s])|(?=(?<!\.)\s+[^.])|$ patterns From 61479aa4ea5e476dfdb3f3481a45de411f1786bd Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Wed, 1 Oct 2014 17:51:23 +0200 Subject: [PATCH 08/16] Match ALL_CAPS constants in dotted names Previously, `self.CONSTANT` was not matched. --- PythonImproved.YAML-tmLanguage | 11 +++++++---- PythonImproved.tmLanguage | 25 +++++++++++++++++-------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/PythonImproved.YAML-tmLanguage b/PythonImproved.YAML-tmLanguage index 7db36dc..25ae4f7 100644 --- a/PythonImproved.YAML-tmLanguage +++ b/PythonImproved.YAML-tmLanguage @@ -45,10 +45,6 @@ patterns: captures: '1': {name: punctuation.definition.comment.python} -- name: constant.other.allcaps.python - match: \b([A-Z_][A-Z0-9_]*)\b(?![\.|\(]) - comment: Match identifiers in ALL_CAPS as constants. - - name: constant.numeric.integer.long.hexadecimal.python match: \b(?i:(0x\h*)L) @@ -431,6 +427,11 @@ patterns: - include: $self repository: + allcaps_constant: + name: constant.other.allcaps.python + match: \b([A-Z_][A-Z0-9_]*)\b(?![\.|\(]) + comment: Match identifiers in ALL_CAPS as constants. + annotated_arguments: begin: \b([a-zA-Z_][a-zA-Z_0-9]*)\s*(:)|(?=\() beginCaptures: @@ -538,6 +539,7 @@ repository: - include: '#magic_function_names' - include: '#magic_variable_names' - include: '#illegal_names' + - include: '#allcaps_constant' - include: '#generic_names' - begin: (?=[A-Za-z_]) end: (?![A-Za-z0-9_]) @@ -549,6 +551,7 @@ repository: - include: '#magic_function_names' - include: '#magic_variable_names' - include: '#language_variables' + - include: '#allcaps_constant' - include: '#generic_names' entity_name_class: diff --git a/PythonImproved.tmLanguage b/PythonImproved.tmLanguage index 06ae88f..50d8346 100644 --- a/PythonImproved.tmLanguage +++ b/PythonImproved.tmLanguage @@ -127,14 +127,6 @@ name comment.line.number-sign.python - - comment - Match identifiers in ALL_CAPS as constants. - match - \b([A-Z_][A-Z0-9_]*)\b(?![\.|\(]) - name - constant.other.allcaps.python - match \b(?i:(0x\h*)L) @@ -1242,6 +1234,15 @@ repository + allcaps_constant + + comment + Match identifiers in ALL_CAPS as constants. + match + \b([A-Z_][A-Z0-9_]*)\b(?![\.|\(]) + name + constant.other.allcaps.python + annotated_arguments begin @@ -1554,6 +1555,10 @@ include #illegal_names + + include + #allcaps_constant + include #generic_names @@ -1595,6 +1600,10 @@ include #language_variables + + include + #allcaps_constant + include #generic_names From 17ea9f5aaaa9757e7c821ceb3180ccd3ea883bdc Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Wed, 1 Oct 2014 17:54:42 +0200 Subject: [PATCH 09/16] `print` must be an ident if followed by `,` Or well, it's only valid syntax in Python 3. --- PythonImproved.YAML-tmLanguage | 2 +- PythonImproved.tmLanguage | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/PythonImproved.YAML-tmLanguage b/PythonImproved.YAML-tmLanguage index 25ae4f7..9522349 100644 --- a/PythonImproved.YAML-tmLanguage +++ b/PythonImproved.YAML-tmLanguage @@ -311,7 +311,7 @@ patterns: - comment: Py2 print statement that should only be matched after function calls name: keyword.other.print.python - match: \b(print)(?=\s)(?!\s*\.) # Otherwise this would qualify as a dotted name, which is matched later + match: \b(print)(?=\s)(?!\s*[.,]) # Otherwise this would qualify as a dotted name, which is matched later - name: meta.item-access.python begin: (?=[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[a-zA-Z_][a-zA-Z_0-9]*)*\s*\[) diff --git a/PythonImproved.tmLanguage b/PythonImproved.tmLanguage index 50d8346..368fec1 100644 --- a/PythonImproved.tmLanguage +++ b/PythonImproved.tmLanguage @@ -885,7 +885,7 @@ comment Py2 print statement that should only be matched after function calls match - \b(print)(?=\s)(?!\s*\.) + \b(print)(?=\s)(?!\s*[.,]) name keyword.other.print.python From ee007cb46f1b5b49fc15007448efb9409833869c Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Wed, 1 Oct 2014 17:59:08 +0200 Subject: [PATCH 10/16] Dotted names may span multiple lines Even though we can't really match that correctly, at least match the leading dot on a new line or after a function call. That's more correct. Example: ident = (value.dotted .__init__) --- PythonImproved.YAML-tmLanguage | 2 +- PythonImproved.tmLanguage | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/PythonImproved.YAML-tmLanguage b/PythonImproved.YAML-tmLanguage index 9522349..605e28d 100644 --- a/PythonImproved.YAML-tmLanguage +++ b/PythonImproved.YAML-tmLanguage @@ -527,7 +527,7 @@ repository: - include: '#string_quoted_single' dotted_name: - begin: (?=\b[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[A-Za-z_][A-Za-z0-9_]*)*) + begin: (?=(?:\.\s*)?[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[A-Za-z_][A-Za-z0-9_]*)*) end: (?![A-Za-z0-9_\.\s])|(?=(?dotted_name begin - (?=\b[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[A-Za-z_][A-Za-z0-9_]*)*) + (?=(?:\.\s*)?[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[A-Za-z_][A-Za-z0-9_]*)*) contentName meta.dotted-name.python end From c9ad6547be827d151607d3e65aecae045cfba7d9 Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Wed, 1 Oct 2014 18:01:16 +0200 Subject: [PATCH 11/16] Add my personal testing file It could be of use for regression testing later, or whatever. In any case it should illustrate most of the changes of my pull request and can easily be extended. --- highlight_test.pyw | 47 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 highlight_test.pyw diff --git a/highlight_test.pyw b/highlight_test.pyw new file mode 100644 index 0000000..7de4e5d --- /dev/null +++ b/highlight_test.pyw @@ -0,0 +1,47 @@ +list(map(str, range(100))) +[str(i) for i in range(100)] + +super(arg1=1)(arg2=2) +list abs, map +some.Ellipsis NotImplemented + +print (file=None) print . __class__ +print keyword print __init__ + +__init__ . something(arg=22) + +callback(print, print , print) + +.__init__ +__init__ [2] + + +@ deco . __init__ (call=some) # rator +def function(): + pass + + +@dêco(func=call) +def sômething(): + pass + + +@normal.decorator +class Class(): + + @wraps(method)#comment + def wrapper(self): + pass + +ccesŝ[2] # currently matched as "list" + +sômething(arg=22) + +CONSTANT +dotted.CONSTANT +constant.after().FUNCTION + +some.Error y.s.e.Exception + +ident = (value.dotted + .__init__) From b57a0095223908ff6c257f9ce8354323767b20e1 Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Tue, 23 Dec 2014 02:16:20 +0100 Subject: [PATCH 12/16] Match `print` before a line break as Py3 ident --- PythonImproved.YAML-tmLanguage | 3 ++- PythonImproved.tmLanguage | 2 +- highlight_test.pyw | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/PythonImproved.YAML-tmLanguage b/PythonImproved.YAML-tmLanguage index 605e28d..78b6324 100644 --- a/PythonImproved.YAML-tmLanguage +++ b/PythonImproved.YAML-tmLanguage @@ -311,7 +311,8 @@ patterns: - comment: Py2 print statement that should only be matched after function calls name: keyword.other.print.python - match: \b(print)(?=\s)(?!\s*[.,]) # Otherwise this would qualify as a dotted name, which is matched later + match: \b(print)(?=\s)(?!\s*([.,]|$)) # Otherwise this would qualify as a dotted name (ident), + # which is matched later - name: meta.item-access.python begin: (?=[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[a-zA-Z_][a-zA-Z_0-9]*)*\s*\[) diff --git a/PythonImproved.tmLanguage b/PythonImproved.tmLanguage index 49e2aa2..8371ade 100644 --- a/PythonImproved.tmLanguage +++ b/PythonImproved.tmLanguage @@ -885,7 +885,7 @@ comment Py2 print statement that should only be matched after function calls match - \b(print)(?=\s)(?!\s*[.,]) + \b(print)(?=\s)(?!\s*([.,]|$)) name keyword.other.print.python diff --git a/highlight_test.pyw b/highlight_test.pyw index 7de4e5d..eb18db0 100644 --- a/highlight_test.pyw +++ b/highlight_test.pyw @@ -10,7 +10,8 @@ print keyword print __init__ __init__ . something(arg=22) -callback(print, print , print) +callback(print, print , print + print) .__init__ __init__ [2] From 05b34f3a6f847e5a3bc8a488ca3b26f3dd22c9bd Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Tue, 23 Dec 2014 02:16:42 +0100 Subject: [PATCH 13/16] Few more test cases --- highlight_test.pyw | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/highlight_test.pyw b/highlight_test.pyw index eb18db0..d514e80 100644 --- a/highlight_test.pyw +++ b/highlight_test.pyw @@ -3,7 +3,7 @@ list(map(str, range(100))) super(arg1=1)(arg2=2) list abs, map -some.Ellipsis NotImplemented +some.Ellipsis NotImplemented # must be root print (file=None) print . __class__ print keyword print __init__ @@ -15,6 +15,7 @@ callback(print, print , print .__init__ __init__ [2] +super().__init__ @ deco . __init__ (call=some) # rator @@ -32,10 +33,11 @@ class Class(): @wraps(method)#comment def wrapper(self): + (self, __class__) pass +#unicode: ccesŝ[2] # currently matched as "list" - sômething(arg=22) CONSTANT From f17fb743f0eb9d76e5497c120123e8a2087a672a Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Wed, 4 Feb 2015 15:26:48 +0100 Subject: [PATCH 14/16] Fix "from . import" and add more test cases --- PythonImproved.YAML-tmLanguage | 13 +++++++++++-- PythonImproved.tmLanguage | 13 +++++++++++-- highlight_test.pyw | 22 +++++++++++++++++++--- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/PythonImproved.YAML-tmLanguage b/PythonImproved.YAML-tmLanguage index 78b6324..b68d2f9 100644 --- a/PythonImproved.YAML-tmLanguage +++ b/PythonImproved.YAML-tmLanguage @@ -528,8 +528,17 @@ repository: - include: '#string_quoted_single' dotted_name: - begin: (?=(?:\.\s*)?[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[A-Za-z_][A-Za-z0-9_]*)*) - end: (?![A-Za-z0-9_\.\s])|(?=(?dotted_name begin - (?=(?:\.\s*)?[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[A-Za-z_][A-Za-z0-9_]*)*) + (?x) +(?=(?:\.(?!\s+import)\s*)? # exlcude "from . import" + [A-Za-z_][A-Za-z0-9_]* + (?:\s*\.\s*[A-Za-z_][A-Za-z0-9_]*)* +) + contentName meta.dotted-name.python end - (?![A-Za-z0-9_\.\s])|(?=(?<!\.)\s+[^.])|$ + (?x) + (?![A-Za-z0-9_\.\s]) +|(?=(?<!\.)\s+[^.]) +|$ + patterns diff --git a/highlight_test.pyw b/highlight_test.pyw index d514e80..d6b16c1 100644 --- a/highlight_test.pyw +++ b/highlight_test.pyw @@ -31,20 +31,36 @@ def sômething(): @normal.decorator class Class(): - @wraps(method)#comment + @wraps(method)# comment def wrapper(self): (self, __class__) pass -#unicode: +# unicode: ccesŝ[2] # currently matched as "list" sômething(arg=22) CONSTANT dotted.CONSTANT constant.after().FUNCTION +CONSTANT[1] +# in theory, these could be constants as well, but they are not highlighted +# for now +CONSTANT() +CONSTANT.s -some.Error y.s.e.Exception +some.Error +y.s.e.SomeException ident = (value.dotted .__init__) + +# this should not break +from . import sublimerepl_build_system_hack +# but this could +from .s. import repl + + +class Demo( + Base Fun): + pass From bb3185f9617872524d4a6ba5d196df9c476cf6d7 Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Fri, 11 Sep 2015 17:32:04 +0200 Subject: [PATCH 15/16] unicode --- PythonImproved.YAML-tmLanguage | 70 +++++++++++++++++----------------- PythonImproved.tmLanguage | 70 +++++++++++++++++----------------- 2 files changed, 70 insertions(+), 70 deletions(-) diff --git a/PythonImproved.YAML-tmLanguage b/PythonImproved.YAML-tmLanguage index b68d2f9..b408bd7 100644 --- a/PythonImproved.YAML-tmLanguage +++ b/PythonImproved.YAML-tmLanguage @@ -22,7 +22,7 @@ patterns: match: (meta|models)\.(Admin|AutoField|BigIntegerField|BinaryField|BooleanField|CharField|CommaSeparatedIntegerField|DateField|DateTimeField|DecimalField|EmailField|FileField|FilePathField|FloatField|ForeignKey|ImageField|IntegerField|IPAddressField|GenericIPAddressField|ManyToManyField|NullBooleanField|OneToOneField|PhoneNumberField|PositiveIntegerField|PositiveSmallIntegerField|SlugField|SmallIntegerField|TextField|TimeField|URLField|USStateField|XMLField)\b - name: support.other.django.module - match: 'django(\.[A-Za-z_][A-Za-z0-9_]+){0,} ' + match: 'django(\.[\p{Alpha}_][\p{Word}_]+){0,} ' - name: variable.other.django.settings comment: updated for 1.7, backwards-compatible with 1.6 @@ -32,7 +32,7 @@ patterns: match: (get_list_or_404|get_object_or_404|load_and_render|loader|render_to_response|render)\b - name: support.function.django.model - match: '[A-Za-z_][A-Za-z0-9_]+\.get_(object|list|iterator|count|values|values_iterator|in_bulk)\b' + match: '[\p{Alpha}_][\p{Word}_]+\.get_(object|list|iterator|count|values|values_iterator|in_bulk)\b' - name: comment.line.note.python match: (#)\s*(BUG|FIXME|TODO|XXX).*$\n? @@ -116,7 +116,7 @@ patterns: - name: meta.class.old-style.python contentName: entity.name.type.class.python - begin: \s*(class)\s+(?=[a-zA-Z_][a-zA-Z_0-9]*\s*\:) + begin: \s*(class)\s+(?=[\p{Alpha}_][a-zA-Z_0-9]*\s*\:) beginCaptures: '1': {name: storage.type.class.python} end: \s*(:) @@ -126,7 +126,7 @@ patterns: - include: '#entity_name_class' - name: meta.class.python - begin: \s*(class)\s+(?=[a-zA-Z_][a-zA-Z_0-9]*\s*\() + begin: \s*(class)\s+(?=[\p{Alpha}_][a-zA-Z_0-9]*\s*\() beginCaptures: '1': {name: storage.type.class.python} end: (\))\s*(?:(\:)|(.*$\n?)) @@ -136,8 +136,8 @@ patterns: '3': {name: invalid.illegal.missing-section-begin.python} patterns: - contentName: entity.name.type.class.python - begin: (?=[A-Za-z_][A-Za-z0-9_]*) - end: (?![A-Za-z0-9_]) + begin: (?=[\p{Alpha}_][\p{Word}_]*) + end: (?![\p{Word}_]) patterns: - include: '#entity_name_class' - contentName: meta.class.inheritance.python @@ -155,7 +155,7 @@ patterns: - include: $self - name: meta.class.python - begin: \s*(class)\s+(?=[a-zA-Z_][a-zA-Z_0-9]) + begin: \s*(class)\s+(?=[\p{Alpha}_][a-zA-Z_0-9]) beginCaptures: '1': {name: storage.type.class.python} end: (\()|\s*($\n?|#.*$\n?) @@ -164,13 +164,13 @@ patterns: '2': {name: invalid.illegal.missing-inheritance.python} patterns: - contentName: entity.name.type.class.python - begin: (?=[A-Za-z_][A-Za-z0-9_]*) - end: (?![A-Za-z0-9_]) + begin: (?=[\p{Alpha}_][\p{Word}_]*) + end: (?![\p{Word}_]) patterns: - include: '#entity_name_function' - name: meta.function.python - begin: \s*(def)\s+(?=[A-Za-z_][A-Za-z0-9_]*\s*\() + begin: \s*(def)\s+(?=[\p{Alpha}_][\p{Word}_]*\s*\() beginCaptures: '1': {name: storage.type.function.python} end: (\:) @@ -178,8 +178,8 @@ patterns: '1': {name: punctuation.section.function.begin.python} patterns: - contentName: entity.name.function.python - begin: (?=[A-Za-z_][A-Za-z0-9_]*) - end: (?![A-Za-z0-9_]) + begin: (?=[\p{Alpha}_][\p{Word}_]*) + end: (?![\p{Word}_]) patterns: - include: '#entity_name_function' - contentName: meta.function.parameters.python @@ -191,7 +191,7 @@ patterns: - include: '#annotated_arguments' - include: '#keyword_arguments' - include: '#comments' - - match: \b(?:(self|cls)|([a-zA-Z_][a-zA-Z_0-9]*))\s*(?:(,)|(?=[\n\)])) + - match: \b(?:(self|cls)|([\p{Alpha}_][a-zA-Z_0-9]*))\s*(?:(,)|(?=[\n\)])) captures: '1': {name: variable.parameter.function.language.python} '2': {name: variable.parameter.function.python} @@ -205,7 +205,7 @@ patterns: - include: $self - name: meta.function.python - begin: \s*(def)\s+(?=[A-Za-z_][A-Za-z0-9_]*) + begin: \s*(def)\s+(?=[\p{Alpha}_][\p{Word}_]*) beginCaptures: '1': {name: storage.type.function.python} end: (\()|\s*($\n?|#.*$\n?) @@ -214,8 +214,8 @@ patterns: '2': {name: invalid.illegal.missing-parameters.python} patterns: - contentName: entity.name.function.python - begin: (?=[A-Za-z_][A-Za-z0-9_]*) - end: (?![A-Za-z0-9_]) + begin: (?=[\p{Alpha}_][\p{Word}_]*) + end: (?![\p{Word}_]) patterns: - include: '#entity_name_function' @@ -234,7 +234,7 @@ patterns: end: (?=\:) patterns: - include: '#keyword_arguments' - - match: \b([a-zA-Z_][a-zA-Z_0-9]*)\s*(?:(,)|(?=[\n\)\:])) + - match: \b([\p{Alpha}_][a-zA-Z_0-9]*)\s*(?:(,)|(?=[\n\)\:])) captures: '1': {name: variable.parameter.function.python} '2': {name: punctuation.separator.parameters.python} @@ -242,7 +242,7 @@ patterns: - comment: a decorator may be a function call which returns a decorator. name: meta.function.decorator-call.python contentName: entity.name.function.decorator.python - begin: ^\s*(?=@\s*[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[A-Za-z_][A-Za-z_0-9]*)*\s*\() + begin: ^\s*(?=@\s*[\p{Alpha}_][\p{Word}_]*(?:\s*\.\s*[\p{Alpha}_][A-Za-z_0-9]*)*\s*\() end: (?<=\)) patterns: - begin: '@' @@ -264,7 +264,7 @@ patterns: - name: meta.function.decorator.python contentName: entity.name.function.decorator.python - begin: ^\s*(?=@\s*[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[a-zA-Z_][a-zA-Z_0-9]*)*) + begin: ^\s*(?=@\s*[\p{Alpha}_][\p{Word}_]*(?:\s*\.\s*[\p{Alpha}_][a-zA-Z_0-9]*)*) end: (?=\s*(#|$)) patterns: - begin: '@' @@ -291,7 +291,7 @@ patterns: # - include: '#generic_object_names' - name: meta.function-call.python - begin: (?=\b[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[A-Za-z_][A-Za-z0-9_]*)*\s*\() + begin: (?=\b[\p{Alpha}_][\p{Word}_]*(?:\s*\.\s*[\p{Alpha}_][\p{Word}_]*)*\s*\() end: (\)) endCaptures: '1': {name: punctuation.definition.arguments.end.python} @@ -315,7 +315,7 @@ patterns: # which is matched later - name: meta.item-access.python - begin: (?=[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[a-zA-Z_][a-zA-Z_0-9]*)*\s*\[) + begin: (?=[\p{Alpha}_][\p{Word}_]*(?:\s*\.\s*[\p{Alpha}_][a-zA-Z_0-9]*)*\s*\[) end: (\]) endCaptures: '1': {name: punctuation.definition.arguments.end.python} @@ -430,11 +430,11 @@ patterns: repository: allcaps_constant: name: constant.other.allcaps.python - match: \b([A-Z_][A-Z0-9_]*)\b(?![\.|\(]) + match: \b([\p{Upper}_][A-Z0-9_]*)\b(?![\.|\(]) comment: Match identifiers in ALL_CAPS as constants. annotated_arguments: - begin: \b([a-zA-Z_][a-zA-Z_0-9]*)\s*(:)|(?=\() + begin: \b([\p{Alpha}_][a-zA-Z_0-9]*)\s*(:)|(?=\() beginCaptures: '1': {name: variable.parameter.function.python} '2': {name: punctuation.separator.annotation.python} @@ -455,7 +455,7 @@ repository: endCaptures: '1': {name: punctuation.definition.parameters-group.end.python} patterns: - - begin: \b([a-zA-Z_][a-zA-Z_0-9]*)\s*(:) + - begin: \b([\p{Alpha}_][a-zA-Z_0-9]*)\s*(:) beginCaptures: '1': {name: variable.parameter.function.python} '2': {name: punctuation.separator.annotation.python} @@ -464,7 +464,7 @@ repository: '1': {name: punctuation.separator.parameters.python} patterns: - include: $self - - begin: \b([a-zA-Z_][a-zA-Z_0-9]*) + - begin: \b([\p{Alpha}_][a-zA-Z_0-9]*) beginCaptures: '1': {name: variable.parameter.function.python} end: \s*(?:(,)|(?=$\n?|\))) @@ -512,7 +512,7 @@ repository: constant_placeholder: name: constant.other.placeholder.python - match: (?i:%(\([a-z_]+\))?#?0?\-?[ ]?\+?([0-9]*|\*)(\.([0-9]*|\*))?[hL]?[a-z%]) + match: (?i:%(\([\p{Upper}_]+\))?#?0?\-?[ ]?\+?([0-9]*|\*)(\.([0-9]*|\*))?[hL]?[a-z%]) docstrings: patterns: @@ -530,9 +530,9 @@ repository: dotted_name: begin: | (?x) - (?=(?:\.(?!\s+import)\s*)? # exlcude "from . import" - [A-Za-z_][A-Za-z0-9_]* - (?:\s*\.\s*[A-Za-z_][A-Za-z0-9_]*)* + (?=(?:\.(?!\s+import)\s*)? + [\p{Alpha}_][\p{Word}_]* + (?:\s*\.\s*[\p{Alpha}_][\p{Word}_]*)* ) end: | (?x) @@ -544,15 +544,15 @@ repository: - begin: \. beginCaptures: '0': {name: meta.dot.python} - end: (?![A-Za-z0-9_]) + end: (?![\p{Word}_]) patterns: - include: '#magic_function_names' - include: '#magic_variable_names' - include: '#illegal_names' - include: '#allcaps_constant' - include: '#generic_names' - - begin: (?=[A-Za-z_]) - end: (?![A-Za-z0-9_]) + - begin: (?=[\p{Alpha}_]) + end: (?![\p{Word}_]) patterns: - include: '#builtin_functions' - include: '#builtin_types' @@ -610,17 +610,17 @@ repository: generic_names: name: meta.generic-name.python - match: '[A-Za-z_][A-Za-z0-9_]*' + match: '[\p{Alpha}_][\p{Word}_]*' generic_object_names: - match: (\.\b([A-Za-z_][A-Za-z0-9_]*)\b(?!\(|\[)|\b([A-Za-z_][A-Za-z0-9_]*)\b\.) + match: (\.\b([\p{Alpha}_][\p{Word}_]*)\b(?!\(|\[)|\b([\p{Alpha}_][\p{Word}_]*)\b\.) illegal_names: name: invalid.illegal.name.python match: \b(and|as|assert|break|class|continue|def|del|elif|else|except|exec|finally|for|from|global|if|import|in|is|lambda|not|or|pass|print|raise|return|try|while|with|yield)\b keyword_arguments: - begin: \b([a-zA-Z_][a-zA-Z_0-9]*)\s*(=)(?!=) + begin: \b([\p{Alpha}_][a-zA-Z_0-9]*)\s*(=)(?!=) beginCaptures: '1': {name: variable.parameter.function.keyword.python} '2': {name: keyword.operator.assignment.python} diff --git a/PythonImproved.tmLanguage b/PythonImproved.tmLanguage index eda821c..5edb178 100644 --- a/PythonImproved.tmLanguage +++ b/PythonImproved.tmLanguage @@ -70,7 +70,7 @@ match - django(\.[A-Za-z_][A-Za-z0-9_]+){0,} + django(\.[\p{Alpha}_][\p{Word}_]+){0,} name support.other.django.module @@ -90,7 +90,7 @@ match - [A-Za-z_][A-Za-z0-9_]+\.get_(object|list|iterator|count|values|values_iterator|in_bulk)\b + [\p{Alpha}_][\p{Word}_]+\.get_(object|list|iterator|count|values|values_iterator|in_bulk)\b name support.function.django.model @@ -267,7 +267,7 @@ begin - \s*(class)\s+(?=[a-zA-Z_][a-zA-Z_0-9]*\s*\:) + \s*(class)\s+(?=[\p{Alpha}_][a-zA-Z_0-9]*\s*\:) beginCaptures 1 @@ -300,7 +300,7 @@ begin - \s*(class)\s+(?=[a-zA-Z_][a-zA-Z_0-9]*\s*\() + \s*(class)\s+(?=[\p{Alpha}_][a-zA-Z_0-9]*\s*\() beginCaptures 1 @@ -335,11 +335,11 @@ begin - (?=[A-Za-z_][A-Za-z0-9_]*) + (?=[\p{Alpha}_][\p{Word}_]*) contentName entity.name.type.class.python end - (?![A-Za-z0-9_]) + (?![\p{Word}_]) patterns @@ -394,7 +394,7 @@ begin - \s*(class)\s+(?=[a-zA-Z_][a-zA-Z_0-9]) + \s*(class)\s+(?=[\p{Alpha}_][a-zA-Z_0-9]) beginCaptures 1 @@ -424,11 +424,11 @@ begin - (?=[A-Za-z_][A-Za-z0-9_]*) + (?=[\p{Alpha}_][\p{Word}_]*) contentName entity.name.type.class.python end - (?![A-Za-z0-9_]) + (?![\p{Word}_]) patterns @@ -441,7 +441,7 @@ begin - \s*(def)\s+(?=[A-Za-z_][A-Za-z0-9_]*\s*\() + \s*(def)\s+(?=[\p{Alpha}_][\p{Word}_]*\s*\() beginCaptures 1 @@ -466,11 +466,11 @@ begin - (?=[A-Za-z_][A-Za-z0-9_]*) + (?=[\p{Alpha}_][\p{Word}_]*) contentName entity.name.function.python end - (?![A-Za-z0-9_]) + (?![\p{Word}_]) patterns @@ -528,7 +528,7 @@ match - \b(?:(self|cls)|([a-zA-Z_][a-zA-Z_0-9]*))\s*(?:(,)|(?=[\n\)])) + \b(?:(self|cls)|([\p{Alpha}_][a-zA-Z_0-9]*))\s*(?:(,)|(?=[\n\)])) @@ -562,7 +562,7 @@ begin - \s*(def)\s+(?=[A-Za-z_][A-Za-z0-9_]*) + \s*(def)\s+(?=[\p{Alpha}_][\p{Word}_]*) beginCaptures 1 @@ -592,11 +592,11 @@ begin - (?=[A-Za-z_][A-Za-z0-9_]*) + (?=[\p{Alpha}_][\p{Word}_]*) contentName entity.name.function.python end - (?![A-Za-z0-9_]) + (?![\p{Word}_]) patterns @@ -670,7 +670,7 @@ match - \b([a-zA-Z_][a-zA-Z_0-9]*)\s*(?:(,)|(?=[\n\)\:])) + \b([\p{Alpha}_][a-zA-Z_0-9]*)\s*(?:(,)|(?=[\n\)\:])) @@ -678,7 +678,7 @@ begin - ^\s*(?=@\s*[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[A-Za-z_][A-Za-z_0-9]*)*\s*\() + ^\s*(?=@\s*[\p{Alpha}_][\p{Word}_]*(?:\s*\.\s*[\p{Alpha}_][A-Za-z_0-9]*)*\s*\() comment a decorator may be a function call which returns a decorator. contentName @@ -749,7 +749,7 @@ begin - ^\s*(?=@\s*[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[a-zA-Z_][a-zA-Z_0-9]*)*) + ^\s*(?=@\s*[\p{Alpha}_][\p{Word}_]*(?:\s*\.\s*[\p{Alpha}_][a-zA-Z_0-9]*)*) contentName entity.name.function.decorator.python end @@ -824,7 +824,7 @@ begin - (?=\b[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[A-Za-z_][A-Za-z0-9_]*)*\s*\() + (?=\b[\p{Alpha}_][\p{Word}_]*(?:\s*\.\s*[\p{Alpha}_][\p{Word}_]*)*\s*\() end (\)) endCaptures @@ -891,7 +891,7 @@ begin - (?=[A-Za-z_][A-Za-z0-9_]*(?:\s*\.\s*[a-zA-Z_][a-zA-Z_0-9]*)*\s*\[) + (?=[\p{Alpha}_][\p{Word}_]*(?:\s*\.\s*[\p{Alpha}_][a-zA-Z_0-9]*)*\s*\[) end (\]) endCaptures @@ -1239,14 +1239,14 @@ comment Match identifiers in ALL_CAPS as constants. match - \b([A-Z_][A-Z0-9_]*)\b(?![\.|\(]) + \b([\p{Upper}_][A-Z0-9_]*)\b(?![\.|\(]) name constant.other.allcaps.python annotated_arguments begin - \b([a-zA-Z_][a-zA-Z_0-9]*)\s*(:)|(?=\() + \b([\p{Alpha}_][a-zA-Z_0-9]*)\s*(:)|(?=\() beginCaptures 1 @@ -1314,7 +1314,7 @@ begin - \b([a-zA-Z_][a-zA-Z_0-9]*)\s*(:) + \b([\p{Alpha}_][a-zA-Z_0-9]*)\s*(:) beginCaptures 1 @@ -1348,7 +1348,7 @@ begin - \b([a-zA-Z_][a-zA-Z_0-9]*) + \b([\p{Alpha}_][a-zA-Z_0-9]*) beginCaptures 1 @@ -1478,7 +1478,7 @@ constant_placeholder match - (?i:%(\([a-z_]+\))?#?0?\-?[ ]?\+?([0-9]*|\*)(\.([0-9]*|\*))?[hL]?[a-z%]) + (?i:%(\([\p{Upper}_]+\))?#?0?\-?[ ]?\+?([0-9]*|\*)(\.([0-9]*|\*))?[hL]?[a-z%]) name constant.other.placeholder.python @@ -1522,9 +1522,9 @@ begin (?x) -(?=(?:\.(?!\s+import)\s*)? # exlcude "from . import" - [A-Za-z_][A-Za-z0-9_]* - (?:\s*\.\s*[A-Za-z_][A-Za-z0-9_]*)* +(?=(?:\.(?!\s+import)\s*)? + [\p{Alpha}_][\p{Word}_]* + (?:\s*\.\s*[\p{Alpha}_][\p{Word}_]*)* ) contentName @@ -1549,7 +1549,7 @@ end - (?![A-Za-z0-9_]) + (?![\p{Word}_]) patterns @@ -1576,9 +1576,9 @@ begin - (?=[A-Za-z_]) + (?=[\p{Alpha}_]) end - (?![A-Za-z0-9_]) + (?![\p{Word}_]) patterns @@ -1752,14 +1752,14 @@ generic_names match - [A-Za-z_][A-Za-z0-9_]* + [\p{Alpha}_][\p{Word}_]* name meta.generic-name.python generic_object_names match - (\.\b([A-Za-z_][A-Za-z0-9_]*)\b(?!\(|\[)|\b([A-Za-z_][A-Za-z0-9_]*)\b\.) + (\.\b([\p{Alpha}_][\p{Word}_]*)\b(?!\(|\[)|\b([\p{Alpha}_][\p{Word}_]*)\b\.) illegal_names @@ -1771,7 +1771,7 @@ keyword_arguments begin - \b([a-zA-Z_][a-zA-Z_0-9]*)\s*(=)(?!=) + \b([\p{Alpha}_][a-zA-Z_0-9]*)\s*(=)(?!=) beginCaptures 1 From f2262213f05f0080012387c8deb1b4951b9930ce Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Fri, 11 Sep 2015 18:10:24 +0200 Subject: [PATCH 16/16] Restructure tests to use syntax testing features --- highlight_test.pyw | 66 ----------- tests/syntax_test_dotted-names.py | 175 ++++++++++++++++++++++++++++++ tests/todo.py | 31 ++++++ 3 files changed, 206 insertions(+), 66 deletions(-) delete mode 100644 highlight_test.pyw create mode 100644 tests/syntax_test_dotted-names.py create mode 100644 tests/todo.py diff --git a/highlight_test.pyw b/highlight_test.pyw deleted file mode 100644 index d6b16c1..0000000 --- a/highlight_test.pyw +++ /dev/null @@ -1,66 +0,0 @@ -list(map(str, range(100))) -[str(i) for i in range(100)] - -super(arg1=1)(arg2=2) -list abs, map -some.Ellipsis NotImplemented # must be root - -print (file=None) print . __class__ -print keyword print __init__ - -__init__ . something(arg=22) - -callback(print, print , print - print) - -.__init__ -__init__ [2] -super().__init__ - - -@ deco . __init__ (call=some) # rator -def function(): - pass - - -@dêco(func=call) -def sômething(): - pass - - -@normal.decorator -class Class(): - - @wraps(method)# comment - def wrapper(self): - (self, __class__) - pass - -# unicode: -ccesŝ[2] # currently matched as "list" -sômething(arg=22) - -CONSTANT -dotted.CONSTANT -constant.after().FUNCTION -CONSTANT[1] -# in theory, these could be constants as well, but they are not highlighted -# for now -CONSTANT() -CONSTANT.s - -some.Error -y.s.e.SomeException - -ident = (value.dotted - .__init__) - -# this should not break -from . import sublimerepl_build_system_hack -# but this could -from .s. import repl - - -class Demo( - Base Fun): - pass diff --git a/tests/syntax_test_dotted-names.py b/tests/syntax_test_dotted-names.py new file mode 100644 index 0000000..e1fb79e --- /dev/null +++ b/tests/syntax_test_dotted-names.py @@ -0,0 +1,175 @@ +# SYNTAX TEST "Packages/PythonImproved/PythonImproved.tmLanguage" +# <- source.python comment + +# ident/dotted-name stuff and function calls +list(map(str, range(100))) +# ^ meta.dotted-name support.type +# ^ meta.dotted-name support.function.builtin +# ^ meta.dotted-name support.type +# ^ meta.dotted-name support.function.builtin +# ^ meta.function-call meta.function-call.arguments meta.function-call meta.function-call.arguments meta.function-call meta.function-call.arguments constant.numeric.integer.decimal + +[str(i) for i in range(100)] +# ^ meta.dotted-name support.type +# ^ meta.dotted-name meta.generic-name +# ^ support.function.builtin + +super(arg1=1)(arg2=2) +# ^ meta.dotted-name support.type +# ^ variable.parameter.function.keyword +# ^ variable.parameter.function.keyword + +list, abs, map, some.map +# ^ meta.dotted-name support.type +# ^ meta.dotted-name support.function.builtin +# ^ meta.dotted-name support.function.builtin +# ^ meta.dotted-name meta.generic-name +# ^ meta.dotted-name meta.dot - meta.generic-name +# ^ meta.dotted-name meta.generic-name + +True, None, False, NotImplemented +# ^ constant.language +# ^ constant.language +# ^ constant.language +# ^ constant.language + +some.Ellipsis, other.NotImplemented # must be root element +# ^ -constant.language +# ^ -constant.language +# ^ -constant.language +# ^ -constant.language + + +__init__ . something(arg=22) +# ^ meta.function-call meta.dotted-name support.function.magic +# ^ meta.dotted-name meta.dot +# ^ meta.dotted-name meta.generic-name +__init__ [2] +# ^ meta.item-access meta.dotted-name support.function.magic +super().__init__[2] +# ^ meta.dotted-name meta.dot +# ^ meta.dotted-name support.function.magic +ident = (value.dotted +# ^ meta.dotted-name meta.generic-name +# ^ meta.dotted-name meta.dot +# ^ meta.dotted-name meta.generic-name + .__init__) +# ^ meta.dotted-name meta.dot +# ^ meta.dotted-name support.function.magic + + +some.Error # should match entire line, by design +y.s.e.SomeException + + + +# "print" keyword vs statement +print (file=None) +# ^ meta.function-call meta.dotted-name support.function.builtin +# ^ meta.function-call meta.dotted-name support.function.builtin +# ^ meta.function-call meta.function-call.arguments variable.parameter.function.keyword + +print . __class__ +# ^ meta.dotted-name support.function.builtin +# ^ meta.dotted-name meta.dot +# ^ meta.dotted-name support.variable.magic + +print keyword +# ^ keyword.other.print + +print __init__ +# ^ keyword.other.print +# ^ support.function.magic + +callback(print, print , print +# ^ support.function.builtin +# ^ support.function.builtin +# ^ support.function.builtin + , + print) +# ^ support.function.builtin + +# ALL_CAPS constants +CONSTANT +# ^ constant.other.allcaps +dotted.CONSTANT +# ^ -constant.other.allcaps +# ^ constant.other.allcaps +constant.after().FUNCTION +# ^ -constant.other.allcaps +# ^ constant.other.allcaps +CONSTANT[1] +# ^ constant.other.allcaps + +# in theory, these could be constants as well, but they are not highlighted +# for now (they prolly should) +CONSTANT() +CONSTANT.s + + +# functions +def func(var1, + # <- variable.parameter.function + # comment + # <- comment + var2=123): + # <- variable.parameter.function + pass + + +# classes +class Demo( + Base Fun): + # <- entity.other.inherited-class + pass + + +# decorators +@ deco . __init__ (call=some) # rator +# <- meta.function.decorator-call entity.name.function.decorator punctuation.definition.decorator +# ^ entity.name.function.decorator meta.dotted-name meta.generic-name +# ^ entity.name.function.decorator meta.dotted-name meta.dot +# ^ entity.name.function.decorator meta.dotted-name support.function.magic +# ^ meta.function.decorator.arguments variable.parameter.function.keyword +def function(): + pass + + +@normal.decorator +# <- meta.function.decorator entity.name.function.decorator punctuation.definition.decorator +# ^ entity.name.function.decorator meta.dotted-name meta.generic-name +class Class(): + + @wraps(method, 12)# comment +# ^ entity.name.function.decorator meta.dotted-name meta.generic-name +# ^ meta.function.decorator.arguments constant +# ^ comment + def wrapper(self): + (self, __class__) + pass + + +# imports +from . import sublimerepl_build_system_hack # this is legit +# ^ -invalid.illegal.name +from .s. import repl # but this is invalid +# ^ meta.dotted-name meta.generic-name +# ^ meta.dotted-name invalid.illegal.name +from a import s as b +# ^ keyword.control.import +# ^ keyword.control.import +# ^ keyword.control.import + + +# unicode +ccesŝ[2] +# ^ meta.item-access meta.dotted-name meta.generic-name +sômething(arg=22) +# ^ meta.function-call meta.dotted-name meta.generic-name + + +@dêco(func=call) +# ^ meta.function.decorator-call meta.dotted-name meta.generic-name +def sômething(): +# ^ entity.name.function meta.generic-name + pass diff --git a/tests/todo.py b/tests/todo.py new file mode 100644 index 0000000..1e2e174 --- /dev/null +++ b/tests/todo.py @@ -0,0 +1,31 @@ + + +# ####################### # +# DOES NOT WORK CORRECTLY # +# ####################### # + +except AnError as b: # <- should be keyword.other + pass +with open() as c: # <- should be keyword.other + pass +yield from something() # <- should be keyword.control.flow +raise AnError() from e # <- should be keyword.other ? + +some.False # 'False' should be invalid + + +def func(var='multi' + 'line'): + pass + + +@ deco [2] # no item access match +def funct‍ion(): + pass + + +super().__init__[2] # [2] currently matched as list and not item-access + +# inline sql string (needs .sublime-syntax) +'INSERT INTO a table INT -- inline comment\n' +'continuation?'