From 49ea74f858bb29a73d4ea899e2ab0cff16ce8c01 Mon Sep 17 00:00:00 2001 From: Anton van Wezenbeek Date: Wed, 30 Aug 2023 11:33:09 +0200 Subject: [PATCH 1/4] improved lilypond lexer, and added folding --- CHANGELOG.md | 1 + data/wex-lexers-macro.xml | 16 + data/wex-lexers.xml | 16 +- external/lexilla.patch | 3 + src/ex/vi/commands-motion.cpp | 2 + src/syntax/wex/lex-lilypond-util.h | 154 +++++++++ src/syntax/wex/lex-lilypond.cpp | 538 +++++++++++++++++++++++++++++ src/syntax/wex/lex-lilypond.h | 173 ++++++++++ test/syntax/test-lexer.cpp | 3 + 9 files changed, 903 insertions(+), 3 deletions(-) create mode 100644 src/syntax/wex/lex-lilypond-util.h create mode 100644 src/syntax/wex/lex-lilypond.cpp create mode 100644 src/syntax/wex/lex-lilypond.h diff --git a/CHANGELOG.md b/CHANGELOG.md index 335883af57..bbbd3ef050 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - stc::vi_command: argument now takes wex::line_data - removed some not necessary copy constructors - moved window.h and control.h to factory +- lilypond lexer improved ### Fixed - ctags::find and empty tag finds next tag diff --git a/data/wex-lexers-macro.xml b/data/wex-lexers-macro.xml index 69927d2e86..6dd4a874f7 100644 --- a/data/wex-lexers-macro.xml +++ b/data/wex-lexers-macro.xml @@ -1151,6 +1151,22 @@ Copyright: (c) 2009-2023, Anton van Wezenbeek + + + + + + + + + + + + + + + + diff --git a/data/wex-lexers.xml b/data/wex-lexers.xml index 41b66c4203..4badb51e95 100644 --- a/data/wex-lexers.xml +++ b/data/wex-lexers.xml @@ -1216,14 +1216,19 @@ www smtp snmp snmp-server ipv4 ipv6 tftp ftp echo dscp isakmp nntp gopher pop3 i - + 1 1 1 - + + + 1 + 1 + 1 + @@ -1326,6 +1331,11 @@ www smtp snmp snmp-server ipv4 ipv6 tftp ftp echo dscp isakmp nntp gopher pop3 i + + + build default include + + @@ -1568,7 +1578,7 @@ www smtp snmp snmp-server ipv4 ipv6 tftp ftp echo dscp isakmp nntp gopher pop3 i - + diff --git a/external/lexilla.patch b/external/lexilla.patch index 519ea004fa..4e1a49fbf9 100644 --- a/external/lexilla.patch +++ b/external/lexilla.patch @@ -7,6 +7,7 @@ index 07dc6c1c..2e9eacdf 100644 using namespace Scintilla; +#include "../../../../../../src/syntax/wex/lex-rfw.cpp" ++#include "../../../../../../src/syntax/wex/lex-lilypond.cpp" + #define HERE_DELIM_MAX 256 @@ -19,6 +20,7 @@ index bd15d392..8ecb9599 100644 extern LexerModule lmRaku; extern LexerModule lmREBOL; extern LexerModule lmRegistry; ++extern LexerModule lmLilyPond; +extern LexerModule lmRFW; extern LexerModule lmRuby; extern LexerModule lmRust; @@ -27,6 +29,7 @@ index bd15d392..8ecb9599 100644 &lmRaku, &lmREBOL, &lmRegistry, ++ &lmLilyPond, + &lmRFW, &lmRuby, &lmRust, diff --git a/src/ex/vi/commands-motion.cpp b/src/ex/vi/commands-motion.cpp index 7a760abff7..5f9b24b7c5 100644 --- a/src/ex/vi/commands-motion.cpp +++ b/src/ex/vi/commands-motion.cpp @@ -746,6 +746,8 @@ void wex::vi::visual_extend(int begin_pos, int end_pos) const default: get_stc()->SetCurrentPos(end_pos); + get_stc() + ->SelectNone(); // appearently previous call selects to the new pos .. break; } } diff --git a/src/syntax/wex/lex-lilypond-util.h b/src/syntax/wex/lex-lilypond-util.h new file mode 100644 index 0000000000..2f1490f241 --- /dev/null +++ b/src/syntax/wex/lex-lilypond-util.h @@ -0,0 +1,154 @@ +//////////////////////////////////////////////////////////////////////////////// +// Name: lex-lilypond-util.h +// Purpose: Implementation of class lilypond +// Author: Anton van Wezenbeek +// Copyright: (c) 2023 Anton van Wezenbeek +//////////////////////////////////////////////////////////////////////////////// + +#pragma once + +namespace Scintilla +{ +/// Offers some general methods. +class lilypond +{ +public: + /// Static interface + + static bool is_letter(int ch); + static bool is_special(int ch); + + /// Other methods + + /// Constructor. + lilypond(LexAccessor& s) + : m_styler(s) + { + ; + } + + bool is_tag_valid(Sci_Position& i, Sci_Position l) const; + bool last_word_is(Sci_Position start, const char* needle) const; + bool last_word_is_match_env(Sci_Position pos) const; + bool next_not_blank_is(Sci_Position i, char needle) const; + +private: + LexAccessor& m_styler; +}; + +// inline implementation lex_rfw_access + +inline bool lilypond::is_letter(int ch) +{ + return IsASCII(ch) && isalpha(ch); +} + +inline bool lilypond::is_special(int ch) +{ + return (ch == '#') || (ch == '$') || (ch == '%') || (ch == '&') || + (ch == '_') || (ch == '{') || (ch == '}') || (ch == ' '); +} + +inline bool lilypond::is_tag_valid(Sci_Position& i, Sci_Position l) const +{ + while (i < l) + { + if (m_styler.SafeGetCharAt(i) == '{') + { + while (i < l) + { + i++; + if (m_styler.SafeGetCharAt(i) == '}') + { + return true; + } + else if ( + !is_letter(m_styler.SafeGetCharAt(i)) && + m_styler.SafeGetCharAt(i) != '*') + { + return false; + } + } + } + else if (!isblank(m_styler.SafeGetCharAt(i))) + { + return false; + } + i++; + } + return false; +} + +inline bool lilypond::next_not_blank_is(Sci_Position i, char needle) const +{ + char ch; + while (i < m_styler.Length()) + { + ch = m_styler.SafeGetCharAt(i); + if (!isspace(ch) && ch != '*') + { + if (ch == needle) + return true; + else + return false; + } + i++; + } + return false; +} + +inline bool lilypond::last_word_is(Sci_Position start, const char* needle) const +{ + Sci_PositionU i = 0; + Sci_PositionU l = static_cast(strlen(needle)); + Sci_Position ini = start - l + 1; + char s[32]; + + while (i < l && i < 31) + { + s[i] = m_styler.SafeGetCharAt(ini + i); + i++; + } + s[i] = '\0'; + + return (strcmp(s, needle) == 0); +} + +inline bool lilypond::last_word_is_match_env(Sci_Position pos) const +{ + Sci_Position i, j; + char s[32]; + const char* mathEnvs[] = { + "align", + "alignat", + "flalign", + "gather", + "multiline", + "displaymath", + "eqnarray", + "equation"}; + if (m_styler.SafeGetCharAt(pos) != '}') + return false; + for (i = pos - 1; i >= 0; --i) + { + if (m_styler.SafeGetCharAt(i) == '{') + break; + if (pos - i >= 20) + return false; + } + if (i < 0 || i == pos - 1) + return false; + ++i; + for (j = 0; i + j < pos; ++j) + s[j] = m_styler.SafeGetCharAt(i + j); + s[j] = '\0'; + if (j == 0) + return false; + if (s[j - 1] == '*') + s[--j] = '\0'; + for (i = 0; i < static_cast(sizeof(mathEnvs) / sizeof(const char*)); ++i) + if (strcmp(s, mathEnvs[i]) == 0) + return true; + return false; +} +}; // namespace Scintilla diff --git a/src/syntax/wex/lex-lilypond.cpp b/src/syntax/wex/lex-lilypond.cpp new file mode 100644 index 0000000000..7a1808f486 --- /dev/null +++ b/src/syntax/wex/lex-lilypond.cpp @@ -0,0 +1,538 @@ +//////////////////////////////////////////////////////////////////////////////// +// Name: lex-lilypond.cpp +// Purpose: Implementation of Scintilla::lex_lillypond +// Author: Anton van Wezenbeek +// Copyright: (c) 2023 Anton van Wezenbeek +//////////////////////////////////////////////////////////////////////////////// + +#include "lex-lilypond.h" +#include "lex-lilypond-util.h" + +lex_lilypond::lex_lilypond() + : DefaultLexer(name(), language()) +{ +} + +int lex_lilypond::mode_to_state(int mode) const +{ + switch (mode) + { + case 1: + return SCE_L_MATH; + case 2: + return SCE_L_MATH2; + default: + return SCE_L_DEFAULT; + } +} + +// Change folding state while processing a line +// Return the level before the first relevant command +void SCI_METHOD lex_lilypond::Fold( + Sci_PositionU startPos, + Sci_Position length, + int, + IDocument* pAccess) +{ + const char* structWords[7] = { + "part", + "chapter", + "section", + "subsection", + "subsubsection", + "paragraph", + "subparagraph"}; + + LexAccessor styler(pAccess); + Sci_PositionU endPos = startPos + length; + Sci_Position curLine = styler.GetLine(startPos); + fold_save save; + + get_save(curLine - 1, save); + + do + { + char ch, buf[16]; + Sci_Position i, j; + int lev = -1; + bool needFold = false; + for (i = static_cast(startPos); + i < static_cast(endPos); + ++i) + { + ch = styler.SafeGetCharAt(i); + if (ch == '\r' || ch == '\n') + break; + if (ch == '{') + { + if (lev < 0) + lev = save.save_to_int(); + ++save.m_open_begins[save.m_level]; + needFold = true; + } + else if (ch == '}') + { + while (save.m_level > 0 && save.m_open_begins[save.m_level] == 0) + --save.m_level; + if (lev < 0) + lev = save.save_to_int(); + if (save.m_open_begins[save.m_level] > 0) + --save.m_open_begins[save.m_level]; + } + else if (ch != '\\' || styler.StyleAt(i) != SCE_L_COMMAND) + continue; + + for (j = 0; j < 15 && i + 1 < static_cast(endPos); ++j, ++i) + { + buf[j] = styler.SafeGetCharAt(i + 1); + if (!lilypond::is_letter(buf[j])) + break; + } + buf[j] = '\0'; + if (strcmp(buf, "begin") == 0) + { + if (lev < 0) + lev = save.save_to_int(); + ++save.m_open_begins[save.m_level]; + needFold = true; + } + else if (strcmp(buf, "end") == 0) + { + while (save.m_level > 0 && save.m_open_begins[save.m_level] == 0) + --save.m_level; + if (lev < 0) + lev = save.save_to_int(); + if (save.m_open_begins[save.m_level] > 0) + --save.m_open_begins[save.m_level]; + } + else + { + for (j = 0; j < 7; ++j) + if (strcmp(buf, structWords[j]) == 0) + break; + if (j >= 7) + continue; + save.m_level = j; // level before the command + for (j = save.m_level + 1; j < save.m_open_begins.size(); ++j) + { + save.m_open_begins[save.m_level] += save.m_open_begins[j]; + save.m_open_begins[j] = 0; + } + if (lev < 0) + lev = save.save_to_int(); + ++save.m_level; // level after the command + needFold = true; + } + } + if (lev < 0) + lev = save.save_to_int(); + if (needFold) + lev |= SC_FOLDLEVELHEADERFLAG; + styler.SetLevel(curLine, lev); + set_saves(curLine, save); + ++curLine; + startPos = styler.LineStart(curLine); + + if (static_cast(startPos) == styler.Length()) + { + lev = save.save_to_int(); + styler.SetLevel(curLine, lev); + set_saves(curLine, save); + resize_saves(curLine); + } + } while (startPos < endPos); + + styler.Flush(); +} + +// There are cases not handled correctly, like $abcd\textrm{what is $x+y$}z+w$. +// But I think it's already good enough. +void SCI_METHOD lex_lilypond::Lex( + Sci_PositionU startPos, + Sci_Position length, + int initStyle, + IDocument* pAccess) +{ + // startPos is assumed to be the first character of a line + LexAccessor styler(pAccess); + styler.StartAt(startPos); + int mode = get_mode(styler.GetLine(startPos) - 1); + int state = initStyle; + if ( + state == SCE_L_ERROR || state == SCE_L_SHORTCMD || + state == SCE_L_SPECIAL) // should not happen + state = mode_to_state(mode); + + char chNext = styler.SafeGetCharAt(startPos); + char chVerbatimDelim = '\0'; + styler.StartSegment(startPos); + Sci_Position lengthDoc = startPos + length; + + for (Sci_Position i = startPos; i < lengthDoc; i++) + { + char ch = chNext; + chNext = styler.SafeGetCharAt(i + 1); + + if (styler.IsLeadByte(ch)) + { + i++; + chNext = styler.SafeGetCharAt(i + 1); + continue; + } + + if (ch == '\r' || ch == '\n') + set_modes(styler.GetLine(i), mode); + + switch (state) + { + case SCE_L_DEFAULT: + switch (ch) + { + case '\\': + styler.ColourTo(i - 1, state); + if (lilypond::is_letter(chNext)) + { + state = SCE_L_COMMAND; + } + else if (lilypond::is_special(chNext)) + { + styler.ColourTo(i + 1, SCE_L_SPECIAL); + i++; + chNext = styler.SafeGetCharAt(i + 1); + } + else if (chNext == '\r' || chNext == '\n') + { + styler.ColourTo(i, SCE_L_ERROR); + } + else if (IsASCII(chNext)) + { + styler.ColourTo(i + 1, SCE_L_SHORTCMD); + if (chNext == '(') + { + mode = 1; + state = SCE_L_MATH; + } + else if (chNext == '[') + { + mode = 2; + state = SCE_L_MATH2; + } + i++; + chNext = styler.SafeGetCharAt(i + 1); + } + break; + case '$': + styler.ColourTo(i - 1, state); + if (chNext == '$') + { + styler.ColourTo(i + 1, SCE_L_SHORTCMD); + mode = 2; + state = SCE_L_MATH2; + i++; + chNext = styler.SafeGetCharAt(i + 1); + } + else + { + styler.ColourTo(i, SCE_L_SHORTCMD); + mode = 1; + state = SCE_L_MATH; + } + break; + case '%': + styler.ColourTo(i - 1, state); + state = SCE_L_COMMENT; + break; + } + break; + // These 3 will never be reached. + case SCE_L_ERROR: + case SCE_L_SPECIAL: + case SCE_L_SHORTCMD: + break; + case SCE_L_COMMAND: + if (!lilypond::is_letter(chNext)) + { + styler.ColourTo(i, state); + lilypond lp(styler); + if (lp.next_not_blank_is(i + 1, '[')) + { + state = SCE_L_CMDOPT; + } + else if (lp.last_word_is(i, "\\begin")) + { + state = SCE_L_TAG; + } + else if (lp.last_word_is(i, "\\end")) + { + state = SCE_L_TAG2; + } + else if ( + lp.last_word_is(i, "\\verb") && chNext != '*' && chNext != ' ') + { + chVerbatimDelim = chNext; + state = SCE_L_VERBATIM; + } + else + { + state = mode_to_state(mode); + } + } + break; + case SCE_L_CMDOPT: + if (ch == ']') + { + styler.ColourTo(i, state); + state = mode_to_state(mode); + } + break; + case SCE_L_TAG: + if (lilypond lp(styler); lp.is_tag_valid(i, lengthDoc)) + { + styler.ColourTo(i, state); + state = mode_to_state(mode); + if (lp.last_word_is(i, "{verbatim}")) + { + state = SCE_L_VERBATIM; + } + else if (lp.last_word_is(i, "{lstlisting}")) + { + state = SCE_L_VERBATIM; + } + else if (lp.last_word_is(i, "{comment}")) + { + state = SCE_L_COMMENT2; + } + else if (lp.last_word_is(i, "{math}") && mode == 0) + { + mode = 1; + state = SCE_L_MATH; + } + else if (lp.last_word_is_match_env(i) && mode == 0) + { + mode = 2; + state = SCE_L_MATH2; + } + } + else + { + styler.ColourTo(i, SCE_L_ERROR); + state = mode_to_state(mode); + ch = styler.SafeGetCharAt(i); + if (ch == '\r' || ch == '\n') + set_modes(styler.GetLine(i), mode); + } + chNext = styler.SafeGetCharAt(i + 1); + break; + case SCE_L_TAG2: + if (lilypond(styler).is_tag_valid(i, lengthDoc)) + { + styler.ColourTo(i, state); + state = mode_to_state(mode); + } + else + { + styler.ColourTo(i, SCE_L_ERROR); + state = mode_to_state(mode); + ch = styler.SafeGetCharAt(i); + if (ch == '\r' || ch == '\n') + set_modes(styler.GetLine(i), mode); + } + chNext = styler.SafeGetCharAt(i + 1); + break; + case SCE_L_MATH: + switch (ch) + { + case '\\': + styler.ColourTo(i - 1, state); + if (lilypond::is_letter(chNext)) + { + Sci_Position match = i + 3; + if (lilypond lp(styler); lp.last_word_is(match, "\\end")) + { + match++; + if (lp.is_tag_valid(match, lengthDoc)) + { + if (lp.last_word_is(match, "{math}")) + mode = 0; + } + } + state = SCE_L_COMMAND; + } + else if (lilypond::is_special(chNext)) + { + styler.ColourTo(i + 1, SCE_L_SPECIAL); + i++; + chNext = styler.SafeGetCharAt(i + 1); + } + else if (chNext == '\r' || chNext == '\n') + { + styler.ColourTo(i, SCE_L_ERROR); + } + else if (IsASCII(chNext)) + { + if (chNext == ')') + { + mode = 0; + state = SCE_L_DEFAULT; + } + styler.ColourTo(i + 1, SCE_L_SHORTCMD); + i++; + chNext = styler.SafeGetCharAt(i + 1); + } + break; + case '$': + styler.ColourTo(i - 1, state); + styler.ColourTo(i, SCE_L_SHORTCMD); + mode = 0; + state = SCE_L_DEFAULT; + break; + case '%': + styler.ColourTo(i - 1, state); + state = SCE_L_COMMENT; + break; + } + break; + case SCE_L_MATH2: + switch (ch) + { + case '\\': + styler.ColourTo(i - 1, state); + if (lilypond::is_letter(chNext)) + { + Sci_Position match = i + 3; + if (lilypond lp(styler); lp.last_word_is(match, "\\end")) + { + match++; + if (lp.is_tag_valid(match, lengthDoc)) + { + if (lp.last_word_is_match_env(match)) + mode = 0; + } + } + state = SCE_L_COMMAND; + } + else if (lilypond::is_special(chNext)) + { + styler.ColourTo(i + 1, SCE_L_SPECIAL); + i++; + chNext = styler.SafeGetCharAt(i + 1); + } + else if (chNext == '\r' || chNext == '\n') + { + styler.ColourTo(i, SCE_L_ERROR); + } + else if (IsASCII(chNext)) + { + if (chNext == ']') + { + mode = 0; + state = SCE_L_DEFAULT; + } + styler.ColourTo(i + 1, SCE_L_SHORTCMD); + i++; + chNext = styler.SafeGetCharAt(i + 1); + } + break; + case '$': + styler.ColourTo(i - 1, state); + if (chNext == '$') + { + styler.ColourTo(i + 1, SCE_L_SHORTCMD); + i++; + chNext = styler.SafeGetCharAt(i + 1); + mode = 0; + state = SCE_L_DEFAULT; + } + else + { // This may not be an error, e.g. + // \begin{equation}\text{$a$}\end{equation} + styler.ColourTo(i, SCE_L_SHORTCMD); + } + break; + case '%': + styler.ColourTo(i - 1, state); + state = SCE_L_COMMENT; + break; + } + break; + case SCE_L_COMMENT: + if (ch == '\r' || ch == '\n') + { + styler.ColourTo(i - 1, state); + state = mode_to_state(mode); + } + break; + case SCE_L_COMMENT2: + if (ch == '\\') + { + Sci_Position match = i + 3; + if (lilypond lp(styler); lp.last_word_is(match, "\\end")) + { + match++; + if (lp.is_tag_valid(match, lengthDoc)) + { + if (lp.last_word_is(match, "{comment}")) + { + styler.ColourTo(i - 1, state); + state = SCE_L_COMMAND; + } + } + } + } + break; + case SCE_L_VERBATIM: + if (ch == '\\') + { + Sci_Position match = i + 3; + if (lilypond lp(styler); lp.last_word_is(match, "\\end")) + { + match++; + if (lp.is_tag_valid(match, lengthDoc)) + { + if (lp.last_word_is(match, "{verbatim}")) + { + styler.ColourTo(i - 1, state); + state = SCE_L_COMMAND; + } + else if (lp.last_word_is(match, "{lstlisting}")) + { + styler.ColourTo(i - 1, state); + state = SCE_L_COMMAND; + } + } + } + } + else if (chNext == chVerbatimDelim) + { + styler.ColourTo(i + 1, state); + state = mode_to_state(mode); + chVerbatimDelim = '\0'; + i++; + chNext = styler.SafeGetCharAt(i + 1); + } + else if (chVerbatimDelim != '\0' && (ch == '\n' || ch == '\r')) + { + styler.ColourTo(i, SCE_L_ERROR); + state = mode_to_state(mode); + chVerbatimDelim = '\0'; + } + break; + } + } + + if (lengthDoc == styler.Length()) + { + resize_modes(styler.GetLine(lengthDoc - 1)); + } + + styler.ColourTo(lengthDoc - 1, state); + styler.Flush(); +} + +static const char* const emptyWordListDesc[] = {0}; + +LexerModule lmLilyPond( + lex_lilypond::language(), + lex_lilypond::get, + lex_lilypond::name(), + emptyWordListDesc); diff --git a/src/syntax/wex/lex-lilypond.h b/src/syntax/wex/lex-lilypond.h new file mode 100644 index 0000000000..0e8a1ed25d --- /dev/null +++ b/src/syntax/wex/lex-lilypond.h @@ -0,0 +1,173 @@ +//////////////////////////////////////////////////////////////////////////////// +// Name: lex-lilypond.h +// Purpose: Declaration of Scintilla::lex_lilypond +// Based on Scintilla::LexLatex +// Author: Anton van Wezenbeek +// Copyright: (c) 2023 Anton van Wezenbeek +//////////////////////////////////////////////////////////////////////////////// + +#pragma once + +namespace Scintilla +{ +/// Collects bool options +class lex_options +{ + friend class lex_option_set; + +public: + bool fold() const { return m_fold; }; + bool fold_comment() const { return m_fold_comment; }; + bool fold_compact() const { return m_fold_compact; }; + +private: + bool m_fold{false}, m_fold_comment{false}, m_fold_compact{false}; +}; + +/// Collects option set +class lex_option_set : public OptionSet +{ +public: + /// Default constructor. + lex_option_set() + { + DefineProperty("fold", &lex_options::m_fold); + DefineProperty("fold.comment", &lex_options::m_fold_comment); + DefineProperty("fold.compact", &lex_options::m_fold_compact); + } +}; + +/// The lilypond lexer class. +class lex_lilypond : public DefaultLexer +{ +public: + /// Static interface. + + /// Returns and creates the lexers object. + static inline ILexer5* get() { return new lex_lilypond(); } + + /// Returns language. + static inline int language() { return SCLEX_AUTOMATIC; }; + + /// Returns lexer name. + static inline const char* name() { return "lilypond"; }; + +private: + struct fold_save + { + fold_save() + : m_open_begins(8) + { + ; + }; + + int save_to_int() const + { + int sum = 0; + for (int i = 0; i <= m_level; ++i) + sum += m_open_begins[i]; + return ((sum + m_level + SC_FOLDLEVELBASE) & SC_FOLDLEVELNUMBERMASK); + } + + std::vector m_open_begins; + int m_level{0}; + }; + + /// Default constructor. + lex_lilypond(); + + /// Overide methods. + + void SCI_METHOD Fold( + Sci_PositionU startPos, + Sci_Position length, + int initStyle, + IDocument* pAccess) override; + + void SCI_METHOD Lex( + Sci_PositionU startPos, + Sci_Position length, + int initStyle, + IDocument* pAccess) override; + + const char* SCI_METHOD PropertyGet(const char* key) override + { + return m_option_set.PropertyGet(key); + }; + + const char* SCI_METHOD PropertyNames() override + { + return m_option_set.PropertyNames(); + }; + + Sci_Position SCI_METHOD PropertySet(const char* key, const char* val) override + { + if (m_option_set.PropertySet(&m_options, key, val)) + { + return 0; + } + + return -1; + }; + + int SCI_METHOD PropertyType(const char* name) override + { + return m_option_set.PropertyType(name); + }; + + /// Other methods. + + int get_mode(Sci_Position line) const + { + if (line >= 0 && line < static_cast(m_modes.size())) + return m_modes[line]; + return 0; + } + + void get_save(Sci_Position line, fold_save& save) const + { + if (line >= 0 && line < static_cast(m_saves.size())) + save = m_saves[line]; + else + { + save.m_level = 0; + for (int i = 0; i < 8; ++i) + save.m_open_begins[i] = 0; + } + } + + int mode_to_state(int mode) const; + + void resize_modes(Sci_Position numLines) + { + if (static_cast(m_modes.size()) > numLines * 2 + 256) + m_modes.resize(numLines + 128); + } + + void resize_saves(Sci_Position numLines) + { + if (static_cast(m_saves.size()) > numLines * 2 + 256) + m_saves.resize(numLines + 128); + } + + void set_modes(Sci_Position line, int mode) + { + if (line >= static_cast(m_modes.size())) + m_modes.resize(line + 1, 0); + m_modes[line] = mode; + } + + void set_saves(Sci_Position line, const fold_save& save) + { + if (line >= static_cast(m_saves.size())) + m_saves.resize(line + 1); + m_saves[line] = save; + } + + lex_options m_options; + lex_option_set m_option_set; + + std::vector m_modes; + std::vector m_saves; +}; +} // namespace Scintilla diff --git a/test/syntax/test-lexer.cpp b/test/syntax/test-lexer.cpp index 6dc407ae3e..38ea3e5e64 100644 --- a/test/syntax/test-lexer.cpp +++ b/test/syntax/test-lexer.cpp @@ -196,6 +196,9 @@ TEST_CASE("wex::lexer") REQUIRE(lexer.display_lexer() == "cpp"); REQUIRE(lexer.scintilla_lexer() == "cpp"); + REQUIRE(lexer.set("lilypond")); + REQUIRE(lexer.display_lexer() == "lilypond"); + REQUIRE(lexer.set("rfw")); REQUIRE(lexer.display_lexer() == "rfw"); REQUIRE(lexer.is_keyword("Documentation")); From 80a4e237069eb7ced1691ee1261506ade3c07cf2 Mon Sep 17 00:00:00 2001 From: Anton van Wezenbeek Date: Wed, 30 Aug 2023 14:11:28 +0200 Subject: [PATCH 2/4] reduced some double code blocks --- src/syntax/wex/lex-lilypond-util.h | 75 ++++++++++++++--- src/syntax/wex/lex-lilypond.cpp | 124 +++++++++++------------------ src/syntax/wex/lex-lilypond.h | 5 +- 3 files changed, 118 insertions(+), 86 deletions(-) diff --git a/src/syntax/wex/lex-lilypond-util.h b/src/syntax/wex/lex-lilypond-util.h index 2f1490f241..f168c2b581 100644 --- a/src/syntax/wex/lex-lilypond-util.h +++ b/src/syntax/wex/lex-lilypond-util.h @@ -16,7 +16,6 @@ class lilypond /// Static interface static bool is_letter(int ch); - static bool is_special(int ch); /// Other methods @@ -27,9 +26,18 @@ class lilypond ; } + bool is_special(char& ch, Sci_Position& no) const; bool is_tag_valid(Sci_Position& i, Sci_Position l) const; + + bool last_word_check( + Sci_Position start, + const char* needle, + const std::vector& next, + Sci_Position length, + int& state) const; bool last_word_is(Sci_Position start, const char* needle) const; bool last_word_is_match_env(Sci_Position pos) const; + bool next_not_blank_is(Sci_Position i, char needle) const; private: @@ -43,10 +51,19 @@ inline bool lilypond::is_letter(int ch) return IsASCII(ch) && isalpha(ch); } -inline bool lilypond::is_special(int ch) +inline bool lilypond::is_special(char& ch, Sci_Position& i) const { - return (ch == '#') || (ch == '$') || (ch == '%') || (ch == '&') || - (ch == '_') || (ch == '{') || (ch == '}') || (ch == ' '); + if ( + (ch == '#') || (ch == '$') || (ch == '%') || (ch == '&') || (ch == '_') || + (ch == '{') || (ch == '}') || (ch == ' ')) + { + m_styler.ColourTo(i + 1, SCE_L_SPECIAL); + i++; + ch = m_styler.SafeGetCharAt(i + 1); + return true; + } + + return false; } inline bool lilypond::is_tag_valid(Sci_Position& i, Sci_Position l) const @@ -97,13 +114,44 @@ inline bool lilypond::next_not_blank_is(Sci_Position i, char needle) const return false; } +inline bool lilypond::last_word_check( + Sci_Position start, + const char* needle, + const std::vector& checks, + Sci_Position lengthDoc, + int& state) const +{ + Sci_Position match = start + 3; + + if (last_word_is(match, needle)) + { + match++; + + if (is_tag_valid(match, lengthDoc)) + { + for (const auto& w : checks) + { + if (last_word_is(match, std::string("{" + w + "}").c_str())) + { + m_styler.ColourTo(start - 1, state); + state = SCE_L_COMMAND; + return true; + } + } + } + } + + return false; +} + inline bool lilypond::last_word_is(Sci_Position start, const char* needle) const { - Sci_PositionU i = 0; - Sci_PositionU l = static_cast(strlen(needle)); - Sci_Position ini = start - l + 1; - char s[32]; + const Sci_PositionU l = static_cast(strlen(needle)); + const Sci_Position ini = start - l + 1; + char s[32]; + + Sci_PositionU i = 0; while (i < l && i < 31) { s[i] = m_styler.SafeGetCharAt(ini + i); @@ -118,7 +166,8 @@ inline bool lilypond::last_word_is_match_env(Sci_Position pos) const { Sci_Position i, j; char s[32]; - const char* mathEnvs[] = { + + const char* mathEnvs[] = { "align", "alignat", "flalign", @@ -127,8 +176,10 @@ inline bool lilypond::last_word_is_match_env(Sci_Position pos) const "displaymath", "eqnarray", "equation"}; + if (m_styler.SafeGetCharAt(pos) != '}') return false; + for (i = pos - 1; i >= 0; --i) { if (m_styler.SafeGetCharAt(i) == '{') @@ -136,19 +187,25 @@ inline bool lilypond::last_word_is_match_env(Sci_Position pos) const if (pos - i >= 20) return false; } + if (i < 0 || i == pos - 1) return false; + ++i; for (j = 0; i + j < pos; ++j) s[j] = m_styler.SafeGetCharAt(i + j); + s[j] = '\0'; if (j == 0) return false; + if (s[j - 1] == '*') s[--j] = '\0'; + for (i = 0; i < static_cast(sizeof(mathEnvs) / sizeof(const char*)); ++i) if (strcmp(s, mathEnvs[i]) == 0) return true; + return false; } }; // namespace Scintilla diff --git a/src/syntax/wex/lex-lilypond.cpp b/src/syntax/wex/lex-lilypond.cpp index 7a1808f486..345ae53178 100644 --- a/src/syntax/wex/lex-lilypond.cpp +++ b/src/syntax/wex/lex-lilypond.cpp @@ -8,11 +8,39 @@ #include "lex-lilypond.h" #include "lex-lilypond-util.h" +#define STATE_ERROR() \ + styler.ColourTo(i, SCE_L_ERROR); \ + state = mode_to_state(mode); \ + ch = styler.SafeGetCharAt(i); \ + if (ch == '\r' || ch == '\n') \ + set_modes(styler.GetLine(i), mode); + lex_lilypond::lex_lilypond() : DefaultLexer(name(), language()) { } +void lex_lilypond::fold_dec(int& lev, fold_save& save) const +{ + while (save.m_level > 0 && save.m_open_begins[save.m_level] == 0) + --save.m_level; + + if (lev < 0) + lev = save.to_int(); + + if (save.m_open_begins[save.m_level] > 0) + --save.m_open_begins[save.m_level]; +} + +void lex_lilypond::fold_inc(int& lev, fold_save& save, bool& need) const +{ + if (lev < 0) + lev = save.to_int(); + + ++save.m_open_begins[save.m_level]; + need = true; +} + int lex_lilypond::mode_to_state(int mode) const { switch (mode) @@ -65,19 +93,11 @@ void SCI_METHOD lex_lilypond::Fold( break; if (ch == '{') { - if (lev < 0) - lev = save.save_to_int(); - ++save.m_open_begins[save.m_level]; - needFold = true; + fold_inc(lev, save, needFold); } else if (ch == '}') { - while (save.m_level > 0 && save.m_open_begins[save.m_level] == 0) - --save.m_level; - if (lev < 0) - lev = save.save_to_int(); - if (save.m_open_begins[save.m_level] > 0) - --save.m_open_begins[save.m_level]; + fold_dec(lev, save); } else if (ch != '\\' || styler.StyleAt(i) != SCE_L_COMMAND) continue; @@ -91,19 +111,11 @@ void SCI_METHOD lex_lilypond::Fold( buf[j] = '\0'; if (strcmp(buf, "begin") == 0) { - if (lev < 0) - lev = save.save_to_int(); - ++save.m_open_begins[save.m_level]; - needFold = true; + fold_inc(lev, save, needFold); } else if (strcmp(buf, "end") == 0) { - while (save.m_level > 0 && save.m_open_begins[save.m_level] == 0) - --save.m_level; - if (lev < 0) - lev = save.save_to_int(); - if (save.m_open_begins[save.m_level] > 0) - --save.m_open_begins[save.m_level]; + fold_dec(lev, save); } else { @@ -119,13 +131,13 @@ void SCI_METHOD lex_lilypond::Fold( save.m_open_begins[j] = 0; } if (lev < 0) - lev = save.save_to_int(); + lev = save.to_int(); ++save.m_level; // level after the command needFold = true; } } if (lev < 0) - lev = save.save_to_int(); + lev = save.to_int(); if (needFold) lev |= SC_FOLDLEVELHEADERFLAG; styler.SetLevel(curLine, lev); @@ -135,7 +147,7 @@ void SCI_METHOD lex_lilypond::Fold( if (static_cast(startPos) == styler.Length()) { - lev = save.save_to_int(); + lev = save.to_int(); styler.SetLevel(curLine, lev); set_saves(curLine, save); resize_saves(curLine); @@ -194,11 +206,8 @@ void SCI_METHOD lex_lilypond::Lex( { state = SCE_L_COMMAND; } - else if (lilypond::is_special(chNext)) + else if (lilypond(styler).is_special(chNext, i)) { - styler.ColourTo(i + 1, SCE_L_SPECIAL); - i++; - chNext = styler.SafeGetCharAt(i + 1); } else if (chNext == '\r' || chNext == '\n') { @@ -315,11 +324,7 @@ void SCI_METHOD lex_lilypond::Lex( } else { - styler.ColourTo(i, SCE_L_ERROR); - state = mode_to_state(mode); - ch = styler.SafeGetCharAt(i); - if (ch == '\r' || ch == '\n') - set_modes(styler.GetLine(i), mode); + STATE_ERROR(); } chNext = styler.SafeGetCharAt(i + 1); break; @@ -331,11 +336,7 @@ void SCI_METHOD lex_lilypond::Lex( } else { - styler.ColourTo(i, SCE_L_ERROR); - state = mode_to_state(mode); - ch = styler.SafeGetCharAt(i); - if (ch == '\r' || ch == '\n') - set_modes(styler.GetLine(i), mode); + STATE_ERROR(); } chNext = styler.SafeGetCharAt(i + 1); break; @@ -358,11 +359,8 @@ void SCI_METHOD lex_lilypond::Lex( } state = SCE_L_COMMAND; } - else if (lilypond::is_special(chNext)) + else if (lilypond(styler).is_special(chNext, i)) { - styler.ColourTo(i + 1, SCE_L_SPECIAL); - i++; - chNext = styler.SafeGetCharAt(i + 1); } else if (chNext == '\r' || chNext == '\n') { @@ -411,11 +409,8 @@ void SCI_METHOD lex_lilypond::Lex( } state = SCE_L_COMMAND; } - else if (lilypond::is_special(chNext)) + else if (lilypond(styler).is_special(chNext, i)) { - styler.ColourTo(i + 1, SCE_L_SPECIAL); - i++; - chNext = styler.SafeGetCharAt(i + 1); } else if (chNext == '\r' || chNext == '\n') { @@ -465,42 +460,19 @@ void SCI_METHOD lex_lilypond::Lex( case SCE_L_COMMENT2: if (ch == '\\') { - Sci_Position match = i + 3; - if (lilypond lp(styler); lp.last_word_is(match, "\\end")) - { - match++; - if (lp.is_tag_valid(match, lengthDoc)) - { - if (lp.last_word_is(match, "{comment}")) - { - styler.ColourTo(i - 1, state); - state = SCE_L_COMMAND; - } - } - } + lilypond(styler) + .last_word_check(i, "\\end", {"comment"}, lengthDoc, state); } break; case SCE_L_VERBATIM: if (ch == '\\') { - Sci_Position match = i + 3; - if (lilypond lp(styler); lp.last_word_is(match, "\\end")) - { - match++; - if (lp.is_tag_valid(match, lengthDoc)) - { - if (lp.last_word_is(match, "{verbatim}")) - { - styler.ColourTo(i - 1, state); - state = SCE_L_COMMAND; - } - else if (lp.last_word_is(match, "{lstlisting}")) - { - styler.ColourTo(i - 1, state); - state = SCE_L_COMMAND; - } - } - } + lilypond(styler).last_word_check( + i, + "\\end", + {"verbatim", "lstlisting"}, + lengthDoc, + state); } else if (chNext == chVerbatimDelim) { diff --git a/src/syntax/wex/lex-lilypond.h b/src/syntax/wex/lex-lilypond.h index 0e8a1ed25d..2ec478d904 100644 --- a/src/syntax/wex/lex-lilypond.h +++ b/src/syntax/wex/lex-lilypond.h @@ -61,7 +61,7 @@ class lex_lilypond : public DefaultLexer ; }; - int save_to_int() const + int to_int() const { int sum = 0; for (int i = 0; i <= m_level; ++i) @@ -117,6 +117,9 @@ class lex_lilypond : public DefaultLexer /// Other methods. + void fold_dec(int& lev, fold_save& save) const; + void fold_inc(int& lev, fold_save& save, bool& need) const; + int get_mode(Sci_Position line) const { if (line >= 0 && line < static_cast(m_modes.size())) From 2e3615d0bb5588046e31e87c3bdcfdc944917511 Mon Sep 17 00:00:00 2001 From: Anton van Wezenbeek Date: Wed, 30 Aug 2023 16:25:43 +0200 Subject: [PATCH 3/4] more double code blocks fixes --- src/syntax/wex/lex-lilypond-util.h | 52 +++++++++++++++++------------- src/syntax/wex/lex-lilypond.cpp | 30 ++++++----------- 2 files changed, 40 insertions(+), 42 deletions(-) diff --git a/src/syntax/wex/lex-lilypond-util.h b/src/syntax/wex/lex-lilypond-util.h index f168c2b581..3b1e62ec8e 100644 --- a/src/syntax/wex/lex-lilypond-util.h +++ b/src/syntax/wex/lex-lilypond-util.h @@ -98,16 +98,11 @@ inline bool lilypond::is_tag_valid(Sci_Position& i, Sci_Position l) const inline bool lilypond::next_not_blank_is(Sci_Position i, char needle) const { - char ch; while (i < m_styler.Length()) { - ch = m_styler.SafeGetCharAt(i); - if (!isspace(ch) && ch != '*') + if (char ch = m_styler.SafeGetCharAt(i); !isspace(ch) && ch != '*') { - if (ch == needle) - return true; - else - return false; + return ch == needle; } i++; } @@ -129,15 +124,26 @@ inline bool lilypond::last_word_check( if (is_tag_valid(match, lengthDoc)) { - for (const auto& w : checks) + if (checks.empty()) { - if (last_word_is(match, std::string("{" + w + "}").c_str())) + if (last_word_is_match_env(match)) { - m_styler.ColourTo(start - 1, state); state = SCE_L_COMMAND; return true; } } + else + { + for (const auto& w : checks) + { + if (last_word_is(match, std::string("{" + w + "}").c_str())) + { + m_styler.ColourTo(start - 1, state); + state = SCE_L_COMMAND; + return true; + } + } + } } } @@ -167,16 +173,6 @@ inline bool lilypond::last_word_is_match_env(Sci_Position pos) const Sci_Position i, j; char s[32]; - const char* mathEnvs[] = { - "align", - "alignat", - "flalign", - "gather", - "multiline", - "displaymath", - "eqnarray", - "equation"}; - if (m_styler.SafeGetCharAt(pos) != '}') return false; @@ -202,9 +198,21 @@ inline bool lilypond::last_word_is_match_env(Sci_Position pos) const if (s[j - 1] == '*') s[--j] = '\0'; - for (i = 0; i < static_cast(sizeof(mathEnvs) / sizeof(const char*)); ++i) - if (strcmp(s, mathEnvs[i]) == 0) + for (const auto& word : std::vector{ + "align", + "alignat", + "flalign", + "gather", + "multiline", + "displaymath", + "eqnarray", + "equation"}) + { + if (word == s) + { return true; + } + } return false; } diff --git a/src/syntax/wex/lex-lilypond.cpp b/src/syntax/wex/lex-lilypond.cpp index 345ae53178..2f072b590d 100644 --- a/src/syntax/wex/lex-lilypond.cpp +++ b/src/syntax/wex/lex-lilypond.cpp @@ -62,7 +62,7 @@ void SCI_METHOD lex_lilypond::Fold( int, IDocument* pAccess) { - const char* structWords[7] = { + const std::vector structWords{ "part", "chapter", "section", @@ -119,10 +119,10 @@ void SCI_METHOD lex_lilypond::Fold( } else { - for (j = 0; j < 7; ++j) - if (strcmp(buf, structWords[j]) == 0) + for (j = 0; j < structWords.size(); ++j) + if (strcmp(buf, structWords[j].c_str()) == 0) break; - if (j >= 7) + if (j >= structWords.size()) continue; save.m_level = j; // level before the command for (j = save.m_level + 1; j < save.m_open_begins.size(); ++j) @@ -347,15 +347,10 @@ void SCI_METHOD lex_lilypond::Lex( styler.ColourTo(i - 1, state); if (lilypond::is_letter(chNext)) { - Sci_Position match = i + 3; - if (lilypond lp(styler); lp.last_word_is(match, "\\end")) + if (lilypond lp(styler); + lp.last_word_check(i, "\\end", {"math"}, lengthDoc, state)) { - match++; - if (lp.is_tag_valid(match, lengthDoc)) - { - if (lp.last_word_is(match, "{math}")) - mode = 0; - } + mode = 0; } state = SCE_L_COMMAND; } @@ -397,15 +392,10 @@ void SCI_METHOD lex_lilypond::Lex( styler.ColourTo(i - 1, state); if (lilypond::is_letter(chNext)) { - Sci_Position match = i + 3; - if (lilypond lp(styler); lp.last_word_is(match, "\\end")) + if (lilypond lp(styler); + lp.last_word_check(i, "\\end", {}, lengthDoc, state)) { - match++; - if (lp.is_tag_valid(match, lengthDoc)) - { - if (lp.last_word_is_match_env(match)) - mode = 0; - } + mode = 0; } state = SCE_L_COMMAND; } From e599d1cd41e1b01fd2058bc0ccb8a88ed857b168 Mon Sep 17 00:00:00 2001 From: Anton van Wezenbeek Date: Thu, 31 Aug 2023 19:24:31 +0200 Subject: [PATCH 4/4] added handle_match to fix another duplicate --- src/syntax/wex/lex-lilypond.cpp | 105 +++++++++++++++----------------- 1 file changed, 49 insertions(+), 56 deletions(-) diff --git a/src/syntax/wex/lex-lilypond.cpp b/src/syntax/wex/lex-lilypond.cpp index 2f072b590d..3642db41f2 100644 --- a/src/syntax/wex/lex-lilypond.cpp +++ b/src/syntax/wex/lex-lilypond.cpp @@ -15,6 +15,53 @@ if (ch == '\r' || ch == '\n') \ set_modes(styler.GetLine(i), mode); +void handle_math( + int lengthDoc, + int& mode, + Sci_Position& i, + LexAccessor& styler, + int& state, + char& chNext) +{ + styler.ColourTo(i - 1, state); + + if (lilypond::is_letter(chNext)) + { + if (lilypond lp(styler); lp.last_word_check( + i, + "\\end", + state == SCE_L_MATH2 ? std::vector{"math"} : + std::vector{}, + lengthDoc, + state)) + { + mode = 0; + } + state = SCE_L_COMMAND; + } + else if (lilypond(styler).is_special(chNext, i)) + { + } + else if (chNext == '\r' || chNext == '\n') + { + styler.ColourTo(i, SCE_L_ERROR); + } + else if (IsASCII(chNext)) + { + if ( + (state == SCE_L_MATH && chNext == ')') || + (state == SCE_L_MATH2 && chNext == ']')) + { + mode = 0; + state = SCE_L_DEFAULT; + } + + styler.ColourTo(i + 1, SCE_L_SHORTCMD); + i++; + chNext = styler.SafeGetCharAt(i + 1); + } +} + lex_lilypond::lex_lilypond() : DefaultLexer(name(), language()) { @@ -344,34 +391,7 @@ void SCI_METHOD lex_lilypond::Lex( switch (ch) { case '\\': - styler.ColourTo(i - 1, state); - if (lilypond::is_letter(chNext)) - { - if (lilypond lp(styler); - lp.last_word_check(i, "\\end", {"math"}, lengthDoc, state)) - { - mode = 0; - } - state = SCE_L_COMMAND; - } - else if (lilypond(styler).is_special(chNext, i)) - { - } - else if (chNext == '\r' || chNext == '\n') - { - styler.ColourTo(i, SCE_L_ERROR); - } - else if (IsASCII(chNext)) - { - if (chNext == ')') - { - mode = 0; - state = SCE_L_DEFAULT; - } - styler.ColourTo(i + 1, SCE_L_SHORTCMD); - i++; - chNext = styler.SafeGetCharAt(i + 1); - } + handle_math(lengthDoc, mode, i, styler, state, chNext); break; case '$': styler.ColourTo(i - 1, state); @@ -389,34 +409,7 @@ void SCI_METHOD lex_lilypond::Lex( switch (ch) { case '\\': - styler.ColourTo(i - 1, state); - if (lilypond::is_letter(chNext)) - { - if (lilypond lp(styler); - lp.last_word_check(i, "\\end", {}, lengthDoc, state)) - { - mode = 0; - } - state = SCE_L_COMMAND; - } - else if (lilypond(styler).is_special(chNext, i)) - { - } - else if (chNext == '\r' || chNext == '\n') - { - styler.ColourTo(i, SCE_L_ERROR); - } - else if (IsASCII(chNext)) - { - if (chNext == ']') - { - mode = 0; - state = SCE_L_DEFAULT; - } - styler.ColourTo(i + 1, SCE_L_SHORTCMD); - i++; - chNext = styler.SafeGetCharAt(i + 1); - } + handle_math(lengthDoc, mode, i, styler, state, chNext); break; case '$': styler.ColourTo(i - 1, state);