Add support for Amazon Ion (#1017)

lib/rouge/demos/ion

@@ -0,0 +1,20 @@
+/**
+ * Sample Ion document.
+ */
+{
+ key: "value",
+ 'float': 1_234e-5,
+ "annotation": type:: null.string,
+ list: 'things':: ['symbol', nan, +inf, -inf, "other"],
+ sexp: (concat ('symbol' [list] "string" null)),
+ date: date:: 1970-01-01T00:00Z,
+ "long string": str:: (
+ '''
+ long strings are neat
+ '''
+ ),
+ struct: {
+ nested: { value: int:: 12_345 },
+ },
+ blob: {{ SGVsbG8sIFdvcmxkIQ== }}, // Hello, World!
+}

lib/rouge/lexers/ion.rb

@@ -0,0 +1,175 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+ module Lexers
+ class Ion < RegexLexer
+ title 'Ion'
+ desc 'Amazon Ion (https://amazon-ion.github.io/ion-docs)'
+ tag 'ion'
+ filenames '*.ion'
+ mimetypes 'application/ion'
+
+ escapes = %r{\\(?:[\\abtR"'/?\\]|x\h{2}|u\h{4}|U\h{8})}
+ q = %r{'(?:#{escapes}|\\'|[^"\n\r])+'}
+ qq = %r{"(?:#{escapes}|\\"|[^"\n\r])+"}
+ qqq = %r{'''}
+ symbol = %r{[A-Za-z_\$](?:[0-9A-Za-z_\$])*}
+
+ state :qq do
+ rule qq, Literal::String::Double
+ end
+
+ state :qqq do
+ rule %r{'''}, Literal::String::Double, :pop!
+ rule %r{[^']+}m, Literal::String::Double
+ rule %r{'}, Literal::String::Double
+ end
+
+ state :quotes do
+ rule qqq, Literal::String::Double, :qqq
+ mixin :qq
+ rule symbol, Literal::String::Symbol
+ rule q, Literal::String::Symbol
+ end
+
+ state :annotation do
+ annotation = %r{(?:[\u{0020}-\u{0026}]|[\u{0028}-\u{005B}]|[\u{005D}-\u{FFFF}]|[\t\b\f ])+}
+ rule %r{('#{annotation}'|#{symbol})(\s*)(::)} do
+ groups Name::Decorator, Text::Whitespace, Operator
+ end
+ end
+
+ state :comments do
+ rule %r{/\*.*?\*/}m, Comment::Multiline
+ rule %r{//.*?$}, Comment::Single
+ end
+
+ state :constants do
+ rule %r{(?:true|false)\b}, Name::Builtin
+ rule %r{null(?:\.(?:blob|bool|clob|decimal|float|int|list|null|sexp|string|struct|symbol|timestamp))?\b}, Name::Builtin
+ end
+
+ state :numbers do
+ rule %r{0b[01]+(?:_[01]+)*\b}, Literal::Number::Bin
+ rule %r{0x\h+(?:_\h+)*\b}, Literal::Number::Hex
+ rule %r{(?:nan|[+-]inf)\b}, Literal::Number::Float
+
+ integer = %r{-?(?:0|[1-9]\d*(?:_\d+)*)}
+ rule %r{#{integer}[.dD][+-]?(?:#{integer})*(?:[dDeE][+-]?#{integer})?}, Literal::Number::Float
+ rule %r{#{integer}[dDeE][+-]?#{integer}}, Literal::Number::Float
+ rule integer, Literal::Number::Integer
+ end
+
+ state :timestamps do
+ year = %r{000[1-9]|00[1-9]\d|0[1-9]\d{2}|[1-9]\d{3}}
+ month = %r{0[1-9]|1[0-2]}
+ day = %r{0[1-9]|[12]\d|3[01]}
+ date = %r{#{year}-#{month}-#{day}}
+
+ hour = %r{[01]\d|2[0-3]}
+ minute = %r{[0-5]\d}
+ second = %r{[0-5]\d(?:\.\d+)?}
+ offset = %r{Z|[+-]#{hour}:#{minute}}
+ time = %r{#{hour}:#{minute}(?::#{second})?#{offset}}
+
+ rule %r{#{date}(T#{time}?)?|#{year}(?:-#{month})?T}, Literal::Date
+ end
+
+ state :whitespace do
+ rule %r{\s+}, Text::Whitespace
+ end
+
+ state :blob do
+ rule %r/}}/, Punctuation::Indicator, :pop!
+
+ rule qqq, Literal::String::Double, :qqq
+ mixin :qq
+ mixin :whitespace
+
+ # no attempt to validate the Base64 blob
+ rule %r{(?:[A-Za-z0-9/\+=]+)}, Literal
+ end
+
+ state :containers do
+ rule %r/{{/, Punctuation::Indicator, :blob
+ rule %r{\[}, Punctuation::Indicator, :list
+ rule %r{\(}, Punctuation::Indicator, :sexp
+ rule %r/{/, Punctuation::Indicator, :struct
+ end
+
+ state :list do
+ rule %r{]}, Punctuation::Indicator, :pop!
+
+ mixin :containers
+ mixin :comments
+ mixin :annotation
+ mixin :whitespace
+ mixin :constants
+ mixin :timestamps
+ mixin :numbers
+ mixin :quotes
+
+ rule %r{[,=;]}, Punctuation
+ end
+
+ state :sexp do
+ rule %r{\)}, Punctuation::Indicator, :pop!
+ rule %r{(?:\+\+|--|<<|>>|\&\&|\.\.|\|\||[-+\*/=<>|&$^.#!%?@`~])}, Operator
+
+ mixin :containers
+ mixin :comments
+ mixin :annotation
+ mixin :whitespace
+ mixin :constants
+ mixin :timestamps
+ mixin :numbers
+ mixin :quotes
+ end
+
+ state :struct do
+ rule %r/}/, Punctuation::Indicator, :pop!
+
+ rule %r{(#{q}|#{qq}|#{symbol})(\s*)(:)} do
+ groups Name::Label, Text::Whitespace, Punctuation
+ push :value
+ end
+
+ mixin :containers
+ mixin :comments
+ mixin :whitespace
+ end
+
+ state :value do
+ mixin :containers
+ mixin :comments
+ mixin :whitespace
+ mixin :annotation
+ mixin :constants
+ mixin :timestamps
+ mixin :numbers
+ mixin :quotes
+
+ rule %r{,}, Punctuation, :pop!
+
+ rule %r/(})/ do
+ groups Punctuation::Indicator
+ pop!(2)
+ end
+ end
+
+ state :root do
+ rule %r{\s*\A\$(?:ion_1_0|ion_symbol_table)\b}, Name::Builtin::Pseudo
+
+ mixin :comments
+ mixin :annotation
+ mixin :constants
+ mixin :timestamps
+ mixin :numbers
+ mixin :containers
+ mixin :quotes
+ mixin :whitespace
+ end
+ end
+ end
+end

spec/lexers/ion_spec.rb

@@ -0,0 +1,199 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+describe Rouge::Lexers::Ion do
+ let(:subject) { Rouge::Lexers::Ion.new }
+
+ describe 'guessing' do
+ include Support::Guessing
+
+ it 'guesses by filename' do
+ assert_guess :filename => 'file.ion'
+ end
+
+ it 'guesses by mimetype' do
+ assert_guess :mimetype => 'application/ion'
+ end
+ end
+
+ describe 'lexing' do
+ include Support::Lexing
+
+ it 'handles a basic file' do
+ ion = <<~ION
+ /**
+ * comment
+ */
+ 'annotation' :: {
+ a: 1,
+ 'b': 2.0,
+ "c": float:: 1.23e-4,
+ "d": 'null':: null.symbol,
+ "e": 'timestamps':: [
+ 1970-01-01T00:00Z,
+ 1970T,
+ ],
+ f: 'constants and such':: [
+ nancy, nan,
+ "+infinity", +inf, -inf,
+ truee, false, true,
+ null.null, null,
+ nullish,
+ ],
+ "g": (add [0b101, 0xdeadbeef, -1_000] "hello"),
+ h: { i: '''string''', },
+ j: {{ aW9u }},
+ }
+ ION
+
+ assert_no_errors ion
+
+ assert_tokens_equal ion,
+ ["Comment.Multiline", "/**\n * comment\n */"],
+ ["Text.Whitespace", "\n"],
+ ["Name.Decorator", "'annotation'"],
+ ["Text.Whitespace", " "],
+ ["Operator", "::"],
+ ["Text.Whitespace", " "],
+ ["Punctuation.Indicator", "{"],
+ ["Text.Whitespace", "\n "],
+ ["Name.Label", "a"],
+ ["Punctuation", ":"],
+ ["Text.Whitespace", " "],
+ ["Literal.Number.Integer", "1"],
+ ["Punctuation", ","],
+ ["Text.Whitespace", "\n "],
+ ["Name.Label", "'b'"],
+ ["Punctuation", ":"],
+ ["Text.Whitespace", " "],
+ ["Literal.Number.Float", "2.0"],
+ ["Punctuation", ","],
+ ["Text.Whitespace", "\n "],
+ ["Name.Label", "\"c\""],
+ ["Punctuation", ":"],
+ ["Text.Whitespace", " "],
+ ["Name.Decorator", "float"],
+ ["Operator", "::"],
+ ["Text.Whitespace", " "],
+ ["Literal.Number.Float", "1.23e-4"],
+ ["Punctuation", ","],
+ ["Text.Whitespace", "\n "],
+ ["Name.Label", "\"d\""],
+ ["Punctuation", ":"],
+ ["Text.Whitespace", " "],
+ ["Name.Decorator", "'null'"],
+ ["Operator", "::"],
+ ["Text.Whitespace", " "],
+ ["Name.Builtin", "null.symbol"],
+ ["Punctuation", ","],
+ ["Text.Whitespace", "\n "],
+ ["Name.Label", "\"e\""],
+ ["Punctuation", ":"],
+ ["Text.Whitespace", " "],
+ ["Name.Decorator", "'timestamps'"],
+ ["Operator", "::"],
+ ["Text.Whitespace", " "],
+ ["Punctuation.Indicator", "["],
+ ["Text.Whitespace", "\n "],
+ ["Literal.Date", "1970-01-01T00:00Z"],
+ ["Punctuation", ","],
+ ["Text.Whitespace", "\n "],
+ ["Literal.Date", "1970T"],
+ ["Punctuation", ","],
+ ["Text.Whitespace", "\n "],
+ ["Punctuation.Indicator", "]"],
+ ["Punctuation", ","],
+ ["Text.Whitespace", "\n "],
+ ["Name.Label", "f"],
+ ["Punctuation", ":"],
+ ["Text.Whitespace", " "],
+ ["Name.Decorator", "'constants and such'"],
+ ["Operator", "::"],
+ ["Text.Whitespace", " "],
+ ["Punctuation.Indicator", "["],
+ ["Text.Whitespace", "\n "],
+ ["Literal.String.Symbol", "nancy"],
+ ["Punctuation", ","],
+ ["Text.Whitespace", " "],
+ ["Literal.Number.Float", "nan"],
+ ["Punctuation", ","],
+ ["Text.Whitespace", "\n "],
+ ["Literal.String.Double", "\"+infinity\""],
+ ["Punctuation", ","],
+ ["Text.Whitespace", " "],
+ ["Literal.Number.Float", "+inf"],
+ ["Punctuation", ","],
+ ["Text.Whitespace", " "],
+ ["Literal.Number.Float", "-inf"],
+ ["Punctuation", ","],
+ ["Text.Whitespace", "\n "],
+ ["Literal.String.Symbol", "truee"],
+ ["Punctuation", ","],
+ ["Text.Whitespace", " "],
+ ["Name.Builtin", "false"],
+ ["Punctuation", ","],
+ ["Text.Whitespace", " "],
+ ["Name.Builtin", "true"],
+ ["Punctuation", ","],
+ ["Text.Whitespace", "\n "],
+ ["Name.Builtin", "null.null"],
+ ["Punctuation", ","],
+ ["Text.Whitespace", " "],
+ ["Name.Builtin", "null"],
+ ["Punctuation", ","],
+ ["Text.Whitespace", "\n "],
+ ["Literal.String.Symbol", "nullish"],
+ ["Punctuation", ","],
+ ["Text.Whitespace", "\n "],
+ ["Punctuation.Indicator", "]"],
+ ["Punctuation", ","],
+ ["Text.Whitespace", "\n "],
+ ["Name.Label", "\"g\""],
+ ["Punctuation", ":"],
+ ["Text.Whitespace", " "],
+ ["Punctuation.Indicator", "("],
+ ["Literal.String.Symbol", "add"],
+ ["Text.Whitespace", " "],
+ ["Punctuation.Indicator", "["],
+ ["Literal.Number.Bin", "0b101"],
+ ["Punctuation", ","],
+ ["Text.Whitespace", " "],
+ ["Literal.Number.Hex", "0xdeadbeef"],
+ ["Punctuation", ","],
+ ["Text.Whitespace", " "],
+ ["Literal.Number.Integer", "-1_000"],
+ ["Punctuation.Indicator", "]"],
+ ["Text.Whitespace", " "],
+ ["Literal.String.Double", "\"hello\""],
+ ["Punctuation.Indicator", ")"],
+ ["Punctuation", ","],
+ ["Text.Whitespace", "\n "],
+ ["Name.Label", "h"],
+ ["Punctuation", ":"],
+ ["Text.Whitespace", " "],
+ ["Punctuation.Indicator", "{"],
+ ["Text.Whitespace", " "],
+ ["Name.Label", "i"],
+ ["Punctuation", ":"],
+ ["Text.Whitespace", " "],
+ ["Literal.String.Double", "'''string'''"],
+ ["Punctuation", ","],
+ ["Text.Whitespace", " "],
+ ["Punctuation.Indicator", "}"],
+ ["Punctuation", ","],
+ ["Text.Whitespace", "\n "],
+ ["Name.Label", "j"],
+ ["Punctuation", ":"],
+ ["Text.Whitespace", " "],
+ ["Punctuation.Indicator", "{{"],
+ ["Text.Whitespace", " "],
+ ["Literal", "aW9u"],
+ ["Text.Whitespace", " "],
+ ["Punctuation.Indicator", "}}"],
+ ["Punctuation", ","],
+ ["Text.Whitespace", "\n"],
+ ["Punctuation.Indicator", "}"],
+ ["Text.Whitespace", "\n"]
+ end
+ end
+end