Rewrite lexer (#50)

Part of a rewrite... --------- Co-authored-by: Kristján Oddsson <[email protected]>
keithamus · Oct 27, 2024 · 49ada2a · 49ada2a
1 parent 4f3354e
commit 49ada2a
Show file tree

Hide file tree

Showing 92 changed files with 2,519 additions and 1,850 deletions.
diff --git a/crates/hdx_ast/src/css/properties/mod.rs b/crates/hdx_ast/src/css/properties/mod.rs
@@ -1,9 +1,9 @@
 use std::{default::Default, fmt::Debug, hash::Hash};
 
 use hdx_atom::{atom, Atom};
-use hdx_lexer::Token;
 use hdx_derive::Visitable;
-use hdx_parser::{peek, Declaration, DeclarationValue, Parse, Parser, Result as ParserResult, State};
+use hdx_lexer::{Kind, Token};
+use hdx_parser::{Declaration, DeclarationValue, Parse, Parser, Result as ParserResult, State};
 use hdx_writer::{CssWriter, Result as WriterResult, WriteCss};
 
 use crate::{css::values, syntax::ComponentValues};
@@ -98,21 +98,23 @@ impl<'a> WriteCss<'a> for Property<'a> {
 }
 
 #[inline]
-fn is_computed_token(token: &Token) -> bool {
-	matches!(token, Token::Function(atom) if matches!(
-		atom.to_ascii_lowercase(),
-		atom!("var")
-			| atom!("calc") | atom!("min")
-			| atom!("max") | atom!("clamp")
-			| atom!("round") | atom!("mod")
-			| atom!("rem") | atom!("sin")
-			| atom!("cos") | atom!("tan")
-			| atom!("asin") | atom!("atan")
-			| atom!("atan2") | atom!("pow")
-			| atom!("sqrt") | atom!("hypot")
-			| atom!("log") | atom!("exp")
-			| atom!("abs") | atom!("sign")
-	))
+fn is_computed_token(parser: &mut Parser) -> bool {
+	let token = parser.peek();
+	token.kind() == Kind::Function
+		&& matches!(
+			parser.parse_atom_lower(token),
+			atom!("var")
+				| atom!("calc") | atom!("min")
+				| atom!("max") | atom!("clamp")
+				| atom!("round") | atom!("mod")
+				| atom!("rem") | atom!("sin")
+				| atom!("cos") | atom!("tan")
+				| atom!("asin") | atom!("atan")
+				| atom!("atan2") | atom!("pow")
+				| atom!("sqrt") | atom!("hypot")
+				| atom!("log") | atom!("exp")
+				| atom!("abs") | atom!("sign")
+		)
 }
 
 macro_rules! style_value {
@@ -191,32 +193,33 @@ impl<'a> DeclarationValue<'a> for StyleValue<'a> {
 		if name.starts_with("--") {
 			return Ok(Self::Custom(Custom::parse(parser)?));
 		}
-		if let Token::Ident(atom) = parser.peek() {
-			match atom.to_ascii_lowercase() {
+		let peek = parser.peek();
+		if peek.kind() == Kind::Ident {
+			match parser.parse_atom_lower(peek) {
 				atom!("initial") => {
-					parser.advance();
+					parser.next();
 					return Ok(Self::Initial);
 				}
 				atom!("inherit") => {
-					parser.advance();
+					parser.next();
 					return Ok(Self::Inherit);
 				}
 				atom!("unset") => {
-					parser.advance();
+					parser.next();
 					return Ok(Self::Unset);
 				}
 				atom!("revert") => {
-					parser.advance();
+					parser.next();
 					return Ok(Self::Revert);
 				}
 				atom!("revert-layer") => {
-					parser.advance();
+					parser.next();
 					return Ok(Self::RevertLayer);
 				}
 				_ => {}
 			}
 		}
-		if is_computed_token(parser.peek()) {
+		if is_computed_token(parser) {
 			return Ok(Self::Computed(Computed::parse(parser)?));
 		}
 		macro_rules! parse_declaration_value {
@@ -228,16 +231,17 @@ impl<'a> DeclarationValue<'a> for StyleValue<'a> {
 						&$atom => {
 							let checkpoint = parser.checkpoint();
 							if let Ok(val) = values::$name::parse(parser) {
-								if peek!(parser, Token::Semicolon | Token::RightCurly | Token::Eof | Token::Delim('!')) {
+								let peek = parser.peek();
+								if peek.kind() == Kind::Eof || matches!(peek.char(), Some(';' | '}' | '!')) {
 									return Ok(Self::$name(val))
 								}
 							}
-							if is_computed_token(parser.peek()) {
+							if is_computed_token(parser) {
 								parser.rewind(checkpoint);
 								Self::Computed(Computed::parse(parser)?)
 							} else {
 								parser.rewind(checkpoint);
-								if is_computed_token(parser.peek()) {
+								if is_computed_token(parser) {
 									Self::Computed(Computed::parse(parser)?)
 								} else {
 									Self::Unknown(Unknown::parse(parser)?)

diff --git a/crates/hdx_ast/src/css/rules/charset.rs b/crates/hdx_ast/src/css/rules/charset.rs
@@ -1,9 +1,9 @@
 use hdx_atom::{atom, Atomizable};
 use hdx_derive::Atomizable;
-use hdx_lexer::{Include, QuoteStyle, Token};
+use hdx_lexer::{Include, Kind, QuoteStyle};
 use hdx_parser::{
 	diagnostics::{self},
-	expect, unexpected, Parse, Parser, Result as ParserResult,
+	expect, expect_ignore_case, unexpected, Parse, Parser, Result as ParserResult,
 };
 use hdx_writer::{write_css, CssWriter, OutputOption, Result as WriterResult, WriteCss};
 
@@ -65,18 +65,20 @@ pub enum Charset {
 
 impl<'a> Parse<'a> for Charset {
 	fn parse(parser: &mut Parser<'a>) -> ParserResult<Self> {
-		expect!(parser.next(), Token::AtKeyword(atom!("charset")));
-		expect!(parser.next_with(Include::Whitespace), Token::Whitespace);
-		match parser.next_with(Include::Whitespace) {
-			Token::String(atom, QuoteStyle::Double) => {
-				if let Some(rule) = Self::from_atom(atom) {
-					expect!(parser.next_with(Include::Whitespace), Token::Semicolon);
+		expect_ignore_case!(parser.next(), Kind::AtKeyword, atom!("charset"));
+		expect!(parser.next_with(Include::Whitespace), Kind::Whitespace);
+		let token = parser.next_with(Include::Whitespace);
+		match token.kind() {
+			Kind::String if token.quote_style() == QuoteStyle::Double => {
+				let atom = parser.parse_atom(token);
+				if let Some(rule) = Self::from_atom(&atom) {
+					expect!(parser.next_with(Include::Whitespace), Kind::Semicolon);
 					Ok(rule)
 				} else {
-					Err(diagnostics::UnexpectedCharset(atom.clone(), parser.span()))?
+					Err(diagnostics::UnexpectedCharset(atom, parser.span()))?
 				}
 			}
-			token => unexpected!(parser, token),
+			_ => unexpected!(parser, token),
 		}
 	}
 }

diff --git a/crates/hdx_ast/src/css/rules/font_face.rs b/crates/hdx_ast/src/css/rules/font_face.rs
@@ -11,7 +11,7 @@ pub struct FontFace<'a>(Vec<'a, Spanned<FontProperty<'a>>>);
 
 impl<'a> Parse<'a> for FontFace<'a> {
 	fn parse(parser: &mut Parser<'a>) -> ParserResult<Self> {
-		expect_ignore_case!(parser.next(), Token::AtKeyword(atom!("font-face")));
+		expect_ignore_case!(parser.next(), Kind::AtKeyword, atom!("font-face"));
 		Ok(Self(Self::parse_rule_list(parser)?))
 	}
 }

diff --git a/crates/hdx_ast/src/css/rules/keyframes.rs b/crates/hdx_ast/src/css/rules/keyframes.rs
@@ -1,5 +1,5 @@
 use hdx_atom::{atom, Atom};
-use hdx_lexer::{QuoteStyle, Token};
+use hdx_lexer::{Kind, QuoteStyle};
 use hdx_parser::{
 	diagnostics, discard, expect, expect_ignore_case, unexpected, unexpected_ident, AtRule, Parse, Parser,
 	Result as ParserResult, Spanned, Vec,
@@ -19,7 +19,7 @@ pub struct Keyframes<'a> {
 
 impl<'a> Parse<'a> for Keyframes<'a> {
 	fn parse(parser: &mut Parser<'a>) -> ParserResult<Self> {
-		expect_ignore_case!(parser.next(), Token::AtKeyword(atom!("keyframes")));
+		expect_ignore_case!(parser.next(), Kind::AtKeyword, atom!("keyframes"));
 		let span = parser.span();
 		match Self::parse_at_rule(parser)? {
 			(Some(name), Some(rules)) => Ok(Self { name, rules }),
@@ -60,14 +60,14 @@ impl KeyframeName {
 impl<'a> Parse<'a> for KeyframeName {
 	fn parse(parser: &mut Parser<'a>) -> ParserResult<Self> {
 		match parser.next() {
-			Token::Ident(atom) => {
+			Kind::Ident(atom) => {
 				if Self::valid_ident(atom) {
 					Ok(Self(atom.clone(), QuoteStyle::None))
 				} else {
 					unexpected_ident!(parser, atom)
 				}
 			}
-			Token::String(atom, quote_style) => Ok(Self(atom.clone(), *quote_style)),
+			Kind::String(atom, quote_style) => Ok(Self(atom.clone(), *quote_style)),
 			token => unexpected!(parser, token),
 		}
 	}
@@ -85,10 +85,10 @@ pub struct KeyframeList<'a>(Vec<'a, Spanned<Keyframe<'a>>>);
 
 impl<'a> Parse<'a> for KeyframeList<'a> {
 	fn parse(parser: &mut Parser<'a>) -> ParserResult<Self> {
-		expect!(parser.next(), Token::LeftCurly);
+		expect!(parser.next(), Kind::LeftCurly);
 		let mut rules = parser.new_vec();
 		loop {
-			if discard!(parser, Token::RightCurly) {
+			if discard!(parser, Kind::RightCurly) {
 				return Ok(Self(rules));
 			}
 			rules.push(Keyframe::parse_spanned(parser)?);
@@ -120,17 +120,17 @@ impl<'a> Parse<'a> for Keyframe<'a> {
 		let mut selector = smallvec![];
 		loop {
 			selector.push(KeyframeSelector::parse(parser)?);
-			if discard!(parser, Token::LeftCurly | Token::Eof) {
+			if discard!(parser, Kind::LeftCurly | Kind::Eof) {
 				break;
 			}
-			if !discard!(parser, Token::Comma) {
+			if !discard!(parser, Kind::Comma) {
 				unexpected!(parser, parser.peek());
 			}
 		}
 		let mut properties = parser.new_vec();
 		loop {
-			discard!(parser, Token::Semicolon);
-			if discard!(parser, Token::RightCurly | Token::Eof) {
+			discard!(parser, Kind::Semicolon);
+			if discard!(parser, Kind::RightCurly | Kind::Eof) {
 				break;
 			}
 			properties.push(Property::parse_spanned(parser)?);
@@ -172,14 +172,22 @@ pub enum KeyframeSelector {
 
 impl<'a> Parse<'a> for KeyframeSelector {
 	fn parse(parser: &mut Parser<'a>) -> ParserResult<Self> {
-		match parser.next() {
-			Token::Ident(atom) => match atom.to_ascii_lowercase() {
+		let token = parser.next();
+		match token.kind() {
+			Kind::Ident => match parser.parse_atom_lower(token) {
 				atom!("from") => Ok(KeyframeSelector::From),
 				atom!("to") => Ok(KeyframeSelector::To),
-				_ => unexpected_ident!(parser, atom),
+				atom => unexpected_ident!(parser, atom),
 			},
-			Token::Dimension(n, atom!("%"), _) if *n >= 0.0 && *n <= 100.0 => Ok(Self::Percent(n.into())),
-			token => unexpected!(parser, token),
+			Kind::Dimension => {
+				let n = parser.parse_number(token);
+				if matches!(parser.parse_atom(token), atom!("%")) && n >= 0.0 && n <= 100.0 {
+					Ok(Self::Percent(n.into()))
+				} else {
+					unexpected!(parser, token)
+				}
+			},
+			_ => unexpected!(parser, token),
 		}
 	}
 }

diff --git a/crates/hdx_ast/src/css/rules/media/features/hack.rs b/crates/hdx_ast/src/css/rules/media/features/hack.rs
@@ -1,5 +1,5 @@
 use hdx_atom::atom;
-use hdx_lexer::Token;
+use hdx_lexer::Kind;
 use hdx_parser::{expect, expect_ignore_case, peek, unexpected, Parse, Parser, Result as ParserResult};
 use hdx_writer::{write_css, CssWriter, Result as WriterResult, WriteCss};
 
@@ -13,19 +13,19 @@ impl<'a> Parse<'a> for HackMediaFeature {
 	fn parse(parser: &mut Parser<'a>) -> ParserResult<Self> {
 		expect_ignore_case! { parser.next(), Token::Ident(_):
 			atom!("min-width") => {
-				expect!(parser.next(), Token::Colon);
+				expect!(parser.next(), Kind::Colon);
 				let (a, b, c, d, e) = (
 					parser.legacy_peek_next_char(0),
 					parser.legacy_peek_next_char(1),
 					parser.legacy_peek_next_char(2),
 					parser.legacy_peek_next_char(3),
 					parser.legacy_peek_next_char(4)
 				);
-				if peek!(parser, Token::Dimension(_, _, _)) &&
+				if peek!(parser, Kind::Dimension) &&
 				(matches!((a, b, c, d), (Some('0'), Some('\\'), Some('0'), Some(' ') | Some(')') | None))) ||
 				(matches!((a, b, c, d, e), (Some(' '), Some('0'), Some('\\'), Some('0'), Some(' ') | Some(')') | None)))
 				{
-					parser.advance();
+					parser.next();
 					return Ok(Self::IEBackslashZero);
 				}
 				unexpected!(parser, parser.peek())