Skip to content

Commit

Permalink
refactor Tokens to be 16b
Browse files Browse the repository at this point in the history
  • Loading branch information
@keithamus
keithamus committed Feb 3, 2024
1 parent fc72ab4 commit 40faef6
Show file tree
Hide file tree
Showing 16 changed files with 685 additions and 688 deletions.
1 change: 1 addition & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions crates/hdx_ast/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ hdx_atomizable_derive = { workspace = true }
# Use OXC Allocator until https://github.com/fitzgen/bumpalo/pull/210 is resolved
oxc_allocator = { workspace = true }
bumpalo = { workspace = true, features = ["collections", "boxed"] }
miette = { workspace = true }

serde = { workspace = true, optional = true }
serde_json = { workspace = true, optional = true }
Expand Down
3 changes: 2 additions & 1 deletion crates/hdx_ast/src/css/qualified_rule.rs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
use hdx_lexer::Span;
#[cfg(feature = "serde")]
use serde::Serialize;

use crate::Span;

#[derive(Debug, Hash)]
#[cfg_attr(feature = "serde", derive(Serialize), serde(tag = "type", rename_all = "camelCase"))]
pub struct QualifiedRule<'a> {
Expand Down
3 changes: 1 addition & 2 deletions crates/hdx_ast/src/css/rules/page.rs
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
use hdx_lexer::Span;
use oxc_allocator::{Box, Vec};
#[cfg(feature = "serde")]
use serde::Serialize;

use crate::{
atom, css::properties::Property, Atom, Atomizable, Spanned, Specificity, ToSpecificity,
atom, css::properties::Property, Atom, Atomizable, Span, Spanned, Specificity, ToSpecificity,
};

// https://drafts.csswg.org/cssom-1/#csspagerule
Expand Down
6 changes: 3 additions & 3 deletions crates/hdx_ast/src/lib.rs
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
#![feature(slice_concat_trait)]

extern crate hdx_atomizable_derive;

pub use hdx_atomizable_derive::Atomizable;
#[cfg(feature = "serde")]
use serde::Serialize;

pub mod css;
pub mod span;
pub mod traits;

pub(crate) use hdx_atom::{atom, Atom, Atomizable};
pub(crate) use hdx_lexer::{PairWise, Span, Token};
pub(crate) use hdx_lexer::{PairWise, Token};
pub(crate) use oxc_allocator::{Allocator, Box, Vec};
pub(crate) use span::Span;
pub use traits::Unit;

#[derive(Debug, PartialEq, Hash)]
Expand Down
0 crates/hdx_lexer/src/span.rs → crates/hdx_ast/src/span.rs
View file
File renamed without changes.
260 changes: 130 additions & 130 deletions crates/hdx_lexer/src/constants.rs
View file
Original file line number Diff line number Diff line change
@@ -1,134 +1,134 @@
use super::Kind;
use super::Token;

pub const SURROGATE_RANGE: std::ops::RangeInclusive<u32> = 0xd800..=0xdfff;

pub const SINGLE_CHAR_TOKENS: &[Kind; 128] = &[
/* 0 */ Kind::Undetermined,
/* 1 */ Kind::Undetermined,
/* 2 */ Kind::Undetermined,
/* 3 */ Kind::Undetermined,
/* 4 */ Kind::Undetermined,
/* 5 */ Kind::Undetermined,
/* 6 */ Kind::Undetermined,
/* 7 */ Kind::Undetermined,
/* 8 */ Kind::Undetermined,
/* 9 */ Kind::Undetermined,
/* 10 */ Kind::Undetermined,
/* 11 */ Kind::Undetermined,
/* 12 */ Kind::Undetermined,
/* 13 */ Kind::Undetermined,
/* 14 */ Kind::Undetermined,
/* 15 */ Kind::Undetermined,
/* 16 */ Kind::Undetermined,
/* 17 */ Kind::Undetermined,
/* 18 */ Kind::Undetermined,
/* 19 */ Kind::Undetermined,
/* 20 */ Kind::Undetermined,
/* 21 */ Kind::Undetermined,
/* 22 */ Kind::Undetermined,
/* 23 */ Kind::Undetermined,
/* 24 */ Kind::Undetermined,
/* 25 */ Kind::Undetermined,
/* 26 */ Kind::Undetermined,
/* 27 */ Kind::Undetermined,
/* 28 */ Kind::Undetermined,
/* 29 */ Kind::Undetermined,
/* 30 */ Kind::Undetermined,
/* 31 */ Kind::Undetermined,
/* 32 */ Kind::Undetermined,
/* 33 */ Kind::Undetermined,
/* 34 */ Kind::Undetermined,
/* 35 */ Kind::Undetermined,
/* 36 */ Kind::Undetermined,
/* 37 */ Kind::Undetermined,
/* 38 */ Kind::Undetermined,
/* 39 */ Kind::Undetermined,
/* 40 */ Kind::LeftParen, // 0x28
/* 41 */ Kind::RightParen, // 0x29
/* 42 */ Kind::Undetermined,
/* 43 */ Kind::Undetermined,
/* 44 */ Kind::Comma, // 0x2C
/* 45 */ Kind::Undetermined,
/* 46 */ Kind::Undetermined,
/* 47 */ Kind::Undetermined,
/* 48 */ Kind::Undetermined,
/* 49 */ Kind::Undetermined,
/* 50 */ Kind::Undetermined,
/* 51 */ Kind::Undetermined,
/* 52 */ Kind::Undetermined,
/* 53 */ Kind::Undetermined,
/* 54 */ Kind::Undetermined,
/* 55 */ Kind::Undetermined,
/* 56 */ Kind::Undetermined,
/* 57 */ Kind::Undetermined,
/* 58 */ Kind::Colon, // 0x3A
/* 59 */ Kind::Semicolon, // 0x3B
/* 60 */ Kind::Undetermined,
/* 61 */ Kind::Undetermined,
/* 62 */ Kind::Undetermined, // 0x3E
/* 63 */ Kind::Undetermined,
/* 64 */ Kind::Undetermined,
/* 65 */ Kind::Undetermined,
/* 66 */ Kind::Undetermined,
/* 67 */ Kind::Undetermined,
/* 68 */ Kind::Undetermined,
/* 69 */ Kind::Undetermined,
/* 70 */ Kind::Undetermined,
/* 71 */ Kind::Undetermined,
/* 72 */ Kind::Undetermined,
/* 73 */ Kind::Undetermined,
/* 74 */ Kind::Undetermined,
/* 75 */ Kind::Undetermined,
/* 76 */ Kind::Undetermined,
/* 77 */ Kind::Undetermined,
/* 78 */ Kind::Undetermined,
/* 79 */ Kind::Undetermined,
/* 80 */ Kind::Undetermined,
/* 81 */ Kind::Undetermined,
/* 82 */ Kind::Undetermined,
/* 83 */ Kind::Undetermined,
/* 84 */ Kind::Undetermined,
/* 85 */ Kind::Undetermined,
/* 86 */ Kind::Undetermined,
/* 87 */ Kind::Undetermined,
/* 88 */ Kind::Undetermined,
/* 89 */ Kind::Undetermined,
/* 90 */ Kind::Undetermined,
/* 91 */ Kind::LeftSquare, // 0x5B
/* 92 */ Kind::Undetermined,
/* 93 */ Kind::RightSquare, // 0x5D
/* 94 */ Kind::Undetermined,
/* 95 */ Kind::Undetermined,
/* 96 */ Kind::Undetermined,
/* 97 */ Kind::Undetermined,
/* 98 */ Kind::Undetermined,
/* 99 */ Kind::Undetermined,
/* 100 */ Kind::Undetermined,
/* 101 */ Kind::Undetermined,
/* 102 */ Kind::Undetermined,
/* 103 */ Kind::Undetermined,
/* 104 */ Kind::Undetermined,
/* 105 */ Kind::Undetermined,
/* 106 */ Kind::Undetermined,
/* 107 */ Kind::Undetermined,
/* 108 */ Kind::Undetermined,
/* 109 */ Kind::Undetermined,
/* 110 */ Kind::Undetermined,
/* 111 */ Kind::Undetermined,
/* 112 */ Kind::Undetermined,
/* 113 */ Kind::Undetermined,
/* 114 */ Kind::Undetermined,
/* 115 */ Kind::Undetermined,
/* 116 */ Kind::Undetermined,
/* 117 */ Kind::Undetermined,
/* 118 */ Kind::Undetermined,
/* 119 */ Kind::Undetermined,
/* 120 */ Kind::Undetermined,
/* 121 */ Kind::Undetermined,
/* 122 */ Kind::Undetermined,
/* 123 */ Kind::LeftCurly, // 0x7B
/* 124 */ Kind::Undetermined,
/* 125 */ Kind::RightCurly, // 0x7D
/* 126 */ Kind::Delim, // 0x7E
/* 127 */ Kind::Undetermined,
pub const SINGLE_CHAR_TOKENS: &[Token; 128] = &[
/* 0 */ Token::Undetermined,
/* 1 */ Token::Undetermined,
/* 2 */ Token::Undetermined,
/* 3 */ Token::Undetermined,
/* 4 */ Token::Undetermined,
/* 5 */ Token::Undetermined,
/* 6 */ Token::Undetermined,
/* 7 */ Token::Undetermined,
/* 8 */ Token::Undetermined,
/* 9 */ Token::Undetermined,
/* 10 */ Token::Undetermined,
/* 11 */ Token::Undetermined,
/* 12 */ Token::Undetermined,
/* 13 */ Token::Undetermined,
/* 14 */ Token::Undetermined,
/* 15 */ Token::Undetermined,
/* 16 */ Token::Undetermined,
/* 17 */ Token::Undetermined,
/* 18 */ Token::Undetermined,
/* 19 */ Token::Undetermined,
/* 20 */ Token::Undetermined,
/* 21 */ Token::Undetermined,
/* 22 */ Token::Undetermined,
/* 23 */ Token::Undetermined,
/* 24 */ Token::Undetermined,
/* 25 */ Token::Undetermined,
/* 26 */ Token::Undetermined,
/* 27 */ Token::Undetermined,
/* 28 */ Token::Undetermined,
/* 29 */ Token::Undetermined,
/* 30 */ Token::Undetermined,
/* 31 */ Token::Undetermined,
/* 32 */ Token::Undetermined,
/* 33 */ Token::Undetermined,
/* 34 */ Token::Undetermined,
/* 35 */ Token::Undetermined,
/* 36 */ Token::Undetermined,
/* 37 */ Token::Undetermined,
/* 38 */ Token::Undetermined,
/* 39 */ Token::Undetermined,
/* 40 */ Token::LeftParen, // 0x28
/* 41 */ Token::RightParen, // 0x29
/* 42 */ Token::Undetermined,
/* 43 */ Token::Undetermined,
/* 44 */ Token::Comma, // 0x2C
/* 45 */ Token::Undetermined,
/* 46 */ Token::Undetermined,
/* 47 */ Token::Undetermined,
/* 48 */ Token::Undetermined,
/* 49 */ Token::Undetermined,
/* 50 */ Token::Undetermined,
/* 51 */ Token::Undetermined,
/* 52 */ Token::Undetermined,
/* 53 */ Token::Undetermined,
/* 54 */ Token::Undetermined,
/* 55 */ Token::Undetermined,
/* 56 */ Token::Undetermined,
/* 57 */ Token::Undetermined,
/* 58 */ Token::Colon, // 0x3A
/* 59 */ Token::Semicolon, // 0x3B
/* 60 */ Token::Undetermined,
/* 61 */ Token::Undetermined,
/* 62 */ Token::Undetermined, // 0x3E
/* 63 */ Token::Undetermined,
/* 64 */ Token::Undetermined,
/* 65 */ Token::Undetermined,
/* 66 */ Token::Undetermined,
/* 67 */ Token::Undetermined,
/* 68 */ Token::Undetermined,
/* 69 */ Token::Undetermined,
/* 70 */ Token::Undetermined,
/* 71 */ Token::Undetermined,
/* 72 */ Token::Undetermined,
/* 73 */ Token::Undetermined,
/* 74 */ Token::Undetermined,
/* 75 */ Token::Undetermined,
/* 76 */ Token::Undetermined,
/* 77 */ Token::Undetermined,
/* 78 */ Token::Undetermined,
/* 79 */ Token::Undetermined,
/* 80 */ Token::Undetermined,
/* 81 */ Token::Undetermined,
/* 82 */ Token::Undetermined,
/* 83 */ Token::Undetermined,
/* 84 */ Token::Undetermined,
/* 85 */ Token::Undetermined,
/* 86 */ Token::Undetermined,
/* 87 */ Token::Undetermined,
/* 88 */ Token::Undetermined,
/* 89 */ Token::Undetermined,
/* 90 */ Token::Undetermined,
/* 91 */ Token::LeftSquare, // 0x5B
/* 92 */ Token::Undetermined,
/* 93 */ Token::RightSquare, // 0x5D
/* 94 */ Token::Undetermined,
/* 95 */ Token::Undetermined,
/* 96 */ Token::Undetermined,
/* 97 */ Token::Undetermined,
/* 98 */ Token::Undetermined,
/* 99 */ Token::Undetermined,
/* 100 */ Token::Undetermined,
/* 101 */ Token::Undetermined,
/* 102 */ Token::Undetermined,
/* 103 */ Token::Undetermined,
/* 104 */ Token::Undetermined,
/* 105 */ Token::Undetermined,
/* 106 */ Token::Undetermined,
/* 107 */ Token::Undetermined,
/* 108 */ Token::Undetermined,
/* 109 */ Token::Undetermined,
/* 110 */ Token::Undetermined,
/* 111 */ Token::Undetermined,
/* 112 */ Token::Undetermined,
/* 113 */ Token::Undetermined,
/* 114 */ Token::Undetermined,
/* 115 */ Token::Undetermined,
/* 116 */ Token::Undetermined,
/* 117 */ Token::Undetermined,
/* 118 */ Token::Undetermined,
/* 119 */ Token::Undetermined,
/* 120 */ Token::Undetermined,
/* 121 */ Token::Undetermined,
/* 122 */ Token::Undetermined,
/* 123 */ Token::LeftCurly, // 0x7B
/* 124 */ Token::Undetermined,
/* 125 */ Token::RightCurly, // 0x7D
/* 126 */ Token::Undetermined, // 0x7E
/* 127 */ Token::Undetermined,
];
Loading

0 comments on commit 40faef6

Please sign in to comment.