Skip to content

Commit

Permalink
Update TypeScript lexer to allow nested generics (#1002)
Browse files Browse the repository at this point in the history 
#425 shows that generics are
not correctly identified in general, but they are rather being treated
as JSX elements. I proposed a simple solution in the comments by adding
a space between `<` and the word next to it, but I believe most people
will either not find the solution or some of them will find it rather
unappealing.

For this reason, I made the JSX rules recursive and added a `","`
`Punctuation` token inside so that there can be a number of generics
used, as well as allowing nested generics. While I am not really fond of
this hack, given that generics are already treated as JSX elements, I
think this is a fair and easy enough solution for most cases.

#### Before
<img width="359" alt="Screenshot 2024-09-20 at 9 28 05 PM"
src="https://github.com/user-attachments/assets/b03c2c8a-3278-438b-8803-00eb62cc4a17">

#### With spacing solution
<img width="392" alt="Screenshot 2024-09-20 at 9 30 13 PM"
src="https://github.com/user-attachments/assets/89289476-c92a-41df-b893-5ab289fa96aa">

#### With recursive JSX and `","` `Punctuation` token
<img width="362" alt="Screenshot 2024-09-20 at 9 55 11 PM"
src="https://github.com/user-attachments/assets/d77d892e-667d-4fb4-93cf-8227d5bd4b17">
  • Loading branch information
@fredrare
fredrare authored Sep 21, 2024
1 parent 10c0c52 commit a56e228
Show file tree
Hide file tree
Showing 3 changed files with 131 additions and 3 deletions.
10 changes: 8 additions & 2 deletions lexers/embedded/typescript.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,12 @@
</rule>
</state>
<state name="tag">
<rule>
<include state="jsx"/>
</rule>
<rule pattern=",">
<token type="Punctuation"/>
</rule>
<rule>
<include state="commentsandwhitespace"/>
</rule>
Expand Down Expand Up @@ -171,7 +177,7 @@
</rule>
<rule pattern="(?=/)">
<token type="Text"/>
<push state="#pop" state="badregex"/>
<push state="badregex"/>
</rule>
<rule>
<pop depth="1"/>
Expand Down Expand Up @@ -276,4 +282,4 @@
</rule>
</state>
</rules>
</lexer>
</lexer>
13 changes: 13 additions & 0 deletions lexers/testdata/typescript.actual
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,16 @@ const moduleName = modules.map();
declare module "fs" {}
declare module "@custom/plugin" {}
declare module "../../compiler/types" {}

type TestType<T> = {
[Key keyof T]: Required<Pick<T, Key>>
}

const TestComponent = ({ a, b }) => (
<Component width={100} prop={{ a, ...b }} attr="text">
<span>This is a component</span>
<SomethingElse />
</Component>
)

export { TestComponent }
111 changes: 110 additions & 1 deletion lexers/testdata/typescript.expected
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,5 +33,114 @@
{"type":"NameOther","value":"\"../../compiler/types\""},
{"type":"Text","value":" "},
{"type":"Punctuation","value":"{}"},
{"type":"Text","value":"\n"}
{"type":"Text","value":"\n\n"},
{"type":"KeywordReserved","value":"type"},
{"type":"Text","value":" "},
{"type":"NameOther","value":"TestType"},
{"type":"Punctuation","value":"\u003c"},
{"type":"NameTag","value":"T"},
{"type":"Punctuation","value":"\u003e"},
{"type":"Text","value":" "},
{"type":"Operator","value":"="},
{"type":"Text","value":" "},
{"type":"Punctuation","value":"{"},
{"type":"Text","value":"\n "},
{"type":"Punctuation","value":"["},
{"type":"NameOther","value":"Key"},
{"type":"Text","value":" "},
{"type":"Keyword","value":"keyof"},
{"type":"Text","value":" "},
{"type":"NameOther","value":"T"},
{"type":"Punctuation","value":"]"},
{"type":"Operator","value":":"},
{"type":"Text","value":" "},
{"type":"NameOther","value":"Required"},
{"type":"Punctuation","value":"\u003c"},
{"type":"NameTag","value":"Pick"},
{"type":"Punctuation","value":"\u003c"},
{"type":"NameTag","value":"T"},
{"type":"Punctuation","value":","},
{"type":"Text","value":" "},
{"type":"NameAttribute","value":"Key"},
{"type":"Punctuation","value":"\u003e\u003e"},
{"type":"Text","value":"\n"},
{"type":"Punctuation","value":"}"},
{"type":"Text","value":"\n\n"},
{"type":"KeywordReserved","value":"const"},
{"type":"Text","value":" "},
{"type":"NameOther","value":"TestComponent"},
{"type":"Text","value":" "},
{"type":"Operator","value":"="},
{"type":"Text","value":" "},
{"type":"Punctuation","value":"({"},
{"type":"Text","value":" "},
{"type":"NameOther","value":"a"},
{"type":"Punctuation","value":","},
{"type":"Text","value":" "},
{"type":"NameOther","value":"b"},
{"type":"Text","value":" "},
{"type":"Punctuation","value":"})"},
{"type":"Text","value":" "},
{"type":"Operator","value":"=\u003e"},
{"type":"Text","value":" "},
{"type":"Punctuation","value":"("},
{"type":"Text","value":"\n "},
{"type":"Punctuation","value":"\u003c"},
{"type":"NameTag","value":"Component"},
{"type":"Text","value":" "},
{"type":"NameAttribute","value":"width"},
{"type":"Operator","value":"="},
{"type":"Punctuation","value":"{"},
{"type":"LiteralNumberInteger","value":"100"},
{"type":"Punctuation","value":"}"},
{"type":"Text","value":" "},
{"type":"NameAttribute","value":"prop"},
{"type":"Operator","value":"="},
{"type":"Punctuation","value":"{{"},
{"type":"Text","value":" "},
{"type":"NameOther","value":"a"},
{"type":"Punctuation","value":","},
{"type":"Text","value":" "},
{"type":"Punctuation","value":"..."},
{"type":"NameOther","value":"b"},
{"type":"Text","value":" "},
{"type":"Punctuation","value":"}}"},
{"type":"Text","value":" "},
{"type":"NameAttribute","value":"attr"},
{"type":"Operator","value":"="},
{"type":"LiteralString","value":"\"text\""},
{"type":"Punctuation","value":"\u003e"},
{"type":"Text","value":"\n "},
{"type":"Punctuation","value":"\u003c"},
{"type":"NameTag","value":"span"},
{"type":"Punctuation","value":"\u003e"},
{"type":"NameOther","value":"This"},
{"type":"Text","value":" "},
{"type":"Keyword","value":"is"},
{"type":"Text","value":" "},
{"type":"NameOther","value":"a"},
{"type":"Text","value":" "},
{"type":"NameOther","value":"component"},
{"type":"Punctuation","value":"\u003c/"},
{"type":"NameTag","value":"span"},
{"type":"Punctuation","value":"\u003e"},
{"type":"Text","value":"\n "},
{"type":"Punctuation","value":"\u003c"},
{"type":"NameTag","value":"SomethingElse"},
{"type":"Text","value":" "},
{"type":"Punctuation","value":"/\u003e"},
{"type":"Text","value":"\n "},
{"type":"Punctuation","value":"\u003c/"},
{"type":"NameTag","value":"Component"},
{"type":"Punctuation","value":"\u003e"},
{"type":"Text","value":"\n"},
{"type":"Punctuation","value":")"},
{"type":"Text","value":"\n\n"},
{"type":"KeywordReserved","value":"export"},
{"type":"Text","value":" "},
{"type":"Punctuation","value":"{"},
{"type":"Text","value":" "},
{"type":"NameOther","value":"TestComponent"},
{"type":"Text","value":" "},
{"type":"Punctuation","value":"}"}
]

0 comments on commit a56e228

Please sign in to comment.