Fix commenting tokenization

Author: neriyaco

src/parser/parser.ts

@@ -21,13 +21,7 @@ export class Parser {
   }
 
   public parse() {
-    this.tokenStream = new TokenStream(
-      this.tokenizer.tokenize().filter((t) => {
-        return (
-          t.type !== TokenType.LineComment && t.type !== TokenType.BlockComment
-        );
-      })
-    );
+    this.tokenStream = new TokenStream(this.tokenizer.tokenize());
     const obj = this.parseValue();
     this.eat(TokenType.EOF);
     this.lateReferences.forEach((ref) => {

src/tokenizer/tokenizer.ts

@@ -66,7 +66,7 @@ export class Tokenizer {
           tokens.push(this.tokenizeNull());
           break;
         case char === "/":
-          tokens.push(this.tokenizeComment());
+          this.tokenizeComment();
           break;
         default:
           this.throwUnexpectedCharacter(char);
@@ -254,20 +254,16 @@ export class Tokenizer {
     return this.makeToken(TokenType.ObjectReference, name);
   }
 
-  private tokenizeComment(): Token {
+  private tokenizeComment() {
     this.next();
-    const nextChar = this.next();
+    const nextChar = this.input.peek();
     if (nextChar === "/") {
-      return this.makeToken(
-        TokenType.LineComment,
-        "//" + this.readSingleLineComment()
-      );
+      this.readSingleLineComment();
+      return;
     }
     if (nextChar === "*") {
-      return this.makeToken(
-        TokenType.BlockComment,
-        "/*" + this.readMultiLineComment()
-      );
+      this.readMultiLineComment();
+      return;
     }
     this.throwUnexpectedCharacter(nextChar);
   }
@@ -312,32 +308,31 @@ export class Tokenizer {
   }
 
   private readSingleLineComment() {
-    let value = "";
     while (!this.input.eof()) {
       const char = this.next();
       if (Characters.isLineBreak(char)) {
-        break;
+        this.nextLine();
+        return;
       }
-      value += char;
     }
-
-    return value;
   }
 
   private readMultiLineComment() {
-    let value = "";
+    this.next();
     while (!this.input.eof()) {
       const char = this.next();
+      if (Characters.isLineBreak(char)) {
+        this.nextLine();
+        continue;
+      }
       if (char === "*") {
         const nextChar = this.input.peek();
         if (nextChar === "/") {
           this.next();
-          break;
+          return;
         }
       }
-      value += char;
     }
-    return value;
   }
 
   private makeToken(type: TokenType, value: string): Token {

tests/simple.rson

@@ -13,16 +13,18 @@
       $namedObject, // This is a reference to the named object
       5,
     ],
-    true,
+    true, // This is a reference to the named object
     false,
     null
     /**
       * This is a comment
       */
+      /**/
   ],
   "object": {
     "key": "value"
   }(namedObject),
   "objectRef": $namedObject,
   "objectRef2": $ref1,
-}
+}
+//