Use the correct token ctor for shebangs

This fixes #267.
jank-lang · Feb 19, 2025 · 88b3033 · 88b3033
1 parent 607bef3
commit 88b3033
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 20 deletions.
diff --git a/compiler+runtime/src/cpp/jank/read/lex.cpp b/compiler+runtime/src/cpp/jank/read/lex.cpp
@@ -107,7 +107,6 @@ namespace jank::read::lex
   {
   }
 
-#ifdef JANK_TEST
   token::token(movable_position const &s,
                movable_position const &e,
                token_kind const k,
@@ -119,6 +118,7 @@ namespace jank::read::lex
   {
   }
 
+#ifdef JANK_TEST
   token::token(size_t const offset, size_t const width, token_kind const k)
     : start{ offset, 1, offset + 1 }
     , end{ offset + width, 1, offset + width + 1 }
@@ -1161,14 +1161,14 @@ namespace jank::read::lex
                 ++pos;
                 if(pos == token_start + 2zu)
                 {
-                  return ok(token{ token_start, 1, token_kind::comment, ""sv });
+                  return ok(token{ token_start, pos, token_kind::comment, ""sv });
                 }
                 else
                 {
                   auto const length{ pos - token_start - 2 };
                   native_persistent_string_view const comment{ file.data() + token_start + 2,
                                                                length };
-                  return ok(token{ token_start, length, token_kind::comment, comment });
+                  return ok(token{ token_start, pos, token_kind::comment, comment });
                 }
               }
             default:

diff --git a/compiler+runtime/test/cpp/jank/read/lex.cpp b/compiler+runtime/test/cpp/jank/read/lex.cpp
@@ -1642,7 +1642,7 @@ namespace jank::read::lex
           native_vector<result<token, error_ptr>> const tokens(p.begin(), p.end());
           CHECK(tokens
                 == make_tokens({
-                  { 0, 1, token_kind::comment, ""sv }
+                  { 0, 2, token_kind::comment, ""sv }
           }));
         }
 
@@ -1652,8 +1652,8 @@ namespace jank::read::lex
           native_vector<result<token, error_ptr>> const tokens(p.begin(), p.end());
           CHECK(tokens
                 == make_tokens({
-                  { 0, 1, token_kind::comment, ""sv },
-                  { 3, 1, token_kind::comment, ""sv }
+                  {           0,           2, token_kind::comment, ""sv },
+                  { { 3, 2, 1 }, { 5, 2, 3 }, token_kind::comment, ""sv }
           }));
         }
 
@@ -1663,7 +1663,7 @@ namespace jank::read::lex
           native_vector<result<token, error_ptr>> const tokens(p.begin(), p.end());
           CHECK(tokens
                 == make_tokens({
-                  { 0, 3, token_kind::comment, "foo"sv },
+                  { 0, 5, token_kind::comment, "foo"sv },
           }));
         }
 
@@ -1673,7 +1673,7 @@ namespace jank::read::lex
           native_vector<result<token, error_ptr>> const tokens(p.begin(), p.end());
           CHECK(tokens
                 == make_tokens({
-                  { 0, 9, token_kind::comment, "foo #!bar"sv }
+                  { 0, 11, token_kind::comment, "foo #!bar"sv }
           }));
         }
 
@@ -1683,7 +1683,7 @@ namespace jank::read::lex
           native_vector<result<token, error_ptr>> const tokens(p.begin(), p.end());
           CHECK(tokens
                 == make_tokens({
-                  { 0, 6, token_kind::comment, "foo #!"sv }
+                  { 0, 8, token_kind::comment, "foo #!"sv }
           }));
         }
 
@@ -1693,7 +1693,7 @@ namespace jank::read::lex
           native_vector<result<token, error_ptr>> const tokens(p.begin(), p.end());
           CHECK(tokens
                 == make_tokens({
-                  { 0, 7, token_kind::comment, "#!#!foo"sv }
+                  { 0, 9, token_kind::comment, "#!#!foo"sv }
           }));
         }
 
@@ -1705,7 +1705,7 @@ namespace jank::read::lex
                 == make_tokens({
                   { 0, 1, token_kind::integer,     1ll },
                   { 2, 1, token_kind::integer,     2ll },
-                  { 4, 3, token_kind::comment, "foo"sv }
+                  { 4, 5, token_kind::comment, "foo"sv }
           }));
         }
 
@@ -1716,7 +1716,7 @@ namespace jank::read::lex
           CHECK(tokens
                 == make_tokens({
                   {           0,           1, token_kind::integer,     1ll },
-                  {           2,           3, token_kind::comment, "foo"sv },
+                  {           2,           5, token_kind::comment, "foo"sv },
                   { { 8, 2, 1 }, { 9, 2, 2 }, token_kind::integer,     2ll }
           }));
         }
@@ -1727,8 +1727,8 @@ namespace jank::read::lex
           native_vector<result<token, error_ptr>> const tokens(p.begin(), p.end());
           CHECK(tokens
                 == make_tokens({
-                  { 0, 3, token_kind::comment, "foo"sv },
-                  { 6, 3, token_kind::comment, "bar"sv },
+                  {           0,            5, token_kind::comment, "foo"sv },
+                  { { 6, 2, 1 }, { 11, 2, 6 }, token_kind::comment, "bar"sv },
           }));
         }
 
@@ -1739,7 +1739,7 @@ namespace jank::read::lex
           CHECK(tokens
                 == make_tokens({
                   { 0, 1, token_kind::reader_macro },
-                  { 1, 3, token_kind::comment, "foo"sv },
+                  { 1, 5, token_kind::comment, "foo"sv },
           }));
         }
 
@@ -1749,7 +1749,7 @@ namespace jank::read::lex
           native_vector<result<token, error_ptr>> const tokens(p.begin(), p.end());
           CHECK(tokens
                 == make_tokens({
-                  { 0, 4, token_kind::comment, "!foo"sv },
+                  { 0, 6, token_kind::comment, "!foo"sv },
           }));
         }
 
@@ -1759,7 +1759,7 @@ namespace jank::read::lex
           native_vector<result<token, error_ptr>> const tokens(p.begin(), p.end());
           CHECK(tokens
                 == make_tokens({
-                  { 0, 4, token_kind::comment, "#foo"sv },
+                  { 0, 6, token_kind::comment, "#foo"sv },
           }));
         }
 
@@ -1769,7 +1769,7 @@ namespace jank::read::lex
           native_vector<result<token, error_ptr>> const tokens(p.begin(), p.end());
           CHECK(tokens
                 == make_tokens({
-                  { 0, 7, token_kind::comment, "(+ 1 1)"sv },
+                  { 0, 9, token_kind::comment, "(+ 1 1)"sv },
           }));
         }
 
@@ -1779,7 +1779,7 @@ namespace jank::read::lex
           native_vector<result<token, error_ptr>> const tokens(p.begin(), p.end());
           CHECK(tokens
                 == make_tokens({
-                  { 0, 5, token_kind::comment, "\"foo\""sv },
+                  { 0, 7, token_kind::comment, "\"foo\""sv },
           }));
         }
 
@@ -1790,7 +1790,7 @@ namespace jank::read::lex
           CHECK(tokens
                 == make_tokens({
                   { 0, 1, token_kind::open_paren },
-                  { 1, 1, token_kind::comment, ")"sv },
+                  { 1, 3, token_kind::comment, ")"sv },
           }));
         }
       }