fix(json): reject unpaired surrogates in string lexing instead of aborting

Fixes #4049.

parse aborted (panic, not ParseError) on strings mixing a raw lone
trailing surrogate with any escape sequence, e.g. the 5-code-unit text
" U+DC00 \n ": lex_string_slow's flush sliced with the checked
ctx.input[start:end], which aborts when the code unit at a slice
boundary is a trailing surrogate. Meanwhile the escape-free fast path
silently *accepted* raw lone surrogates, producing ill-formed strings.

Per the design rule that MoonBit Strings stay Unicode well-formed
(unsafe_to_char is indeed unsafe), the string lexer now rejects every
unpaired surrogate with the documented ParseError (InvalidChar) instead
of either aborting or letting it through:

- raw lone leading/trailing surrogates are rejected on both the fast
  path and the slow path;
- an escaped leading surrogate (\uD8xx) must be immediately followed by
  an escaped trailing surrogate; the pair is combined into one scalar
  value, anything else is rejected. BEHAVIOR CHANGE: "\uD800" alone
  previously parsed into an ill-formed lone-surrogate string and is now
  a parse error;
- well-formed pairs (raw or escaped) parse exactly as before;
- flush now slices with the bounds-check-only view(start_offset~,
  end_offset~), so no abort path remains in string lexing.

Deterministic regression tests in lex_string_test.mbt cover every
rejection shape (raw, escaped, mixed raw/escaped halves) plus the
still-accepted well-formed pairs. Full repo suite green on wasm-gc; the
json suite green on wasm-gc, js, and native.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: bobzhang

json/lex_string.mbt

@@ -17,7 +17,10 @@ fn ParseContext::lex_string(ctx : ParseContext) -> String raise ParseError {
   let string_start = ctx.offset
   // Fast path for ordinary strings: scan raw UTF-16 code units and materialize
   // the slice directly when there are no escapes or control characters.
-  for i in string_start..<ctx.end_offset {
+  for i = string_start {
+    if i >= ctx.end_offset {
+      break
+    }
     let c = ctx.input.unsafe_get(i)
     if c == '"' {
       ctx.offset = i + 1
@@ -29,7 +32,21 @@ fn ParseContext::lex_string(ctx : ParseContext) -> String raise ParseError {
       // \t) are invalid inside a JSON string.
       ctx.offset = i + 1
       ctx.invalid_char(shift=-1)
+    } else if c.is_leading_surrogate() {
+      if i + 1 < ctx.end_offset &&
+        ctx.input.unsafe_get(i + 1).is_trailing_surrogate() {
+        continue i + 2
+      }
+      // MoonBit strings stay Unicode well-formed, so an unpaired surrogate
+      // must be rejected rather than smuggled into the parsed string.
+      ctx.offset = i + 1
+      ctx.invalid_char(shift=-1)
+    } else if c.is_trailing_surrogate() {
+      // A bare trailing surrogate can never start a surrogate pair.
+      ctx.offset = i + 1
+      ctx.invalid_char(shift=-1)
     }
+    continue i + 1
   }
   raise InvalidEof
 }
@@ -39,8 +56,15 @@ fn ParseContext::lex_string_slow(ctx : ParseContext) -> String raise ParseError
   let buf = StringBuilder()
   let mut start = ctx.offset
   fn flush(end : Int) {
-    if start > 0 && end > start {
-      buf.write_view(ctx.input[start:end])
+    if end > start {
+      // `view(start_offset~, end_offset~)` only bounds-checks. The checked
+      // `ctx.input[start:end]` would additionally abort on a trailing
+      // surrogate at a boundary; that cannot happen here because every code
+      // unit in the flushed run has already been validated (unpaired
+      // surrogates raise a ParseError before any flush spans them), but the
+      // unchecked slice keeps this loop's totality independent of that
+      // validation order.
+      buf.write_view(ctx.input.view(start_offset=start, end_offset=end))
     }
   }
 
@@ -63,19 +87,55 @@ fn ParseContext::lex_string_slow(ctx : ParseContext) -> String raise ParseError
           Some('/') => buf.write_char('/')
           Some('u') => {
             let c = ctx.lex_hex_digits(4)
-            buf.write_char(c.unsafe_to_char())
+            if c is (0xD800..=0xDBFF) {
+              // A leading-surrogate escape is only meaningful as the first
+              // half of an escaped surrogate pair; combine it with the
+              // immediately following trailing-surrogate escape into one
+              // Unicode scalar value. Anything else would put an unpaired
+              // surrogate into the result, which MoonBit strings disallow.
+              match ctx.read_char() {
+                Some('\\') => ()
+                Some(_) => ctx.invalid_char(shift=-1)
+                None => raise InvalidEof
+              }
+              match ctx.read_char() {
+                Some('u') => ()
+                Some(_) => ctx.invalid_char(shift=-1)
+                None => raise InvalidEof
+              }
+              let c2 = ctx.lex_hex_digits(4)
+              if c2 is (0xDC00..=0xDFFF) {
+                let combined = (c << 10) + c2 - 0x35fdc00
+                buf.write_char(combined.unsafe_to_char())
+              } else {
+                ctx.invalid_char(shift=-1)
+              }
+            } else if c is (0xDC00..=0xDFFF) {
+              // A bare trailing-surrogate escape can never form a scalar
+              // value.
+              ctx.invalid_char(shift=-1)
+            } else {
+              buf.write_char(c.unsafe_to_char())
+            }
           }
           Some(_) => ctx.invalid_char(shift=-1)
           None => raise InvalidEof
         }
         start = ctx.offset
       }
-      Some(ch) =>
-        if ch.to_int() < 32 {
+      Some(ch) => {
+        let code = ch.to_int()
+        if code < 32 {
+          ctx.invalid_char(shift=-1)
+        } else if code is (0xD800..=0xDFFF) {
+          // `read_char` only yields a surrogate-range value when the code
+          // unit is unpaired; keep parsed strings Unicode well-formed by
+          // rejecting it.
           ctx.invalid_char(shift=-1)
         } else {
           continue
         }
+      }
       None => raise InvalidEof
     }
   }

json/lex_string_test.mbt

@@ -89,3 +89,52 @@ test "lex_hex_digits accepts all hex digit ranges" {
     ),
   )
 }
+
+///|
+/// Regression for #4049. MoonBit strings must stay Unicode well-formed, so
+/// the string lexer rejects every unpaired surrogate with a ParseError.
+/// Previously the escape-free fast path *accepted* raw lone surrogates
+/// (producing ill-formed strings), and combining one with an escape sequence
+/// aborted the process (checked slicing in the slow path's `flush` hit a
+/// surrogate at a slice boundary).
+test "unpaired surrogates are rejected with a clean parse error" {
+  let lone_high = String::from_array([(0xD800).unsafe_to_char()])
+  let lone_low = String::from_array([(0xDC00).unsafe_to_char()])
+  // Raw lone surrogates, escape-free (fast path; used to be accepted).
+  assert_false(@json.valid("\"" + lone_high + "\""))
+  assert_false(@json.valid("\"" + lone_low + "\""))
+  assert_false(@json.valid("\"" + lone_low + lone_high + "\""))
+  // Raw lone surrogates next to escapes (slow path; used to abort).
+  assert_false(@json.valid("\"" + lone_low + "\\n\""))
+  assert_false(@json.valid("\"\\n" + lone_low + "\""))
+  assert_false(@json.valid("\"" + lone_high + "\\t\""))
+  // Escaped lone surrogates (used to produce ill-formed strings).
+  assert_false(@json.valid("\"\\uD800\""))
+  assert_false(@json.valid("\"\\uDC00\""))
+  assert_false(@json.valid("\"\\uD800\\uD800\""))
+  assert_false(@json.valid("\"\\uD800x\""))
+  // Mixed raw/escaped halves do not pair up.
+  assert_false(@json.valid("\"\\uD800" + lone_low + "\""))
+  assert_false(@json.valid("\"" + lone_high + "\\uDC00\""))
+  // The failure is the documented ParseError, not an abort.
+  debug_inspect(
+    expect_parse_error("\"\\uDC00\"", "expected InvalidChar"),
+    content=(
+      #|InvalidChar({ line: 1, column: 6 }, '0')
+    ),
+  )
+}
+
+///|
+/// Well-formed surrogate pairs still parse, raw or escaped (including a pair
+/// split across two `\uXXXX` escapes), and other strings are unaffected.
+test "well-formed surrogate pairs still parse" {
+  assert_true(@json.parse("\"\\uD83D\\uDE00\"") == Json::string("\u{1F600}"))
+  assert_true(@json.parse("\"\u{1F600}\"") == Json::string("\u{1F600}"))
+  assert_true(
+    @json.parse("\"a\\uD83D\\uDE00b\\n\"") == Json::string("a\u{1F600}b\n"),
+  )
+  // Roundtrip through stringify for an astral character next to an escape.
+  let json = Json::string("\u{10FFFF}\\\u{1F600}")
+  assert_true(@json.parse(json.stringify()) == json)
+}