ondemand: able to iterate large json files

- fixed typo bug in
  ValueIterator.get_type() which was
  causing mysterious problems
  beyond just number parsing
- created recursive_iterate_json() in
  debug_main.zig (not in repo) which
  is getting through twitter.json and a
  large package-lock.json file
- add build flag -Dondemand which
  chooses the ondemand parser for
  validation
- ondemand validation matches dom
  validation except for 3 big integer
  values

Author: travisstaloch

.vscode/launch.json

@@ -4,37 +4,18 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
-        // {
-        //     "name": "(gdb) Launch",
-        //     "type": "cppdbg",
-        //     "request": "launch",
-        //     "program": "${workspaceFolder}/zig-out/bin/zimdjson",
-        //     "args": ["test/test.json"],
-        //     "stopAtEntry": true,
-        //     "cwd": "${workspaceFolder}",
-        //     "environment": [],
-        //     "externalConsole": false,
-        //     "MIMode": "gdb",
-        //     "setupCommands": [
-        //         {
-        //             "description": "Enable pretty-printing for gdb",
-        //             "text": "-enable-pretty-printing",
-        //             "ignoreFailures": true
-        //         }
-        //     ]
-        // },
         {
             "type": "lldb",
             "request": "launch",
             "name": "Debug",
-            "program": "${workspaceFolder}/zig-out/bin/zimdjson",
-            // "args": ["test/test2.json"],
+            "program": "${workspaceFolder}/zig-out/bin/dbg",
             "args": [
-                // "../JSONTestSuite/test_parsing/y_string_1_2_3_bytes_UTF-8_sequences.json",
-                "../JSONTestSuite/test_parsing//y_string_utf8.json",
+                // "../JSONTestSuite/test_parsing/i_structure_500_nested_arrays.json",
+                "../JSONTestSuite/test_parsing/y_array_arraysWithSpaces.json",
+                // "../../c/simdjson/jsonexamples/twitter.json",
             ],
             "cwd": "${workspaceFolder}",
-            "preLaunchTask": "zig build",
+            "preLaunchTask": "zig build debug",
         }
     ]
 }

README.md

@@ -5,9 +5,9 @@ This is a port of [simdjson](https://github.com/simdjson/simdjson), a high perfo
 
 
 # requirements
-A CPU with both AVX2 and CLMUL is required (Haswell from 2013 onwards should do for Intel, for AMD a Ryzen/EPYC CPU (Q1 2017) should be sufficient).  Macos is (not yet) supported.
+A CPU with both AVX2 and CLMUL is required (Haswell from 2013 onwards should do for Intel, for AMD a Ryzen/EPYC CPU (Q1 2017) should be sufficient).  
 
-No fallback for unsupported CPUs is provided.
+No fallback for unsupported CPUs is provided (yet).
 
 # usage
 ```console

build.zig

@@ -1,11 +1,12 @@
 const std = @import("std");
 
-fn setup(step: *std.build.LibExeObjStep, mode: std.builtin.Mode, target: anytype, step_size: u8) void {
+fn setup(step: *std.build.LibExeObjStep, mode: std.builtin.Mode, target: anytype, step_size: u8, ondemand: bool) void {
     step.addCSourceFile("src/utils.c", &[_][]const u8{ "-Wall", "-Wextra", "-Werror", "-O3" });
     step.setTarget(target);
     step.linkLibC();
     step.setBuildMode(mode);
     step.addBuildOption(u8, "step_size", step_size);
+    step.addBuildOption(bool, "ondemand", ondemand);
 }
 
 pub fn build(b: *std.build.Builder) void {
@@ -20,19 +21,25 @@ pub fn build(b: *std.build.Builder) void {
         "how many bytes of input to read at per StructuralIndexer step.  must be either 64 or 128",
     ) orelse 64;
 
+    const ondemand = b.option(
+        bool,
+        "ondemand",
+        "use the ondemand parser for validation",
+    ) orelse false;
+
     const lib = b.addStaticLibrary("simdjzon", "src/simdjzon.zig");
-    setup(lib, mode, target, step_size);
+    setup(lib, mode, target, step_size, ondemand);
     lib.install();
 
     var main_tests = b.addTest("src/tests.zig");
-    setup(main_tests, mode, target, step_size);
-    // main_tests.setFilter("ondemand");
+    setup(main_tests, mode, target, step_size, ondemand);
+    // main_tests.setFilter("ondemand array iteration");
 
     const test_step = b.step("test", "Run tests");
     test_step.dependOn(&main_tests.step);
 
     const exe = b.addExecutable("simdjzon", "src/main.zig");
-    setup(exe, mode, target, step_size);
+    setup(exe, mode, target, step_size, ondemand);
     exe.install();
 
     const run_cmd = exe.run();

src/Logger.zig

@@ -1,15 +1,16 @@
 const std = @import("std");
 const mem = std.mem;
-const main = @import("main.zig");
+const common = @import("common.zig");
 const Ondemand = @import("ondemand.zig");
-const println = main.println;
-const print = main.print;
+const println = common.println;
+const print = common.print;
 usingnamespace @import("llvm_intrinsics.zig");
 
 depth: u8 = 0,
 
 const Logger = @This();
 
+pub const MAX_DEPTH = 30;
 const LOG_EVENT_LEN = 20;
 const LOG_BUFFER_LEN = 30;
 const LOG_SMALL_BUFFER_LEN = 10;
@@ -22,15 +23,15 @@ fn pad_with(comptime s: []const u8, comptime pad_byte: u8, comptime len: u8) [le
 }
 
 fn pad_with_alloc(s: []const u8, pad_byte: u8, len: u8, allocator: *mem.Allocator) []const u8 {
-    var buf = allocator.alloc(u8, len) catch unreachable;
+    var buf = allocator.alloc(u8, len) catch return s;
     std.mem.set(u8, buf, pad_byte);
     std.mem.copy(u8, buf, s[0..std.math.min(s.len, buf.len)]);
     return buf;
 }
 
 pub fn start(log: *Logger, iter: anytype) void {
     _ = iter;
-    if (main.debug) {
+    if (common.debug) {
         log.depth = 0;
         const event_txt = pad_with("Event", ' ', LOG_EVENT_LEN);
         const buffer_txt = pad_with("Buffer", ' ', LOG_BUFFER_LEN);
@@ -51,18 +52,22 @@ fn printable_char(c: u8) u8 {
 
 pub fn line_fmt(log: *Logger, iter: anytype, title_prefix: []const u8, title: []const u8, comptime detail_fmt: []const u8, detail_args: anytype) void {
     var buf: [0x100]u8 = undefined;
-    log.line(iter, title_prefix, title, std.fmt.bufPrint(&buf, detail_fmt, detail_args) catch &buf);
+    log.line(iter, title_prefix, title, std.fmt.bufPrint(&buf, detail_fmt, detail_args) catch return);
 }
 
-// TODO: remove catch unreachables
 pub fn line(log: *Logger, iter: anytype, title_prefix: []const u8, title: []const u8, detail: []const u8) void {
+    if (iter.depth >= Logger.MAX_DEPTH) return;
     var log_buf: [0x100]u8 = undefined;
     var log_buf2: [LOG_BUFFER_LEN]u8 = undefined;
-    if (!main.debug) return;
+    if (!common.debug) return;
 
     var log_fba = std.heap.FixedBufferAllocator.init(&log_buf);
     const depth_padding = pad_with_alloc("", ' ', @intCast(u8, if (log.depth < 0x0f) log.depth * 2 else 0xff), &log_fba.allocator);
-    const titles = std.fmt.allocPrint(&log_fba.allocator, "{s}{s}{s}", .{ depth_padding, title_prefix, title }) catch unreachable;
+    const titles = std.fmt.allocPrint(
+        &log_fba.allocator,
+        "{s}{s}{s}",
+        .{ depth_padding, title_prefix, title },
+    ) catch return;
     const p1 = pad_with_alloc(titles, ' ', LOG_EVENT_LEN, &log_fba.allocator);
     print("| {s} ", .{p1});
     const current_index = if (iter.at_beginning()) null else iter.next_structural() - 1;
@@ -112,7 +117,7 @@ pub fn line(log: *Logger, iter: anytype, title_prefix: []const u8, title: []cons
     if (current_index) |ci| {
         print("| {s} ", .{
             pad_with_alloc(
-                std.fmt.bufPrint(&log_buf2, "{}", .{ci[0]}) catch unreachable,
+                std.fmt.bufPrint(&log_buf2, "{}", .{ci[0]}) catch return,
                 ' ',
                 LOG_INDEX_LEN,
                 &log_fba.allocator,
@@ -135,22 +140,22 @@ pub fn value2(log: *Logger, iter: anytype, typ: []const u8, detail: []const u8,
 
 pub fn start_value(log: *Logger, iter: anytype, typ: []const u8) void {
     log.line(iter, "+", typ, "");
-    if (main.debug) log.depth = sat_add_u8(log.depth, 1);
+    if (common.debug) log.depth = sat_add_u8(log.depth, 1);
 }
 pub fn end_value(log: *Logger, iter: anytype, typ: []const u8) void {
-    if (main.debug) log.depth = sat_sub_u8(log.depth, 1);
+    if (common.debug) log.depth = sat_sub_u8(log.depth, 1);
     log.line(iter, "-", typ, "");
 }
 
 pub fn err(log: *Logger, iter: anytype, err_msg: []const u8) void {
     _ = iter;
     _ = log;
-    if (main.debug) std.log.err("{s}", .{err_msg});
+    if (common.debug) std.log.err("{s}", .{err_msg});
 }
 pub fn err_fmt(log: *Logger, iter: anytype, comptime fmt: []const u8, args: anytype) void {
     _ = iter;
     _ = log;
-    if (main.debug) std.log.err(fmt, args);
+    if (common.debug) std.log.err(fmt, args);
 }
 
 pub fn event(log: *Logger, iter: anytype, typ: []const u8, detail: []const u8, delta: i32, depth_delta: i32) void {

src/main.zig

@@ -2,17 +2,17 @@ const build_options = @import("build_options");
 const std = @import("std");
 
 const os = std.os;
-usingnamespace @import("common.zig");
+const common = @import("common.zig");
 const simdjzon = @import("simdjzon.zig");
 const dom = simdjzon.dom;
+const ondemand = simdjzon.ondemand;
 pub const step_size = build_options.step_size;
 
-pub fn main() !u8 {
+pub fn domMain() !u8 {
     var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
     defer arena.deinit();
     const allocator = &arena.allocator;
     var parser: dom.Parser = undefined;
-    // debug = true;
 
     if (os.argv.len == 1) {
         var stdin = std.io.getStdIn().reader();
@@ -37,3 +37,102 @@ pub fn main() !u8 {
     std.log.debug("parse valid", .{});
     return 0;
 }
+
+pub fn ondemandMain() !u8 {
+    var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+    defer arena.deinit();
+    const allocator = &arena.allocator;
+    var parser: ondemand.Parser = undefined;
+    defer if (parser.src.* == .file) parser.src.file.close();
+
+    if (os.argv.len == 1) {
+        var stdin = std.io.getStdIn().reader();
+        const input = try stdin.readAllAlloc(allocator, std.math.maxInt(u32));
+        var src = std.io.StreamSource{ .buffer = std.io.fixedBufferStream(input) };
+        parser = try ondemand.Parser.init(&src, allocator, "<stdin>", .{});
+    } else if (os.argv.len == 2) {
+        const filepath = std.mem.span(os.argv[1]);
+        const file = try std.fs.cwd().openFile(filepath, .{ .read = true });
+
+        var src = std.io.StreamSource{ .file = file };
+        parser = try ondemand.Parser.init(&src, allocator, filepath, .{});
+    } else {
+        std.log.err("Too many arguments.  Please provide input via filename or stdin", .{});
+        return 1;
+    }
+
+    defer parser.deinit();
+
+    var doc = try parser.iterate();
+    // common.debug = true;
+    var string_buf: [0x1000]u8 = undefined;
+    const end_index = recursive_iterate_json(&doc, 1, parser.parser.max_depth, &string_buf) catch |err| switch (err) {
+        // error.EndOfStream => {},
+        else => {
+            std.log.err("parse failed. {s}", .{@errorName(err)});
+            return 1;
+        },
+    };
+
+    if (end_index != doc.iter.last_document_position()[0]) {
+        std.log.err("More than one JSON value at the root of the document, or extra characters at the end of the JSON!", .{});
+        return 1;
+    }
+    std.log.debug("parse valid", .{});
+    return 0;
+}
+
+inline fn recursive_iterate_json(element: anytype, depth: u16, max_depth: u16, string_buf: []u8) common.Error!u32 {
+    if (depth >= max_depth) return error.DEPTH_ERROR;
+    var iter = switch (@TypeOf(element)) {
+        *ondemand.Document => element.iter,
+        *ondemand.Value => element.iter.iter,
+        else => unreachable,
+    };
+    switch (try element.get_type()) {
+        .array => {
+            var arr = try element.get_array();
+            var it = arr.iterator();
+            while (try it.next()) |*child| {
+                _ = try recursive_iterate_json(child, depth + 1, max_depth, string_buf);
+            }
+            return (it.iter.iter.token.index - 1)[0];
+        },
+        .object => {
+            var obj = try element.get_object();
+            var it = obj.iterator();
+            var key: [0x1000]u8 = undefined;
+            while (try it.next(&key)) |*field| {
+                _ = try recursive_iterate_json(&field.value, depth + 1, max_depth, string_buf);
+            }
+            return (it.iter.iter.token.index - 1)[0];
+        },
+        .number => {
+            // FIXME: clean this up to match dom behavior of big int values
+            // failing to match JSONTestSuite on:
+            //   i_number_too_big_neg_int.json
+            //   i_number_too_big_pos_int.json
+            //   i_number_very_big_negative_int.json
+
+            switch ((try iter.peek_delta(0, 1))[0]) {
+                '-' => _ = element.get_int(i64) catch {
+                    _ = try element.get_double();
+                },
+                else => _ = element.get_int(u64) catch {
+                    _ = try element.get_double();
+                },
+            }
+        },
+        .string => _ = try element.get_string(string_buf),
+        .bool => _ = try element.get_bool(),
+        .nul => if (!(try element.is_null())) return error.INCORRECT_TYPE,
+    }
+    return (iter.token.index - 1)[0];
+}
+
+pub fn main() !u8 {
+    return if (build_options.ondemand)
+        ondemandMain()
+    else
+        domMain();
+}

src/number_parsing.zig

@@ -3,14 +3,14 @@ const mem = std.mem;
 usingnamespace @import("vector_types.zig");
 usingnamespace @import("llvm_intrinsics.zig");
 const main = @import("main.zig");
-const Error = main.Error;
-const println = main.println;
 const CharUtils = @import("string_parsing.zig").CharUtils;
 const dom = @import("dom.zig");
 const Iterator = dom.Iterator;
 const TapeBuilder = dom.TapeBuilder;
 const TapeType = dom.TapeType;
 const common = @import("common.zig");
+const Error = common.Error;
+const println = common.println;
 
 fn INVALID_NUMBER(src: [*]const u8) Error {
     _ = src;
@@ -1829,7 +1829,7 @@ pub fn parse_integer(src: [*]const u8) !u64 {
         } else if (src[0] != '1' or i <= std.math.maxInt(i64)) return error.INCORRECT_TYPE;
     }
 
-    return if (negative) (~i + 1) else i;
+    return if (negative) (~i +% 1) else i;
 }
 
 pub fn parse_double(src_: [*]const u8) !f64 {