add comments support

ony · Jun 29, 2016 · 2979e06 · 2979e06
1 parent 9e73b2a
commit 2979e06
Show file tree

Hide file tree

Showing 5 changed files with 215 additions and 24 deletions.
diff --git a/inc/pjson.h b/inc/pjson.h
@@ -39,7 +39,7 @@ typedef struct {
     const char *chunk;
     const char *chunk_end;
 
-    int state;
+    int state, state0; /* current and saved state */
     const char *ptr; /* current position withing chunk */
 
     union {

diff --git a/src/pjson_general.h b/src/pjson_general.h
@@ -97,8 +97,9 @@ static bool pj_poll_tok(pj_parser_ref parser, pj_token *token)
         switch (*p)
         {
         case '\t': case '\n': case '\r': case ' ':
-            parser->ptr = ++p;
-            return pj_space(parser, token, S_INIT);
+            return pj_space(parser, token, p+1, S_INIT);
+        case '/':
+            return pj_comment_start(parser, token, p+1, S_INIT);
         case 'n':
             parser->state = S_N;
             parser->ptr = ++p;
@@ -146,8 +147,9 @@ static bool pj_poll_tok(pj_parser_ref parser, pj_token *token)
         switch (*p)
         {
         case '\t': case '\n': case '\r': case ' ':
-            parser->ptr = ++p;
-            return pj_space(parser, token, S_COMMA);
+            return pj_space(parser, token, p+1, S_COMMA);
+        case '/':
+            return pj_comment_start(parser, token, p+1, S_COMMA);
         case 'n':
             parser->state = S_N;
             parser->ptr = ++p;
@@ -211,8 +213,9 @@ static bool pj_poll_tok(pj_parser_ref parser, pj_token *token)
         switch (*p)
         {
         case '\t': case '\n': case '\r': case ' ':
-            parser->ptr = ++p;
-            return pj_space(parser, token, s);
+            return pj_space(parser, token, p+1, s);
+        case '/':
+            return pj_comment_start(parser, token, p+1, s);
 
         case ',':
             parser->ptr = ++p;
@@ -241,6 +244,15 @@ static bool pj_poll_tok(pj_parser_ref parser, pj_token *token)
             return false;
         }
 
+    case S_COMMENT_START:
+        return pj_comment_start(parser, token, p, parser->state0);
+    case S_COMMENT_LINE:
+        return pj_comment_line(parser, token, p, parser->state0);
+    case S_COMMENT_REGION:
+        return pj_comment_region(parser, token, p, parser->state0);
+    case S_COMMENT_END:
+        return pj_comment_end(parser, token, p, parser->state0);
+
     default:
         assert(!"invalid state"); /* improperly initialized parser? */
         abort();

diff --git a/src/pjson_space.h b/src/pjson_space.h
@@ -23,33 +23,160 @@
 #include "pjson.h"
 #include "pjson_general.h"
 
-static bool pj_isspace(char c)
+static bool pj_comment_line(pj_parser_ref parser, pj_token *token, const char *p, state s)
 {
-    switch (c)
+    TRACE_FUNC();
+    const char * const p_end = parser->chunk_end;
+
+    for (;;)
     {
-    case '\t': case '\n': case '\r': case ' ':
-        return true;
-    default:
-        return false;
+        if (p == p_end)
+        {
+            parser->ptr = p;
+            parser->chunk = p;
+            parser->state = S_COMMENT_LINE;
+            parser->state0 = s;
+            token->token_type = PJ_STARVING;
+            return false;
+        }
+
+        switch (*p)
+        {
+        case '\n':
+            parser->ptr = p+1;
+            parser->chunk = p+1;
+            parser->state = s;
+            return pj_poll_tok(parser, token);
+        default:
+            ++p;
+        }
     }
 }
 
-static bool pj_space(pj_parser_ref parser, pj_token *token, state s)
+static bool pj_comment_end(pj_parser_ref parser, pj_token *token, const char *p, state s);
+
+static bool pj_comment_region(pj_parser_ref parser, pj_token *token, const char *p, state s)
 {
-    const char *p = parser->ptr;
+    TRACE_FUNC();
     const char * const p_end = parser->chunk_end;
-    for (; p != p_end && pj_isspace(*p); ++p);
-    parser->ptr = p;
-    parser->chunk = p;
-    parser->state = s;
-    if (p == p_end)
+
+    for (;;)
     {
-        token->token_type = PJ_STARVING;
-        return false;
+        if (p == p_end)
+        {
+            parser->ptr = p;
+            parser->chunk = p;
+            parser->state = S_COMMENT_REGION;
+            parser->state0 = s;
+            token->token_type = PJ_STARVING;
+            return false;
+        }
+
+        switch (*p)
+        {
+        case '*':
+            return pj_comment_end(parser, token, p+1, s);
+
+        default:
+            ++p;
+        }
     }
-    else
+}
+
+static bool pj_comment_end(pj_parser_ref parser, pj_token *token, const char *p, state s)
+{
+    TRACE_FUNC();
+    const char * const p_end = parser->chunk_end;
+
+    for (;;)
     {
-        return pj_poll_tok(parser, token);
+        if (p == p_end)
+        {
+            parser->ptr = p;
+            parser->chunk = p;
+            parser->state = S_COMMENT_END;
+            parser->state0 = s;
+            token->token_type = PJ_STARVING;
+            return false;
+        }
+
+        switch (*p)
+        {
+        case '*':
+            ++p;
+            break;
+        case '/':
+            parser->ptr = p+1;
+            parser->chunk = p+1;
+            parser->state = s;
+            return pj_poll_tok(parser, token);
+
+        default:
+            return pj_comment_region(parser, token, p+1, s);
+        }
+    }
+}
+
+static bool pj_comment_start(pj_parser_ref parser, pj_token *token, const char *p, state s)
+{
+    TRACE_FUNC();
+    const char * const p_end = parser->chunk_end;
+
+    for (;;)
+    {
+        if (p == p_end)
+        {
+            parser->ptr = p;
+            parser->chunk = p;
+            parser->state = S_COMMENT_START;
+            parser->state0 = s;
+            token->token_type = PJ_STARVING;
+            return false;
+        }
+
+        switch (*p)
+        {
+        case '*':
+            return pj_comment_region(parser, token, p+1, s);
+        case '/':
+            return pj_comment_line(parser, token, p+1, s);
+
+        default:
+            pj_err_tok(parser, token);
+            return false;
+        }
+    }
+}
+
+static bool pj_space(pj_parser_ref parser, pj_token *token, const char *p, state s)
+{
+    TRACE_FUNC();
+    const char * const p_end = parser->chunk_end;
+
+    for (;;)
+    {
+        if (p == p_end)
+        {
+            parser->ptr = p;
+            parser->chunk = p;
+            parser->state = s;
+            token->token_type = PJ_STARVING;
+            return false;
+        }
+
+        switch (*p)
+        {
+        case '\t': case '\n': case '\r': case ' ':
+            ++p;
+            break;
+        case '/':
+            return pj_comment_start(parser, token, p+1, s);
+        default:
+            parser->ptr = p;
+            parser->chunk = p;
+            parser->state = s;
+            return pj_poll_tok(parser, token);
+        }
     }
 }
 

diff --git a/src/pjson_state.h b/src/pjson_state.h
@@ -44,6 +44,9 @@ typedef enum {
     S_UNICODE, S_UNICODE_FINISH = S_UNICODE + 4, /* 4 hex digits */
     S_UNICODE_ESC, /* handle surrogate pairs */
     S_STR_VALUE, /* str may end up as key */
+
+    /* other stuff */
+    S_COMMENT_START, S_COMMENT_LINE, S_COMMENT_REGION, S_COMMENT_END,
 } state;
 
 #define F_BUF 0x100

diff --git a/test/simple.cpp b/test/simple.cpp
@@ -139,3 +139,52 @@ TEST(simple, unicode_str_in_array)
     pj_poll(&parser, tokens.data(), tokens.size());
     EXPECT_EQ( PJ_END, tokens[0].token_type );
 }
+
+TEST(simple, comments)
+{
+    pj_parser parser;
+    array<char, 256> buf;
+    pj_init(&parser, buf.data(), buf.size());
+
+    pj_feed(&parser, "[// comment\n\"// Hello /* comment */\" /* comment */]");
+
+    array<pj_token, 4> tokens;
+
+    pj_poll(&parser, tokens.data(), tokens.size());
+    EXPECT_EQ( PJ_TOK_ARR, tokens[0].token_type );
+    ASSERT_EQ( PJ_TOK_STR, tokens[1].token_type );
+    EXPECT_EQ( std::string("// Hello /* comment */"), std::string(tokens[1].str, tokens[1].len) );
+    EXPECT_EQ( PJ_TOK_ARR_E, tokens[2].token_type );
+    EXPECT_EQ( PJ_STARVING, tokens[3].token_type );
+    pj_feed_end(&parser);
+    pj_poll(&parser, tokens.data(), tokens.size());
+    EXPECT_EQ( PJ_END, tokens[0].token_type );
+}
+
+TEST(simple, comments_chunked)
+{
+    pj_parser parser;
+    array<char, 256> buf;
+    pj_init(&parser, buf.data(), buf.size());
+
+    pj_feed(&parser, "[/");
+
+    array<pj_token, 4> tokens;
+
+    pj_poll(&parser, tokens.data(), tokens.size());
+    EXPECT_EQ( PJ_TOK_ARR, tokens[0].token_type );
+    ASSERT_EQ( PJ_STARVING, tokens[1].token_type );
+    pj_feed(&parser, "/ 4,\n5,\n/");
+    pj_poll(&parser, tokens.data(), tokens.size());
+    ASSERT_EQ( PJ_TOK_NUM, tokens[0].token_type );
+    EXPECT_EQ( "5", std::string(tokens[0].str, tokens[0].len) );
+    ASSERT_EQ( PJ_STARVING, tokens[1].token_type );
+    pj_feed(&parser, "* 6, *");
+    pj_poll(&parser, tokens.data(), tokens.size());
+    ASSERT_EQ( PJ_STARVING, tokens[0].token_type );
+    pj_feed(&parser, "/ 7 ");
+    pj_poll(&parser, tokens.data(), tokens.size());
+    ASSERT_EQ( PJ_TOK_NUM, tokens[0].token_type );
+    EXPECT_EQ( "7", std::string(tokens[0].str, tokens[0].len) );
+    EXPECT_EQ( PJ_STARVING, tokens[1].token_type );
+}