Add an option to dump the tokens from the tokenizer

thradams · Feb 21, 2024 · 036ebce · 036ebce · thradams · Feb 21, 2024
1 parent e64ae41
commit 036ebce
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 2 deletions.
diff --git a/src/lib.c b/src/lib.c
@@ -772,6 +772,11 @@ struct options
     */
     bool visual_studio_ouput_format;
 
+    /*
+      -dump-tokens
+    */
+    bool dump_tokens;
+
     /*
       -o filename
       defines the ouputfile when 1 file is used
@@ -9826,6 +9831,12 @@ int fill_options(struct options* options,
             continue;
         }
 
+        if (strcmp(argv[i], "-dump-tokens") == 0)
+        {
+            options->dump_tokens = true;
+            continue;
+        }
+
         printf("unknown option '%s'", argv[i]);
         return 1;
     }
@@ -28226,6 +28237,11 @@ int compile_one_file(const char* file_name,
 
 		tokens = tokenizer(&tctx, content, file_name, 0, TK_FLAG_NONE);
 
+		if (options->dump_tokens)
+		{
+                    print_tokens(tokens.head);
+                }
+
 		ast.token_list = preprocessor(&prectx, &tokens, 0);
 		if (prectx.n_errors > 0) throw;
 
@@ -28435,7 +28451,7 @@ int compile(int argc, const char** argv, struct report* report)
 		if (argv[i][0] == '-')
 			continue;
 		no_files++;
-		char output_file[400] = { 0 };
+		char output_file[MYMAX_PATH] = { 0 };
 
 		if (!options.no_output)
 		{

diff --git a/src/options.c b/src/options.c
@@ -301,6 +301,12 @@ int fill_options(struct options* options,
             continue;
         }
 
+        if (strcmp(argv[i], "-dump-tokens") == 0)
+        {
+            options->dump_tokens = true;
+            continue;
+        }
+
         printf("unknown option '%s'", argv[i]);
         return 1;
     }

diff --git a/src/options.h b/src/options.h
@@ -225,6 +225,11 @@ struct options
     */
     bool visual_studio_ouput_format;
 
+    /*
+      -dump-tokens
+    */
+    bool dump_tokens;
+
     /*
       -o filename
       defines the ouputfile when 1 file is used

diff --git a/src/parser.c b/src/parser.c
@@ -6734,6 +6734,11 @@ int compile_one_file(const char* file_name,
 
 		tokens = tokenizer(&tctx, content, file_name, 0, TK_FLAG_NONE);
 
+		if (options->dump_tokens)
+		{
+                    print_tokens(tokens.head);
+                }
+
 		ast.token_list = preprocessor(&prectx, &tokens, 0);
 		if (prectx.n_errors > 0) throw;
 
@@ -6943,7 +6948,7 @@ int compile(int argc, const char** argv, struct report* report)
 		if (argv[i][0] == '-')
 			continue;
 		no_files++;
-		char output_file[400] = { 0 };
+		char output_file[MYMAX_PATH] = { 0 };
 
 		if (!options.no_output)
 		{