#7130 WIP - not fully stable yet

Author: tobiu

.github/epic-string-based-templates.md

@@ -182,4 +182,40 @@ Ensured that falsy values (e.g., `false`, `null`, `undefined`) in template inter
 **Description:**
 To improve the developer experience for those familiar with React, a major refactoring was undertaken. The framework's core `render()` method was renamed to `initVnode()` to more accurately reflect its purpose of creating the initial VNode and mounting the component. This freed up the `render` name, allowing `createTemplateVdom()` to be renamed to `render()`, providing a more intuitive and familiar API for functional components using HTML templates. This change also included renaming the `rendered` property to `vnodeInitialized`, the `autoRender` config to `autoInitVnode`, and the `rendering` flag to `isVnodeInitializing` to maintain semantic consistency throughout the framework.
 
+### 15. Create a Robust VDOM-to-String Serializer
+
+**Status: To Do**
+
+**Description:**
+The `JSON.stringify` + regex method for generating the VDOM string during the build process is flawed. It incorrectly handles object keys that are not valid JavaScript identifiers (e.g., `data-foo`) and produces non-idiomatic code (quoted keys). A dedicated serializer is required for correctness and code quality.
+
+**Implementation Details:**
+- **Tool:** A new, custom utility module.
+- **Location:** `buildScripts/util/vdomToString.mjs`.
+- **Method:**
+    1. The utility will export a `vdomToString(vdom)` function that recursively traverses the VDOM object.
+    2. It will check if each object key is a valid JavaScript identifier.
+    3. Valid identifiers will be written to the output string without quotes (e.g., `tag:`).
+    4. Invalid identifiers (e.g., `data-foo`) will be correctly wrapped in single quotes (e.g., `'data-foo':`).
+    5. It will handle the build-time placeholders for runtime expressions, outputting them as raw, unquoted code.
+    6. This new utility will completely replace the `JSON.stringify` and subsequent regex calls in the build scripts.
+
+### 16. Refactor Build-Time Parser to be AST-Based for Robustness
+
+**Status: To Do**
+
+**Description:**
+The current build-time approach, which uses regular expressions to find and replace templates, has proven to be brittle and incorrect. It cannot handle nested `html` templates and can be easily fooled by JSDoc comments or strings that happen to contain the `html`` sequence, leading to build failures. To ensure correctness and consistency with the runtime environment, the build process must be refactored to use a proper JavaScript parser.
+
+**Implementation Details:**
+- **Tools:** `acorn` (to parse JS into an Abstract Syntax Tree) and `astring` (to generate JS code from the AST).
+- **Method:**
+    1. In the build script, for each `.mjs` file, use `acorn` to parse the entire file content into an AST.
+    2. Traverse the AST, specifically looking for `TaggedTemplateExpression` nodes where the `tag` is an `Identifier` with the name `html`.
+    3. Process these template nodes recursively (post-order traversal) to correctly handle nested templates from the inside out.
+    4. The logic from `HtmlTemplateProcessorLogic` will be used to convert the template into its VDOM object representation.
+    5. The original `TaggedTemplateExpression` node in the AST will be replaced with a new AST node representing the generated VDOM object (using an object-to-AST converter).
+    6. Finally, use `astring` to generate the final, correct JavaScript code from the modified AST.
+    7. This new, robust process will replace the fragile regex-based `replace` loop.
+
 

buildScripts/buildESModules.mjs

@@ -1,12 +1,64 @@
+import {generate}           from 'astring';
 import fs                   from 'fs-extra';
 import path                 from 'path';
-import {minify as minifyJs} from 'terser';
+import * as acorn           from 'acorn';
+import * as Terser          from 'terser';
 import {minifyHtml}         from './util/minifyHtml.mjs';
+import { processHtmlTemplateLiteral } from './util/templateBuildProcessor.mjs';
+import { vdomToString }     from './util/vdomToString.mjs';
+
+// A simple JSON to AST converter
+function jsonToAst(json) {
+    if (json === null) {
+        return { type: 'Literal', value: null };
+    }
+    switch (typeof json) {
+        case 'string':
+            const exprMatch = json.match(/##__NEO_EXPR__(.*)##__NEO_EXPR__##/);
+            if (exprMatch) {
+                // This is a raw expression, parse it as a standalone expression
+                try {
+                    const body = acorn.parse(exprMatch[1], {ecmaVersion: 'latest'}).body;
+                    if (body.length > 0 && body[0].type === 'ExpressionStatement') {
+                        return body[0].expression;
+                    }
+                } catch (e) {
+                    console.error(`Failed to parse expression: ${exprMatch[1]}`, e);
+                    // Fallback to literal string if parsing fails
+                    return { type: 'Literal', value: json };
+                }
+            }
+            return { type: 'Literal', value: json };
+        case 'number':
+        case 'boolean':
+            return { type: 'Literal', value: json };
+        case 'object':
+            if (Array.isArray(json)) {
+                return {
+                    type: 'ArrayExpression',
+                    elements: json.map(jsonToAst)
+                };
+            }
+            const properties = Object.entries(json).map(([key, value]) => {
+                const keyNode = /^[a-zA-Z_$][a-zA-Z0-9_$]*$/.test(key)
+                    ? { type: 'Identifier', name: key }
+                    : { type: 'Literal', value: key };
+                return {
+                    type: 'Property',
+                    key: keyNode,
+                    value: jsonToAst(value),
+                    kind: 'init',
+                    computed: keyNode.type === 'Literal'
+                };
+            });
+            return { type: 'ObjectExpression', properties };
+        default:
+            return { type: 'Literal', value: null }; // for undefined, function, etc.
+    }
+}
 
 const
     outputBasePath   = 'dist/esm/',
-    // Regex to find import statements with 'node_modules' in the path
-    // It captures the entire import statement (excluding the leading 'import') and the path itself.
     regexImport      = /(import(?:\s*(?:[\w*{}\n\r\t, ]+from\s*)?|\s*\(\s*)?)(["'`])((?:(?!\2).)*node_modules(?:(?!\2).)*)\2/g,
     root             = path.resolve(),
     requireJson      = path => JSON.parse(fs.readFileSync(path, 'utf-8')),
@@ -22,131 +74,129 @@ if (insideNeo) {
     inputDirectories = ['apps', 'docs', 'node_modules/neo.mjs/src', 'src']
 }
 
-/**
- * @param {String} match
- * @param {String} p1 will be "import {marked}    from " (or similar, including the 'import' keyword and everything up to the first quote)
- * @param {String} p2 will be the quote character (', ", or `)
- * @param {String} p3 will be the original path string (e.g., '../../../../node_modules/marked/lib/marked.esm.js')
- * @returns {String}
- */
 function adjustImportPathHandler(match, p1, p2, p3) {
     let newPath;
-
     if (p3.includes('/node_modules/neo.mjs/')) {
         newPath = p3.replace('/node_modules/neo.mjs/', '/')
     } else {
-        newPath = '../../' + p3; // Prepend 2 levels up
+        newPath = '../../' + p3;
     }
-
-    // Reconstruct the import statement with the new path
     return p1 + p2 + newPath + p2
 }
 
-/**
- *
- * @param {String} inputDir
- * @param {String} outputDir
- * @returns {Promise<void>}
- */
 async function minifyDirectory(inputDir, outputDir) {
     if (fs.existsSync(inputDir)) {
         fs.mkdirSync(outputDir, {recursive: true});
-
         const dirents = fs.readdirSync(inputDir, {recursive: true, withFileTypes: true});
-
         for (const dirent of dirents) {
-            // Intended to skip the docs/output folder, since the content is already minified
             if (dirent.path.includes('/docs/output/')) {
                 continue
             }
-
             if (dirent.isFile()) {
                 const
                     inputPath    = path.join(dirent.path, dirent.name),
                     relativePath = path.relative(inputDir, inputPath),
                     outputPath   = path.join(outputDir, relativePath),
                     content      = fs.readFileSync(inputPath, 'utf8');
-
                 await minifyFile(content, outputPath)
-            }
-            // Copy resources folders
-            else if (dirent.name === 'resources') {
+            } else if (dirent.name === 'resources') {
                 const
                     inputPath    = path.join(dirent.path, dirent.name),
                     relativePath = path.relative(inputDir, inputPath),
                     outputPath   = path.join(outputDir, relativePath);
-
                 fs.mkdirSync(path.dirname(outputPath), {recursive: true});
-
                 fs.copySync(inputPath, outputPath);
-
-                // Minify all JSON files inside the copied folder
                 const resourcesEntries = fs.readdirSync(outputPath, {recursive: true, withFileTypes: true});
-
                 for (const resource of resourcesEntries) {
-                    if (resource.isFile()) {
-                        if (resource.name.endsWith('.json')) {
-                            const
-                                resourcePath = path.join(resource.path, resource.name),
-                                content      = fs.readFileSync(resourcePath, 'utf8');
-
-                            fs.writeFileSync(resourcePath, JSON.stringify(JSON.parse(content)))
-                        }
+                    if (resource.isFile() && resource.name.endsWith('.json')) {
+                        const
+                            resourcePath = path.join(resource.path, resource.name),
+                            content      = fs.readFileSync(resourcePath, 'utf8');
+                        fs.writeFileSync(resourcePath, JSON.stringify(JSON.parse(content)))
                     }
                 }
             }
         }
     }
 }
 
-/**
- * @param {String} content
- * @param {String} outputPath
- * @returns {Promise<void>}
- */
 async function minifyFile(content, outputPath) {
     fs.mkdirSync(path.dirname(outputPath), {recursive: true});
-
     try {
-        // Minify JSON files
         if (outputPath.endsWith('.json')) {
             const jsonContent = JSON.parse(content);
-
             if (outputPath.endsWith('neo-config.json')) {
                 Object.assign(jsonContent, {
                     basePath      : '../../' + jsonContent.basePath,
                     environment   : 'dist/esm',
                     mainPath      : './Main.mjs',
                     workerBasePath: jsonContent.basePath + 'src/worker/'
                 });
-
                 if (!insideNeo) {
                     jsonContent.appPath = jsonContent.appPath.substring(6)
                 }
             }
-
             fs.writeFileSync(outputPath, JSON.stringify(jsonContent));
             console.log(`Minified JSON: ${outputPath}`)
-        }
-        // Minify HTML files
-        else if (outputPath.endsWith('.html')) {
+        } else if (outputPath.endsWith('.html')) {
             const minifiedContent = await minifyHtml(content);
-
             fs.writeFileSync(outputPath, minifiedContent);
             console.log(`Minified HTML: ${outputPath}`)
-        }
-        // Minify JS files
-        else if (outputPath.endsWith('.mjs')) {
+        } else if (outputPath.endsWith('.mjs')) {
             let adjustedContent = content.replace(regexImport, adjustImportPathHandler);
 
-            const result = await minifyJs(adjustedContent, {
-                module: true,
-                compress: {
-                    dead_code: true
-                },
-                mangle: {
-                    toplevel: true
+            // AST-based processing for html templates
+            const ast = acorn.parse(adjustedContent, {ecmaVersion: 'latest', sourceType: 'module'});
+
+            // Simple post-order traversal
+            const nodesToProcess = [];
+            function walk(node, parent, key, index) {
+                if (!node) return;
+                Object.entries(node).forEach(([key, value]) => {
+                    if (Array.isArray(value)) {
+                        value.forEach((child, i) => walk(child, node, key, i));
+                    } else if (typeof value === 'object' && value !== null) {
+                        walk(value, node, key);
+                    }
+                });
+                nodesToProcess.push({node, parent, key, index});
+            }
+            walk(ast);
+
+            let hasChanges = false;
+            for (const {node, parent, key, index} of nodesToProcess) {
+                if (node.type === 'TaggedTemplateExpression' && node.tag.type === 'Identifier' && node.tag.name === 'html') {
+                    const templateLiteral = node.quasi;
+                    const strings = templateLiteral.quasis.map(q => q.value.cooked);
+                    const expressionCodeStrings = templateLiteral.expressions.map(expr => adjustedContent.substring(expr.start, expr.end));
+                    
+                    const componentNameMap = {};
+                    expressionCodeStrings.forEach(exprCode => {
+                        if (/^[A-Z][a-zA-Z0-9]*$/.test(exprCode.trim())) {
+                            componentNameMap[exprCode.trim()] = { __neo_component_name__: exprCode.trim() };
+                        }
+                    });
+
+                    const vdom = await processHtmlTemplateLiteral(strings, expressionCodeStrings, componentNameMap);
+                    const vdomAst = jsonToAst(vdom);
+
+                    if (parent) {
+                        if (index !== undefined) {
+                            parent[key][index] = vdomAst;
+                        } else {
+                            parent[key] = vdomAst;
+                        }
+                        hasChanges = true;
+                    }
                 }
+            }
+
+            let currentContent = hasChanges ? generate(ast) : adjustedContent;
+
+            const result = await Terser.minify(currentContent, {
+                module: true,
+                compress: { dead_code: true },
+                mangle: { toplevel: true }
             });
 
             fs.writeFileSync(outputPath, result.code);
@@ -161,27 +211,22 @@ const
     swContent = fs.readFileSync(path.resolve(root, 'ServiceWorker.mjs'), 'utf8'),
     promises  = [minifyFile(swContent, path.resolve(root, outputBasePath, 'ServiceWorker.mjs'))];
 
-// Execute the minification
 inputDirectories.forEach(folder => {
     const outputPath = path.resolve(root, outputBasePath, folder.replace('node_modules/neo.mjs/', ''));
-
     promises.push(minifyDirectory(path.resolve(root, folder), outputPath)
         .catch(err => {
             console.error('dist/esm Minification failed:', err);
-            process.exit(1) // Exit with error code
+            process.exit(1)
         })
     )
 });
 
 Promise.all(promises).then(() => {
-    // Copying the already skipped and minified docs/output folder
     const docsOutputPath = path.resolve(root, 'docs/output');
-
     if (fs.existsSync(docsOutputPath)) {
         fs.copySync(docsOutputPath, path.resolve(root, outputBasePath, 'docs/output'))
     }
-
     const processTime = (Math.round((new Date - startDate) * 100) / 100000).toFixed(2);
     console.log(`\nTotal time for dist/esm: ${processTime}s`);
     process.exit()
-})
+})