TreeFileSystem.ts refactor and add parsersOnly flag for faster compil…

…ation speed
breck7 · May 8, 2024 · 844629f · 844629f
1 parent d59d71a
commit 844629f
Show file tree

Hide file tree

Showing 9 changed files with 146 additions and 105 deletions.
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "jtree",
-  "version": "76.2.0",
+  "version": "77.0.0",
   "description": "Simplify your code with Tree Notation. This jtree package includes a Tree Notation parser, compiler-compiler, and virtual machine for Tree Languages, as well as sample languages, implemented in TypeScript.",
   "types": "./built/jtree.node.d.ts",
   "main": "./products/TreeNode.js",

diff --git a/products/TreeFileSystem.browser.js b/products/TreeFileSystem.browser.js
@@ -1,4 +1,8 @@
 const GRAMMAR_EXTENSION = ".grammar"
+const parserRegex = /^[a-zA-Z0-9_]+Parser/gm
+// A regex to check if a multiline string has a line that starts with "import ".
+const importRegex = /^import /gm
+const importOnlyRegex = /^importOnly/
 class DiskWriter {
   constructor() {
     this.fileCache = {}
@@ -86,22 +90,26 @@ class TreeFileSystem {
     }
     return _treeCache[absoluteFilePath]
   }
-  _doesFileHaveGrammarDefinitions(absoluteFilePath) {
-    if (!absoluteFilePath) return false
-    const { _grammarExpandersCache } = this
-    if (_grammarExpandersCache[absoluteFilePath] === undefined) _grammarExpandersCache[absoluteFilePath] = !!this._storage.read(absoluteFilePath).match(/^[a-zA-Z0-9_]+Parser/gm)
-    return _grammarExpandersCache[absoluteFilePath]
-  }
-  _evaluateImports(absoluteFilePath) {
+  _assembleFile(absoluteFilePath) {
     const { _expandedImportCache } = this
     if (_expandedImportCache[absoluteFilePath]) return _expandedImportCache[absoluteFilePath]
-    // A regex to check if a multiline string has a line that starts with "import ".
-    const importRegex = /^import /gm
-    const code = this.read(absoluteFilePath)
-    if (!code.match(importRegex))
+    let code = this.read(absoluteFilePath)
+    const isImportOnly = importOnlyRegex.test(code)
+    // Strip any parsers
+    const stripIt = code.includes("// parsersOnly") // temporary perf hack
+    if (stripIt)
+      code = code
+        .split("\n")
+        .filter(line => line.startsWith("import "))
+        .join("\n")
+    const filepathsWithParserDefinitions = []
+    if (this._doesFileHaveGrammarDefinitions(absoluteFilePath)) filepathsWithParserDefinitions.push(absoluteFilePath)
+    if (!importRegex.test(code))
       return {
-        code,
-        importFilePaths: []
+        afterImportPass: code,
+        isImportOnly,
+        importFilePaths: [],
+        filepathsWithParserDefinitions
       }
     let importFilePaths = []
     const lines = code.split("\n")
@@ -111,22 +119,31 @@ class TreeFileSystem {
       if (line.match(importRegex)) {
         const relativeFilePath = line.replace("import ", "")
         const absoluteImportFilePath = this.join(folder, relativeFilePath)
-        const expandedFile = this._evaluateImports(absoluteImportFilePath)
-        replacements.push({ lineNumber, code: expandedFile.code })
+        const expandedFile = this._assembleFile(absoluteImportFilePath)
         importFilePaths.push(absoluteImportFilePath)
         importFilePaths = importFilePaths.concat(expandedFile.importFilePaths)
+        replacements.push({ lineNumber, code: expandedFile.afterImportPass })
       }
     })
     replacements.forEach(replacement => {
       const { lineNumber, code } = replacement
       lines[lineNumber] = code
     })
+    const combinedLines = lines.join("\n")
     _expandedImportCache[absoluteFilePath] = {
-      code: lines.join("\n"),
-      importFilePaths
+      importFilePaths,
+      isImportOnly,
+      afterImportPass: combinedLines,
+      filepathsWithParserDefinitions: importFilePaths.filter(filename => this._doesFileHaveGrammarDefinitions(filename)).concat(filepathsWithParserDefinitions)
     }
     return _expandedImportCache[absoluteFilePath]
   }
+  _doesFileHaveGrammarDefinitions(absoluteFilePath) {
+    if (!absoluteFilePath) return false
+    const { _grammarExpandersCache } = this
+    if (_grammarExpandersCache[absoluteFilePath] === undefined) _grammarExpandersCache[absoluteFilePath] = !!this._storage.read(absoluteFilePath).match(parserRegex)
+    return _grammarExpandersCache[absoluteFilePath]
+  }
   _getOneGrammarParserFromFiles(filePaths, baseGrammarCode) {
     const parserDefinitionRegex = /^[a-zA-Z0-9_]+Parser/
     const asOneFile = filePaths
@@ -164,18 +181,12 @@ class TreeFileSystem {
     }
     return _parserCache[key]
   }
-  evaluateImports(absoluteFilePath, defaultParserCode) {
-    const importResults = this._evaluateImports(absoluteFilePath)
-    const results = {
-      originalFileAsTree: this._getFileAsTree(absoluteFilePath),
-      afterImportPass: importResults.code
-    }
-    if (!defaultParserCode) return results
-    const filepathsWithParserDefinitions = importResults.importFilePaths.filter(filename => this._doesFileHaveGrammarDefinitions(filename))
-    if (this._doesFileHaveGrammarDefinitions(absoluteFilePath)) filepathsWithParserDefinitions.push(absoluteFilePath)
+  assembleFile(absoluteFilePath, defaultParserCode) {
+    const assembledFile = this._assembleFile(absoluteFilePath)
+    if (!defaultParserCode) return assembledFile
     // BUILD CUSTOM COMPILER, IF THERE ARE CUSTOM GRAMMAR NODES DEFINED
-    if (filepathsWithParserDefinitions.length) results.parser = this.getParser(filepathsWithParserDefinitions, defaultParserCode).parser
-    return results
+    if (assembledFile.filepathsWithParserDefinitions.length) assembledFile.parser = this.getParser(assembledFile.filepathsWithParserDefinitions, defaultParserCode).parser
+    return assembledFile
   }
 }
 window.TreeFileSystem = TreeFileSystem
diff --git a/products/TreeFileSystem.js b/products/TreeFileSystem.js
@@ -7,6 +7,10 @@ const { HandGrammarProgram } = require("../products/GrammarLanguage.js")
 const grammarParser = require("../products/grammar.nodejs.js")
 const { posix } = require("../products/Path.js")
 const GRAMMAR_EXTENSION = ".grammar"
+const parserRegex = /^[a-zA-Z0-9_]+Parser/gm
+// A regex to check if a multiline string has a line that starts with "import ".
+const importRegex = /^import /gm
+const importOnlyRegex = /^importOnly/
 class DiskWriter {
   constructor() {
     this.fileCache = {}
@@ -94,22 +98,26 @@ class TreeFileSystem {
     }
     return _treeCache[absoluteFilePath]
   }
-  _doesFileHaveGrammarDefinitions(absoluteFilePath) {
-    if (!absoluteFilePath) return false
-    const { _grammarExpandersCache } = this
-    if (_grammarExpandersCache[absoluteFilePath] === undefined) _grammarExpandersCache[absoluteFilePath] = !!this._storage.read(absoluteFilePath).match(/^[a-zA-Z0-9_]+Parser/gm)
-    return _grammarExpandersCache[absoluteFilePath]
-  }
-  _evaluateImports(absoluteFilePath) {
+  _assembleFile(absoluteFilePath) {
     const { _expandedImportCache } = this
     if (_expandedImportCache[absoluteFilePath]) return _expandedImportCache[absoluteFilePath]
-    // A regex to check if a multiline string has a line that starts with "import ".
-    const importRegex = /^import /gm
-    const code = this.read(absoluteFilePath)
-    if (!code.match(importRegex))
+    let code = this.read(absoluteFilePath)
+    const isImportOnly = importOnlyRegex.test(code)
+    // Strip any parsers
+    const stripIt = code.includes("// parsersOnly") // temporary perf hack
+    if (stripIt)
+      code = code
+        .split("\n")
+        .filter(line => line.startsWith("import "))
+        .join("\n")
+    const filepathsWithParserDefinitions = []
+    if (this._doesFileHaveGrammarDefinitions(absoluteFilePath)) filepathsWithParserDefinitions.push(absoluteFilePath)
+    if (!importRegex.test(code))
       return {
-        code,
-        importFilePaths: []
+        afterImportPass: code,
+        isImportOnly,
+        importFilePaths: [],
+        filepathsWithParserDefinitions
       }
     let importFilePaths = []
     const lines = code.split("\n")
@@ -119,22 +127,31 @@ class TreeFileSystem {
       if (line.match(importRegex)) {
         const relativeFilePath = line.replace("import ", "")
         const absoluteImportFilePath = this.join(folder, relativeFilePath)
-        const expandedFile = this._evaluateImports(absoluteImportFilePath)
-        replacements.push({ lineNumber, code: expandedFile.code })
+        const expandedFile = this._assembleFile(absoluteImportFilePath)
         importFilePaths.push(absoluteImportFilePath)
         importFilePaths = importFilePaths.concat(expandedFile.importFilePaths)
+        replacements.push({ lineNumber, code: expandedFile.afterImportPass })
       }
     })
     replacements.forEach(replacement => {
       const { lineNumber, code } = replacement
       lines[lineNumber] = code
     })
+    const combinedLines = lines.join("\n")
     _expandedImportCache[absoluteFilePath] = {
-      code: lines.join("\n"),
-      importFilePaths
+      importFilePaths,
+      isImportOnly,
+      afterImportPass: combinedLines,
+      filepathsWithParserDefinitions: importFilePaths.filter(filename => this._doesFileHaveGrammarDefinitions(filename)).concat(filepathsWithParserDefinitions)
     }
     return _expandedImportCache[absoluteFilePath]
   }
+  _doesFileHaveGrammarDefinitions(absoluteFilePath) {
+    if (!absoluteFilePath) return false
+    const { _grammarExpandersCache } = this
+    if (_grammarExpandersCache[absoluteFilePath] === undefined) _grammarExpandersCache[absoluteFilePath] = !!this._storage.read(absoluteFilePath).match(parserRegex)
+    return _grammarExpandersCache[absoluteFilePath]
+  }
   _getOneGrammarParserFromFiles(filePaths, baseGrammarCode) {
     const parserDefinitionRegex = /^[a-zA-Z0-9_]+Parser/
     const asOneFile = filePaths
@@ -172,18 +189,12 @@ class TreeFileSystem {
     }
     return _parserCache[key]
   }
-  evaluateImports(absoluteFilePath, defaultParserCode) {
-    const importResults = this._evaluateImports(absoluteFilePath)
-    const results = {
-      originalFileAsTree: this._getFileAsTree(absoluteFilePath),
-      afterImportPass: importResults.code
-    }
-    if (!defaultParserCode) return results
-    const filepathsWithParserDefinitions = importResults.importFilePaths.filter(filename => this._doesFileHaveGrammarDefinitions(filename))
-    if (this._doesFileHaveGrammarDefinitions(absoluteFilePath)) filepathsWithParserDefinitions.push(absoluteFilePath)
+  assembleFile(absoluteFilePath, defaultParserCode) {
+    const assembledFile = this._assembleFile(absoluteFilePath)
+    if (!defaultParserCode) return assembledFile
     // BUILD CUSTOM COMPILER, IF THERE ARE CUSTOM GRAMMAR NODES DEFINED
-    if (filepathsWithParserDefinitions.length) results.parser = this.getParser(filepathsWithParserDefinitions, defaultParserCode).parser
-    return results
+    if (assembledFile.filepathsWithParserDefinitions.length) assembledFile.parser = this.getParser(assembledFile.filepathsWithParserDefinitions, defaultParserCode).parser
+    return assembledFile
   }
 }
 

diff --git a/products/TreeNode.browser.js b/products/TreeNode.browser.js
@@ -2555,7 +2555,7 @@ TreeNode.iris = `sepal_length,sepal_width,petal_length,petal_width,species
 4.9,2.5,4.5,1.7,virginica
 5.1,3.5,1.4,0.2,setosa
 5,3.4,1.5,0.2,setosa`
-TreeNode.getVersion = () => "76.2.0"
+TreeNode.getVersion = () => "77.0.0"
 class AbstractExtendibleTreeNode extends TreeNode {
   _getFromExtended(firstWordPath) {
     const hit = this._getNodeFromExtended(firstWordPath)

diff --git a/products/TreeNode.js b/products/TreeNode.js
@@ -2545,7 +2545,7 @@ TreeNode.iris = `sepal_length,sepal_width,petal_length,petal_width,species
 4.9,2.5,4.5,1.7,virginica
 5.1,3.5,1.4,0.2,setosa
 5,3.4,1.5,0.2,setosa`
-TreeNode.getVersion = () => "76.2.0"
+TreeNode.getVersion = () => "77.0.0"
 class AbstractExtendibleTreeNode extends TreeNode {
   _getFromExtended(firstWordPath) {
     const hit = this._getNodeFromExtended(firstWordPath)

diff --git a/releaseNotes.scroll b/releaseNotes.scroll
@@ -4,6 +4,10 @@ startColumns 4
 
 Here's a list of the notable changes in JTree.
 
+# 77.0.0 2023-5-08
+- 🎉 `assembleFile` (formerly `evaluateImports`) is now faster by stripping Parser Definitions from assembled files, using them only in the returned parsers. (There are still many speed improvements to be had here)
+- ⚠️ BREAKING: The `TreeFileSystem.evaluateImports` method is now `TreeFileSystem.assembleFile`, and the interface of the returned object has changed.
+
 # 76.2.0 2023-5-03
 - 🎉 only ship bare minimum files in npm package
 

diff --git a/treeFileSystem/TreeFileSystem.test.ts b/treeFileSystem/TreeFileSystem.test.ts
@@ -10,7 +10,7 @@ const testTree: treeNotationTypes.testTree = {}
 testTree.disk = equal => {
   const tfs = new TreeFileSystem()
   // Arrange/Act/Assert
-  equal(tfs.evaluateImports(path.join(__dirname, "..", "readme.scroll")).afterImportPass.length > 0, true)
+  equal(tfs.assembleFile(path.join(__dirname, "..", "readme.scroll")).afterImportPass.length > 0, true)
 }
 
 testTree.inMemory = equal => {
@@ -23,8 +23,8 @@ testTree.inMemory = equal => {
   }
   const tfs = new TreeFileSystem(files)
   equal(tfs.dirname("/"), "/")
-  equal(tfs.evaluateImports("/main").afterImportPass, "world\nciao")
-  equal(tfs.evaluateImports("/nested/deep/relative").afterImportPass, "world\nciao")
+  equal(tfs.assembleFile("/main").afterImportPass, "world\nciao")
+  equal(tfs.assembleFile("/nested/deep/relative").afterImportPass, "world\nciao")
 }
 
 /*NODE_JS_ONLY*/ if (!module.parent) TestRacer.testSingleFile(__filename, testTree)