Skip to content

Commit

Permalink
Faster fusion
Browse files Browse the repository at this point in the history
  • Loading branch information
breck7 committed Feb 12, 2025
1 parent 38d3483 commit e2517ff
Show file tree
Hide file tree
Showing 10 changed files with 379 additions and 225 deletions.
4 changes: 2 additions & 2 deletions fusion/Fusion.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -106,10 +106,10 @@ testParticles.circularImports = async equal => {
"/g.scroll": ""
}
const tfs = new Fusion(files)
const result = await tfs.fuseFile("/a.scroll")
equal(result.fused.includes("Circular import detected"), true, "Should have detected circularImports")
const result2 = await tfs.fuseFile("/c.scroll")
equal(result2.fused.includes("Circular import detected"), true, "Should have detected circularImports")
const result = await tfs.fuseFile("/a.scroll")
equal(result.fused.includes("Circular import detected"), true, "Should have detected circularImports")
const result3 = await tfs.fuseFile("/d.scroll")
equal(result3.fused.includes("Circular import detected"), false, "No circularImports detected")
}
Expand Down
180 changes: 101 additions & 79 deletions fusion/Fusion.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,11 @@ interface OpenedFile {

interface FusedFile {
fused: string // codeWithoutImportsNorParserDefinitions
footers: string[]
importFilePaths: string[]
footers?: string[]
importFilePaths?: string[]
isImportOnly: boolean
parser?: particlesTypes.particle
filepathsWithParserDefinitions: string[]
filepathsWithParserDefinitions?: string[]
exists: boolean
circularImportError?: boolean
}
Expand Down Expand Up @@ -101,7 +101,6 @@ class DiskWriter implements Storage {
fileCache: { [filepath: string]: OpenedFile } = {}

async _read(absolutePath: particlesTypes.filepath) {
const { fileCache } = this
if (isUrl(absolutePath)) {
const result = await fetchWithCache(absolutePath)
return {
Expand All @@ -112,16 +111,27 @@ class DiskWriter implements Storage {
}
}

if (!fileCache[absolutePath]) {
const exists = await fs
.access(absolutePath)
.then(() => true)
.catch(() => false)
if (exists) {
const [content, stats] = await Promise.all([fs.readFile(absolutePath, "utf8").then(content => content.replace(/\r/g, "")), fs.stat(absolutePath)])
fileCache[absolutePath] = { absolutePath, exists: true, content, stats }
} else {
fileCache[absolutePath] = { absolutePath, exists: false, content: "", stats: { mtimeMs: 0, ctimeMs: 0 } }
const { fileCache } = this
if (fileCache[absolutePath]) return fileCache[absolutePath]
try {
const stats = await fs.stat(absolutePath)
const content = await fs.readFile(absolutePath, {
encoding: "utf8",
flag: "r" // explicit read flag
})
const normalizedContent = content.includes("\r") ? content.replace(/\r/g, "") : content
fileCache[absolutePath] = {
absolutePath,
exists: true,
content: normalizedContent,
stats
}
} catch (error) {
fileCache[absolutePath] = {
absolutePath,
exists: false,
content: "",
stats: { mtimeMs: 0, ctimeMs: 0 }
}
}
return fileCache[absolutePath]
Expand Down Expand Up @@ -313,7 +323,7 @@ class FusionFile {
fusedFile = await fileSystem.fuseFile(filePath, defaultParserCode)
this.importOnly = fusedFile.isImportOnly
fusedCode = fusedFile.fused
if (fusedFile.footers.length) fusedCode += "\n" + fusedFile.footers.join("\n")
if (fusedFile.footers) fusedCode += "\n" + fusedFile.footers.join("\n")
this.dependencies = fusedFile.importFilePaths
this.fusedFile = fusedFile
}
Expand Down Expand Up @@ -394,7 +404,6 @@ class Fusion implements Storage {
private _storage: Storage
private _particleCache: { [filepath: string]: typeof Particle } = {}
private _parserCache: { [concatenatedFilepaths: string]: any } = {}
private _expandedImportCache: { [filepath: string]: FusedFile } = {}
private _parsersExpandersCache: { [filepath: string]: boolean } = {}

private async _getFileAsParticles(absoluteFilePathOrUrl: string) {
Expand All @@ -406,10 +415,52 @@ class Fusion implements Storage {
return _particleCache[absoluteFilePathOrUrl]
}

private async _fuseFile(absoluteFilePathOrUrl: string, importStack: string[] = []): Promise<FusedFile> {
const { _expandedImportCache } = this
if (_expandedImportCache[absoluteFilePathOrUrl]) return _expandedImportCache[absoluteFilePathOrUrl]
getImports(particle, absoluteFilePathOrUrl, importStack) {
const folder = this.dirname(absoluteFilePathOrUrl)
const results = particle
.filter(particle => particle.getLine().match(importRegex))
.map(async importParticle => {
const rawPath = importParticle.getLine().replace("import ", "")
let absoluteImportFilePath = this.join(folder, rawPath)
if (isUrl(rawPath)) absoluteImportFilePath = rawPath
else if (isUrl(folder)) absoluteImportFilePath = folder + "/" + rawPath

if (importStack.includes(absoluteImportFilePath) || absoluteImportFilePath === absoluteFilePathOrUrl) {
const circularImportError = `Circular import detected: ${[...importStack, absoluteImportFilePath].join(" -> ")}`
return {
expandedFile: circularImportError,
exists: true,
absoluteImportFilePath,
importParticle,
circularImportError,
lineCount: particle.numberOfLines
}
}

const expandedFile = await this._fuseFile(absoluteImportFilePath, [...importStack, absoluteFilePathOrUrl])
const exists = await this.exists(absoluteImportFilePath)
return {
expandedFile,
exists,
absoluteImportFilePath,
importParticle,
lineCount: expandedFile.lineCount
}
})
return Promise.all(results)
}

private _pendingFuseRequests: { [filepath: string]: Promise<FusedFile> } = {}

private async _fuseFile(absoluteFilePathOrUrl: string, importStack: string[]): Promise<FusedFile> {
const { _pendingFuseRequests } = this
if (_pendingFuseRequests[absoluteFilePathOrUrl]) return _pendingFuseRequests[absoluteFilePathOrUrl]

_pendingFuseRequests[absoluteFilePathOrUrl] = this._fuseFile2(absoluteFilePathOrUrl, importStack)
return _pendingFuseRequests[absoluteFilePathOrUrl]
}

private async _fuseFile2(absoluteFilePathOrUrl: string, importStack: string[]): Promise<FusedFile> {
const [code, exists] = await Promise.all([this.read(absoluteFilePathOrUrl), this.exists(absoluteFilePathOrUrl)])

const isImportOnly = importOnlyRegex.test(code)
Expand All @@ -425,69 +476,35 @@ class Fusion implements Storage {
.join("\n")
: code

const lineCount = processedCode.split("\n").length
const lineCount = (processedCode.match(/\n/g) || []).length + 1

const filepathsWithParserDefinitions = []
let filepathsWithParserDefinitions: string[]
if (await this._doesFileHaveParsersDefinitions(absoluteFilePathOrUrl)) {
filepathsWithParserDefinitions.push(absoluteFilePathOrUrl)
filepathsWithParserDefinitions = [absoluteFilePathOrUrl]
}

if (!importRegex.test(processedCode)) {
return {
fused: processedCode,
footers: [],
isImportOnly,
importFilePaths: [],
filepathsWithParserDefinitions,
exists,
lineCount
}
}

const particle = new Particle(processedCode)
const folder = this.dirname(absoluteFilePathOrUrl)

// Fetch all imports in parallel
const importParticles = particle.filter(particle => particle.getLine().match(importRegex))
const importResults = importParticles.map(async importParticle => {
const rawPath = importParticle.getLine().replace("import ", "")
let absoluteImportFilePath = this.join(folder, rawPath)
if (isUrl(rawPath)) absoluteImportFilePath = rawPath
else if (isUrl(folder)) absoluteImportFilePath = folder + "/" + rawPath

if (importStack.includes(absoluteFilePathOrUrl)) {
const circularImportError = `Circular import detected: ${[...importStack, absoluteFilePathOrUrl].join(" -> ")}`
return {
expandedFile: circularImportError,
exists: true,
absoluteImportFilePath,
importParticle,
circularImportError,
lineCount
}
}

// todo: race conditions
const [expandedFile, exists] = await Promise.all([this._fuseFile(absoluteImportFilePath, [...importStack, absoluteFilePathOrUrl]), this.exists(absoluteImportFilePath)])
return {
expandedFile,
exists,
absoluteImportFilePath,
importParticle,
lineCount: expandedFile.lineCount
}
})

const imported = await Promise.all(importResults)
const imported = await this.getImports(particle, absoluteFilePathOrUrl, importStack)

// Assemble all imports
let importFilePaths: string[] = []
let footers: string[] = []
let footers: string[]
let hasCircularImportError = false
imported.forEach(importResults => {
const { importParticle, absoluteImportFilePath, expandedFile, exists, circularImportError, lineCount } = importResults
importFilePaths.push(absoluteImportFilePath)
importFilePaths = importFilePaths.concat(expandedFile.importFilePaths)
if (expandedFile.importFilePaths) importFilePaths = importFilePaths.concat(expandedFile.importFilePaths)

const originalLine = importParticle.getLine()
importParticle.setLine("imported " + absoluteImportFilePath)
Expand All @@ -499,37 +516,41 @@ class Fusion implements Storage {
importParticle.set("circularImportError", circularImportError)
}

footers = footers.concat(expandedFile.footers)
if (importParticle.has("footer")) footers.push(expandedFile.fused)
else importParticle.insertLinesAfter(expandedFile.fused)
if (expandedFile.footers) footers = (footers || []).concat(expandedFile.footers)
if (importParticle.has("footer")) {
footers = footers || []
footers.push(expandedFile.fused)
} else importParticle.insertLinesAfter(expandedFile.fused)
})

const existStates = await Promise.all(importFilePaths.map(file => this.exists(file)))

const allImportsExist = !existStates.some(exists => !exists)
const importFilepathsWithParserDefinitions = (
await Promise.all(
importFilePaths.map(async filename => ({
filename,
hasParser: await this._doesFileHaveParsersDefinitions(filename)
}))
)
)
.filter(result => result.hasParser)
.map(result => result.filename)

_expandedImportCache[absoluteFilePathOrUrl] = {
if (importFilepathsWithParserDefinitions.length) {
filepathsWithParserDefinitions = filepathsWithParserDefinitions || []
filepathsWithParserDefinitions.concat(importFilepathsWithParserDefinitions)
}

return {
importFilePaths,
isImportOnly,
fused: particle.toString(),
lineCount,
footers,
circularImportError: hasCircularImportError,
exists: allImportsExist,
filepathsWithParserDefinitions: (
await Promise.all(
importFilePaths.map(async filename => ({
filename,
hasParser: await this._doesFileHaveParsersDefinitions(filename)
}))
)
)
.filter(result => result.hasParser)
.map(result => result.filename)
.concat(filepathsWithParserDefinitions)
filepathsWithParserDefinitions
}

return _expandedImportCache[absoluteFilePathOrUrl]
}

private async _doesFileHaveParsersDefinitions(absoluteFilePathOrUrl: particlesTypes.filepath) {
Expand Down Expand Up @@ -590,11 +611,11 @@ class Fusion implements Storage {
}

async fuseFile(absoluteFilePathOrUrl: string, defaultParserCode?: string): Promise<FusedFile> {
const fusedFile = await this._fuseFile(absoluteFilePathOrUrl)
const fusedFile = await this._fuseFile(absoluteFilePathOrUrl, [])

if (!defaultParserCode) return fusedFile

if (fusedFile.filepathsWithParserDefinitions.length) {
if (fusedFile.filepathsWithParserDefinitions) {
const parser = await this.getParser(fusedFile.filepathsWithParserDefinitions, defaultParserCode)
fusedFile.parser = parser.parser
}
Expand Down Expand Up @@ -652,6 +673,7 @@ class Fusion implements Storage {
}

folderCache = {}
// todo: this is weird. i know we evolved our way here but we should step back and clean this up.
async getLoadedFilesInFolder(folderPath, extension) {
folderPath = Utils.ensureFolderEndsInSlash(folderPath)
if (this.folderCache[folderPath]) return this.folderCache[folderPath]
Expand Down
85 changes: 85 additions & 0 deletions fusion/perf.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
#!/usr/bin/env node

/*
This file contains a simple set of perf tests that can be run manually to keep fusion perf in check.
*/

// rm perf.cpuprofile; rm perf.heapprofile; node --cpu-prof --cpu-prof-name=perf.cpuprofile --heap-prof --heap-prof-name=perf.heapprofile perf.js

const fs = require("fs")
const path = require("path")
const { Utils } = require("../products/Utils.js")
const { Timer } = Utils
const { Particle } = require("../products/Particle.js")
const { Fusion } = require("../products/Fusion.js")
const { ScrollFile } = require("scroll-cli")

class PerfTest {
constructor(folderPath) {
this.folderPath = folderPath
this.timer = new Timer()
this.files = []
this.simpleStrings = []
this.particles = []
this.fusedFiles = []
this.scrollFiles = []
}

gatherFiles() {
this.files = fs
.readdirSync(this.folderPath)
.filter(file => file.endsWith(".scroll"))
.map(file => path.join(this.folderPath, file))
console.log(`Found ${this.files.length} .scroll files`)
this.timer.tick("Finding files")
return this
}

readToStrings() {
this.simpleStrings = this.files.map(file => fs.readFileSync(file, "utf8"))
this.timer.tick("Reading files to strings")
return this
}

parseToParticles() {
this.particles = this.simpleStrings.map(str => new Particle(str))
this.timer.tick("Parsing to Particles")
return this
}

async fuseFiles() {
const fusion = new Fusion()
this.fusedFiles = await Promise.all(this.files.map(file => fusion.fuseFile(file)))
this.timer.tick("Fusing files")
return this
}

parseAsScroll() {
this.scrollFiles = this.simpleStrings.map(str => new ScrollFile(str))
this.timer.tick("Parsing as Scroll")
return this
}

printMemoryUsage() {
const used = process.memoryUsage()
console.log("\nMemory Usage:")
for (let key in used) {
console.log(`${key}: ${Math.round((used[key] / 1024 / 1024) * 100) / 100} MB`)
}
}

async runAll() {
console.log("Starting performance tests...\n")
this.gatherFiles().readToStrings().parseToParticles()
await this.fuseFiles()
this.parseAsScroll()

console.log(`\nTotal time: ${this.timer.getTotalElapsedTime()}ms`)
this.printMemoryUsage()
}
}

// Run the tests
const dir = "/Users/breck/pldb.io/concepts"
const perfTest = new PerfTest(dir)
perfTest.runAll().catch(console.error)
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "scrollsdk",
"version": "101.1.1",
"version": "101.2.0",
"description": "This npm package includes the Particles class, the Parsers compiler-compiler, a Parsers IDE, and more, all implemented in Particles, Parsers, and TypeScript.",
"types": "./built/scrollsdk.node.d.ts",
"main": "./products/Particle.js",
Expand Down
Loading

0 comments on commit e2517ff

Please sign in to comment.