allow switching between macros in the same namespace easily (#172)

core/src/index.ts

@@ -8,6 +8,7 @@ import {
   Node,
   NodeDefinition,
   NodeOrMacroDefinition,
+  MacroNodeDefinition,
 } from "./node";
 
 export * from "./connect";
@@ -29,6 +30,8 @@ export type NodesCollection = OMap<Node>;
 
 export type NodesDefCollection = OMap<NodeDefinition>;
 
+export type MacrosDefCollection = OMap<MacroNodeDefinition<any>>;
+
 export type CustomNodesCollection = OMap<CustomNode>;
 
 export interface NodeLibraryGroup {
@@ -45,6 +48,11 @@ export type ImportablesResult = {
   errors: { path: string; message: string }[];
 };
 
+export type ImportableMacrosResult = {
+  importableMacros: Record<string, MacrosDefCollection>;
+  errors: { path: string; message: string }[];
+};
+
 export interface FlowJob {
   flow: FlydeFlow;
   id: string;

core/src/node/macro-node.ts

@@ -126,7 +126,7 @@ export type MacroEditorConfigDefinition =
   | MacroEditorConfigStructured;
 
 export interface MacroNode<T> extends NodeMetadata {
-  definitionBuilder: (data: T) => Omit<CodeNodeDefinition, "id">;
+  definitionBuilder: (data: T) => Omit<CodeNodeDefinition, "id" | "namespace">;
   runFnBuilder: (data: T) => CodeNode["run"];
   defaultData: T;
 
@@ -178,14 +178,14 @@ export const isMacroNodeDefinition = (
 };
 
 export function processMacroNodeInstance(
-  namespace: string,
+  prefix: string,
   macro: MacroNode<any>,
   instance: MacroNodeInstance
 ) {
   const metaData = macro.definitionBuilder(instance.macroData);
   const runFn = macro.runFnBuilder(instance.macroData);
 
-  const id = `${namespace}${macro.id}__${instance.id}`;
+  const id = `${prefix}${macro.id}__${instance.id}`;
 
   const resolvedNode: CodeNode = {
     ...metaData,

dev-server/src/service/resolve-dependent-packages-macros.ts

@@ -0,0 +1,64 @@
+import {
+  MacrosDefCollection,
+  NodesDefCollection,
+  isMacroNode,
+} from "@flyde/core";
+import {
+  deserializeFlow,
+  isCodeNodePath,
+  macroNodeToDefinition,
+  resolveCodeNodeDependencies,
+  resolveImportablePaths,
+} from "@flyde/resolver";
+import { readFileSync } from "fs";
+
+export async function resolveDependentPackagesMacros(
+  rootPath: string,
+  flydeDependencies: string[]
+) {
+  return flydeDependencies.reduce<Record<string, MacrosDefCollection>>(
+    (acc, dep) => {
+      try {
+        const paths = resolveImportablePaths(rootPath, dep);
+        const nodes = paths.reduce((acc, filePath) => {
+          if (isCodeNodePath(filePath)) {
+            const obj = resolveCodeNodeDependencies(filePath).nodes.reduce(
+              (obj, { node }) => {
+                // Only include macro nodes
+                if (isMacroNode(node)) {
+                  return {
+                    ...obj,
+                    [node.id]: macroNodeToDefinition(node, filePath),
+                  };
+                }
+                return obj;
+              },
+              {}
+            );
+
+            return { ...acc, ...obj };
+          }
+          try {
+            const flow = deserializeFlow(
+              readFileSync(filePath, "utf8"),
+              filePath
+            );
+            // Only include macro nodes
+            if (isMacroNode(flow.node)) {
+              return { ...acc, [flow.node.id]: flow.node };
+            }
+            return acc;
+          } catch (e) {
+            console.error(`Skipping corrupt flow at ${filePath}, error: ${e}`);
+            return acc;
+          }
+        }, {});
+        return { ...acc, [dep]: nodes };
+      } catch (e) {
+        console.log(`skipping invalid dependency ${dep}`);
+        return acc;
+      }
+    },
+    {}
+  );
+}

dev-server/src/service/scan-importable-macros.ts

@@ -0,0 +1,108 @@
+import { join, relative } from "path";
+import {
+  isCodeNodePath,
+  resolveCodeNodeDependencies,
+  deserializeFlow,
+  macroNodeToDefinition,
+} from "@flyde/resolver";
+
+import {
+  debugLogger,
+  isMacroNode,
+  ImportablesResult,
+  ImportableMacrosResult,
+  MacrosDefCollection,
+  MacroNode,
+} from "@flyde/core";
+import { scanFolderStructure } from "./scan-folders-structure";
+import { FlydeFile } from "../fs-helper/shared";
+import { getFlydeDependencies } from "./get-flyde-dependencies";
+import * as StdLib from "@flyde/stdlib/dist/all";
+import { resolveDependentPackagesMacros } from "./resolve-dependent-packages-macros";
+
+export interface CorruptScannedNode {
+  type: "corrupt";
+  error: string;
+}
+
+export async function scanImportableMacros(
+  rootPath: string,
+  filename: string
+): Promise<ImportableMacrosResult> {
+  const fileRoot = join(rootPath, filename);
+
+  const localFiles = getLocalFlydeFiles(rootPath);
+
+  const depsNames = await getFlydeDependencies(rootPath);
+
+  const depsNodes = await resolveDependentPackagesMacros(rootPath, depsNames);
+
+  let builtInStdLib: Record<string, MacrosDefCollection> = {};
+  if (!depsNames.includes("@flyde/stdlib")) {
+    debugLogger("Using built-in stdlib");
+
+    const nodes = Object.fromEntries(
+      Object.entries(StdLib)
+        .filter((pair) => isMacroNode(pair[1]))
+        .map(([id, node]) => [
+          id,
+          macroNodeToDefinition(node as MacroNode<any>, ""),
+        ])
+    ) as MacrosDefCollection;
+    builtInStdLib = {
+      "@flyde/stdlib": nodes,
+    };
+  }
+
+  let allErrors: ImportablesResult["errors"] = [];
+
+  const localMacros = localFiles
+    .filter((file) => !file.relativePath.endsWith(filename))
+    .reduce<Record<string, MacrosDefCollection>>((acc, file) => {
+      if (isCodeNodePath(file.fullPath)) {
+        const { errors, nodes } = resolveCodeNodeDependencies(file.fullPath);
+        allErrors.push(
+          ...errors.map((err) => ({ path: file.fullPath, message: err }))
+        );
+
+        const nodesObj = nodes.reduce((obj, { node }) => {
+          if (isMacroNode(node)) {
+            obj[node.id] = macroNodeToDefinition(node, file.fullPath);
+          }
+          return obj;
+        }, {});
+        const relativePath = relative(join(fileRoot, ".."), file.fullPath);
+        acc[relativePath] ??= {};
+        acc[relativePath] = {
+          ...acc[relativePath],
+          ...nodesObj,
+        };
+
+        return acc;
+      }
+
+      return acc; // Skip non-code nodes
+    }, {});
+
+  return {
+    importableMacros: { ...builtInStdLib, ...depsNodes, ...localMacros },
+    errors: allErrors,
+  };
+}
+
+function getLocalFlydeFiles(rootPath: string) {
+  const structure = scanFolderStructure(rootPath);
+
+  const localFlydeFiles: FlydeFile[] = [];
+  const queue = [...structure];
+  while (queue.length) {
+    const item = queue.pop();
+    if (item.isFolder === true) {
+      queue.push(...item.children);
+    } else if (item.isFlyde || item.isFlydeCode) {
+      localFlydeFiles.push(item as FlydeFile);
+    }
+  }
+
+  return localFlydeFiles;
+}

editor/src/integrated-flow-manager/IntegratedFlowManager.tsx

@@ -392,14 +392,30 @@ export const IntegratedFlowManager: React.FC<IntegratedFlowManagerProps> = (
     [_onRequestHistory, debuggerClient]
   );
 
+  const onRequestSiblingNodes = React.useCallback<
+    DependenciesContextData["onRequestSiblingNodes"]
+  >(
+    (macro) => {
+      return ports.onRequestSiblingNodes({ macro });
+    },
+    [ports]
+  );
+
   const dependenciesContextValue = React.useMemo<DependenciesContextData>(
     () => ({
       resolvedDependencies: currentResolvedDeps,
       onImportNode,
       onRequestImportables: queryImportables,
       libraryData,
+      onRequestSiblingNodes,
     }),
-    [currentResolvedDeps, onImportNode, queryImportables, libraryData]
+    [
+      currentResolvedDeps,
+      onImportNode,
+      queryImportables,
+      libraryData,
+      onRequestSiblingNodes,
+    ]
   );
 
   return (

editor/src/vscode-ports.ts

@@ -93,6 +93,9 @@ export const createVsCodePorts = (): EditorPorts => {
     getLibraryData: async () => {
       return postMessageCallback("getLibraryData", {});
     },
+    onRequestSiblingNodes: async (dto) => {
+      return postMessageCallback("onRequestSiblingNodes", dto);
+    },
   };
 };
 

editor/src/web-ports.ts

@@ -63,5 +63,8 @@ export const createWebPorts = ({
     getLibraryData: async () => {
       return devServerClient.getLibraryData();
     },
+    onRequestSiblingNodes: async () => {
+      return [];
+    },
   };
 };

flow-editor/src/flow-editor/DependenciesContext.tsx

@@ -4,6 +4,7 @@ import {
   Pos,
   NodeLibraryData,
   ImportablesResult,
+  MacroNodeDefinition,
 } from "@flyde/core";
 import { createContext, useContext } from "react";
 
@@ -25,13 +26,17 @@ export interface DependenciesContextData {
   ) => Promise<ResolvedDependenciesDefinitions>;
   onRequestImportables: () => Promise<LocalImportableResult>;
   libraryData: NodeLibraryData;
+  onRequestSiblingNodes: (
+    macro: MacroNodeDefinition<any>
+  ) => Promise<MacroNodeDefinition<any>[]>;
 }
 
 const DependenciesContext = createContext<DependenciesContextData>({
   resolvedDependencies: {},
   onImportNode: () => Promise.reject(new Error("Not implemented")),
   onRequestImportables: () => Promise.reject(new Error("Not implemented")),
   libraryData: { groups: [] },
+  onRequestSiblingNodes: () => Promise.resolve([]),
 });
 
 export const DependenciesContextProvider = DependenciesContext.Provider;

flow-editor/src/flow-editor/ports/ports.ts

@@ -7,6 +7,7 @@ import {
   noop,
   FlowJob,
   ImportablesResult,
+  MacroNodeDefinition,
 } from "@flyde/core";
 import { ReportEvent } from "./analytics";
 import { toastMsg } from "../../toaster";
@@ -58,6 +59,10 @@ export interface EditorPorts {
     prompt: string;
   }) => Promise<{ importableNode: ImportableSource }>;
   getLibraryData: () => Promise<NodeLibraryData>;
+
+  onRequestSiblingNodes: (dto: {
+    macro: MacroNodeDefinition<any>;
+  }) => Promise<MacroNodeDefinition<any>[]>;
 }
 
 const toastNotImplemented: any = (method: string) => async () => {
@@ -83,6 +88,7 @@ export const defaultPorts: EditorPorts = {
   reportEvent: noop,
   generateNodeFromPrompt: toastNotImplemented("generateNodeFromPrompt"),
   getLibraryData: () => Promise.resolve({ groups: [] }),
+  onRequestSiblingNodes: () => Promise.resolve([]),
 };
 
 export const PortsContext = createContext<EditorPorts>(defaultPorts);