allow switching between macros in the same namespace easily (#172)

core/src/index.ts

@@ -8,6 +8,7 @@ import {
  Node,
  NodeDefinition,
  NodeOrMacroDefinition,
+ MacroNodeDefinition,
 } from "./node";
 
 export * from "./connect";
@@ -29,6 +30,8 @@ export type NodesCollection = OMap<Node>;
 
 export type NodesDefCollection = OMap<NodeDefinition>;
 
+export type MacrosDefCollection = OMap<MacroNodeDefinition<any>>;
+
 export type CustomNodesCollection = OMap<CustomNode>;
 
 export interface NodeLibraryGroup {
@@ -45,6 +48,11 @@ export type ImportablesResult = {
  errors: { path: string; message: string }[];
 };
 
+export type ImportableMacrosResult = {
+ importableMacros: Record<string, MacrosDefCollection>;
+ errors: { path: string; message: string }[];
+};
+
 export interface FlowJob {
  flow: FlydeFlow;
  id: string;

core/src/node/macro-node.ts

@@ -126,7 +126,7 @@ export type MacroEditorConfigDefinition =
  | MacroEditorConfigStructured;
 
 export interface MacroNode<T> extends NodeMetadata {
- definitionBuilder: (data: T) => Omit<CodeNodeDefinition, "id">;
+ definitionBuilder: (data: T) => Omit<CodeNodeDefinition, "id" | "namespace">;
  runFnBuilder: (data: T) => CodeNode["run"];
  defaultData: T;
 
@@ -178,14 +178,14 @@ export const isMacroNodeDefinition = (
 };
 
 export function processMacroNodeInstance(
- namespace: string,
+ prefix: string,
  macro: MacroNode<any>,
  instance: MacroNodeInstance
 ) {
  const metaData = macro.definitionBuilder(instance.macroData);
  const runFn = macro.runFnBuilder(instance.macroData);
 
- const id = `${namespace}${macro.id}__${instance.id}`;
+ const id = `${prefix}${macro.id}__${instance.id}`;
 
  const resolvedNode: CodeNode = {
  ...metaData,

dev-server/src/service/resolve-dependent-packages-macros.ts

@@ -0,0 +1,64 @@
+import {
+ MacrosDefCollection,
+ NodesDefCollection,
+ isMacroNode,
+} from "@flyde/core";
+import {
+ deserializeFlow,
+ isCodeNodePath,
+ macroNodeToDefinition,
+ resolveCodeNodeDependencies,
+ resolveImportablePaths,
+} from "@flyde/resolver";
+import { readFileSync } from "fs";
+
+export async function resolveDependentPackagesMacros(
+ rootPath: string,
+ flydeDependencies: string[]
+) {
+ return flydeDependencies.reduce<Record<string, MacrosDefCollection>>(
+ (acc, dep) => {
+ try {
+ const paths = resolveImportablePaths(rootPath, dep);
+ const nodes = paths.reduce((acc, filePath) => {
+ if (isCodeNodePath(filePath)) {
+ const obj = resolveCodeNodeDependencies(filePath).nodes.reduce(
+ (obj, { node }) => {
+ // Only include macro nodes
+ if (isMacroNode(node)) {
+ return {
+ ...obj,
+ [node.id]: macroNodeToDefinition(node, filePath),
+ };
+ }
+ return obj;
+ },
+ {}
+ );
+
+ return { ...acc, ...obj };
+ }
+ try {
+ const flow = deserializeFlow(
+ readFileSync(filePath, "utf8"),
+ filePath
+ );
+ // Only include macro nodes
+ if (isMacroNode(flow.node)) {
+ return { ...acc, [flow.node.id]: flow.node };
+ }
+ return acc;
+ } catch (e) {
+ console.error(`Skipping corrupt flow at ${filePath}, error: ${e}`);
+ return acc;
+ }
+ }, {});
+ return { ...acc, [dep]: nodes };
+ } catch (e) {
+ console.log(`skipping invalid dependency ${dep}`);
+ return acc;
+ }
+ },
+ {}
+ );
+}

dev-server/src/service/scan-importable-macros.ts

@@ -0,0 +1,108 @@
+import { join, relative } from "path";
+import {
+ isCodeNodePath,
+ resolveCodeNodeDependencies,
+ deserializeFlow,
+ macroNodeToDefinition,
+} from "@flyde/resolver";
+
+import {
+ debugLogger,
+ isMacroNode,
+ ImportablesResult,
+ ImportableMacrosResult,
+ MacrosDefCollection,
+ MacroNode,
+} from "@flyde/core";
+import { scanFolderStructure } from "./scan-folders-structure";
+import { FlydeFile } from "../fs-helper/shared";
+import { getFlydeDependencies } from "./get-flyde-dependencies";
+import * as StdLib from "@flyde/stdlib/dist/all";
+import { resolveDependentPackagesMacros } from "./resolve-dependent-packages-macros";
+
+export interface CorruptScannedNode {
+ type: "corrupt";
+ error: string;
+}
+
+export async function scanImportableMacros(
+ rootPath: string,
+ filename: string
+): Promise<ImportableMacrosResult> {
+ const fileRoot = join(rootPath, filename);
+
+ const localFiles = getLocalFlydeFiles(rootPath);
+
+ const depsNames = await getFlydeDependencies(rootPath);
+
+ const depsNodes = await resolveDependentPackagesMacros(rootPath, depsNames);
+
+ let builtInStdLib: Record<string, MacrosDefCollection> = {};
+ if (!depsNames.includes("@flyde/stdlib")) {
+ debugLogger("Using built-in stdlib");
+
+ const nodes = Object.fromEntries(
+ Object.entries(StdLib)
+ .filter((pair) => isMacroNode(pair[1]))
+ .map(([id, node]) => [
+ id,
+ macroNodeToDefinition(node as MacroNode<any>, ""),
+ ])
+ ) as MacrosDefCollection;
+ builtInStdLib = {
+ "@flyde/stdlib": nodes,
+ };
+ }
+
+ let allErrors: ImportablesResult["errors"] = [];
+
+ const localMacros = localFiles
+ .filter((file) => !file.relativePath.endsWith(filename))
+ .reduce<Record<string, MacrosDefCollection>>((acc, file) => {
+ if (isCodeNodePath(file.fullPath)) {
+ const { errors, nodes } = resolveCodeNodeDependencies(file.fullPath);
+ allErrors.push(
+ ...errors.map((err) => ({ path: file.fullPath, message: err }))
+ );
+
+ const nodesObj = nodes.reduce((obj, { node }) => {
+ if (isMacroNode(node)) {
+ obj[node.id] = macroNodeToDefinition(node, file.fullPath);
+ }
+ return obj;
+ }, {});
+ const relativePath = relative(join(fileRoot, ".."), file.fullPath);
+ acc[relativePath] ??= {};
+ acc[relativePath] = {
+ ...acc[relativePath],
+ ...nodesObj,
+ };
+
+ return acc;
+ }
+
+ return acc; // Skip non-code nodes
+ }, {});
+
+ return {
+ importableMacros: { ...builtInStdLib, ...depsNodes, ...localMacros },
+ errors: allErrors,
+ };
+}
+
+function getLocalFlydeFiles(rootPath: string) {
+ const structure = scanFolderStructure(rootPath);
+
+ const localFlydeFiles: FlydeFile[] = [];
+ const queue = [...structure];
+ while (queue.length) {
+ const item = queue.pop();
+ if (item.isFolder === true) {
+ queue.push(...item.children);
+ } else if (item.isFlyde || item.isFlydeCode) {
+ localFlydeFiles.push(item as FlydeFile);
+ }
+ }
+
+ return localFlydeFiles;
+}

editor/src/integrated-flow-manager/IntegratedFlowManager.tsx

@@ -392,14 +392,30 @@ export const IntegratedFlowManager: React.FC<IntegratedFlowManagerProps> = (
  [_onRequestHistory, debuggerClient]
  );
 
+ const onRequestSiblingNodes = React.useCallback<
+ DependenciesContextData["onRequestSiblingNodes"]
+ >(
+ (macro) => {
+ return ports.onRequestSiblingNodes({ macro });
+ },
+ [ports]
+ );
+
  const dependenciesContextValue = React.useMemo<DependenciesContextData>(
  () => ({
  resolvedDependencies: currentResolvedDeps,
  onImportNode,
  onRequestImportables: queryImportables,
  libraryData,
+ onRequestSiblingNodes,
  }),
- [currentResolvedDeps, onImportNode, queryImportables, libraryData]
+ [
+ currentResolvedDeps,
+ onImportNode,
+ queryImportables,
+ libraryData,
+ onRequestSiblingNodes,
+ ]
  );
 
  return (

editor/src/vscode-ports.ts

@@ -93,6 +93,9 @@ export const createVsCodePorts = (): EditorPorts => {
  getLibraryData: async () => {
  return postMessageCallback("getLibraryData", {});
  },
+ onRequestSiblingNodes: async (dto) => {
+ return postMessageCallback("onRequestSiblingNodes", dto);
+ },
  };
 };
 

editor/src/web-ports.ts

@@ -63,5 +63,8 @@ export const createWebPorts = ({
  getLibraryData: async () => {
  return devServerClient.getLibraryData();
  },
+ onRequestSiblingNodes: async () => {
+ return [];
+ },
  };
 };

flow-editor/src/flow-editor/DependenciesContext.tsx

@@ -4,6 +4,7 @@ import {
  Pos,
  NodeLibraryData,
  ImportablesResult,
+ MacroNodeDefinition,
 } from "@flyde/core";
 import { createContext, useContext } from "react";
 
@@ -25,13 +26,17 @@ export interface DependenciesContextData {
  ) => Promise<ResolvedDependenciesDefinitions>;
  onRequestImportables: () => Promise<LocalImportableResult>;
  libraryData: NodeLibraryData;
+ onRequestSiblingNodes: (
+ macro: MacroNodeDefinition<any>
+ ) => Promise<MacroNodeDefinition<any>[]>;
 }
 
 const DependenciesContext = createContext<DependenciesContextData>({
  resolvedDependencies: {},
  onImportNode: () => Promise.reject(new Error("Not implemented")),
  onRequestImportables: () => Promise.reject(new Error("Not implemented")),
  libraryData: { groups: [] },
+ onRequestSiblingNodes: () => Promise.resolve([]),
 });
 
 export const DependenciesContextProvider = DependenciesContext.Provider;

flow-editor/src/flow-editor/ports/ports.ts

@@ -7,6 +7,7 @@ import {
  noop,
  FlowJob,
  ImportablesResult,
+ MacroNodeDefinition,
 } from "@flyde/core";
 import { ReportEvent } from "./analytics";
 import { toastMsg } from "../../toaster";
@@ -58,6 +59,10 @@ export interface EditorPorts {
  prompt: string;
  }) => Promise<{ importableNode: ImportableSource }>;
  getLibraryData: () => Promise<NodeLibraryData>;
+
+ onRequestSiblingNodes: (dto: {
+ macro: MacroNodeDefinition<any>;
+ }) => Promise<MacroNodeDefinition<any>[]>;
 }
 
 const toastNotImplemented: any = (method: string) => async () => {
@@ -83,6 +88,7 @@ export const defaultPorts: EditorPorts = {
  reportEvent: noop,
  generateNodeFromPrompt: toastNotImplemented("generateNodeFromPrompt"),
  getLibraryData: () => Promise.resolve({ groups: [] }),
+ onRequestSiblingNodes: () => Promise.resolve([]),
 };
 
 export const PortsContext = createContext<EditorPorts>(defaultPorts);