From a213bb8050c6ea854905080ab39d26c5b8bf72ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20B=C3=BCschel?= <13087421+tobiasbueschel@users.noreply.github.com> Date: Sat, 30 Nov 2024 00:49:26 +0800 Subject: [PATCH] feat: set up json-schema-to-diagram --- .editorconfig | 12 ++++ .gitattributes | 1 + .github/FUNDING.yml | 1 + .github/workflows/main.yml | 53 +++++++++++++++ .gitignore | 3 + .npmrc | 1 + LICENSE | 9 +++ index.d.ts | 56 ++++++++++++++++ index.js | 95 ++++++++++++++++++++++++++ index.test.js | 134 +++++++++++++++++++++++++++++++++++++ package.json | 68 +++++++++++++++++++ readme.md | 111 ++++++++++++++++++++++++++++++ 12 files changed, 544 insertions(+) create mode 100644 .editorconfig create mode 100644 .gitattributes create mode 100644 .github/FUNDING.yml create mode 100644 .github/workflows/main.yml create mode 100644 .gitignore create mode 100644 .npmrc create mode 100644 LICENSE create mode 100644 index.d.ts create mode 100644 index.js create mode 100644 index.test.js create mode 100644 package.json create mode 100644 readme.md diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..1c6314a --- /dev/null +++ b/.editorconfig @@ -0,0 +1,12 @@ +root = true + +[*] +indent_style = tab +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[*.yml] +indent_style = space +indent_size = 2 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6313b56 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..a0120c1 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: tobiasbueschel diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..f04b911 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,53 @@ +name: CI & Publish +on: + - push + - pull_request + +permissions: + contents: read + +jobs: + test: + name: Node.js ${{ matrix.node-version }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + node-version: + - 22 + - 20 + - 18 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + - run: npm install + - run: npm test + + release: + name: Release + runs-on: ubuntu-latest + needs: test + if: github.ref == 'refs/heads/main' + permissions: + contents: write # to be able to publish a GitHub release + issues: write # to be able to comment on released issues + pull-requests: write # to be able to comment on released pull requests + id-token: write # to enable use of OIDC for npm provenance + steps: + - name: Checkout + uses: actions/checkout@v1 + - name: Setup Node.js + uses: actions/setup-node@v1 + with: + node-version: 22 + - name: Install dependencies + run: npm install + - name: Verify the integrity of provenance attestations and registry signatures for installed dependencies + run: npm audit signatures + - name: Release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + run: npx semantic-release diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b96b206 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +node_modules +yarn.lock +.vscode/ diff --git a/.npmrc b/.npmrc new file mode 100644 index 0000000..43c97e7 --- /dev/null +++ b/.npmrc @@ -0,0 +1 @@ +package-lock=false diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..68a5b0f --- /dev/null +++ b/LICENSE @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) 2024 Tobias Büschel + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/index.d.ts b/index.d.ts new file mode 100644 index 0000000..92ce0f5 --- /dev/null +++ b/index.d.ts @@ -0,0 +1,56 @@ +import { Model } from "@ai-sdk/openai"; + +export type Options = { + /** + * The path to the file to update. + * @default "./README.md" + */ + readonly filePath?: string; + + /** + * The start marker of the diagram in the file. + * @default "" + */ + readonly startMarker?: string; + + /** + * The end marker of the diagram in the file. + * @default "" + */ + readonly endMarker?: string; + + /** + * The system prompt for the LLM. + */ + readonly systemPrompt?: string; + + /** + * The LLM model to use. + * @default openai("gpt-4o") + */ + readonly model?: Model; + + /** + * The JSON schema of the tools. + */ + readonly jsonSchema: object; +}; + +/** + * Generate a mermaid diagram from a JSON schema of tools. + * + * @param options - The options for the diagram generation. + * @returns A promise that resolves when the diagram is successfully updated. + * + * @example + * ```javascript + * import jsonSchemaToDiagram from 'json-schema-to-diagram'; + * + * const options = { + * jsonSchema: { /* your JSON schema here *\/ }, + * }; + * + * await jsonSchemaToDiagram(options); + * ``` + */ +export default function jsonSchemaToDiagram(options: Options): Promise; diff --git a/index.js b/index.js new file mode 100644 index 0000000..dcce15c --- /dev/null +++ b/index.js @@ -0,0 +1,95 @@ +import { readFile, writeFile } from "fs/promises"; +import path from "path"; +import { generateObject } from "ai"; +import { openai } from "@ai-sdk/openai"; +import { z } from "zod"; + +const DEFAULT_OPTIONS = { + startMarker: "", + endMarker: "", + filePath: "./README.md", + systemPrompt: ` + You are a helpful assistant that generates mermaid diagrams based on a JSON schema of tools. + + Use a flowchart diagram with left-to-right orientation. + Level 1 of the flowchart should be the category of tools that are similar, everything branches out from there + Level 2 of the flowchart should contain individual tools (use bold text for tool names) + Between level 1 and level 2, there should be a text link description of the tool + Level 3 of the flowchart should contain the tools' parameters in a single box with
between each parameter + The flowchart should be in the same language as the one used in the file + `, + model: openai("gpt-4o"), +}; + +/** + * Generate a mermaid diagram from a JSON schema of tools. + * + * @param {Object} options - The options for the diagram generation. + * @param {string} [options.filePath] - The path to the file to update. + * @param {string} [options.startMarker] - The start marker of the diagram in the file. + * @param {string} [options.endMarker] - The end marker of the diagram in the file. + * @param {string} [options.systemPrompt] - The system prompt for the LLM. + * @param {Model} [options.model] - The LLM model to use. + * @param {Object} options.jsonSchema - The JSON schema of the tools. + */ +export default async function jsonSchemaToDiagram(options) { + if (!process.env.OPENAI_API_KEY) { + throw new Error("OPENAI_API_KEY is not set"); + } + + if (typeof options !== "object" || options === null) { + throw new Error(`Expected a non-null object`); + } + + const config = { ...DEFAULT_OPTIONS, ...options }; + const { startMarker, endMarker, filePath, systemPrompt, model, jsonSchema } = + config; + + const hasAllRequiredOptions = + filePath && startMarker && endMarker && systemPrompt && model && jsonSchema; + + // If no filepath is provided, throw an error + if (!hasAllRequiredOptions) { + throw new Error("Missing required options"); + } + + const fileToUpdatePath = path.join(__dirname, filePath); + + try { + const data = await readFile(fileToUpdatePath, "utf8"); + + const { + object: { mermaidDiagramString }, + } = await generateObject({ + model, + system: systemPrompt, + prompt: `Here is the JSON schema: ${JSON.stringify(jsonSchema)}`, + schema: z.object({ + mermaidDiagramString: z + .string() + .describe( + "Contains the created diagram strictly following the mermaid syntax." + ), + }), + }); + + const diagramContent = `\`\`\`mermaid\n${mermaidDiagramString}\n\`\`\`\n`; + + const regex = new RegExp(`${startMarker}[\\s\\S]*?${endMarker}`, "g"); + + if (!regex.test(data)) { + console.error(`Markers not found in ${filePath}.`); + return; + } + + const newContent = data.replace( + regex, + `${startMarker}\n${diagramContent}${endMarker}` + ); + + await writeFile(fileToUpdatePath, newContent, "utf8"); + console.log(`Diagram successfully updated in ${filePath}`); + } catch (err) { + console.error(`Error processing ${filePath}:`, err); + } +} diff --git a/index.test.js b/index.test.js new file mode 100644 index 0000000..6c260b7 --- /dev/null +++ b/index.test.js @@ -0,0 +1,134 @@ +import { describe, beforeEach, it, expect, vi, afterEach } from "vitest"; +import path from "path"; +import { fs, vol } from "memfs"; + +vi.mock("node:fs"); +vi.mock("node:fs/promises"); + +describe("jsonSchemaToDiagram", () => { + let jsonSchemaToDiagram; + const startMarker = ""; + const endMarker = ""; + + beforeEach(async () => { + vi.restoreAllMocks(); + process.env.OPENAI_API_KEY = "test-api-key"; + vol.reset(); + + vol.fromJSON({ + "./README.md": + "\nOld content\n", + }); + + vi.mock("ai", () => ({ + __esModule: true, + generateObject: vi.fn().mockResolvedValue({ + object: { mermaidDiagramString: "graph TD; A-->B;" }, + }), + })); + + const module = await import("./index.js"); + jsonSchemaToDiagram = module.default; + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("throws error if OPENAI_API_KEY is not set", async () => { + delete process.env.OPENAI_API_KEY; + await expect(jsonSchemaToDiagram({})).rejects.toThrow( + "OPENAI_API_KEY is not set" + ); + }); + + it("throws error if options is not an object", async () => { + await expect(jsonSchemaToDiagram(null)).rejects.toThrow( + "Expected a non-null object" + ); + }); + + it("throws error if required options are missing", async () => { + await expect(jsonSchemaToDiagram({})).rejects.toThrow( + "Missing required options" + ); + }); + + it.todo("updates file with generated mermaid diagram", async () => { + const filePath = "./README.md"; + const systemPrompt = "test system prompt"; + const model = "test model"; + const jsonSchema = { tools: [] }; + const mermaidDiagramString = "graph TD; A-->B;"; + + vi.spyOn(fs, "writeFile").mockImplementation(() => { + return Promise.resolve(); + }); + + await jsonSchemaToDiagram({ + filePath, + startMarker, + endMarker, + systemPrompt, + model, + jsonSchema, + }); + + const expectedContent = `${startMarker}\n\`\`\`mermaid\n${mermaidDiagramString}\n\`\`\`\n${endMarker}`; + expect(fs.writeFile).toHaveBeenCalledWith( + path.join(__dirname, filePath), + expectedContent, + "utf8" + ); + }); + + it.todo("logs error if markers are not found in file", async () => { + const filePath = "./README.md"; + const systemPrompt = "test system prompt"; + const model = "test model"; + const jsonSchema = { tools: [] }; + + vol.fromJSON({ + "./README.md": "No markers here", + }); + + vi.spyOn(console, "error").mockImplementation(() => {}); + + await jsonSchemaToDiagram({ + filePath, + startMarker, + endMarker, + systemPrompt, + model, + jsonSchema, + }); + + expect(console.error).toHaveBeenCalledWith( + `Markers not found in ${filePath}:`, + expect.any(Error) + ); + }); + + it("logs error if there is an error processing the file", async () => { + const filePath = "./README.md"; + const systemPrompt = "test system prompt"; + const model = "test model"; + const jsonSchema = { tools: [] }; + + vi.spyOn(console, "error").mockImplementation(() => {}); + + await jsonSchemaToDiagram({ + filePath, + startMarker, + endMarker, + systemPrompt, + model, + jsonSchema, + }); + + expect(console.error).toHaveBeenCalledWith( + `Error processing ${filePath}:`, + expect.any(Error) + ); + }); +}); diff --git a/package.json b/package.json new file mode 100644 index 0000000..20a4e97 --- /dev/null +++ b/package.json @@ -0,0 +1,68 @@ +{ + "name": "json-schema-to-diagram", + "version": "0.0.0", + "description": "Generate Mermaid diagrams from JSON schemas e.g., for function calling tools.", + "license": "MIT", + "repository": "tobiasbueschel/json-schema-to-diagram", + "funding": "https://github.com/sponsors/tobiasbueschel", + "author": { + "name": "Tobias Büschel", + "url": "https://github.com/tobiasbueschel" + }, + "type": "module", + "exports": { + "types": "./index.d.ts", + "default": "./index.js" + }, + "sideEffects": false, + "engines": { + "node": ">=18" + }, + "scripts": { + "test": "vitest" + }, + "files": [ + "index.js", + "index.d.ts" + ], + "keywords": [ + "jsonschema", + "mermaid", + "diagram", + "openai", + "function calling", + "ai" + ], + "dependencies": { + "@ai-sdk/openai": "^1.0.5", + "ai": "^4.0.9" + }, + "devDependencies": { + "@commitlint/config-conventional": "^19.6.0", + "@semantic-release/changelog": "^6.0.3", + "@semantic-release/git": "^10.0.1", + "@semantic-release/github": "^11.0.1", + "@semantic-release/npm": "^12.0.1", + "@semantic-release/release-notes-generator": "^14.0.1", + "memfs": "^4.14.0", + "semantic-release": "^24.2.0", + "typescript": "^5.5.4", + "vitest": "^2.1.6", + "zod-to-json-schema": "^3.23.5" + }, + "release": { + "plugins": [ + "@semantic-release/commit-analyzer", + "@semantic-release/release-notes-generator", + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + [ + "@semantic-release/git", + { + "message": "chore: release ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}" + } + ] + ] + } +} diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..b00d993 --- /dev/null +++ b/readme.md @@ -0,0 +1,111 @@ +# json-schema-to-diagram + +> 🧜‍♀️ Generate Mermaid diagrams from JSON schemas of e.g., function calling tools. + +This is a simple tool that uses OpenAI's LLM to generate a Mermaid diagram from a JSON schema of tools. It can be used to visualize your agentic architecture through a diagram that shows all your tools and their parameters. + +You can for example use this tool to generate a diagram and append it to your README.md or use it in your project's documentation. + +## Example Diagram + +Here's how a Mermaid diagram generated from a JSON schema of [OpenAI function calling tools](https://platform.openai.com/docs/guides/function-calling) looks like: + +```mermaid +flowchart LR + A["Data Tools"] + A --> |"Retrieve real-time stock market data for a company"| B["**getStockData**"] + B --> C["companySymbol"] + A --> |"Upload user-specific data to a storage system"| D["**uploadData**"] + D --> E["data
userId"] + A --> |"Extract the transcript of a YouTube video"| F["**getYouTubeTranscript**"] + F --> G["videoId"] + A --> |"Perform web crawling to gather relevant information"| H["**crawlWebsite**"] + H --> I["url"] + + J["Communication Tools"] + J --> |"Send an automated email to the desired recipient"| K["**sendEmail**"] + K --> L["recipient
subject
body"] + J --> |"Take a screenshot of the current application window"| M["**takeScreenshot**"] + M --> N["windowId"] + + O["Development Tools"] + O --> |"Create a React component based on the given specifications"| P["**generateReactComponent**"] + P --> Q["specifications"] + O --> |"Develop comprehensive documentation for a codebase"| R["**generateCodeDocumentation**"] + R --> S["codebaseId"] + O --> |"Write a unit test for a specified function"| T["**generateUnitTest**"] + T --> U["functionName"] + O --> |"Create an integration test for a module"| V["**generateIntegrationTest**"] + V --> W["moduleId"] +``` + +## Install + +```sh +npm install --save-dev json-schema-to-diagram +``` + +## Usage + +```js +import jsonSchemaToDiagram from "json-schema-to-diagram"; + +const TOOLS = { + // JSON schema of your function calling tools + // For example your OpenAI function calling tools +}; + +// Append the diagram to your README.md file +(async () => { + try { + await jsonSchemaToDiagram({ + filePath: "./README.md", + jsonSchema: TOOLS, + }); + console.log("Diagram appended successfully."); + } catch (error) { + console.error("Error appending diagram:", error); + process.exit(1); + } +})(); +``` + +## API + +### generateDiagram(options) + +#### options.filePath + +Type: `string` + +The path to the file to update (default: `./README.md`). + +#### options.startMarker + +Type: `string` + +The start marker of the diagram in the file (default: ``). + +#### options.endMarker + +Type: `string` + +The end marker of the diagram in the file (default: ``). + +#### options.systemPrompt + +Type: `string` + +The system prompt for the LLM (default: `You are a helpful assistant that generates mermaid diagrams based on a JSON schema of tools.`). + +#### options.model + +Type: `Model` + +The LLM model to use (default: `openai("gpt-4o")`). This uses [ai](https://github.com/vercel/ai) under the hood. + +#### options.jsonSchema + +Type: `string` + +The JSON schema of the tools.