App routes file source tree using code analysis

api/src/index.node.ts

@@ -14,10 +14,9 @@ import {
   USER_PROJECT_ROOT_DIR,
 } from "./constants.js";
 import * as schema from "./db/schema.js";
-import { hasValidAiConfig } from "./lib/ai/index.js";
 import { getAuthServer } from "./lib/fp-services/server.js";
 import { setupRealtimeService } from "./lib/realtime/index.js";
-import { getInferenceConfig, getSetting } from "./lib/settings/index.js";
+import { getSetting } from "./lib/settings/index.js";
 import { resolveWebhoncUrl } from "./lib/utils.js";
 import * as webhonc from "./lib/webhonc/index.js";
 import logger from "./logger/index.js";
@@ -114,10 +113,4 @@ if (proxyRequestsEnabled ?? false) {
   await webhonc.start();
 }
 
-// check settings if ai is enabled, and proactively start the typescript language server
-const inferenceConfig = await getInferenceConfig(db);
-const aiEnabled = hasValidAiConfig(inferenceConfig);
-
-if (aiEnabled) {
-  enableCodeAnalysis();
-}
+enableCodeAnalysis();

api/src/lib/app-routes.ts → api/src/lib/app-routes/index.ts

@@ -1,7 +1,12 @@
 import { and, eq } from "drizzle-orm";
 import type { LibSQLDatabase } from "drizzle-orm/libsql";
 import { z } from "zod";
-import * as schema from "../db/schema.js";
+import * as schema from "../../db/schema.js";
+import type {
+  AppRouteWithFileName,
+  AppRoutesTreeResponse,
+  TreeNode,
+} from "./types.js";
 
 const { appRoutes } = schema;
 
@@ -82,3 +87,71 @@ export async function reregisterRoutes(
     }
   });
 }
+/**
+ * Builds a route tree structure for a file tree browser in the UI.
+ *
+ * This function processes an array of route entries and organizes them into a tree
+ * structure based on their file paths. Routes without a `fileName` are added to an
+ * "unmatched" list. The resulting tree is optimized to consolidate nodes with a single
+ * child into a single path segment.
+ */
+export function buildRouteTree(
+  entries: Array<AppRouteWithFileName>,
+): AppRoutesTreeResponse {
+  const tree: Array<TreeNode> = [];
+  const unmatched: Array<schema.AppRoute> = [];
+
+  for (const entry of entries) {
+    if (!entry.fileName) {
+      unmatched.push(entry);
+    } else {
+      const filePathParts = entry.fileName.split("/");
+      let currentNodeArray = tree;
+
+      for (const [index, part] of filePathParts.entries()) {
+        let childNode = currentNodeArray.find((child) => child.path === part);
+
+        if (!childNode) {
+          childNode = { path: part, routes: [], children: [] };
+          currentNodeArray.push(childNode);
+        }
+
+        if (index === filePathParts.length - 1) {
+          childNode.routes.push(entry);
+        }
+
+        currentNodeArray = childNode.children;
+      }
+    }
+  }
+
+  return { tree: consolidateSingleChildNodes(tree), unmatched };
+}
+
+/**
+ * Optimizes the structure of a route tree by consolidating nodes with a single child.
+ *
+ * This function traverses the tree and combines consecutive nodes with a single child
+ * into a single node with a concatenated path. It is used to simplify the visual
+ * representation of the file tree in the UI.
+ */
+function consolidateSingleChildNodes(nodes: Array<TreeNode>): Array<TreeNode> {
+  return nodes.map((node) => {
+    if (node.children.length === 1) {
+      const child = node.children[0];
+
+      return consolidateSingleChildNodes([
+        {
+          path: `${node.path}/${child.path}`,
+          routes: [...node.routes, ...child.routes],
+          children: child.children,
+        },
+      ])[0];
+    }
+
+    return {
+      ...node,
+      children: consolidateSingleChildNodes(node.children),
+    };
+  });
+}

api/src/lib/app-routes/types.ts

@@ -0,0 +1,16 @@
+import type { AppRoute } from "../../db/schema.js";
+
+export type AppRoutesTreeResponse = {
+  tree: Array<TreeNode>;
+  unmatched: Array<AppRoute>;
+};
+
+export type AppRouteWithFileName = AppRoute & {
+  fileName?: string;
+};
+
+export type TreeNode = {
+  path: string;
+  routes: Array<AppRouteWithFileName>;
+  children: Array<TreeNode>;
+};

api/src/routes/app-routes.ts

@@ -3,14 +3,21 @@ import { and, eq } from "drizzle-orm";
 import { type Context, Hono } from "hono";
 import { env } from "hono/adapter";
 import { z } from "zod";
+import type { RouteEntryId } from "../../../packages/source-analysis/dist/types.js";
 import {
   type NewAppRequest,
   appRequests,
   appResponses,
   appRoutes,
   appRoutesInsertSchema,
 } from "../db/schema.js";
-import { reregisterRoutes, schemaProbedRoutes } from "../lib/app-routes.js";
+import {
+  buildRouteTree,
+  reregisterRoutes,
+  schemaProbedRoutes,
+} from "../lib/app-routes/index.js";
+import type { AppRouteWithFileName } from "../lib/app-routes/types.js";
+import { getResult } from "../lib/code-analysis.js";
 import {
   OTEL_TRACE_ID_REGEX,
   generateOtelTraceId,
@@ -41,7 +48,7 @@ const app = new Hono<{ Bindings: Bindings; Variables: Variables }>();
 
 app.get("/v0/app-routes", async (ctx) => {
   const db = ctx.get("db");
-  const routes = await db.select().from(appRoutes);
+  const routes = await db.query.appRoutes.findMany();
   const baseUrl = resolveServiceArg(
     env(ctx).FPX_SERVICE_TARGET as string,
     "http://localhost:8787",
@@ -52,6 +59,54 @@ app.get("/v0/app-routes", async (ctx) => {
   });
 });
 
+app.get("/v0/app-routes-file-tree", async (ctx) => {
+  const db = ctx.get("db");
+  const routes = await db.query.appRoutes.findMany({
+    where: (appRoutes, { and, eq }) =>
+      and(
+        eq(appRoutes.currentlyRegistered, true),
+        eq(appRoutes.handlerType, "route"),
+        eq(appRoutes.routeOrigin, "discovered"),
+      ),
+  });
+
+  const result = await getResult();
+
+  const routeEntries = [];
+  for (const currentRoute of routes) {
+    const url = new URL("http://localhost");
+    url.pathname = currentRoute.path ?? "";
+    const request = new Request(url, {
+      method: currentRoute.method ?? "",
+    });
+    result.resetHistory();
+    const response = await result.currentApp.fetch(request);
+    const responseText = await response.text();
+
+    if (responseText !== "Ok") {
+      logger.warn("Failed to fetch route for context expansion", responseText);
+      continue;
+    }
+
+    const history = result.getHistory();
+    const routeEntryId = history[history.length - 1];
+    const routeEntry = result.getRouteEntryById(routeEntryId as RouteEntryId);
+
+    routeEntries.push({
+      ...currentRoute,
+      fileName: routeEntry?.fileName,
+    });
+  }
+
+  const tree = buildRouteTree(
+    routeEntries.filter(
+      (route) => route?.fileName !== undefined,
+    ) as Array<AppRouteWithFileName>,
+  );
+
+  return ctx.json(tree);
+});
+
 /**
  * Allow users to manually refresh the app routes list
  */

packages/source-analysis/README.md

@@ -2,35 +2,35 @@
 
 This package is intended for finding the hono routes/app instances in a give codebase. This was initially created to provide context for requests to an LLM. In studio you can use an LLM to generate parameters for your request. However there is more that studio can do with this knowledge, things like providing links to the source code of an endpoint, grouping routes based on location in the source code and more.
 
-This package uses the user's typescript library and configuration (but falls back to the bundled typescript version and the default compiler configuration). Both: the typescript language service as well as the compiler API is being used. The language service is used to find references within the code base and the other for getting the AST tree, access the type checker, etc. 
+This package uses the user's typescript library and configuration (but falls back to the bundled typescript version and the default compiler configuration). Both: the typescript language service as well as the compiler API is being used. The language service is used to find references within the code base and the other for getting the AST tree, access the type checker, etc.
 
 ## Known limitations
 
 Given the current approach, the project this package is trying to analyze should:
 
-* use typescript
-* use standard typescript file extensions (right now only `.ts` and `.tsx` is used)
+- use typescript
+- use standard typescript file extensions (right now only `.ts` and `.tsx` is used)
 
 ## Goal of analyzing the source code
 
-The goal is to find all routes of a users' hono application, knowing that we'd like to find out which code is actually used for that. This way we can provide an LLM with context around what the application accepts/uses (like database schema, zod validation schemas, etc). 
+The goal is to find all routes of a users' hono application, knowing that we'd like to find out which code is actually used for that. This way we can provide an LLM with context around what the application accepts/uses (like database schema, zod validation schemas, etc).
 
 ## Usage
 
 The typical way of using it is as follows:
 
-``` ts
+```ts
 import { createRoutesMonitor } from "@fiberplane/source-analysis"
 
 const location = process.cwd();
 const monitor = createRoutesMonitor(location);
 monitor.start()
   .then(() => {
-    // All files have been found 
+    // All files have been found
     console.log('ready');
 
     // By default the monitor will immediately start to analyze the code
-    // and it will be made available via the `lastSuccessfulResult` 
+    // and it will be made available via the `lastSuccessfulResult`
     console.log(monitor.lastSuccessfulResult);
 
     // Force a manual update:
@@ -50,9 +50,9 @@ monitor.start()
   })
 ```
 
-The result of a successful analysis is a RoutesResult. This is an instance of a class that can be used to get a hono app which mimics all routes that have been found. When the hono instance receives a request, the `RoutesResult` instance will be updated with which routes have been called. You can then call the `getFilesForHistory` method on the `RoutesResult` instance. This will generate a string containing (most) of the code that could be involved if that request was made to the actual application. 
+The result of a successful analysis is a RoutesResult. This is an instance of a class that can be used to get a hono app which mimics all routes that have been found. When the hono instance receives a request, the `RoutesResult` instance will be updated with which routes have been called. You can then call the `getFilesForHistory` method on the `RoutesResult` instance. This will generate a string containing (most) of the code that could be involved if that request was made to the actual application.
 
-``` ts
+```ts
 /* index.ts */
 import { cors } from "hono/cors";
 import { getProfile as getUserProfile } from "./db";
@@ -96,27 +96,24 @@ export async function getProfile() {
 
 As can be seen above it concatenates all files into a single file with some comments around the actually involved filenames.
 
-
 ## How it works
 
-
 There are several strategies that we've been thinking about, one approach is to specify the main entry of an application, find the hono app in that file and go through the source code from there. The downside to this, is that the entry file would need to be specified (or found out by our code base).
 An alternative approach is to analyze all source code and find variables that are of the `Hono<` generic type and use some heuristics to determine the entry point/main app. This is the strategy this package uses.
 On a high level what needs to happen is as follows:
-1. extracting routes. Go through all code and map all apps, routes, middleware and links between apps. Also keep track of what code is related to the app/route/middleware and keep track of what code that code might refer to (etc). This data is stored in the `ResourceManager`. 
+
+1. extracting routes. Go through all code and map all apps, routes, middleware and links between apps. Also keep track of what code is related to the app/route/middleware and keep track of what code that code might refer to (etc). This data is stored in the `ResourceManager`.
 2. analyzing routes. Once all code has been converted into our own data structure, we find the hono apps and see which one has the most routes/entries (and if routes refer to each other their value is added as well). The app with the highest number is treated as the entry point.
 3. the intermediate result: `RoutesResult`. This contains:
 
-    *  a reference to a hono app which can be used to find out what code is executed for a given method/request 
-    * `getFilesForHistory()` method that can be called to see all code that can be executed for a request to an endpoint.
-    * `resetHistory()` method so you can reset the result to the initial state.
-
+   - a reference to a hono app which can be used to find out what code is executed for a given method/request
+   - `getFilesForHistory()` method that can be called to see all code that can be executed for a request to an endpoint.
+   - `resetHistory()` method so you can reset the result to the initial state.
 
 ### Extracting routes, the data structures
 
 The `@fiberplane/source-analysis` package uses several key data structures to represent the elements of your TypeScript project. Understanding these data structures can be helpful if you want to work in this package.
 
-
 #### `RouteTree`
 
 The `RouteTree` type represents a reference to an `app` instance created with `new Hono()`. It contains a list of `entries`, which includes information about calls to the `app` (such as `app.get()`, `app.use()`, and `app.route()`).
@@ -146,16 +143,16 @@ export type RouteTreeEntry = RouteEntry | RouteTreeReference | MiddlewareEntry;
 
 In order to capture/store information about other code the packages uses two other data structures:
 
-- `SourceReference` This is a reference to section of code (like a function, a constant, etc).  A source reference can refer to one or more `SourceReference` as well as one or more `ModuleReferences`
-- A `ModuleReference` represents a link to another file/external package and is source code (part of) an import statement. 
+- `SourceReference` This is a reference to section of code (like a function, a constant, etc). A source reference can refer to one or more `SourceReference` as well as one or more `ModuleReferences`
+- A `ModuleReference` represents a link to another file/external package and is source code (part of) an import statement.
 
 All data structures (apart from `ModuleReference` can contain references to either modules or source references) . All resources are stored using their id in a `Map` in the ResourceManager. Although `ID`s are basically strings -which always start with the basic type like `ROUTE_TREE`, `SOURCE_REFERENCE`, etc - in typescript land they are made specific using [type tagging](https://medium.com/@KevinBGreene/surviving-the-typescript-ecosystem-branding-and-type-tagging-6cf6e516523d) (using type-fest). This way typescript is aware what kind of a resource is being returned when calling `ResourceManager.getResource`.
 
 ## Extracting the code for a route
 
-After creating the resources based on the source code, the main entry endpoint is determined. That main entry point is then used in the `RoutesResult` to set up a separate Hono app. This hono app is never actually listening on a port but is used to send requests to.  The reconstructed app could be written out something like this:
+After creating the resources based on the source code, the main entry endpoint is determined. That main entry point is then used in the `RoutesResult` to set up a separate Hono app. This hono app is never actually listening on a port but is used to send requests to. The reconstructed app could be written out something like this:
 
-``` ts
+```ts
 type HistoryId =
   | RouteEntryId
   | RouteTreeId
@@ -187,7 +184,7 @@ Now, if the `routingApp` handles a request, the history array will contain all I
 
 ## Performance
 
-On smaller projects, the source code analysis is quite fast, it might take even less than 100ms. 
+On smaller projects, the source code analysis is quite fast, it might take even less than 100ms.
 However on larger code bases it some calls can suddenly become slow/expensive. One such call is `getProgram`, it's a method on the language service and returns a compiler instance. Even on tiny projects this call can take ~50ms, so the approach taken here is to call `getProgram` as few times as possible and as early as possible. Another thing is the `tsconfig.json` has quite a significant performance impact, especially specifying types using `compilerOptions.types`. Adding `node` types for the simplest project in the test suite added 5ms to the test (bumping it to around 50ms). This brings us to another optimization that was done for larger projects: caching results for `readFile`, `fileExists` and `directoryExists`. These calls happen a lot, while typescript is trying to figure out for instance whether a package exists and where.
 So in case of this package, we cache the results of this calls while parsing the source code and reset it when files change on the filesystem.
 
@@ -197,7 +194,7 @@ It would be nice if we could do things like incremental analysis, right now the
 
 ## Open issues/known limitations:
 
-* Test if path aliases (`compilerOptions.paths`) actually work (https://www.typescriptlang.org/tsconfig/#paths)
-* ensure export statements are included in the generated result. Right now for instance apps are found, but we don't really check if/how they are exported, cases where a variable declared and the export is done elsewhere are probably not covered.
-* classes have not been thoroughly tested. This means it could break code 
-* `.on` calls aren't being handled
+- Test if path aliases (`compilerOptions.paths`) actually work (https://www.typescriptlang.org/tsconfig/#paths)
+- ensure export statements are included in the generated result. Right now for instance apps are found, but we don't really check if/how they are exported, cases where a variable declared and the export is done elsewhere are probably not covered.
+- classes have not been thoroughly tested. This means it could break code
+- `.on` calls aren't being handled

packages/source-analysis/src/RoutesResult/RoutesResult.ts

@@ -4,6 +4,7 @@ import type { ResourceManager } from "../ResourceManager";
 import { logger } from "../logger";
 import type {
   MiddlewareEntryId,
+  RouteEntry,
   RouteEntryId,
   RouteTree,
   RouteTreeId,
@@ -191,4 +192,8 @@ export class RoutesResult {
   public getFilesForHistory() {
     return generate(this._resourceManager, this._history, this._includeIds);
   }
+
+  public getRouteEntryById(id: RouteEntryId): RouteEntry | undefined {
+    return this._resourceManager.getResource(id);
+  }
 }