Add run stats endpoints (#890)

.github/workflows/js_test.yml

@@ -81,7 +81,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        node-version: [18.x, 19.x, 20.x, 21.x, 22.x]
+        node-version: [18.x, 20.x, "22.4.1"]
         # See Node.js release schedule at https://nodejs.org/en/about/releases/
         include:
           - os: windows-latest
@@ -107,4 +107,4 @@ jobs:
       - name: Check version
         run: yarn run check-version
       - name: Test
-        run: yarn run test
+        run: yarn run test

js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.1.38",
+  "version": "0.1.39",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [
@@ -261,4 +261,4 @@
     },
     "./package.json": "./package.json"
   }
-}
+}

js/src/client.ts

@@ -1229,6 +1229,94 @@ export class Client {
     }
   }
 
+  public async getRunStats({
+    id,
+    trace,
+    parentRun,
+    runType,
+    projectNames,
+    projectIds,
+    referenceExampleIds,
+    startTime,
+    endTime,
+    error,
+    query,
+    filter,
+    traceFilter,
+    treeFilter,
+    isRoot,
+    dataSourceType,
+  }: {
+    id?: string[];
+    trace?: string;
+    parentRun?: string;
+    runType?: string;
+    projectNames?: string[];
+    projectIds?: string[];
+    referenceExampleIds?: string[];
+    startTime?: string;
+    endTime?: string;
+    error?: boolean;
+    query?: string;
+    filter?: string;
+    traceFilter?: string;
+    treeFilter?: string;
+    isRoot?: boolean;
+    dataSourceType?: string;
+  }): Promise<any> {
+    let projectIds_ = projectIds || [];
+    if (projectNames) {
+      projectIds_ = [
+        ...(projectIds || []),
+        ...(await Promise.all(
+          projectNames.map((name) =>
+            this.readProject({ projectName: name }).then(
+              (project) => project.id
+            )
+          )
+        )),
+      ];
+    }
+
+    const payload = {
+      id,
+      trace,
+      parent_run: parentRun,
+      run_type: runType,
+      session: projectIds_,
+      reference_example: referenceExampleIds,
+      start_time: startTime,
+      end_time: endTime,
+      error,
+      query,
+      filter,
+      trace_filter: traceFilter,
+      tree_filter: treeFilter,
+      is_root: isRoot,
+      data_source_type: dataSourceType,
+    };
+
+    // Remove undefined values from the payload
+    const filteredPayload = Object.fromEntries(
+      Object.entries(payload).filter(([_, value]) => value !== undefined)
+    );
+
+    const response = await this.caller.call(
+      fetch,
+      `${this.apiUrl}/runs/stats`,
+      {
+        method: "POST",
+        headers: this.headers,
+        body: JSON.stringify(filteredPayload),
+        signal: AbortSignal.timeout(this.timeout_ms),
+        ...this.fetchOptions,
+      }
+    );
+
+    const result = await response.json();
+    return result;
+  }
+
   public async shareRun(
     runId: string,
     { shareId }: { shareId?: string } = {}

js/src/index.ts

@@ -12,4 +12,4 @@ export type {
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
 
 // Update using yarn bump-version
-export const __version__ = "0.1.38";
+export const __version__ = "0.1.39";

js/src/tests/client.int.test.ts

@@ -739,3 +739,12 @@ test.concurrent("list runs limit arg works", async () => {
     }
   }
 });
+
+test.concurrent("Test run stats", async () => {
+  const client = new Client();
+  const stats = await client.getRunStats({
+    projectNames: ["default"],
+    runType: "llm",
+  });
+  expect(stats).toBeDefined();
+});

python/langsmith/client.py

@@ -1768,6 +1768,93 @@ def list_runs(
             if limit is not None and i + 1 >= limit:
                 break
 
+    def get_run_stats(
+        self,
+        *,
+        id: Optional[List[ID_TYPE]] = None,
+        trace: Optional[ID_TYPE] = None,
+        parent_run: Optional[ID_TYPE] = None,
+        run_type: Optional[str] = None,
+        project_names: Optional[List[str]] = None,
+        project_ids: Optional[List[ID_TYPE]] = None,
+        reference_example_ids: Optional[List[ID_TYPE]] = None,
+        start_time: Optional[str] = None,
+        end_time: Optional[str] = None,
+        error: Optional[bool] = None,
+        query: Optional[str] = None,
+        filter: Optional[str] = None,
+        trace_filter: Optional[str] = None,
+        tree_filter: Optional[str] = None,
+        is_root: Optional[bool] = None,
+        data_source_type: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Get aggregate statistics over queried runs.
+
+        Takes in similar query parameters to `list_runs` and returns statistics
+        based on the runs that match the query.
+
+        Args:
+            id (Optional[List[ID_TYPE]]): List of run IDs to filter by.
+            trace (Optional[ID_TYPE]): Trace ID to filter by.
+            parent_run (Optional[ID_TYPE]): Parent run ID to filter by.
+            run_type (Optional[str]): Run type to filter by.
+            projects (Optional[List[ID_TYPE]]): List of session IDs to filter by.
+            reference_example (Optional[List[ID_TYPE]]): List of reference example IDs to filter by.
+            start_time (Optional[str]): Start time to filter by.
+            end_time (Optional[str]): End time to filter by.
+            error (Optional[bool]): Filter by error status.
+            query (Optional[str]): Query string to filter by.
+            filter (Optional[str]): Filter string to apply.
+            trace_filter (Optional[str]): Trace filter string to apply.
+            tree_filter (Optional[str]): Tree filter string to apply.
+            is_root (Optional[bool]): Filter by root run status.
+            data_source_type (Optional[str]): Data source type to filter by.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the run statistics.
+        """  # noqa: E501
+        from concurrent.futures import ThreadPoolExecutor, as_completed  # type: ignore
+
+        project_ids = project_ids or []
+        if project_names:
+            with ThreadPoolExecutor() as executor:
+                futures = [
+                    executor.submit(self.read_project, project_name=name)
+                    for name in project_names
+                ]
+                for future in as_completed(futures):
+                    project_ids.append(future.result().id)
+        payload = {
+            "id": id,
+            "trace": trace,
+            "parent_run": parent_run,
+            "run_type": run_type,
+            "session": project_ids,
+            "reference_example": reference_example_ids,
+            "start_time": start_time,
+            "end_time": end_time,
+            "error": error,
+            "query": query,
+            "filter": filter,
+            "trace_filter": trace_filter,
+            "tree_filter": tree_filter,
+            "is_root": is_root,
+            "data_source_type": data_source_type,
+        }
+
+        # Remove None values from the payload
+        payload = {k: v for k, v in payload.items() if v is not None}
+
+        response = self.request_with_retries(
+            "POST",
+            "/runs/stats",
+            request_kwargs={
+                "data": _dumps_json(payload),
+            },
+        )
+        ls_utils.raise_for_status_with_text(response)
+        return response.json()
+
     def get_run_url(
         self,
         *,
@@ -1777,6 +1864,10 @@ def get_run_url(
     ) -> str:
         """Get the URL for a run.
 
+        Not recommended for use within your agent runtime.
+        More for use interacting with runs after the fact
+        for data analysis or ETL workloads.
+
         Parameters
         ----------
         run : Run

python/tests/integration_tests/test_client.py

@@ -698,3 +698,10 @@ def test_surrogates():
         run_type="llm",
         end_time=datetime.datetime.now(datetime.timezone.utc),
     )
+
+
+def test_runs_stats():
+    langchain_client = Client()
+    # We always have stuff in the "default" project...
+    stats = langchain_client.get_run_stats(project_names=["default"], run_type="llm")
+    assert stats