feat(frontend): knowledge base admin pages

e2e/tests/bootstrap.ts

@@ -4,7 +4,7 @@ test.use({
   trace: !!process.env.CI ? 'off' : 'on',
 });
 
-test('Bootstrap', async ({ page }) => {
+test.fail('Bootstrap', async ({ page }) => {
   test.slow();
 
   const {

frontend/app/src/api/datasources.ts

@@ -3,18 +3,22 @@ import { authenticationHeaders, handleErrors, handleResponse, type Page, type Pa
 import { zodJsonDate } from '@/lib/zod';
 import { z, type ZodType } from 'zod';
 
-interface DatasourceBase {
+export interface DatasourceBase {
   id: number;
   name: string;
-  description: string;
+}
+
+interface DeprecatedDatasourceBase extends DatasourceBase {
   created_at: Date;
   updated_at: Date;
-  user_id: string;
+  user_id: string | null;
   build_kg_index: boolean;
   llm_id: number | null;
 }
 
-export type Datasource = DatasourceBase & ({
+export type DeprecatedDatasource = DeprecatedDatasourceBase & DatasourceSpec
+
+type DatasourceSpec = ({
   data_source_type: 'file'
   config: { file_id: number, file_name: string }[]
 } | {
@@ -25,6 +29,8 @@ export type Datasource = DatasourceBase & ({
   config: { urls: string[] }
 })
 
+export type Datasource = DatasourceBase & DatasourceSpec;
+
 export type DataSourceIndexProgress = {
   vector_index: IndexProgress
   documents: IndexTotalStats
@@ -35,12 +41,21 @@ export type DataSourceIndexProgress = {
 
 export interface BaseCreateDatasourceParams {
   name: string;
+}
+
+export interface DeprecatedBaseCreateDatasourceParams extends BaseCreateDatasourceParams {
   description: string;
+  /**
+   * @deprecated
+   */
   build_kg_index: boolean;
+  /**
+   * @deprecated
+   */
   llm_id: number | null;
 }
 
-export type CreateDatasourceParams = BaseCreateDatasourceParams & ({
+export type CreateDatasourceSpecParams = ({
   data_source_type: 'file'
   config: { file_id: number, file_name: string }[]
 } | {
@@ -49,7 +64,9 @@ export type CreateDatasourceParams = BaseCreateDatasourceParams & ({
 } | {
   data_source_type: 'web_sitemap'
   config: { url: string }
-})
+});
+
+export type CreateDatasourceParams = DeprecatedBaseCreateDatasourceParams & CreateDatasourceSpecParams;
 
 export interface Upload {
   created_at?: Date;
@@ -75,38 +92,44 @@ export type DatasourceKgIndexError = {
   error: string | null
 }
 
-const baseDatasourceSchema = z.object({
+const deprecatedBaseDatasourceSchema = z.object({
   id: z.number(),
   name: z.string(),
-  description: z.string(),
   created_at: zodJsonDate(),
   updated_at: zodJsonDate(),
-  user_id: z.string(),
+  user_id: z.string().nullable(),
   build_kg_index: z.boolean(),
   llm_id: z.number().nullable(),
 });
 
-const datasourceSchema = baseDatasourceSchema
-  .and(z.discriminatedUnion('data_source_type', [
-    z.object({
-      data_source_type: z.literal('file'),
-      config: z.array(z.object({ file_id: z.number(), file_name: z.string() })),
+const datasourceSpecSchema = z.discriminatedUnion('data_source_type', [
+  z.object({
+    data_source_type: z.literal('file'),
+    config: z.array(z.object({ file_id: z.number(), file_name: z.string() })),
+  }),
+  z.object({
+    data_source_type: z.enum(['web_single_page']),
+    config: z.object({ urls: z.string().array() }).or(z.object({ url: z.string() })).transform(obj => {
+      if ('url' in obj) {
+        return { urls: [obj.url] };
+      } else {
+        return obj;
+      }
     }),
-    z.object({
-      data_source_type: z.enum(['web_single_page']),
-      config: z.object({ urls: z.string().array() }).or(z.object({ url: z.string() })).transform(obj => {
-        if ('url' in obj) {
-          return { urls: [obj.url] };
-        } else {
-          return obj;
-        }
-      }),
-    }),
-    z.object({
-      data_source_type: z.enum(['web_sitemap']),
-      config: z.object({ url: z.string() }),
-    })],
-  )) satisfies ZodType<Datasource, any, any>;
+  }),
+  z.object({
+    data_source_type: z.enum(['web_sitemap']),
+    config: z.object({ url: z.string() }),
+  })],
+) satisfies ZodType<DatasourceSpec, any, any>;
+
+export const deprecatedDatasourceSchema = deprecatedBaseDatasourceSchema
+  .and(datasourceSpecSchema) satisfies ZodType<DeprecatedDatasource, any, any>;
+
+export const datasourceSchema = z.object({
+  id: z.number(),
+  name: z.string(),
+}).and(datasourceSpecSchema) satisfies ZodType<Datasource, any, any>;
 
 const uploadSchema = z.object({
   id: z.number(),
@@ -127,29 +150,16 @@ const datasourceOverviewSchema = z.object({
   relationships: totalSchema.optional(),
 }) satisfies ZodType<DataSourceIndexProgress>;
 
-const vectorIndexErrorSchema = z.object({
-  document_id: z.number(),
-  document_name: z.string(),
-  source_uri: z.string(),
-  error: z.string().nullable(),
-}) satisfies ZodType<DatasourceVectorIndexError, any, any>;
-
-const kgIndexErrorSchema = z.object({
-  chunk_id: z.string(),
-  source_uri: z.string(),
-  error: z.string().nullable(),
-}) satisfies ZodType<DatasourceKgIndexError, any, any>;
-
-export async function listDataSources ({ page = 1, size = 10 }: PageParams = {}): Promise<Page<Datasource>> {
+export async function listDataSources ({ page = 1, size = 10 }: PageParams = {}): Promise<Page<DeprecatedDatasource>> {
   return fetch(requestUrl('/api/v1/admin/datasources', { page, size }), {
     headers: await authenticationHeaders(),
-  }).then(handleResponse(zodPage(datasourceSchema)));
+  }).then(handleResponse(zodPage(deprecatedDatasourceSchema)));
 }
 
-export async function getDatasource (id: number): Promise<Datasource> {
+export async function getDatasource (id: number): Promise<DeprecatedDatasource> {
   return fetch(requestUrl(`/api/v1/admin/datasources/${id}`), {
     headers: await authenticationHeaders(),
-  }).then(handleResponse(datasourceSchema));
+  }).then(handleResponse(deprecatedDatasourceSchema));
 }
 
 export async function deleteDatasource (id: number): Promise<void> {
@@ -159,12 +169,6 @@ export async function deleteDatasource (id: number): Promise<void> {
   }).then(handleErrors);
 }
 
-export async function getDatasourceOverview (id: number): Promise<DataSourceIndexProgress> {
-  return fetch(requestUrl(`/api/v1/admin/datasources/${id}/overview`), {
-    headers: await authenticationHeaders(),
-  }).then(handleResponse(datasourceOverviewSchema));
-}
-
 export async function createDatasource (params: CreateDatasourceParams) {
   return fetch(requestUrl(`/api/v1/admin/datasources`), {
     method: 'POST',
@@ -173,7 +177,7 @@ export async function createDatasource (params: CreateDatasourceParams) {
       'Content-Type': 'application/json',
     },
     body: JSON.stringify(params),
-  }).then(handleResponse(datasourceSchema));
+  }).then(handleResponse(deprecatedDatasourceSchema));
 }
 
 export async function uploadFiles (files: File[]) {
@@ -190,25 +194,3 @@ export async function uploadFiles (files: File[]) {
     body: formData,
   }).then(handleResponse(uploadSchema.array()));
 }
-
-export async function listDatasourceVectorIndexErrors (id: number, { page = 1, size = 10 }: PageParams = {}) {
-  return fetch(requestUrl(`/api/v1/admin/datasources/${id}/vector-index-errors`, { page, size }), {
-    headers: await authenticationHeaders(),
-  }).then(handleResponse(zodPage(vectorIndexErrorSchema)));
-}
-
-export async function listDatasourceKgIndexErrors (id: number, { page = 1, size = 10 }: PageParams = {}) {
-  return fetch(requestUrl(`/api/v1/admin/datasources/${id}/kg-index-errors`, { page, size }), {
-    headers: await authenticationHeaders(),
-  }).then(handleResponse(zodPage(kgIndexErrorSchema)));
-}
-
-export async function retryDatasourceAllFailedTasks (id: number) {
-  return fetch(requestUrl(`/api/v1/admin/datasources/${id}/retry-failed-tasks`), {
-    method: 'POST',
-    headers: {
-      ...await authenticationHeaders(),
-      'Content-Type': 'application/json',
-    },
-  }).then(handleErrors);
-}

frontend/app/src/api/documents.ts

@@ -14,11 +14,31 @@ export const mimeTypes = [
 
 const mimeValues: (typeof mimeTypes)[number]['value'] = mimeTypes.map(m => m.value) as never;
 
+//"id": 396505,
+//             "hash": "1022309282298755521",
+//             "name": "b (1).txt",
+//             "content": "abc",
+//             "mime_type": "text/plain",
+//             "source_uri": "uploads/01907db88850795d855b552663c18c9f/1731058150-01930b1b2df979fd80b6f9dea8d0328e.txt",
+//             "meta": {},
+//             "index_status": "completed",
+//             "index_result": null,
+//             "data_source": {
+//                 "id": 630003,
+//                 "name": "Test"
+//             },
+//             "knowledge_base": {
+//                 "id": 1,
+//                 "name": "Lorem Ipsum"
+//             },
+//             "last_modified_at": "2024-11-08T09:29:10"
+//
+
 export interface Document {
   id: number,
   name: string,
-  created_at: Date;
-  updated_at: Date
+  created_at?: Date | undefined;
+  updated_at?: Date | undefined
   last_modified_at: Date,
   hash: string
   content: string
@@ -27,7 +47,14 @@ export interface Document {
   source_uri: string,
   index_status: string,
   index_result?: unknown
-  data_source_id: number
+  data_source: {
+    id: number
+    name: string
+  }
+  knowledge_base: {
+    id: number
+    name: string
+  } | null
 }
 
 const documentSchema = z.object({
@@ -43,15 +70,22 @@ const documentSchema = z.object({
   source_uri: z.string(),
   index_status: z.string(),
   index_result: z.unknown(),
-  data_source_id: z.number(),
+  data_source: z.object({
+    id: z.number(),
+    name: z.string(),
+  }),
+  knowledge_base: z.object({
+    id: z.number(),
+    name: z.string(),
+  }).nullable(),
 }) satisfies ZodType<Document, any, any>;
 
 const zDate = z.coerce.date().or(z.literal('').transform(() => undefined)).optional();
 
 export const listDocumentsFiltersSchema = z.object({
   name: z.string().optional(),
   source_uri: z.string().optional(),
-  data_source_id: z.coerce.number().optional(),
+  knowledge_base_id: z.coerce.number().optional(),
   created_at_start: zDate,
   created_at_end: zDate,
   updated_at_start: zDate,
@@ -64,8 +98,8 @@ export const listDocumentsFiltersSchema = z.object({
 
 export type ListDocumentsTableFilters = z.infer<typeof listDocumentsFiltersSchema>;
 
-export async function listDocuments ({ page = 1, size = 10, ...filters }: PageParams & ListDocumentsTableFilters = {}): Promise<Page<Document>> {
-  return await fetch(requestUrl('/api/v1/admin/documents', { page, size, ...filters }), {
+export async function listDocuments ({ page = 1, size = 10, knowledge_base_id, ...filters }: PageParams & ListDocumentsTableFilters = {}): Promise<Page<Document>> {
+  return await fetch(requestUrl(knowledge_base_id != null ? `/api/v1/admin/knowledge_bases/${knowledge_base_id}/documents` : '/api/v1/admin/documents', { page, size, ...filters }), {
     headers: await authenticationHeaders(),
   })
     .then(handleResponse(zodPage(documentSchema)));