Skip to content

Commit

Permalink
feat(frontend): knowledge base datasource management pages (#408)
Browse files Browse the repository at this point in the history
  • Loading branch information
634750802 authored Nov 27, 2024
1 parent a9ea734 commit 463ada3
Show file tree
Hide file tree
Showing 66 changed files with 893 additions and 909 deletions.
73 changes: 24 additions & 49 deletions frontend/app/src/api/datasources.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,6 @@ export interface DatasourceBase {
name: string;
}

interface DeprecatedDatasourceBase extends DatasourceBase {
created_at: Date;
updated_at: Date;
user_id: string | null;
build_kg_index: boolean;
llm_id: number | null;
}

export type DeprecatedDatasource = DeprecatedDatasourceBase & DatasourceSpec

type DatasourceSpec = ({
data_source_type: 'file'
config: { file_id: number, file_name: string }[]
Expand All @@ -43,18 +33,6 @@ export interface BaseCreateDatasourceParams {
name: string;
}

export interface DeprecatedBaseCreateDatasourceParams extends BaseCreateDatasourceParams {
description: string;
/**
* @deprecated
*/
build_kg_index: boolean;
/**
* @deprecated
*/
llm_id: number | null;
}

export type CreateDatasourceSpecParams = ({
data_source_type: 'file'
config: { file_id: number, file_name: string }[]
Expand All @@ -66,7 +44,7 @@ export type CreateDatasourceSpecParams = ({
config: { url: string }
});

export type CreateDatasourceParams = DeprecatedBaseCreateDatasourceParams & CreateDatasourceSpecParams;
export type CreateDatasourceParams = BaseCreateDatasourceParams & CreateDatasourceSpecParams;

export interface Upload {
created_at?: Date;
Expand All @@ -92,16 +70,6 @@ export type DatasourceKgIndexError = {
error: string | null
}

const deprecatedBaseDatasourceSchema = z.object({
id: z.number(),
name: z.string(),
created_at: zodJsonDate(),
updated_at: zodJsonDate(),
user_id: z.string().nullable(),
build_kg_index: z.boolean(),
llm_id: z.number().nullable(),
});

const datasourceSpecSchema = z.discriminatedUnion('data_source_type', [
z.object({
data_source_type: z.literal('file'),
Expand All @@ -123,9 +91,6 @@ const datasourceSpecSchema = z.discriminatedUnion('data_source_type', [
})],
) satisfies ZodType<DatasourceSpec, any, any>;

export const deprecatedDatasourceSchema = deprecatedBaseDatasourceSchema
.and(datasourceSpecSchema) satisfies ZodType<DeprecatedDatasource, any, any>;

export const datasourceSchema = z.object({
id: z.number(),
name: z.string(),
Expand All @@ -141,43 +106,53 @@ const uploadSchema = z.object({
created_at: zodJsonDate().optional(),
updated_at: zodJsonDate().optional(),
}) satisfies ZodType<Upload, any, any>;

const datasourceOverviewSchema = z.object({
z.object({
vector_index: indexSchema,
documents: totalSchema,
chunks: totalSchema,
kg_index: indexSchema.optional(),
relationships: totalSchema.optional(),
}) satisfies ZodType<DataSourceIndexProgress>;

export async function listDataSources ({ page = 1, size = 10 }: PageParams = {}): Promise<Page<DeprecatedDatasource>> {
return fetch(requestUrl('/api/v1/admin/datasources', { page, size }), {
export async function listDataSources (kbId: number, { page = 1, size = 10 }: PageParams = {}): Promise<Page<Datasource>> {
return fetch(requestUrl(`/api/v1/admin/knowledge_bases/${kbId}/datasources`, { page, size }), {
headers: await authenticationHeaders(),
}).then(handleResponse(zodPage(deprecatedDatasourceSchema)));
}).then(handleResponse(zodPage(datasourceSchema)));
}

export async function getDatasource (id: number): Promise<DeprecatedDatasource> {
return fetch(requestUrl(`/api/v1/admin/datasources/${id}`), {
export async function getDatasource (kbId: number, id: number): Promise<Datasource> {
return fetch(requestUrl(`/api/v1/admin/knowledge_bases/${kbId}/datasources/${id}`), {
headers: await authenticationHeaders(),
}).then(handleResponse(deprecatedDatasourceSchema));
}).then(handleResponse(datasourceSchema));
}

export async function deleteDatasource (id: number): Promise<void> {
await fetch(requestUrl(`/api/v1/admin/datasources/${id}`), {
export async function deleteDatasource (kbId: number, id: number): Promise<void> {
await fetch(requestUrl(`/api/v1/admin/knowledge_bases/${kbId}/datasources/${id}`), {
method: 'DELETE',
headers: await authenticationHeaders(),
}).then(handleErrors);
}

export async function createDatasource (params: CreateDatasourceParams) {
return fetch(requestUrl(`/api/v1/admin/datasources`), {
export async function createDatasource (kbId: number, params: CreateDatasourceParams) {
return fetch(requestUrl(`/api/v1/admin/knowledge_bases/${kbId}/datasources`), {
method: 'POST',
headers: {
...await authenticationHeaders(),
'Content-Type': 'application/json',
},
body: JSON.stringify(params),
}).then(handleResponse(deprecatedDatasourceSchema));
}).then(handleResponse(datasourceSchema));
}

export async function updateDatasource (kbId: number, id: number, params: { name: string }) {
return fetch(requestUrl(`/api/v1/admin/knowledge_bases/${kbId}/datasources/${id}`), {
method: 'PUT',
headers: {
...await authenticationHeaders(),
'Content-Type': 'application/json',
},
body: JSON.stringify(params),
}).then(handleResponse(datasourceSchema));
}

export async function uploadFiles (files: File[]) {
Expand Down

This file was deleted.

21 changes: 0 additions & 21 deletions frontend/app/src/app/(main)/(admin)/datasources/[id]/page.tsx

This file was deleted.

13 changes: 0 additions & 13 deletions frontend/app/src/app/(main)/(admin)/datasources/page.tsx

This file was deleted.

14 changes: 0 additions & 14 deletions frontend/app/src/app/(main)/(admin)/documents/page.tsx

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
'use client';

import { AdminPageHeading } from '@/components/admin-page-heading';
import { CreateDatasourceForm } from '@/components/datasource/create-datasource-form';
import { mutateKnowledgeBases, useKnowledgeBase } from '@/components/knowledge-base/hooks';
import { Loader2Icon } from 'lucide-react';
import { useRouter } from 'next/navigation';

export default function NewKnowledgeBaseDataSourcePage ({ params }: { params: { id: string } }) {
const id = parseInt(decodeURIComponent(params.id));
const { knowledgeBase } = useKnowledgeBase(id);
const router = useRouter();

return (
<>
<AdminPageHeading
breadcrumbs={[
{ title: 'Knowledge Bases', url: '/knowledge-bases' },
{ title: knowledgeBase?.name ?? <Loader2Icon className="size-4 animate-spin repeat-infinite" />, url: `/knowledge-bases/${id}` },
{ title: 'DataSources', url: `/knowledge-bases/${id}/data-sources` },
{ title: 'New' },
]}
/>
<CreateDatasourceForm
knowledgeBaseId={id}
onCreated={() => {
router.back();
mutateKnowledgeBases();
}}
/>
</>
);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
'use client';

import { DatasourceCard } from '@/components/datasource/datasource-card';
import { DatasourceCreateOption } from '@/components/datasource/datasource-create-option';
import { NoDatasourcePlaceholder } from '@/components/datasource/no-datasource-placeholder';
import { useAllKnowledgeBaseDataSources } from '@/components/knowledge-base/hooks';
import { Skeleton } from '@/components/ui/skeleton';
import { FileDownIcon, GlobeIcon, PaperclipIcon } from 'lucide-react';

export default function KnowledgeBaseDataSourcesPage ({ params }: { params: { id: string } }) {
const id = parseInt(decodeURIComponent(params.id));
const { data: dataSources, isLoading } = useAllKnowledgeBaseDataSources(id);

return (
<div className="space-y-8 max-w-screen-sm">
<section className="space-y-4">
<h3>Create Data Source</h3>
<div className="grid md:grid-cols-3 gap-4">
<DatasourceCreateOption
knowledgeBaseId={id}
type="file"
icon={<PaperclipIcon className="size-4 flex-shrink-0" />}
title="Files"
>
Upload files
</DatasourceCreateOption>
<DatasourceCreateOption
knowledgeBaseId={id}
type="web_single_page"
icon={<FileDownIcon className="size-4 flex-shrink-0" />}
title="Web Pages"
>
Select pages.
</DatasourceCreateOption>
<DatasourceCreateOption
knowledgeBaseId={id}
type="web_sitemap"
icon={<GlobeIcon className="size-4 flex-shrink-0" />}
title="Website by sitemap"
>
Select web sitemap.
</DatasourceCreateOption>
</div>
</section>
<section className="space-y-4">
<h3>Browse existing Data Sources</h3>
{isLoading && <Skeleton className="h-20 rounded-lg" />}
{dataSources?.map(datasource => (
<DatasourceCard key={datasource.id} knowledgeBaseId={id} datasource={datasource} />
))}
{dataSources?.length === 0 && (
<NoDatasourcePlaceholder />
)}
</section>
</div>
);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
import { KnowledgeBaseIndexProgress } from '@/components/knowledge-base/knowledge-base-index';

export default function KnowledgeBaseIndexProgressPage ({ params }: { params: { id: string } }) {
const id = parseInt(decodeURIComponent(params.id));

return (
<section className="space-y-2">
<h3 className="text-lg font-medium">Index Progress</h3>
<KnowledgeBaseIndexProgress id={id} />
</section>
);
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ export default function CreateSynopsisEntityPage ({ params }: { params: { id: st

return (
<>
<NextLink href='.'>
<NextLink href=''>
Back
</NextLink>
<GraphCreateEntity knowledgeBaseId={kbId} onCreated={handleCreateEntity} />
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
import { GraphEditor } from '@/components/graph/GraphEditor';

export default function KnowledgeGraphExplorerPage ({ params }: { params: { id: string } }) {
const id = parseInt(decodeURIComponent(params.id));

return (
<section className="space-y-2">
<GraphEditor knowledgeBaseId={id} />
</section>
);
}
Loading

0 comments on commit 463ada3

Please sign in to comment.