[ユースケースビルダー] マルチモーダル対応 (ファイル添付対応)

packages/cdk/lambda/useCaseBuilder/useCaseBuilderRepository.ts

@@ -157,6 +157,7 @@ export const createUseCase = async (
     promptTemplate: content.promptTemplate,
     inputExamples: content.inputExamples,
     fixedModelId: content.fixedModelId,
+    fileUpload: content.fileUpload,
     isShared: false,
   };
 
@@ -258,13 +259,14 @@ export const updateUseCase = async (
         dataType: useCaseInTable.dataType,
       },
       UpdateExpression:
-        'set title = :title, promptTemplate = :promptTemplate, description = :description, inputExamples = :inputExamples, fixedModelId = :fixedModelId',
+        'set title = :title, promptTemplate = :promptTemplate, description = :description, inputExamples = :inputExamples, fixedModelId = :fixedModelId, fileUpload = :fileUpload',
       ExpressionAttributeValues: {
         ':title': content.title,
         ':promptTemplate': content.promptTemplate,
         ':description': content.description ?? '',
         ':inputExamples': content.inputExamples ?? [],
         ':fixedModelId': content.fixedModelId ?? '',
+        ':fileUpload': !!content.fileUpload,
       },
     })
   );

packages/types/src/useCaseBuilder.d.ts

@@ -19,6 +19,7 @@ export type UseCaseContent = {
   promptTemplate: string;
   inputExamples?: UseCaseInputExample[];
   fixedModelId?: string;
+  fileUpload?: boolean;
 };
 
 // Table に記録されている内容

packages/web/src/components/useCaseBuilder/UseCaseBuilderView.tsx

@@ -1,4 +1,10 @@
-import React, { useCallback, useEffect, useMemo, useState } from 'react';
+import React, {
+  useCallback,
+  useEffect,
+  useMemo,
+  useState,
+  useRef,
+} from 'react';
 import Select from '../Select';
 import Button from '../Button';
 import useChat from '../../hooks/useChat';
@@ -15,7 +21,7 @@ import ButtonShare from './ButtonShare';
 import ButtonUseCaseEdit from './ButtonUseCaseEdit';
 import Skeleton from '../Skeleton';
 import useMyUseCases from '../../hooks/useCaseBuilder/useMyUseCases';
-import { UseCaseInputExample } from 'generative-ai-use-cases-jp';
+import { UseCaseInputExample, FileLimit } from 'generative-ai-use-cases-jp';
 import {
   NOLABEL,
   extractPlaceholdersFromPromptTemplate,
@@ -25,18 +31,51 @@ import {
 } from '../../utils/UseCaseBuilderUtils';
 import useRagKnowledgeBaseApi from '../../hooks/useRagKnowledgeBaseApi';
 import useRagApi from '../../hooks/useRagApi';
+import useFiles from '../../hooks/useFiles';
+import ZoomUpImage from '../ZoomUpImage';
+import ZoomUpVideo from '../ZoomUpVideo';
+import FileCard from '../FileCard';
+import { PiPaperclip, PiSpinnerGap } from 'react-icons/pi';
 
 const ragEnabled: boolean = import.meta.env.VITE_APP_RAG_ENABLED === 'true';
 const ragKnowledgeBaseEnabled: boolean =
   import.meta.env.VITE_APP_RAG_KNOWLEDGE_BASE_ENABLED === 'true';
 
+// pages/ChatPage.tsx に合わせている
+// 差分が生まれた場合は更新する
+const fileLimit: FileLimit = {
+  accept: {
+    doc: [
+      '.csv',
+      '.doc',
+      '.docx',
+      '.html',
+      '.md',
+      '.pdf',
+      '.txt',
+      '.xls',
+      '.xlsx',
+      '.gif',
+    ],
+    image: ['.jpg', '.jpeg', '.png', '.webp'],
+    video: ['.mkv', '.mov', '.mp4', '.webm'],
+  },
+  maxFileCount: 5,
+  maxFileSizeMB: 4.5,
+  maxImageFileCount: 20,
+  maxImageFileSizeMB: 3.75,
+  maxVideoFileCount: 1,
+  maxVideoFileSizeMB: 25, // 25 MB for base64 input (TODO: up to 1 GB through S3)
+};
+
 type Props = {
   modelId?: string;
   title: string;
   promptTemplate: string;
   description?: string;
   inputExamples?: UseCaseInputExample[];
   fixedModelId: string;
+  fileUpload: boolean;
   isLoading?: boolean;
 } & (
   | {
@@ -118,6 +157,16 @@ const UseCaseBuilderView: React.FC<Props> = (props) => {
   const { updateRecentUseUseCase } = useMyUseCases();
   const { retrieve: retrieveKendra } = useRagApi();
   const { retrieve: retrieveKnowledgeBase } = useRagKnowledgeBaseApi();
+  const {
+    uploadedFiles,
+    uploadFiles,
+    checkFiles,
+    deleteUploadedFile,
+    uploading,
+    errorMessages: fileErrorMessages,
+    clear: clearFiles,
+  } = useFiles();
+
   const [errorMessages, setErrorMessages] = useState<string[]>([]);
 
   const placeholders = useMemo(() => {
@@ -207,8 +256,10 @@ const UseCaseBuilderView: React.FC<Props> = (props) => {
       }
     }
 
+    tmpErrorMessages.push(...fileErrorMessages);
+
     setErrorMessages(tmpErrorMessages);
-  }, [setErrorMessages, items, textFormItems]);
+  }, [setErrorMessages, items, textFormItems, fileErrorMessages]);
 
   const onClickExec = useCallback(async () => {
     if (loading) return;
@@ -259,7 +310,14 @@ const UseCaseBuilderView: React.FC<Props> = (props) => {
       }
     }
 
-    postChat(prompt, true);
+    postChat(
+      prompt,
+      true,
+      undefined,
+      undefined,
+      undefined,
+      uploadedFiles.length > 0 ? uploadedFiles : undefined
+    );
     if (!props.previewMode) {
       updateRecentUseUseCase(props.useCaseId);
     }
@@ -275,13 +333,15 @@ const UseCaseBuilderView: React.FC<Props> = (props) => {
     retrieveKendra,
     retrieveKnowledgeBase,
     setText,
+    uploadedFiles,
   ]);
 
   // リセット
   const onClickClear = useCallback(() => {
     clear(textFormUniqueLabels);
     clearChat();
-  }, [clear, clearChat, textFormUniqueLabels]);
+    clearFiles();
+  }, [clear, clearChat, clearFiles, textFormUniqueLabels]);
 
   const disabledExec = useMemo(() => {
     if (props.isLoading || loading) {
@@ -304,6 +364,43 @@ const UseCaseBuilderView: React.FC<Props> = (props) => {
     [setValue]
   );
 
+  const accept = useMemo(() => {
+    if (!modelId) return [];
+    const feature = MODELS.modelFeatureFlags[modelId];
+    return [
+      ...(feature.doc ? fileLimit.accept.doc : []),
+      ...(feature.image ? fileLimit.accept.image : []),
+      ...(feature.video ? fileLimit.accept.video : []),
+    ];
+  }, [modelId]);
+
+  useEffect(() => {
+    checkFiles(fileLimit, accept);
+  }, [accept, checkFiles]);
+
+  const fileInput = useRef<HTMLInputElement | null>(null);
+
+  const onChangeFile = async (e: React.ChangeEvent<HTMLInputElement>) => {
+    const files = e.target.files;
+    if (files) {
+      uploadFiles(Array.from(files), fileLimit, accept);
+
+      if (fileInput.current) {
+        fileInput.current.value = '';
+      }
+    }
+  };
+
+  const deleteFile = useCallback(
+    (fileUrl: string) => {
+      console.log('delete', fileUrl);
+      if (fileLimit && accept) {
+        deleteUploadedFile(fileUrl, fileLimit, accept);
+      }
+    },
+    [deleteUploadedFile, accept]
+  );
+
   return (
     <div>
       <div className="mb-4 flex flex-col-reverse text-xl font-semibold md:flex-row">
@@ -389,6 +486,79 @@ const UseCaseBuilderView: React.FC<Props> = (props) => {
               </div>
             ))}
           </div>
+
+          {props.fileUpload && (
+            <div className="mb-3 flex flex-col">
+              <label>
+                <input
+                  hidden
+                  type="file"
+                  accept={accept.join(',')}
+                  onChange={onChangeFile}
+                  ref={fileInput}
+                />
+                <div
+                  className={`${uploading ? 'bg-gray-300' : 'bg-aws-smile cursor-pointer '} flex w-fit items-center justify-center rounded-lg border px-2 py-1 text-white`}>
+                  {uploading ? (
+                    <PiSpinnerGap className="animate-spin" />
+                  ) : (
+                    <PiPaperclip />
+                  )}
+                  ファイル添付
+                </div>
+              </label>
+
+              {uploadedFiles.length > 0 && (
+                <div className="my-2 flex flex-wrap gap-2">
+                  {uploadedFiles.map((uploadedFile, idx) => {
+                    if (uploadedFile.type === 'image') {
+                      return (
+                        <ZoomUpImage
+                          key={idx}
+                          src={uploadedFile.base64EncodedData}
+                          loading={uploadedFile.uploading}
+                          deleting={uploadedFile.deleting}
+                          size="s"
+                          error={uploadedFile.errorMessages.length > 0}
+                          onDelete={() => {
+                            deleteFile(uploadedFile.s3Url ?? '');
+                          }}
+                        />
+                      );
+                    } else if (uploadedFile.type === 'video') {
+                      return (
+                        <ZoomUpVideo
+                          key={idx}
+                          src={uploadedFile.base64EncodedData}
+                          loading={uploadedFile.uploading}
+                          deleting={uploadedFile.deleting}
+                          size="s"
+                          error={uploadedFile.errorMessages.length > 0}
+                          onDelete={() => {
+                            deleteFile(uploadedFile.s3Url ?? '');
+                          }}
+                        />
+                      );
+                    } else {
+                      return (
+                        <FileCard
+                          key={idx}
+                          filename={uploadedFile.name}
+                          loading={uploadedFile.uploading}
+                          deleting={uploadedFile.deleting}
+                          size="s"
+                          error={uploadedFile.errorMessages.length > 0}
+                          onDelete={() => {
+                            deleteFile(uploadedFile.s3Url ?? '');
+                          }}
+                        />
+                      );
+                    }
+                  })}
+                </div>
+              )}
+            </div>
+          )}
         </>
       )}
       <div className="flex flex-1 items-end justify-between">

packages/web/src/hooks/useCaseBuilder/useMyUseCases.ts

@@ -51,6 +51,7 @@ const useMyUseCases = () => {
       description?: string;
       inputExamples?: UseCaseInputExample[];
       fixedModelId?: string;
+      fileUpload?: boolean;
     }) => {
       return createUseCase(params).finally(() => {
         mutateMyUseCases();
@@ -63,6 +64,7 @@ const useMyUseCases = () => {
       description?: string;
       inputExamples?: UseCaseInputExample[];
       fixedModelId?: string;
+      fileUpload?: boolean;
     }) => {
       // 一覧の更新
       const index = findIndex(params.useCaseId);
@@ -83,6 +85,7 @@ const useMyUseCases = () => {
         description: params.description,
         inputExamples: params.inputExamples,
         fixedModelId: params.fixedModelId,
+        fileUpload: params.fileUpload,
       }).finally(() => {
         mutateMyUseCases();
         mutateFavoriteUseCases();