Merge pull request #271 from l3vels/fix/fine-tuning

Fix/fine tuning

apps/server/services/schedule.py

@@ -1,6 +1,6 @@
 from datetime import datetime, timedelta
 
-import arrow
+# import arrow
 from fastapi_sqlalchemy import db
 
 from models.chat import ChatModel

apps/ui/package.json

@@ -44,6 +44,7 @@
     "notistack": "^2.0.5",
     "npm": "^9.6.2",
     "openai": "^3.2.1",
+    "papaparse": "^5.4.1",
     "react": "^18.2.0",
     "react-avatar": "^5.0.3",
     "react-cookie": "^4.1.1",

apps/ui/src/components/ImportFile/ImportFile.tsx

@@ -14,12 +14,13 @@ import { t } from 'i18next'
 
 const ImportFile = ({ setFieldValue, value = '' }: { setFieldValue: any; value?: string }) => {
   const {
-    // handleFileChange,
     step,
     parsedData,
     setStep,
     handleUploadJson,
-    handleConvertData,
+    handleConvertJson,
+    handleUploadCsv,
+    handleConvertCSVtoJSON,
     fileIsLoading,
   } = useImportFile({
     setFieldValue: setFieldValue,
@@ -29,18 +30,21 @@ const ImportFile = ({ setFieldValue, value = '' }: { setFieldValue: any; value?:
 
   useEffect(() => {
     if (value.length > 0) {
-      // Replace 'fileUrl' with the actual URL of the file you want to read.
       const fileUrl = value
 
       fetch(fileUrl)
         .then(response => {
           if (!response.ok) {
             throw new Error(`Failed to fetch file: ${response.status} ${response.statusText}`)
           }
-          return response.text() // or response.json() for JSON files, response.blob() for binary files, etc.
+          return response.text()
         })
         .then(data => {
-          handleConvertData(data) // Update the state with the file content
+          if (fileUrl.endsWith('.json')) {
+            handleConvertJson(data)
+          } else if (fileUrl.endsWith('.csv')) {
+            handleConvertCSVtoJSON(data)
+          }
         })
         .catch(error => {
           console.error('Error fetching file:', error)
@@ -57,7 +61,7 @@ const ImportFile = ({ setFieldValue, value = '' }: { setFieldValue: any; value?:
               {t('download-template')}
             </ButtonTertiary>
 
-            {/* <UploadButton onChange={handleFileChange} isLoading={false} label={t('upload-csv')} /> */}
+            <UploadButton onChange={handleUploadCsv} isLoading={false} label={t('upload-csv')} />
             <UploadButton
               onChange={handleUploadJson}
               isLoading={fileIsLoading}

apps/ui/src/components/ImportFile/ReviewImport.tsx

@@ -9,14 +9,14 @@ import styled from 'styled-components'
 import Button from '@l3-lib/ui-core/dist/Button'
 
 import Table from 'components/Table'
-import { ButtonTertiary } from 'components/Button/Button'
+import { ButtonPrimary, ButtonTertiary } from 'components/Button/Button'
 import { t } from 'i18next'
 import { useDownloadTemplate } from './useDownloadTemplate'
 
 const ReviewImport = ({ data, setStep: startOver }: { data: any[]; setStep: any }) => {
   const { handleDownloadTemplate } = useDownloadTemplate()
 
-  const { formik, keys, options, step, response, setStep } = useReviewImport(data)
+  const { formik, step, response, setStep } = useReviewImport(data)
 
   const columns = [
     {
@@ -57,9 +57,9 @@ const ReviewImport = ({ data, setStep: startOver }: { data: any[]; setStep: any
                 {/* <ButtonTertiary onClick={formik.handleSubmit} size={Button.sizes.SMALL}>
                   Save
                 </ButtonTertiary> */}
-                <ButtonTertiary onClick={() => startOver(0)} size={Button.sizes.SMALL}>
+                <ButtonPrimary onClick={() => startOver(0)} size={Button.sizes.SMALL}>
                   {t('start-over')}
-                </ButtonTertiary>
+                </ButtonPrimary>
               </StyledButtonContainer>
               {/* <StyledHeaderContainer itemLength={itemLength}>
                 {keys.map((item: any, index: number) => (

apps/ui/src/components/ImportFile/useImportFile.ts

@@ -1,6 +1,7 @@
 import { ToastContext } from 'contexts'
 import useUploadFile from 'hooks/useUploadFile'
 import React, { useContext } from 'react'
+import Papa from 'papaparse'
 
 const useImportFile = ({ setFieldValue }: { setFieldValue: any }) => {
   const { setToast } = useContext(ToastContext)
@@ -11,7 +12,7 @@ const useImportFile = ({ setFieldValue }: { setFieldValue: any }) => {
 
   const { uploadFile } = useUploadFile()
 
-  const handleConvertData = (data: any) => {
+  const handleConvertJson = (data: any) => {
     const dataArray = JSON.parse(data)
     const convertedData = dataArray.map((item: any) => ({
       System: item.System,
@@ -22,21 +23,17 @@ const useImportFile = ({ setFieldValue }: { setFieldValue: any }) => {
     setStep(1)
   }
 
-  const handleUploadJson = async (e: any) => {
-    setFileIsLoading(true)
-    const { files } = e.target
+  const handleConvertCSVtoJSON = (csvString: string) => {
+    const { data, errors } = Papa.parse(csvString, {
+      header: true, // Set this to true if the CSV file has a header row
+      skipEmptyLines: true, // Skip empty lines in CSV
+    })
 
-    if (!files) return
-
-    const file = files[0]
-
-    if (file.type !== 'application/json')
-      return setToast({
-        message: 'File must be JSON!',
-        type: 'negative',
-        open: true,
-      })
+    setParsedData(data)
+    setStep(1)
+  }
 
+  const handleUploadFile = async (files: any) => {
     const promises = []
 
     for (const file of files) {
@@ -55,57 +52,62 @@ const useImportFile = ({ setFieldValue }: { setFieldValue: any }) => {
     const uploadedFiles = await Promise.all(promises)
 
     setFieldValue('fine_tuning_file_url', uploadedFiles?.[0].url)
+  }
 
-    if (file) {
-      const reader = new FileReader()
-      reader.onload = (e: any) => {
-        const data = e.target.result
-
-        handleConvertData(data)
-      }
-      reader.readAsText(file)
-    }
+  const handleUploadJson = async (event: any) => {
+    const { files } = event.target
+    const file = files[0]
 
-    setFileIsLoading(false)
-  }
+    if (file.type !== 'application/json')
+      return setToast({
+        message: 'File must be JSON!',
+        type: 'negative',
+        open: true,
+      })
 
-  const handleFileChange = async (e: any) => {
-    const { files } = e.target
+    handleUploadFile(files)
 
-    if (!files) return
+    const reader = new FileReader()
 
-    const promises = []
+    reader.onload = (event: any) => {
+      const data = event.target.result
 
-    for (const file of files) {
-      promises.push(
-        uploadFile(
-          {
-            name: file.name,
-            type: file.type,
-            size: file.size,
-          },
-          file,
-        ),
-      )
+      handleConvertJson(data)
     }
+    reader.readAsText(file)
+  }
 
-    const uploadedFiles = await Promise.all(promises)
+  const handleUploadCsv = async (event: any) => {
+    const { files } = event.target
+    const file = files[0]
 
-    setFieldValue('fine_tuning_file_url', uploadedFiles?.[0].url)
+    if (file.type !== 'text/csv')
+      return setToast({
+        message: 'File must be CSV!',
+        type: 'negative',
+        open: true,
+      })
 
-    // const response = await parseCsvToJson(files[0], [])
-    // console.log('response', response)
-    // setParsedData()
-    setStep(1)
+    handleUploadFile(files)
+
+    const reader = new FileReader()
+
+    reader.onload = (event: any) => {
+      const csvString = event.target.result
+      handleConvertCSVtoJSON(csvString)
+    }
+
+    reader.readAsText(file)
   }
 
   return {
-    handleFileChange,
+    handleUploadCsv,
     handleUploadJson,
     step,
     parsedData,
     setStep,
-    handleConvertData,
+    handleConvertJson,
+    handleConvertCSVtoJSON,
     fileIsLoading,
   }
 }

apps/ui/src/components/Table/Table.tsx

@@ -26,21 +26,21 @@ const Table = ({ columns, data }: TableProps) => {
   const defaultColumn = useMemo(
     () => ({
       width: 300,
-      minWidth: 100,
+      // minWidth: 100,
       // maxWidth: 100,
     }),
     [],
   )
 
   const { getTableProps, getTableBodyProps, headerGroups, rows, prepareRow } = useTable(
     {
+      defaultColumn,
       columns,
       data,
-      defaultColumn,
     },
-    useResizeColumns,
     useFlexLayout,
     useBlockLayout,
+    useResizeColumns,
   )
 
   return (
@@ -55,6 +55,7 @@ const Table = ({ columns, data }: TableProps) => {
                   {...column.getHeaderProps()}
                   {...column.getResizerProps()}
                   minWidth={column.minWidth}
+                  maxWidth={column.maxWidth}
                   width={column.width}
                 >
                   <TypographyPrimary
@@ -91,6 +92,13 @@ const StyledRoot = styled.div`
   height: 100%;
   overflow: auto;
 
+  ::-webkit-scrollbar {
+    width: 0; /* This will hide the scrollbar */
+  }
+  body {
+    scrollbar-width: none; /* This will hide the scrollbar */
+  }
+
   border-radius: 24px;
   max-height: calc(100vh - 300px);
 `
@@ -109,7 +117,7 @@ const StyledThead = styled.thead`
   background: #fff;
 
   display: flex;
-  z-index: 100000;
+  z-index: 1;
   position: sticky;
   top: 0px;
   margin: 0 0 0 0;

apps/ui/src/components/UploadButton.tsx

@@ -6,7 +6,7 @@ import Button from '@l3-lib/ui-core/dist/Button'
 import Loader from '@l3-lib/ui-core/dist/Loader'
 
 import { StyledAddIcon } from 'pages/Navigation/MainNavigation'
-import { ButtonSecondary, ButtonTertiary } from './Button/Button'
+import { ButtonPrimary } from './Button/Button'
 
 type UploadButtonProps = {
   onChange: (event: ChangeEvent<HTMLInputElement>) => void
@@ -27,9 +27,9 @@ const UploadButton = ({ onChange, isLoading, label }: UploadButtonProps) => {
       {isLoading ? (
         <Loader size={20} />
       ) : label ? (
-        <ButtonTertiary onClick={onAddButtonClick} size={Button.sizes.SMALL}>
+        <ButtonPrimary onClick={onAddButtonClick} size={Button.sizes.SMALL}>
           {label}
-        </ButtonTertiary>
+        </ButtonPrimary>
       ) : (
         <IconButton
           size={IconButton.sizes.SMALL}