From 973cbb91c268164d0351273eef6cfb924f37b4b2 Mon Sep 17 00:00:00 2001 From: Samuel N Cui Date: Thu, 5 Oct 2023 02:11:37 +0800 Subject: [PATCH] feat: add more manage features --- apis/file_get.go | 2 +- entity/utils.go | 30 +---- frontend/package.json | 4 +- frontend/src/actions.ts | 14 +- frontend/src/api.ts | 4 +- frontend/src/components/toolbarInfo.tsx | 19 +++ frontend/src/init.tsx | 4 +- frontend/src/pages/backup.tsx | 8 +- frontend/src/pages/file.tsx | 27 +++- frontend/src/pages/restore.tsx | 43 +++--- frontend/src/pages/tapes.tsx | 20 ++- go.mod | 12 +- go.sum | 28 ++-- library/file.go | 58 ++++++++ library/library.go | 53 ++++++-- resource/db.go | 23 ++-- resource/sqlite_cgo.go.bak | 14 ++ resource/sqlite_fix.go.bak | 156 ++++++++++++++++++++++ resource/{sqlite.go => sqlite_pure_go.go} | 0 resource/sqlite_pure_go.go.bak | 14 ++ 20 files changed, 423 insertions(+), 110 deletions(-) create mode 100644 frontend/src/components/toolbarInfo.tsx create mode 100644 resource/sqlite_cgo.go.bak create mode 100644 resource/sqlite_fix.go.bak rename resource/{sqlite.go => sqlite_pure_go.go} (100%) create mode 100644 resource/sqlite_pure_go.go.bak diff --git a/apis/file_get.go b/apis/file_get.go index b5e9346..9560f9e 100644 --- a/apis/file_get.go +++ b/apis/file_get.go @@ -24,7 +24,7 @@ func (api *API) FileGet(ctx context.Context, req *entity.FileGetRequest) (*entit return nil, err } - children, err := api.lib.List(ctx, req.Id) + children, err := api.lib.ListWithSize(ctx, req.Id) if err != nil { return nil, err } diff --git a/entity/utils.go b/entity/utils.go index 1a024ef..c8d7c69 100644 --- a/entity/utils.go +++ b/entity/utils.go @@ -3,30 +3,14 @@ package entity import ( "database/sql/driver" "fmt" - reflect "reflect" - sync "sync" "github.com/modern-go/reflect2" "google.golang.org/protobuf/proto" ) -var ( - typeMap sync.Map -) - // Scan implement database/sql.Scanner func Scan(dst proto.Message, src interface{}) error { - cacheKey := reflect2.RTypeOf(dst) - typ, has := loadType(cacheKey) - if !has { - ptrType := reflect.TypeOf(dst) - if ptrType.Kind() != reflect.Ptr { - return fmt.Errorf("scan dst is not an ptr, has= %T", dst) - } - - typ = reflect2.Type2(ptrType.Elem()) - storeType(cacheKey, typ) - } + typ := reflect2.TypeOf(dst).(reflect2.PtrType).Elem() typ.Set(dst, typ.New()) var buf []byte @@ -59,15 +43,3 @@ func Value(src proto.Message) (driver.Value, error) { } return buf, nil } - -func loadType(key uintptr) (reflect2.Type, bool) { - i, has := typeMap.Load(key) - if !has { - return nil, false - } - return i.(reflect2.Type), true -} - -func storeType(key uintptr, typ reflect2.Type) { - typeMap.Store(key, typ) -} diff --git a/frontend/package.json b/frontend/package.json index 4ea0ffb..a300337 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -10,8 +10,8 @@ "gen-proto": "protoc --ts_out ./src/entity --proto_path ../entity/ `ls ../entity/*.proto` && ./src/entity/gen_index.sh" }, "dependencies": { - "@aperturerobotics/chonky": "^0.2.6", - "@aperturerobotics/chonky-icon-fontawesome": "^0.2.2", + "@samuelncui/chonky": "^0.2.7", + "@samuelncui/chonky-icon-fontawesome": "^0.2.7", "@emotion/react": "^11.10.4", "@emotion/styled": "^11.10.4", "@fortawesome/fontawesome-svg-core": "^1.2.32", diff --git a/frontend/src/actions.ts b/frontend/src/actions.ts index 3e9aaa6..2692ec4 100644 --- a/frontend/src/actions.ts +++ b/frontend/src/actions.ts @@ -1,6 +1,6 @@ -import { FileData, FileArray, FileAction } from "@aperturerobotics/chonky"; -import { defineFileAction } from "@aperturerobotics/chonky"; -import { ChonkyActions } from "@aperturerobotics/chonky"; +import { FileData, FileArray, FileAction } from "@samuelncui/chonky"; +import { defineFileAction } from "@samuelncui/chonky"; +import { ChonkyActions } from "@samuelncui/chonky"; type RenameFileState = { contextMenuTriggerFile: FileData; @@ -46,3 +46,11 @@ export const CreateRestoreJobAction = defineFileAction({ toolbar: true, }, } as FileAction); + +export const TrimLibraryAction = defineFileAction({ + id: "trim_library", + button: { + name: "Trim Library", + toolbar: true, + }, +} as FileAction); diff --git a/frontend/src/api.ts b/frontend/src/api.ts index 4d17ca9..a90bb4c 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -1,4 +1,4 @@ -import { FileData } from "@aperturerobotics/chonky"; +import { FileData } from "@samuelncui/chonky"; import { GrpcWebFetchTransport } from "@protobuf-ts/grpcweb-transport"; import { ServiceClient, File, SourceFile, Tape, Position } from "./entity"; @@ -90,7 +90,7 @@ export function convertTapes(tapes: Array): FileData[] { selectable: true, draggable: false, droppable: false, - size: 0, + size: Number(tape.writenBytes), modDate: moment.unix(Number(tape.createTime)).toDate(), isTape: true, }; diff --git a/frontend/src/components/toolbarInfo.tsx b/frontend/src/components/toolbarInfo.tsx new file mode 100644 index 0000000..32b7fd3 --- /dev/null +++ b/frontend/src/components/toolbarInfo.tsx @@ -0,0 +1,19 @@ +import { memo } from "react"; +import Typography from "@mui/material/Typography"; +import { FileArray } from "@samuelncui/chonky"; + +import { formatFilesize } from "../tools"; + +export interface ToobarInfoProps { + files?: FileArray; +} + +export const ToobarInfo: React.FC = memo((props) => { + return ( +
+ + {formatFilesize((props.files || []).reduce((total, file) => total + (file?.size ? file.size : 0), 0))} + +
+ ); +}); diff --git a/frontend/src/init.tsx b/frontend/src/init.tsx index 01686b6..c58efec 100644 --- a/frontend/src/init.tsx +++ b/frontend/src/init.tsx @@ -1,5 +1,5 @@ -import { setChonkyDefaults } from "@aperturerobotics/chonky"; -import { ChonkyIconFA } from "@aperturerobotics/chonky-icon-fontawesome"; +import { setChonkyDefaults } from "@samuelncui/chonky"; +import { ChonkyIconFA } from "@samuelncui/chonky-icon-fontawesome"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { faPencilAlt } from "@fortawesome/free-solid-svg-icons/faPencilAlt"; diff --git a/frontend/src/pages/backup.tsx b/frontend/src/pages/backup.tsx index 1fc2d54..14e08f4 100644 --- a/frontend/src/pages/backup.tsx +++ b/frontend/src/pages/backup.tsx @@ -2,8 +2,8 @@ import { useState, useEffect, useMemo, useCallback, FC, useRef, RefObject } from import Grid from "@mui/material/Grid"; import Box from "@mui/material/Box"; -import { FileBrowser, FileNavbar, FileToolbar, FileList, FileContextMenu, FileArray, FileBrowserHandle } from "@aperturerobotics/chonky"; -import { ChonkyActions, ChonkyFileActionData, FileData } from "@aperturerobotics/chonky"; +import { FileBrowser, FileNavbar, FileToolbar, FileList, FileContextMenu, FileArray, FileBrowserHandle } from "@samuelncui/chonky"; +import { ChonkyActions, ChonkyFileActionData, FileData } from "@samuelncui/chonky"; import { cli, convertSourceFiles } from "../api"; import { Root } from "../api"; @@ -26,7 +26,6 @@ const useBackupSourceBrowser = (source: RefObject) => { const onFileAction = useCallback( (data: ChonkyFileActionData) => { - console.log("source", data); switch (data.id) { case ChonkyActions.OpenFiles.id: (async () => { @@ -72,7 +71,7 @@ const useBackupTargetBrowser = () => { const [files, setFiles] = useState(Array(0)); const [folderChain, setFolderChan] = useState([ { - id: "0", + id: "backup_waitlist", name: "Backup Waitlist", isDir: true, openable: true, @@ -84,7 +83,6 @@ const useBackupTargetBrowser = () => { const onFileAction = useCallback( (data: ChonkyFileActionData) => { - console.log("target", data); switch (data.id) { case ChonkyActions.DeleteFiles.id: (() => { diff --git a/frontend/src/pages/file.tsx b/frontend/src/pages/file.tsx index 89b7be3..a7c1cf1 100644 --- a/frontend/src/pages/file.tsx +++ b/frontend/src/pages/file.tsx @@ -2,12 +2,13 @@ import { useState, useRef, useEffect, useMemo, useCallback } from "react"; import Grid from "@mui/material/Grid"; import Box from "@mui/material/Box"; -import { FullFileBrowser, FileBrowserProps, FileBrowserHandle, FileArray } from "@aperturerobotics/chonky"; -import { ChonkyActions, ChonkyFileActionData } from "@aperturerobotics/chonky"; +import { FileBrowser as ChonckFileBrowser, FileNavbar, FileToolbar, FileList, FileContextMenu, FileArray, FileBrowserHandle } from "@samuelncui/chonky"; +import { ChonkyActions, ChonkyFileActionData } from "@samuelncui/chonky"; import { cli, convertFiles } from "../api"; import { Root } from "../api"; import { RenameFileAction, RefreshListAction } from "../actions"; +import { ToobarInfo } from "../components/toolbarInfo"; import { useDetailModal, DetailModal } from "./file-detail"; import { FileGetReply } from "../entity"; @@ -152,6 +153,9 @@ const useFileBrowser = (storageKey: string, refreshAll: () => Promise, ope ); const fileActions = useMemo(() => [ChonkyActions.CreateFolder, ChonkyActions.DeleteFiles, ChonkyActions.MoveFiles, RenameFileAction, RefreshListAction], []); + const totalSize = useMemo(() => { + return files.reduce((total, file) => total + (file?.size ? file.size : 0), 0); + }, [files]); return { files, @@ -160,6 +164,7 @@ const useFileBrowser = (storageKey: string, refreshAll: () => Promise, ope fileActions, defaultFileViewActionId: ChonkyActions.EnableListView.id, doubleClickDelay: 300, + totalSize, }; }; @@ -185,10 +190,24 @@ export const FileBrowser = () => { - + + + + + + + + - + + + + + + + + diff --git a/frontend/src/pages/restore.tsx b/frontend/src/pages/restore.tsx index 6a7bde8..e4b6e69 100644 --- a/frontend/src/pages/restore.tsx +++ b/frontend/src/pages/restore.tsx @@ -2,22 +2,24 @@ import { useState, useEffect, useMemo, useCallback, FC, useRef, RefObject } from import Grid from "@mui/material/Grid"; import Box from "@mui/material/Box"; -import { FileBrowser, FileNavbar, FileToolbar, FileList, FileContextMenu, FileArray, FileBrowserHandle } from "@aperturerobotics/chonky"; -import { ChonkyActions, ChonkyFileActionData, FileData } from "@aperturerobotics/chonky"; +import { FileBrowser, FileNavbar, FileToolbar, FileList, FileContextMenu, FileArray, FileBrowserHandle } from "@samuelncui/chonky"; +import { ChonkyActions, ChonkyFileActionData, FileData } from "@samuelncui/chonky"; + +import { ToobarInfo } from "../components/toolbarInfo"; import { cli, convertFiles } from "../api"; import { Root } from "../api"; import { AddFileAction, RefreshListAction, CreateRestoreJobAction } from "../actions"; import { JobCreateRequest, JobRestoreParam, Source } from "../entity"; -const useRestoreSourceBrowser = (source: RefObject) => { +const useRestoreSourceBrowser = (target: RefObject) => { const [files, setFiles] = useState(Array(1).fill(null)); const [folderChain, setFolderChan] = useState([Root]); const openFolder = useCallback(async (id: string) => { const [file, folderChain] = await Promise.all([cli.fileGet({ id: BigInt(id) }).response, cli.fileListParents({ id: BigInt(id) }).response]); - setFiles(convertFiles(file.children)); + setFiles(convertFiles(file.children).map((file) => ({ ...file, droppable: false }))); setFolderChan([Root, ...convertFiles(folderChain.parents)]); }, []); useEffect(() => { @@ -26,7 +28,6 @@ const useRestoreSourceBrowser = (source: RefObject) => { const onFileAction = useCallback( (data: ChonkyFileActionData) => { - console.log("source", data); switch (data.id) { case ChonkyActions.OpenFiles.id: (async () => { @@ -45,8 +46,8 @@ const useRestoreSourceBrowser = (source: RefObject) => { return; case ChonkyActions.EndDragNDrop.id: - (() => { - if (!source.current) { + (async () => { + if (!target.current) { return; } @@ -54,15 +55,22 @@ const useRestoreSourceBrowser = (source: RefObject) => { .filter((file): file is FileData => !!file && file.id !== "0") .map((file) => file.name) .join("/"); - source.current.requestFileAction(AddFileAction, { - ...data.payload, - selectedFiles: data.payload.selectedFiles.map((file) => ({ ...file, name: base + "/" + file.name })), - }); + + const selectedFiles = data.payload.selectedFiles.map((file) => ({ + ...file, + name: base ? base + "/" + file.name : file.name, + openable: false, + draggable: false, + })); + await target.current.requestFileAction(AddFileAction, { ...data.payload, selectedFiles }); })(); + return; } + + console.log("source done", data); }, - [openFolder, source, folderChain], + [openFolder, target, folderChain], ); const fileActions = useMemo(() => [ChonkyActions.StartDragNDrop, RefreshListAction], []); @@ -81,7 +89,7 @@ const useRestoreTargetBrowser = () => { const [files, setFiles] = useState(Array(0)); const [folderChain, setFolderChan] = useState([ { - id: "0", + id: "restore_waitlist", name: "Restore Waitlist", isDir: true, openable: true, @@ -93,7 +101,6 @@ const useRestoreTargetBrowser = () => { const onFileAction = useCallback( (data: ChonkyFileActionData) => { - console.log("target", data); switch (data.id) { case ChonkyActions.DeleteFiles.id: (() => { @@ -141,7 +148,9 @@ export const RestoreBrowser = () => { - + + + @@ -149,7 +158,9 @@ export const RestoreBrowser = () => { - + + + diff --git a/frontend/src/pages/tapes.tsx b/frontend/src/pages/tapes.tsx index 3b9a9de..cdda864 100644 --- a/frontend/src/pages/tapes.tsx +++ b/frontend/src/pages/tapes.tsx @@ -1,13 +1,12 @@ import { useState, useEffect, useMemo, useCallback, FC, useRef, RefObject } from "react"; -import moment from "moment"; import Grid from "@mui/material/Grid"; import Box from "@mui/material/Box"; -import { FileBrowser, FileNavbar, FileToolbar, FileList, FileContextMenu, FileArray, FileBrowserHandle } from "@aperturerobotics/chonky"; -import { ChonkyActions, ChonkyFileActionData, FileData } from "@aperturerobotics/chonky"; +import { FileBrowser, FileNavbar, FileToolbar, FileList, FileContextMenu, FileArray, FileBrowserHandle } from "@samuelncui/chonky"; +import { ChonkyActions, ChonkyFileActionData, FileData } from "@samuelncui/chonky"; import { cli, Root, convertTapes, convertPositions } from "../api"; -import { TapeListRequest, Source, Tape, Position } from "../entity"; +import { TrimLibraryAction } from "../actions"; export const TapesType = "tapes"; @@ -49,7 +48,6 @@ const useTapesSourceBrowser = (source: RefObject) => { const reply = await cli.tapeGetPositions({ id: BigInt(tapeIDStr), directory: dir }).response; const files = convertPositions(reply.positions); - console.log("refresh jobs list, target= ", target, "tape_id= ", tapeIDStr, "dir= ", dir, "reply= ", reply, "files= ", files); setFiles(files); const targetFolderChain = []; @@ -105,12 +103,22 @@ const useTapesSourceBrowser = (source: RefObject) => { await openFolder(current); })(); return; + case TrimLibraryAction.id: + (async () => { + if (!confirm(`Empty pointer in library will be trimed, may cause data loss. Are you sure?`)) { + return; + } + + console.log(await cli.libraryTrim({ trimFile: true, trimPosition: true }).response); + alert("Trim Library Success!"); + })(); + return; } }, [openFolder, source, folderChain], ); - const fileActions = useMemo(() => [ChonkyActions.DeleteFiles], []); + const fileActions = useMemo(() => [ChonkyActions.DeleteFiles, TrimLibraryAction], []); return { files, diff --git a/go.mod b/go.mod index a7293d8..0124b4e 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 github.com/samber/lo v1.38.1 - github.com/samuelncui/acp v0.0.0-20230929123032-b9f8584ad50c + github.com/samuelncui/acp v0.0.0-20231004032618-b13b740940ae github.com/sirupsen/logrus v1.9.3 google.golang.org/grpc v1.53.0 google.golang.org/protobuf v1.30.0 @@ -29,6 +29,12 @@ require ( gorm.io/gorm v1.25.2 ) +replace ( + github.com/glebarez/sqlite => github.com/samuelncui/gorm-sqlite v0.0.0-20231004150052-7f8c4fd3e561 + gorm.io/driver/sqlite => github.com/samuelncui/gorm-sqlite v0.0.0-20231004151052-c8fdb51ac7b9 + gorm.io/gorm => github.com/samuelncui/gorm v0.0.0-20231004143348-3fe5335dfd1e +) + require ( github.com/bytedance/sonic v1.8.0 // indirect github.com/cenkalti/backoff/v4 v4.1.1 // indirect @@ -53,7 +59,7 @@ require ( github.com/lestrrat-go/strftime v1.0.6 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect - github.com/mattn/go-sqlite3 v1.14.16 // indirect + github.com/mattn/go-sqlite3 v1.14.17 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/moby/sys/mountinfo v0.6.2 // indirect @@ -63,7 +69,7 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/rs/cors v1.7.0 // indirect - github.com/samuelncui/godf v0.0.0-20230927093204-37ea5acb9fc1 // indirect + github.com/samuelncui/godf v0.0.0-20231004032257-e436410ad5a0 // indirect github.com/schollz/progressbar/v3 v3.13.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.9 // indirect diff --git a/go.sum b/go.sum index 2c8c61d..7150a12 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,6 @@ github.com/gin-gonic/gin v1.9.0 h1:OjyFBKICoexlu99ctXNR2gg+c5pKrKMuyjgARg9qeY8= github.com/gin-gonic/gin v1.9.0/go.mod h1:W1Me9+hsUSyj3CePGrd1/QrKJMSJ1Tu/0hFEH89961k= github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo= github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k= -github.com/glebarez/sqlite v1.9.0 h1:Aj6bPA12ZEx5GbSF6XADmCkYXlljPNUY+Zf1EQxynXs= -github.com/glebarez/sqlite v1.9.0/go.mod h1:YBYCoyupOao60lzp1MVBLEjZfgkq0tdB1voAQ09K9zw= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -205,7 +203,6 @@ github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LF github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -261,9 +258,8 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= -github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= +github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= @@ -369,10 +365,16 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/samuelncui/acp v0.0.0-20230929123032-b9f8584ad50c h1:xJVq1UOaqjI3JVGUQvT+w6584UdEBGzxy7WN8XXuSnk= -github.com/samuelncui/acp v0.0.0-20230929123032-b9f8584ad50c/go.mod h1:HDBJGNFN6yd3kWuCU5eKaCICvmCwVWb6AzFS+wSKyWQ= -github.com/samuelncui/godf v0.0.0-20230927093204-37ea5acb9fc1 h1:K2m4b66nzupWlkfUPJKIw2tgz4aDociv5XwtlynwbzI= -github.com/samuelncui/godf v0.0.0-20230927093204-37ea5acb9fc1/go.mod h1:lGc26yUHA5Fr2Cm/FzlkwCQJ9VtBUK9cue56biDDnWo= +github.com/samuelncui/acp v0.0.0-20231004032618-b13b740940ae h1:+qtA0L1BCdGRMX+TpTk/uhfHluqw/hlyGobUZpHCeOc= +github.com/samuelncui/acp v0.0.0-20231004032618-b13b740940ae/go.mod h1:hGza8YRSjJZqNNT4INmxVM/26VJnr31LRZNwULopZTA= +github.com/samuelncui/godf v0.0.0-20231004032257-e436410ad5a0 h1:Xp01x8L8AAhrMkZpHKezRC1Hv0sXDCkpahXd3OORFLg= +github.com/samuelncui/godf v0.0.0-20231004032257-e436410ad5a0/go.mod h1:lGc26yUHA5Fr2Cm/FzlkwCQJ9VtBUK9cue56biDDnWo= +github.com/samuelncui/gorm v0.0.0-20231004143348-3fe5335dfd1e h1:zgYmcGYZFiEEWinCmOgPn/1tFg5ziGpkLH5AK41dByw= +github.com/samuelncui/gorm v0.0.0-20231004143348-3fe5335dfd1e/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +github.com/samuelncui/gorm-sqlite v0.0.0-20231004150052-7f8c4fd3e561 h1:IF5M8kCRKUPXR0RXkmUwZ+wyrY7/u6LpToV07d7VaTE= +github.com/samuelncui/gorm-sqlite v0.0.0-20231004150052-7f8c4fd3e561/go.mod h1:YBYCoyupOao60lzp1MVBLEjZfgkq0tdB1voAQ09K9zw= +github.com/samuelncui/gorm-sqlite v0.0.0-20231004151052-c8fdb51ac7b9 h1:BDJLjGVwAWw+XC5it/EqMSP0WiNAy/lmQfWDyHWOXLE= +github.com/samuelncui/gorm-sqlite v0.0.0-20231004151052-c8fdb51ac7b9/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4= github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE= github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= @@ -619,12 +621,6 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/mysql v1.3.6 h1:BhX1Y/RyALb+T9bZ3t07wLnPZBukt+IRkMn8UZSNbGM= gorm.io/driver/mysql v1.3.6/go.mod h1:sSIebwZAVPiT+27jK9HIwvsqOGKx3YMPmrA3mBJR10c= -gorm.io/driver/sqlite v1.3.6 h1:Fi8xNYCUplOqWiPa3/GuCeowRNBRGTf62DEmhMDHeQQ= -gorm.io/driver/sqlite v1.3.6/go.mod h1:Sg1/pvnKtbQ7jLXxfZa+jSHvoX8hoZA8cn4xllOMTgE= -gorm.io/gorm v1.23.4/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= -gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= -gorm.io/gorm v1.25.2 h1:gs1o6Vsa+oVKG/a9ElL3XgyGfghFfkKA2SInQaCyMho= -gorm.io/gorm v1.25.2/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/library/file.go b/library/file.go index a727650..b64a0fc 100644 --- a/library/file.go +++ b/library/file.go @@ -7,10 +7,12 @@ import ( "fmt" "io/fs" "path" + "sort" "strings" "time" mapset "github.com/deckarep/golang-set/v2" + "github.com/samber/lo" "github.com/sirupsen/logrus" "gorm.io/gorm" ) @@ -340,6 +342,30 @@ func (l *Library) List(ctx context.Context, parentID int64) ([]*File, error) { return l.list(ctx, l.db.WithContext(ctx), parentID) } +func (l *Library) ListWithSize(ctx context.Context, parentID int64) ([]*File, error) { + all, err := l.listAll(ctx, l.db.WithContext(ctx), parentID) + if err != nil { + return nil, err + } + + mapping := lo.GroupBy(all, func(file *File) int64 { return file.ParentID }) + var fetchSize func(file *File) + fetchSize = func(file *File) { + for _, child := range mapping[file.ID] { + fetchSize(child) + file.Size += child.Size + } + } + + files := mapping[parentID] + for _, f := range files { + fetchSize(f) + } + + sort.Slice(files, func(i, j int) bool { return files[i].Name < files[j].Name }) + return files, nil +} + func (l *Library) list(ctx context.Context, tx *gorm.DB, parentID int64) ([]*File, error) { files := make([]*File, 0, 4) if r := tx.Where("parent_id = ?", parentID).Order("name").Find(&files); r.Error != nil { @@ -348,6 +374,38 @@ func (l *Library) list(ctx context.Context, tx *gorm.DB, parentID int64) ([]*Fil return files, nil } +func (l *Library) listAll(ctx context.Context, tx *gorm.DB, parentIDs ...int64) ([]*File, error) { + files := make([]*File, 0, 4) + + current := parentIDs + for { + batch := make([]*File, 0, 4) + if r := tx.Where("parent_id IN (?)", current).Find(&batch); r.Error != nil { + return nil, fmt.Errorf("find files fail, %w", r.Error) + } + + if len(batch) == 0 { + break + } + + files = append(files, batch...) + next := make([]int64, 0, 4) + for _, f := range batch { + if !fs.FileMode(f.Mode).IsDir() { + continue + } + next = append(next, f.ID) + } + + if len(next) == 0 { + break + } + current = next + } + + return files, nil +} + func (l *Library) ListParents(ctx context.Context, id int64) ([]*File, error) { return l.listParnets(ctx, l.db.WithContext(ctx), id) } diff --git a/library/library.go b/library/library.go index 5a2586a..2c5a30a 100644 --- a/library/library.go +++ b/library/library.go @@ -4,6 +4,9 @@ import ( "context" "encoding/json" "fmt" + "io/fs" + "sort" + "strings" mapset "github.com/deckarep/golang-set/v2" "github.com/modern-go/reflect2" @@ -157,26 +160,56 @@ func (l *Library) Trim(ctx context.Context, position, file bool) error { } current = files[len(files)-1].ID - fileIDs := lo.Map(files, func(f *File, _ int) int64 { return f.ID }) + fileIDs := lo.Map( + lo.Filter(files, func(f *File, _ int) bool { return fs.FileMode(f.Mode).IsRegular() }), + func(f *File, _ int) int64 { return f.ID }, + ) positions, err := l.MGetPositionByFileID(ctx, fileIDs...) if err != nil { return fmt.Errorf("mget position by file id fail, %w", err) } - needDelete := make([]int64, 0) - for _, file := range files { - if posis, has := positions[file.ID]; has && len(posis) > 0 { + needDeleteFileIDs := make([]int64, 0) + needDeletePositionIDs := make([]int64, 0) + for _, fileID := range fileIDs { + posis, has := positions[fileID] + if !has || len(posis) == 0 { + needDeleteFileIDs = append(needDeleteFileIDs, fileID) continue } - needDelete = append(needDelete, file.ID) + if len(posis) == 1 { + continue + } + + sort.Slice(posis, func(i int, j int) bool { + ii, jj := posis[i], posis[j] + if ii.TapeID != jj.TapeID { + return ii.TapeID < jj.TapeID + } + if ii.Path != jj.Path { + return strings.ReplaceAll(ii.Path, "/", "\x00") < strings.ReplaceAll(jj.Path, "/", "\x00") + } + return ii.WriteTime.After(jj.WriteTime) + }) + for idx, posi := range posis { + if idx == 0 { + continue + } + if posis[idx-1].TapeID == posi.TapeID && posis[idx-1].Path == posi.Path { + needDeletePositionIDs = append(needDeletePositionIDs, posi.ID) + } + } } - if len(needDelete) == 0 { - continue + if len(needDeleteFileIDs) > 0 { + if r := l.db.WithContext(ctx).Where("id IN (?)", needDeleteFileIDs).Delete(ModelFile); r.Error != nil { + return fmt.Errorf("delete files fail, err= %w", r.Error) + } } - - if r := l.db.WithContext(ctx).Where("id IN (?)", needDelete).Delete(ModelFile); r.Error != nil { - return fmt.Errorf("delete files fail, err= %w", r.Error) + if len(needDeletePositionIDs) > 0 { + if r := l.db.WithContext(ctx).Where("id IN (?)", needDeletePositionIDs).Delete(ModelPosition); r.Error != nil { + return fmt.Errorf("delete positions fail, err= %w", r.Error) + } } } diff --git a/resource/db.go b/resource/db.go index 9ae2edf..b4e84a9 100644 --- a/resource/db.go +++ b/resource/db.go @@ -35,15 +35,16 @@ func NewDBConn(dialect, dsn string) (*gorm.DB, error) { return db, nil } -func SQLEscape(sql string) string { - dest := make([]byte, 0, 2*len(sql)) - var escape byte - for i := 0; i < len(sql); i++ { - c := sql[i] +func SQLEscape(str string) string { + runes := []rune(str) + result := make([]rune, 0, len(runes)) - escape = 0 + var escape rune + for i := 0; i < len(runes); i++ { + r := runes[i] - switch c { + escape = 0 + switch r { case 0: /* Must be escaped for 'mysql' */ escape = '0' case '\n': /* Must be escaped for logs */ @@ -56,16 +57,16 @@ func SQLEscape(sql string) string { escape = '\'' case '"': /* Better safe than sorry */ escape = '"' - case '\032': //十进制26,八进制32,十六进制1a, /* This gives problems on Win32 */ + case '\032': // This gives problems on Win32 escape = 'Z' } if escape != 0 { - dest = append(dest, '\\', escape) + result = append(result, '\\', escape) } else { - dest = append(dest, c) + result = append(result, r) } } - return string(dest) + return string(result) } diff --git a/resource/sqlite_cgo.go.bak b/resource/sqlite_cgo.go.bak new file mode 100644 index 0000000..cca455d --- /dev/null +++ b/resource/sqlite_cgo.go.bak @@ -0,0 +1,14 @@ +//go:build !((darwin && amd64) || (darwin && arm64) || (freebsd && amd64) || (linux && arm) || (linux && arm64) || (linux && 386) || (linux && amd64) || (linux && s390x) || (windows && amd64)) + +package resource + +import ( + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +type innerSQLiteMigrator = sqlite.Migrator + +func openSQLite(dsn string) gorm.Dialector { + return &sqliteDialector{sqlite.Open(dsn)} +} diff --git a/resource/sqlite_fix.go.bak b/resource/sqlite_fix.go.bak new file mode 100644 index 0000000..22aa530 --- /dev/null +++ b/resource/sqlite_fix.go.bak @@ -0,0 +1,156 @@ +package resource + +import ( + "fmt" + "strings" + + "gorm.io/gorm" + "gorm.io/gorm/clause" + "gorm.io/gorm/migrator" + "gorm.io/gorm/schema" +) + +// fix primary key autoincrement problem +type sqliteDialector struct { + gorm.Dialector +} + +func (dialector *sqliteDialector) DataTypeOf(field *schema.Field) string { + switch field.DataType { + case schema.Int, schema.Uint: + if field.AutoIncrement { + // https://www.sqlite.org/autoinc.html + return "integer PRIMARY KEY AUTOINCREMENT" + } + } + + return dialector.Dialector.DataTypeOf(field) +} + +func (dialector *sqliteDialector) Migrator(db *gorm.DB) gorm.Migrator { + return sqliteMigrator{innerSQLiteMigrator{Migrator: migrator.Migrator{Config: migrator.Config{ + DB: db, + Dialector: dialector, + CreateIndexAfterCreateTable: true, + }}}} +} + +type sqliteMigrator struct { + innerSQLiteMigrator +} + +// CreateTable create table in database for values +func (m sqliteMigrator) CreateTable(values ...interface{}) error { + for _, value := range m.ReorderModels(values, false) { + tx := m.DB.Session(&gorm.Session{}) + if err := m.RunWithValue(value, func(stmt *gorm.Statement) (err error) { + var ( + createTableSQL = "CREATE TABLE ? (" + values = []interface{}{m.CurrentTable(stmt)} + hasPrimaryKeyInDataType bool + ) + + for _, dbName := range stmt.Schema.DBNames { + field := stmt.Schema.FieldsByDBName[dbName] + if !field.IgnoreMigration { + createTableSQL += "? ?" + hasPrimaryKeyInDataType = hasPrimaryKeyInDataType || strings.Contains(strings.ToUpper(m.DataTypeOf(field)), "PRIMARY KEY") + values = append(values, clause.Column{Name: dbName}, m.DB.Migrator().FullDataTypeOf(field)) + createTableSQL += "," + } + } + + if !hasPrimaryKeyInDataType && len(stmt.Schema.PrimaryFields) > 0 { + createTableSQL += "PRIMARY KEY ?," + primaryKeys := make([]interface{}, 0, len(stmt.Schema.PrimaryFields)) + for _, field := range stmt.Schema.PrimaryFields { + primaryKeys = append(primaryKeys, clause.Column{Name: field.DBName}) + } + + values = append(values, primaryKeys) + } + + for _, idx := range stmt.Schema.ParseIndexes() { + if m.CreateIndexAfterCreateTable { + defer func(value interface{}, name string) { + if err == nil { + err = tx.Migrator().CreateIndex(value, name) + } + }(value, idx.Name) + } else { + if idx.Class != "" { + createTableSQL += idx.Class + " " + } + createTableSQL += "INDEX ? ?" + + if idx.Comment != "" { + createTableSQL += fmt.Sprintf(" COMMENT '%s'", idx.Comment) + } + + if idx.Option != "" { + createTableSQL += " " + idx.Option + } + + createTableSQL += "," + values = append(values, clause.Column{Name: idx.Name}, tx.Migrator().(migrator.BuildIndexOptionsInterface).BuildIndexOptions(idx.Fields, stmt)) + } + } + + if !m.DB.DisableForeignKeyConstraintWhenMigrating && !m.DB.IgnoreRelationshipsWhenMigrating { + for _, rel := range stmt.Schema.Relationships.Relations { + if rel.Field.IgnoreMigration { + continue + } + if constraint := rel.ParseConstraint(); constraint != nil { + if constraint.Schema == stmt.Schema { + sql, vars := buildConstraint(constraint) + createTableSQL += sql + "," + values = append(values, vars...) + } + } + } + } + + for _, chk := range stmt.Schema.ParseCheckConstraints() { + createTableSQL += "CONSTRAINT ? CHECK (?)," + values = append(values, clause.Column{Name: chk.Name}, clause.Expr{SQL: chk.Constraint}) + } + + createTableSQL = strings.TrimSuffix(createTableSQL, ",") + + createTableSQL += ")" + + if tableOption, ok := m.DB.Get("gorm:table_options"); ok { + createTableSQL += fmt.Sprint(tableOption) + } + + err = tx.Exec(createTableSQL, values...).Error + return err + }); err != nil { + return err + } + } + return nil +} + +func buildConstraint(constraint *schema.Constraint) (sql string, results []interface{}) { + sql = "CONSTRAINT ? FOREIGN KEY ? REFERENCES ??" + if constraint.OnDelete != "" { + sql += " ON DELETE " + constraint.OnDelete + } + + if constraint.OnUpdate != "" { + sql += " ON UPDATE " + constraint.OnUpdate + } + + var foreignKeys, references []interface{} + for _, field := range constraint.ForeignKeys { + foreignKeys = append(foreignKeys, clause.Column{Name: field.DBName}) + } + + for _, field := range constraint.References { + references = append(references, clause.Column{Name: field.DBName}) + } + results = append(results, clause.Table{Name: constraint.Name}, foreignKeys, clause.Table{Name: constraint.ReferenceSchema.Table}, references) + return +} diff --git a/resource/sqlite.go b/resource/sqlite_pure_go.go similarity index 100% rename from resource/sqlite.go rename to resource/sqlite_pure_go.go diff --git a/resource/sqlite_pure_go.go.bak b/resource/sqlite_pure_go.go.bak new file mode 100644 index 0000000..dde2d29 --- /dev/null +++ b/resource/sqlite_pure_go.go.bak @@ -0,0 +1,14 @@ +//go:build (darwin && amd64) || (darwin && arm64) || (freebsd && amd64) || (linux && arm) || (linux && arm64) || (linux && 386) || (linux && amd64) || (linux && s390x) || (windows && amd64) + +package resource + +import ( + "github.com/glebarez/sqlite" + "gorm.io/gorm" +) + +type innerSQLiteMigrator = sqlite.Migrator + +func openSQLite(dsn string) gorm.Dialector { + return &sqliteDialector{sqlite.Open(dsn)} +}