From 61c78e32517257aeb67aa8d6ab339868b19e6235 Mon Sep 17 00:00:00 2001 From: beebeeoii Date: Wed, 10 Aug 2022 09:37:20 +0800 Subject: [PATCH 1/5] fix: mismatch in canvas file names --- pkg/api/files.go | 2 +- pkg/interfaces/File.go | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/pkg/api/files.go b/pkg/api/files.go index 798b640..8c0e8b3 100644 --- a/pkg/api/files.go +++ b/pkg/api/files.go @@ -165,7 +165,7 @@ func (filesRequest FilesRequest) GetFiles() ([]File, error) { files = append(files, File{ Id: strconv.Itoa(fileObject.Id), - Name: appFile.CleanseFolderFileName(fileObject.Name), + Name: appFile.CleanseFolderFileName(fileObject.DisplayName), LastUpdated: lastUpdated, Ancestors: ancestors, DownloadUrl: fileObject.Url, diff --git a/pkg/interfaces/File.go b/pkg/interfaces/File.go index 126cb9c..c3a0b86 100644 --- a/pkg/interfaces/File.go +++ b/pkg/interfaces/File.go @@ -2,8 +2,16 @@ package interfaces // TODO Documentation type CanvasFileObject struct { - Id int `json:"id"` - Name string `json:"filename"` + Id int `json:"id"` + Name string `json:"filename"` + // DisplayName is the name that is seen on Canvas Web. It can differ + // from the actual name of the file being uploaded. + // Using DisplayName would be more accurate as Professors might + // set the DisplayName to contain more information for the File. + // For eg. filename can be "Tutorial1.pdf" but DisplayName can be + // "Tutorial1_HW1.pdf" to show that Tutorial 1 is to be submitted as + // a graded Homework. + DisplayName string `json:"display_name"` UUID string `json:"uuid"` Url string `json:"url"` HiddenForUser bool `json:"hidden_for_user"` From 2e53d91bf026bd7bc224c25b59560ff8d20a0bdc Mon Sep 17 00:00:00 2001 From: beebeeoii Date: Wed, 10 Aug 2022 09:39:09 +0800 Subject: [PATCH 2/5] fix: find but not download canvas root folder --- pkg/api/files.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pkg/api/files.go b/pkg/api/files.go index 8c0e8b3..60f1feb 100644 --- a/pkg/api/files.go +++ b/pkg/api/files.go @@ -78,10 +78,21 @@ func (foldersRequest FoldersRequest) GetFolders() ([]Folder, error) { } for _, folderObject := range response { + // All the folders and files of a module are stored under the "course files" folder. + // We do not want to download that folder as we just want the folders and files in that + // folder. + // + // The "course files" folder resembles the 'home directory' of a module. + downloadable := !folderObject.HiddenForUser + + if folderObject.FullName == "course files" { + downloadable = false + } + folders = append(folders, Folder{ Id: strconv.Itoa(folderObject.Id), Name: appFile.CleanseFolderFileName(folderObject.Name), - Downloadable: !folderObject.HiddenForUser, + Downloadable: downloadable, HasSubFolder: folderObject.FoldersCount > 0, Ancestors: ancestors, IsRootFolder: folderObject.ParentFolderId == 0 && From 5fc21a768de7a004b64f3ec43b3f5226676433ef Mon Sep 17 00:00:00 2001 From: beebeeoii Date: Wed, 10 Aug 2022 09:39:40 +0800 Subject: [PATCH 3/5] enhancement: GetRootFiles recursively --- pkg/api/files.go | 106 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/pkg/api/files.go b/pkg/api/files.go index 60f1feb..cf08fad 100644 --- a/pkg/api/files.go +++ b/pkg/api/files.go @@ -142,6 +142,112 @@ func (foldersRequest FoldersRequest) GetFolders() ([]Folder, error) { return folders, nil } +// GetRootFiles returns a slice of File objects and nested File objects that are in a Folder. +// It will traverse all nested folders and return all nested files. +func (foldersRequest FoldersRequest) GetRootFiles() ([]File, error) { + files := []File{} + + if foldersRequest.Request.Token == "" { + return files, nil + } + + switch builder := foldersRequest.Builder.(type) { + case Module: + // Module exists but its contents are restricted to be downloaded. + if !builder.IsAccessible { + return files, nil + } + + // Retrieving of folders in main folder is only required for Luminus + // as Canvas already returns its + if foldersRequest.Request.Url.Platform != constants.Luminus { + break + } + + moduleMainFolder := Folder{ + Id: builder.Id, + Name: builder.Name, + Downloadable: true, + HasSubFolder: true, // doesn't matter + Ancestors: []string{}, // main folder does not have any ancestors + } + subFilesReq, subFilesReqErr := BuildFilesRequest( + foldersRequest.Request.Token, + foldersRequest.Request.Url.Platform, + moduleMainFolder, + ) + if subFilesReqErr != nil { + return files, subFilesReqErr + } + + subFiles, subFilesErr := subFilesReq.GetFiles() + if subFilesErr != nil { + return files, subFilesErr + } + + files = append(files, subFiles...) + case Folder: + // Folder exists but its contents are restricted to be downloaded. + if !builder.Downloadable { + return files, nil + } + + subFilesReq, subFilesReqErr := BuildFilesRequest( + foldersRequest.Request.Token, + foldersRequest.Request.Url.Platform, + builder, + ) + if subFilesReqErr != nil { + return files, subFilesReqErr + } + + subFiles, subFilesErr := subFilesReq.GetFiles() + if subFilesErr != nil { + return files, subFilesErr + } + + files = append(files, subFiles...) + + if !builder.HasSubFolder { + break + } + } + + subFoldersReq, subFoldersReqErr := BuildFoldersRequest( + foldersRequest.Request.Token, + foldersRequest.Request.Url.Platform, + foldersRequest.Builder, + ) + if subFoldersReqErr != nil { + return files, subFoldersReqErr + } + + subFolders, subFoldersErr := subFoldersReq.GetFolders() + if subFoldersErr != nil { + return files, subFoldersErr + } + + for _, subFolder := range subFolders { + nestedFoldersReq, nestedFoldersReqErr := BuildFoldersRequest( + foldersRequest.Request.Token, + foldersRequest.Request.Url.Platform, + subFolder, + ) + if nestedFoldersReqErr != nil { + return files, nestedFoldersReqErr + } + + nestedFiles, nestedFilesErr := nestedFoldersReq.GetRootFiles() + if nestedFilesErr != nil { + return files, nestedFilesErr + } + + files = append(files, nestedFiles...) + } + + return files, nil +} + func (filesRequest FilesRequest) GetFiles() ([]File, error) { files := []File{} From 3ba90bd0413f1c3807648b1cd5412343c68c97c3 Mon Sep 17 00:00:00 2001 From: beebeeoii Date: Wed, 10 Aug 2022 09:43:17 +0800 Subject: [PATCH 4/5] fix: builder for Canvas from Module to Folder --- pkg/api/request.go | 49 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/pkg/api/request.go b/pkg/api/request.go index 4531e0d..e22d5f2 100644 --- a/pkg/api/request.go +++ b/pkg/api/request.go @@ -90,22 +90,61 @@ func BuildModulesRequest(token string, platform constants.Platform) (ModulesRequ func BuildFoldersRequest(token string, platform constants.Platform, builder interface{}) (FoldersRequest, error) { var url string - switch builder := builder.(type) { + switch b := builder.(type) { case Module: switch p := platform; p { case constants.Canvas: - url = fmt.Sprintf(constants.CANVAS_MODULE_FOLDERS_ENDPOINT, builder.Id) + url = fmt.Sprintf(constants.CANVAS_MODULE_FOLDERS_ENDPOINT, b.Id) + folderRequest := FoldersRequest{ + Request: Request{ + Method: GET_METHOD, + Token: token, + Url: interfaces.Url{ + Url: url, + Platform: platform, + }, + UserAgent: USER_AGENT, + }, + Builder: b, + } + + folders, foldersErr := folderRequest.GetFolders() + if foldersErr != nil { + return folderRequest, foldersErr + } + + var rootFolderId string + for _, folder := range folders { + if folder.Name == "course files" { + rootFolderId = folder.Id + break + } + } + + if rootFolderId == "" { + return folderRequest, foldersErr + } + + url = fmt.Sprintf(constants.CANVAS_FOLDERS_ENDPOINT, b.Id) + + builder = Folder{ + Id: rootFolderId, + Name: b.ModuleCode, + Downloadable: b.IsAccessible, + HasSubFolder: true, + Ancestors: []string{}, + } case constants.Luminus: - url = fmt.Sprintf(FOLDER_URL_ENDPOINT, builder.Id) + url = fmt.Sprintf(FOLDER_URL_ENDPOINT, b.Id) default: return FoldersRequest{}, errors.New("invalid platform provided") } case Folder: switch p := platform; p { case constants.Canvas: - url = fmt.Sprintf(constants.CANVAS_FOLDERS_ENDPOINT, builder.Id) + url = fmt.Sprintf(constants.CANVAS_FOLDERS_ENDPOINT, b.Id) case constants.Luminus: - url = fmt.Sprintf(FOLDER_URL_ENDPOINT, builder.Id) + url = fmt.Sprintf(FOLDER_URL_ENDPOINT, b.Id) default: return FoldersRequest{}, errors.New("invalid platform provided") } From ee4c170e698eec479599543256390a588852ed14 Mon Sep 17 00:00:00 2001 From: beebeeoii Date: Wed, 10 Aug 2022 09:43:40 +0800 Subject: [PATCH 5/5] refactor: utilise GetRootFIles --- internal/cron/cron.go | 110 +++++++++++------------------------------- 1 file changed, 29 insertions(+), 81 deletions(-) diff --git a/internal/cron/cron.go b/internal/cron/cron.go index 9c9fd56..ea8b68b 100644 --- a/internal/cron/cron.go +++ b/internal/cron/cron.go @@ -173,67 +173,47 @@ func createJob(frequency int) (*gocron.Job, error) { return } - canvasFolders := []api.Folder{} + lmsFiles := []api.File{} for _, module := range canvasModules { - // Note that there is no need to ensure that the files in the module's root - // folder are downloaded. They are already downloaded as those files are - // already put into a folder "course files" by Canvas which are downloaded. - // Check out files.go GetFiles() for more details. - folders, canvasFoldersErr := getFolders(tokensData.CanvasToken.CanvasApiToken, constants.Canvas, module) - if canvasFoldersErr != nil { - // TODO Somehow collate this error and display to user at the end - // notifications.NotificationChannel <- notifications.Notification{Title: "Sync", Content: canvasFoldersErr.Error()} - logs.Logger.Errorln(canvasFoldersErr) + foldersReq, foldersReqErr := api.BuildFoldersRequest( + tokensData.CanvasToken.CanvasApiToken, + constants.Canvas, + module, + ) + if foldersReqErr != nil { + logs.Logger.Errorln(foldersReqErr) } - canvasFolders = append(canvasFolders, folders...) - } - luminusFolders := []api.Folder{} - for _, module := range luminusModules { - // This ensures that files in the module's root folder are downloaded as well. - moduleMainFolder := api.Folder{ - Id: module.Id, - Name: module.Name, - Downloadable: module.IsAccessible, - HasSubFolder: true, // doesn't matter - Ancestors: []string{}, // main folder does not have any ancestors + files, foldersErr := foldersReq.GetRootFiles() + if foldersErr != nil { + logs.Logger.Errorln(foldersErr) } - folders, luminusFoldersErr := getFolders(tokensData.LuminusToken.JwtToken, constants.Luminus, module) - if luminusFoldersErr != nil { - // TODO Somehow collate this error and display to user at the end - // notifications.NotificationChannel <- notifications.Notification{Title: "Sync", Content: luminusFoldersErr.Error()} - logs.Logger.Errorln(luminusFoldersErr) - } - luminusFolders = append(luminusFolders, moduleMainFolder) - luminusFolders = append(luminusFolders, folders...) + + lmsFiles = append(lmsFiles, files...) } - nFilesToUpdate := 0 - filesUpdated := []api.File{} + for _, module := range luminusModules { + foldersReq, foldersReqErr := api.BuildFoldersRequest( + tokensData.LuminusToken.JwtToken, + constants.Luminus, + module, + ) + if foldersReqErr != nil { + logs.Logger.Errorln(foldersReqErr) + } - files := []api.File{} - for _, folder := range canvasFolders { - canvasFiles, canvasFilesErr := getFiles(tokensData.CanvasToken.CanvasApiToken, constants.Canvas, folder) - if canvasFilesErr != nil { - // TODO Somehow collate this error and display to user at the end - // notifications.NotificationChannel <- notifications.Notification{Title: "Sync", Content: canvasFilesErr.Error()} - logs.Logger.Errorln(canvasFilesErr) + files, foldersErr := foldersReq.GetRootFiles() + if foldersErr != nil { + logs.Logger.Errorln(foldersErr) } - files = append(files, canvasFiles...) + lmsFiles = append(lmsFiles, files...) } - for _, folder := range luminusFolders { - luminusFiles, luminusFilesErr := getFiles(tokensData.LuminusToken.JwtToken, constants.Luminus, folder) - if luminusFilesErr != nil { - // TODO Somehow collate this error and display to user at the end - // notifications.NotificationChannel <- notifications.Notification{Title: "Sync", Content: luminusFilesErr.Error()} - logs.Logger.Errorln(luminusFilesErr) - } - files = append(files, luminusFiles...) - } + nFilesToUpdate := 0 + filesUpdated := []api.File{} - for _, file := range files { + for _, file := range lmsFiles { key := fmt.Sprintf("%s/%s", strings.Join(file.Ancestors, "/"), file.Name) localLastUpdated := currentFiles[key].LastUpdated platformLastUpdated := file.LastUpdated @@ -328,38 +308,6 @@ func getModules(token string, platform constants.Platform) ([]api.Module, error) return modules, nil } -func getFolders(token string, platform constants.Platform, module api.Module) ([]api.Folder, error) { - folders := []api.Folder{} - - foldersReq, foldersReqErr := api.BuildFoldersRequest(token, platform, module) - if foldersReqErr != nil { - return folders, foldersReqErr - } - - folders, foldersErr := foldersReq.GetFolders() - if foldersErr != nil { - return folders, foldersErr - } - - return folders, nil -} - -func getFiles(token string, platform constants.Platform, folder api.Folder) ([]api.File, error) { - files := []api.File{} - - filesReq, filesReqErr := api.BuildFilesRequest(token, platform, folder) - if filesReqErr != nil { - return files, filesReqErr - } - - files, filesErr := filesReq.GetFiles() - if filesErr != nil { - return files, filesErr - } - - return files, nil -} - func getGrades(modules []api.Module) ([]api.Grade, error) { grades := []api.Grade{}