Skip to content

Commit

Permalink
Merge pull request #70 from Beebeeoii/fix-nested-folders-not-retrieved
Browse files Browse the repository at this point in the history 
fix: Nested folders not retrieved / in wrong directory
  • Loading branch information
Jia Wei Lee authored Aug 10, 2022
2 parents eeb51a0 + ee4c170 commit e6d58b9
Show file tree
Hide file tree
Showing 4 changed files with 202 additions and 90 deletions.
110 changes: 29 additions & 81 deletions internal/cron/cron.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,67 +173,47 @@ func createJob(frequency int) (*gocron.Job, error) {
return
}

canvasFolders := []api.Folder{}
lmsFiles := []api.File{}
for _, module := range canvasModules {
// Note that there is no need to ensure that the files in the module's root
// folder are downloaded. They are already downloaded as those files are
// already put into a folder "course files" by Canvas which are downloaded.
// Check out files.go GetFiles() for more details.
folders, canvasFoldersErr := getFolders(tokensData.CanvasToken.CanvasApiToken, constants.Canvas, module)
if canvasFoldersErr != nil {
// TODO Somehow collate this error and display to user at the end
// notifications.NotificationChannel <- notifications.Notification{Title: "Sync", Content: canvasFoldersErr.Error()}
logs.Logger.Errorln(canvasFoldersErr)
foldersReq, foldersReqErr := api.BuildFoldersRequest(
tokensData.CanvasToken.CanvasApiToken,
constants.Canvas,
module,
)
if foldersReqErr != nil {
logs.Logger.Errorln(foldersReqErr)
}
canvasFolders = append(canvasFolders, folders...)
}

luminusFolders := []api.Folder{}
for _, module := range luminusModules {
// This ensures that files in the module's root folder are downloaded as well.
moduleMainFolder := api.Folder{
Id: module.Id,
Name: module.Name,
Downloadable: module.IsAccessible,
HasSubFolder: true, // doesn't matter
Ancestors: []string{}, // main folder does not have any ancestors
files, foldersErr := foldersReq.GetRootFiles()
if foldersErr != nil {
logs.Logger.Errorln(foldersErr)
}
folders, luminusFoldersErr := getFolders(tokensData.LuminusToken.JwtToken, constants.Luminus, module)
if luminusFoldersErr != nil {
// TODO Somehow collate this error and display to user at the end
// notifications.NotificationChannel <- notifications.Notification{Title: "Sync", Content: luminusFoldersErr.Error()}
logs.Logger.Errorln(luminusFoldersErr)
}
luminusFolders = append(luminusFolders, moduleMainFolder)
luminusFolders = append(luminusFolders, folders...)

lmsFiles = append(lmsFiles, files...)
}

nFilesToUpdate := 0
filesUpdated := []api.File{}
for _, module := range luminusModules {
foldersReq, foldersReqErr := api.BuildFoldersRequest(
tokensData.LuminusToken.JwtToken,
constants.Luminus,
module,
)
if foldersReqErr != nil {
logs.Logger.Errorln(foldersReqErr)
}

files := []api.File{}
for _, folder := range canvasFolders {
canvasFiles, canvasFilesErr := getFiles(tokensData.CanvasToken.CanvasApiToken, constants.Canvas, folder)
if canvasFilesErr != nil {
// TODO Somehow collate this error and display to user at the end
// notifications.NotificationChannel <- notifications.Notification{Title: "Sync", Content: canvasFilesErr.Error()}
logs.Logger.Errorln(canvasFilesErr)
files, foldersErr := foldersReq.GetRootFiles()
if foldersErr != nil {
logs.Logger.Errorln(foldersErr)
}
files = append(files, canvasFiles...)

lmsFiles = append(lmsFiles, files...)
}

for _, folder := range luminusFolders {
luminusFiles, luminusFilesErr := getFiles(tokensData.LuminusToken.JwtToken, constants.Luminus, folder)
if luminusFilesErr != nil {
// TODO Somehow collate this error and display to user at the end
// notifications.NotificationChannel <- notifications.Notification{Title: "Sync", Content: luminusFilesErr.Error()}
logs.Logger.Errorln(luminusFilesErr)
}
files = append(files, luminusFiles...)
}
nFilesToUpdate := 0
filesUpdated := []api.File{}

for _, file := range files {
for _, file := range lmsFiles {
key := fmt.Sprintf("%s/%s", strings.Join(file.Ancestors, "/"), file.Name)
localLastUpdated := currentFiles[key].LastUpdated
platformLastUpdated := file.LastUpdated
Expand Down Expand Up @@ -328,38 +308,6 @@ func getModules(token string, platform constants.Platform) ([]api.Module, error)
return modules, nil
}

func getFolders(token string, platform constants.Platform, module api.Module) ([]api.Folder, error) {
folders := []api.Folder{}

foldersReq, foldersReqErr := api.BuildFoldersRequest(token, platform, module)
if foldersReqErr != nil {
return folders, foldersReqErr
}

folders, foldersErr := foldersReq.GetFolders()
if foldersErr != nil {
return folders, foldersErr
}

return folders, nil
}

func getFiles(token string, platform constants.Platform, folder api.Folder) ([]api.File, error) {
files := []api.File{}

filesReq, filesReqErr := api.BuildFilesRequest(token, platform, folder)
if filesReqErr != nil {
return files, filesReqErr
}

files, filesErr := filesReq.GetFiles()
if filesErr != nil {
return files, filesErr
}

return files, nil
}

func getGrades(modules []api.Module) ([]api.Grade, error) {
grades := []api.Grade{}

Expand Down
121 changes: 119 additions & 2 deletions pkg/api/files.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,21 @@ func (foldersRequest FoldersRequest) GetFolders() ([]Folder, error) {
}

for _, folderObject := range response {
// All the folders and files of a module are stored under the "course files" folder.
// We do not want to download that folder as we just want the folders and files in that
// folder.
//
// The "course files" folder resembles the 'home directory' of a module.
downloadable := !folderObject.HiddenForUser

if folderObject.FullName == "course files" {
downloadable = false
}

folders = append(folders, Folder{
Id: strconv.Itoa(folderObject.Id),
Name: appFile.CleanseFolderFileName(folderObject.Name),
Downloadable: !folderObject.HiddenForUser,
Downloadable: downloadable,
HasSubFolder: folderObject.FoldersCount > 0,
Ancestors: ancestors,
IsRootFolder: folderObject.ParentFolderId == 0 &&
Expand Down Expand Up @@ -131,6 +142,112 @@ func (foldersRequest FoldersRequest) GetFolders() ([]Folder, error) {
return folders, nil
}

// GetRootFiles returns a slice of File objects and nested File objects that are in a Folder.
// It will traverse all nested folders and return all nested files.
func (foldersRequest FoldersRequest) GetRootFiles() ([]File, error) {
files := []File{}

if foldersRequest.Request.Token == "" {
return files, nil
}

switch builder := foldersRequest.Builder.(type) {
case Module:
// Module exists but its contents are restricted to be downloaded.
if !builder.IsAccessible {
return files, nil
}

// Retrieving of folders in main folder is only required for Luminus
// as Canvas already returns its
if foldersRequest.Request.Url.Platform != constants.Luminus {
break
}

moduleMainFolder := Folder{
Id: builder.Id,
Name: builder.Name,
Downloadable: true,
HasSubFolder: true, // doesn't matter
Ancestors: []string{}, // main folder does not have any ancestors
}
subFilesReq, subFilesReqErr := BuildFilesRequest(
foldersRequest.Request.Token,
foldersRequest.Request.Url.Platform,
moduleMainFolder,
)
if subFilesReqErr != nil {
return files, subFilesReqErr
}

subFiles, subFilesErr := subFilesReq.GetFiles()
if subFilesErr != nil {
return files, subFilesErr
}

files = append(files, subFiles...)
case Folder:
// Folder exists but its contents are restricted to be downloaded.
if !builder.Downloadable {
return files, nil
}

subFilesReq, subFilesReqErr := BuildFilesRequest(
foldersRequest.Request.Token,
foldersRequest.Request.Url.Platform,
builder,
)
if subFilesReqErr != nil {
return files, subFilesReqErr
}

subFiles, subFilesErr := subFilesReq.GetFiles()
if subFilesErr != nil {
return files, subFilesErr
}

files = append(files, subFiles...)

if !builder.HasSubFolder {
break
}
}

subFoldersReq, subFoldersReqErr := BuildFoldersRequest(
foldersRequest.Request.Token,
foldersRequest.Request.Url.Platform,
foldersRequest.Builder,
)
if subFoldersReqErr != nil {
return files, subFoldersReqErr
}

subFolders, subFoldersErr := subFoldersReq.GetFolders()
if subFoldersErr != nil {
return files, subFoldersErr
}

for _, subFolder := range subFolders {
nestedFoldersReq, nestedFoldersReqErr := BuildFoldersRequest(
foldersRequest.Request.Token,
foldersRequest.Request.Url.Platform,
subFolder,
)
if nestedFoldersReqErr != nil {
return files, nestedFoldersReqErr
}

nestedFiles, nestedFilesErr := nestedFoldersReq.GetRootFiles()
if nestedFilesErr != nil {
return files, nestedFilesErr
}

files = append(files, nestedFiles...)
}

return files, nil
}

func (filesRequest FilesRequest) GetFiles() ([]File, error) {
files := []File{}

Expand Down Expand Up @@ -165,7 +282,7 @@ func (filesRequest FilesRequest) GetFiles() ([]File, error) {

files = append(files, File{
Id: strconv.Itoa(fileObject.Id),
Name: appFile.CleanseFolderFileName(fileObject.Name),
Name: appFile.CleanseFolderFileName(fileObject.DisplayName),
LastUpdated: lastUpdated,
Ancestors: ancestors,
DownloadUrl: fileObject.Url,
Expand Down
49 changes: 44 additions & 5 deletions pkg/api/request.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,22 +90,61 @@ func BuildModulesRequest(token string, platform constants.Platform) (ModulesRequ
func BuildFoldersRequest(token string, platform constants.Platform, builder interface{}) (FoldersRequest, error) {
var url string

switch builder := builder.(type) {
switch b := builder.(type) {
case Module:
switch p := platform; p {
case constants.Canvas:
url = fmt.Sprintf(constants.CANVAS_MODULE_FOLDERS_ENDPOINT, builder.Id)
url = fmt.Sprintf(constants.CANVAS_MODULE_FOLDERS_ENDPOINT, b.Id)
folderRequest := FoldersRequest{
Request: Request{
Method: GET_METHOD,
Token: token,
Url: interfaces.Url{
Url: url,
Platform: platform,
},
UserAgent: USER_AGENT,
},
Builder: b,
}

folders, foldersErr := folderRequest.GetFolders()
if foldersErr != nil {
return folderRequest, foldersErr
}

var rootFolderId string
for _, folder := range folders {
if folder.Name == "course files" {
rootFolderId = folder.Id
break
}
}

if rootFolderId == "" {
return folderRequest, foldersErr
}

url = fmt.Sprintf(constants.CANVAS_FOLDERS_ENDPOINT, b.Id)

builder = Folder{
Id: rootFolderId,
Name: b.ModuleCode,
Downloadable: b.IsAccessible,
HasSubFolder: true,
Ancestors: []string{},
}
case constants.Luminus:
url = fmt.Sprintf(FOLDER_URL_ENDPOINT, builder.Id)
url = fmt.Sprintf(FOLDER_URL_ENDPOINT, b.Id)
default:
return FoldersRequest{}, errors.New("invalid platform provided")
}
case Folder:
switch p := platform; p {
case constants.Canvas:
url = fmt.Sprintf(constants.CANVAS_FOLDERS_ENDPOINT, builder.Id)
url = fmt.Sprintf(constants.CANVAS_FOLDERS_ENDPOINT, b.Id)
case constants.Luminus:
url = fmt.Sprintf(FOLDER_URL_ENDPOINT, builder.Id)
url = fmt.Sprintf(FOLDER_URL_ENDPOINT, b.Id)
default:
return FoldersRequest{}, errors.New("invalid platform provided")
}
Expand Down
12 changes: 10 additions & 2 deletions pkg/interfaces/File.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,16 @@ package interfaces

// TODO Documentation
type CanvasFileObject struct {
Id int `json:"id"`
Name string `json:"filename"`
Id int `json:"id"`
Name string `json:"filename"`
// DisplayName is the name that is seen on Canvas Web. It can differ
// from the actual name of the file being uploaded.
// Using DisplayName would be more accurate as Professors might
// set the DisplayName to contain more information for the File.
// For eg. filename can be "Tutorial1.pdf" but DisplayName can be
// "Tutorial1_HW1.pdf" to show that Tutorial 1 is to be submitted as
// a graded Homework.
DisplayName string `json:"display_name"`
UUID string `json:"uuid"`
Url string `json:"url"`
HiddenForUser bool `json:"hidden_for_user"`
Expand Down

0 comments on commit e6d58b9

Please sign in to comment.