Skip to content

Commit

Permalink
DEVEX-2122 List folder content in a single /listFolder request (#84)
Browse files Browse the repository at this point in the history 
* List folder objects and describe content in a single /listFolder request

* GHA: update MacOS runner version

* Print request duration when verbose=2

* Bump version to 1.3.0 for release
  • Loading branch information
@wormsik
wormsik authored Feb 15, 2024
1 parent 75a7652 commit e3f92d8
Show file tree
Hide file tree
Showing 4 changed files with 72 additions and 82 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/master.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ jobs:
macos:
name: Macos build
runs-on: macos-10.15
runs-on: macos-latest
env:
GOOS: darwin
GOARCH: amd64
Expand Down
127 changes: 61 additions & 66 deletions dx_describe.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"log"
"net/http"
"strings"
"time"

// The dxda package has the get-environment code
"github.com/dnanexus/dxda"
Expand All @@ -33,6 +34,24 @@ type DxDescribeDataObject struct {
Properties map[string]string
}

func ConvertDescribeRawToDataObject(
projectId string,
descRaw DxDescribeRaw) DxDescribeDataObject {
return DxDescribeDataObject{
Id: descRaw.Id,
ProjId: projectId,
Name: descRaw.Name,
State: descRaw.State,
ArchivalState: descRaw.ArchivalState,
Folder: descRaw.Folder,
Size: descRaw.Size,
CtimeSeconds: descRaw.CreatedMillisec / 1000,
MtimeSeconds: descRaw.ModifiedMillisec / 1000,
Tags: descRaw.Tags,
Properties: descRaw.Properties,
}
}

// https://documentation.dnanexus.com/developer/api/data-containers/projects#api-method-project-xxxx-describe
type FileUploadParameters struct {
MinimumPartSize int64 `json:"minimumPartSize"`
Expand Down Expand Up @@ -165,21 +184,7 @@ func submit(

var files = make(map[string]DxDescribeDataObject)
for _, descRawTop := range reply.Results {
descRaw := descRawTop.Describe

desc := DxDescribeDataObject{
Id: descRaw.Id,
ProjId: descRaw.ProjId,
Name: descRaw.Name,
State: descRaw.State,
ArchivalState: descRaw.ArchivalState,
Folder: descRaw.Folder,
Size: descRaw.Size,
CtimeSeconds: descRaw.CreatedMillisec / 1000,
MtimeSeconds: descRaw.ModifiedMillisec / 1000,
Tags: descRaw.Tags,
Properties: descRaw.Properties,
}
desc := ConvertDescribeRawToDataObject(descRawTop.Describe.ProjId, descRawTop.Describe)
//fmt.Printf("%v\n", desc)
files[desc.Id] = desc
}
Expand Down Expand Up @@ -224,9 +229,10 @@ func DxDescribeBulkObjects(
}

type ListFolderRequest struct {
Folder string `json:"folder"`
Only string `json:"only"`
IncludeHidden bool `json:"includeHidden"`
Folder string `json:"folder"`
Only string `json:"only"`
IncludeHidden bool `json:"includeHidden"`
Describe map[string]bool `json:"describe"`
}

type ListFolderResponse struct {
Expand All @@ -235,87 +241,76 @@ type ListFolderResponse struct {
}

type ObjInfo struct {
Id string `json:"id"`
}

type DxListFolder struct {
objIds []string
subdirs []string
Id string `json:"id"`
Describe DxDescribeRaw `json:"describe"`
}

// Issue a /project-xxxx/listFolder API call. Get
// back a list of object-ids and sub-directories.
func listFolder(
func DxDescribeFolder(
ctx context.Context,
httpClient *http.Client,
dxOptions *Options,
dxEnv *dxda.DXEnvironment,
projectId string,
dir string) (*DxListFolder, error) {

folder string) (*DxFolder, error) {
request := ListFolderRequest{
Folder: dir,
Folder: folder,
Only: "all",
IncludeHidden: false,
}
request.Describe = map[string]bool{
"id": true,
"name": true,
"state": true,
"archivalState": true,
"folder": true,
"created": true,
"modified": true,
"size": true,
"tags": true,
"properties": true,
}

var payload []byte
payload, err := json.Marshal(request)
if err != nil {
log.Printf("listFolder(%s) payload marshalling error %s", folder, err.Error())
return nil, err
}
dxRequest := fmt.Sprintf("%s/listFolder", projectId)
reqStartTime := time.Now()
repJs, err := dxda.DxAPI(ctx, httpClient, NumRetriesDefault, dxEnv, dxRequest, string(payload))
if dxOptions.VerboseLevel > 1 {
log.Printf("listFolder(%s) API call duration %s", folder, time.Now().Sub(reqStartTime))
}
if err != nil {
log.Printf("listFolder(%s) request error %s", folder, err.Error())
return nil, err
}
var reply ListFolderResponse
if err := json.Unmarshal(repJs, &reply); err != nil {
log.Printf("listFolder(%s) response unmarshalling error %s", folder, err.Error())
return nil, err
}
var objIds []string
for _, objInfo := range reply.Objects {
objIds = append(objIds, objInfo.Id)
}
retval := DxListFolder{
objIds: objIds,
subdirs: reply.Folders,
dataObjects := make(map[string]DxDescribeDataObject)
for _, oDesc := range reply.Objects {
dataObjects[oDesc.Id] = ConvertDescribeRawToDataObject(projectId, oDesc.Describe)
}
return &retval, nil
}

func DxDescribeFolder(
ctx context.Context,
httpClient *http.Client,
dxEnv *dxda.DXEnvironment,
projectId string,
folder string) (*DxFolder, error) {
// The listFolder API call returns a list of object ids and folders.
// We could describe the objects right here, but we do that separately.
folderInfo, err := listFolder(ctx, httpClient, dxEnv, projectId, folder)
if err != nil {
log.Printf("listFolder(%s) error %s", folder, err.Error())
return nil, err
var folderInfo *DxFolder
folderInfo = &DxFolder{
path: folder,
dataObjects: dataObjects,
subdirs: reply.Folders,
}

// limit the number of directory elements
numElementsInDir := len(folderInfo.objIds)
numElementsInDir := len(folderInfo.dataObjects)
if numElementsInDir > MaxDirSize {
return nil, fmt.Errorf(
"Too many elements (%d) in a directory, the limit is %d",
numElementsInDir, MaxDirSize)
}
dxObjs, err := DxDescribeBulkObjects(ctx, httpClient, dxEnv, projectId, folderInfo.objIds)
if err != nil {
log.Printf("describeBulkObjects(%v) error %s", folderInfo.objIds, err.Error())
return nil, err
}
dataObjects := make(map[string]DxDescribeDataObject)
for _, oDesc := range dxObjs {
dataObjects[oDesc.Id] = oDesc
}
return &DxFolder{
path: folder,
dataObjects: dataObjects,
subdirs: folderInfo.subdirs,
}, nil
return folderInfo, nil
}

type RequestDescribeProject struct {
Expand Down
21 changes: 9 additions & 12 deletions metadata_db.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,8 +102,8 @@ func (mdb *MetadataDb) opClose(oph *OpHandle) {

// Split a path into a parent and child. For example:
//
// /A/B/C -> "/A/B", "C"
// / -> "", "/"
// /A/B/C -> "/A/B", "C"
// / -> "", "/"
func splitPath(fullPath string) (parentDir string, basename string) {
if fullPath == "/" {
// The anomalous case.
Expand Down Expand Up @@ -497,7 +497,6 @@ func (mdb *MetadataDb) lookupDirByInode(oph *OpHandle, parent string, dname stri
}

// search for a file with a particular inode.
//
func (mdb *MetadataDb) LookupByInode(ctx context.Context, oph *OpHandle, inode int64) (Node, bool, error) {
// point lookup in the namespace table
sqlStmt := fmt.Sprintf(`
Expand Down Expand Up @@ -554,7 +553,6 @@ func (mdb *MetadataDb) LookupDirByInode(ctx context.Context, oph *OpHandle, inod
}

// Find information on a directory by searching on its full name.
//
func (mdb *MetadataDb) lookupDirByName(oph *OpHandle, dirname string) (string, string, error) {
parentDir, basename := splitPath(dirname)
if mdb.options.Verbose {
Expand Down Expand Up @@ -700,10 +698,10 @@ func (mdb *MetadataDb) directoryReadAllEntries(

// Create an entry representing one remote file. This has
// several use cases:
// 1) Create a singleton file from the manifest
// 2) Create a new file, and upload it later to the platform
// 1. Create a singleton file from the manifest
// 2. Create a new file, and upload it later to the platform
// (the file-id will be the empty string "")
// 3) Discover a file in a directory, which may actually be a link to another file.
// 3. Discover a file in a directory, which may actually be a link to another file.
func (mdb *MetadataDb) createDataObject(
oph *OpHandle,
kind int,
Expand Down Expand Up @@ -975,7 +973,7 @@ func (mdb *MetadataDb) directoryReadFromDNAx(
}

// describe all (closed) files
dxDir, err := DxDescribeFolder(ctx, oph.httpClient, &mdb.dxEnv, projId, projFolder)
dxDir, err := DxDescribeFolder(ctx, oph.httpClient, &mdb.options, &mdb.dxEnv, projId, projFolder)
if err != nil {
fmt.Printf(err.Error())
fmt.Printf("reading directory frmo DNAx error")
Expand Down Expand Up @@ -1398,9 +1396,9 @@ func (mdb *MetadataDb) UpdateFileLocalPath(
}

// Move a file
// 1) Can move a file from one directory to another,
// or leave it in the same directory
// 2) Can change the filename.
// 1. Can move a file from one directory to another,
// or leave it in the same directory
// 2. Can change the filename.
func (mdb *MetadataDb) MoveFile(
ctx context.Context,
oph *OpHandle,
Expand Down Expand Up @@ -1488,7 +1486,6 @@ func (mdb *MetadataDb) execModifyRecord(oph *OpHandle, r MoveRecord) error {
//
// From the shell we issue the command:
// $ mv A D/K/
//
func (mdb *MetadataDb) MoveDir(
ctx context.Context,
oph *OpHandle,
Expand Down
4 changes: 1 addition & 3 deletions util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ const (
NumRetriesDefault = 10
InitialUploadPartSize = 16 * MiB
MaxUploadPartSize = 700 * MiB
Version = "v1.2.0"
Version = "v1.3.0"
)
const (
InodeInvalid = 0
Expand Down Expand Up @@ -194,7 +194,6 @@ type DeadFile struct {
//
// Not that not only files have attributes, applets and workflows
// have them too.
//
type DirtyFileInfo struct {
Inode int64
dirtyData bool
Expand Down Expand Up @@ -273,7 +272,6 @@ func LogMsg(moduleName string, a string, args ...interface{}) {
log.Printf("%s %s: %s", Time2string(now), moduleName, msg)
}

//
// 1024 => 1KB
// 10240 => 10KB
// 1100000 => 1MB
Expand Down

0 comments on commit e3f92d8

Please sign in to comment.