diff --git a/.gitignore b/.gitignore index b0caef6..c3cdf2b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ main __pycache__ ENV -cmd_results.txt \ No newline at end of file +cmd_results.txt +diff.txt \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..6ceb839 --- /dev/null +++ b/Makefile @@ -0,0 +1,2 @@ +/go/bin/dxfuse : $(wildcard *.go) + go build -o /go/bin/dxfuse /go/src/github.com/dnanexus/dxfuse/cli/main.go diff --git a/README.md b/README.md index 695644b..2f5fcb6 100644 --- a/README.md +++ b/README.md @@ -36,16 +36,22 @@ is dropped, and a warning is emitted to the log. dxfuse approximates a normal POSIX filesystem, but does not always have the same semantics. For example: 1. Metadata like last access time are not supported 2. Directories have approximate create/modify times. This is because DNAx does not keep such attributes for directories. -3. Files are immutable, which means that they cannot be overwritten. -4. A newly written file is located locally. When it is closed, it becomes read-only, and is uploaded to the cloud. There are several limitations currently: - Primarily intended for Linux, but can be used on OSX -- Intended to operate on platform workers - Limits directories to 10,000 elements - Updates to the project emanating from other machines are not reflected locally - Rename does not allow removing the target file or directory. This is because this cannot be done automatically by dnanexus. +- Does not support hard links + +Updates to files are batched and asynchronously applied to the cloud +object system. For example, if `foo.txt` is updated, the changes will +not be immediately visible to another user looking at the platform +object directly. Because platform files are immutable, even a minor +modification requires rewriting the entire file, creating a new +version. This is an inherent limitation, making file update +inefficient. ## Implementation @@ -87,21 +93,32 @@ download methods were (1) `dx cat`, and (2) `cat` from a dxfuse mount point. # Building To build the code from source, you'll need, at the very least, the `go` and `git` tools. -Assuming the go directory is `/go`, then, clone the code with: +install dependencies: +``` +go get github.com/google/subcommands +go get golang.org/x/sync/semaphore +go install github.com/google/subcommands +go get github.com/dnanexus/dxda +go install github.com/dnanexus/dxda +go install github.com/dnanexus/dxda/cmd/dx-download- +``` + +Assuming the go directory is `/go`, clone the code with: ``` +cd /go/src/github.com/dnanexus/dxfuse git clone git@github.com:dnanexus/dxfuse.git ``` Build the code: ``` -go build -o /go/bin/dxfuse /go/src/github.com/dnanexus/cmd/main.go +go build -o /go/bin/dxfuse /go/src/github.com/dnanexus/dxfuse/cli/main.go ``` # Usage To mount a dnanexus project `mammals` on local directory `/home/jonas/foo` do: ``` -sudo dxfuse -uid $(id -u) -gid $(id -g) /home/jonas/foo mammals +sudo -E dxfuse -uid $(id -u) -gid $(id -g) /home/jonas/foo mammals ``` The bootstrap process has some asynchrony, so it could take it a @@ -111,12 +128,12 @@ the `verbose` flag. Debugging output is written to the log, which is placed at `/var/log/dxfuse.log`. The maximal verbosity level is 2. ``` -sudo dxfuse -verbose 1 MOUNT-POINT PROJECT-NAME +sudo -E dxfuse -verbose 1 MOUNT-POINT PROJECT-NAME ``` Project ids can be used instead of project names. To mount several projects, say, `mammals`, `fish`, and `birds`, do: ``` -sudo dxfuse /home/jonas/foo mammals fish birds +sudo -E dxfuse /home/jonas/foo mammals fish birds ``` This will create the directory hierarchy: @@ -135,15 +152,22 @@ To stop the dxfuse process do: sudo umount MOUNT-POINT ``` +There are situations where you want the background process to +synchronously update all modified and newly created files. For example, before shutting down a machine, +or unmounting the filesystem. This can be done by issuing the command: +``` +$ sudo dxfuse -sync +``` + ## Extended attributes (xattrs) -DNXa data objects have properties and tags, these are exposed as POSIX extended attributes. The package we use for testing is `xattr` which is native on MacOS (OSX), and can be installed with `sudo apt-get install xattr` on Linux. Xattrs can be written and removed. The examples here use `xattr`, although other tools will work just as well. +DNXa data objects have properties and tags, these are exposed as POSIX extended attributes. Xattrs can be read, written, and removed. The package we use here is `attr`, it can installed with `sudo apt-get install attr` on Linux. On OSX the `xattr` package comes packaged with the base operating system, and can be used to the same effect. -DNAx tags and properties are prefixed. For example, if `zebra.txt` is a file then `xattr -l zebra.txt` will print out all the tags, properties, and attributes that have no POSIX equivalent. These are split into three correspnding prefixes _tag_, _prop_, and _base_ all under the `user` Linux namespace. +DNAx tags and properties are prefixed. For example, if `zebra.txt` is a file then `attr -l zebra.txt` will print out all the tags, properties, and attributes that have no POSIX equivalent. These are split into three correspnding prefixes _tag_, _prop_, and _base_ all under the `user` Linux namespace. Here `zebra.txt` has no properties or tags. ``` -$ xattr -l zebra.txt +$ attr -l zebra.txt base.state: closed base.archivalState: live @@ -152,24 +176,24 @@ base.id: file-xxxx Add a property named `family` with value `mammal` ``` -$ xattr -w prop.family mammal zebra.txt +$ attr -s prop.family -V mammal zebra.txt ``` Add a tag `africa` ``` -$ xattr -w tag.africa XXX zebra.txt +$ attr -s tag.africa -V XXX zebra.txt ``` Remove the `family` property: ``` -$ xattr -d prop.family zebra.txt +$ attr -r prop.family zebra.txt ``` -You cannot modify any _base.*_ attribute, these are read-only. Currently, setting and deleting xattrs can be done only for files that are closed on the platform. +You cannot modify _base.*_ attributes, these are read-only. Currently, setting and deleting xattrs can be done only for files that are closed on the platform. ## Mac OS (OSX) -For OSX you will need to install [OSXFUSE](http://osxfuse.github.com/). Note that Your Milage May Vary (YMMV) on this platform, we are focused on Linux currently. +For OSX you will need to install [OSXFUSE](http://osxfuse.github.com/). Note that Your Milage May Vary (YMMV) on this platform, we are mostly focused on Linux. # Common problems @@ -178,3 +202,23 @@ If a project appears empty, or is missing files, it could be that the dnanexus t If you do not set the `uid` and `gid` options then creating hard links will fail on Linux. This is because it will fail the kernel's permissions check. There is no natural match for DNAnexus applets and workflows, so they are presented as block devices. They do not behave like block devices, but the shell colors them differently from files and directories. + +Mmap doesn't work all that well with FUSE ([stack overflow issue](https://stackoverflow.com/questions/46839807/mmap-no-such-device)). For example, trying to memory-map (mmap) a file with python causes an error. + +``` +>>> import mmap +>>> fd = open('/home/orodeh/MNT/dxfuse_test_data/README.md', 'r') +>>> mmap.mmap(fp.fileno(), 0, mmap.PROT_READ) +Traceback (most recent call last): + File "", line 1, in + OSError: [Errno 19] No such device +``` + +A workaround is to make the mapping private: + +``` +>>> import mmap +>>> fd = open('/home/orodeh/MNT/dxfuse_test_data/README.md', 'r') +>>> mmap.mmap(fd.fileno(), 0, prot=mmap.PROT_READ, flags=mmap.MAP_PRIVATE, offset=0) +>>> fd.readline() +``` diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index cac6beb..b171100 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,10 +1,13 @@ # Release Notes ## v0.19 -Improvements to extended attributes (xattrs). The testing tool we use is `xattr`, which is native on MacOS (OSX), and can be installed with `sudo apt-get install xattr` on Linux. +- *Experimental support for overwriting files*. There is a limit of 16 MiB on a file that is undergoes modification. This is because it needs to first be downloaded in its entirety, before allowing any changes. It will then be uploaded to the platform. This is an expensive operation that is required because DNAnexus files are immutable. -- Xattrs can be written and removed, with the current limitation that this works only for closed files. -- Tags and properties are namespaced. For example, if `zebra.txt` is a normal text file with no DNAx tags or properties then `xattr -l` will print out all the tags, properties, and extra attributes that have no POSIX equivalent. This is split into three namespaces: _base_, _prop_, and _tag_. +- Removed support for hard links. The combination of hard-links, cloning on DNAx, and writable files does not work at the moment. + +- Improvements to extended attributes (xattrs). The testing tool we use is `xattr`, which is native on MacOS (OSX), and can be installed with `sudo apt-get install xattr` on Linux. Xattrs can be written and removed. + +Tags and properties are namespaced. For example, if `zebra.txt` is a normal text file with no DNAx tags or properties then `xattr -l` will print out all the tags, properties, and extra attributes that have no POSIX equivalent. This is split into three namespaces: _base_, _prop_, and _tag_. ``` $ xattr -l zebra.txt diff --git a/cli/main.go b/cli/main.go index 7190d35..d77e005 100644 --- a/cli/main.go +++ b/cli/main.go @@ -42,6 +42,7 @@ func usage() { var ( debugFuseFlag = flag.Bool("debugFuse", false, "Tap into FUSE debugging information") + fsSync = flag.Bool("sync", false, "Sychronize the filesystem and exit") gid = flag.Int("gid", -1, "User group id (gid)") help = flag.Bool("help", false, "display program options") readOnly = flag.Bool("readOnly", false, "mount the filesystem in read-only mode") @@ -55,6 +56,10 @@ func lookupProject(dxEnv *dxda.DXEnvironment, projectIdOrName string) (string, e // This is a project ID return projectIdOrName, nil } + if strings.HasPrefix(projectIdOrName, "container-") { + // This is a container ID + return projectIdOrName, nil + } // This is a project name, describe it, and // return the project-id. @@ -179,6 +184,11 @@ func parseCmdLineArgs() Config { fmt.Println(dxfuse.Version) os.Exit(0) } + if *fsSync { + cmdClient := dxfuse.NewCmdClient() + cmdClient.Sync() + os.Exit(0) + } if *help { usage() os.Exit(0) diff --git a/cmd_client.go b/cmd_client.go new file mode 100644 index 0000000..dda2e1a --- /dev/null +++ b/cmd_client.go @@ -0,0 +1,33 @@ +package dxfuse + +import ( + "fmt" + "net/rpc" + "os" +) + +type CmdClient struct { +} + +// Sending commands with a client +// +func NewCmdClient() *CmdClient { + return &CmdClient{} +} + +func (client *CmdClient) Sync() { + rpcClient, err := rpc.Dial("tcp", fmt.Sprintf(":%d", CmdPort)) + if err != nil { + fmt.Printf("could not connect to the dxfuse server: %s", err.Error()) + os.Exit(1) + } + defer rpcClient.Close() + + // Synchronous call + var reply bool + err = rpcClient.Call("CmdServerBox.GetLine", "sync", &reply) + if err != nil { + fmt.Printf("sync error: %s", err.Error()) + os.Exit(1) + } +} diff --git a/cmd_server.go b/cmd_server.go new file mode 100644 index 0000000..09553cf --- /dev/null +++ b/cmd_server.go @@ -0,0 +1,83 @@ +/* Accept commands from the dxfuse_tools program. The only command +* right now is sync, but this is the place to implement additional +* ones to come in the future. +*/ +package dxfuse + +import ( + "fmt" + "log" + "net" + "net/rpc" +) + +const ( + // A port number for accepting commands + CmdPort = 7205 +) + +type CmdServer struct { + options Options + sybx *SyncDbDx + inbound *net.TCPListener +} + +// A separate structure used for exporting through RPC +type CmdServerBox struct { + cmdSrv *CmdServer +} + +func NewCmdServer(options Options, sybx *SyncDbDx) *CmdServer { + cmdServer := &CmdServer{ + options: options, + sybx : sybx, + inbound : nil, + } + return cmdServer +} + +// write a log message, and add a header +func (cmdSrv *CmdServer) log(a string, args ...interface{}) { + LogMsg("CmdServer", a, args...) +} + +func (cmdSrv *CmdServer) Init() { + addy, err := net.ResolveTCPAddr("tcp", fmt.Sprintf(":%d", CmdPort)) + if err != nil { + log.Fatal(err) + } + + inbound, err := net.ListenTCP("tcp", addy) + if err != nil { + log.Fatal(err) + } + cmdSrv.inbound = inbound + + cmdSrvBox := &CmdServerBox{ + cmdSrv : cmdSrv, + } + rpc.Register(cmdSrvBox) + go rpc.Accept(inbound) + + cmdSrv.log("started command server, accepting external commands") +} + +func (cmdSrv *CmdServer) Close() { + cmdSrv.inbound.Close() +} + +// Note: all export functions from this module have to have this format. +// Nothing else will work with the RPC package. +func (box *CmdServerBox) GetLine(arg string, reply *bool) error { + cmdSrv := box.cmdSrv + cmdSrv.log("Received line %s", arg) + switch arg { + case "sync": + cmdSrv.sybx.CmdSync() + default: + cmdSrv.log("Unknown command") + } + + *reply = true + return nil +} diff --git a/doc/Internals.md b/doc/Internals.md index 94feeb0..6c3f216 100644 --- a/doc/Internals.md +++ b/doc/Internals.md @@ -1,4 +1,4 @@ -## The Database Schema +# The Database Schema A local sqlite3 database is used to store filesystem information discovered by querying DNAnexus. @@ -7,20 +7,22 @@ The `data_objects` table maintains information for files, applets, workflows, an | field name | SQL type | description | | --- | --- | -- | -| inode | bigint | local filesystem i-node, cannot change | | kind | int | type of file: regular, symbolic link, other | | id | text | The DNAx object-id | | proj\_id | text | A project id for the file | | state | text | the file state (open/closing/closed) | | archival\_state | text | archival state of this file | +| inode | bigint | local filesystem i-node, cannot change | | size | bigint | size of the file in bytes | | ctime | bigint | creation time | | mtime | bigint | modification time | | mode | int | Unix permission bits | -| nlink | int | number of hard links to this file | | tags | text | DNAx tags for this object, encoded as a JSON array | | properties | text | DNAx properties for this object, encoded as JSON | -| inline\_data | text | holds the path for a symlink, if it has a local copy, this is the path | +| symlink | text | holds the path for a symlink (if symlink) | +| local\_path | text | if file has a local copy, this is the path | +| dirty\_data | int | has the data been modified? (only files) | +| dirty\_metadata | int | have the tags or properties been modified? | It stores `stat` information on a data object, and maps it to an inode. The inode is the primary key, and it cannot change once @@ -35,6 +37,12 @@ The `archival_state` is relevant for files only. It can have one of four values: `live`, `archival`, `archived`, `unarchiving`. A file can be accessed only when it is in the `live` state. +The `state` can be one of `open`, `closing`, `closed`. It applies to all data objects. + +The `id` will be empty when a file is first created. It will be populated when it is first +uploaded to the platform. Every update the id will change. This is because DNAx files are immutable, +and changing, even a single byte, requires rewriting the entire file, generating a new id. + The `namespace` table stores information on the directory structure. | field name | SQL type | description | @@ -42,7 +50,7 @@ The `namespace` table stores information on the directory structure. | parent | text | the parent folder | | name | text | directory/file name | | obj\_type | int | directory=1, data-object=2 | -| inode | bigint | local filesystem i-node, cannot change | +| inode | bigint | local filesystem i-node, cannot change | For example, directory `/A/B/C` is represented with the record: ``` @@ -122,26 +130,20 @@ A hard link is an entry in the namespace that points to an existing data object. single i-node can have multiple namespace entries, so it cannot serve as a primary key. -## Sequential Prefetch +# Sequential Prefetch Performing prefetch for sequential streams incurs overhead and costs memory. The goal of the prefetch module is: *if a file is read from start to finish, we want to be able to read it in large network requests*. What follows is a simplified description of the algorithm. In order for a file to be eligible for streaming it has to be at -8MiB. A bitmap is maintained for areas accessed. If the first metabyte +8MiB. A bitmap is maintained for areas accessed. If a complete metabyte is accessed, prefetch is started. This entails sending multiple asynchronous IO to fetch 4MiB of data. As long as the data is fully read, prefetch continues. If a file is not accessed for more -than five minutes, or, access is outside the prefetched area, the process stops. +than five minutes, or, access is outside the prefetched area, the process halts. It will start again if sequential access is detected down the road. -## File upload and creation - -It is possible to create new files. These are written to the local disk, and uploaded when they -are closed. Since DNAnexus files are immutable, once a file is closed, it becomes read only. The local -copy is not erased, it is accessed when performing read IOs. - -## Manifest +# Manifest The *manifest* option specifies the initial snapshot of the filesystem tree as a JSON file. The database is initialized from this snapshot, @@ -202,3 +204,33 @@ will create the directory structure: ``` Browsing through directory `Cards/J`, is equivalent to traversing the remote `proj-1019001:/Joker` folder. + + +# File creation and modification + +dxfuse allows creating new files and modifing existing files, +inspite of the fact that only immutable files exist on DNAx. The +mismatch between what the filesystem allows (updating a file), and +what is available natively on the platform makes the update operation +expensive. + +When a file is first created it is written to the local disk and +marked dirty in the database. In order to modify an existing file it +is downloaded in its entirety to the local disk, modified locally, and +marked dirty. A background daemon scans the database periodically and +uploads dirty files to the platform. If a file `foo` already exists as +object `file-xxxx`, a new version of it is uploaded, and when done, +the database is modified to point to the new version. It is then +possible to either erase the old version, or keep it as an old +snapshot. + +There are situations where you want the background process to +synchronously update all modified and newly created files. For example, before shutting down a machine, +or unmounting the filesystem. This can be done by issuing the command: +``` +$ dxfuse -sync +``` + +Metadata such as xattrs is updated with a similar scheme. The database +is updated, and the inode is marked `dirtyMetadata`. The background daemon then +updates the attributes asynchronously. diff --git a/doc/Procedures.md b/doc/Procedures.md index c167e73..c239014 100644 --- a/doc/Procedures.md +++ b/doc/Procedures.md @@ -5,11 +5,10 @@ - Update release notes and README.md - Make sure the version number in `utils.go` is correct. It is used when building the release. -- Merge onto master branch, make sure [travis tests](https://travis-ci.org/dnanexus/dxfuse) pass. The travis builds should create new executables under `dxfuse_test_data:/releases/$version`. +- Merge onto master branch, make sure [travis tests](https://travis-ci.org/dnanexus/dxfuse) pass. - Tag release with new version: ``` git tag $version git push origin $version ``` -- Update [releases](https://github.com/dnanexus/dxfuse/releases) github page, - use the `Draft a new release` button, and upload executables. +- Update [releases](https://github.com/dnanexus/dxfuse/releases) github page, use the `Draft a new release` button, and upload executables. diff --git a/dx_describe.go b/dx_describe.go index c660494..bb6bdd3 100644 --- a/dx_describe.go +++ b/dx_describe.go @@ -51,6 +51,7 @@ type DxDescribePrj struct { CtimeSeconds int64 MtimeSeconds int64 UploadParams FileUploadParameters + Level int // one of VIEW, UPLOAD, CONTRIBUTE, ADMINISTER } // a DNAx directory. It holds files and sub-directories. @@ -99,8 +100,7 @@ func submit( ctx context.Context, httpClient *retryablehttp.Client, dxEnv *dxda.DXEnvironment, - fileIds []string, - closedFilesOnly bool) (map[string]DxDescribeDataObject, error) { + fileIds []string) (map[string]DxDescribeDataObject, error) { // Limit the number of fields returned, because by default we // get too much information, which is a burden on the server side. @@ -177,8 +177,7 @@ func DxDescribeBulkObjects( ctx context.Context, httpClient *retryablehttp.Client, dxEnv *dxda.DXEnvironment, - objIds []string, - closedFilesOnly bool) (map[string]DxDescribeDataObject, error) { + objIds []string) (map[string]DxDescribeDataObject, error) { var gMap = make(map[string]DxDescribeDataObject) if len(objIds) == 0 { return gMap, nil @@ -197,7 +196,7 @@ func DxDescribeBulkObjects( batches = append(batches, objIds) for _, objIdBatch := range(batches) { - m, err := submit(ctx, httpClient, dxEnv, objIdBatch, closedFilesOnly) + m, err := submit(ctx, httpClient, dxEnv, objIdBatch) if err != nil { return nil, err } @@ -275,9 +274,7 @@ func DxDescribeFolder( httpClient *retryablehttp.Client, dxEnv *dxda.DXEnvironment, projectId string, - folder string, - closedFilesOnly bool) (*DxFolder, error) { - + folder string) (*DxFolder, error) { // The listFolder API call returns a list of object ids and folders. // We could describe the objects right here, but we do that separately. folderInfo, err := listFolder(ctx, httpClient, dxEnv, projectId, folder) @@ -293,7 +290,7 @@ func DxDescribeFolder( numElementsInDir, MaxDirSize) } - dxObjs, err := DxDescribeBulkObjects(ctx, httpClient, dxEnv, folderInfo.objIds, closedFilesOnly) + dxObjs, err := DxDescribeBulkObjects(ctx, httpClient, dxEnv, folderInfo.objIds) if err != nil { log.Printf("describeBulkObjects(%v) error %s", folderInfo.objIds, err.Error()) return nil, err @@ -323,6 +320,19 @@ type ReplyDescribeProject struct { CreatedMillisec int64 `json:"created"` ModifiedMillisec int64 `json:"modified"` UploadParams FileUploadParameters `json:"fileUploadParameters"` + Level string `json:"level"` +} + +func projectPermissionsToInt(perm string) int { + switch perm { + case "VIEW": return PERM_VIEW + case "UPLOAD": return PERM_UPLOAD + case "CONTRIBUTE": return PERM_CONTRIBUTE + case "ADMINISTER": return PERM_ADMINISTER + } + + log.Panicf("Unknown project permission %s", perm) + return 0 } func DxDescribeProject( @@ -341,6 +351,7 @@ func DxDescribeProject( "created" : true, "modified" : true, "fileUploadParameters" : true, + "level" : true, } var payload []byte payload, err := json.Marshal(request) @@ -359,7 +370,7 @@ func DxDescribeProject( return nil, err } - prj := DxDescribePrj { + prj := DxDescribePrj{ Id : reply.Id, Name : reply.Name, Region : reply.Region, @@ -368,6 +379,7 @@ func DxDescribeProject( CtimeSeconds : reply.CreatedMillisec / 1000, MtimeSeconds : reply.ModifiedMillisec/ 1000, UploadParams : reply.UploadParams, + Level : projectPermissionsToInt(reply.Level), } return &prj, nil } @@ -377,11 +389,10 @@ func DxDescribe( ctx context.Context, httpClient *retryablehttp.Client, dxEnv *dxda.DXEnvironment, - objId string, - closedFilesOnly bool) (DxDescribeDataObject, error) { + objId string) (DxDescribeDataObject, error) { var objectIds []string objectIds = append(objectIds, objId) - m, err := DxDescribeBulkObjects(ctx, httpClient, dxEnv, objectIds, closedFilesOnly) + m, err := DxDescribeBulkObjects(ctx, httpClient, dxEnv, objectIds) if err != nil { return DxDescribeDataObject{}, err } diff --git a/dx_ops.go b/dx_ops.go index 0a897b8..f57fc31 100644 --- a/dx_ops.go +++ b/dx_ops.go @@ -12,6 +12,11 @@ import ( "github.com/hashicorp/go-retryablehttp" ) +const ( + fileCloseWaitTime = 5 * time.Second + fileCloseMaxWaitTime = 10 * time.Minute +) + type DxOps struct { dxEnv dxda.DXEnvironment options Options @@ -44,6 +49,9 @@ func (ops *DxOps) DxFolderNew( httpClient *retryablehttp.Client, projId string, folder string) error { + if ops.options.Verbose { + ops.log("new-folder %s:%s", projId, folder) + } var request RequestFolderNew request.ProjId = projId @@ -88,6 +96,9 @@ func (ops *DxOps) DxFolderRemove( httpClient *retryablehttp.Client, projId string, folder string) error { + if ops.options.Verbose { + ops.log("remove-folder %s:%s", projId, folder) + } var request RequestFolderRemove request.ProjId = projId @@ -132,6 +143,9 @@ func (ops *DxOps) DxRemoveObjects( httpClient *retryablehttp.Client, projId string, objectIds []string) error { + if ops.options.Verbose { + ops.log("Removing %d objects from project %s", len(objectIds), projId) + } var request RequestRemoveObjects request.Objects = objectIds @@ -152,7 +166,7 @@ func (ops *DxOps) DxRemoveObjects( return err } - var reply ReplyFolderRemove + var reply ReplyRemoveObjects if err := json.Unmarshal(repJs, &reply); err != nil { return err } @@ -179,6 +193,9 @@ func (ops *DxOps) DxFileNew( projId string, fname string, folder string) (string, error) { + if ops.options.Verbose { + ops.log("file-new %s:%s/%s", projId, folder, fname) + } var request RequestNewFile request.ProjId = projId @@ -208,8 +225,10 @@ func (ops *DxOps) DxFileNew( func (ops *DxOps) DxFileCloseAndWait( ctx context.Context, httpClient *retryablehttp.Client, - fid string, - verbose bool) error { + fid string) error { + if ops.options.Verbose { + ops.log("file close-and-wait %s", fid) + } _, err := dxda.DxAPI( ctx, @@ -226,7 +245,7 @@ func (ops *DxOps) DxFileCloseAndWait( start := time.Now() deadline := start.Add(fileCloseMaxWaitTime) for true { - fDesc, err := DxDescribe(ctx, httpClient, &ops.dxEnv, fid, false) + fDesc, err := DxDescribe(ctx, httpClient, &ops.dxEnv, fid) if err != nil { return err } @@ -236,7 +255,7 @@ func (ops *DxOps) DxFileCloseAndWait( return nil case "closing": // not done yet. - if verbose { + if ops.options.Verbose { elapsed := time.Now().Sub(start) ops.log("Waited %s for file %s to close", elapsed.String(), fid) } @@ -332,6 +351,9 @@ func (ops *DxOps) DxRename( projId string, fileId string, newName string) error { + if ops.options.Verbose { + ops.log("file rename %s:%s %s", projId, fileId, newName) + } var request RequestRename request.ProjId = projId @@ -518,18 +540,15 @@ type ReplySetProperties struct { Id string `json:"id"` } -func (ops *DxOps) DxSetProperty( +func (ops *DxOps) DxSetProperties( ctx context.Context, httpClient *retryablehttp.Client, projId string, objId string, - key string, - value *string) error { + props map[string](*string)) error { var request RequestSetProperties request.ProjId = projId - props := make(map[string](*string)) - props[key] = value request.Properties = props payload, err := json.Marshal(request) @@ -562,17 +581,15 @@ type ReplyAddTags struct { Id string `json:"id"` } -func (ops *DxOps) DxAddTag( +func (ops *DxOps) DxAddTags( ctx context.Context, httpClient *retryablehttp.Client, projId string, objId string, - key string) error { + tags []string) error { var request RequestAddTags request.ProjId = projId - tags := make([]string, 1) - tags[0] = key request.Tags = tags payload, err := json.Marshal(request) @@ -606,17 +623,15 @@ type ReplyRemoveTags struct { Id string `json:"id"` } -func (ops *DxOps) DxRemoveTag( +func (ops *DxOps) DxRemoveTags( ctx context.Context, httpClient *retryablehttp.Client, projId string, objId string, - key string) error { + tags []string) error { var request RequestRemoveTags request.ProjId = projId - tags := make([]string, 1) - tags[0] = key request.Tags = tags payload, err := json.Marshal(request) diff --git a/dxfuse.go b/dxfuse.go index 51d972b..6de06e0 100644 --- a/dxfuse.go +++ b/dxfuse.go @@ -32,6 +32,90 @@ const ( XATTR_BASE = "base" ) +type Filesys struct { + // inherit empty implementations for all the filesystem + // methods we do not implement + fuseutil.NotImplementedFileSystem + + // configuration information for accessing dnanexus servers + dxEnv dxda.DXEnvironment + + // various options + options Options + + // A file holding a sqlite3 database with all the files and + // directories collected thus far. + dbFullPath string + + // Lock for protecting shared access to the database + mutex *sync.Mutex + + // a pool of http clients, for short requests, such as file creation, + // or file describe. + httpClientPool chan(*retryablehttp.Client) + + // metadata database + mdb *MetadataDb + + // prefetch state for all files + pgs *PrefetchGlobalState + + // sync daemon + sybx *SyncDbDx + + // API to dx + ops *DxOps + + // A way to send external commands to the filesystem + cmdSrv *CmdServer + + // description for each mounted project + projId2Desc map[string]DxDescribePrj + + // all open files + fhCounter uint64 + fhTable map[fuseops.HandleID]*FileHandle + + // all open directories + dhCounter uint64 + dhTable map[fuseops.HandleID]*DirHandle + + tmpFileCounter uint64 + + // is the the system shutting down (unmounting) + shutdownCalled bool +} + +// Files can be in two access modes: remote-read-only or local-read-write +const ( + // read only file that is on the cloud + AM_RO_Remote = 1 + + // file that has a local copy and can be modified. + // updates will be propagated with the background daemon. + AM_RW_Local = 2 +) + +type FileHandle struct { + // a lock allowing multiple readers or a single writer. + accessMode int + inode int64 + size int64 // this is up-to-date only for remote files + hid fuseops.HandleID + + // URL used for downloading file ranges. + // Used for read-only files. + url *DxDownloadURL + + // A file-descriptor for files with a local copy + fd *os.File +} + +type DirHandle struct { + d Dir + entries []fuseutil.Dirent +} + func NewDxfuse( dxEnv dxda.DXEnvironment, manifest Manifest, @@ -46,14 +130,13 @@ func NewDxfuse( dxEnv : dxEnv, options: options, dbFullPath : DatabaseFile, - mutex : sync.Mutex{}, + mutex : &sync.Mutex{}, httpClientPool: httpIoPool, ops : NewDxOps(dxEnv, options), fhCounter : 1, fhTable : make(map[fuseops.HandleID]*FileHandle), dhCounter : 1, dhTable : make(map[fuseops.HandleID]*DirHandle), - nonce : NewNonce(), tmpFileCounter : 0, shutdownCalled : false, } @@ -85,12 +168,12 @@ func NewDxfuse( return nil, err } - oph := fsys.OpOpen() + oph := fsys.opOpen() if err := fsys.mdb.PopulateRoot(context.TODO(), oph, manifest); err != nil { - fsys.OpClose(oph) + fsys.opClose(oph) return nil, err } - fsys.OpClose(oph) + fsys.opClose(oph) fsys.pgs = NewPrefetchGlobalState(options.VerboseLevel, dxEnv) @@ -114,13 +197,15 @@ func NewDxfuse( } projId2Desc[pDesc.Id] = *pDesc } + fsys.projId2Desc = projId2Desc + + // initialize sync daemon + fsys.sybx = NewSyncDbDx(options, dxEnv, projId2Desc, mdb, fsys.mutex) - // initialize background upload state - fsys.fugs = NewFileUploadGlobalState(options, dxEnv, projId2Desc) + // create an endpoint for communicating with the user + fsys.cmdSrv = NewCmdServer(options, fsys.sybx) + fsys.cmdSrv.Init() - // Provide the upload module with a reference to the database. - // This is needed to report the end of an upload. - fsys.fugs.mdb = mdb return fsys, nil } @@ -129,7 +214,54 @@ func (fsys *Filesys) log(a string, args ...interface{}) { LogMsg("dxfuse", a, args...) } -func (fsys *Filesys) OpOpen() *OpHandle { +func (fsys *Filesys) Shutdown() { + if fsys.shutdownCalled { + // shutdown has already been called. + // We are not waiting for anything, and just + // unmounting the filesystem here. + fsys.log("Shutdown called a second time, skipping the normal sequence") + return + } + fsys.shutdownCalled = true + + // Close the sql database. + // + // If there is an error, we report it. There is nothing actionable + // to do with it. + // + // We do not remove the metadata database file, so it could be inspected offline. + fsys.log("Shutting down dxfuse") + + // stop any background operations the metadata database may be running. + fsys.mdb.Shutdown() + + // stop the running threads in the prefetch module + fsys.pgs.Shutdown() + + // close the command server, this frees up the port + fsys.cmdSrv.Close() + + // Stop the synchronization daemon. Do not complete + // outstanding operations. + if fsys.sybx != nil { + fsys.sybx.Shutdown() + } +} + +// check if a user has sufficient permissions to read/write a project +func (fsys *Filesys) checkProjectPermissions(projId string, requiredPerm int) bool { + if fsys.options.ReadOnly { + // if the filesystem is mounted read-only, we + // allow only operations that require VIEW level access + if requiredPerm > PERM_VIEW { + return false + } + } + pDesc := fsys.projId2Desc[projId] + return pDesc.Level >= requiredPerm +} + +func (fsys *Filesys) opOpen() *OpHandle { txn, err := fsys.mdb.BeginTxn() if err != nil { log.Panic("Could not open transaction") @@ -143,7 +275,7 @@ func (fsys *Filesys) OpOpen() *OpHandle { } } -func (fsys *Filesys) OpClose(oph *OpHandle) { +func (fsys *Filesys) opClose(oph *OpHandle) { fsys.httpClientPool <- oph.httpClient if oph.err == nil { @@ -159,36 +291,6 @@ func (fsys *Filesys) OpClose(oph *OpHandle) { } } -func (fsys *Filesys) Shutdown() { - if fsys.shutdownCalled { - // shutdown has already been called. - // We are not waiting for anything, and just - // unmounting the filesystem here. - fsys.log("Shutdown called a second time, skipping the normal sequence") - return - } - fsys.shutdownCalled = true - - // Close the sql database. - // - // If there is an error, we report it. There is nothing actionable - // to do with it. - // - // We do not remove the metadata database file, so it could be inspected offline. - fsys.log("Shutting down dxfuse") - - // stop any background operations the metadata database may be running. - fsys.mdb.Shutdown() - - // stop the running threads in the prefetch module - fsys.pgs.Shutdown() - - // complete pending uploads - if !fsys.options.ReadOnly { - fsys.fugs.Shutdown() - } -} - func (fsys *Filesys) dxErrorToFilesystemError(dxErr dxda.DxError) error { switch dxErr.EType { case "InvalidInput": @@ -222,6 +324,7 @@ func (fsys *Filesys) translateError(err error) error { } func (fsys *Filesys) StatFS(ctx context.Context, op *fuseops.StatFSOp) error { + //return fuse.ENOSYS return nil } @@ -243,9 +346,9 @@ func (fsys *Filesys) calcExpirationTime(a fuseops.InodeAttributes) time.Time { func (fsys *Filesys) LookUpInode(ctx context.Context, op *fuseops.LookUpInodeOp) error { fsys.mutex.Lock() + oph := fsys.opOpen() + defer fsys.opClose(oph) defer fsys.mutex.Unlock() - oph := fsys.OpOpen() - defer fsys.OpClose(oph) parentDir, ok, err := fsys.mdb.LookupDirByInode(ctx, oph, int64(op.Parent)) if err != nil { @@ -281,9 +384,9 @@ func (fsys *Filesys) LookUpInode(ctx context.Context, op *fuseops.LookUpInodeOp) func (fsys *Filesys) GetInodeAttributes(ctx context.Context, op *fuseops.GetInodeAttributesOp) error { fsys.mutex.Lock() + oph := fsys.opOpen() + defer fsys.opClose(oph) defer fsys.mutex.Unlock() - oph := fsys.OpOpen() - defer fsys.OpClose(oph) // Grab the inode. node, ok, err := fsys.mdb.LookupByInode(ctx, oph, int64(op.Inode)) @@ -309,9 +412,9 @@ func (fsys *Filesys) GetInodeAttributes(ctx context.Context, op *fuseops.GetInod // otherwise, this is a permission error. func (fsys *Filesys) SetInodeAttributes(ctx context.Context, op *fuseops.SetInodeAttributesOp) error { fsys.mutex.Lock() + oph := fsys.opOpen() + defer fsys.opClose(oph) defer fsys.mutex.Unlock() - oph := fsys.OpOpen() - defer fsys.OpClose(oph) // Grab the inode. node, ok, err := fsys.mdb.LookupByInode(ctx, oph, int64(op.Inode)) @@ -334,6 +437,9 @@ func (fsys *Filesys) SetInodeAttributes(ctx context.Context, op *fuseops.SetInod // can't modify directory attributes return syscall.EPERM } + if !fsys.checkProjectPermissions(file.ProjId, PERM_VIEW) { + return syscall.EPERM + } // we know it is a file. // check if this is a read-only file. @@ -354,15 +460,15 @@ func (fsys *Filesys) SetInodeAttributes(ctx context.Context, op *fuseops.SetInod attrs.Mtime = *op.Mtime } // we don't handle atime - if err := fsys.mdb.UpdateFile(ctx, oph, file, int64(attrs.Size), attrs.Mtime, attrs.Mode); err != nil { + err = fsys.mdb.UpdateFileAttrs(ctx, oph, file.Inode, int64(attrs.Size), attrs.Mtime, &attrs.Mode); + if err != nil { fsys.log("database error in OpenFile %s", err.Error()) return fuse.EIO } if op.Size != nil && *op.Size != oldSize { // The size changed, truncate the file - localPath := file.InlineData - if err := os.Truncate(localPath, int64(*op.Size)); err != nil { + if err := os.Truncate(file.LocalPath, int64(*op.Size)); err != nil { fsys.log("Error truncating inode=%d from %d to %d", op.Inode, oldSize, op.Size) return err @@ -381,7 +487,7 @@ func (fsys *Filesys) SetInodeAttributes(ctx context.Context, op *fuseops.SetInod func (fsys *Filesys) removeFileHandlesWithInode(inode int64) { handles := make([]fuseops.HandleID, 0) for hid, fh := range fsys.fhTable { - if fh.f.Inode == inode { + if fh.inode == inode { handles = append(handles, hid) } } @@ -422,9 +528,9 @@ func (fsys *Filesys) ForgetInode(ctx context.Context, op *fuseops.ForgetInodeOp) func (fsys *Filesys) MkDir(ctx context.Context, op *fuseops.MkDirOp) error { fsys.mutex.Lock() + oph := fsys.opOpen() + defer fsys.opClose(oph) defer fsys.mutex.Unlock() - oph := fsys.OpOpen() - defer fsys.OpClose(oph) if fsys.options.Verbose { fsys.log("CreateDir(%s)", op.Name) @@ -451,6 +557,9 @@ func (fsys *Filesys) MkDir(ctx context.Context, op *fuseops.MkDirOp) error { // The directory already exists return fuse.EEXIST } + if !fsys.checkProjectPermissions(parentDir.ProjId, PERM_CONTRIBUTE) { + return syscall.EPERM + } // The mode must be 777 for fuse to work properly // We -ignore- the mode set by the user. @@ -506,12 +615,12 @@ func (fsys *Filesys) MkDir(ctx context.Context, op *fuseops.MkDirOp) error { func (fsys *Filesys) RmDir(ctx context.Context, op *fuseops.RmDirOp) error { fsys.mutex.Lock() + oph := fsys.opOpen() + defer fsys.opClose(oph) defer fsys.mutex.Unlock() - oph := fsys.OpOpen() - defer fsys.OpClose(oph) if fsys.options.Verbose { - fsys.log("Remove Dir(%s)", op.Name) + fsys.log("RemoveDir(%s)", op.Name) } // the parent is supposed to be a directory @@ -535,6 +644,9 @@ func (fsys *Filesys) RmDir(ctx context.Context, op *fuseops.RmDirOp) error { // The directory does not exist return fuse.ENOENT } + if !fsys.checkProjectPermissions(parentDir.ProjId, PERM_CONTRIBUTE) { + return syscall.EPERM + } var childDir Dir switch childNode.(type) { @@ -595,13 +707,21 @@ func (fsys *Filesys) insertIntoDirHandleTable(dh *DirHandle) fuseops.HandleID { return did } +// Create a file in a protected directory, used only +// by dxfuse. +func (fsys *Filesys) createLocalPath(filename string) string { + cnt := atomic.AddUint64(&fsys.tmpFileCounter, 1) + localPath := fmt.Sprintf("%s/%d_%s", CreatedFilesDir, cnt) + return localPath +} + // A CreateRequest asks to create and open a file (not a directory). // func (fsys *Filesys) CreateFile(ctx context.Context, op *fuseops.CreateFileOp) error { fsys.mutex.Lock() + oph := fsys.opOpen() + defer fsys.opClose(oph) defer fsys.mutex.Unlock() - oph := fsys.OpOpen() - defer fsys.OpClose(oph) if fsys.options.Verbose { fsys.log("CreateFile(%s)", op.Name) @@ -630,31 +750,20 @@ func (fsys *Filesys) CreateFile(ctx context.Context, op *fuseops.CreateFileOp) e // The file already exists return fuse.EEXIST } + if !fsys.checkProjectPermissions(parentDir.ProjId, PERM_UPLOAD) { + return syscall.EPERM + } // we now know that the parent directory exists, and the file // does not. - // Create a temporary file in a protected directory, used only - // by dxfuse. - cnt := atomic.AddUint64(&fsys.tmpFileCounter, 1) - localPath := fmt.Sprintf("%s/%d_%s", CreatedFilesDir, cnt, op.Name) - - // create the file object on the platform. - fileId, err := fsys.ops.DxFileNew( - ctx, oph.httpClient, fsys.nonce.String(), - parentDir.ProjId, op.Name, parentDir.ProjFolder) - if err != nil { - fsys.log("Error in creating file (%s:%s/%s) on dnanexus: %s", - parentDir.ProjId, parentDir.ProjFolder, op.Name, - err.Error()) - oph.RecordError(err) - return fsys.translateError(err) - } - - file, err := fsys.mdb.CreateFile(ctx, oph, &parentDir, fileId, op.Name, op.Mode, localPath) + // Create a local file to hold the data + localPath := fsys.createLocalPath(op.Name) + file, err := fsys.mdb.CreateFile(ctx, oph, &parentDir, op.Name, op.Mode, localPath) if err != nil { return err } + // The file will be created on the platform asynchronously. // Set up attributes for the child. now := time.Now() @@ -688,10 +797,10 @@ func (fsys *Filesys) CreateFile(ctx context.Context, op *fuseops.CreateFileOp) e } fh := FileHandle{ - fKind : RW_File, - f : file, + accessMode : AM_RW_Local, + inode : file.Inode, + size : file.Size, url : nil, - localPath : &localPath, fd : writer, } op.Handle = fsys.insertIntoFileHandleTable(&fh) @@ -699,98 +808,8 @@ func (fsys *Filesys) CreateFile(ctx context.Context, op *fuseops.CreateFileOp) e } func (fsys *Filesys) CreateLink(ctx context.Context, op *fuseops.CreateLinkOp) error { - fsys.mutex.Lock() - defer fsys.mutex.Unlock() - oph := fsys.OpOpen() - defer fsys.OpClose(oph) - - if fsys.options.Verbose { - fsys.log("CreateLink (inode=%d) -> (parent-inode=%d name=%s)", - op.Target, op.Parent, op.Name) - } - - // parent is supposed to be a directory - parentDir, ok, err := fsys.mdb.LookupDirByInode(ctx, oph, int64(op.Parent)) - if err != nil { - return err - } - if !ok { - // parent directory does not exist - return fuse.ENOENT - } - - // Make sure the destination doesn't already exist - _, ok, err = fsys.mdb.LookupInDir(ctx, oph, &parentDir, op.Name) - if err != nil { - return err - } - if ok { - // The link file already exists - return fuse.EEXIST - } - - // make sure that target node exists - targetNode, ok, err := fsys.mdb.LookupByInode(ctx, oph, int64(op.Target)) - if err != nil { - return err - } - if !ok { - return fuse.ENOENT - } - - var targetFile File - switch targetNode.(type) { - case Dir: - // can't make a hard link to a directory - return fuse.EINVAL - case File: - targetFile = targetNode.(File) - } - - if targetFile.Name != op.Name { - fsys.log("cloning is only allowed if the destination and source names are the same") - return fuse.EINVAL - } - - if fsys.options.Verbose { - fsys.log("CreateLink %s/%s -> %s", - parentDir.FullPath, op.Name, targetFile.Name) - } - - // create a link on the platform. This is done with the clone call. - ok, err = fsys.ops.DxClone( - ctx, oph.httpClient, - targetFile.ProjId, // source project - targetFile.Id, // source id - parentDir.ProjId, // destination project id - parentDir.ProjFolder) // destination folder - if err != nil { - fsys.log("dx clone error %s", err.Error()) - oph.RecordError(err) - return fsys.translateError(err) - } - if !ok { - fsys.log("(%s) object not cloned because it already exists in the target project (%s)", - targetFile.Id, parentDir.ProjId) - return syscall.EINVAL - } - - destFile, err := fsys.mdb.CreateLink(ctx, oph, targetFile, parentDir, op.Name) - if err != nil { - fsys.log("database error in create-link %s", err.Error()) - return fuse.EIO - } - - // fill in child information - op.Entry.Child = destFile.GetInode() - op.Entry.Attributes = destFile.GetAttrs() - - // We don't spontaneously mutate, so the kernel can cache as long as it wants - // (since it also handles invalidation). - op.Entry.AttributesExpiration = fsys.calcExpirationTime(op.Entry.Attributes) - op.Entry.EntryExpiration = op.Entry.AttributesExpiration - - return nil + // not supporting creation of hard links now + return fuse.ENOSYS } func (fsys *Filesys) renameFile( @@ -800,6 +819,18 @@ func (fsys *Filesys) renameFile( newParentDir Dir, file File, newName string) error { + err := fsys.mdb.MoveFile(ctx, oph, file.Inode, newParentDir, newName) + if err != nil { + fsys.log("database error in rename") + return fuse.EIO + } + + if file.Id == "" { + // The file has not been uploaded to the platform yet + return nil + } + + // The file is on the platform, we need to move it on the backend. if oldParentDir.Inode == newParentDir.Inode { // /file-xxxx/rename API call err := fsys.ops.DxRename(ctx, oph.httpClient, file.ProjId, file.Id, newName) @@ -827,13 +858,6 @@ func (fsys *Filesys) renameFile( } } - - err := fsys.mdb.MoveFile(ctx, oph, file.Inode, newParentDir, newName) - if err != nil { - fsys.log("database error in rename") - return fuse.EIO - } - return nil } @@ -898,9 +922,9 @@ func (fsys *Filesys) renameDir( func (fsys *Filesys) Rename(ctx context.Context, op *fuseops.RenameOp) error { fsys.mutex.Lock() + oph := fsys.opOpen() + defer fsys.opClose(oph) defer fsys.mutex.Unlock() - oph := fsys.OpOpen() - defer fsys.OpClose(oph) if fsys.options.Verbose { fsys.log("Rename (inode=%d name=%s) -> (inode=%d, name=%s)", @@ -954,6 +978,9 @@ a rename. You will need to issue a separate remove operation prior to rename. `) return syscall.EPERM } + if !fsys.checkProjectPermissions(oldParentDir.ProjId, PERM_CONTRIBUTE) { + return syscall.EPERM + } oldDir := filepath.Clean(oldParentDir.FullPath + "/" + op.OldName) if oldDir == "/" { @@ -994,7 +1021,7 @@ a rename. You will need to issue a separate remove operation prior to rename. } return fsys.renameDir(ctx, oph, oldParentDir, newParentDir, srcDir, op.NewName) default: - log.Panic(fmt.Sprintf("bad type for srcNode %v", srcNode)) + log.Panicf("bad type for srcNode %v", srcNode) } return nil } @@ -1002,9 +1029,9 @@ a rename. You will need to issue a separate remove operation prior to rename. // Decrement the link count, and remove the file if it hits zero. func (fsys *Filesys) Unlink(ctx context.Context, op *fuseops.UnlinkOp) error { fsys.mutex.Lock() + oph := fsys.opOpen() + defer fsys.opClose(oph) defer fsys.mutex.Unlock() - oph := fsys.OpOpen() - defer fsys.OpClose(oph) if fsys.options.Verbose { fsys.log("Unlink(%s)", op.Name) @@ -1029,6 +1056,9 @@ func (fsys *Filesys) Unlink(ctx context.Context, op *fuseops.UnlinkOp) error { // The file does not exist return fuse.ENOENT } + if !fsys.checkProjectPermissions(parentDir.ProjId, PERM_CONTRIBUTE) { + return syscall.EPERM + } var fileToRemove File switch childNode.(type) { @@ -1038,11 +1068,17 @@ func (fsys *Filesys) Unlink(ctx context.Context, op *fuseops.UnlinkOp) error { // can't unlink a directory return fuse.EINVAL } - check(fileToRemove.Nlink > 0) - // Report to the upload module, that we are cancelling the upload - // for this file. - fsys.fugs.CancelUpload(fileToRemove.Id) + if err := fsys.mdb.Unlink(ctx, oph, fileToRemove); err != nil { + fsys.log("database error in unlink %s", err.Error()) + return fuse.EIO + } + + // The file has not been created on the platform yet, there is no need to + // remove it + if fileToRemove.Id == "" { + return nil + } // remove the file on the platform objectIds := make([]string, 1) @@ -1054,11 +1090,6 @@ func (fsys *Filesys) Unlink(ctx context.Context, op *fuseops.UnlinkOp) error { return fsys.translateError(err) } - if err := fsys.mdb.Unlink(ctx, oph, fileToRemove); err != nil { - fsys.log("database error in unlink %s", err.Error()) - return fuse.EIO - } - return nil } @@ -1140,9 +1171,9 @@ func (fsys *Filesys) readEntireDir(ctx context.Context, oph *OpHandle, dir Dir) // COMMON for drivers func (fsys *Filesys) OpenDir(ctx context.Context, op *fuseops.OpenDirOp) error { fsys.mutex.Lock() + oph := fsys.opOpen() + defer fsys.opClose(oph) defer fsys.mutex.Unlock() - oph := fsys.OpOpen() - defer fsys.OpClose(oph) // the parent is supposed to be a directory dir, ok, err := fsys.mdb.LookupDirByInode(ctx, oph, int64(op.Inode)) @@ -1213,19 +1244,24 @@ func (fsys *Filesys) ReleaseDirHandle(ctx context.Context, op *fuseops.ReleaseDi // === // File handling // -func (fsys *Filesys) openRegularFile(ctx context.Context, oph *OpHandle, op *fuseops.OpenFileOp, f File) (*FileHandle, error) { - if f.InlineData != "" { +func (fsys *Filesys) openRegularFile( + ctx context.Context, + oph *OpHandle, + op *fuseops.OpenFileOp, + f File) (*FileHandle, error) { + + if f.LocalPath != "" { // a regular file that has a local copy - reader, err := os.Open(f.InlineData) + reader, err := os.OpenFile(f.LocalPath, os.O_RDWR, 0644) if err != nil { - fsys.log("Could not open local file %s, err=%s", f.InlineData, err.Error()) + fsys.log("Could not open local file %s, err=%s", f.LocalPath, err.Error()) return nil, err } fh := &FileHandle{ - fKind: RO_LocalCopy, - f : f, + accessMode: AM_RW_Local, + inode : f.Inode, + size : f.Size, url: nil, - localPath : &f.InlineData, fd : reader, } @@ -1247,21 +1283,23 @@ func (fsys *Filesys) openRegularFile(ctx context.Context, oph *OpHandle, op *fus json.Unmarshal(body, &u) fh := &FileHandle{ - fKind: RO_Remote, - f : f, + accessMode: AM_RO_Remote, + inode : f.Inode, + size : f.Size, url: &u, - localPath : nil, fd : nil, } return fh, nil } +// Note: What happens if the file is opened for writing? +// func (fsys *Filesys) OpenFile(ctx context.Context, op *fuseops.OpenFileOp) error { fsys.mutex.Lock() + oph := fsys.opOpen() + defer fsys.opClose(oph) defer fsys.mutex.Unlock() - oph := fsys.OpOpen() - defer fsys.OpClose(oph) if fsys.options.Verbose { fsys.log("OpenFile inode=%d", op.Inode) @@ -1313,13 +1351,13 @@ func (fsys *Filesys) OpenFile(ctx context.Context, op *fuseops.OpenFileOp) error // directly. There is no need to generate a preauthenticated // URL. fh = &FileHandle{ - fKind : RO_Remote, - f : file, + accessMode : AM_RO_Remote, + inode : file.Inode, + size : file.Size, url : &DxDownloadURL{ - URL : file.InlineData, + URL : file.Symlink, Headers : nil, }, - localPath : nil, fd : nil, } default: @@ -1333,9 +1371,9 @@ func (fsys *Filesys) OpenFile(ctx context.Context, op *fuseops.OpenFileOp) error op.KeepPageCache = true op.UseDirectIO = true - if fh.fKind == RO_Remote { + if fh.accessMode == AM_RO_Remote { // Create an entry in the prefetch table, if the file is eligable - fsys.pgs.CreateStreamEntry(fh.hid, fh.f, *fh.url) + fsys.pgs.CreateStreamEntry(fh.hid, file, *fh.url) } return nil } @@ -1351,7 +1389,7 @@ func (fsys *Filesys) getWritableFD(ctx context.Context, handle fuseops.HandleID) return nil, fuse.EINVAL } - if fh.fKind != RW_File { + if fh.accessMode != AM_RW_Local { // This isn't a writeable file, there is no dirty data to flush return nil, nil } @@ -1362,21 +1400,23 @@ func (fsys *Filesys) getWritableFD(ctx context.Context, handle fuseops.HandleID) } +// read a remote immutable file +// func (fsys *Filesys) readRemoteFile(ctx context.Context, op *fuseops.ReadFileOp, fh *FileHandle) error { // This is a regular file reqSize := int64(len(op.Dst)) - if fh.f.Size == 0 || reqSize == 0 { + if fh.size == 0 || reqSize == 0 { // The file is empty return nil } - if fh.f.Size <= op.Offset { + if fh.size <= op.Offset { // request is beyond the size of the file return nil } endOfs := op.Offset + reqSize - 1 // make sure we don't go over the file size - lastByteInFile := fh.f.Size - 1 + lastByteInFile := fh.size - 1 endOfs = MinInt64(lastByteInFile, endOfs) // See if the data has already been prefetched. @@ -1399,8 +1439,8 @@ func (fsys *Filesys) readRemoteFile(ctx context.Context, op *fuseops.ReadFileOp, // add an extent in the file that we want to read headers["Range"] = fmt.Sprintf("bytes=%d-%d", op.Offset, endOfs) if fsys.options.Verbose { - fsys.log("network read (%s %d) ofs=%d len=%d endOfs=%d lastByteInFile=%d", - fh.f.Name, fh.f.Inode, op.Offset, reqSize, endOfs, lastByteInFile) + fsys.log("network read (inode=%d) ofs=%d len=%d endOfs=%d lastByteInFile=%d", + fh.inode, op.Offset, reqSize, endOfs, lastByteInFile) } // Take an http client from the pool. Return it when done. @@ -1426,67 +1466,129 @@ func (fsys *Filesys) ReadFile(ctx context.Context, op *fuseops.ReadFileOp) error } fsys.mutex.Unlock() - // TODO: is there a scenario where two threads will run into a conflict - // because one is holding the handle, and the other is mutating it? - - switch fh.f.Kind { - case FK_Regular: - if fh.fKind == RO_Remote { - return fsys.readRemoteFile(ctx, op, fh) - } else { - // the file has a local copy - n, err := fh.fd.ReadAt(op.Dst, op.Offset) - if err == io.ErrUnexpectedEOF || err == io.EOF { - // we don't report EOF to FUSE, it notices - // it because the number of bytes read is smaller - // than requested. - err = nil - } - op.BytesRead = n - return err - } - - case FK_Symlink: + switch fh.accessMode { + case AM_RO_Remote: return fsys.readRemoteFile(ctx, op, fh) - + case AM_RW_Local: + // the file has a local copy + n, err := fh.fd.ReadAt(op.Dst, op.Offset) + if err == io.ErrUnexpectedEOF || err == io.EOF { + // we don't report EOF to FUSE, it notices + // it because the number of bytes read is smaller + // than requested. + err = nil + } + op.BytesRead = n + return err default: - // can only read files - return syscall.EPERM + log.Panicf("Invalid file access mode %d", fh.accessMode) + return fuse.EIO } } - -func (fsys *Filesys) findWritableFileHandle(handle fuseops.HandleID) (*FileHandle, error) { +// This is tricky. We have a remote file that we want +// to update. This requires: +// 1. downloading the entire file +// 2. changing the file-handle +// +// This conversion is one way. +// +// The download could be long for large files, so we +// may need to either limit file size, or. We are holding the global lock +// while we are downloading the file. This really needs to be improved. +func (fsys *Filesys) prepareFileForWrite(ctx context.Context, op *fuseops.WriteFileOp) (*FileHandle, error) { fsys.mutex.Lock() + oph := fsys.opOpen() + defer fsys.opClose(oph) defer fsys.mutex.Unlock() - // Here, we start from the file handle - fh,ok := fsys.fhTable[handle] + fh,ok := fsys.fhTable[op.Handle] if !ok { // invalid file handle. It doesn't exist in the table return nil, fuse.EINVAL } - if fh.fKind != RW_File { - // This file isn't open for writing - return nil, syscall.EPERM + if fh.accessMode == AM_RW_Local { + // file is already local + return fh, nil } - if fh.fd == nil { - log.Panic("file descriptor is empty") + if fh.size > WritableFileSizeLimit { + fsys.log("File (inode=%d) is larger than the maximum supported writable file limit", + WritableFileSizeLimit) + return nil, fuse.EINVAL + } + + // We need to convert to a local file. + if fsys.options.Verbose { + fsys.log("Downloading inode=%d to local disk, only then can we modify it", fh.inode) + } + + // Do not perform prefetch on a file that has a local copy + fsys.pgs.RemoveStreamEntry(fh.hid) + + // do not hold the global lock while downloading the file + + // create the local file + localPath := fsys.createLocalPath("") + fd, err := os.OpenFile(localPath, os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + return nil, err + } + + err = fsys.pgs.DownloadEntireFile(oph.httpClient, fh.inode, fh.size, *fh.url, fd, localPath) + if err != nil { + fsys.log("failed to download file inode=%d", fh.inode, err.Error()) + // Should we erase the partial file to save space? + return nil, err + } + + // update the database, match the file with its local path + err = fsys.mdb.UpdateFileLocalPath(context.TODO(), oph, fh.inode, localPath) + if err != nil { + fsys.log("database error in updating file for write %s", err.Error()) + return nil, oph.RecordError(fuse.EIO) } + + // update the file-handle. + fh.accessMode = AM_RW_Local + fh.url = nil + fh.fd = fd + return fh, nil } // Writes to files. // -// A file is created locally, and writes go to the local location. When -// the file is closed, it becomes read only, and is then uploaded to the cloud. +// If the file has not been downloaded yet, we need to first download it. We are +// locking the entire filesystem while we are doing this, which will need to be improved. +// +// Note: the file-open operation doesn't state if the file is going to be opened for +// reading or writing. func (fsys *Filesys) WriteFile(ctx context.Context, op *fuseops.WriteFileOp) error { - fh,err := fsys.findWritableFileHandle(op.Handle) + fh, err := fsys.prepareFileForWrite(ctx, op) if err != nil { + fsys.log("Error while converting file from remote-read-only to a local file") return err } + // we are not holding the global lock while we are doing IO - _, err = fh.fd.WriteAt(op.Data, op.Offset) + nBytes, err := fh.fd.WriteAt(op.Data, op.Offset) + + // Try to efficiently calculate the size and mtime, instead + // of doing a filesystem call. + fSize := MaxInt64(op.Offset + int64(nBytes), fh.size) + mtime := time.Now() + + // Update the file attributes in the database (size, mtime) + fsys.mutex.Lock() + oph := fsys.opOpen() + defer fsys.opClose(oph) + defer fsys.mutex.Unlock() + + if err := fsys.mdb.UpdateFileAttrs(ctx, oph, fh.inode, fSize, mtime, nil); err != nil { + fsys.log("database error in updating attributes for WriteFile %s", err.Error()) + return fuse.EIO + } + return err } @@ -1526,9 +1628,9 @@ func (fsys *Filesys) SyncFile(ctx context.Context, op *fuseops.SyncFileOp) error func (fsys *Filesys) ReleaseFileHandle(ctx context.Context, op *fuseops.ReleaseFileHandleOp) error { fsys.mutex.Lock() + oph := fsys.opOpen() + defer fsys.opClose(oph) defer fsys.mutex.Unlock() - oph := fsys.OpOpen() - defer fsys.OpClose(oph) fh, ok := fsys.fhTable[op.Handle] if !ok { @@ -1540,22 +1642,22 @@ func (fsys *Filesys) ReleaseFileHandle(ctx context.Context, op *fuseops.ReleaseF delete(fsys.fhTable, op.Handle) // Clear the state involved with this open file descriptor - switch fh.fKind { - case RO_Remote: + switch fh.accessMode { + case AM_RO_Remote: // Read-only file that is accessed remotely fsys.pgs.RemoveStreamEntry(fh.hid) return nil - case RW_File: + case AM_RW_Local: // A new file created locally. We need to upload it // to the platform. if fsys.options.Verbose { - fsys.log("Close new file(%s)", fh.f.Name) + fsys.log("Close new file (inode=%d)", fh.inode) } // flush and close the local file - // We leave the local file in place. This allows reading from - // it, without accessing the network. + // We leave the local file in place. This allows read/write + // without additional network transfers. if err := fh.fd.Sync(); err != nil { return err } @@ -1564,36 +1666,11 @@ func (fsys *Filesys) ReleaseFileHandle(ctx context.Context, op *fuseops.ReleaseF } fh.fd = nil - // make this file read only - if err := os.Chmod(*fh.localPath, fileReadOnlyMode); err != nil { - return err - } - - fInfo, err := os.Lstat(*fh.localPath) - if err != nil { - return err - } - fileSize := fInfo.Size() - modTime := fInfo.ModTime() - - // update database entry - if err := fsys.mdb.UpdateFile(ctx, oph, fh.f, fileSize, modTime, fileReadOnlyMode); err != nil { - fsys.log("database error in OpenFile %s", err.Error()) - return fuse.EIO - } - - // initiate a background request to upload the file to the cloud - return fsys.fugs.UploadFile(fh.f, fileSize) - - case RO_LocalCopy: - // Read-only file with a local copy - if fh.fd != nil { - fh.fd.Close() - } + // the sync daemon will upload the file asynchronously return nil default: - log.Panicf("Invalid file kind %d", fh.fKind) + log.Panicf("Invalid file access mode %d", fh.accessMode) } return nil } @@ -1637,9 +1714,9 @@ func (fsys *Filesys) xattrParseName(name string) (string, string, error) { func (fsys *Filesys) RemoveXattr(ctx context.Context, op *fuseops.RemoveXattrOp) error { fsys.mutex.Lock() + oph := fsys.opOpen() + defer fsys.opClose(oph) defer fsys.mutex.Unlock() - oph := fsys.OpOpen() - defer fsys.OpClose(oph) if fsys.options.Verbose { fsys.log("RemoveXattr %d", op.Inode) @@ -1650,6 +1727,9 @@ func (fsys *Filesys) RemoveXattr(ctx context.Context, op *fuseops.RemoveXattrOp) if err != nil { return err } + if !fsys.checkProjectPermissions(file.ProjId, PERM_CONTRIBUTE) { + return syscall.EPERM + } // look for the attribute namespace, attrName, err := fsys.xattrParseName(op.Name) @@ -1705,18 +1785,7 @@ func (fsys *Filesys) RemoveXattr(ctx context.Context, op *fuseops.RemoveXattrOp) fsys.log("database error in RemoveXattr: %s", err.Error()) return fuse.EIO } - - // set it on the platform. - switch namespace { - case XATTR_TAG: - return fsys.ops.DxRemoveTag(ctx, oph.httpClient, file.ProjId, file.Id, attrName) - case XATTR_PROP: - return fsys.ops.DxSetProperty(ctx, oph.httpClient, file.ProjId, file.Id, attrName, nil) - default: - log.Panicf("sanity: invalid namespace %s", namespace) - return fuse.EINVAL - } - + return nil } @@ -1735,9 +1804,9 @@ func (fsys *Filesys) getXattrFill(op *fuseops.GetXattrOp, val_str string) error func (fsys *Filesys) GetXattr(ctx context.Context, op *fuseops.GetXattrOp) error { fsys.mutex.Lock() + oph := fsys.opOpen() + defer fsys.opClose(oph) defer fsys.mutex.Unlock() - oph := fsys.OpOpen() - defer fsys.OpClose(oph) if fsys.options.Verbose { fsys.log("GetXattr %d", op.Inode) @@ -1796,9 +1865,9 @@ func (fsys *Filesys) GetXattr(ctx context.Context, op *fuseops.GetXattrOp) error // Make a list of all the extended attributes func (fsys *Filesys) ListXattr(ctx context.Context, op *fuseops.ListXattrOp) error { fsys.mutex.Lock() + oph := fsys.opOpen() + defer fsys.opClose(oph) defer fsys.mutex.Unlock() - oph := fsys.OpOpen() - defer fsys.OpClose(oph) if fsys.options.Verbose { fsys.log("ListXattr %d", op.Inode) @@ -1847,9 +1916,9 @@ func (fsys *Filesys) ListXattr(ctx context.Context, op *fuseops.ListXattrOp) err func (fsys *Filesys) SetXattr(ctx context.Context, op *fuseops.SetXattrOp) error { fsys.mutex.Lock() + oph := fsys.opOpen() + defer fsys.opClose(oph) defer fsys.mutex.Unlock() - oph := fsys.OpOpen() - defer fsys.OpClose(oph) if fsys.options.Verbose { fsys.log("SetXattr %d", op.Inode) @@ -1871,8 +1940,15 @@ func (fsys *Filesys) SetXattr(ctx context.Context, op *fuseops.SetXattrOp) error file = node.(File) case Dir: // directories do not have attributes + // + // Note: we may want to change this for directories + // representing projects. This would allow reporting project + // tags and properties. return syscall.EINVAL } + if !fsys.checkProjectPermissions(file.ProjId, PERM_CONTRIBUTE) { + return syscall.EPERM + } // Check if the property already exists // convert @@ -1946,21 +2022,5 @@ func (fsys *Filesys) SetXattr(ctx context.Context, op *fuseops.SetXattrOp) error fsys.log("database error in SetXattr %s", err.Error()) return fuse.EIO } - - // set it on the platform. - switch namespace { - case XATTR_TAG: - if !attrExists { - return fsys.ops.DxAddTag(ctx, oph.httpClient, file.ProjId, file.Id, attrName) - } else { - // already tagged - return nil - } - case XATTR_PROP: - value := string(op.Value) - return fsys.ops.DxSetProperty(ctx, oph.httpClient, file.ProjId, file.Id, attrName, &value) - default: - log.Panicf("sanity: invalid namespace %s", namespace) - return fuse.EINVAL - } + return nil } diff --git a/file_upload.go b/file_upload.go deleted file mode 100644 index fa66024..0000000 --- a/file_upload.go +++ /dev/null @@ -1,430 +0,0 @@ -package dxfuse - -import ( - "context" - "errors" - "fmt" - "os" - "sync" - "time" - - "github.com/dnanexus/dxda" - "github.com/hashicorp/go-retryablehttp" - "github.com/jacobsa/fuse" -) - -type Chunk struct { - fileId string - index int - data []byte - fwg *sync.WaitGroup - - // output from the operation - err error -} - -type FileUploadReq struct { - id string - partSize int64 - uploadParams FileUploadParameters - localPath string - fileSize int64 -} - -type FileUploadGlobalState struct { - dxEnv dxda.DXEnvironment - options Options - projId2Desc map[string]DxDescribePrj - fileUploadQueue chan FileUploadReq - chunkQueue chan *Chunk - wg sync.WaitGroup - mutex sync.Mutex - mdb *MetadataDb - ops *DxOps - - // list of files undergoing upload. If the flag is true, the file should be - // uploaded. If it is false, the upload was cancelled. - ongoingOps map[string]bool -} - -const ( - chunkMaxQueueSize = 10 - - numFileThreads = 4 - numBulkDataThreads = 8 - minChunkSize = 16 * MiB - - fileCloseWaitTime = 5 * time.Second - fileCloseMaxWaitTime = 10 * time.Minute -) - -func NewFileUploadGlobalState( - options Options, - dxEnv dxda.DXEnvironment, - projId2Desc map[string]DxDescribePrj) *FileUploadGlobalState { - - // the chunk queue size should be at least the size of the thread - // pool. - chunkQueueSize := MaxInt(numBulkDataThreads, chunkMaxQueueSize) - - fugs := &FileUploadGlobalState{ - dxEnv : dxEnv, - options : options, - projId2Desc : projId2Desc, - fileUploadQueue : make(chan FileUploadReq), - - // limit the size of the chunk queue, so we don't - // have too many chunks stored in memory. - chunkQueue : make(chan *Chunk, chunkQueueSize), - - mutex : sync.Mutex{}, - ops : NewDxOps(dxEnv, options), - ongoingOps : make(map[string]bool, 0), - } - - // Create a bunch of threads - fugs.wg.Add(numFileThreads) - for i := 0; i < numFileThreads; i++ { - go fugs.createFileWorker() - } - - fugs.wg.Add(numBulkDataThreads) - for i := 0; i < numBulkDataThreads; i++ { - go fugs.bulkDataWorker() - } - - return fugs -} - -// write a log message, and add a header -func (fugs *FileUploadGlobalState) log(a string, args ...interface{}) { - LogMsg("file_upload", a, args...) -} - -func (fugs *FileUploadGlobalState) Shutdown() { - // signal all upload threads to stop - close(fugs.fileUploadQueue) - close(fugs.chunkQueue) - - // wait for all of them to complete - fugs.wg.Wait() -} - -func (fugs *FileUploadGlobalState) CancelUpload(fileId string) { - fugs.mutex.Lock() - fugs.mutex.Unlock() - _, ok := fugs.ongoingOps[fileId] - if !ok { - // The file is not being uploaded. We are done. - return - } - fugs.ongoingOps[fileId] = false -} - -// A worker dedicated to performing data-upload operations -func (fugs *FileUploadGlobalState) bulkDataWorker() { - // A fixed http client - client := dxda.NewHttpClient(true) - - for true { - chunk, ok := <- fugs.chunkQueue - if !ok { - fugs.wg.Done() - return - } - if fugs.options.Verbose { - fugs.log("Uploading chunk=%d len=%d", chunk.index, len(chunk.data)) - } - - // upload the data, and store the error code in the chunk - // data structure. - chunk.err = fugs.ops.DxFileUploadPart( - context.TODO(), - client, - chunk.fileId, chunk.index, chunk.data) - - // release the memory used by the chunk, we no longer - // need it. The file-thread is going to check the error code, - // so the struct itself remains alive. - chunk.data = nil - chunk.fwg.Done() - } -} - - -func divideRoundUp(x int64, y int64) int64 { - return (x + y - 1) / y -} - -// Check if a part size can work for a file -func checkPartSizeSolution(param FileUploadParameters, fileSize int64, partSize int64) bool { - if partSize < param.MinimumPartSize { - return false - } - if partSize > param.MaximumPartSize { - return false - } - numParts := divideRoundUp(fileSize, partSize) - if numParts > param.MaximumNumParts { - return false - } - return true -} - -func (fugs *FileUploadGlobalState) calcPartSize(param FileUploadParameters, fileSize int64) (int64, error) { - if param.MaximumFileSize < fileSize { - return 0, errors.New( - fmt.Sprintf("File is too large, the limit is %d, and the file is %d", - param.MaximumFileSize, fileSize)) - } - - // The minimal number of parts we'll need for this file - minNumParts := divideRoundUp(fileSize, param.MaximumPartSize) - - if minNumParts > param.MaximumNumParts { - return 0, errors.New( - fmt.Sprintf("We need at least %d parts for the file, but the limit is %d", - minNumParts, param.MaximumNumParts)) - } - - // now we know that there is a solution. We'll try to use a small part size, - // to reduce memory requirements. However, we don't want really small parts, which is why - // we use [minChunkSize]. - preferedChunkSize := divideRoundUp(param.MinimumPartSize, minChunkSize) * minChunkSize - for preferedChunkSize < param.MaximumPartSize { - if (checkPartSizeSolution(param, fileSize, preferedChunkSize)) { - return preferedChunkSize, nil - } - preferedChunkSize *= 2 - } - - // nothing smaller will work, we need to use the maximal file size - return param.MaximumPartSize, nil -} - -// read a range in a file -func readLocalFileExtent(filename string, ofs int64, len int) ([]byte, error) { - fReader, err := os.Open(filename) - if err != nil { - return nil, err - } - defer fReader.Close() - - buf := make([]byte, len) - recvLen, err := fReader.ReadAt(buf, ofs) - if err != nil { - return nil, err - } - if recvLen != len { - panic(fmt.Sprintf("short read, got %d bytes instead of %d", - recvLen, len)) - } - return buf, nil -} - -// Upload the parts. Small files are uploaded synchronously, large -// files are uploaded by worker threads. -// -// note: chunk indexes start at 1 (not zero) -func (fugs *FileUploadGlobalState) uploadFileData( - client *retryablehttp.Client, - upReq FileUploadReq) error { - if upReq.fileSize == 0 { - panic("The file is empty") - } - - if upReq.fileSize <= upReq.partSize { - // This is a small file, upload it synchronously. - // This ensures that only large chunks are uploaded by the bulk-threads, - // improving fairness. - data, err := readLocalFileExtent(upReq.localPath, 0, int(upReq.fileSize)) - if err != nil { - return err - } - return fugs.ops.DxFileUploadPart( - context.TODO(), - client, - upReq.id, 1, data) - } - - // a large file, with more than a single chunk - var fileWg sync.WaitGroup - fileEndOfs := upReq.fileSize - 1 - ofs := int64(0) - cIndex := 1 - fileParts := make([]*Chunk, 0) - for ofs <= fileEndOfs { - chunkEndOfs := MinInt64(ofs + upReq.partSize - 1, fileEndOfs) - chunkLen := chunkEndOfs - ofs - buf, err := readLocalFileExtent(upReq.localPath, ofs, int(chunkLen)) - if err != nil { - return err - } - chunk := &Chunk{ - fileId : upReq.id, - index : cIndex, - data : buf, - fwg : &fileWg, - err : nil, - } - // enqueue an upload request. This can block, if there - // are many chunks. - fileWg.Add(1) - fugs.chunkQueue <- chunk - fileParts = append(fileParts, chunk) - - ofs += upReq.partSize - cIndex++ - } - - // wait for all requests to complete - fileWg.Wait() - - // check the error codes - var finalErr error - for _, chunk := range(fileParts) { - if chunk.err != nil { - fugs.log("failed to upload file %s part %d, error=%s", - chunk.fileId, chunk.index, chunk.err.Error()) - finalErr = chunk.err - } - } - - return finalErr -} - -func (fugs *FileUploadGlobalState) createEmptyFile( - httpClient *retryablehttp.Client, - upReq FileUploadReq) error { - // The file is empty - if upReq.uploadParams.EmptyLastPartAllowed { - // we need to upload an empty part, only - // then can we close the file - ctx := context.TODO() - err := fugs.ops.DxFileUploadPart(ctx, httpClient, upReq.id, 1, make([]byte, 0)) - if err != nil { - fugs.log("error uploading empty chunk to file %s", upReq.id) - return err - } - } else { - // The file can have no parts. - } - return nil -} - -func (fugs *FileUploadGlobalState) uploadFileDataAndWait( - client *retryablehttp.Client, - upReq FileUploadReq) error { - if fugs.options.Verbose { - fugs.log("Upload file-size=%d part-size=%d", upReq.fileSize, upReq.partSize) - } - - if upReq.fileSize == 0 { - // Create an empty file - if err := fugs.createEmptyFile(client, upReq); err != nil { - return err - } - } else { - // loop over the parts, and upload them - if err := fugs.uploadFileData(client, upReq); err != nil { - return err - } - } - - if fugs.options.Verbose { - fugs.log("Closing %s", upReq.id) - } - ctx := context.TODO() - return fugs.ops.DxFileCloseAndWait(ctx, client, upReq.id, fugs.options.Verbose) -} - -// check if the upload has been cancelled -func (fugs *FileUploadGlobalState) shouldUpload(fileid string) bool { - fugs.mutex.Lock() - defer fugs.mutex.Unlock() - - return fugs.ongoingOps[fileid] -} - -func (fugs *FileUploadGlobalState) uploadComplete(fileid string) { - fugs.mutex.Lock() - defer fugs.mutex.Unlock() - - delete(fugs.ongoingOps, fileid) -} - -func (fugs *FileUploadGlobalState) createFileWorker() { - // A fixed http client. The idea is to be able to reuse http connections. - client := dxda.NewHttpClient(true) - - for true { - upReq, ok := <-fugs.fileUploadQueue - if !ok { - fugs.wg.Done() - return - } - - // check if the upload has been cancelled - if !fugs.shouldUpload(upReq.id) { - fugs.log("file %s was removed, no need to upload", upReq.id) - fugs.uploadComplete(upReq.id) - continue - } - - err := fugs.uploadFileDataAndWait(client, upReq) - - if err != nil { - // Upload failed. Do not erase the local copy. - // - // Note: we have not entire eliminated the race condition - // between uploading and removing a file. We may still - // get errors in good path cases. - if (fugs.shouldUpload(upReq.id)) { - fugs.log("Error during upload of file %s", upReq.id) - fugs.log(err.Error()) - } - fugs.uploadComplete(upReq.id) - continue - } - - // Update the database to point to the remote file copy. This saves - // space on the local drive. - //fugs.mdb.UpdateFileMakeRemote(context.TODO(), upReq.id) - - fugs.uploadComplete(upReq.id) - } -} - -// enqueue a request to upload the file. This will happen in the background. Since -// we don't erase the local file, there is no rush. -func (fugs *FileUploadGlobalState) UploadFile(f File, fileSize int64) error { - projDesc, ok := fugs.projId2Desc[f.ProjId] - if !ok { - panic(fmt.Sprintf("project %s not found", f.ProjId)) - } - - partSize, err := fugs.calcPartSize(projDesc.UploadParams, fileSize) - if err != nil { - fugs.log(` -There is a problem with the file size, it cannot be uploaded -to the platform due to part size constraints. Error=%s`, - err.Error()) - return fuse.EINVAL - } - - // Add the file to the "being-uploaded" list. - // we will need this in the corner case of deleting the file while - // it is being uploaded. - fugs.mutex.Lock() - fugs.ongoingOps[f.Id] = true - fugs.mutex.Unlock() - - fugs.fileUploadQueue <- FileUploadReq{ - id : f.Id, - partSize : partSize, - uploadParams : projDesc.UploadParams, - localPath : f.InlineData, - fileSize : fileSize, - } - return nil -} diff --git a/manifest.go b/manifest.go index 4693d70..51a4573 100644 --- a/manifest.go +++ b/manifest.go @@ -22,6 +22,7 @@ type ManifestFile struct { // These may not be provided by the user. Then, we // need to query DNAx for the information. + State string `json:"state,omitempty"` ArchivalState string `json:"archivalState,omitempty"` Fname string `json:"fname,omitempty"` Size int64 `json:"size,omitempty"` @@ -300,7 +301,9 @@ func (m *Manifest) FillInMissingFields(ctx context.Context, dxEnv dxda.DXEnviron // Make a list of all the files that are missing details fileIds := make(map[string]bool) for _, fl := range m.Files { - if fl.Fname == "" || + if fl.State == "" || + fl.ArchivalState == "" || + fl.Fname == "" || fl.Size == 0 || fl.CtimeSeconds == 0 || fl.MtimeSeconds == 0 { @@ -311,7 +314,7 @@ func (m *Manifest) FillInMissingFields(ctx context.Context, dxEnv dxda.DXEnviron for fId, _ := range fileIds { fileIdList = append(fileIdList, fId) } - dataObjs, err := DxDescribeBulkObjects(ctx, tmpHttpClient, &dxEnv, fileIdList, true) + dataObjs, err := DxDescribeBulkObjects(ctx, tmpHttpClient, &dxEnv, fileIdList) if err != nil { return err } @@ -321,10 +324,16 @@ func (m *Manifest) FillInMissingFields(ctx context.Context, dxEnv dxda.DXEnviron fl := &m.Files[i] fDesc, ok := dataObjs[fl.FileId] if ok { + if fDesc.State != "closed" { + return fmt.Errorf("File %s is not closed, it is %s", + fDesc.Id, fDesc.State) + } + // This file was missing details if fl.Fname == "" { fl.Fname = fDesc.Name } + fl.State = fDesc.State fl.ArchivalState = fDesc.ArchivalState fl.Size = fDesc.Size fl.CtimeSeconds = fDesc.CtimeSeconds diff --git a/metadata_db.go b/metadata_db.go index 3d116e2..5252c32 100644 --- a/metadata_db.go +++ b/metadata_db.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "log" + "math" "path/filepath" "os" "strings" @@ -60,11 +61,38 @@ func (mdb *MetadataDb) log(a string, args ...interface{}) { LogMsg("metadata_db", a, args...) } -// open a transaction func (mdb *MetadataDb) BeginTxn() (*sql.Tx, error) { return mdb.db.Begin() } +func (mdb *MetadataDb) opOpen() *OpHandle { + txn, err := mdb.db.Begin() + if err != nil { + log.Panic("Could not open transaction") + } + + return &OpHandle{ + httpClient : nil, + txn : txn, + err : nil, + } +} + +func (mdb *MetadataDb) opClose(oph *OpHandle) { + if oph.err == nil { + err := oph.txn.Commit() + if err != nil { + log.Panic("could not commit transaction") + } + } else { + err := oph.txn.Rollback() + if err != nil { + log.Panic("could not rollback transaction") + } + } +} + + // Construct a local sql database that holds metadata for // a large number of dx:files. This metadata_db will be consulted // when performing dxfuse operations. For example, a read-dir is @@ -85,20 +113,6 @@ func splitPath(fullPath string) (parentDir string, basename string) { } } -func boolToInt(b bool) int { - if b { - return 1 - } - return 0 -} - -func intToBool(x int) bool { - if x > 0 { - return true - } - return false -} - // Marshal a DNAx object tags to/from a string that // is stored in a database table type MTags struct { @@ -183,10 +197,12 @@ func (mdb *MetadataDb) init2(txn *sql.Tx) error { ctime bigint, mtime bigint, mode int, - nlink int, tags text, properties text, - inline_data text, + symlink text, + local_path text, + dirty_data int, + dirty_metadata int, PRIMARY KEY (inode) ); ` @@ -196,14 +212,22 @@ func (mdb *MetadataDb) init2(txn *sql.Tx) error { } sqlStmt = ` - CREATE INDEX id_index - ON data_objects (id); + CREATE INDEX idx_dirty_data + ON data_objects (dirty_data); ` if _, err := txn.Exec(sqlStmt); err != nil { mdb.log(err.Error()) - return fmt.Errorf("Could not create index id_index on table data_objects") + return fmt.Errorf("Could not create index on dirty_data column in table data_objects") } + sqlStmt = ` + CREATE INDEX idx_dirty_metadata + ON data_objects (dirty_metadata); + ` + if _, err := txn.Exec(sqlStmt); err != nil { + mdb.log(err.Error()) + return fmt.Errorf("Could not create index on dirty_metdata column in table data_objects") + } // Create a table for the namespace relationships. All members of a directory // are listed here under their parent. Linking all the tables are the inode numbers. @@ -326,6 +350,13 @@ func (mdb *MetadataDb) Init() error { return nil } +func (mdb *MetadataDb) Shutdown() { + if err := mdb.db.Close(); err != nil { + mdb.log(err.Error()) + mdb.log("Error closing the sqlite database %s", mdb.dbFullPath) + } +} + // Allocate an inode number. These must remain stable during the // lifetime of the mount. // @@ -335,43 +366,6 @@ func (mdb *MetadataDb) allocInodeNum() int64 { return mdb.inodeCnt } -// search for a file by Id. If the file exists, return its inode and link-count. Otherwise, -// return 0, 0. -func (mdb *MetadataDb) lookupDataObjectById(oph *OpHandle, fId string) (int64, int, bool, error) { - // point lookup in the files table - sqlStmt := fmt.Sprintf(` - SELECT inode,nlink - FROM data_objects - WHERE id = '%s';`, - fId) - rows, err := oph.txn.Query(sqlStmt) - if err != nil { - mdb.log("lookupDataObjectById fId=%s err=%s", fId, err.Error()) - return InodeInvalid, 0, false, oph.RecordError(err) - } - - var nlink int - var inode int64 - numRows := 0 - for rows.Next() { - rows.Scan(&inode, &nlink) - numRows++ - } - rows.Close() - - switch numRows { - case 0: - // this file doesn't exist in the database - return InodeInvalid, 0, false, nil - case 1: - // correct, there is exactly one such file - return inode, nlink, true, nil - default: - log.Panicf("Found %d data-objects with Id %s", numRows, fId) - return 0, 0, false, nil - } -} - // search for a file with a particular inode // // This is important for a file with multiple hard links. The @@ -380,7 +374,7 @@ func (mdb *MetadataDb) lookupDataObjectById(oph *OpHandle, fId string) (int64, i func (mdb *MetadataDb) lookupDataObjectByInode(oph *OpHandle, oname string, inode int64) (File, bool, error) { // point lookup in the files table sqlStmt := fmt.Sprintf(` - SELECT kind,id,proj_id,state,archival_state,size,ctime,mtime,mode,nlink,tags,properties,inline_data + SELECT kind,id,proj_id,state,archival_state,size,ctime,mtime,mode,tags,properties,symlink,local_path, dirty_data, dirty_metadata FROM data_objects WHERE inode = '%d';`, inode) @@ -402,12 +396,16 @@ func (mdb *MetadataDb) lookupDataObjectByInode(oph *OpHandle, oname string, inod var mtime int64 var props string var tags string - rows.Scan(&f.Kind, &f.Id, &f.ProjId, &f.State, &f.ArchivalState, &f.Size, &ctime, &mtime, &f.Mode, &f.Nlink, - &tags, &props, &f.InlineData) + var dirtyData int + var dirtyMetadata int + rows.Scan(&f.Kind, &f.Id, &f.ProjId, &f.State, &f.ArchivalState, &f.Size, &ctime, &mtime, &f.Mode, + &tags, &props, &f.Symlink, &f.LocalPath, &dirtyData, &dirtyMetadata) f.Ctime = SecondsToTime(ctime) f.Mtime = SecondsToTime(mtime) f.Tags = tagsUnmarshal(tags) f.Properties = propertiesUnmarshal(props) + f.dirtyData = intToBool(dirtyData) + f.dirtyMetadata = intToBool(dirtyMetadata) numRows++ } rows.Close() @@ -420,7 +418,7 @@ func (mdb *MetadataDb) lookupDataObjectByInode(oph *OpHandle, oname string, inod // found exactly one file return f, true, nil default: - log.Panicf("Found %d data-objects with name %s", numRows, oname) + log.Panicf("Found %d data-objects with inode=%d (name %s)", numRows, inode, oname) return File{}, false, nil } } @@ -487,8 +485,7 @@ func (mdb *MetadataDb) lookupDirByInode(oph *OpHandle, parent string, dname stri // search for a file with a particular inode. // -// Note: a file with multiple hard links will appear several times. -func (mdb *MetadataDb) LookupByInodeAll(ctx context.Context, oph *OpHandle, inode int64) ([]Node, error) { +func (mdb *MetadataDb) LookupByInode(ctx context.Context, oph *OpHandle, inode int64) (Node, bool, error) { // point lookup in the namespace table sqlStmt := fmt.Sprintf(` SELECT parent,name,obj_type @@ -497,86 +494,112 @@ func (mdb *MetadataDb) LookupByInodeAll(ctx context.Context, oph *OpHandle, inod inode) rows, err := oph.txn.Query(sqlStmt) if err != nil { - mdb.log("LookupByInodeAll: error in query err=%s", err.Error()) - return nil, oph.RecordError(err) + mdb.log("LookupByInode: error in query err=%s", err.Error()) + return nil, false, oph.RecordError(err) } - var parents []string - var names []string - var obj_types []int + var parent string + var name string + var obj_type int + numRows := 0 for rows.Next() { - var p string - var n string - var o int - rows.Scan(&p, &n, &o) - - parents = append(parents, p) - names = append(names, n) - obj_types = append(obj_types, o) + rows.Scan(&parent, &name, &obj_type) + numRows++ } rows.Close() + if numRows == 0 { + return nil, false, nil + } + if numRows > 1 { + log.Panicf("More than one node with inode=%d", inode) + return nil, false, nil + } - if len(parents) == 0 { - return nil, nil - } - - var nodes []Node - for i, obj_type := range(obj_types) { - var node Node - var ok bool - var err error - - switch obj_type { - case nsDirType: - node, ok, err = mdb.lookupDirByInode(oph, parents[i], names[i], inode) - case nsDataObjType: - // This is important for a file with multiple hard links. The - // parent directory determines which project the file belongs to. - node, ok, err = mdb.lookupDataObjectByInode(oph, names[i], inode) - default: - log.Panicf("Invalid type %d in namespace table", obj_type) - } - - if err != nil { - return nil, err - } - if ok { - nodes = append(nodes, node) - } + switch obj_type { + case nsDirType: + return mdb.lookupDirByInode(oph, parent, name, inode) + case nsDataObjType: + // This is important for a file with multiple hard links. The + // parent directory determines which project the file belongs to. + return mdb.lookupDataObjectByInode(oph, name, inode) + default: + log.Panicf("Invalid type %d in namespace table", obj_type) + return nil, false, nil } - return nodes, nil } func (mdb *MetadataDb) LookupDirByInode(ctx context.Context, oph *OpHandle, inode int64) (Dir, bool, error) { - nodes,err := mdb.LookupByInodeAll(ctx, oph, inode) + node, ok, err := mdb.LookupByInode(ctx, oph, inode) + if !ok { + return Dir{}, false, err + } if err != nil { return Dir{}, false, err } - switch len(nodes) { - case 0: - return Dir{}, false, nil - case 1: - dir := nodes[0].(Dir) - return dir, true, nil - default: - log.Panicf("found multiple directories for inode=%d", inode) - return Dir{}, false, nil - } + dir := node.(Dir) + return dir, true, nil } -// Return one of the hits for an inode. A file that has multiple hard links will appear -// multiple times, and we return only the first hit. -func (mdb *MetadataDb) LookupByInode(ctx context.Context, oph *OpHandle, inode int64) (Node, bool, error) { - nodes, err := mdb.LookupByInodeAll(ctx, oph, inode) +// Find information on a directory by searching on its full name. +// +func (mdb *MetadataDb) lookupDirByName(oph *OpHandle, dirname string) (string, string, error) { + parentDir, basename := splitPath(dirname) + if mdb.options.Verbose { + mdb.log("lookupDirByName (%s)", dirname) + } + + // Extract information for all the subdirectories + sqlStmt := fmt.Sprintf(` + SELECT dirs.proj_id, dirs.proj_folder, nm.obj_type + FROM directories as dirs + JOIN namespace as nm + ON dirs.inode = nm.inode + WHERE nm.parent = '%s' AND nm.name = '%s' ;`, + parentDir, basename) + rows, err := oph.txn.Query(sqlStmt) if err != nil { - return File{}, false, oph.RecordError(err) + return "", "", err } - if len(nodes) == 0 { - return File{}, false, nil + + numRows := 0 + var projId string + var projFolder string + var objType int + for rows.Next() { + numRows++ + rows.Scan(&projId, &projFolder, &objType) + if objType != nsDirType { + log.Panicf("looking for a directory, but found a file") + } } - return nodes[0], true, nil + rows.Close() + + if numRows != 1 { + log.Panicf("looking for directory %s, and found %d of them", + dirname, numRows) + } + return projId, projFolder, nil +} + +// We wrote a new version of this file, creating a new file-id. +func (mdb *MetadataDb) UpdateInodeFileId(inode int64, fileId string) error { + oph := mdb.opOpen() + defer mdb.opClose(oph) + + sqlStmt := fmt.Sprintf(` + UPDATE data_objects + SET id = '%s' + WHERE inode = '%d';`, + fileId, inode) + if _, err := oph.txn.Exec(sqlStmt); err != nil { + mdb.log("UpdateInodeFileId Error updating data_object table, %s", + err.Error()) + return err + } + return nil } + // The directory is in the database, read it in its entirety. func (mdb *MetadataDb) directoryReadAllEntries( oph *OpHandle, @@ -623,7 +646,7 @@ func (mdb *MetadataDb) directoryReadAllEntries( // Extract information for all the files sqlStmt = fmt.Sprintf(` - SELECT dos.kind, dos.id, dos.proj_id, dos.state, dos.archival_state, dos.inode, dos.size, dos.ctime, dos.mtime, dos.mode, dos.nlink, dos.tags, dos.properties, dos.inline_data, namespace.name + SELECT dos.kind, dos.id, dos.proj_id, dos.state, dos.archival_state, dos.inode, dos.size, dos.ctime, dos.mtime, dos.mode, dos.tags, dos.properties, dos.symlink, dos.local_path, dos.dirty_data, dos.dirty_metadata, namespace.name FROM data_objects as dos JOIN namespace ON dos.inode = namespace.inode @@ -645,13 +668,18 @@ func (mdb *MetadataDb) directoryReadAllEntries( var tags string var props string var mode int - rows.Scan(&f.Kind,&f.Id, &f.ProjId, &f.State, &f.ArchivalState, &f.Inode, &f.Size, &ctime, &mtime, &mode, &f.Nlink, - &tags, &props, &f.InlineData,&f.Name) + var dirtyData int + var dirtyMetadata int + rows.Scan(&f.Kind,&f.Id, &f.ProjId, &f.State, &f.ArchivalState, &f.Inode, + &f.Size, &ctime, &mtime, &mode, + &tags, &props, &f.Symlink, &f.LocalPath, &dirtyData, &dirtyMetadata, &f.Name) f.Ctime = SecondsToTime(ctime) f.Mtime = SecondsToTime(mtime) f.Tags = tagsUnmarshal(tags) f.Properties = propertiesUnmarshal(props) f.Mode = os.FileMode(mode) + f.dirtyData = intToBool(dirtyData) + f.dirtyMetadata = intToBool(dirtyMetadata) files[f.Name] = f } @@ -665,16 +693,13 @@ func (mdb *MetadataDb) directoryReadAllEntries( // several use cases: // 1) Create a singleton file from the manifest // 2) Create a new file, and upload it later to the platform +// (the file-id will be the empty string "") // 3) Discover a file in a directory, which may actually be a link to another file. -const ( - CDO_MUST_BE_NEW = 1 - CDO_ALREADY_EXISTS = 2 - CDO_NEUTRAL = 3 -) func (mdb *MetadataDb) createDataObject( oph *OpHandle, - flag int, kind int, + dirtyData bool, + dirtyMetadata bool, projId string, state string, archivalState string, @@ -687,63 +712,34 @@ func (mdb *MetadataDb) createDataObject( mode os.FileMode, parentDir string, fname string, - inlineData string) (int64, error) { + symlink string, + localPath string) (int64, error) { if mdb.options.VerboseLevel > 1 { mdb.log("createDataObject %s:%s %s", projId, objId, filepath.Clean(parentDir + "/" + fname)) } + // File doesn't exist, we need to choose a new inode number. + // Note: it is on stable storage, and will not change. + inode := mdb.allocInodeNum() - inode, nlink, ok, err := mdb.lookupDataObjectById(oph, objId) - if err != nil { - return 0, err - } - - if !ok { - if flag == CDO_ALREADY_EXISTS { - log.Panicf("Object %s:%s should already exists, but does not", - projId, objId) - } - - // File doesn't exist, we need to choose a new inode number. - // NOte: it is on stable storage, and will not change. - inode = mdb.allocInodeNum() - - // marshal tags and properties - mTags := tagsMarshal(tags) - mProps := propertiesMarshal(properties) - - // Create an entry for the file - sqlStmt := fmt.Sprintf(` - INSERT INTO data_objects - VALUES ('%d', '%s', '%s', '%s', '%s', '%d', '%d', '%d', '%d', '%d', '%d', '%s', '%s', '%s');`, - kind, objId, projId, state, archivalState, inode, size, ctime, mtime, int(mode), 1, - mTags, mProps, - inlineData) - if _, err := oph.txn.Exec(sqlStmt); err != nil { - mdb.log(err.Error()) - mdb.log("Error inserting into data objects table") - return 0, oph.RecordError(err) - } - } else { - if flag == CDO_MUST_BE_NEW { - log.Panicf("Object %s:%s must not be already in the database", - projId, objId) - } + // marshal tags and properties + mTags := tagsMarshal(tags) + mProps := propertiesMarshal(properties) - // File already exists, we need to increase the link count - sqlStmt := fmt.Sprintf(` - UPDATE data_objects - SET nlink = '%d' - WHERE id = '%s';`, - nlink + 1, objId) - if _, err := oph.txn.Exec(sqlStmt); err != nil { - mdb.log("Error updating data_object table, incrementing the link number err=%s", - err.Error()) - return 0, oph.RecordError(err) - } + // Create an entry for the file + sqlStmt := fmt.Sprintf(` + INSERT INTO data_objects + VALUES ('%d', '%s', '%s', '%s', '%s', '%d', '%d', '%d', '%d', '%d', '%s', '%s', '%s', '%s', '%d', '%d');`, + kind, objId, projId, state, archivalState, inode, size, ctime, mtime, int(mode), + mTags, mProps, symlink, localPath, + boolToInt(dirtyData), boolToInt(dirtyMetadata)) + if _, err := oph.txn.Exec(sqlStmt); err != nil { + mdb.log(err.Error()) + mdb.log("Error inserting into data objects table") + return 0, oph.RecordError(err) } - sqlStmt := fmt.Sprintf(` + sqlStmt = fmt.Sprintf(` INSERT INTO namespace VALUES ('%s', '%s', '%d', '%d');`, parentDir, fname, nsDataObjType, inode) @@ -884,7 +880,7 @@ func (mdb *MetadataDb) kindOfFile(o DxDescribeDataObject) int { return kind } -func inlineDataOfFile(kind int, o DxDescribeDataObject) string { +func symlinkOfFile(kind int, o DxDescribeDataObject) string { if kind == FK_Regular && len(o.SymlinkPath) > 0 { // A symbolic link kind = FK_Symlink @@ -924,12 +920,13 @@ func (mdb *MetadataDb) populateDir( for _, o := range dxObjs { kind := mdb.kindOfFile(o) - inlineData := inlineDataOfFile(kind, o) + symlink := symlinkOfFile(kind, o) _, err := mdb.createDataObject( oph, - CDO_NEUTRAL, kind, + false, + false, o.ProjId, o.State, o.ArchivalState, @@ -942,7 +939,8 @@ func (mdb *MetadataDb) populateDir( fileReadWriteMode, dirPath, o.Name, - inlineData) + symlink, + "") if err != nil { return oph.RecordError(err) } @@ -1000,7 +998,7 @@ func (mdb *MetadataDb) directoryReadFromDNAx( } // describe all (closed) files - dxDir, err := DxDescribeFolder(ctx, oph.httpClient, &mdb.dxEnv, projId, projFolder, true) + dxDir, err := DxDescribeFolder(ctx, oph.httpClient, &mdb.dxEnv, projId, projFolder) if err != nil { fmt.Printf(err.Error()) fmt.Printf("reading directory frmo DNAx error") @@ -1195,8 +1193,9 @@ func (mdb *MetadataDb) PopulateRoot(ctx context.Context, oph *OpHandle, manifest for _, fl := range manifest.Files { _, err := mdb.createDataObject( oph, - CDO_NEUTRAL, FK_Regular, + false, + false, fl.ProjId, "closed", fl.ArchivalState, @@ -1209,6 +1208,7 @@ func (mdb *MetadataDb) PopulateRoot(ctx context.Context, oph *OpHandle, manifest fileReadOnlyMode, fl.Parent, fl.Fname, + "", "") if err != nil { mdb.log(err.Error()) @@ -1246,7 +1246,6 @@ func (mdb *MetadataDb) CreateFile( ctx context.Context, oph *OpHandle, dir *Dir, - fileId string, fname string, mode os.FileMode, localPath string) (File, error) { @@ -1255,23 +1254,31 @@ func (mdb *MetadataDb) CreateFile( dir.FullPath, fname, localPath, dir.ProjId) } + // We are creating a fake DNAx file on the local machine. + // Its state doesn't quite make sense: + // 1. live, that means not archived + // 2. closed, + // 3. empty, without any data + // 4. no object ID nowSeconds := time.Now().Unix() inode, err := mdb.createDataObject( oph, - CDO_MUST_BE_NEW, FK_Regular, + true, // file is dirty, it should be uploaded. + false, dir.ProjId, "closed", "live", - fileId, + "", // no file-id yet 0, /* the file is empty */ nowSeconds, nowSeconds, - nil, // A local file doesn't have tags or properties + nil, // A local file initially doesn't have tags or properties nil, mode, dir.FullPath, fname, + "", localPath) if err != nil { mdb.log("CreateFile error creating data object") @@ -1281,7 +1288,7 @@ func (mdb *MetadataDb) CreateFile( // 3. return a File structure return File{ Kind: FK_Regular, - Id : fileId, + Id : "", ProjId : dir.ProjId, ArchivalState : "live", Name : fname, @@ -1290,138 +1297,102 @@ func (mdb *MetadataDb) CreateFile( Ctime : SecondsToTime(nowSeconds), Mtime : SecondsToTime(nowSeconds), Mode : mode, - Nlink : 1, - InlineData : localPath, + Symlink: "", + LocalPath : localPath, }, nil } -// We know that -// 1) the parent directory exists and is populated -// 2) the target file -// 3) the source i-node exists -func (mdb *MetadataDb) CreateLink(ctx context.Context, oph *OpHandle, srcFile File, dstParent Dir, name string) (File, error) { - // insert into the database - _, err := mdb.createDataObject( - oph, - CDO_ALREADY_EXISTS, - FK_Regular, - dstParent.ProjId, - srcFile.State, - srcFile.ArchivalState, - srcFile.Id, - srcFile.Size, - int64(srcFile.Ctime.Second()), - int64(srcFile.Mtime.Second()), - nil, // cloning does not copy the tags or properties - nil, - srcFile.Mode, - dstParent.FullPath, - name, - "") - if err != nil { - mdb.log("CreateLink error creating data object") - return File{}, oph.RecordError(err) - } - - // 3. return a File structure - return File{ - Kind: FK_Regular, - Id : srcFile.Id, - ProjId : dstParent.ProjId, - ArchivalState : srcFile.ArchivalState, - Name : name, - Size : srcFile.Size, - Inode : srcFile.Inode, - Ctime : srcFile.Ctime, - Mtime : srcFile.Mtime, - Mode : srcFile.Mode, - Nlink : srcFile.Nlink + 1, - InlineData : "", - }, nil -} - -// reduce link count by one. If it reaches zero, delete the file. -// // TODO: take into account the case of ForgetInode, and files that are open, but unlinked. +// +// on this file system, since we don't keep track of link count, this amount to removing the file. func (mdb *MetadataDb) Unlink(ctx context.Context, oph *OpHandle, file File) error { - nlink := file.Nlink - 1 - if nlink > 0 { - // reduce one from the link count. It is still positive, - // so there is nothing else to do - sqlStmt := fmt.Sprintf(` - UPDATE data_objects - SET nlink = '%d' - WHERE inode = '%d'`, - nlink, file.Inode) - if _, err := oph.txn.Exec(sqlStmt); err != nil { - mdb.log(err.Error()) - mdb.log("could not reduce the link count for inode=%d to %d", - file.Inode, nlink) - return oph.RecordError(err) - } - } else { - // the link hit zero, we can remove the file - sqlStmt := fmt.Sprintf(` + sqlStmt := fmt.Sprintf(` DELETE FROM namespace WHERE inode='%d';`, + file.Inode) + if _, err := oph.txn.Exec(sqlStmt); err != nil { + mdb.log(err.Error()) + mdb.log("could not delete row for inode=%d from the namespace table", file.Inode) - if _, err := oph.txn.Exec(sqlStmt); err != nil { - mdb.log(err.Error()) - mdb.log("could not delete row for inode=%d from the namespace table", - file.Inode) - return oph.RecordError(err) - } + return oph.RecordError(err) + } - sqlStmt = fmt.Sprintf(` + sqlStmt = fmt.Sprintf(` DELETE FROM data_objects WHERE inode='%d';`, + file.Inode) + if _, err := oph.txn.Exec(sqlStmt); err != nil { + mdb.log(err.Error()) + mdb.log("could not delete row for inode=%d from the data_objects table", file.Inode) - if _, err := oph.txn.Exec(sqlStmt); err != nil { - mdb.log(err.Error()) - mdb.log("could not delete row for inode=%d from the data_objects table", - file.Inode) - return oph.RecordError(err) - } - - if file.Kind == RW_File || file.Kind == RO_LocalCopy { - // remove the file data so it does not take up space on disk. - // This might be undergoing upload at the moment. Removing the local - // file will cause the download to fail early, which is what we - // want. - if err := os.Remove(file.InlineData); err != nil { - mdb.log(err.Error()) - } - } + return oph.RecordError(err) } return nil } -func (mdb *MetadataDb) UpdateFile( +func (mdb *MetadataDb) UpdateFileAttrs( ctx context.Context, oph *OpHandle, - f File, + inode int64, fileSize int64, modTime time.Time, - mode os.FileMode) error { + mode *os.FileMode) error { + modTimeSec := modTime.Unix() + + sqlStmt := "" + if mode == nil { + // don't update the mode + if mdb.options.Verbose { + mdb.log("Update inode=%d size=%d", inode, fileSize) + } + sqlStmt = fmt.Sprintf(` + UPDATE data_objects + SET size = '%d', mtime='%d', dirty_data='1' + WHERE inode = '%d';`, + fileSize, modTimeSec, inode) + } else { + if mdb.options.Verbose { + mdb.log("Update inode=%d size=%d mode=%d", inode, fileSize, mode) + } + sqlStmt = fmt.Sprintf(` + UPDATE data_objects + SET size = '%d', mtime='%d', mode='%d', dirty_data='1' + WHERE inode = '%d';`, + fileSize, modTimeSec, int(*mode), inode) + } + + if _, err := oph.txn.Exec(sqlStmt); err != nil { + mdb.log(err.Error()) + mdb.log("UpdateFile error executing transaction") + return oph.RecordError(err) + } + return nil +} + +func (mdb *MetadataDb) UpdateFileLocalPath( + ctx context.Context, + oph *OpHandle, + inode int64, + localPath string) error { if mdb.options.Verbose { - mdb.log("Update file=%v size=%d mode=%d", f, fileSize, mode) + mdb.log("Update inode=%d localPath=%s", inode, localPath) } - modTimeSec := modTime.Unix() sqlStmt := fmt.Sprintf(` UPDATE data_objects - SET size = '%d', mtime='%d', mode='%d' + SET local_path = '%s' WHERE inode = '%d';`, - fileSize, modTimeSec, int(mode), f.Inode) + localPath, inode) if _, err := oph.txn.Exec(sqlStmt); err != nil { mdb.log(err.Error()) - mdb.log("UpdateFile error executing transaction") + mdb.log("UpdateFileLocalPath error executing transaction") return oph.RecordError(err) } return nil } + // Move a file // 1) Can move a file from one directory to another, // or leave it in the same directory @@ -1620,9 +1591,9 @@ func (mdb *MetadataDb) UpdateFileTagsAndProperties( // update the database sqlStmt := fmt.Sprintf(` UPDATE data_objects - SET tags='%s', properties='%s' - WHERE id = '%s';`, - mTags, mProps, file.Id) + SET tags = '%s', properties = '%s', dirty_metadata = '1' + WHERE inode = '%d';`, + mTags, mProps, file.Inode) if _, err := oph.txn.Exec(sqlStmt); err != nil { mdb.log(err.Error()) @@ -1632,9 +1603,97 @@ func (mdb *MetadataDb) UpdateFileTagsAndProperties( return nil } -func (mdb *MetadataDb) Shutdown() { - if err := mdb.db.Close(); err != nil { - mdb.log(err.Error()) - mdb.log("Error closing the sqlite database %s", mdb.dbFullPath) +// Get a list of all the dirty files, and reset the table. The files can be modified again, +// which will set the flag to true. +func (mdb *MetadataDb) DirtyFilesGetAndReset(flag int) ([]DirtyFileInfo, error) { + oph := mdb.opOpen() + defer mdb.opClose(oph) + + var loThreshSec int64 = 0 + switch flag { + case DIRTY_FILES_ALL: + // we want to find all dirty files + loThreshSec = math.MaxInt64 + case DIRTY_FILES_INACTIVE: + // we only want recently inactive files. Otherwise, + // we'll be writing way too much. + loThreshSec = time.Now().Unix() + loThreshSec -= int64(FileWriteInactivityThresh.Seconds()) + } + + // join all the tables so we can get the file attributes, the + // directory it lives under, and which project-folder this + // corresponds to. + sqlStmt := fmt.Sprintf(` + SELECT dos.kind, + dos.inode, + dos.dirty_data as dirty_data, + dos.dirty_metadata as dirty_metadata, + dos.id, + dos.size, + dos.mtime as mtime, + dos.local_path, + dos.tags, + dos.properties, + namespace.name, + namespace.parent + FROM data_objects as dos + JOIN namespace + ON dos.inode = namespace.inode + WHERE (dirty_data = '1' OR dirty_metadata = '1') AND (mtime < '%d') ;`, + loThreshSec) + + rows, err := oph.txn.Query(sqlStmt) + if err != nil { + mdb.log("DirtyFilesGetAllAndReset err=%s", err.Error()) + return nil, err + } + + var fAr []DirtyFileInfo + for rows.Next() { + var f DirtyFileInfo + var kind int + var dirtyData int + var dirtyMetadata int + var tags string + var props string + + rows.Scan(&kind, &f.Inode, &dirtyData, &dirtyMetadata, &f.Id, + &f.FileSize, + &f.Mtime, + &f.LocalPath, &tags, &props, &f.Name, &f.Directory) + f.dirtyData = intToBool(dirtyData) + f.dirtyMetadata = intToBool(dirtyMetadata) + f.Tags = tagsUnmarshal(tags) + f.Properties = propertiesUnmarshal(props) + + if kind != FK_Regular { + log.Panicf("Non regular file has dirty data; kind=%d %v", kind, f) + } + fAr = append(fAr, f) + } + rows.Close() + + // Figure out the project folder for each file + for i, _ := range(fAr) { + projId, projFolder, err := mdb.lookupDirByName(oph, fAr[i].Directory) + if err != nil { + return nil, err + } + fAr[i].ProjId = projId + fAr[i].ProjFolder = projFolder + } + + // erase the flag from the entire table + sqlStmt = fmt.Sprintf(` + UPDATE data_objects + SET dirty_data = '0', dirty_metadata = '0' + WHERE (dirty_data = '1' OR dirty_metadata = '1') AND (mtime < '%d') ;`, + loThreshSec) + + if _, err := oph.txn.Exec(sqlStmt); err != nil { + mdb.log("Error erasing dirty_data|dirty_metadata flags (%s)", err.Error()) + return nil, err } + return fAr, nil } diff --git a/prefetch.go b/prefetch.go index ab15c18..10fccea 100644 --- a/prefetch.go +++ b/prefetch.go @@ -5,9 +5,11 @@ package dxfuse // we need to do is check the map. import ( "context" + "errors" "fmt" "log" "math/bits" + "os" "runtime" "sync" "sync/atomic" @@ -60,7 +62,8 @@ const ( // A request that one of the IO-threads will pick up type IoReq struct { hid fuseops.HandleID - f File + inode int64 + size int64 url DxDownloadURL ioSize int64 // The io size @@ -107,7 +110,9 @@ type PrefetchFileMetadata struct { // the file being tracked hid fuseops.HandleID - f File + inode int64 + id string + size int64 url DxDownloadURL state int @@ -136,10 +141,6 @@ type PrefetchGlobalState struct { ioCounter uint64 } -func fileDesc(f File) string { - return fmt.Sprintf("%d", f.Inode) -} - // presumption: there is some intersection func (iov Iovec) intersectBuffer(startOfs int64, endOfs int64) []byte { // these are offsets in the entire file @@ -166,7 +167,7 @@ func (iov Iovec) stateString() string { // write a log message, and add a header func (pfm *PrefetchFileMetadata) log(a string, args ...interface{}) { - hdr := fmt.Sprintf("prefetch(%d,%s)", pfm.hid, fileDesc(pfm.f)) + hdr := fmt.Sprintf("prefetch(%d,%s)", pfm.hid, pfm.inode) LogMsg(hdr, a, args...) } @@ -210,14 +211,6 @@ func (pfm *PrefetchFileMetadata) cancelIOs() { } } -func (pfm *PrefetchFileMetadata) reset() { - pfm.log("access is not sequential, reseting stream state inode=%d", pfm.f.Inode) - pfm.cancelIOs() - pfm.hiUserAccessOfs = 0 - pfm.state = PFM_NIL - pfm.cache = Cache{} -} - // write a log message, and add a header func (pgs *PrefetchGlobalState) log(a string, args ...interface{}) { LogMsg("prefetch", a, args...) @@ -284,6 +277,16 @@ func NewPrefetchGlobalState(verboseLevel int, dxEnv dxda.DXEnvironment) *Prefetc return pgs } +func (pgs *PrefetchGlobalState) resetPfm(pfm *PrefetchFileMetadata) { + if pgs.verbose { + pfm.log("access is not sequential, reseting stream state inode=%d", pfm.inode) + } + pfm.cancelIOs() + pfm.hiUserAccessOfs = 0 + pfm.state = PFM_NIL + pfm.cache = Cache{} +} + func (pgs *PrefetchGlobalState) Shutdown() { // signal all prefetch threads to stop close(pgs.ioQueue) @@ -302,7 +305,7 @@ func (pgs *PrefetchGlobalState) Shutdown() { for _, hid := range allHandles { pfm := pgs.getAndLockPfm(hid) if pfm != nil { - pfm.reset() + pgs.resetPfm(pfm) pfm.mutex.Unlock() } } @@ -314,22 +317,16 @@ func (pgs *PrefetchGlobalState) Shutdown() { // we aren't waiting for the periodic cleanup thread. } -func check(value bool) { - if !value { - panic("assertion failed") - } -} - func (pgs *PrefetchGlobalState) reportIfSlowIO( startTs time.Time, - f File, + inode int64, startByte int64, endByte int64) { endTs := time.Now() deltaSec := int(endTs.Sub(startTs).Seconds()) if deltaSec > slowIoThresh { - pgs.log("(%s) slow IO [%d -- %d] %d seconds", - fileDesc(f), startByte, endByte, deltaSec) + pgs.log("(inode=%d) slow IO [%d -- %d] %d seconds", + inode, startByte, endByte, deltaSec) } } @@ -339,7 +336,7 @@ func (pgs *PrefetchGlobalState) readData(client *retryablehttp.Client, ioReq IoR expectedLen := ioReq.endByte - ioReq.startByte + 1 if pgs.verbose { pgs.log("hid=%d (%s) (io=%d) reading extent from DNAx ofs=%d len=%d", - ioReq.hid, fileDesc(ioReq.f), ioReq.id, ioReq.startByte, expectedLen) + ioReq.hid, ioReq.inode, ioReq.id, ioReq.startByte, expectedLen) } headers := make(map[string]string) @@ -358,7 +355,7 @@ func (pgs *PrefetchGlobalState) readData(client *retryablehttp.Client, ioReq IoR defer timer.Stop() startTs := time.Now() - defer pgs.reportIfSlowIO(startTs, ioReq.f, ioReq.startByte, ioReq.endByte) + defer pgs.reportIfSlowIO(startTs, ioReq.inode, ioReq.startByte, ioReq.endByte) for tCnt := 0; tCnt < NumRetriesDefault; tCnt++ { data, err := dxda.DxHttpRequest(ctx, client, 1, "GET", ioReq.url.URL, headers, []byte("{}")) @@ -369,18 +366,18 @@ func (pgs *PrefetchGlobalState) readData(client *retryablehttp.Client, ioReq IoR recvLen := int64(len(data)) if recvLen != expectedLen { // retry (only) in the case of short read - pgs.log("(%s) (io=%d) received length is wrong, got %d, expected %d. Retrying.", - fileDesc(ioReq.f), ioReq.id, recvLen, expectedLen) + pgs.log("(inode=%d) (io=%d) received length is wrong, got %d, expected %d. Retrying.", + ioReq.inode, ioReq.id, recvLen, expectedLen) continue } if pgs.verbose { if err == nil { - pgs.log("(%s) (io=%d) [%d -- %d] returned correctly", - fileDesc(ioReq.f), ioReq.id, ioReq.startByte, ioReq.endByte) + pgs.log("(inode=%d) (io=%d) [%d -- %d] returned correctly", + ioReq.inode, ioReq.id, ioReq.startByte, ioReq.endByte) } else { - pgs.log("(%s) (io=%d) [%d -- %d] returned with error %s", - fileDesc(ioReq.f), ioReq.id, ioReq.startByte, ioReq.endByte, err.Error()) + pgs.log("(inode=%d) (io=%d) [%d -- %d] returned with error %s", + ioReq.inode, ioReq.id, ioReq.startByte, ioReq.endByte, err.Error()) } } return data, err @@ -390,6 +387,54 @@ func (pgs *PrefetchGlobalState) readData(client *retryablehttp.Client, ioReq IoR return nil, fmt.Errorf("Did not receive the data") } +// Download an entire file, and write it to disk. +func (pgs *PrefetchGlobalState) DownloadEntireFile( + client *retryablehttp.Client, + inode int64, + size int64, + url DxDownloadURL, + fd *os.File, + localPath string) error { + if pgs.verbose { + pgs.log("Downloading entire file (inode=%d) to %s", inode, localPath) + } + + endOfs := size - 1 + startByte := int64(0) + for startByte <= endOfs { + endByte := MinInt64(startByte + pgs.prefetchMaxIoSize - 1, endOfs) + iovLen := endByte - startByte + 1 + + // read one chunk of the file + uniqueId := atomic.AddUint64(&pgs.ioCounter, 1) + ioReq := IoReq{ + hid : 0, // Invalid handle, shouldn't be in a table + inode : inode, + size : size, + url : url, + ioSize : iovLen, + startByte : startByte, + endByte : endByte, + id : uniqueId, + } + + data, err := pgs.readData(client, ioReq) + if err != nil { + return err + } + n, err := fd.WriteAt(data, startByte) + if err != nil { + return err + } + if int64(n) != iovLen { + return errors.New("Length of local io-write is wrong") + } + + startByte += pgs.prefetchMaxIoSize + } + return nil +} + // Find the index for this chunk in the cache. The chunks may be different // size, so we need to scan. func findIovecIndex(pfm *PrefetchFileMetadata, ioReq IoReq) int { @@ -463,24 +508,24 @@ func (pgs *PrefetchGlobalState) prefetchIoWorker() { data, err := pgs.readData(client, ioReq) if pgs.verboseLevel >= 2 { - pgs.log("(%s) (io=%d) adding returned data to file", fileDesc(ioReq.f), ioReq.id) + pgs.log("(inode=%d) (io=%d) adding returned data to file", ioReq.inode, ioReq.id) } pfm := pgs.getAndLockPfm(ioReq.hid) if pfm == nil { // file is not tracked anymore - pgs.log("(%s) (io=%d) dropping prefetch IO [%d -- %d], file is no longer tracked", - fileDesc(ioReq.f), ioReq.id, ioReq.startByte, ioReq.endByte) + pgs.log("(inode=%d) (io=%d) dropping prefetch IO [%d -- %d], file is no longer tracked", + ioReq.inode, ioReq.id, ioReq.startByte, ioReq.endByte) continue } if pgs.verboseLevel >= 2 { - pgs.log("(%s) (%d) holding the PFM lock", fileDesc(ioReq.f), ioReq.id) + pgs.log("(inode=%d) (%d) holding the PFM lock", ioReq.inode, ioReq.id) } pgs.addIoReqToCache(pfm, ioReq, data, err) pfm.mutex.Unlock() if pgs.verboseLevel >= 2 { - pgs.log("(%s) (%d) Done", fileDesc(ioReq.f), ioReq.id) + pgs.log("(inode=%d) (%d) Done", ioReq.inode, ioReq.id) } } } @@ -525,7 +570,7 @@ func (pgs *PrefetchGlobalState) tableCleanupWorker() { if !pgs.isWorthIt(pfm, now) { // This stream isn't worth it, release // the cache resources - pfm.reset() + pgs.resetPfm(pfm) } pfm.mutex.Unlock() } @@ -537,19 +582,28 @@ func (pgs *PrefetchGlobalState) tableCleanupWorker() { } } -func (pgs *PrefetchGlobalState) newPrefetchFileMetadata(hid fuseops.HandleID, f File, url DxDownloadURL) *PrefetchFileMetadata { - var pfm PrefetchFileMetadata - pfm.hid = hid - pfm.f = f - pfm.url = url - pfm.lastIoTimestamp = time.Now() - pfm.hiUserAccessOfs = 0 - pfm.mw.timestamp = time.Now() - - // Initial state of the file; no IOs were detected yet - pfm.state = PFM_NIL - - return &pfm +func (pgs *PrefetchGlobalState) newPrefetchFileMetadata( + hid fuseops.HandleID, + f File, + url DxDownloadURL) *PrefetchFileMetadata { + now := time.Now() + return &PrefetchFileMetadata{ + mutex : sync.Mutex{}, + hid : hid, + inode : f.Inode, + id : f.Id, + size : f.Size, + url : url, + state : PFM_NIL, // Initial state of the file; no IOs were detected yet + lastIoTimestamp : now, + hiUserAccessOfs : 0, + mw : MeasureWindow{ + timestamp : now, + numIOs : 0, + numBytesPrefetched : 0, + numPrefetchIOs : 0, + }, + } } // setup so we can detect a sequential stream. @@ -618,11 +672,11 @@ func (pgs *PrefetchGlobalState) RemoveStreamEntry(hid fuseops.HandleID) { pfm := pgs.getAndLockPfm(hid) if pfm != nil { if pgs.verbose { - pgs.log("RemoveStreamEntry (%d, %s, %d)", hid, pfm.f.Name, pfm.f.Inode) + pgs.log("RemoveStreamEntry (%d, inode=%d)", hid, pfm.inode) } // wake up any waiting synchronous user IOs - pfm.reset() + pgs.resetPfm(pfm) pfm.mutex.Unlock() // remove from the table @@ -721,7 +775,7 @@ func (pgs *PrefetchGlobalState) moveCacheWindow(pfm *PrefetchFileMetadata, iovIn // // stretch the cache forward, but don't go over the file size. Add place holder // io-vectors, waiting for prefetch IOs to return. - lastByteInFile := pfm.f.Size - 1 + lastByteInFile := pfm.size - 1 for i := 0; i < nReadAheadChunks; i++ { startByte := (pfm.cache.endByte + 1) + (int64(i) * int64(pfm.cache.prefetchIoSize)) @@ -742,7 +796,8 @@ func (pgs *PrefetchGlobalState) moveCacheWindow(pfm *PrefetchFileMetadata, iovIn uniqueId := atomic.AddUint64(&pgs.ioCounter, 1) pgs.ioQueue <- IoReq{ hid : pfm.hid, - f : pfm.f, + inode : pfm.inode, + size : pfm.size, url : pfm.url, ioSize : iov.ioSize, startByte : iov.startByte, @@ -849,7 +904,7 @@ func (pgs *PrefetchGlobalState) markAccessedAndMaybeStartPrefetch( pgs.moveCacheWindow(pfm, last) // Have we reached the end of the file? - if pfm.cache.endByte >= pfm.f.Size - 1 { + if pfm.cache.endByte >= pfm.size - 1 { pfm.state = PFM_EOF } } @@ -1005,7 +1060,7 @@ func (pgs *PrefetchGlobalState) CacheLookup(hid fuseops.HandleID, startOfs int64 // No data is cached. Only detecting if there is sequential access. ok := pgs.markAccessedAndMaybeStartPrefetch(pfm, startOfs, endOfs) if !ok { - pfm.reset() + pgs.resetPfm(pfm) } return 0 @@ -1016,7 +1071,7 @@ func (pgs *PrefetchGlobalState) CacheLookup(hid fuseops.HandleID, startOfs int64 if retCode == DATA_OUTSIDE_CACHE { // The file is not accessed sequentially. // zero out the cache and start over. - pfm.reset() + pgs.resetPfm(pfm) } return len @@ -1026,7 +1081,7 @@ func (pgs *PrefetchGlobalState) CacheLookup(hid fuseops.HandleID, startOfs int64 return len default: - panic(fmt.Sprintf("bad state %d for fileId=%s", pfm.state, pfm.f.Id)) + log.Panicf("bad state %d for fileId=%s", pfm.state, pfm.id) + return 0 } - } diff --git a/sync_fs_db.go b/sync_fs_db.go new file mode 100644 index 0000000..0bebd90 --- /dev/null +++ b/sync_fs_db.go @@ -0,0 +1,676 @@ +package dxfuse + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "sync" + "time" + + "github.com/dnanexus/dxda" + "github.com/hashicorp/go-retryablehttp" + "github.com/jacobsa/fuse" +) + +const ( + sweepPeriodicTime = 1 * time.Minute +) + +const ( + chunkMaxQueueSize = 10 + + numFileThreads = 4 + numBulkDataThreads = 8 + numMetadataThreads = 2 + minChunkSize = 16 * MiB +) + +type Chunk struct { + fileId string + index int + data []byte + fwg *sync.WaitGroup + + // output from the operation + err error +} + +type FileUpdateReq struct { + dfi DirtyFileInfo + partSize int64 + uploadParams FileUploadParameters +} + +type SyncDbDx struct { + dxEnv dxda.DXEnvironment + options Options + projId2Desc map[string]DxDescribePrj + fileUpdateQueue chan FileUpdateReq + chunkQueue chan *Chunk + sweepStopChan chan struct{} + sweepStoppedChan chan struct{} + wg sync.WaitGroup + mutex *sync.Mutex + mdb *MetadataDb + ops *DxOps + nonce *Nonce +} + +func NewSyncDbDx( + options Options, + dxEnv dxda.DXEnvironment, + projId2Desc map[string]DxDescribePrj, + mdb *MetadataDb, + mutex *sync.Mutex) *SyncDbDx { + // the chunk queue size should be at least the size of the thread + // pool. + chunkQueueSize := MaxInt(numBulkDataThreads, chunkMaxQueueSize) + + // limit the size of the chunk queue, so we don't + // have too many chunks stored in memory. + chunkQueue := make(chan *Chunk, chunkQueueSize) + + sybx := &SyncDbDx{ + dxEnv : dxEnv, + options : options, + projId2Desc : projId2Desc, + fileUpdateQueue : nil, + chunkQueue : chunkQueue, + sweepStopChan : nil, + sweepStoppedChan : nil, + mutex : mutex, + mdb : mdb, + ops : NewDxOps(dxEnv, options), + nonce : NewNonce(), + } + + // bunch of background threads to upload bulk file data. + // + // These are never closed, because they are used during synchronization. + // When we sync the filesystem, we upload all the files. + for i := 0; i < numBulkDataThreads; i++ { + go sybx.bulkDataWorker() + } + + sybx.startBackgroundWorkers() + sybx.startSweepWorker() + + return sybx +} + +// write a log message, and add a header +func (sybx *SyncDbDx) log(a string, args ...interface{}) { + LogMsg("synx_db_dx", a, args...) +} + + +func (sybx *SyncDbDx) startSweepWorker() { + // start a periodic thread to synchronize the database with + // the platform + sybx.sweepStopChan = make(chan struct{}) + sybx.sweepStoppedChan = make(chan struct{}) + go sybx.periodicSync() +} + +func (sybx *SyncDbDx) stopSweepWorker() { + close(sybx.sweepStopChan) + <- sybx.sweepStoppedChan + + sybx.sweepStopChan = nil + sybx.sweepStoppedChan = nil +} + +func (sybx *SyncDbDx) startBackgroundWorkers() { + // Create a bunch of threads to update files and metadata + sybx.fileUpdateQueue = make(chan FileUpdateReq) + for i := 0; i < numFileThreads; i++ { + sybx.wg.Add(1) + go sybx.updateFileWorker() + } +} + +func (sybx *SyncDbDx) stopBackgroundWorkers() { + // signal all upload and modification threads to stop + close(sybx.fileUpdateQueue) + + // wait for all of them to complete + sybx.wg.Wait() + + sybx.fileUpdateQueue = nil +} + +func (sybx *SyncDbDx) Shutdown() { + sybx.stopSweepWorker() + sybx.stopBackgroundWorkers() + close(sybx.chunkQueue) +} + +// A worker dedicated to performing data-upload operations +func (sybx *SyncDbDx) bulkDataWorker() { + // A fixed http client + client := dxda.NewHttpClient(true) + + for true { + chunk, ok := <- sybx.chunkQueue + if !ok { + return + } + if sybx.options.Verbose { + sybx.log("Uploading chunk=%d len=%d", chunk.index, len(chunk.data)) + } + + // upload the data, and store the error code in the chunk + // data structure. + chunk.err = sybx.ops.DxFileUploadPart( + context.TODO(), + client, + chunk.fileId, chunk.index, chunk.data) + + // release the memory used by the chunk, we no longer + // need it. The file-thread is going to check the error code, + // so the struct itself remains alive. + chunk.data = nil + chunk.fwg.Done() + } +} + + +func divideRoundUp(x int64, y int64) int64 { + return (x + y - 1) / y +} + +// Check if a part size can work for a file +func checkPartSizeSolution(param FileUploadParameters, fileSize int64, partSize int64) bool { + if partSize < param.MinimumPartSize { + return false + } + if partSize > param.MaximumPartSize { + return false + } + numParts := divideRoundUp(fileSize, partSize) + if numParts > param.MaximumNumParts { + return false + } + return true +} + +func (sybx *SyncDbDx) calcPartSize(param FileUploadParameters, fileSize int64) (int64, error) { + if param.MaximumFileSize < fileSize { + return 0, errors.New( + fmt.Sprintf("File is too large, the limit is %d, and the file is %d", + param.MaximumFileSize, fileSize)) + } + + // The minimal number of parts we'll need for this file + minNumParts := divideRoundUp(fileSize, param.MaximumPartSize) + + if minNumParts > param.MaximumNumParts { + return 0, errors.New( + fmt.Sprintf("We need at least %d parts for the file, but the limit is %d", + minNumParts, param.MaximumNumParts)) + } + + // now we know that there is a solution. We'll try to use a small part size, + // to reduce memory requirements. However, we don't want really small parts, which is why + // we use [minChunkSize]. + preferedChunkSize := divideRoundUp(param.MinimumPartSize, minChunkSize) * minChunkSize + for preferedChunkSize < param.MaximumPartSize { + if (checkPartSizeSolution(param, fileSize, preferedChunkSize)) { + return preferedChunkSize, nil + } + preferedChunkSize *= 2 + } + + // nothing smaller will work, we need to use the maximal file size + return param.MaximumPartSize, nil +} + +// read a range in a file +func readLocalFileExtent(filename string, ofs int64, len int) ([]byte, error) { + fReader, err := os.Open(filename) + if err != nil { + return nil, err + } + defer fReader.Close() + + buf := make([]byte, len) + recvLen, err := fReader.ReadAt(buf, ofs) + if err != nil { + return nil, err + } + if recvLen != len { + log.Panicf("short read, got %d bytes instead of %d", + recvLen, len) + } + return buf, nil +} + +// Upload the parts. Small files are uploaded synchronously, large +// files are uploaded by worker threads. +// +// note: chunk indexes start at 1 (not zero) +func (sybx *SyncDbDx) uploadFileData( + client *retryablehttp.Client, + upReq FileUpdateReq, + fileId string) error { + if upReq.dfi.FileSize == 0 { + log.Panicf("The file is empty") + } + + if upReq.dfi.FileSize <= upReq.partSize { + // This is a small file, upload it synchronously. + // This ensures that only large chunks are uploaded by the bulk-threads, + // improving fairness. + data, err := readLocalFileExtent(upReq.dfi.LocalPath, 0, int(upReq.dfi.FileSize)) + if err != nil { + return err + } + return sybx.ops.DxFileUploadPart( + context.TODO(), + client, + fileId, 1, data) + } + + // a large file, with more than a single chunk + var fileWg sync.WaitGroup + fileEndOfs := upReq.dfi.FileSize - 1 + ofs := int64(0) + cIndex := 1 + fileParts := make([]*Chunk, 0) + for ofs <= fileEndOfs { + chunkEndOfs := MinInt64(ofs + upReq.partSize - 1, fileEndOfs) + chunkLen := chunkEndOfs - ofs + buf, err := readLocalFileExtent(upReq.dfi.LocalPath, ofs, int(chunkLen)) + if err != nil { + return err + } + chunk := &Chunk{ + fileId : fileId, + index : cIndex, + data : buf, + fwg : &fileWg, + err : nil, + } + // enqueue an upload request. This can block, if there + // are many chunks. + fileWg.Add(1) + sybx.chunkQueue <- chunk + fileParts = append(fileParts, chunk) + + + ofs += upReq.partSize + cIndex++ + } + + // wait for all requests to complete + fileWg.Wait() + + // check the error codes + var finalErr error + for _, chunk := range(fileParts) { + if chunk.err != nil { + sybx.log("failed to upload file %s part %d, error=%s", + chunk.fileId, chunk.index, chunk.err.Error()) + finalErr = chunk.err + } + } + + return finalErr +} + +func (sybx *SyncDbDx) createEmptyFileData( + httpClient *retryablehttp.Client, + upReq FileUpdateReq, + fileId string) error { + // The file is empty + if upReq.uploadParams.EmptyLastPartAllowed { + // we need to upload an empty part, only + // then can we close the file + ctx := context.TODO() + err := sybx.ops.DxFileUploadPart(ctx, httpClient, fileId, 1, make([]byte, 0)) + if err != nil { + sybx.log("error uploading empty chunk to file %s", fileId) + return err + } + } else { + // The file can have no parts. + } + return nil +} + +func (sybx *SyncDbDx) uploadFileDataAndWait( + client *retryablehttp.Client, + upReq FileUpdateReq, + fileId string) error { + if sybx.options.Verbose { + sybx.log("Upload file-size=%d part-size=%d", upReq.dfi.FileSize, upReq.partSize) + } + + if upReq.dfi.FileSize == 0 { + // Create an empty file + if err := sybx.createEmptyFileData(client, upReq, fileId); err != nil { + return err + } + } else { + // loop over the parts, and upload them + if err := sybx.uploadFileData(client, upReq, fileId); err != nil { + return err + } + } + + if sybx.options.Verbose { + sybx.log("Closing %s", fileId) + } + return sybx.ops.DxFileCloseAndWait(context.TODO(), client, fileId) +} + + +// Upload +func (sybx *SyncDbDx) updateFileData( + client *retryablehttp.Client, + upReq FileUpdateReq) (string, error) { + + // We need to lock the filesystem while we are doing this, because + // a race could happen if the directory is removed while the file + // is created. + sybx.mutex.Lock() + + // create the file object on the platform. + fileId, err := sybx.ops.DxFileNew( + context.TODO(), client, sybx.nonce.String(), + upReq.dfi.ProjId, + upReq.dfi.Name, + upReq.dfi.ProjFolder) + if err != nil { + sybx.mutex.Unlock() + // an error could occur here if the directory has been removed + // while we were trying to upload the file. + sybx.log("Error in creating file (%s:%s/%s) on dnanexus: %s", + upReq.dfi.ProjId, upReq.dfi.ProjFolder, upReq.dfi.Name, + err.Error()) + return "", err + } + + // Update the database with the new ID. + sybx.mdb.UpdateInodeFileId(upReq.dfi.Inode, fileId) + sybx.mutex.Unlock() + + // Note: the file may have been deleted while it was being uploaded. + // This means that error could occur here, and they would be legal. + err = sybx.uploadFileDataAndWait(client, upReq, fileId) + if err != nil { + // Upload failed. Do not erase the local copy. + // + sybx.log("Error during upload of file %s: %s", + fileId, err.Error()) + return "", err + } + + // Erase the old file-id. + if upReq.dfi.Id == "" { + // This is the first time we are creating the file, there + // is no older version on the platform. + return fileId, nil + } + + // remove the old version + var oldFileId []string + oldFileId = append(oldFileId, upReq.dfi.Id) + err = sybx.ops.DxRemoveObjects(context.TODO(), client, upReq.dfi.ProjId, oldFileId) + if err != nil { + // TODO: if the file has already been removed on the platform, then + // we will get an error here. + return "", err + } + return fileId, nil +} + +func (sybx *SyncDbDx) updateFileAttributes(client *retryablehttp.Client, dfi DirtyFileInfo) error { + // describe the object state on the platform. The properties/tags have + // changed. + fDesc, err := DxDescribe(context.TODO(), client, &sybx.dxEnv, dfi.Id) + if err != nil { + sybx.log(err.Error()) + sybx.log("Failed ot describe file %v", dfi) + return err + } + + // Figure out the symmetric difference between the on-platform properties, + // and what the filesystem has. + dnaxProps := fDesc.Properties + fsProps := dfi.Properties + opProps := make(map[string]*string) + + for key, dnaxValue := range(dnaxProps) { + fsValue, ok := fsProps[key] + if !ok { + // property was removed + opProps[key] = nil + } else if dnaxValue != fsValue { + // value has changed + opProps[key] = &fsValue + } + } + + for key, fsValue := range(fsProps) { + _, ok := dnaxProps[key] + if !ok { + // a new property + opProps[key] = &fsValue + } else { + // existing property, we already checked that case; + // if the value changed, we set it in the map + } + } + + if len(opProps) > 0 { + if sybx.options.Verbose { + sybx.log("%s symmetric difference between properties %v ^ %v = %v", + dfi.Id, dnaxProps, fsProps, opProps) + } + err := sybx.ops.DxSetProperties(context.TODO(), client, dfi.ProjId, dfi.Id, opProps) + if err != nil { + return err + } + } + + // figure out the symmetric difference between the old and new tags. + dnaxTags := fDesc.Tags + fsTags := dfi.Tags + + // make hash-tables for easy access + dnaxTagsTbl := make(map[string]bool) + for _, tag := range(dnaxTags) { + dnaxTagsTbl[tag] = true + } + fsTagsTbl := make(map[string]bool) + for _, tag := range(fsTags) { + fsTagsTbl[tag] = true + } + + var tagsRemoved []string + for _, tag := range(dnaxTags) { + _, ok := fsTagsTbl[tag] + if !ok { + tagsRemoved = append(tagsRemoved, tag) + } + } + + var tagsAdded []string + for _, tag := range(fsTags) { + _, ok := dnaxTagsTbl[tag] + if !ok { + tagsAdded = append(tagsAdded, tag) + } + } + if sybx.options.Verbose { + if len(tagsAdded) > 0 || len(tagsRemoved) > 0 { + sybx.log("%s symmetric difference between tags %v ^ %v = (added=%v, removed=%v)", + dfi.Id, dnaxTags, fsTags, tagsAdded, tagsRemoved) + } + } + + if len(tagsAdded) != 0 { + err := sybx.ops.DxAddTags(context.TODO(), client, dfi.ProjId, dfi.Id, tagsAdded) + if err != nil { + return err + } + } + if len(tagsRemoved) != 0 { + err := sybx.ops.DxRemoveTags(context.TODO(), client, dfi.ProjId, dfi.Id, tagsRemoved) + if err != nil { + return err + } + } + return nil +} + +func (sybx *SyncDbDx) updateFileWorker() { + // A fixed http client. The idea is to be able to reuse http connections. + client := dxda.NewHttpClient(true) + + for true { + upReq, ok := <-sybx.fileUpdateQueue + if !ok { + sybx.wg.Done() + return + } + if sybx.options.Verbose { + sybx.log("updateFile %v", upReq) + } + + // note: the file-id may be empty ("") if the file + // has just been created on the local machine. + var err error + crntFileId := upReq.dfi.Id + if upReq.dfi.dirtyData { + crntFileId, err = sybx.updateFileData(client, upReq) + if err != nil { + sybx.log("Error in update-data: %s", err.Error()) + continue + } + } + if upReq.dfi.dirtyMetadata { + if crntFileId == "" { + // create an empty file + check(upReq.dfi.FileSize == 0) + crntFileId, err = sybx.updateFileData(client, upReq) + if err != nil { + sybx.log("Error when creating a metadata-only file %s", + err.Error()) + continue + } + } + // file exists, figure out what needs to be + // updated + dfi := upReq.dfi + dfi.Id = crntFileId + sybx.updateFileAttributes(client, dfi) + } + } +} + +// enqueue a request to upload the file. This will happen in the background. Since +// we don't erase the local file, there is no rush. +func (sybx *SyncDbDx) enqueueUpdateFileReq(dfi DirtyFileInfo) error { + projDesc, ok := sybx.projId2Desc[dfi.ProjId] + if !ok { + log.Panicf("project (%s) not found", dfi.ProjId) + } + + partSize, err := sybx.calcPartSize(projDesc.UploadParams, dfi.FileSize) + if err != nil { + sybx.log(` +There is a problem with the file size, it cannot be uploaded +to the platform due to part size constraints. Error=%s`, + err.Error()) + return fuse.EINVAL + } + + sybx.fileUpdateQueue <- FileUpdateReq{ + dfi : dfi, + partSize : partSize, + uploadParams : projDesc.UploadParams, + } + return nil +} + +func (sybx *SyncDbDx) sweep(flag int) error { + if sybx.options.Verbose { + sybx.log("syncing database and platform [") + } + + // find all the dirty files. We need to lock + // the database while we are doing this. + sybx.mutex.Lock() + dirtyFiles, err := sybx.mdb.DirtyFilesGetAndReset(flag) + if err != nil { + sybx.mutex.Unlock() + return err + } + sybx.mutex.Unlock() + + if sybx.options.Verbose { + sybx.log("%d dirty files", len(dirtyFiles)) + } + + // enqueue them on the "to-upload" list + for _, file := range(dirtyFiles) { + sybx.enqueueUpdateFileReq(file) + } + + if sybx.options.Verbose { + sybx.log("]") + } + return nil +} + +func (sybx *SyncDbDx) periodicSync() { + sybx.log("starting sweep thread") + lastSweepTs := time.Now() + for true { + // we need to wake up often to check if + // the sweep has been disabled. + time.Sleep(1 * time.Second) + + select { + default: + // normal case, we weren't stopped + case <- sybx.sweepStopChan: + sybx.log("stopped sweep thread") + close(sybx.sweepStoppedChan) + return + } + + now := time.Now() + if now.Before(lastSweepTs.Add(sweepPeriodicTime)) { + continue + } + lastSweepTs = now + + if err := sybx.sweep(DIRTY_FILES_INACTIVE); err != nil { + sybx.log("Error in sweep: %s", err.Error()) + } + } +} + +func (sybx *SyncDbDx) CmdSync() error { + // we don't want to have two sweeps running concurrently + sybx.stopSweepWorker() + + if err := sybx.sweep(DIRTY_FILES_ALL); err != nil { + sybx.log("Error in sweep: %s", err.Error()) + return err + } + + // now wait for the objects to be created and the data uploaded + sybx.stopBackgroundWorkers() + + // start the background threads again + sybx.startBackgroundWorkers() + sybx.startSweepWorker() + + return nil +} diff --git a/test/Makefile b/test/Makefile index e7bb68f..23faba4 100644 --- a/test/Makefile +++ b/test/Makefile @@ -4,7 +4,7 @@ aws : copy_all dx build benchmark -f --destination dxfuse_test_data:/applets/benchmark dx build correctness -f --destination dxfuse_test_data:/applets/correctness dx build correctness_downloads -f --destination dxfuse_test_data:/applets/correctness_downloads - dx build bam_diff -f --destination dxfuse_test_data:/applets/bam_diff + dx build bio_tools -f --destination dxfuse_test_data:/applets/bio_tools azure: dxfuse copy_all dx build benchmark -f --destination dxfuse_azure_westus:/applets/benchmark @@ -28,7 +28,7 @@ correct_downloads: dxfuse cp -f /go/bin/dxfuse correctness_downloads/resources/usr/bin/ bio : dxfuse - cp -f /go/bin/dxfuse bam_diff/resources/usr/bin/ + cp -f /go/bin/dxfuse bio_tools/resources/usr/bin/ clean : rm -f dxfuse diff --git a/test/bam_diff/.gitignore b/test/bio_tools/.gitignore similarity index 100% rename from test/bam_diff/.gitignore rename to test/bio_tools/.gitignore diff --git a/test/bam_diff/code.sh b/test/bio_tools/code.sh similarity index 83% rename from test/bam_diff/code.sh rename to test/bio_tools/code.sh index 58618ab..36b1171 100644 --- a/test/bam_diff/code.sh +++ b/test/bio_tools/code.sh @@ -67,4 +67,12 @@ main() { end=`date +%s` runtime=$((end-start)) dx-jobutil-add-output --class=string runtime_sambamba "$runtime seconds" + +# echo "samtools split" + # samtools split --threads 1 -u SJAML030069_D1.RNA-Seq.unaccounted_reads.bam -f '%*_%!.%.' > ~/SJAML030069_D1.RNA-Seq.bam +# start=`date +%s` +# samtools split --threads 1 -u SRR10270774_markdup.A.bam -f '%*_%!.%.' > ~/filter_A.bam +# end=`date +%s` +# runtime=$((end-start)) +# dx-jobutil-add-output --class=string runtime_samtools_split "$runtime seconds" } diff --git a/test/bam_diff/dxapp.json b/test/bio_tools/dxapp.json similarity index 61% rename from test/bam_diff/dxapp.json rename to test/bio_tools/dxapp.json index ee63f10..8cc517b 100644 --- a/test/bam_diff/dxapp.json +++ b/test/bio_tools/dxapp.json @@ -1,6 +1,6 @@ { - "name": "bam_diff", - "summary": "check that bam diff works with dxfuse", + "name": "bio_tools", + "summary": "check that various bioinformatics tools work well with dxfuse", "dxapi": "1.0.0", "version": "0.0.1", "inputSpec": [ @@ -13,15 +13,23 @@ "outputSpec": [ { "name" : "runtime_bam_diff", - "class" : "string" + "class" : "string", + "optional" : true }, { "name" : "num_lines", - "class" : "int" + "class" : "int", + "optional" : true }, { "name" : "runtime_sambamba", - "class" : "string" + "class" : "string", + "optional" : true + }, + { + "name" : "runtime_samtools_split", + "class" : "string", + "optional" : true } ], "runSpec": { diff --git a/test/bam_diff/resources/usr/bin/bam b/test/bio_tools/resources/usr/bin/bam similarity index 100% rename from test/bam_diff/resources/usr/bin/bam rename to test/bio_tools/resources/usr/bin/bam diff --git a/test/bam_diff/resources/usr/bin/sambamba b/test/bio_tools/resources/usr/bin/sambamba similarity index 100% rename from test/bam_diff/resources/usr/bin/sambamba rename to test/bio_tools/resources/usr/bin/sambamba diff --git a/test/container_with_files/code.sh b/test/container_with_files/code.sh new file mode 100644 index 0000000..e511ddc --- /dev/null +++ b/test/container_with_files/code.sh @@ -0,0 +1,9 @@ +#!/bin/bash -e + +main() { + mkdir -p out/foo + mkdir -p out/bar + echo "just a test file with junk data" >> out/foo/A.txt + echo "just a test file with junk data" >> out/bar/B.txt + dx-upload-all-outputs +} diff --git a/test/container_with_files/dxapp.json b/test/container_with_files/dxapp.json new file mode 100644 index 0000000..f44cd74 --- /dev/null +++ b/test/container_with_files/dxapp.json @@ -0,0 +1,34 @@ +{ + "name": "container_with_files", + "summary": "create a container with files", + "dxapi": "1.0.0", + "version": "0.0.1", + "inputSpec": [ + { + "name": "verbose", + "class": "boolean", + "optional": true + } + ], + "outputSpec": [ + { + "name" : "foo", + "class" : "file" + }, + { + "name" : "bar", + "class" : "file" + } + ], + "runSpec": { + "interpreter": "bash", + "file": "code.sh", + "distribution": "Ubuntu", + "release": "16.04", + "timeoutPolicy" : { + "*" : { + "hours" : 1 + } + } + } +} diff --git a/test/correctness/code.sh b/test/correctness/code.sh index fef1d74..ef4121c 100644 --- a/test/correctness/code.sh +++ b/test/correctness/code.sh @@ -1,7 +1,5 @@ #!/bin/bash -e -#!/bin/bash -e - ###################################################################### ## constants @@ -63,14 +61,8 @@ function check_file_write_content { echo $content > $write_dir/A.txt ls -l $write_dir/A.txt - # wait for the file to achieve the closed state - while true; do - file_state=$(dx describe $projName:/$target_dir/A.txt --json | grep state | awk '{ gsub("[,\"]", "", $2); print $2 }') - if [[ "$file_state" == "closed" ]]; then - break - fi - sleep 2 - done + echo "synchronizing the filesystem" + sudo $dxfuse -sync echo "file is closed" dx ls -l $projName:/$target_dir/A.txt @@ -90,14 +82,8 @@ function check_file_write_content { touch $write_dir/B.txt ls -l $write_dir/B.txt - # wait for the file to achieve the closed state - while true; do - file_state=$(dx describe $projName:/$target_dir/B.txt --json | grep state | awk '{ gsub("[,\"]", "", $2); print $2 }') - if [[ "$file_state" == "closed" ]]; then - break - fi - sleep 2 - done + echo "synchronizing the filesystem" + sudo $dxfuse -sync echo "file is closed" dx ls -l $projName:/$target_dir/B.txt @@ -277,6 +263,7 @@ function file_create_existing { cd $write_dir echo "happy days" > hello.txt + chmod 444 hello.txt set +e (echo "nothing much" > hello.txt) >& /tmp/cmd_results.txt @@ -500,16 +487,6 @@ function faux_dirs_remove { -function hard_links { - local root_dir=$1 - cd $root_dir - - ln $mountpoint/dxfuse_test_read_only/doc/approaches.md . - stat approaches.md - rm -f approaches.md -} - - function populate_faux_dir { local faux_dir=$1 @@ -535,11 +512,11 @@ function archived_files { local root_dir=$1 num_files=$(ls -1 $root_dir | wc -l) - if [[ $num_files != 2 ]]; then - echo "Should see two live files. Instead, can see $num_files files." + if [[ $num_files != 3 ]]; then + echo "Should see 3 files. Instead, can see $num_files files." exit 1 else - echo "correct, can see two live files" + echo "correct, can see 3 files" fi } @@ -644,9 +621,6 @@ main() { echo "faux dir operations" faux_dirs_remove $mountpoint/$projName/$faux_dir - echo "hard links" - hard_links $mountpoint/$projName/$faux_dir - echo "directory and file with the same name" dir_and_file_with_the_same_name $mountpoint/$projName diff --git a/test/correctness_downloads/code.sh b/test/correctness_downloads/code.sh index bf087a3..8e36502 100755 --- a/test/correctness_downloads/code.sh +++ b/test/correctness_downloads/code.sh @@ -6,8 +6,7 @@ projName="dxfuse_test_data" # larger test for a cloud worker -#dxDirOnProject="correctness" -dxDirOnProject="mini" +dxDirOnProject="correctness" baseDir=$HOME/dxfuse_test mountpoint=${baseDir}/MNT diff --git a/test/local/dx_download_compare.sh b/test/local/dx_download_compare.sh new file mode 100644 index 0000000..2d64668 --- /dev/null +++ b/test/local/dx_download_compare.sh @@ -0,0 +1,84 @@ +###################################################################### +## constants + +projName="dxfuse_test_data" +dxfuse="$GOPATH/bin/dxfuse" +dxDirOnProject="mini" +baseDir=$HOME/dxfuse_test +mountpoint=${baseDir}/MNT + +dxfuseDir=$mountpoint/$projName/$dxDirOnProject +dxpyDir=${baseDir}/dxCopy/$dxDirOnProject + +###################################################################### + +teardown_complete=0 + +# cleanup sequence +function teardown { + if [[ $teardown_complete == 1 ]]; then + return + fi + teardown_complete=1 + + echo "unmounting dxfuse" + cd $HOME + sudo umount $mountpoint +} + +# trap any errors and cleanup +trap teardown EXIT + +###################################################################### + +function dx_download_compare_body { + echo "download recursively with dx download" + parentDxpyDir=$(dirname $dxpyDir) + if [[ ! -d $parentDxpyDir ]]; then + echo "downloading into $parentDxpyDir from $projName:/$dxDirOnProject" + mkdir -p $parentDxpyDir + dx download --no-progress -o $parentDxpyDir -r $projName:/$dxDirOnProject + fi + + # do not exit immediately if there are differences; we want to see the files + # that aren't the same + echo "recursive compare" + diff -r --brief $dxpyDir $dxfuseDir > diff.txt || true + if [[ -s diff.txt ]]; then + echo "Difference in basic file structure" + cat diff.txt + echo "===== dxpy ==== " + tree $dxpyDir + echo + echo "===== dxfuse ==== " + tree $dxfuseDir + exit 1 + fi +} + +function dx_download_compare { + # Get all the DX environment variables, so that dxfuse can use them + echo "loading the dx environment" + + # local machine + rm -f ENV + dx env --bash > ENV + source ENV >& /dev/null + rm -f ENV + + # clean and make fresh directories + mkdir -p $mountpoint + + # Start the dxfuse daemon in the background, and wait for it to initilize. + echo "Mounting dxfuse" + flags="" + if [[ $verbose != "" ]]; then + flags="-verbose 2" + fi + sudo -E $dxfuse -uid $(id -u) -gid $(id -g) $flags $mountpoint dxfuse_test_data + sleep 1 + + dx_download_compare_body + + teardown +} diff --git a/test/local/faux_dirs.sh b/test/local/faux_dirs.sh new file mode 100644 index 0000000..985fec5 --- /dev/null +++ b/test/local/faux_dirs.sh @@ -0,0 +1,132 @@ +###################################################################### +## constants + +projName="dxfuse_test_data" +dxfuse="$GOPATH/bin/dxfuse" +dxDirOnProject="mini" +baseDir=$HOME/dxfuse_test +mountpoint=${baseDir}/MNT + +# Directories created during the test +writeable_dirs=() +###################################################################### + +teardown_complete=0 + +# cleanup sequence +function teardown { + if [[ $teardown_complete == 1 ]]; then + return + fi + teardown_complete=1 + + rm -f cmd_results.txt + + echo "unmounting dxfuse" + cd $HOME + sudo umount $mountpoint + + for d in ${writeable_dirs[@]}; do + dx rm -r $projName:/$d >& /dev/null || true + done +} + +# trap any errors and cleanup +trap teardown EXIT + +###################################################################### + +function faux_dirs_move { + local root_dir=$1 + cd $root_dir + + tree $root_dir + + # cannot move faux directories + set +e + (mv $root_dir/1 $root_dir/2 ) >& /dev/null + rc=$? + if [[ $rc == 0 ]]; then + echo "Error, could not a faux directory" + fi + + # cannot move files into a faux directory + (mv -f $root_dir/NewYork.txt $root_dir/1) >& /dev/null + rc=$? + if [[ $rc == 0 ]]; then + echo "Error, could move a file into a faux directory" + fi + set -e +} + +function faux_dirs_remove { + local root_dir=$1 + cd $root_dir + + # can move a file out of a faux directory + mkdir $root_dir/T + mv $root_dir/1/NewYork.txt $root_dir/T + rm -rf $root_dir/T + + echo "removing faux dir 1" + rm -rf $root_dir/1 +} + + +function populate_faux_dir { + local faux_dir=$1 + + echo "deep dish pizza and sky trains" > /tmp/XXX + echo "nice play chunk" > /tmp/YYY + echo "no more chewing on shoes!" > /tmp/ZZZ + echo "you just won a trip to the Caribbean" > /tmp/VVV + + dx upload /tmp/XXX -p --destination $projName:/$faux_dir/Chicago.txt >& /dev/null + dx upload /tmp/YYY -p --destination $projName:/$faux_dir/Chicago.txt >& /dev/null + dx upload /tmp/ZZZ -p --destination $projName:/$faux_dir/NewYork.txt >& /dev/null + dx upload /tmp/VVV -p --destination $projName:/$faux_dir/NewYork.txt >& /dev/null + rm -f /tmp/XXX /tmp/YYY /tmp/ZZZ /tmp/VVV +} + +###################################################################### + +function faux_dirs { + # Get all the DX environment variables, so that dxfuse can use them + echo "loading the dx environment" + + # local machine + rm -f ENV + dx env --bash > ENV + source ENV >& /dev/null + rm -f ENV + + # clean and make fresh directories + mkdir -p $mountpoint + + # generate random alphanumeric strings + faux_dir=$(cat /dev/urandom | env LC_CTYPE=C LC_ALL=C tr -dc 'a-zA-Z0-9' | fold -w 12 | head -n 1) + faux_dir="faux_$faux_dir" + writeable_dirs=($faux_dir) + for d in ${writeable_dirs[@]}; do + dx rm -r $projName:/$d >& /dev/null || true + done + + populate_faux_dir $faux_dir + + # Start the dxfuse daemon in the background, and wait for it to initilize. + echo "Mounting dxfuse" + flags="" + if [[ $verbose != "" ]]; then + flags="-verbose 2" + fi + sudo -E $dxfuse -uid $(id -u) -gid $(id -g) $flags $mountpoint dxfuse_test_data dxfuse_test_read_only ArchivedStuff + sleep 1 + + echo "faux dirs cannot be moved" + faux_dirs_move $mountpoint/$projName/$faux_dir + + echo "faux dir operations" + faux_dirs_remove $mountpoint/$projName/$faux_dir + + teardown +} diff --git a/test/local/file_overwrite.sh b/test/local/file_overwrite.sh new file mode 100644 index 0000000..a2b6dd4 --- /dev/null +++ b/test/local/file_overwrite.sh @@ -0,0 +1,128 @@ +###################################################################### +## constants + +projName="dxfuse_test_data" +dxfuse="$GOPATH/bin/dxfuse" +baseDir=$HOME/dxfuse_test +mountpoint=${baseDir}/MNT + +# Directories created during the test +writeable_dirs=() + +line1="K2 is the most dangerous mountain to climb in the Himalayas" +line2="One would also like to climb Kilimanjaro and the Everest" + +###################################################################### + +teardown_complete=0 + +# cleanup sequence +function teardown { + if [[ $teardown_complete == 1 ]]; then + return + fi + teardown_complete=1 + + rm -f cmd_results.txt + + echo "unmounting dxfuse" + cd $HOME + sudo umount $mountpoint + + for d in ${writeable_dirs[@]}; do + dx rm -r $projName:/$d >& /dev/null || true + done +} + +# trap any errors and cleanup +trap teardown EXIT + +###################################################################### + +# copy a file and check that platform has the correct content +# +function check_file_write_content { + local top_dir=$1 + local target_dir=$2 + local write_dir=$top_dir/$target_dir + + echo "write_dir = $write_dir" + + # create a small file through the filesystem interface + echo $line1 > $write_dir/A.txt + ls -l $write_dir/A.txt + + echo "synchronizing the filesystem" + sudo $dxfuse -sync + + echo "file is closed" + dx ls -l $projName:/$target_dir/A.txt + + # compare the data + local content=$(dx cat $projName:/$target_dir/A.txt) + if [[ "$content" == "$line1" ]]; then + echo "correct" + else + echo "bad content" + echo "should be: $line1" + echo "found: $content" + fi +} + +function check_overwrite { + local top_dir=$1 + local target_dir=$2 + local write_dir=$top_dir/$target_dir + + echo "write_dir = $write_dir" + + echo $line2 >> $write_dir/A.txt + + cat $write_dir/A.txt +} + +function file_overwrite { + # Get all the DX environment variables, so that dxfuse can use them + echo "loading the dx environment" + + # local machine + rm -f ENV + dx env --bash > ENV + source ENV >& /dev/null + rm -f ENV + + # clean and make fresh directories + mkdir -p $mountpoint + + # generate random alphanumeric strings + base_dir=$(cat /dev/urandom | env LC_CTYPE=C LC_ALL=C tr -dc 'a-zA-Z0-9' | fold -w 12 | head -n 1) + base_dir="base_$base_dir" + writeable_dirs=($base_dir) + for d in ${writeable_dirs[@]}; do + dx rm -r $projName:/$d >& /dev/null || true + done + + dx mkdir $projName:/$base_dir + + # Start the dxfuse daemon in the background, and wait for it to initilize. + echo "Mounting dxfuse" + flags="" + if [[ $verbose != "" ]]; then + flags="-verbose 2" + fi + sudo -E $dxfuse -uid $(id -u) -gid $(id -g) $flags $mountpoint dxfuse_test_data + sleep 1 + + echo "writing a small file" + check_file_write_content $mountpoint/$projName $base_dir + + sudo umount $mountpoint + + # now we are ready for an overwrite experiment + sudo -E $dxfuse -uid $(id -u) -gid $(id -g) $flags $mountpoint dxfuse_test_data + + echo "overwriting a file" + check_overwrite $mountpoint/$projName $base_dir + + teardown +} diff --git a/test/local/file_write_slow.sh b/test/local/file_write_slow.sh new file mode 100644 index 0000000..685e6a9 --- /dev/null +++ b/test/local/file_write_slow.sh @@ -0,0 +1,172 @@ +###################################################################### +## constants + +projName="dxfuse_test_data" +dxfuse="$GOPATH/bin/dxfuse" +baseDir=$HOME/dxfuse_test +mountpoint=${baseDir}/MNT + +# Directories created during the test +writeable_dirs=() +###################################################################### + +teardown_complete=0 + +# cleanup sequence +function teardown { + if [[ $teardown_complete == 1 ]]; then + return + fi + teardown_complete=1 + + rm -f cmd_results.txt + + echo "unmounting dxfuse" + cd $HOME + sudo umount $mountpoint + + for d in ${writeable_dirs[@]}; do + dx rm -r $projName:/$d >& /dev/null || true + done +} + +# trap any errors and cleanup +trap teardown EXIT + +###################################################################### + +# copy a file and check that platform has the correct content +# +function check_file_write_content { + local top_dir=$1 + local target_dir=$2 + local write_dir=$top_dir/$target_dir + local content="nothing much" + + echo "write_dir = $write_dir" + + # create a small file through the filesystem interface + echo $content > $write_dir/A.txt + ls -l $write_dir/A.txt + + echo "synchronizing the filesystem" + sudo $dxfuse -sync + + echo "file is closed" + dx ls -l $projName:/$target_dir/A.txt + + # compare the data + local content2=$(dx cat $projName:/$target_dir/A.txt) + if [[ "$content" == "$content2" ]]; then + echo "correct" + else + echo "bad content" + echo "should be: $content" + echo "found: $content2" + fi + + + # create an empty file + touch $write_dir/B.txt + ls -l $write_dir/B.txt + + echo "synchronizing the filesystem" + sudo $dxfuse -sync + + echo "file is closed" + dx ls -l $projName:/$target_dir/B.txt + + # compare the data + local content3=$(dx cat $projName:/$target_dir/B.txt) + if [[ "$content3" == "" ]]; then + echo "correct" + else + echo "bad content" + echo "should be empty" + echo "found: $content3" + fi +} + +# copy files inside the mounted filesystem +# +function write_files { + local src_dir=$1 + local write_dir=$2 + + echo "write_dir = $write_dir" + ls -l $write_dir + + echo "copying large files" + cp $src_dir/* $write_dir/ + + echo "synchronizing the filesystem" + sudo $dxfuse -sync + + # compare resulting files + echo "comparing files" + local files=$(find $src_dir -type f) + for f in $files; do + b_name=$(basename $f) + diff $f $write_dir/$b_name + done +} + +# check that we can't write to VIEW only project +# +function write_to_read_only_project { + ls $mountpoint/dxfuse_test_read_only + (echo "hello" > $mountpoint/dxfuse_test_read_only/A.txt) >& cmd_results.txt || true + local result=$(cat cmd_results.txt) + + if [[ $result =~ "Operation not permitted" ]]; then + echo "Correct, we should not be able to modify a project to which we have VIEW access" + else + echo "Incorrect, we managed to modify a project to which we have VIEW access" + exit 1 + fi +} + + +function file_write_slow { + # Get all the DX environment variables, so that dxfuse can use them + echo "loading the dx environment" + + # local machine + rm -f ENV + dx env --bash > ENV + source ENV >& /dev/null + rm -f ENV + + # clean and make fresh directories + mkdir -p $mountpoint + + # generate random alphanumeric strings + base_dir=$(cat /dev/urandom | env LC_CTYPE=C LC_ALL=C tr -dc 'a-zA-Z0-9' | fold -w 12 | head -n 1) + base_dir="base_$base_dir" + writeable_dirs=($base_dir) + for d in ${writeable_dirs[@]}; do + dx rm -r $projName:/$d >& /dev/null || true + done + + dx mkdir $projName:/$base_dir + + target_dir=$base_dir/T1 + dx mkdir $projName:/$target_dir + + # Start the dxfuse daemon in the background, and wait for it to initilize. + echo "Mounting dxfuse" + flags="" + if [[ $verbose != "" ]]; then + flags="-verbose 2" + fi + sudo -E $dxfuse -uid $(id -u) -gid $(id -g) $flags $mountpoint dxfuse_test_data + sleep 1 + + echo "can write to a small file" + check_file_write_content $mountpoint/$projName $target_dir + + echo "can write several large files to a directory" + write_files $mountpoint/$projName/large_files $mountpoint/$projName/$target_dir + + teardown +} diff --git a/test/local/fs_test_cases.sh b/test/local/fs_test_cases.sh index 2e483d6..f9f6b5b 100644 --- a/test/local/fs_test_cases.sh +++ b/test/local/fs_test_cases.sh @@ -2,21 +2,11 @@ ## constants projName="dxfuse_test_data" - -if [[ $DX_JOB_ID == "" ]]; then - # small test for a remote laptop - dxDirOnProject="mini" -else - # larger test for a cloud worker - dxDirOnProject="correctness" -fi - +dxfuse="$GOPATH/bin/dxfuse" +dxDirOnProject="mini" baseDir=$HOME/dxfuse_test mountpoint=${baseDir}/MNT -dxfuseDir=$mountpoint/$projName/$dxDirOnProject -dxpyDir=${baseDir}/dxCopy/$dxDirOnProject - # Directories created during the test writeable_dirs=() ###################################################################### @@ -46,91 +36,6 @@ trap teardown EXIT ###################################################################### -# copy a file and check that platform has the correct content -# -function check_file_write_content { - local top_dir=$1 - local target_dir=$2 - local write_dir=$top_dir/$target_dir - local content="nothing much" - - echo "write_dir = $write_dir" - - # create a small file through the filesystem interface - echo $content > $write_dir/A.txt - ls -l $write_dir/A.txt - - # wait for the file to achieve the closed state - while true; do - file_state=$(dx describe $projName:/$target_dir/A.txt --json | grep state | awk '{ gsub("[,\"]", "", $2); print $2 }') - if [[ "$file_state" == "closed" ]]; then - break - fi - sleep 2 - done - - echo "file is closed" - dx ls -l $projName:/$target_dir/A.txt - - # compare the data - local content2=$(dx cat $projName:/$target_dir/A.txt) - if [[ "$content" == "$content2" ]]; then - echo "correct" - else - echo "bad content" - echo "should be: $content" - echo "found: $content2" - fi - - - # create an empty file - touch $write_dir/B.txt - ls -l $write_dir/B.txt - - # wait for the file to achieve the closed state - while true; do - file_state=$(dx describe $projName:/$target_dir/B.txt --json | grep state | awk '{ gsub("[,\"]", "", $2); print $2 }') - if [[ "$file_state" == "closed" ]]; then - break - fi - sleep 2 - done - - echo "file is closed" - dx ls -l $projName:/$target_dir/B.txt - - # compare the data - local content3=$(dx cat $projName:/$target_dir/B.txt) - if [[ "$content3" == "" ]]; then - echo "correct" - else - echo "bad content" - echo "should be empty" - echo "found: $content3" - fi -} - -# copy files inside the mounted filesystem -# -function write_files { - local src_dir=$1 - local write_dir=$2 - - echo "write_dir = $write_dir" - ls -l $write_dir - - echo "copying large files" - cp $src_dir/* $write_dir/ - - # compare resulting files - echo "comparing files" - local files=$(find $src_dir -type f) - for f in $files; do - b_name=$(basename $f) - diff $f $write_dir/$b_name - done -} - # check that we can't write to VIEW only project # function write_to_read_only_project { @@ -203,8 +108,8 @@ function create_remove_dir { tree $write_dir if [[ $flag == "yes" ]]; then - echo "letting the files complete uploading" - sleep 10 + echo "synchronizing the filesystem" + $dxfuse -sync fi echo "removing directory recursively" @@ -274,6 +179,7 @@ function file_create_existing { cd $write_dir echo "happy days" > hello.txt + chmod 444 hello.txt set +e (echo "nothing much" > hello.txt) >& /tmp/cmd_results.txt @@ -459,68 +365,6 @@ function move_dir_to_file { rm -f Y.txt } -function faux_dirs_move { - local root_dir=$1 - cd $root_dir - - tree $root_dir - - # cannot move faux directories - set +e - (mv $root_dir/1 $root_dir/2 ) >& /dev/null - rc=$? - if [[ $rc == 0 ]]; then - echo "Error, could not a faux directory" - fi - - # cannot move files into a faux directory - (mv -f $root_dir/NewYork.txt $root_dir/1) >& /dev/null - rc=$? - if [[ $rc == 0 ]]; then - echo "Error, could move a file into a faux directory" - fi - set -e -} - -function faux_dirs_remove { - local root_dir=$1 - cd $root_dir - - # can move a file out of a faux directory - mkdir $root_dir/T - mv $root_dir/1/NewYork.txt $root_dir/T - rm -rf $root_dir/T - - echo "removing faux dir 1" - rm -rf $root_dir/1 -} - - - -function hard_links { - local root_dir=$1 - cd $root_dir - - ln $mountpoint/dxfuse_test_read_only/doc/approaches.md . - stat approaches.md - rm -f approaches.md -} - - -function populate_faux_dir { - local faux_dir=$1 - - echo "deep dish pizza and sky trains" > /tmp/XXX - echo "nice play chunk" > /tmp/YYY - echo "no more chewing on shoes!" > /tmp/ZZZ - echo "you just won a trip to the Caribbean" > /tmp/VVV - - dx upload /tmp/XXX -p --destination $projName:/$faux_dir/Chicago.txt >& /dev/null - dx upload /tmp/YYY -p --destination $projName:/$faux_dir/Chicago.txt >& /dev/null - dx upload /tmp/ZZZ -p --destination $projName:/$faux_dir/NewYork.txt >& /dev/null - dx upload /tmp/VVV -p --destination $projName:/$faux_dir/NewYork.txt >& /dev/null - rm -f /tmp/XXX /tmp/YYY /tmp/ZZZ /tmp/VVV -} function dir_and_file_with_the_same_name { local root_dir=$1 @@ -547,7 +391,6 @@ function fs_test_cases { dx env --bash > ENV source ENV >& /dev/null rm -f ENV - dxfuse="$GOPATH/bin/dxfuse" # clean and make fresh directories mkdir -p $mountpoint @@ -555,22 +398,16 @@ function fs_test_cases { # generate random alphanumeric strings base_dir=$(cat /dev/urandom | env LC_CTYPE=C LC_ALL=C tr -dc 'a-zA-Z0-9' | fold -w 12 | head -n 1) base_dir="base_$base_dir" - faux_dir=$(cat /dev/urandom | env LC_CTYPE=C LC_ALL=C tr -dc 'a-zA-Z0-9' | fold -w 12 | head -n 1) - faux_dir="faux_$faux_dir" expr_dir=$(cat /dev/urandom | env LC_CTYPE=C LC_ALL=C tr -dc 'a-zA-Z0-9' | fold -w 12 | head -n 1) expr_dir="expr_$expr_dir" - writeable_dirs=($base_dir $faux_dir $expr_dir) + writeable_dirs=($base_dir $expr_dir) for d in ${writeable_dirs[@]}; do dx rm -r $projName:/$d >& /dev/null || true done dx mkdir $projName:/$base_dir - populate_faux_dir $faux_dir dx mkdir $projName:/$expr_dir - target_dir=$base_dir/T1 - dx mkdir $projName:/$target_dir - # Start the dxfuse daemon in the background, and wait for it to initilize. echo "Mounting dxfuse" flags="" @@ -580,12 +417,6 @@ function fs_test_cases { sudo -E $dxfuse -uid $(id -u) -gid $(id -g) $flags $mountpoint dxfuse_test_data dxfuse_test_read_only ArchivedStuff sleep 1 - echo "can write to a small file" - check_file_write_content $mountpoint/$projName $target_dir - -# echo "can write several files to a directory" -# write_files $mountpoint/$projName/$dxDirOnProject/large $mountpoint/$projName/$target_dir - echo "can't write to read-only project" write_to_read_only_project @@ -637,15 +468,6 @@ function fs_test_cases { move_non_existent_dir "$mountpoint/$projName" move_dir_to_file "$mountpoint/$projName" - echo "faux dirs cannot be moved" - faux_dirs_move $mountpoint/$projName/$faux_dir - - echo "faux dir operations" - faux_dirs_remove $mountpoint/$projName/$faux_dir - - echo "hard links" - hard_links $mountpoint/$projName/$faux_dir - echo "directory and file with the same name" dir_and_file_with_the_same_name $mountpoint/$projName diff --git a/test/local/local.sh b/test/local/local.sh index a139d1f..8bcbf5e 100755 --- a/test/local/local.sh +++ b/test/local/local.sh @@ -9,5 +9,17 @@ xattr_test source $CRNT_DIR/manifest_test.sh manifest_test +source $CRNT_DIR/dx_download_compare.sh +dx_download_compare + +source $CRNT_DIR/file_write_slow.sh +file_write_slow + source $CRNT_DIR/fs_test_cases.sh fs_test_cases + +source $CRNT_DIR/faux_dirs.sh +faux_dirs + +source $CRNT_DIR/file_overwrite.sh +file_overwrite diff --git a/test/local/manifest_test.sh b/test/local/manifest_test.sh index 2d753b0..ca4d9f7 100644 --- a/test/local/manifest_test.sh +++ b/test/local/manifest_test.sh @@ -7,7 +7,7 @@ function manifest_test { mkdir -p $mountpoint - sudo -E $dxfuse -verbose 2 -uid $(id -u) -gid $(id -g) $mountpoint $CRNT_DIR/two_files.json + sudo -E $dxfuse -uid $(id -u) -gid $(id -g) $mountpoint $CRNT_DIR/two_files.json sleep 1 tree $mountpoint diff --git a/test/local/xattr_test.sh b/test/local/xattr_test.sh index 7092e8e..8b85530 100644 --- a/test/local/xattr_test.sh +++ b/test/local/xattr_test.sh @@ -30,11 +30,46 @@ trap teardown EXIT ###################################################################### +# wait for the file to achieve the closed state +function close_file { + local path=$1 + while true; do + file_state=$(dx describe $path --json | grep state | awk '{ gsub("[,\"]", "", $2); print $2 }') + if [[ "$file_state" == "closed" ]]; then + break + fi + sleep 1 + done +} + +function setup { + local base_dir=$1 + + # bat.txt + local f=$projName:/$base_dir/bat.txt + echo "flying mammal" | dx upload --destination $f - + dx set_properties $f fly=yes family=mammal eat=omnivore + close_file $f + + # whale.txt + local f=$projName:/$base_dir/whale.txt + echo "The largest mammal on earth" | dx upload --destination $f - + close_file $f + + # Mountains.txt + local f=$projName:/$base_dir/Mountains.txt + echo "K2, Kilimanjaro, Everest, Mckinly" | dx upload --destination $f - + close_file $f + + dx ls -l $projName:/$base_dir +} + function check_bat { local base_dir=$1 + local test_dir=$mountpoint/$projName/$base_dir # Get a list of all the attributes - local bat_all_attrs=$(xattr $base_dir/bat.txt | sort | tr '\n' ' ') + local bat_all_attrs=$(xattr $test_dir/bat.txt | sort | tr '\n' ' ') local bat_all_expected="base.archivalState base.id base.state prop.eat prop.family prop.fly " if [[ $bat_all_attrs != $bat_all_expected ]]; then echo "bat attributes are incorrect" @@ -43,7 +78,7 @@ function check_bat { exit 1 fi - local bat_family=$(xattr -p prop.family $base_dir/bat.txt) + local bat_family=$(xattr -p prop.family $test_dir/bat.txt) local bat_family_expected="mammal" if [[ $bat_family != $bat_family_expected ]]; then echo "bat family is wrong" @@ -53,14 +88,15 @@ function check_bat { fi - xattr -w prop.family carnivore $base_dir/bat.txt - xattr -w prop.family mammal $base_dir/bat.txt + xattr -w prop.family carnivore $test_dir/bat.txt + xattr -w prop.family mammal $test_dir/bat.txt } function check_whale { local base_dir=$1 + local test_dir=$mountpoint/$projName/$base_dir - local whale_all_attrs=$(xattr $base_dir/whale.txt | sort | tr '\n' ' ') + local whale_all_attrs=$(xattr $test_dir/whale.txt | sort | tr '\n' ' ') local whale_all_expected="base.archivalState base.id base.state " if [[ $whale_all_attrs != $whale_all_expected ]]; then echo "whale attributes are incorrect" @@ -72,7 +108,9 @@ function check_whale { function check_new { local base_dir=$1 - local f=$base_dir/Mountains.txt + local test_dir=$mountpoint/$projName/$base_dir + local f=$test_dir/Mountains.txt + local dnaxF=$projName:/$base_dir/Mountains.txt xattr -w prop.family geography $f xattr -w tag.high X $f @@ -95,6 +133,29 @@ function check_new { exit 1 fi + echo "synchronizing filesystem" + $dxfuse -sync + + props=$(dx describe $dnaxF --json | jq -cMS .properties | tr '[]' ' ') + echo "props on platform: $props" + local props_expected='{"family":"geography"}' + if [[ $props != $props_expected ]]; then + echo "$f properties mismatch" + echo " got: $props" + echo " expecting: $props_expected" + exit 1 + fi + + tags=$(dx describe $dnaxF --json | jq -cMS .tags | tr '[]' ' ') + echo "tags on platform: $tags" + local tags_expected=' "high" ' + if [[ $tags != $tags_expected ]]; then + echo "$f tags mismatch" + echo " got: $tags" + echo " expecting: $tags_expected" + exit 1 + fi + xattr -d prop.family $f xattr -d tag.high $f xattr $f @@ -109,16 +170,41 @@ function check_new { fi } + function xattr_test { + # Get all the DX environment variables, so that dxfuse can use them + echo "loading the dx environment" + + # local machine + rm -f ENV + dx env --bash > ENV + source ENV >& /dev/null + rm -f ENV + + # clean and make fresh directories mkdir -p $mountpoint - sudo -E $dxfuse -verbose 2 -uid $(id -u) -gid $(id -g) $mountpoint $projName + # generate random alphanumeric strings + base_dir=$(cat /dev/urandom | env LC_CTYPE=C LC_ALL=C tr -dc 'a-zA-Z0-9' | fold -w 12 | head -n 1) + base_dir="base_$base_dir" + writeable_dirs=($base_dir) + for d in ${writeable_dirs[@]}; do + dx rm -r $projName:/$d >& /dev/null || true + done + dx mkdir $projName:/$base_dir + + setup $base_dir - # This seems to be needed on MacOS + # Start the dxfuse daemon in the background, and wait for it to initilize. + echo "Mounting dxfuse" + flags="" + if [[ $verbose != "" ]]; then + flags="-verbose 2" + fi + sudo -E $dxfuse -uid $(id -u) -gid $(id -g) $flags $mountpoint $projName sleep 1 - local base_dir=$mountpoint/$projName/xattrs - tree $base_dir + tree $mountpoint/$projName/$base_dir check_bat $base_dir check_whale $base_dir diff --git a/test/run_tests.py b/test/run_tests.py index ed9f5ae..cf7097d 100755 --- a/test/run_tests.py +++ b/test/run_tests.py @@ -128,25 +128,24 @@ def run_local_test(): sys.exit(1) def run_correctness(dx_proj, itype, verbose): - run_local_test() correctness = lookup_applet("correctness", dx_proj, "/applets") - bam_diff = lookup_applet("bam_diff", dx_proj, "/applets") + bio_tools = lookup_applet("bio_tools", dx_proj, "/applets") correctness_downloads = lookup_applet("correctness_downloads", dx_proj, "/applets") jobs1 = launch_jobs(dx_proj, correctness, [itype], verbose) - jobs2 = launch_jobs(dx_proj, bam_diff, [itype], verbose) + jobs2 = launch_jobs(dx_proj, bio_tools, [itype], verbose) jobs3 = launch_jobs(dx_proj, correctness_downloads, [itype], verbose) wait_for_completion(jobs1 + jobs2 + jobs3) def run_biotools(dx_proj, itype, verbose): - bam_diff = lookup_applet("bam_diff", dx_proj, "/applets") - jobs2 = launch_jobs(dx_proj, bam_diff, [itype], verbose) + bio_tools = lookup_applet("bio_tools", dx_proj, "/applets") + jobs = launch_jobs(dx_proj, bio_tools, [itype], verbose) wait_for_completion(jobs) def main(): argparser = argparse.ArgumentParser(description="Run benchmarks on several instance types for dxfuse") argparser.add_argument("--project", help="DNAnexus project", default="dxfuse_test_data") - argparser.add_argument("--test", help="which testing suite to run [bench, correct, bio]", + argparser.add_argument("--test", help="which testing suite to run [bench, bio, correct, local]", default="correctness") argparser.add_argument("--size", help="how large should the test be? [small, large]", default="small") @@ -179,6 +178,8 @@ def main(): run_correctness(dx_proj, instance_types[0], args.verbose) elif args.test.startswith("bio"): run_biotools(dx_proj, instance_types[0], args.verbose) + elif args.test.startswith("local"): + run_local_test() else: print("Unknown test {}".format(args.test)) exit(1) diff --git a/util.go b/util.go index 89d4577..13b0968 100644 --- a/util.go +++ b/util.go @@ -5,34 +5,33 @@ import ( "fmt" "log" "os" - "sync" "time" - "github.com/dnanexus/dxda" "github.com/hashicorp/go-retryablehttp" - "github.com/jacobsa/fuse/fuseutil" "github.com/jacobsa/fuse/fuseops" ) const ( - CreatedFilesDir = "/var/dxfuse/created_files" - DatabaseFile = "/var/dxfuse/metadata.db" - HttpClientPoolSize = 4 - LogFile = "/var/log/dxfuse.log" - MaxDirSize = 10 * 1000 - MaxNumFileHandles = 1000 * 1000 - NumRetriesDefault = 3 - Version = "v0.19" + KiB = 1024 + MiB = 1024 * KiB + GiB = 1024 * MiB ) const ( - InodeInvalid = 0 - InodeRoot = fuseops.RootInodeID // This is an OS constant + CreatedFilesDir = "/var/dxfuse/created_files" + DatabaseFile = "/var/dxfuse/metadata.db" + HttpClientPoolSize = 4 + FileWriteInactivityThresh = 5 * time.Minute + WritableFileSizeLimit = 16 * MiB + LogFile = "/var/log/dxfuse.log" + MaxDirSize = 10 * 1000 + MaxNumFileHandles = 1000 * 1000 + NumRetriesDefault = 3 + Version = "v0.19" ) const ( - KiB = 1024 - MiB = 1024 * KiB - GiB = 1024 * MiB + InodeInvalid = 0 + InodeRoot = fuseops.RootInodeID // This is an OS constant ) const ( // It turns out that in order for regular users to be able to create file, @@ -42,6 +41,19 @@ const ( fileReadOnlyMode = 0444 fileReadWriteMode = 0644 ) +const ( + // flags for writing files to disk + DIRTY_FILES_ALL = 14 // all modified files + DIRTY_FILES_INACTIVE = 15 // only files there were unmodified recently +) + +const ( + // Permissions + PERM_VIEW = 1 + PERM_UPLOAD = 2 + PERM_CONTRIBUTE = 3 + PERM_ADMINISTER = 4 +) // A URL generated with the /file-xxxx/download API call, that is // used to download file ranges. @@ -59,55 +71,6 @@ type Options struct { } -type Filesys struct { - // inherit empty implementations for all the filesystem - // methods we do not implement - fuseutil.NotImplementedFileSystem - - // configuration information for accessing dnanexus servers - dxEnv dxda.DXEnvironment - - // various options - options Options - - // A file holding a sqlite3 database with all the files and - // directories collected thus far. - dbFullPath string - - // Lock for protecting shared access to the database - mutex sync.Mutex - - // a pool of http clients, for short requests, such as file creation, - // or file describe. - httpClientPool chan(*retryablehttp.Client) - - // metadata database - mdb *MetadataDb - - // prefetch state for all files - pgs *PrefetchGlobalState - - // background upload state - fugs *FileUploadGlobalState - - // API to dx - ops *DxOps - - // all open files - fhCounter uint64 - fhTable map[fuseops.HandleID]*FileHandle - - // all open directories - dhCounter uint64 - dhTable map[fuseops.HandleID]*DirHandle - - nonce *Nonce - tmpFileCounter uint64 - - // is the the system shutting down (unmounting) - shutdownCalled bool -} - // A node is a generalization over files and directories type Node interface { GetInode() fuseops.InodeID @@ -166,6 +129,9 @@ const ( // A Unix file can stand for any DNAx data object. For example, it could be a workflow or an applet. // We distinguish between them based on the Id (file-xxxx, applet-xxxx, workflow-xxxx, ...). +// +// Note: this struct is immutable by convention. The up-to-date data is always on the database, +// not in memory. type File struct { Kind int // Kind of object this is Id string // Required to build a download URL @@ -185,7 +151,6 @@ type File struct { Ctime time.Time Mtime time.Time Mode os.FileMode // uint32 - Nlink int Uid uint32 Gid uint32 @@ -194,13 +159,19 @@ type File struct { Properties map[string]string // for a symlink, it holds the path. + Symlink string + // For a regular file, a path to a local copy (if any). - InlineData string + LocalPath string + + // is the file modified + dirtyData bool + dirtyMetadata bool } func (f File) GetAttrs() (a fuseops.InodeAttributes) { a.Size = uint64(f.Size) - a.Nlink = uint32(f.Nlink) + a.Nlink = 1 a.Mtime = f.Mtime a.Ctime = f.Ctime a.Mode = f.Mode @@ -214,34 +185,37 @@ func (f File) GetInode() fuseops.InodeID { return fuseops.InodeID(f.Inode) } - -// Files can be opened in read-only mode, or read-write mode. -const ( - RO_Remote = 1 // read only file that is on the cloud - RW_File = 2 // newly created file - RO_LocalCopy = 3 // read only file that has a local copy -) - -type FileHandle struct { - fKind int - f File - hid fuseops.HandleID - - // URL used for downloading file ranges. - // Used for read-only files. - url *DxDownloadURL - - // Local file copy, may be empty. - localPath *string - - // 1. Used for reading from an immutable local copy - // 2. Used for writing to newly created files. - fd *os.File +// A file that is scheduled for removal +type DeadFile struct { + Kind int // Kind of object this is + Id string // Required to build a download URL + ProjId string // Note: this could be a container + Inode int64 + LocalPath string } -type DirHandle struct { - d Dir - entries []fuseutil.Dirent +// Information required to upload file data to the platform. +// It also includes updated tags and properties of a data-object. +// +// Not that not only files have attributes, applets and workflows +// have them too. +// +type DirtyFileInfo struct { + Inode int64 + dirtyData bool + dirtyMetadata bool + + // will be "" for files created locally, and not uploaded yet + Id string + FileSize int64 + Mtime int64 + LocalPath string + Tags []string + Properties map[string]string + Name string + Directory string + ProjFolder string + ProjId string } // A handle used when operating on a filesystem @@ -328,3 +302,24 @@ func BytesToString(numBytes int64) string { } return fmt.Sprintf("%d%s", digits[msd], byteModifier[msd]) } + +func check(value bool) { + if !value { + log.Panicf("assertion failed") + os.Exit(1) + } +} + +func boolToInt(b bool) int { + if b { + return 1 + } + return 0 +} + +func intToBool(x int) bool { + if x > 0 { + return true + } + return false +}