diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index c236467..0bfb72d 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -99,11 +99,12 @@ ignore_release_directories = [ ".stversions" ] # `${XDG_CACHE_HOME:-$HOME/.cache}/rose`. cache_dir = "~/.cache/rose" -# Maximum parallel processes that the read cache updater can spawn. Defaults to -# $(nproc)/2. The higher this number is; the more performant the cache update -# will be. +# Maximum parallel processes that Rose can spawn. Defaults to # $(nproc)/2. +# +# Rose uses this value to limit the max parallelization of read cache updates +# and the number of works that the virtual filesystem can spin up to handle a +# request. max_proc = 4 - ``` # Systemd diff --git a/rose/virtualfs.py b/rose/virtualfs.py index 2834b56..91c4cb3 100644 --- a/rose/virtualfs.py +++ b/rose/virtualfs.py @@ -639,7 +639,7 @@ def mkdir(self, path: Path) -> None: return except ReleaseDoesNotExistError as e: logger.debug( - f"Failed adding release {p.release} to collage {p.collage}: release not found." + f"Failed adding release {p.release} to collage {p.collage}: release not found" ) raise llfuse.FUSEError(errno.ENOENT) from e if p.playlist and p.file is None: @@ -766,7 +766,7 @@ def read(self, fh: int, offset: int, length: int) -> bytes: def write(self, fh: int, offset: int, data: bytes) -> int: logger.debug(f"LOGICAL: Received write for {fh=} {offset=} {len(data)=}") if pap := self.playlist_additions_in_progress.get(fh, None): - logger.debug("Matched write to an in-progress playlist addition.") + logger.debug("Matched write to an in-progress playlist addition") _, _, b = pap del b[offset:] b.extend(data) @@ -776,10 +776,10 @@ def write(self, fh: int, offset: int, data: bytes) -> int: def release(self, fh: int) -> None: if pap := self.playlist_additions_in_progress.get(fh, None): - logger.debug("Matched release to an in-progress playlist addition.") + logger.debug("Matched release to an in-progress playlist addition") playlist, ext, b = pap if not b: - logger.debug("Aborting playlist addition release: no bytes to write.") + logger.debug("Aborting playlist addition release: no bytes to write") return with tempfile.TemporaryDirectory() as tmpdir: audiopath = Path(tmpdir) / f"f{ext}" @@ -899,8 +899,8 @@ def __init__(self, config: Config): # Well, this should be ok for now. I really don't want to track this... we indeed change # inodes across FS restarts. "generation": random.randint(0, 1000000), - # Have a 15 second metadata timeout by default. - "entry_timeout": 15, + # Have a 30 second entry timeout by default. + "entry_timeout": 30, } # We cache some items for getattr and lookup for performance reasons--after a ls, getattr is # serially called for each item in the directory, and sequential 1k SQLite reads is quite @@ -912,7 +912,6 @@ def __init__(self, config: Config): self.getattr_cache: cachetools.TTLCache[int, llfuse.EntryAttributes] self.lookup_cache: cachetools.TTLCache[tuple[int, bytes], llfuse.EntryAttributes] self.reset_getattr_caches() - # We handle state for readdir calls here. Because programs invoke readdir multiple times # with offsets, we end up with many readdir calls for a single directory. However, we do not # want to actually invoke the logical Rose readdir call that many times. So we load it once @@ -921,6 +920,22 @@ def __init__(self, config: Config): # # Map of file handle -> (parent inode, child name, child attributes). self.readdir_cache: dict[int, list[tuple[int, bytes, llfuse.EntryAttributes]]] = {} + # Ghost Files: We pretend some files exist in the filesystem, despite them not actually + # existing. We do this in order to be compatible with the expectations that tools have for + # filesystems. + # + # For example, when we use file writing to add a file to a playlist, that file is + # immediately renamed to its correct playlist-specific filename upon release. However, `cp` + # exits with an error, for it followed up the release with an attempt to set file + # permissions and attributes on a now non-existent file. + # + # In order to pretend to tools that we are a Real Filesystem and not some shitty hack of a + # filesystem, we have these ghost files that exist for a period of time following an + # operation. + # + # We actually want a set of inodes; however, cachetools only has dicts. So we pretend to be + # Rob Pike and implement sets as mappings of inode -> True. + self.ghost_files: cachetools.TTLCache[int, bool] = cachetools.TTLCache(maxsize=9999, ttl=3) def reset_getattr_caches(self) -> None: # When a write happens, clear these caches. These caches are very short-lived and intended @@ -942,9 +957,16 @@ def getattr(self, inode: int, _: Any) -> llfuse.EntryAttributes: # For performance, pull from the getattr cache if possible. with contextlib.suppress(KeyError): attrs = self.getattr_cache[inode] - logger.debug(f"FUSE: Resolved getattr for {inode=} to {attrs.__getstate__()=}.") + logger.debug(f"FUSE: Resolved getattr for {inode=} to {attrs.__getstate__()=}") return attrs + # If this path is a ghost file path; pretend here! + if self.ghost_files.get(inode, False): + logger.debug(f"FUSE: Resolved getattr for {inode=} as ghost file") + attrs = self.rose.stat("file") + attrs["st_ino"] = inode + return self.make_entry_attributes(attrs) + path = self.inodes.get_path(inode) logger.debug(f"FUSE: Resolved getattr {inode=} to {path=}") attrs = self.rose.getattr(path) @@ -957,7 +979,7 @@ def lookup(self, parent_inode: int, name: bytes, _: Any) -> llfuse.EntryAttribut with contextlib.suppress(KeyError): attrs = self.lookup_cache[(parent_inode, name)] logger.debug( - f"FUSE: Resolved lookup for {parent_inode=}/{name=} to {attrs.__getstate__()=}." + f"FUSE: Resolved lookup for {parent_inode=}/{name=} to {attrs.__getstate__()=}" ) return attrs @@ -1006,7 +1028,12 @@ def open(self, inode: int, flags: int, _: Any) -> int: logger.debug(f"FUSE: Received open for {inode=} {flags=}") path = self.inodes.get_path(inode) logger.debug(f"FUSE: Resolved open for {inode=} to {path=}") - return self.rose.open(path, flags) + fh = self.rose.open(path, flags) + # If this was a create operation, and Rose succeeded, flag the filepath as a ghost file and + # _always_ pretend it exists for the following short duration. + if flags & os.O_CREAT == os.O_CREAT: + self.ghost_files[inode] = True + return fh def read(self, fh: int, offset: int, length: int) -> bytes: logger.debug(f"FUSE: Received read for {fh=} {offset=} {length=}") @@ -1039,9 +1066,7 @@ def create( inode = self.inodes.calc_inode(path) logger.debug(f"FUSE: Created inode {inode=} for {path=}; now delegating to open call") fh = self.open(inode, flags, ctx) - # Avoid zombies coming back from an old cache. self.reset_getattr_caches() - attrs = self.rose.stat("file") attrs["st_ino"] = inode return fh, self.make_entry_attributes(attrs) @@ -1051,7 +1076,6 @@ def unlink(self, parent_inode: int, name: bytes, _: Any) -> None: path = self.inodes.get_path(parent_inode, name) logger.debug(f"FUSE: Resolved unlink for {parent_inode=}/{name=} to {path=}") self.rose.unlink(path) - # Avoid zombies coming back from an old cache. self.reset_getattr_caches() self.inodes.remove_path(path) @@ -1060,7 +1084,6 @@ def mkdir(self, parent_inode: int, name: bytes, _mode: int, _: Any) -> llfuse.En path = self.inodes.get_path(parent_inode, name) logger.debug(f"FUSE: Resolved mkdir for {parent_inode=}/{name=} to {path=}") self.rose.mkdir(path) - # Avoid zombies coming back from an old cache. self.reset_getattr_caches() attrs = self.rose.stat("dir") attrs["st_ino"] = self.inodes.calc_inode(path) @@ -1071,7 +1094,6 @@ def rmdir(self, parent_inode: int, name: bytes, _: Any) -> None: path = self.inodes.get_path(parent_inode, name) logger.debug(f"FUSE: Resolved rmdir for {parent_inode=}/{name=} to {path=}") self.rose.rmdir(path) - # Avoid zombies coming back from an old cache. self.reset_getattr_caches() self.inodes.remove_path(path) @@ -1151,7 +1173,7 @@ def mount_virtualfs(c: Config, debug: bool = False) -> None: options.add("debug") llfuse.init(VirtualFS(c), str(c.fuse_mount_dir), options) try: - llfuse.main() + llfuse.main(workers=c.max_proc) except: llfuse.close() raise diff --git a/rose/virtualfs_test.py b/rose/virtualfs_test.py index 83f8ea6..5983da6 100644 --- a/rose/virtualfs_test.py +++ b/rose/virtualfs_test.py @@ -169,7 +169,19 @@ def test_virtual_filesystem_playlist_actions( assert (src / "!playlists" / "New Jeans.toml").is_file() assert not (src / "!playlists" / "New Tee.toml").exists() # Add track to playlist. - shutil.copyfile(release_dir / filename, root / "8. Playlists" / "New Jeans" / filename) + # Use `cp -p` to test the ghost files behavior. A pure copy file will succeed, because it + # stops after the release. However, cp -p also attempts to set some attributes on the moved + # file, which fails if we immediately vanish the file post-release, which the naive + # implementation does. + subprocess.run( + [ + "cp", + "-p", + str(release_dir / filename), + str(root / "8. Playlists" / "New Jeans" / filename), + ], + check=True, + ) assert (root / "8. Playlists" / "New Jeans" / "1. BLACKPINK - Track 1.m4a").is_file() with (src / "!playlists" / "New Jeans.toml").open("r") as fp: assert "BLACKPINK - Track 1.m4a" in fp.read()