Skip to content

Commit

Permalink
add basic ghost files to make cp -p work
Browse files Browse the repository at this point in the history
  • Loading branch information
@azuline
azuline committed Oct 29, 2023
1 parent 9a06335 commit 74b291b
Show file tree
Hide file tree
Showing 3 changed files with 56 additions and 21 deletions.
9 changes: 5 additions & 4 deletions docs/CONFIGURATION.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,11 +99,12 @@ ignore_release_directories = [ ".stversions" ]
# `${XDG_CACHE_HOME:-$HOME/.cache}/rose`.
cache_dir = "~/.cache/rose"

# Maximum parallel processes that the read cache updater can spawn. Defaults to
# $(nproc)/2. The higher this number is; the more performant the cache update
# will be.
# Maximum parallel processes that Rose can spawn. Defaults to # $(nproc)/2.
#
# Rose uses this value to limit the max parallelization of read cache updates
# and the number of works that the virtual filesystem can spin up to handle a
# request.
max_proc = 4

```

# Systemd
Expand Down
54 changes: 38 additions & 16 deletions rose/virtualfs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -639,7 +639,7 @@ def mkdir(self, path: Path) -> None:
return
except ReleaseDoesNotExistError as e:
logger.debug(
f"Failed adding release {p.release} to collage {p.collage}: release not found."
f"Failed adding release {p.release} to collage {p.collage}: release not found"
)
raise llfuse.FUSEError(errno.ENOENT) from e
if p.playlist and p.file is None:
Expand Down Expand Up @@ -766,7 +766,7 @@ def read(self, fh: int, offset: int, length: int) -> bytes:
def write(self, fh: int, offset: int, data: bytes) -> int:
logger.debug(f"LOGICAL: Received write for {fh=} {offset=} {len(data)=}")
if pap := self.playlist_additions_in_progress.get(fh, None):
logger.debug("Matched write to an in-progress playlist addition.")
logger.debug("Matched write to an in-progress playlist addition")
_, _, b = pap
del b[offset:]
b.extend(data)
Expand All @@ -776,10 +776,10 @@ def write(self, fh: int, offset: int, data: bytes) -> int:

def release(self, fh: int) -> None:
if pap := self.playlist_additions_in_progress.get(fh, None):
logger.debug("Matched release to an in-progress playlist addition.")
logger.debug("Matched release to an in-progress playlist addition")
playlist, ext, b = pap
if not b:
logger.debug("Aborting playlist addition release: no bytes to write.")
logger.debug("Aborting playlist addition release: no bytes to write")
return
with tempfile.TemporaryDirectory() as tmpdir:
audiopath = Path(tmpdir) / f"f{ext}"
Expand Down Expand Up @@ -899,8 +899,8 @@ def __init__(self, config: Config):
# Well, this should be ok for now. I really don't want to track this... we indeed change
# inodes across FS restarts.
"generation": random.randint(0, 1000000),
# Have a 15 second metadata timeout by default.
"entry_timeout": 15,
# Have a 30 second entry timeout by default.
"entry_timeout": 30,
}
# We cache some items for getattr and lookup for performance reasons--after a ls, getattr is
# serially called for each item in the directory, and sequential 1k SQLite reads is quite
Expand All @@ -912,7 +912,6 @@ def __init__(self, config: Config):
self.getattr_cache: cachetools.TTLCache[int, llfuse.EntryAttributes]
self.lookup_cache: cachetools.TTLCache[tuple[int, bytes], llfuse.EntryAttributes]
self.reset_getattr_caches()

# We handle state for readdir calls here. Because programs invoke readdir multiple times
# with offsets, we end up with many readdir calls for a single directory. However, we do not
# want to actually invoke the logical Rose readdir call that many times. So we load it once
Expand All @@ -921,6 +920,22 @@ def __init__(self, config: Config):
#
# Map of file handle -> (parent inode, child name, child attributes).
self.readdir_cache: dict[int, list[tuple[int, bytes, llfuse.EntryAttributes]]] = {}
# Ghost Files: We pretend some files exist in the filesystem, despite them not actually
# existing. We do this in order to be compatible with the expectations that tools have for
# filesystems.
#
# For example, when we use file writing to add a file to a playlist, that file is
# immediately renamed to its correct playlist-specific filename upon release. However, `cp`
# exits with an error, for it followed up the release with an attempt to set file
# permissions and attributes on a now non-existent file.
#
# In order to pretend to tools that we are a Real Filesystem and not some shitty hack of a
# filesystem, we have these ghost files that exist for a period of time following an
# operation.
#
# We actually want a set of inodes; however, cachetools only has dicts. So we pretend to be
# Rob Pike and implement sets as mappings of inode -> True.
self.ghost_files: cachetools.TTLCache[int, bool] = cachetools.TTLCache(maxsize=9999, ttl=3)

def reset_getattr_caches(self) -> None:
# When a write happens, clear these caches. These caches are very short-lived and intended
Expand All @@ -942,9 +957,16 @@ def getattr(self, inode: int, _: Any) -> llfuse.EntryAttributes:
# For performance, pull from the getattr cache if possible.
with contextlib.suppress(KeyError):
attrs = self.getattr_cache[inode]
logger.debug(f"FUSE: Resolved getattr for {inode=} to {attrs.__getstate__()=}.")
logger.debug(f"FUSE: Resolved getattr for {inode=} to {attrs.__getstate__()=}")
return attrs

# If this path is a ghost file path; pretend here!
if self.ghost_files.get(inode, False):
logger.debug(f"FUSE: Resolved getattr for {inode=} as ghost file")
attrs = self.rose.stat("file")
attrs["st_ino"] = inode
return self.make_entry_attributes(attrs)

path = self.inodes.get_path(inode)
logger.debug(f"FUSE: Resolved getattr {inode=} to {path=}")
attrs = self.rose.getattr(path)
Expand All @@ -957,7 +979,7 @@ def lookup(self, parent_inode: int, name: bytes, _: Any) -> llfuse.EntryAttribut
with contextlib.suppress(KeyError):
attrs = self.lookup_cache[(parent_inode, name)]
logger.debug(
f"FUSE: Resolved lookup for {parent_inode=}/{name=} to {attrs.__getstate__()=}."
f"FUSE: Resolved lookup for {parent_inode=}/{name=} to {attrs.__getstate__()=}"
)
return attrs

Expand Down Expand Up @@ -1006,7 +1028,12 @@ def open(self, inode: int, flags: int, _: Any) -> int:
logger.debug(f"FUSE: Received open for {inode=} {flags=}")
path = self.inodes.get_path(inode)
logger.debug(f"FUSE: Resolved open for {inode=} to {path=}")
return self.rose.open(path, flags)
fh = self.rose.open(path, flags)
# If this was a create operation, and Rose succeeded, flag the filepath as a ghost file and
# _always_ pretend it exists for the following short duration.
if flags & os.O_CREAT == os.O_CREAT:
self.ghost_files[inode] = True
return fh

def read(self, fh: int, offset: int, length: int) -> bytes:
logger.debug(f"FUSE: Received read for {fh=} {offset=} {length=}")
Expand Down Expand Up @@ -1039,9 +1066,7 @@ def create(
inode = self.inodes.calc_inode(path)
logger.debug(f"FUSE: Created inode {inode=} for {path=}; now delegating to open call")
fh = self.open(inode, flags, ctx)
# Avoid zombies coming back from an old cache.
self.reset_getattr_caches()

attrs = self.rose.stat("file")
attrs["st_ino"] = inode
return fh, self.make_entry_attributes(attrs)
Expand All @@ -1051,7 +1076,6 @@ def unlink(self, parent_inode: int, name: bytes, _: Any) -> None:
path = self.inodes.get_path(parent_inode, name)
logger.debug(f"FUSE: Resolved unlink for {parent_inode=}/{name=} to {path=}")
self.rose.unlink(path)
# Avoid zombies coming back from an old cache.
self.reset_getattr_caches()
self.inodes.remove_path(path)

Expand All @@ -1060,7 +1084,6 @@ def mkdir(self, parent_inode: int, name: bytes, _mode: int, _: Any) -> llfuse.En
path = self.inodes.get_path(parent_inode, name)
logger.debug(f"FUSE: Resolved mkdir for {parent_inode=}/{name=} to {path=}")
self.rose.mkdir(path)
# Avoid zombies coming back from an old cache.
self.reset_getattr_caches()
attrs = self.rose.stat("dir")
attrs["st_ino"] = self.inodes.calc_inode(path)
Expand All @@ -1071,7 +1094,6 @@ def rmdir(self, parent_inode: int, name: bytes, _: Any) -> None:
path = self.inodes.get_path(parent_inode, name)
logger.debug(f"FUSE: Resolved rmdir for {parent_inode=}/{name=} to {path=}")
self.rose.rmdir(path)
# Avoid zombies coming back from an old cache.
self.reset_getattr_caches()
self.inodes.remove_path(path)

Expand Down Expand Up @@ -1151,7 +1173,7 @@ def mount_virtualfs(c: Config, debug: bool = False) -> None:
options.add("debug")
llfuse.init(VirtualFS(c), str(c.fuse_mount_dir), options)
try:
llfuse.main()
llfuse.main(workers=c.max_proc)
except:
llfuse.close()
raise
Expand Down
14 changes: 13 additions & 1 deletion rose/virtualfs_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,19 @@ def test_virtual_filesystem_playlist_actions(
assert (src / "!playlists" / "New Jeans.toml").is_file()
assert not (src / "!playlists" / "New Tee.toml").exists()
# Add track to playlist.
shutil.copyfile(release_dir / filename, root / "8. Playlists" / "New Jeans" / filename)
# Use `cp -p` to test the ghost files behavior. A pure copy file will succeed, because it
# stops after the release. However, cp -p also attempts to set some attributes on the moved
# file, which fails if we immediately vanish the file post-release, which the naive
# implementation does.
subprocess.run(
[
"cp",
"-p",
str(release_dir / filename),
str(root / "8. Playlists" / "New Jeans" / filename),
],
check=True,
)
assert (root / "8. Playlists" / "New Jeans" / "1. BLACKPINK - Track 1.m4a").is_file()
with (src / "!playlists" / "New Jeans.toml").open("r") as fp:
assert "BLACKPINK - Track 1.m4a" in fp.read()
Expand Down

0 comments on commit 74b291b

Please sign in to comment.