diff --git a/rose/cache.py b/rose/cache.py index 86442d4..fbe2c40 100644 --- a/rose/cache.py +++ b/rose/cache.py @@ -269,6 +269,8 @@ def update_cache_for_releases( # Leave as None to update all releases. release_dirs: list[Path] | None = None, force: bool = False, + # For testing. + force_multiprocessing: bool = False, ) -> None: """ Update the read cache to match the data for any passed-in releases. If a directory lacks a @@ -296,9 +298,23 @@ def update_cache_for_releases( and d.name != "!playlists" and d.name not in c.ignore_release_directories ] + if not release_dirs: + logger.info("No-Op: No whitelisted releases passed into update_cache_for_releases") + return logger.info(f"Refreshing the read cache for {len(release_dirs)} releases") logger.debug(f"Refreshing cached data for {', '.join([r.name for r in release_dirs])}") + # If the number of releases changed is less than 50; do not bother with all that multiprocessing + # gunk: instead, directly call the executor. + # + # This has an added benefit of not spawning processes from the virtual filesystem and watchdog + # processes, as those processes always update the cache for one release at a time and are + # multithreaded. Starting other processes from threads is bad! + if not force_multiprocessing and len(release_dirs) < 50: + known_virtual_dirnames_hi: dict[str, bool] = {} + _update_cache_for_releases_executor(c, release_dirs, force, known_virtual_dirnames_hi) + return + # Batch size defaults to equal split across all processes. However, if the number of directories # is small, we shrink the # of processes to save on overhead. num_proc = c.max_proc diff --git a/rose/cache_test.py b/rose/cache_test.py index 47adcea..f23245a 100644 --- a/rose/cache_test.py +++ b/rose/cache_test.py @@ -143,6 +143,18 @@ def test_update_cache_all(config: Config) -> None: assert cursor.fetchone()[0] == 4 +def test_update_cache_multiprocessing(config: Config) -> None: + """Test that the update all function works.""" + shutil.copytree(TEST_RELEASE_1, config.music_source_dir / TEST_RELEASE_1.name) + shutil.copytree(TEST_RELEASE_2, config.music_source_dir / TEST_RELEASE_2.name) + update_cache_for_releases(config, force_multiprocessing=True) + with connect(config) as conn: + cursor = conn.execute("SELECT COUNT(*) FROM releases") + assert cursor.fetchone()[0] == 2 + cursor = conn.execute("SELECT COUNT(*) FROM tracks") + assert cursor.fetchone()[0] == 4 + + def test_update_cache_releases(config: Config) -> None: release_dir = config.music_source_dir / TEST_RELEASE_1.name shutil.copytree(TEST_RELEASE_1, release_dir) diff --git a/rose/cli.py b/rose/cli.py index 096f506..834cc98 100644 --- a/rose/cli.py +++ b/rose/cli.py @@ -8,7 +8,6 @@ """ import logging -import multiprocessing import os import signal from dataclasses import dataclass @@ -95,7 +94,6 @@ def update(ctx: Context, force: bool) -> None: # fmt: on def watch(ctx: Context, foreground: bool) -> None: """Start a watchdog that will auto-refresh the cache on changes in music_source_dir.""" - multiprocessing.set_start_method("spawn") if not foreground: daemonize(pid_path=ctx.config.watchdog_pid_path) @@ -131,7 +129,6 @@ def fs() -> None: # fmt: on def mount(ctx: Context, foreground: bool) -> None: """Mount the virtual library.""" - multiprocessing.set_start_method("spawn") if not foreground: daemonize() diff --git a/rose/cli_test.py b/rose/cli_test.py index 61f8c1d..0c0c582 100644 --- a/rose/cli_test.py +++ b/rose/cli_test.py @@ -1,4 +1,3 @@ -import multiprocessing import os from typing import Any @@ -35,8 +34,6 @@ def mock_exit(x: int) -> None: raise SystemExit(x) monkeypatch.setattr(os, "_exit", mock_exit) - # set_start_method borks a bit in tests; it's not important in this test anyways. - monkeypatch.setattr(multiprocessing, "set_start_method", lambda _: None) ctx = Context(config=config) runner = CliRunner()