From 71404fdfc2b313bb2c0988fd3a547356637f7f04 Mon Sep 17 00:00:00 2001 From: blissful Date: Mon, 9 Oct 2023 14:55:42 -0400 Subject: [PATCH] add database schema & execute migration on command run --- .root | 0 README.md | 27 +++++ conftest.py | 51 +++++++-- flake.nix | 7 +- .../20231009_01_qlEHa-bootstrap.rollback.sql | 14 +++ migrations/20231009_01_qlEHa-bootstrap.sql | 105 ++++++++++++++++++ pyproject.toml | 3 + rose/__main__.py | 35 +++++- rose/cache/database.py | 44 ++++++++ rose/cache/database_test.py | 34 ++++++ rose/{config.py => foundation/conf.py} | 19 +++- .../conf_test.py} | 2 +- rose/{base_error.py => foundation/errors.py} | 0 13 files changed, 328 insertions(+), 13 deletions(-) delete mode 100644 .root create mode 100644 migrations/20231009_01_qlEHa-bootstrap.rollback.sql create mode 100644 migrations/20231009_01_qlEHa-bootstrap.sql create mode 100644 rose/cache/database.py create mode 100644 rose/cache/database_test.py rename rose/{config.py => foundation/conf.py} (58%) rename rose/{config_test.py => foundation/conf_test.py} (93%) rename rose/{base_error.py => foundation/errors.py} (100%) diff --git a/.root b/.root deleted file mode 100644 index e69de29..0000000 diff --git a/README.md b/README.md index a62c45c..e1b2284 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # rose +_In Progress_ + Rose is a Linux music library manager. ## Configuration @@ -13,4 +15,29 @@ The configuration parameters, with examples, are: music_source_dir = "~/.music-src" # The directory to mount the library's virtual filesystem on. fuse_mount_dir = "~/music" +# The directory to write the cache to. Defaults to `${XDG_CACHE_HOME:-$HOME/.cache}/rose`. +cache_dir = "~/.cache/rose" ``` + +## Library Conventions & Expectations + +### Directory Structure + +`$music_source_dir/albums/track.ogg` + +### Supported Extensions + +### Tag Structure + +WIP + +artist1;artist2 feat. artist3 + +BNF TODO + +# Architecture + +todo + +- db is read cache, not source of truth +- filetags and files are source of truth diff --git a/conftest.py b/conftest.py index e7b14ed..3f4d657 100644 --- a/conftest.py +++ b/conftest.py @@ -1,23 +1,60 @@ import logging -import pathlib +import sqlite3 +from collections.abc import Iterator +from pathlib import Path import _pytest.pathlib +import pytest +from click.testing import CliRunner + +from rose.foundation.conf import Config logger = logging.getLogger(__name__) +@pytest.fixture() +def isolated_dir() -> Iterator[Path]: + with CliRunner().isolated_filesystem(): + yield Path.cwd() + + +@pytest.fixture() +def config(isolated_dir: Path) -> Config: + (isolated_dir / "cache").mkdir() + return Config( + music_source_dir=isolated_dir / "source", + fuse_mount_dir=isolated_dir / "mount", + cache_dir=isolated_dir / "cache", + cache_database_path=isolated_dir / "cache" / "cache.sqlite3", + ) + + +def freeze_database_time(conn: sqlite3.Connection) -> None: + """ + This function freezes the CURRENT_TIMESTAMP function in SQLite3 to + "2020-01-01 01:01:01". This should only be used in testing. + """ + conn.create_function( + "CURRENT_TIMESTAMP", + 0, + _return_fake_timestamp, + deterministic=True, + ) + + +def _return_fake_timestamp() -> str: + return "2020-01-01 01:01:01" + + # Pytest has a bug where it doesn't handle namespace packages and treats same-name files -# in different packages as a naming collision. -# -# https://stackoverflow.com/a/72366347 -# Tweaked from ^ to handle our foundation/product split. +# in different packages as a naming collision. https://stackoverflow.com/a/72366347 resolve_pkg_path_orig = _pytest.pathlib.resolve_package_path -namespace_pkg_dirs = [str(d) for d in pathlib.Path(__file__).parent.iterdir() if d.is_dir()] +namespace_pkg_dirs = [str(d) for d in Path(__file__).parent.iterdir() if d.is_dir()] # patched method -def resolve_package_path(path: pathlib.Path) -> pathlib.Path | None: +def resolve_package_path(path: Path) -> Path | None: # call original lookup result = resolve_pkg_path_orig(path) if result is None: diff --git a/flake.nix b/flake.nix index 6631b4e..acf1bc6 100644 --- a/flake.nix +++ b/flake.nix @@ -29,6 +29,10 @@ pytest-cov setuptools ]; + dev-cli = pkgs.writeShellScriptBin "rose" '' + cd $ROSE_ROOT + python -m rose $@ + ''; in { devShells.default = pkgs.mkShell { @@ -40,7 +44,7 @@ done echo "$path" } - export ROSE_ROOT="$(find-up .root)" + export ROSE_ROOT="$(find-up flake.nix)" # We intentionally do not allow installing Python packages to the # global Python environment. Mutable Python installations should be @@ -53,6 +57,7 @@ paths = with pkgs; [ (python.withPackages (_: prod-deps ++ dev-deps)) ruff + dev-cli ]; }) ]; diff --git a/migrations/20231009_01_qlEHa-bootstrap.rollback.sql b/migrations/20231009_01_qlEHa-bootstrap.rollback.sql new file mode 100644 index 0000000..77d8423 --- /dev/null +++ b/migrations/20231009_01_qlEHa-bootstrap.rollback.sql @@ -0,0 +1,14 @@ +-- bootstrap +-- depends: + +DROP TABLE playlists_tracks; +DROP TABLE playlists; +DROP TABLE collections_releases; +DROP TABLE collections; +DROP TABLE tracks_artists; +DROP TABLE releases_artists; +DROP TABLE artists; +DROP TABLE artist_role_enum; +DROP TABLE tracks; +DROP TABLE releases; +DROP TABLE release_type_enum; diff --git a/migrations/20231009_01_qlEHa-bootstrap.sql b/migrations/20231009_01_qlEHa-bootstrap.sql new file mode 100644 index 0000000..db7ea02 --- /dev/null +++ b/migrations/20231009_01_qlEHa-bootstrap.sql @@ -0,0 +1,105 @@ +-- bootstrap +-- depends: + +CREATE TABLE release_type_enum (value TEXT PRIMARY KEY); +INSERT INTO release_type_enum (value) VALUES + ('album'), + ('single'), + ('ep'), + ('compilation'), + ('soundtrack'), + ('live'), + ('remix'), + ('djmix'), + ('mixtape'), + ('other'), + ('unknown'); + +CREATE TABLE releases ( + id INTEGER PRIMARY KEY, + source_path TEXT NOT NULL UNIQUE, + title TEXT NOT NULL, + release_type TEXT NOT NULL REFERENCES release_type_enum(value), + release_year INTEGER +); +CREATE INDEX releases_source_path ON releases(source_path); +CREATE INDEX releases_release_year ON releases(release_year); + +CREATE TABLE tracks ( + id INTEGER PRIMARY KEY, + source_path TEXT NOT NULL UNIQUE, + title TEXT NOT NULL, + release_id INTEGER NOT NULL REFERENCES releases(id), + track_number TEXT NOT NULL, + disc_number TEXT NOT NULL, + duration_seconds INTEGER NOT NULL +); +CREATE INDEX tracks_source_path ON tracks(source_path); +CREATE INDEX tracks_release_id ON tracks(release_id); +CREATE INDEX tracks_ordering ON tracks(release_id, disc_number, track_number); + +CREATE TABLE artists ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL +); +CREATE INDEX artists_name ON artists(name); + +CREATE TABLE artist_role_enum (value TEXT PRIMARY KEY); +INSERT INTO artist_role_enum (value) VALUES + ('main'), + ('feature'), + ('remixer'), + ('producer'), + ('composer'), + ('conductor'), + ('djmixer'); + +CREATE TABLE releases_artists ( + release_id INTEGER REFERENCES releases(id) ON DELETE CASCADE, + artist_id INTEGER REFERENCES artists(id) ON DELETE CASCADE, + role TEXT REFERENCES artist_role_enum(value), + PRIMARY KEY (release_id, artist_id) +); +CREATE INDEX releases_artists_release_id ON releases_artists(release_id); +CREATE INDEX releases_artists_artist_id ON releases_artists(artist_id); + +CREATE TABLE tracks_artists ( + track_id INTEGER REFERENCES tracks(id) ON DELETE CASCADE, + artist_id INTEGER REFERENCES artists(id) ON DELETE CASCADE, + role TEXT REFERENCES artist_role_enum(value), + PRIMARY KEY (track_id, artist_id) +); +CREATE INDEX tracks_artists_track_id ON tracks_artists(track_id); +CREATE INDEX tracks_artists_artist_id ON tracks_artists(artist_id); + +CREATE TABLE collections ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + source_path TEXT UNIQUE NOT NULL +); +CREATE INDEX collections_source_path ON collections(source_path); + +CREATE TABLE collections_releases ( + collection_id INTEGER REFERENCES collections(id) ON DELETE CASCADE, + release_id INTEGER REFERENCES releases(id) ON DELETE CASCADE, + position INTEGER NOT NULL +); +CREATE INDEX collections_releases_collection_id ON collections_releases(collection_id); +CREATE INDEX collections_releases_release_id ON collections_releases(release_id); +CREATE UNIQUE INDEX collections_releases_collection_position ON collections_releases(collection_id, position); + +CREATE TABLE playlists ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + source_path TEXT UNIQUE NOT NULL +); +CREATE INDEX playlists_source_path ON playlists(source_path); + +CREATE TABLE playlists_tracks ( + playlist_id INTEGER REFERENCES playlists(id) ON DELETE CASCADE, + track_id INTEGER REFERENCES tracks(id) ON DELETE CASCADE, + position INTEGER NOT NULL +); +CREATE INDEX playlists_tracks_playlist_id ON playlists_tracks(playlist_id); +CREATE INDEX playlists_tracks_track_id ON playlists_tracks(track_id); +CREATE UNIQUE INDEX playlists_tracks_playlist_position ON playlists_tracks(playlist_id, position); diff --git a/pyproject.toml b/pyproject.toml index 0ae939e..1a34d82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,9 @@ strict = true strict_optional = true explicit_package_bases = true +[[tool.mypy.overrides]] +module = "yoyo" +ignore_missing_imports = true [[tool.mypy.overrides]] module = "setuptools" ignore_missing_imports = true diff --git a/rose/__main__.py b/rose/__main__.py index 08b9fe0..6b915c4 100644 --- a/rose/__main__.py +++ b/rose/__main__.py @@ -1,9 +1,42 @@ +import logging +from dataclasses import dataclass + import click +from rose.cache.database import migrate_database +from rose.foundation.conf import Config + + +@dataclass +class Context: + config: Config + @click.group() -def cli() -> None: +@click.option("--verbose", "-v", is_flag=True) +@click.pass_context +def cli(clickctx: click.Context, verbose: bool) -> None: """A filesystem-driven music library manager.""" + clickctx.obj = Context( + config=Config.read(), + ) + + if verbose: + logging.getLogger().setLevel(logging.DEBUG) + + # Migrate the database on every command invocation. + migrate_database(clickctx.obj.config) + + +@cli.group() +@click.pass_obj +def cache(_: Context) -> None: + """Manage the cached metadata.""" + + +@cache.command() +def reset() -> None: + """Reset the cache and empty the database.""" if __name__ == "__main__": diff --git a/rose/cache/database.py b/rose/cache/database.py new file mode 100644 index 0000000..ab396cf --- /dev/null +++ b/rose/cache/database.py @@ -0,0 +1,44 @@ +import logging +import sqlite3 +from collections.abc import Iterator +from contextlib import contextmanager + +import yoyo + +from rose.foundation.conf import MIGRATIONS_PATH, Config + +logger = logging.getLogger(__name__) + + +@contextmanager +def connect(c: Config) -> Iterator[sqlite3.Connection]: + conn = connect_fn(c) + try: + yield conn + finally: + conn.close() + + +def connect_fn(c: Config) -> sqlite3.Connection: + """Non-context manager version of connect.""" + conn = sqlite3.connect( + c.cache_database_path, + detect_types=sqlite3.PARSE_DECLTYPES, + isolation_level=None, + timeout=15.0, + ) + + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys=ON") + conn.execute("PRAGMA journal_mode=WAL") + + return conn + + +def migrate_database(c: Config) -> None: + db_backend = yoyo.get_backend(f"sqlite:///{c.cache_database_path}") + db_migrations = yoyo.read_migrations(str(MIGRATIONS_PATH)) + + logger.debug("Applying database migrations") + with db_backend.lock(): + db_backend.apply_migrations(db_backend.to_apply(db_migrations)) diff --git a/rose/cache/database_test.py b/rose/cache/database_test.py new file mode 100644 index 0000000..ba1012b --- /dev/null +++ b/rose/cache/database_test.py @@ -0,0 +1,34 @@ +import sqlite3 +from pathlib import Path + +import yoyo + +from conftest import freeze_database_time +from rose.cache.database import migrate_database +from rose.foundation.conf import MIGRATIONS_PATH, Config + + +def test_run_database_migrations(config: Config) -> None: + migrate_database(config) + assert config.cache_database_path.exists() + + with sqlite3.connect(str(config.cache_database_path)) as conn: + freeze_database_time(conn) + cursor = conn.execute("SELECT 1 FROM _yoyo_version") + assert len(cursor.fetchall()) > 0 + + +def test_migrations(isolated_dir: Path) -> None: + """ + Test that, for each migration, the up -> down -> up path doesn't + cause an error. Basically, ladder our way up through the migration + chain. + """ + backend = yoyo.get_backend(f"sqlite:///{isolated_dir / 'db.sqlite3'}") + migrations = yoyo.read_migrations(str(MIGRATIONS_PATH)) + + assert len(migrations) > 0 + for mig in migrations: + backend.apply_one(mig) + backend.rollback_one(mig) + backend.apply_one(mig) diff --git a/rose/config.py b/rose/foundation/conf.py similarity index 58% rename from rose/config.py rename to rose/foundation/conf.py index 335dc9b..d5c49ac 100644 --- a/rose/config.py +++ b/rose/foundation/conf.py @@ -6,10 +6,14 @@ import tomllib -from rose.base_error import RoseError +from rose.foundation.errors import RoseError -CONFIG_HOME = Path(os.environ.get("XDG_CONFIG_HOME", os.environ["HOME"] + "/.config")) -CONFIG_PATH = CONFIG_HOME / "rose" / "config.toml" +XDG_CONFIG_HOME = Path(os.environ.get("XDG_CONFIG_HOME", os.environ["HOME"] + "/.config")) +CONFIG_PATH = XDG_CONFIG_HOME / "rose" / "config.toml" +XDG_CACHE_HOME = Path(os.environ.get("XDG_CACHE_HOME", os.environ["HOME"] + "/.cache")) +CACHE_PATH = XDG_CACHE_HOME / "rose" + +MIGRATIONS_PATH = Path(__file__).resolve().parent.parent.parent / "migrations" class ConfigNotFoundError(RoseError): @@ -24,6 +28,8 @@ class MissingConfigKeyError(RoseError): class Config: music_source_dir: Path fuse_mount_dir: Path + cache_dir: Path + cache_database_path: Path @classmethod def read(cls, config_path_override: Path | None = None) -> Config: @@ -34,10 +40,17 @@ def read(cls, config_path_override: Path | None = None) -> Config: except FileNotFoundError as e: raise ConfigNotFoundError(f"Configuration file not found ({CONFIG_PATH})") from e + cache_dir = CACHE_PATH + if "cache_dir" in data: + cache_dir = Path(data["cache_dir"]).expanduser() + cache_dir.mkdir(exist_ok=True) + try: return cls( music_source_dir=Path(data["music_source_dir"]).expanduser(), fuse_mount_dir=Path(data["fuse_mount_dir"]).expanduser(), + cache_dir=cache_dir, + cache_database_path=cache_dir / "cache.sqlite3", ) except KeyError as e: raise MissingConfigKeyError(f"Missing key in configuration file: {e}") from e diff --git a/rose/config_test.py b/rose/foundation/conf_test.py similarity index 93% rename from rose/config_test.py rename to rose/foundation/conf_test.py index 7f16c24..4989581 100644 --- a/rose/config_test.py +++ b/rose/foundation/conf_test.py @@ -4,7 +4,7 @@ import pytest -from rose.config import Config, ConfigNotFoundError, MissingConfigKeyError +from .conf import Config, ConfigNotFoundError, MissingConfigKeyError def test_config() -> None: diff --git a/rose/base_error.py b/rose/foundation/errors.py similarity index 100% rename from rose/base_error.py rename to rose/foundation/errors.py