From ce2452f86881213d925a868381d54571c26bc68c Mon Sep 17 00:00:00 2001 From: blissful Date: Wed, 11 Oct 2023 10:59:38 -0400 Subject: [PATCH] replace migration system with db erasure --- conftest.py | 15 +++--- flake.nix | 1 - .../20231009_01_qlEHa-bootstrap.rollback.sql | 15 ------ pyproject.toml | 3 -- rose/cache/database.py | 40 ++++++++++++--- rose/cache/database_test.py | 51 +++++++++---------- .../cache/schema.sql | 3 -- rose/foundation/conf.py | 2 +- setup.py | 1 - yoyo.ini | 8 --- 10 files changed, 66 insertions(+), 73 deletions(-) delete mode 100644 migrations/20231009_01_qlEHa-bootstrap.rollback.sql rename migrations/20231009_01_qlEHa-bootstrap.sql => rose/cache/schema.sql (99%) delete mode 100644 yoyo.ini diff --git a/conftest.py b/conftest.py index 37477a7..cd41a03 100644 --- a/conftest.py +++ b/conftest.py @@ -1,3 +1,4 @@ +import hashlib import logging import sqlite3 from collections.abc import Iterator @@ -5,10 +6,9 @@ import _pytest.pathlib import pytest -import yoyo from click.testing import CliRunner -from rose.foundation.conf import MIGRATIONS_PATH, Config +from rose.foundation.conf import SCHEMA_PATH, Config logger = logging.getLogger(__name__) @@ -25,10 +25,13 @@ def config(isolated_dir: Path) -> Config: cache_dir.mkdir() cache_database_path = cache_dir / "cache.sqlite3" - db_backend = yoyo.get_backend(f"sqlite:///{cache_database_path}") - db_migrations = yoyo.read_migrations(str(MIGRATIONS_PATH)) - with db_backend.lock(): - db_backend.apply_migrations(db_backend.to_apply(db_migrations)) + with sqlite3.connect(cache_database_path) as conn: + with SCHEMA_PATH.open("r") as fp: + conn.executescript(fp.read()) + conn.execute("CREATE TABLE _schema_hash (value TEXT PRIMARY KEY)") + with SCHEMA_PATH.open("rb") as fp: + latest_schema_hash = hashlib.sha256(fp.read()).hexdigest() + conn.execute("INSERT INTO _schema_hash (value) VALUES (?)", (latest_schema_hash,)) return Config( music_source_dir=isolated_dir / "source", diff --git a/flake.nix b/flake.nix index f052d8e..92340bd 100644 --- a/flake.nix +++ b/flake.nix @@ -31,7 +31,6 @@ fuse mutagen uuid6-python - yoyo-migrations ]; dev-deps = with python.pkgs; [ black diff --git a/migrations/20231009_01_qlEHa-bootstrap.rollback.sql b/migrations/20231009_01_qlEHa-bootstrap.rollback.sql deleted file mode 100644 index e0e4e53..0000000 --- a/migrations/20231009_01_qlEHa-bootstrap.rollback.sql +++ /dev/null @@ -1,15 +0,0 @@ --- bootstrap --- depends: - -DROP TABLE playlists_tracks; -DROP TABLE playlists; -DROP TABLE collections_releases; -DROP TABLE collections; -DROP TABLE tracks_artists; -DROP TABLE releases_artists; -DROP TABLE artist_role_enum; -DROP TABLE tracks; -DROP TABLE releases_labels; -DROP TABLE releases_genres; -DROP TABLE releases; -DROP TABLE release_type_enum; diff --git a/pyproject.toml b/pyproject.toml index e2b2a8d..47f2031 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,9 +59,6 @@ strict = true strict_optional = true explicit_package_bases = true -[[tool.mypy.overrides]] -module = "yoyo" -ignore_missing_imports = true [[tool.mypy.overrides]] module = "fuse" ignore_missing_imports = true diff --git a/rose/cache/database.py b/rose/cache/database.py index 9faf1e7..9179868 100644 --- a/rose/cache/database.py +++ b/rose/cache/database.py @@ -1,4 +1,5 @@ import binascii +import hashlib import logging import random import sqlite3 @@ -6,9 +7,7 @@ from collections.abc import Iterator from contextlib import contextmanager -import yoyo - -from rose.foundation.conf import MIGRATIONS_PATH, Config +from rose.foundation.conf import SCHEMA_PATH, Config logger = logging.getLogger(__name__) @@ -72,9 +71,34 @@ def connect_fn(c: Config) -> sqlite3.Connection: def migrate_database(c: Config) -> None: - db_backend = yoyo.get_backend(f"sqlite:///{c.cache_database_path}") - db_migrations = yoyo.read_migrations(str(MIGRATIONS_PATH)) + """ + "Migrate" the database. If the schema in the database does not match that on disk, then nuke the + database and recreate it from scratch. Otherwise, no op. - logger.debug("Applying database migrations") - with db_backend.lock(): - db_backend.apply_migrations(db_backend.to_apply(db_migrations)) + We can do this because the database is just a read cache. It is not source-of-truth for any of + its own data. + """ + with SCHEMA_PATH.open("rb") as fp: + latest_schema_hash = hashlib.sha256(fp.read()).hexdigest() + + with connect(c) as conn: + cursor = conn.execute( + """ + SELECT EXISTS( + SELECT * FROM sqlite_master + WHERE type = 'table' AND name = '_schema_hash' + ) + """ + ) + if cursor.fetchone()[0]: + cursor = conn.execute("SELECT value FROM _schema_hash") + if (row := cursor.fetchone()) and row[0] == latest_schema_hash: + # Everything matches! Exit! + return + + c.cache_database_path.unlink(missing_ok=True) + with connect(c) as conn: + with SCHEMA_PATH.open("r") as fp: + conn.executescript(fp.read()) + conn.execute("CREATE TABLE _schema_hash (value TEXT PRIMARY KEY)") + conn.execute("INSERT INTO _schema_hash (value) VALUES (?)", (latest_schema_hash,)) diff --git a/rose/cache/database_test.py b/rose/cache/database_test.py index ba1012b..ff562a9 100644 --- a/rose/cache/database_test.py +++ b/rose/cache/database_test.py @@ -1,34 +1,31 @@ -import sqlite3 -from pathlib import Path +import hashlib -import yoyo +from rose.cache.database import connect, migrate_database +from rose.foundation.conf import SCHEMA_PATH, Config -from conftest import freeze_database_time -from rose.cache.database import migrate_database -from rose.foundation.conf import MIGRATIONS_PATH, Config - -def test_run_database_migrations(config: Config) -> None: +def test_schema(config: Config) -> None: + # Test that the schema successfully bootstraps. + with SCHEMA_PATH.open("rb") as fp: + latest_schema_hash = hashlib.sha256(fp.read()).hexdigest() migrate_database(config) - assert config.cache_database_path.exists() - - with sqlite3.connect(str(config.cache_database_path)) as conn: - freeze_database_time(conn) - cursor = conn.execute("SELECT 1 FROM _yoyo_version") - assert len(cursor.fetchall()) > 0 + with connect(config) as conn: + cursor = conn.execute("SELECT value FROM _schema_hash") + assert cursor.fetchone()[0] == latest_schema_hash -def test_migrations(isolated_dir: Path) -> None: - """ - Test that, for each migration, the up -> down -> up path doesn't - cause an error. Basically, ladder our way up through the migration - chain. - """ - backend = yoyo.get_backend(f"sqlite:///{isolated_dir / 'db.sqlite3'}") - migrations = yoyo.read_migrations(str(MIGRATIONS_PATH)) +def test_migration(config: Config) -> None: + # Test that "migrating" the database correctly migrates it. + config.cache_database_path.unlink() + with connect(config) as conn: + conn.execute("CREATE TABLE _schema_hash (value TEXT PRIMARY KEY)") + conn.execute("INSERT INTO _schema_hash (value) VALUES ('haha')") - assert len(migrations) > 0 - for mig in migrations: - backend.apply_one(mig) - backend.rollback_one(mig) - backend.apply_one(mig) + with SCHEMA_PATH.open("rb") as fp: + latest_schema_hash = hashlib.sha256(fp.read()).hexdigest() + migrate_database(config) + with connect(config) as conn: + cursor = conn.execute("SELECT value FROM _schema_hash") + assert cursor.fetchone()[0] == latest_schema_hash + cursor = conn.execute("SELECT COUNT(*) FROM _schema_hash") + assert cursor.fetchone()[0] == 1 diff --git a/migrations/20231009_01_qlEHa-bootstrap.sql b/rose/cache/schema.sql similarity index 99% rename from migrations/20231009_01_qlEHa-bootstrap.sql rename to rose/cache/schema.sql index 0fa0789..9b1af39 100644 --- a/migrations/20231009_01_qlEHa-bootstrap.sql +++ b/rose/cache/schema.sql @@ -1,6 +1,3 @@ --- bootstrap --- depends: - CREATE TABLE release_type_enum (value TEXT PRIMARY KEY); INSERT INTO release_type_enum (value) VALUES ('album'), diff --git a/rose/foundation/conf.py b/rose/foundation/conf.py index d5c49ac..1f426e0 100644 --- a/rose/foundation/conf.py +++ b/rose/foundation/conf.py @@ -13,7 +13,7 @@ XDG_CACHE_HOME = Path(os.environ.get("XDG_CACHE_HOME", os.environ["HOME"] + "/.cache")) CACHE_PATH = XDG_CACHE_HOME / "rose" -MIGRATIONS_PATH = Path(__file__).resolve().parent.parent.parent / "migrations" +SCHEMA_PATH = Path(__file__).resolve().parent.parent / "cache" / "schema.sql" class ConfigNotFoundError(RoseError): diff --git a/setup.py b/setup.py index 2c39d2e..1c44433 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,5 @@ "fuse-python", "mutagen", "uuid6-python", - "yoyo-migrations", ], ) diff --git a/yoyo.ini b/yoyo.ini deleted file mode 100644 index bd04938..0000000 --- a/yoyo.ini +++ /dev/null @@ -1,8 +0,0 @@ -# This is for development use only. - -[DEFAULT] -sources = migrations -migration_table = _yoyo_migration -batch_mode = off -verbosity = 0 -database = sqlite:///db.sqlite3