Skip to content

Commit

Permalink
move sanitizer to the vfs module only
Browse files Browse the repository at this point in the history
  • Loading branch information
@azuline
azuline committed Apr 20, 2024
1 parent 33ee799 commit 067d87c
Show file tree
Hide file tree
Showing 6 changed files with 169 additions and 125 deletions.
38 changes: 19 additions & 19 deletions conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,15 +118,15 @@ def seeded_cache(config: Config) -> None:
, ('r3', '{dirpaths[2]}', null , '0000-01-01T00:00:00+00:00', '999' , 'Release 3', 'album' , 2021 , 1780 , 'DG-002' , 1 , true , '3');
INSERT INTO releases_genres
(release_id, genre , genre_sanitized, position)
VALUES ('r1' , 'Techno' , 'Techno' , 1)
, ('r1' , 'Deep House', 'Deep House' , 2)
, ('r2' , 'Classical' , 'Classical' , 1);
(release_id, genre , position)
VALUES ('r1' , 'Techno' , 1)
, ('r1' , 'Deep House', 2)
, ('r2' , 'Classical' , 1);
INSERT INTO releases_labels
(release_id, label , label_sanitized, position)
VALUES ('r1' , 'Silk Music' , 'Silk Music' , 1)
, ('r2' , 'Native State', 'Native State' , 1);
(release_id, label , position)
VALUES ('r1' , 'Silk Music' , 1)
, ('r2' , 'Native State', 1);
INSERT INTO tracks
(id , source_path , source_mtime, title , release_id, tracknumber, tracktotal, discnumber, duration_seconds, metahash)
Expand All @@ -136,20 +136,20 @@ def seeded_cache(config: Config) -> None:
, ('t4', '{musicpaths[3]}', '999' , 'Track 1', 'r3' , '01' , 1 , '01' , 120 , '4');
INSERT INTO releases_artists
(release_id, artist , artist_sanitized , role , position)
VALUES ('r1' , 'Techno Man' , 'Techno Man' , 'main' , 1)
, ('r1' , 'Bass Man' , 'Bass Man' , 'main' , 2)
, ('r2' , 'Violin Woman' , 'Violin Woman' , 'main' , 1)
, ('r2' , 'Conductor Woman', 'Conductor Woman', 'guest', 2);
(release_id, artist , role , position)
VALUES ('r1' , 'Techno Man' , 'main' , 1)
, ('r1' , 'Bass Man' , 'main' , 2)
, ('r2' , 'Violin Woman' , 'main' , 1)
, ('r2' , 'Conductor Woman', 'guest', 2);
INSERT INTO tracks_artists
(track_id, artist , artist_sanitized , role , position)
VALUES ('t1' , 'Techno Man' , 'Techno Man' , 'main' , 1)
, ('t1' , 'Bass Man' , 'Bass Man' , 'main' , 2)
, ('t2' , 'Techno Man' , 'Techno Man' , 'main' , 1)
, ('t2' , 'Bass Man' , 'Bass Man' , 'main' , 2)
, ('t3' , 'Violin Woman' , 'Violin Woman' , 'main' , 1)
, ('t3' , 'Conductor Woman', 'Conductor Woman', 'guest', 2);
(track_id, artist , role , position)
VALUES ('t1' , 'Techno Man' , 'main' , 1)
, ('t1' , 'Bass Man' , 'main' , 2)
, ('t2' , 'Techno Man' , 'main' , 1)
, ('t2' , 'Bass Man' , 'main' , 2)
, ('t3' , 'Violin Woman' , 'main' , 1)
, ('t3' , 'Conductor Woman', 'guest', 2);
INSERT INTO collages
(name , source_mtime)
Expand Down
101 changes: 46 additions & 55 deletions rose/cache.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -990,19 +990,13 @@ def _update_cache_for_releases_executor(
)
upd_release_ids.append(release.id)
for pos, genre in enumerate(release.genres):
upd_release_genre_args.append(
[release.id, genre, sanitize_dirname(genre, False), pos]
)
upd_release_genre_args.append([release.id, genre, pos])
for pos, label in enumerate(release.labels):
upd_release_label_args.append(
[release.id, label, sanitize_dirname(label, False), pos]
)
upd_release_label_args.append([release.id, label, pos])
pos = 0
for role, artists in release.releaseartists.items():
for art in artists:
upd_release_artist_args.append(
[release.id, art.name, sanitize_dirname(art.name, False), role, pos]
)
upd_release_artist_args.append([release.id, art.name, role, pos])
pos += 1

if track_ids_to_insert:
Expand All @@ -1028,9 +1022,7 @@ def _update_cache_for_releases_executor(
pos = 0
for role, artists in track.trackartists.items():
for art in artists:
upd_track_artist_args.append(
[track.id, art.name, sanitize_dirname(art.name, False), role, pos]
)
upd_track_artist_args.append([track.id, art.name, role, pos])
pos += 1
logger.debug(f"Release update scheduling loop time {time.time() - loop_start=}")

Expand Down Expand Up @@ -1083,8 +1075,8 @@ def _update_cache_for_releases_executor(
)
conn.execute(
f"""
INSERT INTO releases_genres (release_id, genre, genre_sanitized, position)
VALUES {",".join(["(?,?,?,?)"]*len(upd_release_genre_args))}
INSERT INTO releases_genres (release_id, genre, position)
VALUES {",".join(["(?,?,?)"]*len(upd_release_genre_args))}
""",
_flatten(upd_release_genre_args),
)
Expand All @@ -1098,8 +1090,8 @@ def _update_cache_for_releases_executor(
)
conn.execute(
f"""
INSERT INTO releases_labels (release_id, label, label_sanitized, position)
VALUES {",".join(["(?,?,?,?)"]*len(upd_release_label_args))}
INSERT INTO releases_labels (release_id, label, position)
VALUES {",".join(["(?,?,?)"]*len(upd_release_label_args))}
""",
_flatten(upd_release_label_args),
)
Expand All @@ -1113,8 +1105,8 @@ def _update_cache_for_releases_executor(
)
conn.execute(
f"""
INSERT INTO releases_artists (release_id, artist, artist_sanitized, role, position)
VALUES {",".join(["(?,?,?,?,?)"]*len(upd_release_artist_args))}
INSERT INTO releases_artists (release_id, artist, role, position)
VALUES {",".join(["(?,?,?,?)"]*len(upd_release_artist_args))}
""",
_flatten(upd_release_artist_args),
)
Expand Down Expand Up @@ -1147,8 +1139,8 @@ def _update_cache_for_releases_executor(
)
conn.execute(
f"""
INSERT INTO tracks_artists (track_id, artist, artist_sanitized, role, position)
VALUES {",".join(["(?,?,?,?,?)"]*len(upd_track_artist_args))}
INSERT INTO tracks_artists (track_id, artist, role, position)
VALUES {",".join(["(?,?,?,?)"]*len(upd_track_artist_args))}
""",
_flatten(upd_track_artist_args),
)
Expand Down Expand Up @@ -1689,42 +1681,41 @@ def update_cache_evict_nonexistent_playlists(c: Config) -> None:

def list_releases_delete_this(
c: Config,
sanitized_artist_filter: str | None = None,
sanitized_genre_filter: str | None = None,
sanitized_label_filter: str | None = None,
artist_filter: str | None = None,
genre_filter: str | None = None,
label_filter: str | None = None,
new: bool | None = None,
) -> list[CachedRelease]:
with connect(c) as conn:
query = "SELECT * FROM releases_view WHERE 1=1"
args: list[str | bool] = []
if sanitized_artist_filter:
sanitized_artists: list[str] = [sanitized_artist_filter]
for alias in c.sanitized_artist_aliases_map.get(sanitized_artist_filter, []):
sanitized_artists.append(alias)
if artist_filter:
artists: list[str] = [artist_filter]
for alias in c.artist_aliases_map.get(artist_filter, []):
artists.append(alias)
query += f"""
AND EXISTS (
SELECT * FROM releases_artists
WHERE release_id = id AND artist_sanitized IN ({','.join(['?']*len(sanitized_artists))})
WHERE release_id = id AND artist IN ({','.join(['?']*len(artists))})
)
"""
args.extend(sanitized_artists)
if sanitized_genre_filter:
# TODO(NOW): Umm.. sanitized to not sanitized?
args.extend(artists)
if genre_filter:
query += """
AND EXISTS (
SELECT * FROM releases_genres
WHERE release_id = id AND genre_sanitized = ?
WHERE release_id = id AND genre = ?
)
"""
args.append(sanitized_genre_filter)
if sanitized_label_filter:
args.append(genre_filter)
if label_filter:
query += """
AND EXISTS (
SELECT * FROM releases_labels
WHERE release_id = id AND label_sanitized = ?
WHERE release_id = id AND label = ?
)
"""
args.append(sanitized_label_filter)
args.append(label_filter)
if new is not None:
query += " AND new = ?"
args.append(new)
Expand Down Expand Up @@ -2100,55 +2091,55 @@ def collage_exists(c: Config, collage_name: str) -> bool:
return bool(cursor.fetchone()[0])


def list_artists(c: Config) -> list[tuple[str, str]]:
def list_artists(c: Config) -> list[str]:
with connect(c) as conn:
cursor = conn.execute("SELECT DISTINCT artist, artist_sanitized FROM releases_artists")
return [(row["artist"], row["artist_sanitized"]) for row in cursor]
cursor = conn.execute("SELECT DISTINCT artist FROM releases_artists")
return [row["artist"] for row in cursor]


def artist_exists(c: Config, artist_sanitized: str) -> bool:
args: list[str] = [artist_sanitized]
for alias in c.sanitized_artist_aliases_map.get(artist_sanitized, []):
def artist_exists(c: Config, artist: str) -> bool:
args: list[str] = [artist]
for alias in c.artist_aliases_map.get(artist, []):
args.append(alias)
with connect(c) as conn:
cursor = conn.execute(
f"""
SELECT EXISTS(
SELECT * FROM releases_artists
WHERE artist_sanitized IN ({','.join(['?']*len(args))})
WHERE artist IN ({','.join(['?']*len(args))})
)
""",
args,
)
return bool(cursor.fetchone()[0])


def list_genres(c: Config) -> list[tuple[str, str]]:
def list_genres(c: Config) -> list[str]:
with connect(c) as conn:
cursor = conn.execute("SELECT DISTINCT genre, genre_sanitized FROM releases_genres")
return [(row["genre"], row["genre_sanitized"]) for row in cursor]
cursor = conn.execute("SELECT DISTINCT genre FROM releases_genres")
return [row["genre"] for row in cursor]


def genre_exists(c: Config, genre_sanitized: str) -> bool:
def genre_exists(c: Config, genre: str) -> bool:
with connect(c) as conn:
cursor = conn.execute(
"SELECT EXISTS(SELECT * FROM releases_genres WHERE genre_sanitized = ?)",
(genre_sanitized,),
"SELECT EXISTS(SELECT * FROM releases_genres WHERE genre = ?)",
(genre,),
)
return bool(cursor.fetchone()[0])


def list_labels(c: Config) -> list[tuple[str, str]]:
def list_labels(c: Config) -> list[str]:
with connect(c) as conn:
cursor = conn.execute("SELECT DISTINCT label, label_sanitized FROM releases_labels")
return [(row["label"], row["label_sanitized"]) for row in cursor]
cursor = conn.execute("SELECT DISTINCT label, label FROM releases_labels")
return [row["label"] for row in cursor]


def label_exists(c: Config, label_sanitized: str) -> bool:
def label_exists(c: Config, label: str) -> bool:
with connect(c) as conn:
cursor = conn.execute(
"SELECT EXISTS(SELECT * FROM releases_labels WHERE label_sanitized = ?)",
(label_sanitized,),
"SELECT EXISTS(SELECT * FROM releases_labels WHERE label = ?)",
(label,),
)
return bool(cursor.fetchone()[0])

Expand Down
8 changes: 0 additions & 8 deletions rose/cache.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,26 +29,22 @@ CREATE INDEX releases_new ON releases(new);
CREATE TABLE releases_genres (
release_id TEXT REFERENCES releases(id) ON DELETE CASCADE,
genre TEXT,
genre_sanitized TEXT NOT NULL,
position INTEGER NOT NULL,
PRIMARY KEY (release_id, genre),
UNIQUE (release_id, position)
);
CREATE INDEX releases_genres_release_id_position ON releases_genres(release_id, position);
CREATE INDEX releases_genres_genre ON releases_genres(genre);
CREATE INDEX releases_genres_genre_sanitized ON releases_genres(genre_sanitized);

CREATE TABLE releases_labels (
release_id TEXT REFERENCES releases(id) ON DELETE CASCADE,
label TEXT,
label_sanitized TEXT NOT NULL,
position INTEGER NOT NULL,
PRIMARY KEY (release_id, label),
UNIQUE (release_id, position)
);
CREATE INDEX releases_labels_release_id_position ON releases_labels(release_id, position);
CREATE INDEX releases_labels_label ON releases_labels(label);
CREATE INDEX releases_labels_label_sanitized ON releases_labels(label_sanitized);

CREATE TABLE tracks (
id TEXT PRIMARY KEY,
Expand Down Expand Up @@ -85,28 +81,24 @@ INSERT INTO artist_role_enum (value) VALUES
CREATE TABLE releases_artists (
release_id TEXT REFERENCES releases(id) ON DELETE CASCADE,
artist TEXT,
artist_sanitized TEXT NOT NULL,
role TEXT REFERENCES artist_role_enum(value) NOT NULL,
position INTEGER NOT NULL,
PRIMARY KEY (release_id, artist, role)
UNIQUE (release_id, position)
);
CREATE INDEX releases_artists_release_id_position ON releases_artists(release_id, position);
CREATE INDEX releases_artists_artist ON releases_artists(artist);
CREATE INDEX releases_artists_artist_sanitized ON releases_artists(artist_sanitized);

CREATE TABLE tracks_artists (
track_id TEXT REFERENCES tracks(id) ON DELETE CASCADE,
artist TEXT,
artist_sanitized TEXT NOT NULL,
role TEXT REFERENCES artist_role_enum(value) NOT NULL,
position INTEGER NOT NULL,
PRIMARY KEY (track_id, artist, role),
UNIQUE (track_id, position)
);
CREATE INDEX tracks_artists_track_id_position ON tracks_artists(track_id, position);
CREATE INDEX tracks_artists_artist ON tracks_artists(artist);
CREATE INDEX tracks_artists_artist_sanitized ON tracks_artists(artist_sanitized);

CREATE TABLE collages (
name TEXT PRIMARY KEY,
Expand Down
32 changes: 16 additions & 16 deletions rose/cache_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -458,26 +458,26 @@ def test_update_cache_releases_evicts_relations(config: Config) -> None:
with connect(config) as conn:
conn.execute(
"""
INSERT INTO releases_genres (release_id, genre, genre_sanitized, position)
VALUES ('ilovecarly', 'lalala', 'lalala', 2)
INSERT INTO releases_genres (release_id, genre, position)
VALUES ('ilovecarly', 'lalala', 2)
""",
)
conn.execute(
"""
INSERT INTO releases_labels (release_id, label, label_sanitized, position)
VALUES ('ilovecarly', 'lalala', 'lalala', 1)
INSERT INTO releases_labels (release_id, label, position)
VALUES ('ilovecarly', 'lalala', 1)
""",
)
conn.execute(
"""
INSERT INTO releases_artists (release_id, artist, artist_sanitized, role, position)
VALUES ('ilovecarly', 'lalala', 'lalala', 'main', 1)
INSERT INTO releases_artists (release_id, artist, role, position)
VALUES ('ilovecarly', 'lalala', 'main', 1)
""",
)
conn.execute(
"""
INSERT INTO tracks_artists (track_id, artist, artist_sanitized, role, position)
SELECT id, 'lalala', 'lalala', 'main', 1 FROM tracks
INSERT INTO tracks_artists (track_id, artist, role, position)
SELECT id, 'lalala', 'main', 1 FROM tracks
""",
)
# Second cache refresh.
Expand Down Expand Up @@ -1410,27 +1410,27 @@ def test_get_track_logtext(config: Config) -> None:
def test_list_artists(config: Config) -> None:
artists = list_artists(config)
assert set(artists) == {
("Techno Man", "Techno Man"),
("Bass Man", "Bass Man"),
("Violin Woman", "Violin Woman"),
("Conductor Woman", "Conductor Woman"),
"Techno Man",
"Bass Man",
"Violin Woman",
"Conductor Woman",
}


@pytest.mark.usefixtures("seeded_cache")
def test_list_genres(config: Config) -> None:
genres = list_genres(config)
assert set(genres) == {
("Techno", "Techno"),
("Deep House", "Deep House"),
("Classical", "Classical"),
"Techno",
"Deep House",
"Classical",
}


@pytest.mark.usefixtures("seeded_cache")
def test_list_labels(config: Config) -> None:
labels = list_labels(config)
assert set(labels) == {("Silk Music", "Silk Music"), ("Native State", "Native State")}
assert set(labels) == {"Silk Music", "Native State"}


@pytest.mark.usefixtures("seeded_cache")
Expand Down
Loading

0 comments on commit 067d87c

Please sign in to comment.