From ec44b7ddc6f81a467feff34236b38e2a42330e8a Mon Sep 17 00:00:00 2001 From: blissful Date: Thu, 26 Oct 2023 21:14:56 -0400 Subject: [PATCH] metadata mgmt docs --- README.md | 48 ++++++-- docs/ARCHITECTURE.md | 6 +- docs/CACHE_MAINTENANCE.md | 2 +- docs/CONFIGURATION.md | 4 +- docs/METADATA_MANAGEMENT.md | 240 ++++++++++++++++++++++++++++++++---- docs/PLAYLISTS_COLLAGES.md | 4 +- docs/VIRTUAL_FILESYSTEM.md | 10 +- rose/cache.py | 1 + rose/releases.py | 4 +- rose/tagger.py | 8 +- 10 files changed, 278 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 4e39e51..4081214 100644 --- a/README.md +++ b/README.md @@ -141,15 +141,14 @@ This section contains a concise list of Rosé's features. - Virtual Filesystem - Read audio files and cover art - Modify files and cover art - - Filter releases by album artist, genre, label, and "new"-ness. + - Filter releases by album artist, genre, label, and "new"-ness - Browse and edit collages and playlists - Group artist aliases together - Toggle release "new"-ness - Whitelist/blacklist entries in the artist, genre, and label views - Command Line - - Edit release metadata as a text file. - - Import metadata and cover art from third-party sources: Discogs, - MusicBrainz, Tidal, Deezer, Apple, Junodownload, Beatport, and fanart.tv + - Edit release metadata as a text file + - Import metadata and cover art from third-party sources - Extract embedded cover art to a file - Automatically update metadata via patterns and rules - Collage and playlist management @@ -266,13 +265,34 @@ finally (3) play music! to the configured `fuse_mount_dir`, and you should see your music available in the virtual filesystem! + ```bash + $ cd $fuse_mount_dir + + $ ls -1 + '1. Releases' + '2. Releases - New' + '3. Releases - Recently Added' + '4. Artists' + '5. Genres' + '6. Labels' + '7. Collages' + '8. Playlists' + + $ ls -1 "1. Releases/" + 'BLACKPINK - 2016. SQUARE ONE - Single [K-Pop] {YG Entertainment}' + 'BLACKPINK - 2016. SQUARE TWO - Single [K-Pop] {YG Entertainment}' + 'LOOΠΔ ODD EYE CIRCLE - 2017. Mix & Match - EP [K-Pop] {BlockBerry Creative}' + 'YUZION - 2019. Young Trapper [Hip Hop]' + '{NEW} LOOΠΔ - 2017. Kim Lip - Single [K-Pop]' + ``` + 3. Let's play some music! You should be able to open a music file in your music player of choice. Mine is `mpv`: ```bash - $ mpv "1. Releases/LOOΠΔ ODD EYE CIRCLE - 2017. Mix & Match - EP [K-Pop] {BlockBerry Creative}/04. LOOΠΔ ODD EYE CIRCLE - Chaotic.opus" + $ mpv "1. Releases/LOOΠΔ ODD EYE CIRCLE - 2017. Mix & Match - EP [K-Pop] {BlockBerry Creative}/04. LOOΠΔ ODD EYE CIRCLE - Chaotic.opus" (+) Audio --aid=1 'Chaotic' (opus 2ch 48000Hz) File tags: Artist: LOOΠΔ ODD EYE CIRCLE @@ -301,6 +321,10 @@ We recommend using Rosé with: [mc](https://midnight-commander.org/), and [ranger](https://github.com/ranger/ranger). 2. A media player, such as [mpv](https://mpv.io/). +You also need not use the complete feature set of Rosé. Everything will +continue to work if you only use the virtual filesystem and ignore the +metatdata tooling, and vice versa. + # Learn More For additional documentation, please read the following files: @@ -332,12 +356,14 @@ limitations under the License. # Contributions +Bug fixes are happily accepted! + +However, please do not open a pull request for a new feature without prior +discussion. + Rosé is a pet project that I developed for personal use. Rosé is designed to match my specific needs and constraints, and is never destined to be widely -adopted. - -Bug fix contributions are happily accepted! However, please do not open a pull -request for a new feature without prior discussion. +adopted. Therefore, I will lean towards keeping the feature set focused and +small, and will not add too many features over the lifetime of the project. -Rose is provided as-is: there are no promises of future maintenance or feature -development. +Rosé is provided as-is: I may not maintain it in the future. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index e52a764..c02c686 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -46,7 +46,7 @@ This has some nice consequences: because conflict resolution is then ambiguous. Whereas in Rosé, there are no conflicts. -## Stable Release & Track Identifiers +# Stable Release & Track Identifiers Rosé assigns UUIDs to each release and track in order to identify them across arbitrarily large metadata changes. These UUIDs are persisted to the source @@ -59,7 +59,7 @@ files. - Each track has a custom `roseid` tag. This tag is written to the source audio file. Read the `tagger.py` file for the exact field name used. -## Read Cache Update +# Read Cache Update The read cache update is optimized to minimize the number of disk accesses, as it's a hot path and quite expensive if implemented poorly. @@ -74,7 +74,7 @@ queries to batch the writes. The update process is also parallelizable, so we shard workloads across multiple processes. -## Logging +# Logging Logs are written to stderr and to `${XDG_STATE_HOME:-$HOME/.local/state}/rose/rose.log`. Debug logging can be turned on with the `--verbose/-v` option. Rosé is heavily diff --git a/docs/CACHE_MAINTENANCE.md b/docs/CACHE_MAINTENANCE.md index 6cc0957..9892870 100644 --- a/docs/CACHE_MAINTENANCE.md +++ b/docs/CACHE_MAINTENANCE.md @@ -36,7 +36,7 @@ cache update --force`. It would be pretty annoying if you had to run this command by hand after each metadata update. So Rosé will automatically run this command whenever an update -happens _through_ Rosé. That means: +happens _through_ Rosé. That means: - If a file is modified in the virtual filesystem, a cache update is triggered when the file handle closes. diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index dceff46..9294025 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -58,7 +58,7 @@ fuse_labels_whitelist = [ "xxx", "yyy" ] # These options remove specific entities from their respective top-level # virtual filesystem directories. This is useful if there are a few values you # don't find useful, e.g. a random featuring artist or one super niche genre. -# +# # These options are mutually exclusive with the fuse_*_whitelist options; if # both are specified for a given entity type, the configuration will not # validate. @@ -77,7 +77,7 @@ max_proc = 4 ``` -## Systemd Unit Files +# Systemd If you want Rosé to always be on, you can configure systemd to manage Rosé. Systemd can ensure that Rosé starts on boot and restarts on failure. diff --git a/docs/METADATA_MANAGEMENT.md b/docs/METADATA_MANAGEMENT.md index 265af35..04457f5 100644 --- a/docs/METADATA_MANAGEMENT.md +++ b/docs/METADATA_MANAGEMENT.md @@ -1,24 +1,115 @@ # Managing Your Music Metadata -TODO Intro: motivations, reasons, fields we care about - -## Tag Conventions - -Rosé is lenient in the tags it ingests, but has opinionated conventions for the -tags it writes. Impl detail: how exactly does rose edit the files? - -### Field Mappings - -TODO - -### Multi-Valued Tags +Rosé relies on the metadata embedded in your music files to organize your music +into a useful virtual filesystem. This means that the quality of the music tags +is important for getting the most out of Rosé. + +Therefore, Rosé also provides tools to improve the metadata of your music. +Currently, Rosé provides: + +- A text-based interface for manually modifying release metadata. +- Metadata importing from third-party sources. +- Rules engine to automatically update metadata based on patterns + +In this document, we'll first cover the conventions that Rosé expects and +applies towards tags, and then go through each of the the functionalities +listed above. + +# Tagging Conventions + +This section describes how Rosé reads and writes tags from files. Rosé applies +fairly rigid conventions in the tags it writes, and applies a relaxed version +of those conventions when ingesting tags from audio files. + +## Managed Tags + +Rosé manages the following tags: + +- Release Tags: + - Title + - Album Artists + - Release Year + - Release Type (e.g. Album, EP, Single) + - Genre + - Label +- Track Tags: + - Title + - Artists + - Track Number + - Disc Number + - Rosé ID + +Rosé does not care about any other tags and does not do anything with them. + +## Field Mappings + +Rosé supports three tag container formats: + +- ID3: `.mp3` files +- MP4: `.m4a` files +- Vorbis:`.ogg`, `.opus`, and `.flac` files + +In this section, we will list out the per-container fields that we read/write. +Rosé will only write to a single field for each tag; however, for tags with +multiple conventions out in the rest of the world, Rosé will support reading +from additional fields. + +### MP3 + +| Tag | Field Name | Will Ingest These Fields | +| ------------- | ------------------ | ---------------------------------------------------------------------------------------------------------------------- | +| Release Title | `TALB` | | +| Album Artists | `TPE2` | | +| Release Year | `TDRC` | `TYER` | +| Release Type | `TXXX:RELEASETYPE` | | +| Genre | `TCON` | | +| Label | `TPUB` | | +| Track Title | `TIT2` | | +| Track Artists | `TPE1` | `TPE4` (Remixer), `TCOM` (Composer), `TPE3` (Conductor), `TIPL,IPLS/producer` (producer), `TIPL,IPLS/DJ-mix` (djmixer) | +| Track Number | `TRCK` | | +| Disc Number | `TPOS` | | +| Rose ID | `TXXX:ROSEID` | | + +### MP4 + +| Tag | Field Name | Will Ingest These Fields | +| ------------- | ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Release Title | `\xa9alb` | | +| Album Artists | `aART` | | +| Release Year | `\xa9day` | | +| Release Type | `----:com.apple.iTunes:RELEASETYPE` | | +| Genre | `\xa9gen` | | +| Label | `----:com.apple.iTunes:LABEL` | | +| Track Title | `\xa9nam` | | +| Track Artists | `\xa9ART` | `----:com.apple.iTunes:REMIXER` (Remixer), `\xa9wrt` (Composer), `----:com.apple.iTunes:CONDUCTOR` (Conductor), `----:com.apple.iTunes:PRODUCER` (producer), `----:com.apple.iTunes:DJMIXER` (djmixer) | +| Track Number | `trkn` | | +| Disc Number | `disk` | | +| Rose ID | `----:net.sunsetglow.rose:ID` | | + +### Vorbis + +| Tag | Field Name | Will Ingest These Fields | +| ------------- | -------------- | --------------------------------------------------------------------------------------------------------------- | +| Release Title | `album` | | +| Album Artists | `albumartist` | | +| Release Year | `date` | `year` | +| Release Type | `releasetype` | | +| Genre | `genre` | | +| Label | `organization` | `label`, `recordlabel` | +| Track Title | `title` | | +| Track Artists | `artist` | `remixer` (Remixer), `composer` (Composer), `conductor` (Conductor), `producer` (producer), `djmixer` (djmixer) | +| Track Number | `tracknumber` | | +| Disc Number | `discnumber` | | +| Rose ID | `roseid` | | + +## Multi-Valued Tags Rosé supports multiple values for the artists, genres, and labels tags. Rosé writes a single tag field and with fields concatenated together with a `;` delimiter. For example, `genre=Deep House;Techno`. Rosé does not write one tag per frame due to inconsistent support by other useful programs. -### Artist Tags +## Artist Tags Rosé preserves the artists' role in the artist tag by using specialized delimiters. An example artist tag is: `Pyotr Ilyich Tchaikovsky performed by André Previn;London Symphony Orchestra feat. Barack Obama`. @@ -26,9 +117,9 @@ delimiters. An example artist tag is: `Pyotr Ilyich Tchaikovsky performed by And The artist tag is described by the following grammar: ``` - ::=
+ ::=
::= ' performed by ' - ::= ' pres. ' + ::= ' pres. '
::= ::= ' feat. ' ::= ' remixed by ' @@ -36,14 +127,121 @@ The artist tag is described by the following grammar: ::= string ';' | string ``` -## Manual Editing +Rosé only supports the artist roles: + +- `main` +- `guest` +- `producer` +- `composer` +- `conductor` +- `djmixer` + +Rosé writes a single tag value into the _Track Artists_ and _Album Artists_ +tags. Though some conventions exist for writing each role into its own tag, +Rosé does not follow them, due to inconsistent (mainly nonexistent) support by +other useful programs. + +## Release Type Tags + +Rosé supports tagging the release _type_. The supported values are: + +- `album` +- `single` +- `ep` +- `compilation` +- `soundtrack` +- `live` +- `remix` +- `djmix` +- `mixtape` +- `other` +- `bootleg` +- `demo` +- `unknown` + +# Text-Based Release Editing + +Rosé supports editing a release's metadata as a text file via the +`rose releases edit` command. This command accepts a Release ID or a Release's +Virtual Filesystem Directory Name. + +So for example: + +```bash +$ rose releases edit "LOOΠΔ ODD EYE CIRCLE - 2017. Mix & Match - EP [K-Pop] {BlockBerry Creative}" +$ rose releases edit "018b4ff1-acdf-7ff1-bcd6-67757aea0fed" +``` + +This command opens up a TOML representation of the release's metadata in your +`$EDITOR`. Upon save and exit, the TOML's metadata is written to the file tags. + +Rosé validates the Artist Role and Release Type fields. The values provided +must be one of the supported values. The supported values are documented in +[Artist Tags](#artist-tags) and [Release Type Tags](#release-type-tags). + +An example of the TOML representation is: + +```toml +title = "Mix & Match" +releasetype = "ep" +year = 2017 +genres = [ + "K-Pop", +] +labels = [ + "BlockBerry Creative", +] +artists = [ + { name = "LOOΠΔ ODD EYE CIRCLE", role = "main" }, +] + +[tracks.018b6514-6fb8-729f-bf86-7590187ff377] +disc_number = "1" +track_number = "1" +title = "ODD" +artists = [ + { name = "LOOΠΔ ODD EYE CIRCLE", role = "main" }, +] + +[tracks.018b6514-6fba-7508-8576-c8e82ad4b7bc] +disc_number = "1" +track_number = "2" +title = "Girl Front" +artists = [ + { name = "LOOΠΔ ODD EYE CIRCLE", role = "main" }, +] + +[tracks.018b6514-6fb9-73f1-a139-18ecefcf55da] +disc_number = "1" +track_number = "3" +title = "LOONATIC" +artists = [ + { name = "LOOΠΔ ODD EYE CIRCLE", role = "main" }, +] + +[tracks.018b6514-6fb7-7cc6-9d23-8eaf0b1beee8] +disc_number = "1" +track_number = "4" +title = "Chaotic" +artists = [ + { name = "LOOΠΔ ODD EYE CIRCLE", role = "main" }, +] + +[tracks.018b6514-6fb6-766f-8430-c6ea3f48966d] +disc_number = "1" +track_number = "5" +title = "Starlight" +artists = [ + { name = "LOOΠΔ ODD EYE CIRCLE", role = "main" }, +] +``` -TODO: Text file editing +# Metadata Import & Cover Art Downloading -## Metadata Import & Cover Downloading +In Development -TBD +Sources: Discogs, MusicBrainz, Tidal, Deezer, Apple, Junodownload, Beatport, and fanart.tv -## Rules Engine +# Rules Engine -TBD +In Development diff --git a/docs/PLAYLISTS_COLLAGES.md b/docs/PLAYLISTS_COLLAGES.md index 1c66db5..59bd7dd 100644 --- a/docs/PLAYLISTS_COLLAGES.md +++ b/docs/PLAYLISTS_COLLAGES.md @@ -7,7 +7,7 @@ As Rosé implements playlists and collages in almost the same way, except that one tracks releases and the other tracks tracks, we discuss both collages and playlists together. -## Storage Format +# Storage Format Collages and playlists are stored on-disk in the source directory, in the `!collages` and `!playlists` directories, respectively. Each collage and @@ -57,6 +57,8 @@ update, so that they remain meaningful. The ordering of the releases/tracks is meaningful: they represent the ordering of releases/tracks in the collage/playlist. +# Operations + However, working with this file directly is quite annoying, so Rosé allows you to manage collages and playlists via the command line and the virtual filesystem. In the rest of this document, we'll demonstrate the basic diff --git a/docs/VIRTUAL_FILESYSTEM.md b/docs/VIRTUAL_FILESYSTEM.md index 5f3f7e5..dea51a0 100644 --- a/docs/VIRTUAL_FILESYSTEM.md +++ b/docs/VIRTUAL_FILESYSTEM.md @@ -2,17 +2,17 @@ TODO: Intro?? -## Mounting & Unmounting +# Mounting & Unmounting troubleshooting... edge cases -## Directory Structure +# Directory Structure -## Directory and File Names +# Directory and File Names -## New Releases +# New Releases -## Operations +# Operations > [!NOTE] > Operations on collages and playlists are documented in diff --git a/rose/cache.py b/rose/cache.py index 62b8645..b76290b 100644 --- a/rose/cache.py +++ b/rose/cache.py @@ -213,6 +213,7 @@ class StoredDataFile: "djmix": "DJ-Mix", "mixtape": "Mixtape", "other": "Other", + "demo": "Demo", "unknown": "Unknown", } diff --git a/rose/releases.py b/rose/releases.py index 0aaa700..69a50be 100644 --- a/rose/releases.py +++ b/rose/releases.py @@ -101,7 +101,7 @@ def to_mapping(artists: list[MetadataArtist]) -> ArtistMapping: m = ArtistMapping() for a in artists: try: - getattr(m, a.role).append(a.name) + getattr(m, a.role.lower()).append(a.name) except AttributeError as e: raise UnknownArtistRoleError( f"Failed to write tags: Unknown role for artist {a.name}: {a.role}" @@ -226,7 +226,7 @@ def edit_release(c: Config, release_id_or_virtual_dirname: str) -> None: dirty = True logger.debug(f"Modified tag detected for {t.source_path}: album") if tags.release_type != release_meta.releasetype: - tags.release_type = release_meta.releasetype + tags.release_type = release_meta.releasetype.lower() dirty = True logger.debug(f"Modified tag detected for {t.source_path}: release_type") if tags.year != release_meta.year: diff --git a/rose/tagger.py b/rose/tagger.py index 1d0238d..2413e3a 100644 --- a/rose/tagger.py +++ b/rose/tagger.py @@ -40,8 +40,9 @@ "remix", "djmix", "mixtape", - "other", "bootleg", + "demo", + "other", "unknown", ] @@ -184,6 +185,7 @@ def flush(self, *, validate: bool = True) -> None: if not validate and "pytest" not in sys.modules: raise Exception("Validate can only be turned off by tests.") + self.release_type = (self.release_type or "unknown").lower() if validate and self.release_type not in SUPPORTED_RELEASE_TYPES: raise UnsupportedTagValueTypeError( f"Release type {self.release_type} is not a supported release type.\n" @@ -242,7 +244,7 @@ def _write_tag_with_description(name: str, value: str | None) -> None: m.tags["\xa9alb"] = self.album or "" m.tags["\xa9gen"] = ";".join(self.genre) m.tags["----:com.apple.iTunes:LABEL"] = ";".join(self.label).encode() - m.tags["----:com.apple.iTunes:RELEASETYPE"] = (self.release_type or "").encode() + m.tags["----:com.apple.iTunes:RELEASETYPE"] = self.release_type.encode() m.tags["aART"] = format_artist_string(self.album_artists, self.genre) m.tags["\xa9ART"] = format_artist_string(self.artists, self.genre) # Wipe the alt. role artist tags, since we encode the full artist into the main tag. @@ -297,7 +299,7 @@ def _write_tag_with_description(name: str, value: str | None) -> None: m.tags["album"] = self.album or "" m.tags["genre"] = ";".join(self.genre) m.tags["organization"] = ";".join(self.label) - m.tags["releasetype"] = self.release_type or "" + m.tags["releasetype"] = self.release_type m.tags["albumartist"] = format_artist_string(self.album_artists, self.genre) m.tags["artist"] = format_artist_string(self.artists, self.genre) # Wipe the alt. role artist tags, since we encode the full artist into the main tag.