From 7f138e69c1190ba98ebf616cc2929595312fb02f Mon Sep 17 00:00:00 2001 From: Silverarmor <23619946+Silverarmor@users.noreply.github.com> Date: Mon, 8 Feb 2021 20:16:35 +1300 Subject: [PATCH] Dev to Master (#1171) * Run regressions on CI (#1158) Authored by @aklajnert * Use argparse instead of raw `sys.argv` (#1157) Authored by @aklajnert * FIxed error caused by songs with ':' in name (#1162) Authored by @MikhailZex * Fixed TypeError when downloading tracks (#1177) Authored by @xnetcat * Bump version to 3.3.2 * Lint with flake8, fix code to pass it (#1178) Authored by @aklajnert * Fixed flake8 (#1181) Authored by @aklajnert * Skip song if track is empty. (#1164) Authored by @bee395 Co-authored-by: Andrzej Klajnert Co-authored-by: Michael George Co-authored-by: Jakub <42355410+xnetcat@users.noreply.github.com> Co-authored-by: bee395 --- .github/workflows/spotify-downloader-ci.yml | 51 ++++++++ setup.cfg | 10 +- spotdl/__init__.py | 1 + spotdl/__main__.py | 107 +++++++++------- spotdl/download/__init__.py | 2 +- spotdl/download/downloader.py | 130 ++++++++++---------- spotdl/download/progressHandlers.py | 90 +++++++------- spotdl/search/__init__.py | 8 +- spotdl/search/provider.py | 8 +- spotdl/search/songObj.py | 66 +++++----- spotdl/search/spotifyClient.py | 46 ++++--- spotdl/search/utils.py | 44 ++++--- tests/regressions.py | 2 +- tests/test_entry_point.py | 8 +- tox.ini | 10 +- 15 files changed, 324 insertions(+), 259 deletions(-) diff --git a/.github/workflows/spotify-downloader-ci.yml b/.github/workflows/spotify-downloader-ci.yml index a67037cf0..4793a965b 100644 --- a/.github/workflows/spotify-downloader-ci.yml +++ b/.github/workflows/spotify-downloader-ci.yml @@ -29,3 +29,54 @@ jobs: tox env: PLATFORM: ${{ matrix.platform }} + + mypy: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install dependencies + run: | + python -m pip install tox + - name: Run mypy + run: | + tox -e mypy + + flake8: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install dependencies + run: | + python -m pip install tox + - name: Run flake8 + run: | + tox -e flake8 + + regressions: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install dependencies + run: | + sudo add-apt-repository ppa:jonathonf/ffmpeg-4 -y + sudo apt-get update + sudo apt install ffmpeg -y + python -m pip install -e .[test] + - name: Run tests + run: | + pytest tests/regressions.py diff --git a/setup.cfg b/setup.cfg index 5b2051835..4b38bfc17 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [metadata] -version = 3.3.1 +version = 3.3.2 name = spotdl url = https://github.com/spotDL/spotify-downloader @@ -39,14 +39,16 @@ python_requires = >=3.6 packages = find: [options.extras_require] -test = +test = pytest >= 6.0 pytest-mock >= 3.3.1 pytest-vcr >= 1.0.2 pyfakefs >= 4.3.0 pytest-cov >= 2.10.1 -dev = +dev = tox + mypy==0.790 + flake8==3.8.4 [options.entry_points] console_scripts= @@ -55,3 +57,5 @@ console_scripts= [mypy] ignore_missing_imports = True +[flake8] +max-line-length = 100 diff --git a/spotdl/__init__.py b/spotdl/__init__.py index 0e726177c..bff52acf5 100644 --- a/spotdl/__init__.py +++ b/spotdl/__init__.py @@ -1,6 +1,7 @@ from .__main__ import console_entry_point __all__ = [ + 'console_entry_point', 'search', 'download', ] diff --git a/spotdl/__main__.py b/spotdl/__main__.py index 8a1eaf72e..d1e88041a 100644 --- a/spotdl/__main__.py +++ b/spotdl/__main__.py @@ -1,42 +1,47 @@ #! Basic necessities to get the CLI running -from spotdl.search import spotifyClient -import sys - -#! Song Search from different start points -from spotdl.search.utils import get_playlist_tracks, get_album_tracks, search_for_song -from spotdl.search.songObj import SongObj +import argparse -#! The actual download stuff +# ! The actual download stuff from spotdl.download.downloader import DownloadManager +from spotdl.search import spotifyClient +from spotdl.search.songObj import SongObj +# ! Song Search from different start points +from spotdl.search.utils import get_playlist_tracks, get_album_tracks, search_for_song - -#! Usage is simple - call 'python __main__.py -#! Eg. -#! python __main__.py https://open.spotify.com/playlist/37i9dQZF1DWXhcuQw7KIeM?si=xubKHEBESM27RqGkqoXzgQ 'old gods of asgard Control' https://open.spotify.com/album/2YMWspDGtbDgYULXvVQFM6?si=gF5dOQm8QUSo-NdZVsFjAQ https://open.spotify.com/track/08mG3Y1vljYA6bvDt4Wqkj?si=SxezdxmlTx-CaVoucHmrUA -#! -#! Well, yeah its a pretty long example but, in theory, it should work like a charm. -#! -#! A '.spotdlTrackingFile' is automatically created with the name of the first song in the playlist/album or -#! the name of the song supplied. We don't really re re re-query YTM and SPotify as all relevant details are -#! stored to disk. -#! -#! Files are cleaned up on download failure. -#! -#! All songs are normalized to standard base volume. the soft ones are made louder, the loud ones, softer. -#! -#! The progress bar is synched across multiple-processes (4 processes as of now), getting the progress bar to -#! synch was an absolute pain, each process knows how much 'it' progressed, but the display has to be for the -#! overall progress so, yeah... that took time. -#! -#! spotdl will show you its true speed on longer download's - so make sure you try downloading a playlist. -#! -#! still yet to try and package this but, in theory, there should be no errors. -#! -#! - cheerio! (Michael) -#! -#! P.S. Tell me what you think. Up to your expectations? - -#! Script Help +# ! Usage is simple - call: +# 'python __main__.py +# ! Eg. +# ! python __main__.py +# ! https://open.spotify.com/playlist/37i9dQZF1DWXhcuQw7KIeM?si=xubKHEBESM27RqGkqoXzgQ +# ! 'old gods of asgard Control' +# ! https://open.spotify.com/album/2YMWspDGtbDgYULXvVQFM6?si=gF5dOQm8QUSo-NdZVsFjAQ +# ! https://open.spotify.com/track/08mG3Y1vljYA6bvDt4Wqkj?si=SxezdxmlTx-CaVoucHmrUA +# ! +# ! Well, yeah its a pretty long example but, in theory, it should work like a charm. +# ! +# ! A '.spotdlTrackingFile' is automatically created with the name of the first song in the +# ! playlist/album or the name of the song supplied. We don't really re re re-query YTM and Spotify +# ! as all relevant details are stored to disk. +# ! +# ! Files are cleaned up on download failure. +# ! +# ! All songs are normalized to standard base volume. the soft ones are made louder, +# ! the loud ones, softer. +# ! +# ! The progress bar is synched across multiple-processes (4 processes as of now), getting the +# ! progress bar to synch was an absolute pain, each process knows how much 'it' progressed, +# ! but the display has to be for the overall progress so, yeah... that took time. +# ! +# ! spotdl will show you its true speed on longer download's - so make sure you try +# ! downloading a playlist. +# ! +# ! still yet to try and package this but, in theory, there should be no errors. +# ! +# ! - cheerio! (Michael) +# ! +# ! P.S. Tell me what you think. Up to your expectations? + +# ! Script Help help_notice = ''' To download a song run, spotdl [trackUrl] @@ -63,36 +68,34 @@ You can queue up multiple download tasks by separating the arguments with spaces: spotdl [songQuery1] [albumUrl] [songQuery2] ... (order does not matter) - ex. spotdl 'The Weeknd - Blinding Lights' https://open.spotify.com/playlist/37i9dQZF1E8UXBoz02kGID?si=oGd5ctlyQ0qblj_bL6WWow ... + ex. spotdl 'The Weeknd - Blinding Lights' + https://open.spotify.com/playlist/37i9dQZF1E8UXBoz02kGID?si=oGd5ctlyQ0qblj_bL6WWow ... -spotDL downloads up to 4 songs in parallel, so for a faster experience, download albums and playlist, rather than tracks. +spotDL downloads up to 4 songs in parallel, so for a faster experience, +download albums and playlist, rather than tracks. ''' + def console_entry_point(): ''' This is where all the console processing magic happens. Its super simple, rudimentary even but, it's dead simple & it works. ''' - - if '--help' in sys.argv or '-H' in sys.argv or '-h' in sys.argv or len(sys.argv) == 1: - print(help_notice) - - #! We use 'return None' as a convenient exit/break from the function - return None + arguments = parse_arguments() spotifyClient.initialize( clientId='4fe3fecfe5334023a1472516cc99d805', clientSecret='0f02b7c483c04257984695007a4a8d5c' - ) + ) downloader = DownloadManager() - for request in sys.argv[1:]: + for request in arguments.url: if 'open.spotify.com' in request and 'track' in request: print('Fetching Song...') song = SongObj.from_url(request) - if song.get_youtube_link() != None: + if song.get_youtube_link() is not None: downloader.download_single_song(song) else: print('Skipping %s (%s) as no match could be found on youtube' % ( @@ -126,5 +129,17 @@ def console_entry_point(): downloader.close() + +def parse_arguments(): + parser = argparse.ArgumentParser( + prog="spotdl", + description=help_notice, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("url", type=str, nargs="+") + + return parser.parse_args() + + if __name__ == '__main__': console_entry_point() diff --git a/spotdl/download/__init__.py b/spotdl/download/__init__.py index 3ebd22bf6..c789ddbcd 100644 --- a/spotdl/download/__init__.py +++ b/spotdl/download/__init__.py @@ -1,4 +1,4 @@ __all__ = [ 'progressHandlers', 'downloader' -] \ No newline at end of file +] diff --git a/spotdl/download/downloader.py b/spotdl/download/downloader.py index aed24144b..bcbd06315 100644 --- a/spotdl/download/downloader.py +++ b/spotdl/download/downloader.py @@ -4,22 +4,17 @@ import asyncio import concurrent.futures import sys - -import typing from pathlib import Path - -from pytube import YouTube +# ! The following are not used, they are just here for static typechecking with mypy +from typing import List +from urllib.request import urlopen from mutagen.easyid3 import EasyID3, ID3 from mutagen.id3 import APIC as AlbumCover +from pytube import YouTube -from urllib.request import urlopen - -#! The following are not used, they are just here for static typechecking with mypy -from typing import List - -from spotdl.search.songObj import SongObj from spotdl.download.progressHandlers import DisplayManager, DownloadTracker +from spotdl.search.songObj import SongObj # ========================== @@ -32,7 +27,7 @@ # =========================================================== class DownloadManager(): - #! Big pool sizes on slow connections will lead to more incomplete downloads + # ! Big pool sizes on slow connections will lead to more incomplete downloads poolSize = 4 def __init__(self): @@ -44,15 +39,15 @@ def __init__(self): self.displayManager.clear() if sys.platform == "win32": - #! ProactorEventLoop is required on Windows to run subprocess asynchronously - #! it is default since Python 3.8 but has to be changed for previous versions + # ! ProactorEventLoop is required on Windows to run subprocess asynchronously + # ! it is default since Python 3.8 but has to be changed for previous versions loop = asyncio.ProactorEventLoop() asyncio.set_event_loop(loop) self.loop = asyncio.get_event_loop() - #! semaphore is required to limit concurrent asyncio executions + # ! semaphore is required to limit concurrent asyncio executions self.semaphore = asyncio.Semaphore(self.poolSize) - #! thread pool executor is used to run blocking (CPU-bound) code from a thread + # ! thread pool executor is used to run blocking (CPU-bound) code from a thread self.thread_executor = concurrent.futures.ThreadPoolExecutor( max_workers=self.poolSize) @@ -118,11 +113,11 @@ async def download_song(self, songObj: SongObj) -> None: Downloads, Converts, Normalizes song & embeds metadata as ID3 tags. ''' - #! all YouTube downloads are to .\Temp; they are then converted and put into .\ and - #! finally followed up with ID3 metadata tags + # ! all YouTube downloads are to .\Temp; they are then converted and put into .\ and + # ! finally followed up with ID3 metadata tags - #! we explicitly use the os.path.join function here to ensure download is - #! platform agnostic + # ! we explicitly use the os.path.join function here to ensure download is + # ! platform agnostic # Create a .\Temp folder if not present tempFolder = Path('.', 'Temp') @@ -133,24 +128,24 @@ async def download_song(self, songObj: SongObj) -> None: # build file name of converted file artistStr = '' - #! we eliminate contributing artist names that are also in the song name, else we - #! would end up with things like 'Jetta, Mastubs - I'd love to change the world - #! (Mastubs REMIX).mp3' which is kinda an odd file name. + # ! we eliminate contributing artist names that are also in the song name, else we + # ! would end up with things like 'Jetta, Mastubs - I'd love to change the world + # ! (Mastubs REMIX).mp3' which is kinda an odd file name. for artist in songObj.get_contributing_artists(): if artist.lower() not in songObj.get_song_name().lower(): artistStr += artist + ', ' - #! the ...[:-2] is to avoid the last ', ' appended to artistStr + # ! the ...[:-2] is to avoid the last ', ' appended to artistStr convertedFileName = artistStr[:-2] + ' - ' + songObj.get_song_name() - #! this is windows specific (disallowed chars) + # ! this is windows specific (disallowed chars) for disallowedChar in ['/', '?', '\\', '*', '|', '<', '>']: if disallowedChar in convertedFileName: convertedFileName = convertedFileName.replace( disallowedChar, '') - #! double quotes (") and semi-colons (:) are also disallowed characters but we would - #! like to retain their equivalents, so they aren't removed in the prior loop + # ! double quotes (") and semi-colons (:) are also disallowed characters but we would + # ! like to retain their equivalents, so they aren't removed in the prior loop convertedFileName = convertedFileName.replace( '"', "'").replace(':', '-') @@ -163,8 +158,8 @@ async def download_song(self, songObj: SongObj) -> None: if self.downloadTracker: self.downloadTracker.notify_download_completion(songObj) - #! None is the default return value of all functions, we just explicitly define - #! it here as a continent way to avoid executing the rest of the function. + # ! None is the default return value of all functions, we just explicitly define + # ! it here as a continent way to avoid executing the rest of the function. return None # download Audio from YouTube @@ -194,33 +189,33 @@ async def download_song(self, songObj: SongObj) -> None: # convert downloaded file to MP3 with normalization - #! -af loudnorm=I=-7:LRA applies EBR 128 loudness normalization algorithm with - #! intergrated loudness target (I) set to -17, using values lower than -15 - #! causes 'pumping' i.e. rhythmic variation in loudness that should not - #! exist -loud parts exaggerate, soft parts left alone. - #! - #! dynaudnorm applies dynamic non-linear RMS based normalization, this is what - #! actually normalized the audio. The loudnorm filter just makes the apparent - #! loudness constant - #! - #! apad=pad_dur=2 adds 2 seconds of silence toward the end of the track, this is - #! done because the loudnorm filter clips/cuts/deletes the last 1-2 seconds on - #! occasion especially if the song is EDM-like, so we add a few extra seconds to - #! combat that. - #! - #! -acodec libmp3lame sets the encoded to 'libmp3lame' which is far better - #! than the default 'mp3_mf', '-abr true' automatically determines and passes the - #! audio encoding bitrate to the filters and encoder. This ensures that the - #! sampled length of songs matches the actual length (i.e. a 5 min song won't display - #! as 47 seconds long in your music player, yeah that was an issue earlier.) + # ! -af loudnorm=I=-7:LRA applies EBR 128 loudness normalization algorithm with + # ! intergrated loudness target (I) set to -17, using values lower than -15 + # ! causes 'pumping' i.e. rhythmic variation in loudness that should not + # ! exist -loud parts exaggerate, soft parts left alone. + # ! + # ! dynaudnorm applies dynamic non-linear RMS based normalization, this is what + # ! actually normalized the audio. The loudnorm filter just makes the apparent + # ! loudness constant + # ! + # ! apad=pad_dur=2 adds 2 seconds of silence toward the end of the track, this is + # ! done because the loudnorm filter clips/cuts/deletes the last 1-2 seconds on + # ! occasion especially if the song is EDM-like, so we add a few extra seconds to + # ! combat that. + # ! + # ! -acodec libmp3lame sets the encoded to 'libmp3lame' which is far better + # ! than the default 'mp3_mf', '-abr true' automatically determines and passes the + # ! audio encoding bitrate to the filters and encoder. This ensures that the + # ! sampled length of songs matches the actual length (i.e. a 5 min song won't display + # ! as 47 seconds long in your music player, yeah that was an issue earlier.) command = 'ffmpeg -v quiet -y -i "%s" -acodec libmp3lame -abr true ' \ - f'-b:a {trackAudioStream.bitrate} ' \ + f'-b:a {trackAudioStream.bitrate} ' \ '-af "apad=pad_dur=2, dynaudnorm, loudnorm=I=-17" "%s"' - #! bash/ffmpeg on Unix systems need to have excape char (\) for special characters: \$ - #! alternatively the quotes could be reversed (single <-> double) in the command then - #! the windows special characters needs escaping (^): ^\ ^& ^| ^> ^< ^^ + # ! bash/ffmpeg on Unix systems need to have excape char (\) for special characters: \$ + # ! alternatively the quotes could be reversed (single <-> double) in the command then + # ! the windows special characters needs escaping (^): ^\ ^& ^| ^> ^< ^^ if sys.platform == 'win32': formattedCommand = command % ( @@ -229,14 +224,14 @@ async def download_song(self, songObj: SongObj) -> None: ) else: formattedCommand = command % ( - str(downloadedFilePath).replace('$', '\$'), - str(convertedFilePath).replace('$', '\$') + str(downloadedFilePath).replace('$', r'\$'), + str(convertedFilePath).replace('$', r'\$') ) process = await asyncio.subprocess.create_subprocess_shell(formattedCommand) _ = await process.communicate() - #! Wait till converted file is actually created + # ! Wait till converted file is actually created while True: if convertedFilePath.is_file(): break @@ -308,10 +303,10 @@ def close(self) -> None: self.displayManager.close() async def _download_from_youtube(self, convertedFileName, tempFolder, trackAudioStream): - #! The following function calls blocking code, which would block whole event loop. - #! Therefore it has to be called in a separate thread via ThreadPoolExecutor. This - #! is not a problem, since GIL is released for the I/O operations, so it shouldn't - #! hurt performance. + # ! The following function calls blocking code, which would block whole event loop. + # ! Therefore it has to be called in a separate thread via ThreadPoolExecutor. This + # ! is not a problem, since GIL is released for the I/O operations, so it shouldn't + # ! hurt performance. return await self.loop.run_in_executor( self.thread_executor, self._perform_audio_download, @@ -321,28 +316,29 @@ async def _download_from_youtube(self, convertedFileName, tempFolder, trackAudio ) def _perform_audio_download(self, convertedFileName, tempFolder, trackAudioStream): - #! The actual download, if there is any error, it'll be here, + # ! The actual download, if there is any error, it'll be here, try: - #! pyTube will save the song in .\Temp\$songName.mp4 or .webm, it doesn't save as '.mp3' + # ! pyTube will save the song in .\Temp\$songName.mp4 or .webm, + # ! it doesn't save as '.mp3' downloadedFilePath = trackAudioStream.download( output_path=tempFolder, filename=convertedFileName, skip_existing=False ) return downloadedFilePath - except: - #! This is equivalent to a failed download, we do nothing, the song remains on - #! downloadTrackers download queue and all is well... - #! - #! None is again used as a convenient exit + except: # noqa:E722 + # ! This is equivalent to a failed download, we do nothing, the song remains on + # ! downloadTrackers download queue and all is well... + # ! + # ! None is again used as a convenient exit tempFiles = Path(tempFolder).glob(f'{convertedFileName}.*') for tempFile in tempFiles: tempFile.unlink() return None async def _pool_download(self, song_obj: SongObj): - #! Run asynchronous task in a pool to make sure that all processes - #! don't run at once. + # ! Run asynchronous task in a pool to make sure that all processes + # ! don't run at once. # tasks that cannot acquire semaphore will wait here until it's free # only certain amount of tasks can acquire the semaphore at the same time diff --git a/spotdl/download/progressHandlers.py b/spotdl/download/progressHandlers.py index 2dda5798e..24b39e655 100644 --- a/spotdl/download/progressHandlers.py +++ b/spotdl/download/progressHandlers.py @@ -1,20 +1,19 @@ -#=============== -#=== Imports === -#=============== +# =============== +# === Imports === +# =============== import typing from pathlib import Path +from typing import List from tqdm import tqdm -#! These are not used, they're here for static type checking using mypy +# ! These are not used, they're here for static type checking using mypy from spotdl.search.songObj import SongObj -from typing import List - -#======================= -#=== Display classes === -#======================= +# ======================= +# === Display classes === +# ======================= class SpecializedTQDM(tqdm): @@ -22,31 +21,32 @@ class SpecializedTQDM(tqdm): def format_dict(self): formatDict = super(SpecializedTQDM, self).format_dict - #! 1/rate is the time it takes to finish 1 iteration (secs). The displayManager - #! for which specializedTQDM is built works on the assumption that 100 iterations - #! makes one song downloaded. Hence the time in seconds per song would be - #! 100 * (1/rate) and in minuts would be 100/ (60 * rate) + # ! 1/rate is the time it takes to finish 1 iteration (secs). The displayManager + # ! for which specializedTQDM is built works on the assumption that 100 iterations + # ! makes one song downloaded. Hence the time in seconds per song would be + # ! 100 * (1/rate) and in minuts would be 100/ (60 * rate) if formatDict['rate']: - newRate = '{:.2f}'.format(100 / (60 * formatDict['rate'] )) + newRate = '{:.2f}'.format(100 / (60 * formatDict['rate'])) else: newRate = '~' - formatDict.update(rate_min = (newRate + 'min/' + formatDict['unit'])) + formatDict.update(rate_min=(newRate + 'min/' + formatDict['unit'])) - #! You can now use {rate_min} as a formatting arg to get rate in mins/unit + # ! You can now use {rate_min} as a formatting arg to get rate in mins/unit return formatDict + class DisplayManager(): def __init__(self): - #! specializedTQDM handles most of the display details, displayManager is an - #! additional bit of calculations to ensure that the specializedTQDM progressbar - #! works accurately across downloads from multiple processes + # ! specializedTQDM handles most of the display details, displayManager is an + # ! additional bit of calculations to ensure that the specializedTQDM progressbar + # ! works accurately across downloads from multiple processes self.progressBar = SpecializedTQDM( - total = 100, - dynamic_ncols = True, - bar_format = '{percentage:3.0f}%|{bar}|ETA: {remaining}, {rate_min}', - unit = 'song' + total=100, + dynamic_ncols=True, + bar_format='{percentage:3.0f}%|{bar}|ETA: {remaining}, {rate_min}', + unit='song' ) def set_song_count_to(self, songCount: int) -> None: @@ -59,8 +59,8 @@ def set_song_count_to(self, songCount: int) -> None: download set ''' - #! all calculations are based of the arbitrary choice that 1 song consists of - #! 100 steps/points/iterations + # ! all calculations are based of the arbitrary choice that 1 song consists of + # ! 100 steps/points/iterations self.progressBar.total = songCount * 100 def notify_download_skip(self) -> None: @@ -76,15 +76,15 @@ def pytube_progress_hook(self, stream, chunk, bytes_remaining) -> None: bytes are read from youtube. ''' - #! This will be called until download is complete, i.e we get an overall - #! self.progressBar.update(90) + # ! This will be called until download is complete, i.e we get an overall + # ! self.progressBar.update(90) fileSize = stream.filesize - #! How much of the file was downloaded this iteration scaled put of 90. - #! It's scaled to 90 because, the arbitrary division of each songs 100 - #! iterations is (a) 90 for download (b) 5 for conversion & normalization - #! and (c) 5 for ID3 tag embedding + # ! How much of the file was downloaded this iteration scaled put of 90. + # ! It's scaled to 90 because, the arbitrary division of each songs 100 + # ! iterations is (a) 90 for download (b) 5 for conversion & normalization + # ! and (c) 5 for ID3 tag embedding iterFraction = len(chunk) / fileSize * 90 self.progressBar.update(iterFraction) @@ -101,7 +101,7 @@ def notify_download_completion(self) -> None: updates progresbar to reflect a download being completed ''' - #! Download completion implie ID# tag embedding was just finished + # ! Download completion implie ID# tag embedding was just finished self.progressBar.update(5) def reset(self) -> None: @@ -128,10 +128,9 @@ def close(self) -> None: self.progressBar.close() - -#=============================== -#=== Download tracking class === -#=============================== +# =============================== +# === Download tracking class === +# =============================== class DownloadTracker(): def __init__(self): @@ -159,7 +158,7 @@ def load_tracking_file(self, trackingFilePath: str) -> None: self.saveFile = trackingFile # convert song data dumps to songObj's - #! see, songObj.get_data_dump and songObj.from_dump for more details + # ! see, songObj.get_data_dump and songObj.from_dump for more details for dump in songDataDumps: self.songObjList.append(SongObj.from_dump(dump)) @@ -191,38 +190,33 @@ def backup_to_disk(self): backs up details of songObj's yet to be downloaded to a .spotdlTrackingFile ''' # remove tracking file if no songs left in queue - #! we use 'return None' as a convenient exit point + # ! we use 'return None' as a convenient exit point if len(self.songObjList) == 0: if self.saveFile and self.saveFile.is_file(): self.saveFile.unlink() return None - - - # prepare datadumps of all songObj's yet to be downloaded songDataDumps = [] for song in self.songObjList: songDataDumps.append(song.get_data_dump()) - #! the default naming of a tracking file is $nameOfFirstSOng.spotdlTrackingFile, - #! it needs a little fixing because of disallowed characters in file naming + # ! the default naming of a tracking file is $nameOfFirstSOng.spotdlTrackingFile, + # ! it needs a little fixing because of disallowed characters in file naming if not self.saveFile: songName = self.songObjList[0].get_song_name() - for disallowedChar in ['/', '?', '\\', '*','|', '<', '>']: + for disallowedChar in ['/', '?', '\\', '*', '|', '<', '>']: if disallowedChar in songName: songName = songName.replace(disallowedChar, '') - songName = songName.replace('"', "'").replace(': ', ' - ') + songName = songName.replace('"', "'").replace(':', ' - ') self.saveFile = Path(songName + '.spotdlTrackingFile') - - # backup to file - #! we use 'wb' instead of 'w' to accommodate your fav K-pop/J-pop/Viking music + # ! we use 'wb' instead of 'w' to accommodate your fav K-pop/J-pop/Viking music with open(self.saveFile, 'wb') as file_handle: file_handle.write(str(songDataDumps).encode()) diff --git a/spotdl/search/__init__.py b/spotdl/search/__init__.py index 37fa96741..d17492145 100644 --- a/spotdl/search/__init__.py +++ b/spotdl/search/__init__.py @@ -5,7 +5,7 @@ 'utils' ] -#! You should be able to do all you want with just theese three lines -#! from spotdl.search.spotifyClient import initialize -#! from spotdl.search.songObj import songObj -#! from spotdl.search.utils import * \ No newline at end of file +# ! You should be able to do all you want with just theese three lines +# ! from spotdl.search.spotifyClient import initialize +# ! from spotdl.search.songObj import songObj +# ! from spotdl.search.utils import * diff --git a/spotdl/search/provider.py b/spotdl/search/provider.py index d78459572..93fbe0ee9 100644 --- a/spotdl/search/provider.py +++ b/spotdl/search/provider.py @@ -2,13 +2,13 @@ # === Imports === # =============== +# ! the following are for the search provider to function +import typing from datetime import timedelta from time import strptime # ! Just for static typing from typing import List -# ! the following are for the search provider to function -import typing from rapidfuzz.fuzz import partial_ratio from ytmusicapi import YTMusic @@ -48,7 +48,7 @@ def match_percentage(str1: str, str2: str, score_cutoff: float = 0) -> float: # ! we build new strings that contain only alphanumerical characters and spaces # ! and return the partial_ratio of that - except: + except: # noqa:E722 newStr1 = '' for eachLetter in str1: @@ -95,7 +95,7 @@ def __map_result_to_song_data(result: dict) -> dict: 'name': result['title'], 'type': result['resultType'], 'artist': artists, - 'length': __parse_duration(result['duration']), + 'length': __parse_duration(result.get('duration', None)), 'link': f'https://www.youtube.com/watch?v={video_id}', 'position': 0 } diff --git a/spotdl/search/songObj.py b/spotdl/search/songObj.py index 0f055e36f..221a4aacb 100644 --- a/spotdl/search/songObj.py +++ b/spotdl/search/songObj.py @@ -18,11 +18,11 @@ def __init__(self, rawTrackMeta, rawAlbumMeta, rawArtistMeta, youtubeLink): self.__rawArtistMeta = rawArtistMeta self.__youtubeLink = youtubeLink - #! constructors here are a bit mucky, there are two different constructors for two - #! different use cases, hence the actual __init__ function does not exist + # ! constructors here are a bit mucky, there are two different constructors for two + # ! different use cases, hence the actual __init__ function does not exist - #! Note, since the following are class methods, an instance of songObj is initialized - #! and passed to them + # ! Note, since the following are class methods, an instance of songObj is initialized + # ! and passed to them @classmethod def from_url(cls, spotifyURL: str): # check if URL is a playlist, user, artist or album, if yes raise an Exception, @@ -30,8 +30,6 @@ def from_url(cls, spotifyURL: str): if not ('open.spotify.com' in spotifyURL and 'track' in spotifyURL): raise Exception('passed URL is not that of a track: %s' % spotifyURL) - - # query spotify for song, artist, album details spotifyClient = get_spotify_client() @@ -43,8 +41,6 @@ def from_url(cls, spotifyURL: str): albumId = rawTrackMeta['album']['id'] rawAlbumMeta = spotifyClient.album(albumId) - - # get best match from the given provider songName = rawTrackMeta['name'] @@ -52,7 +48,7 @@ def from_url(cls, spotifyURL: str): duration = round( rawTrackMeta['duration_ms'] / 1000, - ndigits = 3 + ndigits=3 ) contributingArtists = [] @@ -67,19 +63,19 @@ def from_url(cls, spotifyURL: str): duration ) - return cls( + return cls( rawTrackMeta, rawAlbumMeta, rawArtistMeta, youtubeLink ) @classmethod def from_dump(cls, dataDump: dict): - rawTrackMeta = dataDump['rawTrackMeta'] - rawAlbumMeta = dataDump['rawAlbumMeta'] + rawTrackMeta = dataDump['rawTrackMeta'] + rawAlbumMeta = dataDump['rawAlbumMeta'] rawArtistMeta = dataDump['rawAlbumMeta'] - youtubeLink = dataDump['youtubeLink'] + youtubeLink = dataDump['youtubeLink'] - return cls( + return cls( rawTrackMeta, rawAlbumMeta, rawArtistMeta, youtubeLink ) @@ -90,16 +86,16 @@ def __eq__(self, comparedSong) -> bool: else: return False - #================================ - #=== Interface Implementation === - #================================ + # ================================ + # === Interface Implementation === + # ================================ def get_youtube_link(self) -> str: return self.__youtubeLink - #! Song Details: + # ! Song Details: - #! 1. Name + # ! 1. Name def get_song_name(self) -> str: '''' returns songs's name. @@ -107,7 +103,7 @@ def get_song_name(self) -> str: return self.__rawTrackMeta['name'] - #! 2. Track Number + # ! 2. Track Number def get_track_number(self) -> int: ''' returns song's track number from album (as in weather its the first @@ -116,7 +112,7 @@ def get_track_number(self) -> int: return self.__rawTrackMeta['track_number'] - #! 3. Genres + # ! 3. Genres def get_genres(self) -> List[str]: ''' returns a list of possible genres for the given song, the first member @@ -126,15 +122,15 @@ def get_genres(self) -> List[str]: return self.__rawAlbumMeta['genres'] + self.__rawArtistMeta['genres'] - #! 4. Duration + # ! 4. Duration def get_duration(self) -> float: ''' returns duration of song in seconds. ''' - return round(self.__rawTrackMeta['duration_ms'] / 1000, ndigits = 3) + return round(self.__rawTrackMeta['duration_ms'] / 1000, ndigits=3) - #! 5. All involved artists + # ! 5. All involved artists def get_contributing_artists(self) -> List[str]: ''' returns a list of all artists who worked on the song. @@ -154,9 +150,9 @@ def get_contributing_artists(self) -> List[str]: return contributingArtists - #! Album Details: + # ! Album Details: - #! 1. Name + # ! 1. Name def get_album_name(self) -> str: ''' returns name of the album that the song belongs to. @@ -164,7 +160,7 @@ def get_album_name(self) -> str: return self.__rawTrackMeta['album']['name'] - #! 2. All involved artist + # ! 2. All involved artist def get_album_artists(self) -> List[str]: ''' returns list of all artists who worked on the album that @@ -179,7 +175,7 @@ def get_album_artists(self) -> List[str]: return albumArtists - #! 3. Release Year/Date + # ! 3. Release Year/Date def get_album_release(self) -> str: ''' returns date/year of album release depending on what data is available. @@ -187,9 +183,9 @@ def get_album_release(self) -> str: return self.__rawTrackMeta['album']['release_date'] - #! Utilities for genuine use and also for metadata freaks: + # ! Utilities for genuine use and also for metadata freaks: - #! 1. Album Art URL + # ! 1. Album Art URL def get_album_cover_url(self) -> str: ''' returns url of the biggest album art image available. @@ -197,7 +193,7 @@ def get_album_cover_url(self) -> str: return self.__rawTrackMeta['album']['images'][0]['url'] - #! 2. All the details the spotify-api can provide + # ! 2. All the details the spotify-api can provide def get_data_dump(self) -> dict: ''' returns a dictionary containing the spotify-api responses as-is. The @@ -211,11 +207,11 @@ def get_data_dump(self) -> dict: have to look it up seperately when it's already been looked up once? ''' - #! internally the only reason this exists is that it helps in saving to disk + # ! internally the only reason this exists is that it helps in saving to disk return { - 'youtubeLink' : self.__youtubeLink, - 'rawTrackMeta' : self.__rawTrackMeta, - 'rawAlbumMeta' : self.__rawAlbumMeta, + 'youtubeLink': self.__youtubeLink, + 'rawTrackMeta': self.__rawTrackMeta, + 'rawAlbumMeta': self.__rawAlbumMeta, 'rawArtistMeta': self.__rawArtistMeta } diff --git a/spotdl/search/spotifyClient.py b/spotdl/search/spotifyClient.py index 3ba62a11e..80354b34c 100644 --- a/spotdl/search/spotifyClient.py +++ b/spotdl/search/spotifyClient.py @@ -1,29 +1,26 @@ -#=============== -#=== Imports === -#=============== +# =============== +# === Imports === +# =============== from spotipy import Spotify from spotipy.oauth2 import SpotifyClientCredentials +# =============== +# === Globals === +# =============== - -#=============== -#=== Globals === -#=============== - -#! Look through both initialize() and get_spotify_client() for need of masterClient +# ! Look through both initialize() and get_spotify_client() for need of masterClient masterClient = None - -#========================= -#=== The actual action === -#========================= +# ========================= +# === The actual action === +# ========================= def initialize(clientId: str, clientSecret: str): ''' `str` `clientId` : client id from your spotify account - + `str` `clientSecret` : client secret for your client id RETURNS `~` @@ -33,26 +30,25 @@ def initialize(clientId: str, clientSecret: str): ''' # check if initialization has been completed, if yes, raise an Exception - #! None evaluates to False, objects evaluate to True. + # ! None evaluates to False, objects evaluate to True. global masterClient if masterClient: raise Exception('A spotify client has already been initialized') - - # else create and cache a spotify client else: # create Oauth credentials for the SpotifyClient credentialManager = SpotifyClientCredentials( - client_id = clientId, - client_secret = clientSecret + client_id=clientId, + client_secret=clientSecret ) - client = Spotify(client_credentials_manager = credentialManager) - + client = Spotify(client_credentials_manager=credentialManager) + masterClient = client + def get_spotify_client(): ''' RETURNS `Spotify` @@ -62,11 +58,11 @@ def get_spotify_client(): ''' global masterClient - - #! None evaluvates to False, Objects evaluate to True + + # ! None evaluates to False, Objects evaluate to True if masterClient: return masterClient else: - raise Exception('Spotify client not created. Call spotifyClient.initialize' + - '(clientId, clientSecret) first.') \ No newline at end of file + raise Exception('Spotify client not created. Call spotifyClient.initialize' + '(clientId, clientSecret) first.') diff --git a/spotdl/search/utils.py b/spotdl/search/utils.py index b45914388..b802d35bb 100644 --- a/spotdl/search/utils.py +++ b/spotdl/search/utils.py @@ -1,9 +1,10 @@ -from spotdl.search.spotifyClient import get_spotify_client +from typing import List + from spotdl.search.songObj import SongObj +from spotdl.search.spotifyClient import get_spotify_client -from typing import List -def search_for_song(query: str) -> SongObj: +def search_for_song(query: str) -> SongObj: ''' `str` `query` : what you'd type into spotify's search box @@ -14,7 +15,7 @@ def search_for_song(query: str) -> SongObj: spotifyClient = get_spotify_client() # get possible matches from spotify - result = spotifyClient.search(query, type = 'track') + result = spotifyClient.search(query, type='track') # return first result link or if no matches are found, raise and Exception if len(result['tracks']['items']) == 0: @@ -23,12 +24,13 @@ def search_for_song(query: str) -> SongObj: for songResult in result['tracks']['items']: songUrl = 'http://open.spotify.com/track/' + songResult['id'] song = SongObj.from_url(songUrl) - - if song.get_youtube_link() != None: + + if song.get_youtube_link() is not None: return song - + raise Exception('Could not match any of the results on YouTube') - + + def get_album_tracks(albumUrl: str) -> List[SongObj]: ''' `str` `albumUrl` : Spotify Url of the album whose tracks are to be @@ -47,21 +49,22 @@ def get_album_tracks(albumUrl: str) -> List[SongObj]: for track in trackResponse['items']: song = SongObj.from_url('https://open.spotify.com/track/' + track['id']) - - if song.get_youtube_link() != None: + + if song.get_youtube_link() is not None: albumTracks.append(song) - + # check if more tracks are to be passed if trackResponse['next']: trackResponse = spotifyClient.album_tracks( albumUrl, - offset = len(albumTracks) + offset=len(albumTracks) ) else: break - + return albumTracks + def get_playlist_tracks(playlistUrl: str) -> List[SongObj]: ''' `str` `playlistUrl` : Spotify Url of the album whose tracks are to be @@ -79,19 +82,22 @@ def get_playlist_tracks(playlistUrl: str) -> List[SongObj]: while True: for songEntry in playlistResponse['items']: + if songEntry['track'] is None: + continue + song = SongObj.from_url( 'https://open.spotify.com/track/' + songEntry['track']['id']) - - if song.get_youtube_link() != None: + + if song.get_youtube_link() is not None: playlistTracks.append(song) - # check if more tracks are to be passed + # check if more tracks are to be passed if playlistResponse['next']: playlistResponse = spotifyClient.playlist_tracks( playlistUrl, - offset = len(playlistTracks) + offset=playlistResponse['offset'] + playlistResponse['limit'] ) else: break - - return playlistTracks \ No newline at end of file + + return playlistTracks diff --git a/tests/regressions.py b/tests/regressions.py index 1b565115f..e9937fa3f 100644 --- a/tests/regressions.py +++ b/tests/regressions.py @@ -1,6 +1,6 @@ import sys -from spotdl import console_entry_point +from spotdl.__main__ import console_entry_point SONGS = { "https://open.spotify.com/track/6CN3e26iQSj1N5lomh0mfO": "Eminem - Like Toy Soldiers.mp3", diff --git a/tests/test_entry_point.py b/tests/test_entry_point.py index 818d07b82..4b7b497bc 100644 --- a/tests/test_entry_point.py +++ b/tests/test_entry_point.py @@ -31,7 +31,7 @@ def patch_dependencies(mocker, monkeypatch): mocker.patch.object(DownloadManager, "close", autospec=True) -@pytest.mark.parametrize("argument", ["-h", "-H", "--help", None]) +@pytest.mark.parametrize("argument", ["-h", "--help"]) def test_show_help(capsys, monkeypatch, argument): """The --help, -h switches or no arguments should display help message""" @@ -41,10 +41,12 @@ def test_show_help(capsys, monkeypatch, argument): if argument: cli_args.append(argument) monkeypatch.setattr(sys, "argv", cli_args) - console_entry_point() + + with pytest.raises(SystemExit): + console_entry_point() out, _ = capsys.readouterr() - assert out == help_notice + "\n" + assert help_notice in out @pytest.mark.vcr() diff --git a/tox.ini b/tox.ini index 4ce9ff09f..1f01a8a71 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36,py37,py38,py39,mypy +envlist = py36,py37,py38,py39,mypy,flake8 [testenv] deps = .[test] @@ -9,12 +9,16 @@ commands = pytest commands = pytest --disable-vcr --cov=spotdl [testenv:mypy] -deps = mypy==0.790 +deps = .[dev] commands = mypy spotdl +[testenv:flake8] +deps = .[dev] +commands = flake8 spotdl + [gh-actions] python = 3.6: py36 3.7: py37 - 3.8: py38, mypy + 3.8: py38 3.9: py39