From dd60bd5ea564700157a3b253bf663b09d162befa Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Fri, 25 Oct 2024 20:28:38 -0700 Subject: [PATCH 1/8] [DEV] Helper test to reproduce from user logs easier --- tests/e2e/youtube/test_debug_repro.py | 34 ++ tests/resources/file_fixtures/repro.yaml | 694 +++++++++++++++++++++++ 2 files changed, 728 insertions(+) create mode 100644 tests/e2e/youtube/test_debug_repro.py create mode 100644 tests/resources/file_fixtures/repro.yaml diff --git a/tests/e2e/youtube/test_debug_repro.py b/tests/e2e/youtube/test_debug_repro.py new file mode 100644 index 000000000..e782e3c01 --- /dev/null +++ b/tests/e2e/youtube/test_debug_repro.py @@ -0,0 +1,34 @@ +import pytest +import yaml + +from resources import file_fixture_path + +from ytdl_sub.subscriptions.subscription import Subscription + + +@pytest.fixture +def repro_preset_dict(output_directory): + with open(file_fixture_path("repro.yaml"), "r", encoding="utf-8") as yaml_file: + yaml_dict = yaml.safe_load(yaml_file) + out = yaml_dict['presets']['subs'] + del out['preset'] + out['output_options']['output_directory'] = output_directory + return out + + +@pytest.mark.skipif(False, reason="Always skip repro, for local testing only") +class TestReproduce: + def test_single_video_download( + self, + default_config, + repro_preset_dict, + output_directory, + ): + single_video_subscription = Subscription.from_dict( + config=default_config, + preset_name="repro", + preset_dict=repro_preset_dict, + ) + + transaction_log = single_video_subscription.download(dry_run=False) + assert not transaction_log.is_empty diff --git a/tests/resources/file_fixtures/repro.yaml b/tests/resources/file_fixtures/repro.yaml new file mode 100644 index 000000000..2e289401c --- /dev/null +++ b/tests/resources/file_fixtures/repro.yaml @@ -0,0 +1,694 @@ +presets: + subs: + chapters: + embed_chapters: true + download: + - download_reverse: false + url: '{%bilateral_url(url) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url2) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url3) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url4) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url5) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url6) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url7) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url8) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url9) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url10) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url11) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url12) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url13) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url14) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url15) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url16) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url17) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url18) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url19) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url20) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url21) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url22) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url23) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url24) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url25) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url26) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url27) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url28) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url29) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url30) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url31) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url32) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url33) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url34) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url35) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url36) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url37) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url38) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url39) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url40) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url41) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url42) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url43) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url44) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url45) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url46) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url47) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url48) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url49) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url50) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url51) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url52) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url53) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url54) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url55) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url56) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url57) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url58) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url59) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url60) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url61) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url62) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url63) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url64) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url65) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url66) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url67) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url68) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url69) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url70) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url71) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url72) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url73) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url74) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url75) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url76) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url77) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url78) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url79) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url80) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url81) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url82) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url83) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url84) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url85) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url86) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url87) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url88) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url89) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url90) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url91) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url92) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url93) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url94) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url95) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url96) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url97) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url98) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{ %bilateral_url(url99) }' + ytdl_options: + playlist_items: -1:0:-1 + - download_reverse: false + url: '{%bilateral_url(url100) }' + ytdl_options: + playlist_items: -1:0:-1 + nfo_tags: + nfo_name: '{episode_file_path}.nfo' + nfo_root: episodedetails + tags: + aired: '{episode_date_standardized}' + episode: '{episode_number}' + genre: + - '{tv_show_genre}' + mpaa: '{episode_content_rating}' + plot: '{episode_plot}' + season: '{season_number}' + title: '{episode_title}' + year: '{episode_year}' + output_directory_nfo_tags: + nfo_name: tvshow.nfo + nfo_root: tvshow + tags: + genre: + - '{tv_show_genre}' + mpaa: '{tv_show_content_rating}' + title: '{tv_show_name}' + output_options: + file_name: '{episode_file_path}.{ext}' + info_json_name: '{episode_file_path}.{info_json_ext}' + maintain_download_archive: true + output_directory: '{tv_show_directory}/{tv_show_name_sanitized}' + thumbnail_name: '{thumbnail_file_name}' + overrides: + '%is_bilateral_url': '{ %contains( $0, "youtube.com/playlist" ) }' + '%bilateral_url': "{ \n %if(\n %and(\n enable_bilateral_scraping,\n\ + \ true,\n %is_bilateral_url($0)\n \ + \ ),\n $0,\n \"\"\n )\n}" + avatar_uncropped_thumbnail_file_name: '{tv_show_poster_file_name}' + banner_uncropped_thumbnail_file_name: '{tv_show_fanart_file_name}' + enable_bilateral_scraping: true + episode_content_rating: '{tv_show_content_rating}' + episode_date_standardized: '{upload_date_standardized}' + episode_file_name: s{season_number_padded}.e{episode_number_padded} - {file_title} + episode_file_path: '{season_directory_name_sanitized}/{episode_file_name_sanitized}' + episode_number: '{upload_month}{upload_day_padded}{upload_date_index_padded}' + episode_number_padded: '{upload_month_padded}{upload_day_padded}{upload_date_index_padded}' + episode_plot: '{webpage_url} + + + {description}' + episode_title: '{title} - {upload_date_standardized}' + episode_year: '{upload_year}' + file_title: '{title_sanitized}' + file_uid: '{uid_sanitized}' + season_directory_name: Season {season_number_padded} + season_number: '{upload_year}' + season_number_padded: '{season_number}' + season_poster_file_name: season{season_number_padded}-poster.jpg + subscription_array: '{ [] }' + subscription_indent_1: '{tv_show_genre_default}' + subscription_indent_2: '{tv_show_content_rating_default}' + subscription_value: '' + subscription_value_10: '' + subscription_value_100: '' + subscription_value_11: '' + subscription_value_12: '' + subscription_value_13: '' + subscription_value_14: '' + subscription_value_15: '' + subscription_value_16: '' + subscription_value_17: '' + subscription_value_18: '' + subscription_value_19: '' + subscription_value_2: '' + subscription_value_20: '' + subscription_value_21: '' + subscription_value_22: '' + subscription_value_23: '' + subscription_value_24: '' + subscription_value_25: '' + subscription_value_26: '' + subscription_value_27: '' + subscription_value_28: '' + subscription_value_29: '' + subscription_value_3: '' + subscription_value_30: '' + subscription_value_31: '' + subscription_value_32: '' + subscription_value_33: '' + subscription_value_34: '' + subscription_value_35: '' + subscription_value_36: '' + subscription_value_37: '' + subscription_value_38: '' + subscription_value_39: '' + subscription_value_4: '' + subscription_value_40: '' + subscription_value_41: '' + subscription_value_42: '' + subscription_value_43: '' + subscription_value_44: '' + subscription_value_45: '' + subscription_value_46: '' + subscription_value_47: '' + subscription_value_48: '' + subscription_value_49: '' + subscription_value_5: '' + subscription_value_50: '' + subscription_value_51: '' + subscription_value_52: '' + subscription_value_53: '' + subscription_value_54: '' + subscription_value_55: '' + subscription_value_56: '' + subscription_value_57: '' + subscription_value_58: '' + subscription_value_59: '' + subscription_value_6: '' + subscription_value_60: '' + subscription_value_61: '' + subscription_value_62: '' + subscription_value_63: '' + subscription_value_64: '' + subscription_value_65: '' + subscription_value_66: '' + subscription_value_67: '' + subscription_value_68: '' + subscription_value_69: '' + subscription_value_7: '' + subscription_value_70: '' + subscription_value_71: '' + subscription_value_72: '' + subscription_value_73: '' + subscription_value_74: '' + subscription_value_75: '' + subscription_value_76: '' + subscription_value_77: '' + subscription_value_78: '' + subscription_value_79: '' + subscription_value_8: '' + subscription_value_80: '' + subscription_value_81: '' + subscription_value_82: '' + subscription_value_83: '' + subscription_value_84: '' + subscription_value_85: '' + subscription_value_86: '' + subscription_value_87: '' + subscription_value_88: '' + subscription_value_89: '' + subscription_value_9: '' + subscription_value_90: '' + subscription_value_91: '' + subscription_value_92: '' + subscription_value_93: '' + subscription_value_94: '' + subscription_value_95: '' + subscription_value_96: '' + subscription_value_97: '' + subscription_value_98: '' + subscription_value_99: '' + thumbnail_file_name: '{episode_file_path}-thumb.jpg' + tv_show_content_rating: '{subscription_indent_2}' + tv_show_content_rating_default: TV-14 + tv_show_directory: /mnt/streamy/videos/Subs/ + tv_show_fanart_file_name: fanart.jpg + tv_show_genre: Animation + tv_show_genre_default: ytdl-sub + tv_show_name: Filthy Frank + tv_show_poster_file_name: poster.jpg + url: https://www.youtube.com/playlist?list=PLdd_1efceN2FP9USkRvE1v7cgVIewjpB6 + url10: '{subscription_value_10}' + url100: '{subscription_value_100}' + url11: '{subscription_value_11}' + url12: '{subscription_value_12}' + url13: '{subscription_value_13}' + url14: '{subscription_value_14}' + url15: '{subscription_value_15}' + url16: '{subscription_value_16}' + url17: '{subscription_value_17}' + url18: '{subscription_value_18}' + url19: '{subscription_value_19}' + url2: '{subscription_value_2}' + url20: '{subscription_value_20}' + url21: '{subscription_value_21}' + url22: '{subscription_value_22}' + url23: '{subscription_value_23}' + url24: '{subscription_value_24}' + url25: '{subscription_value_25}' + url26: '{subscription_value_26}' + url27: '{subscription_value_27}' + url28: '{subscription_value_28}' + url29: '{subscription_value_29}' + url3: '{subscription_value_3}' + url30: '{subscription_value_30}' + url31: '{subscription_value_31}' + url32: '{subscription_value_32}' + url33: '{subscription_value_33}' + url34: '{subscription_value_34}' + url35: '{subscription_value_35}' + url36: '{subscription_value_36}' + url37: '{subscription_value_37}' + url38: '{subscription_value_38}' + url39: '{subscription_value_39}' + url4: '{subscription_value_4}' + url40: '{subscription_value_40}' + url41: '{subscription_value_41}' + url42: '{subscription_value_42}' + url43: '{subscription_value_43}' + url44: '{subscription_value_44}' + url45: '{subscription_value_45}' + url46: '{subscription_value_46}' + url47: '{subscription_value_47}' + url48: '{subscription_value_48}' + url49: '{subscription_value_49}' + url5: '{subscription_value_5}' + url50: '{subscription_value_50}' + url51: '{subscription_value_51}' + url52: '{subscription_value_52}' + url53: '{subscription_value_53}' + url54: '{subscription_value_54}' + url55: '{subscription_value_55}' + url56: '{subscription_value_56}' + url57: '{subscription_value_57}' + url58: '{subscription_value_58}' + url59: '{subscription_value_59}' + url6: '{subscription_value_6}' + url60: '{subscription_value_60}' + url61: '{subscription_value_61}' + url62: '{subscription_value_62}' + url63: '{subscription_value_63}' + url64: '{subscription_value_64}' + url65: '{subscription_value_65}' + url66: '{subscription_value_66}' + url67: '{subscription_value_67}' + url68: '{subscription_value_68}' + url69: '{subscription_value_69}' + url7: '{subscription_value_7}' + url70: '{subscription_value_70}' + url71: '{subscription_value_71}' + url72: '{subscription_value_72}' + url73: '{subscription_value_73}' + url74: '{subscription_value_74}' + url75: '{subscription_value_75}' + url76: '{subscription_value_76}' + url77: '{subscription_value_77}' + url78: '{subscription_value_78}' + url79: '{subscription_value_79}' + url8: '{subscription_value_8}' + url80: '{subscription_value_80}' + url81: '{subscription_value_81}' + url82: '{subscription_value_82}' + url83: '{subscription_value_83}' + url84: '{subscription_value_84}' + url85: '{subscription_value_85}' + url86: '{subscription_value_86}' + url87: '{subscription_value_87}' + url88: '{subscription_value_88}' + url89: '{subscription_value_89}' + url9: '{subscription_value_9}' + url90: '{subscription_value_90}' + url91: '{subscription_value_91}' + url92: '{subscription_value_92}' + url93: '{subscription_value_93}' + url94: '{subscription_value_94}' + url95: '{subscription_value_95}' + url96: '{subscription_value_96}' + url97: '{subscription_value_97}' + url98: '{subscription_value_98}' + url99: '{subscription_value_99}' + preset: + - _jellyfin_tv_show + - _tv_show_by_date + - _season_by_year + - jellyfin_tv_show_by_date + - season_by_year__episode_by_month_day + - add_subtitles + - include_info_json + - tv_show + subtitles: + allow_auto_generated_subtitles: true + embed_subtitles: true + languages: en + video_tags: + contentRating: '{episode_content_rating}' + date: '{episode_date_standardized}' + episode_id: '{episode_number}' + genre: '{tv_show_genre}' + show: '{tv_show_name}' + synopsis: '{episode_plot}' + title: '{episode_title}' + year: '{episode_year}' + ytdl_options: + break_on_existing: true \ No newline at end of file From f53a22d482b8d5f5b1cc6968003b6dd810bf9684 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Fri, 25 Oct 2024 20:34:49 -0700 Subject: [PATCH 2/8] lint --- tests/e2e/youtube/test_debug_repro.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/e2e/youtube/test_debug_repro.py b/tests/e2e/youtube/test_debug_repro.py index e782e3c01..2aa1b4c1c 100644 --- a/tests/e2e/youtube/test_debug_repro.py +++ b/tests/e2e/youtube/test_debug_repro.py @@ -1,6 +1,5 @@ import pytest import yaml - from resources import file_fixture_path from ytdl_sub.subscriptions.subscription import Subscription @@ -10,9 +9,9 @@ def repro_preset_dict(output_directory): with open(file_fixture_path("repro.yaml"), "r", encoding="utf-8") as yaml_file: yaml_dict = yaml.safe_load(yaml_file) - out = yaml_dict['presets']['subs'] - del out['preset'] - out['output_options']['output_directory'] = output_directory + out = yaml_dict["presets"]["subs"] + del out["preset"] + out["output_options"]["output_directory"] = output_directory return out From 49ea614fb24459af7bf5c0b997f882942f2c58c9 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sun, 27 Oct 2024 14:58:36 -0700 Subject: [PATCH 3/8] Revert "[FEATURE] Support newlines and tabs in scripting strings (#1105)" This reverts commit 80054aa77b84318b63396ea06453c2edefad6f13. --- src/ytdl_sub/script/parser.py | 12 ++---------- tests/unit/script/functions/test_string_functions.py | 1 - tests/unit/script/types/test_string.py | 6 +----- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/src/ytdl_sub/script/parser.py b/src/ytdl_sub/script/parser.py index 0f844e34f..737d6ffa2 100644 --- a/src/ytdl_sub/script/parser.py +++ b/src/ytdl_sub/script/parser.py @@ -301,16 +301,8 @@ def _parse_string(self, str_open_token: str) -> String: self._pos += len(str_open_token) return String(value=string_value) - # Read literal "\n" as newlines - if self._read(increment_pos=False, length=2) == "\\n": - string_value += "\n" - self._pos += 2 - elif self._read(increment_pos=False, length=2) == "\\t": - string_value += "\t" - self._pos += 2 - else: - self._pos += 1 - string_value += ch + self._pos += 1 + string_value += ch raise STRINGS_NOT_CLOSED diff --git a/tests/unit/script/functions/test_string_functions.py b/tests/unit/script/functions/test_string_functions.py index ea3d441d9..e5d898f8c 100644 --- a/tests/unit/script/functions/test_string_functions.py +++ b/tests/unit/script/functions/test_string_functions.py @@ -132,7 +132,6 @@ def test_contains_any(self, value, expected_output): ("no splits", " | ", None, ["no splits"]), ("one | split", " | ", None, ["one", "split"]), ("max | split | one", " | ", 1, ["max", "split | one"]), - ("multiline\ndescription", "\\n", None, ["multiline", "description"]), ], ) def test_split( diff --git a/tests/unit/script/types/test_string.py b/tests/unit/script/types/test_string.py index 08700ace6..c3749845b 100644 --- a/tests/unit/script/types/test_string.py +++ b/tests/unit/script/types/test_string.py @@ -1,7 +1,6 @@ import re import pytest -from unit.script.conftest import single_variable_output from ytdl_sub.script.parser import STRINGS_NOT_CLOSED from ytdl_sub.script.parser import STRINGS_ONLY_ARGS @@ -46,13 +45,10 @@ def test_string_not_arg(self, string: str): ("{%string('backslash \\\\')}", "backslash \\\\"), ("{%string('''triple quote with \" ' \\''')}", "triple quote with \" ' \\"), ('{%string("""triple quote with " \' \\""")}', "triple quote with \" ' \\"), - ("{%string('literal \\n newlines')}", "literal \n newlines"), - ("{%string('supports \t tabs')}", "supports \t tabs"), - ("{%string('literal \\t tabs')}", "literal \t tabs"), ], ) def test_string(self, string: str, expected_string: str): - assert single_variable_output(string) == expected_string + assert Script({"out": string}).resolve() == ScriptOutput({"out": String(expected_string)}) def test_null_is_empty_string(self): assert Script({"out": "{%string(null)}"}).resolve() == ScriptOutput({"out": String("")}) From 6260df805daad119bc8f5ce55e453f4a4693afd5 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sat, 2 Nov 2024 00:07:41 -0700 Subject: [PATCH 4/8] raw yaml repro --- tests/e2e/youtube/test_debug_repro.py | 50 +- tests/resources/file_fixtures/repro.yaml | 743 ++--------------------- 2 files changed, 93 insertions(+), 700 deletions(-) diff --git a/tests/e2e/youtube/test_debug_repro.py b/tests/e2e/youtube/test_debug_repro.py index 2aa1b4c1c..0da3579ed 100644 --- a/tests/e2e/youtube/test_debug_repro.py +++ b/tests/e2e/youtube/test_debug_repro.py @@ -1,3 +1,6 @@ +import json +from typing import Dict + import pytest import yaml from resources import file_fixture_path @@ -5,29 +8,60 @@ from ytdl_sub.subscriptions.subscription import Subscription -@pytest.fixture -def repro_preset_dict(output_directory): - with open(file_fixture_path("repro.yaml"), "r", encoding="utf-8") as yaml_file: - yaml_dict = yaml.safe_load(yaml_file) - out = yaml_dict["presets"]["subs"] +def debug_yaml_to_preset_dict(debug_subscription_dict: Dict, output_directory: str) -> Dict: + out = debug_subscription_dict["presets"]["subs"] del out["preset"] out["output_options"]["output_directory"] = output_directory return out +@pytest.fixture +def debug_log_rerpo(output_directory): + with open(file_fixture_path("repro.yaml"), "r", encoding="utf-8") as yaml_file: + yaml_dict = yaml.safe_load(yaml_file) + return debug_yaml_to_preset_dict( + debug_subscription_dict=yaml_dict, + output_directory=output_directory, + ) + + +@pytest.fixture +def subscription_yaml_preset(default_config, output_directory): + subscriptions = Subscription.from_file_path( + config=default_config, subscription_path=file_fixture_path("repro.yaml") + ) + subscription_dict = json.loads(subscriptions[0].as_yaml()) + return debug_yaml_to_preset_dict( + debug_subscription_dict=subscription_dict, + output_directory=output_directory, + ) + + @pytest.mark.skipif(False, reason="Always skip repro, for local testing only") class TestReproduce: - def test_single_video_download( + def test_debug_log_repro( self, default_config, repro_preset_dict, output_directory, ): - single_video_subscription = Subscription.from_dict( + sub = Subscription.from_dict( config=default_config, preset_name="repro", preset_dict=repro_preset_dict, ) - transaction_log = single_video_subscription.download(dry_run=False) + transaction_log = sub.download(dry_run=False) + assert not transaction_log.is_empty + + def test_subscription_yaml_repro( + self, default_config, subscription_yaml_preset, output_directory + ): + sub = Subscription.from_dict( + config=default_config, + preset_name="repro", + preset_dict=subscription_yaml_preset, + ) + + transaction_log = sub.download(dry_run=False) assert not transaction_log.is_empty diff --git a/tests/resources/file_fixtures/repro.yaml b/tests/resources/file_fixtures/repro.yaml index 2e289401c..50abb9813 100644 --- a/tests/resources/file_fixtures/repro.yaml +++ b/tests/resources/file_fixtures/repro.yaml @@ -1,694 +1,53 @@ -presets: - subs: - chapters: - embed_chapters: true - download: - - download_reverse: false - url: '{%bilateral_url(url) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url2) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url3) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url4) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url5) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url6) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url7) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url8) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url9) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url10) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url11) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url12) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url13) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url14) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url15) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url16) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url17) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url18) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url19) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url20) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url21) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url22) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url23) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url24) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url25) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url26) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url27) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url28) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url29) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url30) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url31) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url32) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url33) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url34) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url35) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url36) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url37) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url38) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url39) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url40) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url41) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url42) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url43) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url44) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url45) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url46) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url47) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url48) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url49) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url50) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url51) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url52) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url53) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url54) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url55) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url56) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url57) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url58) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url59) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url60) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url61) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url62) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url63) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url64) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url65) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url66) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url67) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url68) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url69) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url70) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url71) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url72) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url73) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url74) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url75) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url76) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url77) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url78) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url79) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url80) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url81) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url82) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url83) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url84) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url85) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url86) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url87) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url88) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url89) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url90) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url91) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url92) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url93) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url94) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url95) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url96) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url97) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url98) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{ %bilateral_url(url99) }' - ytdl_options: - playlist_items: -1:0:-1 - - download_reverse: false - url: '{%bilateral_url(url100) }' - ytdl_options: - playlist_items: -1:0:-1 - nfo_tags: - nfo_name: '{episode_file_path}.nfo' - nfo_root: episodedetails - tags: - aired: '{episode_date_standardized}' - episode: '{episode_number}' - genre: - - '{tv_show_genre}' - mpaa: '{episode_content_rating}' - plot: '{episode_plot}' - season: '{season_number}' - title: '{episode_title}' - year: '{episode_year}' - output_directory_nfo_tags: - nfo_name: tvshow.nfo - nfo_root: tvshow - tags: - genre: - - '{tv_show_genre}' - mpaa: '{tv_show_content_rating}' - title: '{tv_show_name}' - output_options: - file_name: '{episode_file_path}.{ext}' - info_json_name: '{episode_file_path}.{info_json_ext}' - maintain_download_archive: true - output_directory: '{tv_show_directory}/{tv_show_name_sanitized}' - thumbnail_name: '{thumbnail_file_name}' - overrides: - '%is_bilateral_url': '{ %contains( $0, "youtube.com/playlist" ) }' - '%bilateral_url': "{ \n %if(\n %and(\n enable_bilateral_scraping,\n\ - \ true,\n %is_bilateral_url($0)\n \ - \ ),\n $0,\n \"\"\n )\n}" - avatar_uncropped_thumbnail_file_name: '{tv_show_poster_file_name}' - banner_uncropped_thumbnail_file_name: '{tv_show_fanart_file_name}' - enable_bilateral_scraping: true - episode_content_rating: '{tv_show_content_rating}' - episode_date_standardized: '{upload_date_standardized}' - episode_file_name: s{season_number_padded}.e{episode_number_padded} - {file_title} - episode_file_path: '{season_directory_name_sanitized}/{episode_file_name_sanitized}' - episode_number: '{upload_month}{upload_day_padded}{upload_date_index_padded}' - episode_number_padded: '{upload_month_padded}{upload_day_padded}{upload_date_index_padded}' - episode_plot: '{webpage_url} +__preset__: + overrides: + music_directory: "hi" +YouTube Full Albums: + = Lofi: + "~Game Chops": + url: "https://www.youtube.com/playlist?list=PLBsm_SagFMmdWnCnrNtLjA9kzfrRkto4i" + chapter_title_regex_array: + - "^\\d+\\.\\.(.*)" # Captures 'title' from '1..title' + chapter_title_capture: >- + { + %regex_capture_many( chapter_title, chapter_title_regex_array, [chapter_title] ) + } + track_title: "{%array_at(chapter_title_capture, 1)}" + + + + = Eastern Bloc: + "Eastern Bloc": + output_options: + migrated_download_archive_name: ".Eastern Bloc-download-archive.json" + + overrides: + title_capture_regex_array: + - "^(.*) - (.*) \\|\\|.*" # Captures artist, album from 'artist - album ||...' + - "^(.*) - (.*) \\[.*" # Captures artist, ablum from 'artist - album [...' + - "^(.*) - (.*)" # Captures artist, ablum from 'artist - album' + title_capture: >- + { + %regex_capture_many( title, title_capture_regex_array ) + } + + description_capture_regex_array: + - "Genre:\\s*(.*)\\s*\n(?:Rec.+|Year):\\s*.*(\\d{4})\\s*\\n" # Captures genre and recorded year + description_capture: >- + { + %regex_capture_many( description, description_capture_regex_array, ) + } + + chapter_title_regex_array: + - "^(?:\\d+\\.\\s*|)(.*)" # Captures title from '1. title' + chapter_title_capture: >- + { + %regex_capture_many( chapter_title, chapter_title_regex_array, [chapter_title] ) + } + + url: "https://www.youtube.com/@heavymetalofeasternbloc" + track_artist: "{%array_at(title_capture, 1)}" + track_album: "{%array_at(title_capture, 2)}" + track_title: "{%array_at(chapter_title_capture, 1)}" + track_year: "{%array_at(description_capture, 1)}" - {description}' - episode_title: '{title} - {upload_date_standardized}' - episode_year: '{upload_year}' - file_title: '{title_sanitized}' - file_uid: '{uid_sanitized}' - season_directory_name: Season {season_number_padded} - season_number: '{upload_year}' - season_number_padded: '{season_number}' - season_poster_file_name: season{season_number_padded}-poster.jpg - subscription_array: '{ [] }' - subscription_indent_1: '{tv_show_genre_default}' - subscription_indent_2: '{tv_show_content_rating_default}' - subscription_value: '' - subscription_value_10: '' - subscription_value_100: '' - subscription_value_11: '' - subscription_value_12: '' - subscription_value_13: '' - subscription_value_14: '' - subscription_value_15: '' - subscription_value_16: '' - subscription_value_17: '' - subscription_value_18: '' - subscription_value_19: '' - subscription_value_2: '' - subscription_value_20: '' - subscription_value_21: '' - subscription_value_22: '' - subscription_value_23: '' - subscription_value_24: '' - subscription_value_25: '' - subscription_value_26: '' - subscription_value_27: '' - subscription_value_28: '' - subscription_value_29: '' - subscription_value_3: '' - subscription_value_30: '' - subscription_value_31: '' - subscription_value_32: '' - subscription_value_33: '' - subscription_value_34: '' - subscription_value_35: '' - subscription_value_36: '' - subscription_value_37: '' - subscription_value_38: '' - subscription_value_39: '' - subscription_value_4: '' - subscription_value_40: '' - subscription_value_41: '' - subscription_value_42: '' - subscription_value_43: '' - subscription_value_44: '' - subscription_value_45: '' - subscription_value_46: '' - subscription_value_47: '' - subscription_value_48: '' - subscription_value_49: '' - subscription_value_5: '' - subscription_value_50: '' - subscription_value_51: '' - subscription_value_52: '' - subscription_value_53: '' - subscription_value_54: '' - subscription_value_55: '' - subscription_value_56: '' - subscription_value_57: '' - subscription_value_58: '' - subscription_value_59: '' - subscription_value_6: '' - subscription_value_60: '' - subscription_value_61: '' - subscription_value_62: '' - subscription_value_63: '' - subscription_value_64: '' - subscription_value_65: '' - subscription_value_66: '' - subscription_value_67: '' - subscription_value_68: '' - subscription_value_69: '' - subscription_value_7: '' - subscription_value_70: '' - subscription_value_71: '' - subscription_value_72: '' - subscription_value_73: '' - subscription_value_74: '' - subscription_value_75: '' - subscription_value_76: '' - subscription_value_77: '' - subscription_value_78: '' - subscription_value_79: '' - subscription_value_8: '' - subscription_value_80: '' - subscription_value_81: '' - subscription_value_82: '' - subscription_value_83: '' - subscription_value_84: '' - subscription_value_85: '' - subscription_value_86: '' - subscription_value_87: '' - subscription_value_88: '' - subscription_value_89: '' - subscription_value_9: '' - subscription_value_90: '' - subscription_value_91: '' - subscription_value_92: '' - subscription_value_93: '' - subscription_value_94: '' - subscription_value_95: '' - subscription_value_96: '' - subscription_value_97: '' - subscription_value_98: '' - subscription_value_99: '' - thumbnail_file_name: '{episode_file_path}-thumb.jpg' - tv_show_content_rating: '{subscription_indent_2}' - tv_show_content_rating_default: TV-14 - tv_show_directory: /mnt/streamy/videos/Subs/ - tv_show_fanart_file_name: fanart.jpg - tv_show_genre: Animation - tv_show_genre_default: ytdl-sub - tv_show_name: Filthy Frank - tv_show_poster_file_name: poster.jpg - url: https://www.youtube.com/playlist?list=PLdd_1efceN2FP9USkRvE1v7cgVIewjpB6 - url10: '{subscription_value_10}' - url100: '{subscription_value_100}' - url11: '{subscription_value_11}' - url12: '{subscription_value_12}' - url13: '{subscription_value_13}' - url14: '{subscription_value_14}' - url15: '{subscription_value_15}' - url16: '{subscription_value_16}' - url17: '{subscription_value_17}' - url18: '{subscription_value_18}' - url19: '{subscription_value_19}' - url2: '{subscription_value_2}' - url20: '{subscription_value_20}' - url21: '{subscription_value_21}' - url22: '{subscription_value_22}' - url23: '{subscription_value_23}' - url24: '{subscription_value_24}' - url25: '{subscription_value_25}' - url26: '{subscription_value_26}' - url27: '{subscription_value_27}' - url28: '{subscription_value_28}' - url29: '{subscription_value_29}' - url3: '{subscription_value_3}' - url30: '{subscription_value_30}' - url31: '{subscription_value_31}' - url32: '{subscription_value_32}' - url33: '{subscription_value_33}' - url34: '{subscription_value_34}' - url35: '{subscription_value_35}' - url36: '{subscription_value_36}' - url37: '{subscription_value_37}' - url38: '{subscription_value_38}' - url39: '{subscription_value_39}' - url4: '{subscription_value_4}' - url40: '{subscription_value_40}' - url41: '{subscription_value_41}' - url42: '{subscription_value_42}' - url43: '{subscription_value_43}' - url44: '{subscription_value_44}' - url45: '{subscription_value_45}' - url46: '{subscription_value_46}' - url47: '{subscription_value_47}' - url48: '{subscription_value_48}' - url49: '{subscription_value_49}' - url5: '{subscription_value_5}' - url50: '{subscription_value_50}' - url51: '{subscription_value_51}' - url52: '{subscription_value_52}' - url53: '{subscription_value_53}' - url54: '{subscription_value_54}' - url55: '{subscription_value_55}' - url56: '{subscription_value_56}' - url57: '{subscription_value_57}' - url58: '{subscription_value_58}' - url59: '{subscription_value_59}' - url6: '{subscription_value_6}' - url60: '{subscription_value_60}' - url61: '{subscription_value_61}' - url62: '{subscription_value_62}' - url63: '{subscription_value_63}' - url64: '{subscription_value_64}' - url65: '{subscription_value_65}' - url66: '{subscription_value_66}' - url67: '{subscription_value_67}' - url68: '{subscription_value_68}' - url69: '{subscription_value_69}' - url7: '{subscription_value_7}' - url70: '{subscription_value_70}' - url71: '{subscription_value_71}' - url72: '{subscription_value_72}' - url73: '{subscription_value_73}' - url74: '{subscription_value_74}' - url75: '{subscription_value_75}' - url76: '{subscription_value_76}' - url77: '{subscription_value_77}' - url78: '{subscription_value_78}' - url79: '{subscription_value_79}' - url8: '{subscription_value_8}' - url80: '{subscription_value_80}' - url81: '{subscription_value_81}' - url82: '{subscription_value_82}' - url83: '{subscription_value_83}' - url84: '{subscription_value_84}' - url85: '{subscription_value_85}' - url86: '{subscription_value_86}' - url87: '{subscription_value_87}' - url88: '{subscription_value_88}' - url89: '{subscription_value_89}' - url9: '{subscription_value_9}' - url90: '{subscription_value_90}' - url91: '{subscription_value_91}' - url92: '{subscription_value_92}' - url93: '{subscription_value_93}' - url94: '{subscription_value_94}' - url95: '{subscription_value_95}' - url96: '{subscription_value_96}' - url97: '{subscription_value_97}' - url98: '{subscription_value_98}' - url99: '{subscription_value_99}' - preset: - - _jellyfin_tv_show - - _tv_show_by_date - - _season_by_year - - jellyfin_tv_show_by_date - - season_by_year__episode_by_month_day - - add_subtitles - - include_info_json - - tv_show - subtitles: - allow_auto_generated_subtitles: true - embed_subtitles: true - languages: en - video_tags: - contentRating: '{episode_content_rating}' - date: '{episode_date_standardized}' - episode_id: '{episode_number}' - genre: '{tv_show_genre}' - show: '{tv_show_name}' - synopsis: '{episode_plot}' - title: '{episode_title}' - year: '{episode_year}' - ytdl_options: - break_on_existing: true \ No newline at end of file From 57c60953367b4fb8b545573d0b280b9616800fd6 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sat, 2 Nov 2024 00:24:48 -0700 Subject: [PATCH 5/8] support sub file --- tests/e2e/youtube/test_debug_repro.py | 12 +++++++----- tests/resources/file_fixtures/repro.yaml | 19 ++++--------------- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/tests/e2e/youtube/test_debug_repro.py b/tests/e2e/youtube/test_debug_repro.py index 0da3579ed..5dc5c4653 100644 --- a/tests/e2e/youtube/test_debug_repro.py +++ b/tests/e2e/youtube/test_debug_repro.py @@ -9,10 +9,12 @@ def debug_yaml_to_preset_dict(debug_subscription_dict: Dict, output_directory: str) -> Dict: - out = debug_subscription_dict["presets"]["subs"] - del out["preset"] - out["output_options"]["output_directory"] = output_directory - return out + presets_level = debug_subscription_dict["presets"] + assert isinstance(presets_level, dict) and len(presets_level) == 1 + subscription = list(presets_level.values())[0] + del subscription["preset"] + subscription["output_options"]["output_directory"] = output_directory + return subscription @pytest.fixture @@ -30,7 +32,7 @@ def subscription_yaml_preset(default_config, output_directory): subscriptions = Subscription.from_file_path( config=default_config, subscription_path=file_fixture_path("repro.yaml") ) - subscription_dict = json.loads(subscriptions[0].as_yaml()) + subscription_dict = yaml.safe_load(subscriptions[0].as_yaml()) return debug_yaml_to_preset_dict( debug_subscription_dict=subscription_dict, output_directory=output_directory, diff --git a/tests/resources/file_fixtures/repro.yaml b/tests/resources/file_fixtures/repro.yaml index 50abb9813..4953f131d 100644 --- a/tests/resources/file_fixtures/repro.yaml +++ b/tests/resources/file_fixtures/repro.yaml @@ -3,24 +3,13 @@ __preset__: music_directory: "hi" YouTube Full Albums: - = Lofi: - "~Game Chops": - url: "https://www.youtube.com/playlist?list=PLBsm_SagFMmdWnCnrNtLjA9kzfrRkto4i" - chapter_title_regex_array: - - "^\\d+\\.\\.(.*)" # Captures 'title' from '1..title' - chapter_title_capture: >- - { - %regex_capture_many( chapter_title, chapter_title_regex_array, [chapter_title] ) - } - track_title: "{%array_at(chapter_title_capture, 1)}" - - = Eastern Bloc: "Eastern Bloc": output_options: migrated_download_archive_name: ".Eastern Bloc-download-archive.json" - + ytdl_options: + max_downloads: 3 overrides: title_capture_regex_array: - "^(.*) - (.*) \\|\\|.*" # Captures artist, album from 'artist - album ||...' @@ -32,10 +21,10 @@ YouTube Full Albums: } description_capture_regex_array: - - "Genre:\\s*(.*)\\s*\n(?:Rec.+|Year):\\s*.*(\\d{4})\\s*\\n" # Captures genre and recorded year + - 'Genre:\s*(.*)\s*\n(?:Rec.+|Year):\s*.*(\d\{4\})\s*\n' # Captures genre and recorded year description_capture: >- { - %regex_capture_many( description, description_capture_regex_array, ) + %regex_capture_many( description, description_capture_regex_array) } chapter_title_regex_array: From 60e7b33cc617592f2164df7c500ea7b21d308ad6 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sat, 2 Nov 2024 00:38:31 -0700 Subject: [PATCH 6/8] yaml --- tests/resources/file_fixtures/repro.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/resources/file_fixtures/repro.yaml b/tests/resources/file_fixtures/repro.yaml index 4953f131d..17ca0b07f 100644 --- a/tests/resources/file_fixtures/repro.yaml +++ b/tests/resources/file_fixtures/repro.yaml @@ -21,7 +21,7 @@ YouTube Full Albums: } description_capture_regex_array: - - 'Genre:\s*(.*)\s*\n(?:Rec.+|Year):\s*.*(\d\{4\})\s*\n' # Captures genre and recorded year + - '{%unescape("Genre:\s*(.*)\s*\n(?:Rec.+|Year):\s*.*(\d\{4\})\s*\n")}' # Captures genre and recorded year description_capture: >- { %regex_capture_many( description, description_capture_regex_array) From 4b70314d0f7dc5d459439beb34474fea2f86f6e5 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sun, 3 Nov 2024 18:58:17 -0800 Subject: [PATCH 7/8] skip --- tests/e2e/youtube/test_debug_repro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/youtube/test_debug_repro.py b/tests/e2e/youtube/test_debug_repro.py index 5dc5c4653..b96f530b2 100644 --- a/tests/e2e/youtube/test_debug_repro.py +++ b/tests/e2e/youtube/test_debug_repro.py @@ -39,7 +39,7 @@ def subscription_yaml_preset(default_config, output_directory): ) -@pytest.mark.skipif(False, reason="Always skip repro, for local testing only") +@pytest.mark.skipif(True, reason="Always skip repro, for local testing only") class TestReproduce: def test_debug_log_repro( self, From 93025c61d2ed74e6f096dd24a37428618c94a3ce Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sun, 3 Nov 2024 19:14:28 -0800 Subject: [PATCH 8/8] repro docs --- docs/source/guides/development/index.rst | 16 ++++++++++++++++ tests/e2e/{youtube => }/test_debug_repro.py | 0 2 files changed, 16 insertions(+) rename tests/e2e/{youtube => }/test_debug_repro.py (100%) diff --git a/docs/source/guides/development/index.rst b/docs/source/guides/development/index.rst index bf6b2c122..d0750e65d 100644 --- a/docs/source/guides/development/index.rst +++ b/docs/source/guides/development/index.rst @@ -62,3 +62,19 @@ is highly recommended. TODO: screenshots of configuration +Debugging +--------- + +Debug Logs +^^^^^^^^^^^^^^^ +Run with ``--log-level debug`` to show all debug logs when running ytdl-sub. + + +Reproducing a Failing Subscription +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Subscriptions will dump their entire *compiled* yaml at the beginning of exeuction +when using ``--log-level debug``. This can be copy-pasted into the file +``resources/file_fixtures/repro.yaml``. + +Running the test ``e2e.test_debug_repro.TestReproduce.test_debug_log_repro`` +will fully reproduce that subscription in order to debug it. \ No newline at end of file diff --git a/tests/e2e/youtube/test_debug_repro.py b/tests/e2e/test_debug_repro.py similarity index 100% rename from tests/e2e/youtube/test_debug_repro.py rename to tests/e2e/test_debug_repro.py