From 5f90bf6a998850884fb1be559b234d534d7cf1e6 Mon Sep 17 00:00:00 2001 From: Andy <88590076+AAndyProgram@users.noreply.github.com> Date: Mon, 8 Apr 2024 07:00:52 +0300 Subject: [PATCH] 2024.4.8.0 YT MusicPlaylistsForm, VideoOptionsForm: add audio bitrate option MediaItem: update type icon; update confirmation dialog for deleting non-single object Track: update extension PlayList: update 'ToString' information for 'MediaItem' YouTubeMediaContainerBase: add size recalculation; add audio bitrate change; embed thumbnail in the extracted 'mp3' as cover art; update 'DownloadCommand' function; include elements' files in XML for non-single items; update 'Delete' function; update files handling; include generated playlists and cover file in the file list YouTubeSettings: add properties 'DefaultAudioEmbedThumbnail_ExtractedFiles', 'DefaultAudioBitrate', 'DefaultAudioBitrate_crf' Exclude 'drc' from parsing results Fix incorrect file reference when the yt-dlp.exe has a different name SCrawler Base.Declarations: hide 'TokenRefreshIntervalProvider' error Base.DeclaredNames: remove 'Header_FB_FRIENDLY_NAME' const (use 'API.Instagram.UserData.GQL') Base.M3U8Base: add 'SkipBroken' argument Base.UserDataBase: add size recalculation (STD) Base.SiteSettingsBase: add 'SettingsVersion' property TDownloader: delete 'RenameOldFileNames' function SiteEditorForm: remove begin/end update of global settings when updating MainFrame: update 'BTT_DOWN_SPEC' tooltip SettingsHostCollection, SettingsHost: move site settings to a personal setting file (delete these settings from the global settings file) DownloadGroupCollection: remove data update during initialization and reindexing SettingsCLS: add 'SettingsVersion' property Add hidden controls API.JustForFans: change m3u8 parsing and downloading algo; remove 'CancellationToken' from m3u8 (replace with 'IThrower') API.Facebook: add option 'RequestsWaitTimer_Any'; add internal option 'DownloadData_Impl'; update GDL names and tokens references; add wait timers API.Threads: add option 'RequestsWaitTimer_Any'; add internal option 'DownloadData_Impl'; update GDL names and tokens references; add wait timers API.Instagram: ADD 'GDL' SUPPORT; add 'UpdateWwwClaim' to 'Declarations.UpdateResponser' and 'UserData'; add additional 'HH_IG_WWW_CLAIM' properties; add 'RequestsWaitTimer_Any' property; add tooltips for timer controls; update 'LastRequests' environment; update information about requests on the label in the settings form; update reels downloading function --- SCrawler.YouTube/Base/Structures.vb | 11 +- SCrawler.YouTube/Base/YouTubeFunctions.vb | 2 +- SCrawler.YouTube/Base/YouTubeSettings.vb | 10 + .../Controls/MusicPlaylistsForm.Designer.vb | 61 ++- .../Controls/MusicPlaylistsForm.resx | 8 + .../Controls/MusicPlaylistsForm.vb | 12 +- SCrawler.YouTube/Controls/VideoOption.vb | 4 + .../Controls/VideoOptionsForm.Designer.vb | 265 +++++----- .../Controls/VideoOptionsForm.resx | 43 +- SCrawler.YouTube/Controls/VideoOptionsForm.vb | 27 +- SCrawler.YouTube/Declarations.vb | 22 + SCrawler.YouTube/Downloader/MediaItem.vb | 33 +- SCrawler.YouTube/Objects/PlayList.vb | 7 +- SCrawler.YouTube/Objects/Track.vb | 18 +- .../Objects/YouTubeMediaContainerBase.vb | 296 ++++++++++-- SCrawler/API/Base/Declarations.vb | 2 +- SCrawler/API/Base/DeclaredNames.vb | 2 - SCrawler/API/Base/M3U8Base.vb | 24 +- SCrawler/API/Base/SiteSettingsBase.vb | 5 +- SCrawler/API/Base/UserDataBase.vb | 7 +- SCrawler/API/Base/YTDLP.vb | 2 +- SCrawler/API/Facebook/SiteSettings.vb | 4 +- SCrawler/API/Facebook/UserData.vb | 81 ++-- SCrawler/API/Instagram/Declarations.vb | 8 +- SCrawler/API/Instagram/SiteSettings.vb | 206 ++++++-- SCrawler/API/Instagram/UserData.GQL.vb | 333 +++++++++++++ SCrawler/API/Instagram/UserData.vb | 455 ++++++++++-------- SCrawler/API/JustForFans/M3U8.vb | 172 +++++-- SCrawler/API/JustForFans/UserData.vb | 2 +- SCrawler/API/ThreadsNet/SiteSettings.vb | 31 +- SCrawler/API/ThreadsNet/UserData.vb | 71 +-- .../Groups/DownloadGroupCollection.vb | 11 +- SCrawler/Download/TDownloader.vb | 24 - SCrawler/Editors/SiteEditorForm.vb | 15 +- SCrawler/MainFrame.Designer.vb | 4 +- SCrawler/MainMod.vb | 1 + .../Attributes/Attributes.vb | 8 + .../Hosts/DownloadableMediaHost.vb | 2 +- .../Hosts/PropertyValueHost.vb | 2 + .../PluginsEnvironment/Hosts/SettingsHost.vb | 53 +- .../Hosts/SettingsHostCollection.vb | 62 ++- SCrawler/SCrawler.vbproj | 1 + SCrawler/SettingsCLS.vb | 4 + 43 files changed, 1795 insertions(+), 616 deletions(-) create mode 100644 SCrawler/API/Instagram/UserData.GQL.vb diff --git a/SCrawler.YouTube/Base/Structures.vb b/SCrawler.YouTube/Base/Structures.vb index 5df756e..4e2cadc 100644 --- a/SCrawler.YouTube/Base/Structures.vb +++ b/SCrawler.YouTube/Base/Structures.vb @@ -81,6 +81,7 @@ Namespace API.YouTube.Base Public Structure MediaObject : Implements IIndexable, IComparable(Of MediaObject) Public Type As Plugin.UserMediaTypes Public ID As String + Public ID_DRC As Boolean Public Extension As String Public Width As Integer Public Height As Integer @@ -110,10 +111,14 @@ Namespace API.YouTube.Base End Function Private Function CompareTo(ByVal Other As MediaObject) As Integer Implements IComparable(Of MediaObject).CompareTo If Type = Other.Type Then - If Width.CompareTo(Other.Width) = 0 Then - Return Size.CompareTo(Other.Size) * -1 + If ID_DRC.CompareTo(Other.ID_DRC) = 0 Then + If Width.CompareTo(Other.Width) = 0 Then + Return Size.CompareTo(Other.Size) * -1 + Else + Return Width.CompareTo(Other.Width) * -1 + End If Else - Return Width.CompareTo(Other.Width) * -1 + Return ID_DRC.CompareTo(Other.ID_DRC) End If Else Return CInt(Type).CompareTo(CInt(Other.Type)) diff --git a/SCrawler.YouTube/Base/YouTubeFunctions.vb b/SCrawler.YouTube/Base/YouTubeFunctions.vb index e41952f..040b53b 100644 --- a/SCrawler.YouTube/Base/YouTubeFunctions.vb +++ b/SCrawler.YouTube/Base/YouTubeFunctions.vb @@ -174,7 +174,7 @@ Namespace API.YouTube.Base ByVal ObjType As YouTubeMediaType, ByVal ChannelTab As YouTubeChannelTab, ByVal IsMusic As Boolean, ByVal UrlAsIs As Boolean) As Boolean Try - Dim command$ = "yt-dlp --write-info-json --skip-download" + Dim command$ = $"{YTDLP_NAME} --write-info-json --skip-download" command.StringAppend(GetCookiesCommand(UseCookies, CookiesFile), " ") If DateAfter.HasValue Then command.StringAppend($"--dateafter {DateAfter.Value:yyyyMMdd}", " ") If DateBefore.HasValue Then command.StringAppend($"--datebefore {DateBefore.Value:yyyyMMdd}", " ") diff --git a/SCrawler.YouTube/Base/YouTubeSettings.vb b/SCrawler.YouTube/Base/YouTubeSettings.vb index 4c3c4fd..b238c97 100644 --- a/SCrawler.YouTube/Base/YouTubeSettings.vb +++ b/SCrawler.YouTube/Base/YouTubeSettings.vb @@ -38,6 +38,7 @@ Namespace API.YouTube.Base Friend ReadOnly Property DownloadLocations As DownloadLocationsCollection Friend ReadOnly Property PlaylistsLocations As DownloadLocationsCollection Public Overridable Property AccountName As String + Private ReadOnly Property SettingsVersion As XMLValue(Of Integer) #Region "Environment" #Region "Programs" Public ReadOnly Property DefaultAudioEmbedThumbnail As XMLValue(Of Boolean) + + Public ReadOnly Property DefaultAudioEmbedThumbnail_ExtractedFiles As XMLValue(Of Boolean) + + Public ReadOnly Property DefaultAudioBitrate As XMLValue(Of Integer) + + Public ReadOnly Property DefaultAudioBitrate_crf As XMLValue(Of Integer) #Region "Music" diff --git a/SCrawler.YouTube/Controls/MusicPlaylistsForm.Designer.vb b/SCrawler.YouTube/Controls/MusicPlaylistsForm.Designer.vb index 0450ce9..912c374 100644 --- a/SCrawler.YouTube/Controls/MusicPlaylistsForm.Designer.vb +++ b/SCrawler.YouTube/Controls/MusicPlaylistsForm.Designer.vb @@ -52,6 +52,7 @@ Namespace API.YouTube.Controls Dim ActionButton15 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton() Dim ActionButton16 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton() Dim ActionButton17 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton() + Dim ActionButton18 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton() Dim TT_MAIN As System.Windows.Forms.ToolTip Me.BTT_DOWN = New System.Windows.Forms.Button() Me.BTT_CANCEL = New System.Windows.Forms.Button() @@ -66,6 +67,7 @@ Namespace API.YouTube.Controls Me.CH_DOWN_LYRICS = New System.Windows.Forms.CheckBox() Me.TXT_OUTPUT_PATH = New PersonalUtilities.Forms.Controls.ComboBoxExtended() Me.CMB_PLS = New PersonalUtilities.Forms.Controls.ComboBoxExtended() + Me.TXT_AUDIO_BITRATE = New PersonalUtilities.Forms.Controls.TextBoxExtended() TP_MAIN = New System.Windows.Forms.TableLayoutPanel() TP_BUTTONS = New System.Windows.Forms.TableLayoutPanel() TP_PLS = New System.Windows.Forms.TableLayoutPanel() @@ -92,6 +94,7 @@ Namespace API.YouTube.Controls CType(Me.TXT_SUBS, System.ComponentModel.ISupportInitialize).BeginInit() CType(Me.TXT_OUTPUT_PATH, System.ComponentModel.ISupportInitialize).BeginInit() CType(Me.CMB_PLS, System.ComponentModel.ISupportInitialize).BeginInit() + CType(Me.TXT_AUDIO_BITRATE, System.ComponentModel.ISupportInitialize).BeginInit() Me.SuspendLayout() ' 'TP_MAIN @@ -106,10 +109,10 @@ Namespace API.YouTube.Controls TP_MAIN.Margin = New System.Windows.Forms.Padding(0) TP_MAIN.Name = "TP_MAIN" TP_MAIN.RowCount = 3 - TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 112.0!)) + TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 140.0!)) TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!)) TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!)) - TP_MAIN.Size = New System.Drawing.Size(434, 289) + TP_MAIN.Size = New System.Drawing.Size(434, 317) TP_MAIN.TabIndex = 0 ' 'TP_BUTTONS @@ -121,14 +124,14 @@ Namespace API.YouTube.Controls TP_BUTTONS.Controls.Add(Me.BTT_DOWN, 1, 0) TP_BUTTONS.Controls.Add(Me.BTT_CANCEL, 2, 0) TP_BUTTONS.Dock = System.Windows.Forms.DockStyle.Fill - TP_BUTTONS.Location = New System.Drawing.Point(0, 264) + TP_BUTTONS.Location = New System.Drawing.Point(0, 292) TP_BUTTONS.Margin = New System.Windows.Forms.Padding(0) TP_BUTTONS.Name = "TP_BUTTONS" TP_BUTTONS.RowCount = 1 TP_BUTTONS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!)) TP_BUTTONS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!)) TP_BUTTONS.Size = New System.Drawing.Size(434, 25) - TP_BUTTONS.TabIndex = 2 + TP_BUTTONS.TabIndex = 1 ' 'BTT_DOWN ' @@ -156,7 +159,7 @@ Namespace API.YouTube.Controls 'SPLITTER_MAIN ' Me.SPLITTER_MAIN.Dock = System.Windows.Forms.DockStyle.Fill - Me.SPLITTER_MAIN.Location = New System.Drawing.Point(3, 115) + Me.SPLITTER_MAIN.Location = New System.Drawing.Point(3, 143) Me.SPLITTER_MAIN.Name = "SPLITTER_MAIN" ' 'SPLITTER_MAIN.Panel1 @@ -272,18 +275,20 @@ Namespace API.YouTube.Controls TP_SETTINGS.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Percent, 100.0!)) TP_SETTINGS.Controls.Add(TP_FORMATS, 0, 1) TP_SETTINGS.Controls.Add(TP_LYRICS, 0, 0) - TP_SETTINGS.Controls.Add(Me.TXT_OUTPUT_PATH, 0, 2) - TP_SETTINGS.Controls.Add(Me.CMB_PLS, 0, 3) + TP_SETTINGS.Controls.Add(Me.TXT_OUTPUT_PATH, 0, 3) + TP_SETTINGS.Controls.Add(Me.CMB_PLS, 0, 4) + TP_SETTINGS.Controls.Add(Me.TXT_AUDIO_BITRATE, 0, 2) TP_SETTINGS.Dock = System.Windows.Forms.DockStyle.Fill TP_SETTINGS.Location = New System.Drawing.Point(0, 0) TP_SETTINGS.Margin = New System.Windows.Forms.Padding(0) TP_SETTINGS.Name = "TP_SETTINGS" - TP_SETTINGS.RowCount = 4 - TP_SETTINGS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 25.0!)) - TP_SETTINGS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 25.0!)) - TP_SETTINGS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 25.0!)) - TP_SETTINGS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 25.0!)) - TP_SETTINGS.Size = New System.Drawing.Size(434, 112) + TP_SETTINGS.RowCount = 5 + TP_SETTINGS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 20.0!)) + TP_SETTINGS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 20.0!)) + TP_SETTINGS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 20.0!)) + TP_SETTINGS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 20.0!)) + TP_SETTINGS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 20.0!)) + TP_SETTINGS.Size = New System.Drawing.Size(434, 140) TP_SETTINGS.TabIndex = 1 ' 'TP_FORMATS @@ -302,7 +307,7 @@ Namespace API.YouTube.Controls TP_FORMATS.RowCount = 1 TP_FORMATS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!)) TP_FORMATS.Size = New System.Drawing.Size(434, 28) - TP_FORMATS.TabIndex = 1 + TP_FORMATS.TabIndex = 5 ' 'TXT_FORMATS_ADDIT ' @@ -371,7 +376,7 @@ Namespace API.YouTube.Controls TP_LYRICS.RowCount = 1 TP_LYRICS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!)) TP_LYRICS.Size = New System.Drawing.Size(434, 28) - TP_LYRICS.TabIndex = 0 + TP_LYRICS.TabIndex = 6 ' 'TXT_SUBS ' @@ -462,7 +467,7 @@ Namespace API.YouTube.Controls Me.TXT_OUTPUT_PATH.Columns.Add(ListColumn1) Me.TXT_OUTPUT_PATH.Columns.Add(ListColumn2) Me.TXT_OUTPUT_PATH.Dock = System.Windows.Forms.DockStyle.Fill - Me.TXT_OUTPUT_PATH.Location = New System.Drawing.Point(3, 59) + Me.TXT_OUTPUT_PATH.Location = New System.Drawing.Point(3, 87) Me.TXT_OUTPUT_PATH.Name = "TXT_OUTPUT_PATH" Me.TXT_OUTPUT_PATH.Size = New System.Drawing.Size(428, 22) Me.TXT_OUTPUT_PATH.TabIndex = 2 @@ -505,23 +510,39 @@ Namespace API.YouTube.Controls Me.CMB_PLS.CaptionVisible = True Me.CMB_PLS.CaptionWidth = 50.0R Me.CMB_PLS.Dock = System.Windows.Forms.DockStyle.Fill - Me.CMB_PLS.Location = New System.Drawing.Point(3, 87) + Me.CMB_PLS.Location = New System.Drawing.Point(3, 115) Me.CMB_PLS.Name = "CMB_PLS" Me.CMB_PLS.Size = New System.Drawing.Size(428, 22) Me.CMB_PLS.TabIndex = 3 Me.CMB_PLS.TextBoxBorderStyle = System.Windows.Forms.BorderStyle.FixedSingle ' + 'TXT_AUDIO_BITRATE + ' + ActionButton18.BackgroundImage = CType(resources.GetObject("ActionButton18.BackgroundImage"), System.Drawing.Image) + ActionButton18.Name = "Clear" + ActionButton18.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Clear + Me.TXT_AUDIO_BITRATE.Buttons.Add(ActionButton18) + Me.TXT_AUDIO_BITRATE.CaptionText = "Audio bitrate" + Me.TXT_AUDIO_BITRATE.CaptionToolTipEnabled = True + Me.TXT_AUDIO_BITRATE.CaptionToolTipText = "Default audio bitrate if you want to change it during download" + Me.TXT_AUDIO_BITRATE.CaptionWidth = 112.0R + Me.TXT_AUDIO_BITRATE.Dock = System.Windows.Forms.DockStyle.Fill + Me.TXT_AUDIO_BITRATE.Location = New System.Drawing.Point(3, 59) + Me.TXT_AUDIO_BITRATE.Name = "TXT_AUDIO_BITRATE" + Me.TXT_AUDIO_BITRATE.Size = New System.Drawing.Size(428, 22) + Me.TXT_AUDIO_BITRATE.TabIndex = 4 + ' 'MusicPlaylistsForm ' Me.AcceptButton = Me.BTT_DOWN Me.AutoScaleDimensions = New System.Drawing.SizeF(6.0!, 13.0!) Me.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font Me.CancelButton = Me.BTT_CANCEL - Me.ClientSize = New System.Drawing.Size(434, 289) + Me.ClientSize = New System.Drawing.Size(434, 317) Me.Controls.Add(TP_MAIN) Me.Icon = Global.SCrawler.My.Resources.SiteYouTube.YouTubeMusicIcon_32 Me.KeyPreview = True - Me.MinimumSize = New System.Drawing.Size(450, 328) + Me.MinimumSize = New System.Drawing.Size(450, 356) Me.Name = "MusicPlaylistsForm" Me.Text = "Albums" TP_MAIN.ResumeLayout(False) @@ -541,6 +562,7 @@ Namespace API.YouTube.Controls CType(Me.TXT_SUBS, System.ComponentModel.ISupportInitialize).EndInit() CType(Me.TXT_OUTPUT_PATH, System.ComponentModel.ISupportInitialize).EndInit() CType(Me.CMB_PLS, System.ComponentModel.ISupportInitialize).EndInit() + CType(Me.TXT_AUDIO_BITRATE, System.ComponentModel.ISupportInitialize).EndInit() Me.ResumeLayout(False) End Sub @@ -557,5 +579,6 @@ Namespace API.YouTube.Controls Private WithEvents CH_DOWN_LYRICS As CheckBox Private WithEvents TXT_OUTPUT_PATH As PersonalUtilities.Forms.Controls.ComboBoxExtended Private WithEvents CMB_PLS As PersonalUtilities.Forms.Controls.ComboBoxExtended + Private WithEvents TXT_AUDIO_BITRATE As PersonalUtilities.Forms.Controls.TextBoxExtended End Class End Namespace \ No newline at end of file diff --git a/SCrawler.YouTube/Controls/MusicPlaylistsForm.resx b/SCrawler.YouTube/Controls/MusicPlaylistsForm.resx index dc6a92c..0d277dd 100644 --- a/SCrawler.YouTube/Controls/MusicPlaylistsForm.resx +++ b/SCrawler.YouTube/Controls/MusicPlaylistsForm.resx @@ -504,6 +504,14 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6LEtW/4flgYiLD1qeX0A AAAASUVORK5CYII= + + + + + iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO + xAAADsQBlSsOGwAAAIZJREFUOE+1j10KwCAMgz2b755xl/IsvnaL2K20UfbDAmEako+ZROSTafjE12Go + tbbB43rK5xSAQq1VYFtmeQBoqZTSreVZvgTknM8yyyjA/qodsDF9gspD2Bj6B+DH+NqzhQQAG+POMnSX + AFuc5QFgn6ClHh5iOQVAKNixyucB8NY0vG9JOzzyhrdq5IRgAAAAAElFTkSuQmCC \ No newline at end of file diff --git a/SCrawler.YouTube/Controls/MusicPlaylistsForm.vb b/SCrawler.YouTube/Controls/MusicPlaylistsForm.vb index e267b26..c509861 100644 --- a/SCrawler.YouTube/Controls/MusicPlaylistsForm.vb +++ b/SCrawler.YouTube/Controls/MusicPlaylistsForm.vb @@ -19,6 +19,7 @@ Namespace API.YouTube.Controls Friend Class MusicPlaylistsForm : Implements IDesignXMLContainer #Region "Declarations" Private MyView As FormView + Private ReadOnly MyFieldsChecker As FieldsChecker Friend Property DesignXML As EContainer Implements IDesignXMLContainer.DesignXML Private Property DesignXMLNodes As String() Implements IDesignXMLContainer.DesignXMLNodes Private Property DesignXMLNodeName As String Implements IDesignXMLContainer.DesignXMLNodeName @@ -48,6 +49,7 @@ Namespace API.YouTube.Controls InitializeComponent() M3U8Files = New List(Of SFile) MyContainer = Container + MyFieldsChecker = New FieldsChecker End Sub #End Region #Region "Form handlers" @@ -61,6 +63,7 @@ Namespace API.YouTube.Controls MyYouTubeSettings.DownloadLocations.PopulateComboBox(TXT_OUTPUT_PATH) MyYouTubeSettings.PlaylistsLocations.PopulateComboBox(CMB_PLS,, True) CMB_PLS.Text = MyYouTubeSettings.LatestPlaylistFile.Value + If MyYouTubeSettings.DefaultAudioBitrate > 0 Then TXT_AUDIO_BITRATE.Text = MyYouTubeSettings.DefaultAudioBitrate.Value CMB_FORMATS.Items.AddRange(AvailableAudioFormats) If MyYouTubeSettings.PlaylistFormSplitterDistance > 0 Then SPLITTER_MAIN.SplitterDistancePercentageSet(MyYouTubeSettings.PlaylistFormSplitterDistance) @@ -113,6 +116,9 @@ Namespace API.YouTube.Controls Text = .PlaylistTitle End If + MyFieldsChecker.AddControl(Of Integer)(TXT_AUDIO_BITRATE, TXT_AUDIO_BITRATE.CaptionText, True) + MyFieldsChecker.EndLoaderOperations() + UpdateSizeText() End With RefillAddit() @@ -120,7 +126,8 @@ Namespace API.YouTube.Controls End Sub Private Sub MusicPlaylistsForm_Closing(sender As Object, e As CancelEventArgs) Handles Me.Closing MyYouTubeSettings.PlaylistFormSplitterDistance.Value = SPLITTER_MAIN.SplitterDistancePercentageGet - MyView.DisposeIfReady() + MyView.DisposeIfReady + MyFieldsChecker.DisposeIfReady M3U8Files.Clear() End Sub Private Sub MusicPlaylistsForm_KeyDown(sender As Object, e As KeyEventArgs) Handles Me.KeyDown @@ -322,7 +329,7 @@ Namespace API.YouTube.Controls Private Sub BTT_DOWN_Click(sender As Object, e As EventArgs) Handles BTT_DOWN.Click If TXT_OUTPUT_PATH.IsEmptyString Then MsgBoxE({"The output path cannot be null.", "Download music"}, vbCritical) - Else + ElseIf MyFieldsChecker.AllParamsOK Then With DirectCast(MyContainer, YouTubeMediaContainerBase) .OutputSubtitlesFormat = IIf(CH_DOWN_LYRICS.Checked, "LRC", String.Empty) If Not TXT_SUBS.Checked Then .PostProcessing_OutputSubtitlesFormats.Clear() @@ -331,6 +338,7 @@ Namespace API.YouTube.Controls .AbsolutePath = TXT_OUTPUT_PATH.Checked .File = TXT_OUTPUT_PATH.Text.CSFileP .M3U8_PlaylistFiles = M3U8FilesFull + .OutputAudioBitrate = AConvert(Of Integer)(TXT_AUDIO_BITRATE.Text, -1) If MyYouTubeSettings.OutputPathAutoChange Then MyYouTubeSettings.OutputPath.Value = .File If MyDownloaderSettings.OutputPathAutoAddPaths Then MyYouTubeSettings.DownloadLocations.Add(.File, False) If Not CMB_PLS.Text.IsEmptyString Then MyYouTubeSettings.PlaylistsLocations.Add(CMB_PLS.Text, False, True) diff --git a/SCrawler.YouTube/Controls/VideoOption.vb b/SCrawler.YouTube/Controls/VideoOption.vb index 71cdd3b..424e861 100644 --- a/SCrawler.YouTube/Controls/VideoOption.vb +++ b/SCrawler.YouTube/Controls/VideoOption.vb @@ -27,6 +27,7 @@ Namespace API.YouTube.Controls Friend Sub New(ByVal m As MediaObject, Optional ByVal SelectedAudio As MediaObject = Nothing) Me.New Const d$ = " " & ChrW(183) & " " + Const DRC$ = Objects.YouTubeMediaContainerBase.DRC MyMedia = m If m.Type = Plugin.UserMediaTypes.Audio Then If m.Bitrate >= 320 Then @@ -38,6 +39,7 @@ Namespace API.YouTube.Controls End If LBL_DEFINITION.Text = $"{m.Bitrate}k" LBL_CODECS.Text = $"{m.Extension} {d} {m.Codec} {d} {m.Bitrate}k" + If Not m.ID.IsEmptyString AndAlso m.ID.StringToLower.Contains(DRC) Then LBL_CODECS.Text &= $" {d} DRC" Else If m.Height >= 1440 Then LBL_DEFINITION_INFO.Text = "Ultra High Definition" @@ -53,7 +55,9 @@ Namespace API.YouTube.Controls LBL_DEFINITION.Text = $"{m.Height}p" LBL_CODECS.Text = $"{m.Extension.StringToUpper}{d}{m.Codec.StringToUpper}{d}{m.FPS}fps{d}{m.Bitrate}k" If Not m.Protocol.IsEmptyString Then LBL_CODECS.Text &= $" ({m.Protocol})" + If Not m.ID.IsEmptyString AndAlso m.ID.StringToLower.Contains(DRC) Then LBL_CODECS.Text &= $"{d}DRC" If Not SelectedAudio.ID.IsEmptyString Then LBL_CODECS.Text &= $" / {SelectedAudio.Extension}{d}{SelectedAudio.Codec}{d}{SelectedAudio.Bitrate}k" + If Not SelectedAudio.ID.IsEmptyString AndAlso SelectedAudio.ID.StringToLower.Contains(DRC) Then LBL_CODECS.Text &= $"{d}DRC" End If Dim sv% = m.Size / 1024 diff --git a/SCrawler.YouTube/Controls/VideoOptionsForm.Designer.vb b/SCrawler.YouTube/Controls/VideoOptionsForm.Designer.vb index f39f9c3..56673c8 100644 --- a/SCrawler.YouTube/Controls/VideoOptionsForm.Designer.vb +++ b/SCrawler.YouTube/Controls/VideoOptionsForm.Designer.vb @@ -47,6 +47,7 @@ Namespace API.YouTube.Controls Dim LBL_FORMAT As System.Windows.Forms.Label Dim LBL_SUBS_FORMAT As System.Windows.Forms.Label Dim TT_MAIN As System.Windows.Forms.ToolTip + Dim TP_FPS_BITRATE As System.Windows.Forms.TableLayoutPanel Dim ActionButton7 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton() Dim ActionButton8 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton() Dim ActionButton9 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton() @@ -57,6 +58,7 @@ Namespace API.YouTube.Controls Dim ActionButton14 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton() Dim ActionButton15 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton() Dim ActionButton16 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton() + Dim ActionButton17 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton() Me.ICON_VIDEO = New System.Windows.Forms.PictureBox() Me.LBL_TITLE = New System.Windows.Forms.Label() Me.TP_HEADER_INFO_2 = New System.Windows.Forms.TableLayoutPanel() @@ -71,6 +73,8 @@ Namespace API.YouTube.Controls Me.OPT_VIDEO = New System.Windows.Forms.RadioButton() Me.OPT_AUDIO = New System.Windows.Forms.RadioButton() Me.LBL_AUDIO_CODEC = New System.Windows.Forms.Label() + Me.TXT_FPS = New PersonalUtilities.Forms.Controls.TextBoxExtended() + Me.TXT_AUDIO_BITRATE = New PersonalUtilities.Forms.Controls.TextBoxExtended() Me.TP_HEADER_BASE = New System.Windows.Forms.TableLayoutPanel() Me.TP_SUBS = New System.Windows.Forms.TableLayoutPanel() Me.TXT_SUBS = New PersonalUtilities.Forms.Controls.TextBoxExtended() @@ -80,7 +84,6 @@ Namespace API.YouTube.Controls Me.CMB_FORMAT = New System.Windows.Forms.ComboBox() Me.CMB_AUDIO_CODEC = New System.Windows.Forms.ComboBox() Me.NUM_RES = New System.Windows.Forms.NumericUpDown() - Me.TXT_FPS = New PersonalUtilities.Forms.Controls.TextBoxExtended() Me.TP_CONTROLS = New System.Windows.Forms.TableLayoutPanel() Me.TXT_SUBS_ADDIT = New PersonalUtilities.Forms.Controls.TextBoxExtended() Me.TXT_EXTRA_AUDIO_FORMATS = New PersonalUtilities.Forms.Controls.TextBoxExtended() @@ -99,6 +102,7 @@ Namespace API.YouTube.Controls LBL_FORMAT = New System.Windows.Forms.Label() LBL_SUBS_FORMAT = New System.Windows.Forms.Label() TT_MAIN = New System.Windows.Forms.ToolTip(Me.components) + TP_FPS_BITRATE = New System.Windows.Forms.TableLayoutPanel() TP_HEADER.SuspendLayout() CType(Me.ICON_VIDEO, System.ComponentModel.ISupportInitialize).BeginInit() TP_HEADER_INFO.SuspendLayout() @@ -112,13 +116,15 @@ Namespace API.YouTube.Controls TP_PLS.SuspendLayout() CType(Me.CMB_PLS, System.ComponentModel.ISupportInitialize).BeginInit() TP_WHAT.SuspendLayout() + TP_FPS_BITRATE.SuspendLayout() + CType(Me.TXT_FPS, System.ComponentModel.ISupportInitialize).BeginInit() + CType(Me.TXT_AUDIO_BITRATE, System.ComponentModel.ISupportInitialize).BeginInit() Me.TP_HEADER_BASE.SuspendLayout() Me.TP_SUBS.SuspendLayout() CType(Me.TXT_SUBS, System.ComponentModel.ISupportInitialize).BeginInit() Me.TP_MAIN.SuspendLayout() Me.TP_OPTIONS.SuspendLayout() CType(Me.NUM_RES, System.ComponentModel.ISupportInitialize).BeginInit() - CType(Me.TXT_FPS, System.ComponentModel.ISupportInitialize).BeginInit() CType(Me.TXT_SUBS_ADDIT, System.ComponentModel.ISupportInitialize).BeginInit() CType(Me.TXT_EXTRA_AUDIO_FORMATS, System.ComponentModel.ISupportInitialize).BeginInit() Me.SuspendLayout() @@ -137,7 +143,7 @@ Namespace API.YouTube.Controls TP_HEADER.Name = "TP_HEADER" TP_HEADER.RowCount = 1 TP_HEADER.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!)) - TP_HEADER.Size = New System.Drawing.Size(719, 63) + TP_HEADER.Size = New System.Drawing.Size(599, 63) TP_HEADER.TabIndex = 0 ' 'ICON_VIDEO @@ -166,7 +172,7 @@ Namespace API.YouTube.Controls TP_HEADER_INFO.RowCount = 2 TP_HEADER_INFO.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 50.0!)) TP_HEADER_INFO.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 50.0!)) - TP_HEADER_INFO.Size = New System.Drawing.Size(589, 63) + TP_HEADER_INFO.Size = New System.Drawing.Size(469, 63) TP_HEADER_INFO.TabIndex = 0 ' 'LBL_TITLE @@ -175,7 +181,7 @@ Namespace API.YouTube.Controls Me.LBL_TITLE.Font = New System.Drawing.Font("Arial", 9.0!, System.Drawing.FontStyle.Bold, System.Drawing.GraphicsUnit.Point, CType(204, Byte)) Me.LBL_TITLE.Location = New System.Drawing.Point(3, 0) Me.LBL_TITLE.Name = "LBL_TITLE" - Me.LBL_TITLE.Size = New System.Drawing.Size(583, 31) + Me.LBL_TITLE.Size = New System.Drawing.Size(463, 31) Me.LBL_TITLE.TabIndex = 0 Me.LBL_TITLE.Text = "Video title" Me.LBL_TITLE.TextAlign = System.Drawing.ContentAlignment.MiddleLeft @@ -197,7 +203,7 @@ Namespace API.YouTube.Controls Me.TP_HEADER_INFO_2.Name = "TP_HEADER_INFO_2" Me.TP_HEADER_INFO_2.RowCount = 1 Me.TP_HEADER_INFO_2.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!)) - Me.TP_HEADER_INFO_2.Size = New System.Drawing.Size(589, 32) + Me.TP_HEADER_INFO_2.Size = New System.Drawing.Size(469, 32) Me.TP_HEADER_INFO_2.TabIndex = 1 ' 'ICON_CLOCK @@ -244,7 +250,7 @@ Namespace API.YouTube.Controls Me.LBL_URL.LinkColor = System.Drawing.Color.FromArgb(CType(CType(0, Byte), Integer), CType(CType(0, Byte), Integer), CType(CType(192, Byte), Integer)) Me.LBL_URL.Location = New System.Drawing.Point(115, 0) Me.LBL_URL.Name = "LBL_URL" - Me.LBL_URL.Size = New System.Drawing.Size(471, 32) + Me.LBL_URL.Size = New System.Drawing.Size(351, 32) Me.LBL_URL.TabIndex = 1 Me.LBL_URL.TabStop = True Me.LBL_URL.Text = "https://www.youtube.com/watch?v=abcdefghijk" @@ -258,14 +264,14 @@ Namespace API.YouTube.Controls TP_FOOTER.Controls.Add(TP_OK_CANCEL, 0, 2) TP_FOOTER.Controls.Add(TP_PLS, 0, 0) TP_FOOTER.Dock = System.Windows.Forms.DockStyle.Fill - TP_FOOTER.Location = New System.Drawing.Point(6, 215) + TP_FOOTER.Location = New System.Drawing.Point(6, 243) TP_FOOTER.Margin = New System.Windows.Forms.Padding(6, 3, 6, 3) TP_FOOTER.Name = "TP_FOOTER" TP_FOOTER.RowCount = 3 TP_FOOTER.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 33.33333!)) TP_FOOTER.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 33.33333!)) TP_FOOTER.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 33.33333!)) - TP_FOOTER.Size = New System.Drawing.Size(709, 81) + TP_FOOTER.Size = New System.Drawing.Size(589, 81) TP_FOOTER.TabIndex = 5 ' 'TP_DESTINATION @@ -281,7 +287,7 @@ Namespace API.YouTube.Controls TP_DESTINATION.Name = "TP_DESTINATION" TP_DESTINATION.RowCount = 1 TP_DESTINATION.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!)) - TP_DESTINATION.Size = New System.Drawing.Size(709, 27) + TP_DESTINATION.Size = New System.Drawing.Size(589, 27) TP_DESTINATION.TabIndex = 0 ' 'TXT_FILE @@ -310,14 +316,14 @@ Namespace API.YouTube.Controls Me.TXT_FILE.Location = New System.Drawing.Point(1, 1) Me.TXT_FILE.Margin = New System.Windows.Forms.Padding(1) Me.TXT_FILE.Name = "TXT_FILE" - Me.TXT_FILE.Size = New System.Drawing.Size(627, 22) + Me.TXT_FILE.Size = New System.Drawing.Size(507, 22) Me.TXT_FILE.TabIndex = 0 Me.TXT_FILE.TextBoxBorderStyle = System.Windows.Forms.BorderStyle.FixedSingle ' 'BTT_BROWSE ' Me.BTT_BROWSE.Dock = System.Windows.Forms.DockStyle.Fill - Me.BTT_BROWSE.Location = New System.Drawing.Point(632, 2) + Me.BTT_BROWSE.Location = New System.Drawing.Point(512, 2) Me.BTT_BROWSE.Margin = New System.Windows.Forms.Padding(3, 2, 3, 2) Me.BTT_BROWSE.Name = "BTT_BROWSE" Me.BTT_BROWSE.Size = New System.Drawing.Size(74, 23) @@ -341,13 +347,13 @@ Namespace API.YouTube.Controls TP_OK_CANCEL.RowCount = 1 TP_OK_CANCEL.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!)) TP_OK_CANCEL.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 27.0!)) - TP_OK_CANCEL.Size = New System.Drawing.Size(709, 27) + TP_OK_CANCEL.Size = New System.Drawing.Size(589, 27) TP_OK_CANCEL.TabIndex = 1 ' 'BTT_DOWN ' Me.BTT_DOWN.Dock = System.Windows.Forms.DockStyle.Fill - Me.BTT_DOWN.Location = New System.Drawing.Point(552, 2) + Me.BTT_DOWN.Location = New System.Drawing.Point(432, 2) Me.BTT_DOWN.Margin = New System.Windows.Forms.Padding(3, 2, 3, 2) Me.BTT_DOWN.Name = "BTT_DOWN" Me.BTT_DOWN.Size = New System.Drawing.Size(74, 23) @@ -359,7 +365,7 @@ Namespace API.YouTube.Controls ' Me.BTT_CANCEL.DialogResult = System.Windows.Forms.DialogResult.Cancel Me.BTT_CANCEL.Dock = System.Windows.Forms.DockStyle.Fill - Me.BTT_CANCEL.Location = New System.Drawing.Point(632, 2) + Me.BTT_CANCEL.Location = New System.Drawing.Point(512, 2) Me.BTT_CANCEL.Margin = New System.Windows.Forms.Padding(3, 2, 3, 2) Me.BTT_CANCEL.Name = "BTT_CANCEL" Me.BTT_CANCEL.Size = New System.Drawing.Size(74, 23) @@ -381,7 +387,7 @@ Namespace API.YouTube.Controls TP_PLS.RowCount = 1 TP_PLS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!)) TP_PLS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 27.0!)) - TP_PLS.Size = New System.Drawing.Size(709, 27) + TP_PLS.Size = New System.Drawing.Size(589, 27) TP_PLS.TabIndex = 2 ' 'CMB_PLS @@ -414,14 +420,14 @@ Namespace API.YouTube.Controls Me.CMB_PLS.Location = New System.Drawing.Point(1, 1) Me.CMB_PLS.Margin = New System.Windows.Forms.Padding(1) Me.CMB_PLS.Name = "CMB_PLS" - Me.CMB_PLS.Size = New System.Drawing.Size(627, 22) + Me.CMB_PLS.Size = New System.Drawing.Size(507, 22) Me.CMB_PLS.TabIndex = 0 Me.CMB_PLS.TextBoxBorderStyle = System.Windows.Forms.BorderStyle.FixedSingle ' 'BTT_PLS_BROWSE ' Me.BTT_PLS_BROWSE.Dock = System.Windows.Forms.DockStyle.Fill - Me.BTT_PLS_BROWSE.Location = New System.Drawing.Point(632, 2) + Me.BTT_PLS_BROWSE.Location = New System.Drawing.Point(512, 2) Me.BTT_PLS_BROWSE.Margin = New System.Windows.Forms.Padding(3, 2, 3, 2) Me.BTT_PLS_BROWSE.Name = "BTT_PLS_BROWSE" Me.BTT_PLS_BROWSE.Size = New System.Drawing.Size(74, 23) @@ -434,20 +440,20 @@ Namespace API.YouTube.Controls ' LB_SEP_1.Anchor = CType((System.Windows.Forms.AnchorStyles.Left Or System.Windows.Forms.AnchorStyles.Right), System.Windows.Forms.AnchorStyles) LB_SEP_1.BackColor = System.Drawing.SystemColors.ControlDark - LB_SEP_1.Location = New System.Drawing.Point(6, 179) + LB_SEP_1.Location = New System.Drawing.Point(6, 207) LB_SEP_1.Margin = New System.Windows.Forms.Padding(6, 0, 6, 0) LB_SEP_1.Name = "LB_SEP_1" - LB_SEP_1.Size = New System.Drawing.Size(709, 1) + LB_SEP_1.Size = New System.Drawing.Size(589, 1) LB_SEP_1.TabIndex = 3 ' 'LB_SEP_2 ' LB_SEP_2.Anchor = CType((System.Windows.Forms.AnchorStyles.Left Or System.Windows.Forms.AnchorStyles.Right), System.Windows.Forms.AnchorStyles) LB_SEP_2.BackColor = System.Drawing.SystemColors.ControlDark - LB_SEP_2.Location = New System.Drawing.Point(6, 209) + LB_SEP_2.Location = New System.Drawing.Point(6, 237) LB_SEP_2.Margin = New System.Windows.Forms.Padding(6, 0, 6, 0) LB_SEP_2.Name = "LB_SEP_2" - LB_SEP_2.Size = New System.Drawing.Size(709, 1) + LB_SEP_2.Size = New System.Drawing.Size(589, 1) LB_SEP_2.TabIndex = 5 ' 'TP_WHAT @@ -519,7 +525,7 @@ Namespace API.YouTube.Controls ' LBL_SUBS_FORMAT.AutoSize = True LBL_SUBS_FORMAT.Dock = System.Windows.Forms.DockStyle.Fill - LBL_SUBS_FORMAT.Location = New System.Drawing.Point(552, 0) + LBL_SUBS_FORMAT.Location = New System.Drawing.Point(432, 0) LBL_SUBS_FORMAT.Name = "LBL_SUBS_FORMAT" LBL_SUBS_FORMAT.Size = New System.Drawing.Size(74, 28) LBL_SUBS_FORMAT.TabIndex = 2 @@ -531,7 +537,7 @@ Namespace API.YouTube.Controls ' Me.LBL_AUDIO_CODEC.AutoSize = True Me.LBL_AUDIO_CODEC.Dock = System.Windows.Forms.DockStyle.Fill - Me.LBL_AUDIO_CODEC.Location = New System.Drawing.Point(552, 0) + Me.LBL_AUDIO_CODEC.Location = New System.Drawing.Point(432, 0) Me.LBL_AUDIO_CODEC.Name = "LBL_AUDIO_CODEC" Me.LBL_AUDIO_CODEC.Size = New System.Drawing.Size(74, 28) Me.LBL_AUDIO_CODEC.TabIndex = 5 @@ -539,6 +545,59 @@ Namespace API.YouTube.Controls Me.LBL_AUDIO_CODEC.TextAlign = System.Drawing.ContentAlignment.MiddleRight TT_MAIN.SetToolTip(Me.LBL_AUDIO_CODEC, "Output Audio Codec") ' + 'TP_FPS_BITRATE + ' + TP_FPS_BITRATE.ColumnCount = 2 + TP_FPS_BITRATE.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Percent, 50.0!)) + TP_FPS_BITRATE.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Percent, 50.0!)) + TP_FPS_BITRATE.Controls.Add(Me.TXT_FPS, 0, 0) + TP_FPS_BITRATE.Controls.Add(Me.TXT_AUDIO_BITRATE, 1, 0) + TP_FPS_BITRATE.Dock = System.Windows.Forms.DockStyle.Fill + TP_FPS_BITRATE.Location = New System.Drawing.Point(6, 93) + TP_FPS_BITRATE.Margin = New System.Windows.Forms.Padding(6, 0, 6, 0) + TP_FPS_BITRATE.Name = "TP_FPS_BITRATE" + TP_FPS_BITRATE.RowCount = 1 + TP_FPS_BITRATE.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!)) + TP_FPS_BITRATE.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!)) + TP_FPS_BITRATE.Size = New System.Drawing.Size(589, 28) + TP_FPS_BITRATE.TabIndex = 6 + ' + 'TXT_FPS + ' + ActionButton7.BackgroundImage = CType(resources.GetObject("ActionButton7.BackgroundImage"), System.Drawing.Image) + ActionButton7.Name = "Clear" + ActionButton7.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Clear + Me.TXT_FPS.Buttons.Add(ActionButton7) + Me.TXT_FPS.CaptionText = "Video FPS" + Me.TXT_FPS.CaptionToolTipEnabled = True + Me.TXT_FPS.CaptionToolTipText = "Set the video FPS by setting the FPS value in this field. Leave blank so as not t" & + "o change" + Me.TXT_FPS.CaptionWidth = 60.0R + Me.TXT_FPS.Dock = System.Windows.Forms.DockStyle.Fill + Me.TXT_FPS.Location = New System.Drawing.Point(3, 2) + Me.TXT_FPS.Margin = New System.Windows.Forms.Padding(3, 2, 3, 3) + Me.TXT_FPS.Name = "TXT_FPS" + Me.TXT_FPS.Size = New System.Drawing.Size(288, 22) + Me.TXT_FPS.TabIndex = 0 + Me.TXT_FPS.TextBoxWidthMinimal = 30 + ' + 'TXT_AUDIO_BITRATE + ' + ActionButton8.BackgroundImage = CType(resources.GetObject("ActionButton8.BackgroundImage"), System.Drawing.Image) + ActionButton8.Name = "Clear" + ActionButton8.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Clear + Me.TXT_AUDIO_BITRATE.Buttons.Add(ActionButton8) + Me.TXT_AUDIO_BITRATE.CaptionText = "Audio bitrate" + Me.TXT_AUDIO_BITRATE.CaptionToolTipEnabled = True + Me.TXT_AUDIO_BITRATE.CaptionToolTipText = "Set the video FPS if you want to change it during download. Leave blank so as not" & + " to change." + Me.TXT_AUDIO_BITRATE.CaptionWidth = 75.0R + Me.TXT_AUDIO_BITRATE.Dock = System.Windows.Forms.DockStyle.Fill + Me.TXT_AUDIO_BITRATE.Location = New System.Drawing.Point(297, 3) + Me.TXT_AUDIO_BITRATE.Name = "TXT_AUDIO_BITRATE" + Me.TXT_AUDIO_BITRATE.Size = New System.Drawing.Size(289, 22) + Me.TXT_AUDIO_BITRATE.TabIndex = 1 + ' 'TP_HEADER_BASE ' Me.TP_HEADER_BASE.CellBorderStyle = System.Windows.Forms.TableLayoutPanelCellBorderStyle.[Single] @@ -553,8 +612,8 @@ Namespace API.YouTube.Controls Me.TP_HEADER_BASE.RowCount = 1 Me.TP_HEADER_BASE.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!)) Me.TP_HEADER_BASE.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 64.0!)) - Me.TP_HEADER_BASE.Size = New System.Drawing.Size(721, 65) - Me.TP_HEADER_BASE.TabIndex = 6 + Me.TP_HEADER_BASE.Size = New System.Drawing.Size(601, 65) + Me.TP_HEADER_BASE.TabIndex = 7 ' 'TP_SUBS ' @@ -566,31 +625,31 @@ Namespace API.YouTube.Controls Me.TP_SUBS.Controls.Add(LBL_SUBS_FORMAT, 1, 0) Me.TP_SUBS.Controls.Add(Me.CMB_SUBS_FORMAT, 2, 0) Me.TP_SUBS.Dock = System.Windows.Forms.DockStyle.Fill - Me.TP_SUBS.Location = New System.Drawing.Point(6, 93) + Me.TP_SUBS.Location = New System.Drawing.Point(6, 121) Me.TP_SUBS.Margin = New System.Windows.Forms.Padding(6, 0, 6, 0) Me.TP_SUBS.Name = "TP_SUBS" Me.TP_SUBS.RowCount = 1 Me.TP_SUBS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!)) - Me.TP_SUBS.Size = New System.Drawing.Size(709, 28) + Me.TP_SUBS.Size = New System.Drawing.Size(589, 28) Me.TP_SUBS.TabIndex = 2 ' 'TXT_SUBS ' - ActionButton7.BackgroundImage = CType(resources.GetObject("ActionButton7.BackgroundImage"), System.Drawing.Image) - ActionButton7.Name = "Open" - ActionButton7.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Open - ActionButton7.ToolTipText = "Choose subtitles" - ActionButton8.BackgroundImage = CType(resources.GetObject("ActionButton8.BackgroundImage"), System.Drawing.Image) - ActionButton8.Name = "Refresh" - ActionButton8.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Refresh - ActionButton8.ToolTipText = "Reset subtitles to initial selected" ActionButton9.BackgroundImage = CType(resources.GetObject("ActionButton9.BackgroundImage"), System.Drawing.Image) - ActionButton9.Name = "Clear" - ActionButton9.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Clear - ActionButton9.ToolTipText = "Clear subtitles selection (don't download subtitles)" - Me.TXT_SUBS.Buttons.Add(ActionButton7) - Me.TXT_SUBS.Buttons.Add(ActionButton8) + ActionButton9.Name = "Open" + ActionButton9.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Open + ActionButton9.ToolTipText = "Choose subtitles" + ActionButton10.BackgroundImage = CType(resources.GetObject("ActionButton10.BackgroundImage"), System.Drawing.Image) + ActionButton10.Name = "Refresh" + ActionButton10.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Refresh + ActionButton10.ToolTipText = "Reset subtitles to initial selected" + ActionButton11.BackgroundImage = CType(resources.GetObject("ActionButton11.BackgroundImage"), System.Drawing.Image) + ActionButton11.Name = "Clear" + ActionButton11.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Clear + ActionButton11.ToolTipText = "Clear subtitles selection (don't download subtitles)" Me.TXT_SUBS.Buttons.Add(ActionButton9) + Me.TXT_SUBS.Buttons.Add(ActionButton10) + Me.TXT_SUBS.Buttons.Add(ActionButton11) Me.TXT_SUBS.CaptionText = "Subtitles" Me.TXT_SUBS.CaptionToolTipEnabled = True Me.TXT_SUBS.CaptionToolTipText = "The selected subtitles will also be downloaded" @@ -599,7 +658,7 @@ Namespace API.YouTube.Controls Me.TXT_SUBS.Dock = System.Windows.Forms.DockStyle.Fill Me.TXT_SUBS.Location = New System.Drawing.Point(3, 3) Me.TXT_SUBS.Name = "TXT_SUBS" - Me.TXT_SUBS.Size = New System.Drawing.Size(543, 22) + Me.TXT_SUBS.Size = New System.Drawing.Size(423, 22) Me.TXT_SUBS.TabIndex = 0 Me.TXT_SUBS.TextBoxReadOnly = True ' @@ -608,7 +667,7 @@ Namespace API.YouTube.Controls Me.CMB_SUBS_FORMAT.Dock = System.Windows.Forms.DockStyle.Fill Me.CMB_SUBS_FORMAT.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList Me.CMB_SUBS_FORMAT.FormattingEnabled = True - Me.CMB_SUBS_FORMAT.Location = New System.Drawing.Point(632, 3) + Me.CMB_SUBS_FORMAT.Location = New System.Drawing.Point(512, 3) Me.CMB_SUBS_FORMAT.Name = "CMB_SUBS_FORMAT" Me.CMB_SUBS_FORMAT.Size = New System.Drawing.Size(74, 21) Me.CMB_SUBS_FORMAT.TabIndex = 1 @@ -618,55 +677,56 @@ Namespace API.YouTube.Controls Me.TP_MAIN.ColumnCount = 1 Me.TP_MAIN.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Percent, 100.0!)) Me.TP_MAIN.Controls.Add(Me.TP_HEADER_BASE, 0, 0) - Me.TP_MAIN.Controls.Add(TP_FOOTER, 0, 8) + Me.TP_MAIN.Controls.Add(TP_FOOTER, 0, 9) Me.TP_MAIN.Controls.Add(Me.TP_OPTIONS, 0, 1) - Me.TP_MAIN.Controls.Add(Me.TP_CONTROLS, 0, 6) - Me.TP_MAIN.Controls.Add(LB_SEP_1, 0, 5) - Me.TP_MAIN.Controls.Add(LB_SEP_2, 0, 7) - Me.TP_MAIN.Controls.Add(Me.TP_SUBS, 0, 2) - Me.TP_MAIN.Controls.Add(Me.TXT_SUBS_ADDIT, 0, 3) - Me.TP_MAIN.Controls.Add(Me.TXT_EXTRA_AUDIO_FORMATS, 0, 4) + Me.TP_MAIN.Controls.Add(Me.TP_CONTROLS, 0, 7) + Me.TP_MAIN.Controls.Add(LB_SEP_1, 0, 6) + Me.TP_MAIN.Controls.Add(LB_SEP_2, 0, 8) + Me.TP_MAIN.Controls.Add(Me.TP_SUBS, 0, 3) + Me.TP_MAIN.Controls.Add(Me.TXT_SUBS_ADDIT, 0, 4) + Me.TP_MAIN.Controls.Add(Me.TXT_EXTRA_AUDIO_FORMATS, 0, 5) + Me.TP_MAIN.Controls.Add(TP_FPS_BITRATE, 0, 2) Me.TP_MAIN.Dock = System.Windows.Forms.DockStyle.Fill Me.TP_MAIN.Location = New System.Drawing.Point(0, 0) Me.TP_MAIN.Name = "TP_MAIN" - Me.TP_MAIN.RowCount = 10 + Me.TP_MAIN.RowCount = 11 Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 65.0!)) Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!)) Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!)) Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!)) Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!)) + Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!)) Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 5.0!)) Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!)) Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 5.0!)) Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 87.0!)) Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle()) - Me.TP_MAIN.Size = New System.Drawing.Size(721, 300) + Me.TP_MAIN.Size = New System.Drawing.Size(601, 328) Me.TP_MAIN.TabIndex = 0 ' 'TP_OPTIONS ' - Me.TP_OPTIONS.ColumnCount = 7 + Me.TP_OPTIONS.ColumnCount = 6 Me.TP_OPTIONS.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Percent, 100.0!)) Me.TP_OPTIONS.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Absolute, 80.0!)) Me.TP_OPTIONS.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Absolute, 80.0!)) Me.TP_OPTIONS.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Absolute, 80.0!)) - Me.TP_OPTIONS.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Absolute, 120.0!)) Me.TP_OPTIONS.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Absolute, 80.0!)) Me.TP_OPTIONS.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Absolute, 80.0!)) + Me.TP_OPTIONS.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Absolute, 20.0!)) Me.TP_OPTIONS.Controls.Add(LBL_FORMAT, 1, 0) Me.TP_OPTIONS.Controls.Add(TP_WHAT, 0, 0) Me.TP_OPTIONS.Controls.Add(Me.CMB_FORMAT, 2, 0) - Me.TP_OPTIONS.Controls.Add(Me.LBL_AUDIO_CODEC, 5, 0) - Me.TP_OPTIONS.Controls.Add(Me.CMB_AUDIO_CODEC, 6, 0) + Me.TP_OPTIONS.Controls.Add(Me.LBL_AUDIO_CODEC, 4, 0) + Me.TP_OPTIONS.Controls.Add(Me.CMB_AUDIO_CODEC, 5, 0) Me.TP_OPTIONS.Controls.Add(Me.NUM_RES, 3, 0) - Me.TP_OPTIONS.Controls.Add(Me.TXT_FPS, 4, 0) Me.TP_OPTIONS.Dock = System.Windows.Forms.DockStyle.Fill Me.TP_OPTIONS.Location = New System.Drawing.Point(6, 65) Me.TP_OPTIONS.Margin = New System.Windows.Forms.Padding(6, 0, 6, 0) Me.TP_OPTIONS.Name = "TP_OPTIONS" Me.TP_OPTIONS.RowCount = 1 Me.TP_OPTIONS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!)) - Me.TP_OPTIONS.Size = New System.Drawing.Size(709, 28) + Me.TP_OPTIONS.Size = New System.Drawing.Size(589, 28) Me.TP_OPTIONS.TabIndex = 1 ' 'CMB_FORMAT @@ -684,7 +744,7 @@ Namespace API.YouTube.Controls Me.CMB_AUDIO_CODEC.Dock = System.Windows.Forms.DockStyle.Fill Me.CMB_AUDIO_CODEC.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList Me.CMB_AUDIO_CODEC.FormattingEnabled = True - Me.CMB_AUDIO_CODEC.Location = New System.Drawing.Point(632, 3) + Me.CMB_AUDIO_CODEC.Location = New System.Drawing.Point(512, 3) Me.CMB_AUDIO_CODEC.Name = "CMB_AUDIO_CODEC" Me.CMB_AUDIO_CODEC.Size = New System.Drawing.Size(74, 21) Me.CMB_AUDIO_CODEC.TabIndex = 3 @@ -701,57 +761,39 @@ Namespace API.YouTube.Controls Me.NUM_RES.TextAlign = System.Windows.Forms.HorizontalAlignment.Center Me.NUM_RES.Value = New Decimal(New Integer() {1080, 0, 0, 0}) ' - 'TXT_FPS - ' - ActionButton10.BackgroundImage = CType(resources.GetObject("ActionButton10.BackgroundImage"), System.Drawing.Image) - ActionButton10.Name = "Clear" - ActionButton10.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Clear - Me.TXT_FPS.Buttons.Add(ActionButton10) - Me.TXT_FPS.CaptionText = "FPS" - Me.TXT_FPS.CaptionToolTipEnabled = True - Me.TXT_FPS.CaptionToolTipText = "You can reduce the video FPS by setting the FPS value in this field." - Me.TXT_FPS.CaptionWidth = 30.0R - Me.TXT_FPS.Dock = System.Windows.Forms.DockStyle.Fill - Me.TXT_FPS.Location = New System.Drawing.Point(432, 2) - Me.TXT_FPS.Margin = New System.Windows.Forms.Padding(3, 2, 3, 3) - Me.TXT_FPS.Name = "TXT_FPS" - Me.TXT_FPS.Size = New System.Drawing.Size(114, 22) - Me.TXT_FPS.TabIndex = 6 - Me.TXT_FPS.TextBoxWidthMinimal = 30 - ' 'TP_CONTROLS ' Me.TP_CONTROLS.ColumnCount = 1 Me.TP_CONTROLS.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Percent, 100.0!)) Me.TP_CONTROLS.Dock = System.Windows.Forms.DockStyle.Fill - Me.TP_CONTROLS.Location = New System.Drawing.Point(3, 182) + Me.TP_CONTROLS.Location = New System.Drawing.Point(3, 210) Me.TP_CONTROLS.Margin = New System.Windows.Forms.Padding(3, 0, 3, 0) Me.TP_CONTROLS.Name = "TP_CONTROLS" Me.TP_CONTROLS.RowCount = 1 Me.TP_CONTROLS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!)) - Me.TP_CONTROLS.Size = New System.Drawing.Size(715, 25) + Me.TP_CONTROLS.Size = New System.Drawing.Size(595, 25) Me.TP_CONTROLS.TabIndex = 0 ' 'TXT_SUBS_ADDIT ' - ActionButton11.BackgroundImage = CType(resources.GetObject("ActionButton11.BackgroundImage"), System.Drawing.Image) - ActionButton11.Enabled = False - ActionButton11.Name = "Open" - ActionButton11.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Open - ActionButton11.ToolTipText = "Choose additional formats" ActionButton12.BackgroundImage = CType(resources.GetObject("ActionButton12.BackgroundImage"), System.Drawing.Image) ActionButton12.Enabled = False - ActionButton12.Name = "Refresh" - ActionButton12.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Refresh - ActionButton12.ToolTipText = "Fill in additional formats from the defaults" + ActionButton12.Name = "Open" + ActionButton12.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Open + ActionButton12.ToolTipText = "Choose additional formats" ActionButton13.BackgroundImage = CType(resources.GetObject("ActionButton13.BackgroundImage"), System.Drawing.Image) ActionButton13.Enabled = False - ActionButton13.Name = "Clear" - ActionButton13.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Clear - ActionButton13.ToolTipText = "Remove all additional formats" - Me.TXT_SUBS_ADDIT.Buttons.Add(ActionButton11) + ActionButton13.Name = "Refresh" + ActionButton13.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Refresh + ActionButton13.ToolTipText = "Fill in additional formats from the defaults" + ActionButton14.BackgroundImage = CType(resources.GetObject("ActionButton14.BackgroundImage"), System.Drawing.Image) + ActionButton14.Enabled = False + ActionButton14.Name = "Clear" + ActionButton14.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Clear + ActionButton14.ToolTipText = "Remove all additional formats" Me.TXT_SUBS_ADDIT.Buttons.Add(ActionButton12) Me.TXT_SUBS_ADDIT.Buttons.Add(ActionButton13) + Me.TXT_SUBS_ADDIT.Buttons.Add(ActionButton14) Me.TXT_SUBS_ADDIT.CaptionMode = PersonalUtilities.Forms.Controls.Base.ICaptionControl.Modes.CheckBox Me.TXT_SUBS_ADDIT.CaptionText = "Additional subtitle formats" Me.TXT_SUBS_ADDIT.CaptionToolTipEnabled = True @@ -759,44 +801,44 @@ Namespace API.YouTube.Controls Me.TXT_SUBS_ADDIT.CaptionWidth = 150.0R Me.TXT_SUBS_ADDIT.ClearTextByButtonClear = False Me.TXT_SUBS_ADDIT.Dock = System.Windows.Forms.DockStyle.Fill - Me.TXT_SUBS_ADDIT.Location = New System.Drawing.Point(6, 124) + Me.TXT_SUBS_ADDIT.Location = New System.Drawing.Point(6, 152) Me.TXT_SUBS_ADDIT.Margin = New System.Windows.Forms.Padding(6, 3, 6, 3) Me.TXT_SUBS_ADDIT.Name = "TXT_SUBS_ADDIT" - Me.TXT_SUBS_ADDIT.Size = New System.Drawing.Size(709, 22) + Me.TXT_SUBS_ADDIT.Size = New System.Drawing.Size(589, 22) Me.TXT_SUBS_ADDIT.TabIndex = 3 Me.TXT_SUBS_ADDIT.Tag = "s" Me.TXT_SUBS_ADDIT.TextBoxReadOnly = True ' 'TXT_EXTRA_AUDIO_FORMATS ' - ActionButton14.BackgroundImage = CType(resources.GetObject("ActionButton14.BackgroundImage"), System.Drawing.Image) - ActionButton14.Enabled = False - ActionButton14.Name = "Open" - ActionButton14.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Open - ActionButton14.ToolTipText = "Choose additional formats" ActionButton15.BackgroundImage = CType(resources.GetObject("ActionButton15.BackgroundImage"), System.Drawing.Image) ActionButton15.Enabled = False - ActionButton15.Name = "Refresh" - ActionButton15.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Refresh - ActionButton15.ToolTipText = "Fill in additional formats from the defaults" + ActionButton15.Name = "Open" + ActionButton15.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Open + ActionButton15.ToolTipText = "Choose additional formats" ActionButton16.BackgroundImage = CType(resources.GetObject("ActionButton16.BackgroundImage"), System.Drawing.Image) ActionButton16.Enabled = False - ActionButton16.Name = "Clear" - ActionButton16.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Clear - ActionButton16.ToolTipText = "Choose additional formats" - Me.TXT_EXTRA_AUDIO_FORMATS.Buttons.Add(ActionButton14) + ActionButton16.Name = "Refresh" + ActionButton16.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Refresh + ActionButton16.ToolTipText = "Fill in additional formats from the defaults" + ActionButton17.BackgroundImage = CType(resources.GetObject("ActionButton17.BackgroundImage"), System.Drawing.Image) + ActionButton17.Enabled = False + ActionButton17.Name = "Clear" + ActionButton17.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Clear + ActionButton17.ToolTipText = "Choose additional formats" Me.TXT_EXTRA_AUDIO_FORMATS.Buttons.Add(ActionButton15) Me.TXT_EXTRA_AUDIO_FORMATS.Buttons.Add(ActionButton16) + Me.TXT_EXTRA_AUDIO_FORMATS.Buttons.Add(ActionButton17) Me.TXT_EXTRA_AUDIO_FORMATS.CaptionMode = PersonalUtilities.Forms.Controls.Base.ICaptionControl.Modes.CheckBox Me.TXT_EXTRA_AUDIO_FORMATS.CaptionText = "Additional audio formats" Me.TXT_EXTRA_AUDIO_FORMATS.CaptionToolTipEnabled = True Me.TXT_EXTRA_AUDIO_FORMATS.CaptionWidth = 150.0R Me.TXT_EXTRA_AUDIO_FORMATS.ClearTextByButtonClear = False Me.TXT_EXTRA_AUDIO_FORMATS.Dock = System.Windows.Forms.DockStyle.Fill - Me.TXT_EXTRA_AUDIO_FORMATS.Location = New System.Drawing.Point(6, 152) + Me.TXT_EXTRA_AUDIO_FORMATS.Location = New System.Drawing.Point(6, 180) Me.TXT_EXTRA_AUDIO_FORMATS.Margin = New System.Windows.Forms.Padding(6, 3, 6, 3) Me.TXT_EXTRA_AUDIO_FORMATS.Name = "TXT_EXTRA_AUDIO_FORMATS" - Me.TXT_EXTRA_AUDIO_FORMATS.Size = New System.Drawing.Size(709, 22) + Me.TXT_EXTRA_AUDIO_FORMATS.Size = New System.Drawing.Size(589, 22) Me.TXT_EXTRA_AUDIO_FORMATS.TabIndex = 4 Me.TXT_EXTRA_AUDIO_FORMATS.Tag = "a" Me.TXT_EXTRA_AUDIO_FORMATS.TextBoxReadOnly = True @@ -807,14 +849,14 @@ Namespace API.YouTube.Controls Me.AutoScaleDimensions = New System.Drawing.SizeF(6.0!, 13.0!) Me.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font Me.CancelButton = Me.BTT_CANCEL - Me.ClientSize = New System.Drawing.Size(721, 300) + Me.ClientSize = New System.Drawing.Size(601, 328) Me.Controls.Add(Me.TP_MAIN) Me.FormBorderStyle = System.Windows.Forms.FormBorderStyle.FixedSingle Me.Icon = Global.SCrawler.My.Resources.SiteYouTube.YouTubeIcon_32 Me.KeyPreview = True Me.MaximizeBox = False Me.MinimizeBox = False - Me.MinimumSize = New System.Drawing.Size(737, 339) + Me.MinimumSize = New System.Drawing.Size(617, 367) Me.Name = "VideoOptionsForm" Me.ShowInTaskbar = False Me.SizeGripStyle = System.Windows.Forms.SizeGripStyle.Hide @@ -834,6 +876,9 @@ Namespace API.YouTube.Controls CType(Me.CMB_PLS, System.ComponentModel.ISupportInitialize).EndInit() TP_WHAT.ResumeLayout(False) TP_WHAT.PerformLayout() + TP_FPS_BITRATE.ResumeLayout(False) + CType(Me.TXT_FPS, System.ComponentModel.ISupportInitialize).EndInit() + CType(Me.TXT_AUDIO_BITRATE, System.ComponentModel.ISupportInitialize).EndInit() Me.TP_HEADER_BASE.ResumeLayout(False) Me.TP_SUBS.ResumeLayout(False) Me.TP_SUBS.PerformLayout() @@ -842,7 +887,6 @@ Namespace API.YouTube.Controls Me.TP_OPTIONS.ResumeLayout(False) Me.TP_OPTIONS.PerformLayout() CType(Me.NUM_RES, System.ComponentModel.ISupportInitialize).EndInit() - CType(Me.TXT_FPS, System.ComponentModel.ISupportInitialize).EndInit() CType(Me.TXT_SUBS_ADDIT, System.ComponentModel.ISupportInitialize).EndInit() CType(Me.TXT_EXTRA_AUDIO_FORMATS, System.ComponentModel.ISupportInitialize).EndInit() Me.ResumeLayout(False) @@ -875,5 +919,6 @@ Namespace API.YouTube.Controls Private WithEvents TXT_FPS As PersonalUtilities.Forms.Controls.TextBoxExtended Private WithEvents CMB_PLS As PersonalUtilities.Forms.Controls.ComboBoxExtended Private WithEvents BTT_PLS_BROWSE As Button + Private WithEvents TXT_AUDIO_BITRATE As PersonalUtilities.Forms.Controls.TextBoxExtended End Class End Namespace \ No newline at end of file diff --git a/SCrawler.YouTube/Controls/VideoOptionsForm.resx b/SCrawler.YouTube/Controls/VideoOptionsForm.resx index 8656187..31b25a8 100644 --- a/SCrawler.YouTube/Controls/VideoOptionsForm.resx +++ b/SCrawler.YouTube/Controls/VideoOptionsForm.resx @@ -377,7 +377,26 @@ False + + False + + + iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO + xAAADsQBlSsOGwAAAIZJREFUOE+1j10KwCAMgz2b755xl/IsvnaL2K20UfbDAmEako+ZROSTafjE12Go + tbbB43rK5xSAQq1VYFtmeQBoqZTSreVZvgTknM8yyyjA/qodsDF9gspD2Bj6B+DH+NqzhQQAG+POMnSX + AFuc5QFgn6ClHh5iOQVAKNixyucB8NY0vG9JOzzyhrdq5IRgAAAAAElFTkSuQmCC + + + + + iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO + xAAADsQBlSsOGwAAAIZJREFUOE+1j10KwCAMgz2b755xl/IsvnaL2K20UfbDAmEako+ZROSTafjE12Go + tbbB43rK5xSAQq1VYFtmeQBoqZTSreVZvgTknM8yyyjA/qodsDF9gspD2Bj6B+DH+NqzhQQAG+POMnSX + AFuc5QFgn6ClHh5iOQVAKNixyucB8NY0vG9JOzzyhrdq5IRgAAAAAElFTkSuQmCC + + + iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO wwAADsMBx2+oZAAAAR5JREFUOE+VkjFqwzAUhn2D9iShRyi+QhYbGujg3ZATZPKYdC6FQhPwlAMkg3dP @@ -388,7 +407,7 @@ cMaRN0UdBBkAAAAASUVORK5CYII= - + iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGOfPtRkwAAACBjSFJNAAB6 JQAAgIMAAPn/AACA6QAAdTAAAOpgAAA6mAAAF2+SX8VGAAAACXBIWXMAAAsTAAALEwEAmpwYAAACOElE @@ -404,15 +423,7 @@ VnR1MIwhwMTCyqEQ37qEmZVDFF0OE/9nAACtFF4Ey6OP+wAAAABJRU5ErkJggg== - - - iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO - xAAADsQBlSsOGwAAAIZJREFUOE+1j10KwCAMgz2b755xl/IsvnaL2K20UfbDAmEako+ZROSTafjE12Go - tbbB43rK5xSAQq1VYFtmeQBoqZTSreVZvgTknM8yyyjA/qodsDF9gspD2Bj6B+DH+NqzhQQAG+POMnSX - AFuc5QFgn6ClHh5iOQVAKNixyucB8NY0vG9JOzzyhrdq5IRgAAAAAElFTkSuQmCC - - - + iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO xAAADsQBlSsOGwAAAIZJREFUOE+1j10KwCAMgz2b755xl/IsvnaL2K20UfbDAmEako+ZROSTafjE12Go @@ -420,7 +431,7 @@ AFuc5QFgn6ClHh5iOQVAKNixyucB8NY0vG9JOzzyhrdq5IRgAAAAAElFTkSuQmCC - + iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO wwAADsMBx2+oZAAAAR5JREFUOE+VkjFqwzAUhn2D9iShRyi+QhYbGujg3ZATZPKYdC6FQhPwlAMkg3dP @@ -431,7 +442,7 @@ cMaRN0UdBBkAAAAASUVORK5CYII= - + iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGOfPtRkwAAACBjSFJNAAB6 JQAAgIMAAPn/AACA6QAAdTAAAOpgAAA6mAAAF2+SX8VGAAAACXBIWXMAAAsTAAALEwEAmpwYAAACOElE @@ -447,7 +458,7 @@ VnR1MIwhwMTCyqEQ37qEmZVDFF0OE/9nAACtFF4Ey6OP+wAAAABJRU5ErkJggg== - + iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO xAAADsQBlSsOGwAAAIZJREFUOE+1j10KwCAMgz2b755xl/IsvnaL2K20UfbDAmEako+ZROSTafjE12Go @@ -455,7 +466,7 @@ AFuc5QFgn6ClHh5iOQVAKNixyucB8NY0vG9JOzzyhrdq5IRgAAAAAElFTkSuQmCC - + iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO wwAADsMBx2+oZAAAAR5JREFUOE+VkjFqwzAUhn2D9iShRyi+QhYbGujg3ZATZPKYdC6FQhPwlAMkg3dP @@ -466,7 +477,7 @@ cMaRN0UdBBkAAAAASUVORK5CYII= - + iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGOfPtRkwAAACBjSFJNAAB6 JQAAgIMAAPn/AACA6QAAdTAAAOpgAAA6mAAAF2+SX8VGAAAACXBIWXMAAAsTAAALEwEAmpwYAAACOElE @@ -482,7 +493,7 @@ VnR1MIwhwMTCyqEQ37qEmZVDFF0OE/9nAACtFF4Ey6OP+wAAAABJRU5ErkJggg== - + iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO xAAADsQBlSsOGwAAAIZJREFUOE+1j10KwCAMgz2b755xl/IsvnaL2K20UfbDAmEako+ZROSTafjE12Go diff --git a/SCrawler.YouTube/Controls/VideoOptionsForm.vb b/SCrawler.YouTube/Controls/VideoOptionsForm.vb index 0978bc0..766121c 100644 --- a/SCrawler.YouTube/Controls/VideoOptionsForm.vb +++ b/SCrawler.YouTube/Controls/VideoOptionsForm.vb @@ -26,7 +26,7 @@ Namespace API.YouTube.Controls Friend Property DesignXML As EContainer Implements IDesignXMLContainer.DesignXML Private Property DesignXMLNodes As String() Implements IDesignXMLContainer.DesignXMLNodes Private Property DesignXMLNodeName As String Implements IDesignXMLContainer.DesignXMLNodeName - Private Const ControlsRow As Integer = 6 + Private Const ControlsRow As Integer = 7 Private ReadOnly Property CNT_PROCESSOR As TableControlsProcessor Friend Property MyContainer As YouTubeMediaContainerBase Private Initialization As Boolean = True @@ -164,11 +164,16 @@ Namespace API.YouTube.Controls If InheritsFromContainer Then If .OutputVideoFPS > 0 Then TXT_FPS.Text = .OutputVideoFPS + If .OutputAudioBitrate > 0 Then TXT_AUDIO_BITRATE.Text = .OutputAudioBitrate Else If MyYouTubeSettings.DefaultVideoFPS > 0 Then TXT_FPS.Text = MyYouTubeSettings.DefaultVideoFPS + If MyYouTubeSettings.DefaultAudioBitrate > 0 Then TXT_AUDIO_BITRATE.Text = MyYouTubeSettings.DefaultAudioBitrate.Value End If - MyFieldsChecker.AddControl(Of Double)(TXT_FPS, TXT_FPS.CaptionText, True, New FpsFieldChecker) - MyFieldsChecker.EndLoaderOperations() + With MyFieldsChecker + .AddControl(Of Double)(TXT_FPS, TXT_FPS.CaptionText, True, New FpsFieldChecker) + .AddControl(Of Integer)(TXT_AUDIO_BITRATE, TXT_AUDIO_BITRATE.CaptionText, True) + .EndLoaderOperations() + End With TP_SUBS.Enabled = .Subtitles.Count > 0 TXT_SUBS_ADDIT.Enabled = .Subtitles.Count > 0 RefillTextBoxes() @@ -327,6 +332,7 @@ Namespace API.YouTube.Controls If Full Then .OutputVideoExtension = CMB_FORMAT.Text.StringToLower .OutputVideoFPS = AConvert(Of Double)(TXT_FPS.Text, YouTubeSettings.FpsFormatProvider.MyProviderDefault, -1) + .OutputAudioBitrate = AConvert(Of Integer)(TXT_AUDIO_BITRATE.Text, -1) .OutputAudioCodec = CMB_AUDIO_CODEC.Text.StringToLower .OutputSubtitlesFormat = CMB_SUBS_FORMAT.Text.StringToLower .IsAudioSelected = OPT_AUDIO.Checked @@ -346,10 +352,12 @@ Namespace API.YouTube.Controls Else f = TXT_FILE.Text End If + f = CleanFileName(f) If f.IsEmptyString Then Throw New ArgumentNullException("File", "The output file cannot be null") With MyContainer .OutputVideoExtension = CMB_FORMAT.Text.StringToLower .OutputVideoFPS = AConvert(Of Double)(TXT_FPS.Text, YouTubeSettings.FpsFormatProvider.MyProviderDefault, -1) + .OutputAudioBitrate = AConvert(Of Integer)(TXT_AUDIO_BITRATE.Text, -1) .OutputAudioCodec = CMB_AUDIO_CODEC.Text.StringToLower .OutputSubtitlesFormat = CMB_SUBS_FORMAT.Text.StringToLower .M3U8_PlaylistFiles = M3U8FilesFull @@ -369,6 +377,7 @@ Namespace API.YouTube.Controls Else .SelectedVideoIndex = -1 .SelectedAudioIndex = cntIndex + .MediaType = UMTypes.Audio End If .FileSetManually = True .File = f @@ -379,6 +388,7 @@ Namespace API.YouTube.Controls Else If OPT_AUDIO.Checked Then .SetMaxResolution(-2) + .MediaType = UMTypes.Audio Else .SetMaxResolution(NUM_RES.Value) End If @@ -595,12 +605,15 @@ Namespace API.YouTube.Controls f = SFile.SelectPath(f, "Select the destination of the video files", EDP.ReturnValue) Else f = TXT_FILE.Text - Dim sPattern$ = $"Video|{AvailableVideoFormats.Select(Function(vf) $"*.{vf.ToLower}").ListToString(";")}" & - $"|Audio|{AvailableAudioFormats.Select(Function(af) $"*.{af.ToLower}").ListToString(";")}" & - "|All Files|*.*" - f = SFile.SaveAs(f, "Select the destination of the video file",,, sPattern, EDP.ReturnValue) + Dim ext$ = f.Extension + Dim sPattern$ = "All Files|*.*|" & + $"Video|{AvailableVideoFormats.Select(Function(vf) $"*.{vf.ToLower}").ListToString(";")}" & + $"|Audio|{AvailableAudioFormats.Select(Function(af) $"*.{af.ToLower}").ListToString(";")}" + f = SFile.SaveAs(f, "Select the destination of the video file",, ext, sPattern, EDP.ReturnValue) + f.Extension = ext End If #Enable Warning + f = CleanFileName(f) If Not f.IsEmptyString Then If e.Button = MouseButtons.Right Then MyYouTubeSettings.DownloadLocations.Add(f, MyDownloaderSettings.OutputPathAskForName) diff --git a/SCrawler.YouTube/Declarations.vb b/SCrawler.YouTube/Declarations.vb index 335f347..2cb61a5 100644 --- a/SCrawler.YouTube/Declarations.vb +++ b/SCrawler.YouTube/Declarations.vb @@ -17,10 +17,21 @@ Namespace API.YouTube Public Const DownloaderDataFolderYouTube As String = DownloadObjects.STDownloader.DownloaderDataFolder & "YouTube\" Friend Const YouTubeDownloadPathDefault As String = "YouTubeDownloads\" Friend Const SimpleArraysFormNode As String = "SimpleFormatsChooserForm" + Private Const YTDLP_DefaultName As String = "yt-dlp" Public Property MyYouTubeSettings As Base.YouTubeSettings Public Property MyCache As CacheKeeper Friend ReadOnly Property MyCacheSettings As New CacheKeeper(DownloaderDataFolderYouTube) With {.DeleteCacheOnDispose = False, .DeleteRootOnDispose = False} Public ReadOnly Property YouTubeCookieNetscapeFile As New SFile($"Settings\Responser_{YouTubeSite}_Cookies_Netscape.txt") + Friend ReadOnly Property YTDLP_NAME As String + Get + Dim n$ = MyYouTubeSettings.YTDLP.Value.Name + If Not n.IsEmptyString Then + Return If(n.ToLower = YTDLP_DefaultName, n, $"""{n}""") + Else + Return YTDLP_DefaultName + End If + End Get + End Property Friend ReadOnly Property AvailableSubtitlesFormats As String() Get Return {"ASS", "LRC", "SRT", "VTT"} @@ -45,6 +56,17 @@ Namespace API.YouTube Friend ReadOnly TitleHtmlConverter As Func(Of String, String) = Function(Input) Input.StringRemoveWinForbiddenSymbols().StringTrim() Friend ReadOnly ProgressProvider As IMyProgressNumberProvider = MyProgressNumberProvider.Percentage Public ReadOnly TrueUrlRegEx As RParams = RParams.DM(Base.YouTubeFunctions.TrueUrlPattern, 0, EDP.ReturnValue) + Friend Function CleanFileName(ByVal f As SFile) As SFile + If Not f.IsEmptyString And Not f.Name.IsEmptyString Then + Dim ff As SFile = f + ff.Name = ff.Name.StringRemoveWinForbiddenSymbols + If Not ff.Name.IsEmptyString Then ff.Name = ff.Name.Replace("%", String.Empty) + If ff.Name.IsEmptyString Then ff.Name = "file" + Return ff + Else + Return f + End If + End Function Private Class TimeToStringConverter : Implements ICustomProvider Private ReadOnly _Provider As New ADateTime("mm\:ss") With {.TimeParseMode = ADateTime.TimeModes.TimeSpan} Private ReadOnly _ProviderWithHours As New ADateTime("h\:mm\:ss") With {.TimeParseMode = ADateTime.TimeModes.TimeSpan} diff --git a/SCrawler.YouTube/Downloader/MediaItem.vb b/SCrawler.YouTube/Downloader/MediaItem.vb index 17eed97..7d9237f 100644 --- a/SCrawler.YouTube/Downloader/MediaItem.vb +++ b/SCrawler.YouTube/Downloader/MediaItem.vb @@ -12,6 +12,7 @@ Imports SCrawler.API.YouTube.Objects Imports SCrawler.API.YouTube.Controls Imports PersonalUtilities.Tools Imports PersonalUtilities.Forms.Toolbars +Imports PersonalUtilities.Functions.Messaging Namespace DownloadObjects.STDownloader Public Delegate Sub MediaItemEventHandler(ByVal Sender As MediaItem, ByVal Container As IYouTubeMediaContainer) @@ -135,7 +136,7 @@ Namespace DownloadObjects.STDownloader LBL_TITLE.Text = .ToString(True) If Not .SiteKey = YouTubeSiteKey And .ContentType = Plugin.UserMediaTypes.Picture Then LBL_INFO.Text = .File.Extension.StringToUpper - ElseIf Not .IsMusic Then + ElseIf Not .IsMusic And Not (.MediaType = Plugin.UserMediaTypes.Audio Or .MediaType = Plugin.UserMediaTypes.AudioPre) Then If .Height > 0 Then LBL_INFO.Text = $"{ .File.Extension.StringToUpper}{d}{ .Height}p" Else @@ -180,10 +181,10 @@ Namespace DownloadObjects.STDownloader With MyContainer If Not .SiteKey = YouTubeSiteKey And .ContentType = Plugin.UserMediaTypes.Picture Then ICON_WHAT.Image = My.Resources.ImagePic_32 - ElseIf Not .IsMusic Then - ICON_WHAT.Image = My.Resources.VideoCamera_32 - Else + ElseIf .IsMusic Or .MediaType = Plugin.UserMediaTypes.Audio Or .MediaType = Plugin.UserMediaTypes.AudioPre Then ICON_WHAT.Image = My.Resources.AudioMusic_32 + Else + ICON_WHAT.Image = My.Resources.VideoCamera_32 End If End With End Sub, EDP.None) @@ -229,7 +230,7 @@ Namespace DownloadObjects.STDownloader .ColumnStyles.Clear() .ColumnCount = 0 If ContainerHasElements Or MyContainer.MediaState = Plugin.UserMediaStates.Downloaded Then - If Not MyContainer.SiteKey = YouTubeSiteKey Then UpdateMediaIcon() + UpdateMediaIcon() If ContainerHasElements Then BTT_OPEN_FOLDER.Visible = False BTT_OPEN_FILE.Visible = False @@ -476,12 +477,28 @@ Namespace DownloadObjects.STDownloader RaiseEvent Removal(Me, MyContainer) End Sub Private Sub BTT_DELETE_FILE_Click(sender As Object, e As EventArgs) Handles BTT_DELETE_FILE.Click - If MsgBoxE({$"Are you sure you want to delete the following {FileOption.ToString.ToLower}:{vbCr}" & - If(FileOption = SFO.File, MyContainer.File.ToString, MyContainer.File.PathWithSeparator), - $"Deleting a {FileOption.ToString.ToLower}"}, vbExclamation,,, {"Process", "Cancel"}) = 0 Then + Dim opt$ + Dim opt2$ = String.Empty + If FileOption = SFO.File Then + opt = "file" + Else + opt = "item" + opt2 = "THE ITEM MAY CONTAIN MULTIPLE FILES" & vbCr + End If + Dim b As New List(Of MsgBoxButton) From {New MsgBoxButton("Process")} + If Not opt2.IsEmptyString Then _ + b.Add(New MsgBoxButton("Show files", "Show files to delete") With { + .IsDialogResultButton = False, + .CallBack = Function(r, m, bb) MsgBoxE(New MMessage($"The following files will be deleted:{vbCr}{vbCr}{MyContainer.Files.ListToString(vbCr)}", + "Files to delete",, vbExclamation) With {.Editable = True})}) + b.Add(New MsgBoxButton("Cancel")) + If MsgBoxE({$"Are you sure you want to delete the following {opt}:{vbCr}{opt2}" & + If(FileOption = SFO.File, MyContainer.File.ToString, MyContainer.ToString(True)), + $"Deleting {opt}"}, vbExclamation,,, b) = 0 Then MyContainer.Delete(True) RaiseEvent Removal(Me, MyContainer) End If + b.Clear() End Sub #End Region #Region "ISupportInitialize Support" diff --git a/SCrawler.YouTube/Objects/PlayList.vb b/SCrawler.YouTube/Objects/PlayList.vb index e418238..552d9b7 100644 --- a/SCrawler.YouTube/Objects/PlayList.vb +++ b/SCrawler.YouTube/Objects/PlayList.vb @@ -19,17 +19,18 @@ Namespace API.YouTube.Objects Dim __title$ = $" - {Title}" If Not s.IsEmptyString Then s = $" [{s}]" If Not PlaylistTitle.IsEmptyString And Not ForMediaItem Then t = $"{PlaylistTitle} - " + Dim c% = {Count, ElementsNumber}.Max If IsMusic Then - If Count <= 1 Then t &= "Single" Else t &= "Album" + If c <= 1 Then t &= "Single" Else t &= "Album" Else t &= "Playlist" End If If Not PlaylistTitle.IsEmptyString And Not ForMediaItem Then t &= $" - {PlaylistTitle}" If PlaylistTitle = Title Then __title = String.Empty If ForMediaItem Then - Return $"{t} ({Count}){__title}" + Return $"{t} ({c}){__title}" Else - Return $"{t} ({Count}){__title} ({AConvert(Of String)(Duration, TimeToStringProvider)}){s}" + Return $"{t} ({c}){__title} ({AConvert(Of String)(Duration, TimeToStringProvider)}){s}" End If End Function Public Overrides Function Parse(ByVal Container As EContainer, ByVal Path As SFile, ByVal IsMusic As Boolean, diff --git a/SCrawler.YouTube/Objects/Track.vb b/SCrawler.YouTube/Objects/Track.vb index 1cee2bb..4de6382 100644 --- a/SCrawler.YouTube/Objects/Track.vb +++ b/SCrawler.YouTube/Objects/Track.vb @@ -27,6 +27,7 @@ Namespace API.YouTube.Objects Else _File.Extension = mp3 End If + _File = CleanFileName(_File) End If End Sub Public Overrides Function ToString(ByVal ForMediaItem As Boolean) As String @@ -46,12 +47,17 @@ Namespace API.YouTube.Objects _ObjectType = Base.YouTubeMediaType.Single Me.IsMusic = IsMusic If MyBase.Parse(Container, Path, IsMusic, Token, Progress) Then - Dim f As SFile = MyYouTubeSettings.OutputPath - If f.IsEmptyString Then f = "YouTubeDownloads\OutputFile.mp3" - Dim ext$ = MyYouTubeSettings.DefaultAudioCodec.Value.StringToLower - If ext.IsEmptyString Then ext = "mp3" - f.Extension = ext - File = f + With MyYouTubeSettings + Dim f As SFile = .OutputPath + If f.IsEmptyString Then f = "YouTubeDownloads\OutputFile.mp3" + Dim ext$ = .DefaultAudioCodecMusic.Value.StringToLower.IfNullOrEmpty(.DefaultAudioCodec.Value.StringToLower) + If ext.IsEmptyString Then ext = "mp3" + f.Extension = ext + 'If f.Name.IsEmptyString Then f.Name = File.Name + File = f + If _File.Extension.IsEmptyString Then _File.Extension = ext + _File = CleanFileName(_File) + End With Return True Else Return False diff --git a/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb b/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb index c598997..aa98218 100644 --- a/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb +++ b/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb @@ -123,6 +123,15 @@ Namespace API.YouTube.Objects Public Property UserTitle As String Implements IYouTubeMediaContainer.UserTitle #End Region #Region "Playlist support" + Private _ElementsNumber As Integer = 0 + Protected Property ElementsNumber As Integer + Get + Return If(HasElements, Count, _ElementsNumber) + End Get + Set(ByVal _ElementsNumber As Integer) + Me._ElementsNumber = _ElementsNumber + End Set + End Property Friend ReadOnly Property Elements As List(Of IYouTubeMediaContainer) Implements IYouTubeMediaContainer.Elements Friend ReadOnly Property HasElements As Boolean Implements IYouTubeMediaContainer.HasElements Get @@ -265,6 +274,18 @@ Namespace API.YouTube.Objects PostProcessing_OutputAudioFormats.RemoveAll(Function(s) s = -1) End If End Sub + Protected _OutputAudioBitrate As Integer = -1 + Friend Property OutputAudioBitrate As Integer + Get + Return _OutputAudioBitrate + End Get + Set(ByVal NewBitrate As Integer) + If Not [Protected] Then + _OutputAudioBitrate = NewBitrate + If HasElements Then Elements.ForEach(Sub(elem) DirectCast(elem, YouTubeMediaContainerBase).OutputAudioBitrate = NewBitrate) + End If + End Set + End Property #End Region #Region "Subtitles" Protected ReadOnly _Subtitles As List(Of Subtitles) @@ -376,10 +397,13 @@ Namespace API.YouTube.Objects End Set End Property Protected _Size As Integer = 0 + Protected _SizeRecalculated As Boolean = False Public Overridable Property Size As Integer Implements IDownloadableMedia.Size Get If HasElements Then Return Elements.Sum(Function(e) If(e.Checked, e.Size, 0)) + ElseIf _SizeRecalculated Then + Return _Size Else If Checked Then If IsMusic And SelectedAudioIndex.ValueBetween(0, MediaObjects.Count - 1) Then @@ -559,7 +583,25 @@ Namespace API.YouTube.Objects If ObjectType = YouTubeMediaType.Single AndAlso Not GetPlayListTitle.IsEmptyString Then _SpecialPath.StringAppend(GetPlayListTitle(), "\") If Elements.Count > 0 Then Elements.ForEach(Sub(e) e.SpecialFolder = Path) End Sub - Protected Friend ReadOnly Property Files As List(Of SFile) Implements IYouTubeMediaContainer.Files + Private ReadOnly _Files As List(Of SFile) + Protected Friend Property Files As List(Of SFile) Implements IYouTubeMediaContainer.Files + Get + If HasElements Then + Return GetFilesFiles() + Else + Return _Files + End If + End Get + Set(ByVal f As List(Of SFile)) + _Files.ListAddList(f, LAP.NotContainsOnly) + End Set + End Property + Protected Overloads Sub AddFile(ByVal f As SFile) + _Files.ListAddValue(f, LAP.NotContainsOnly) + End Sub + Protected Overloads Sub AddFile(ByVal f As IEnumerable(Of SFile)) + _Files.ListAddList(f, LAP.NotContainsOnly) + End Sub Protected _File As SFile Protected Friend Property FileSetManually As Boolean = False Public Property FileIgnorePlaylist As Boolean = False @@ -628,6 +670,14 @@ Namespace API.YouTube.Objects If HasElements And Not IsMusic Then urls.ListAddList(Elements.SelectMany(Function(elem As YouTubeMediaContainerBase) elem.GetFiles()), LAP.NotContainsOnly) Return urls End Function + Private Function GetFilesFiles() As IEnumerable(Of SFile) + Dim f As New List(Of SFile) + If File.Exists Then f.Add(File) + If _Files.Count > 0 Then f.AddRange(_Files) + If ThumbnailFile.Exists Then f.Add(ThumbnailFile) + If HasElements Then f.ListAddList(Elements.SelectMany(Function(elem As YouTubeMediaContainerBase) elem.GetFilesFiles()), LAP.NotContainsOnly) + Return f + End Function Private _M3U8_PlaylistFiles As IEnumerable(Of SFile) = Nothing Friend Property M3U8_PlaylistFiles As IEnumerable(Of SFile) Get @@ -647,6 +697,7 @@ Namespace API.YouTube.Objects Private Const aac As String = "aac" Private Const ac3 As String = "ac3" Protected PostProcessing_AudioAC3 As Boolean = False + Protected PostProcessing_AudioMP3 As Boolean = False Public Overridable ReadOnly Property Command(ByVal WithCookies As Boolean) As String Implements IYouTubeMediaContainer.Command Get If Not File.IsEmptyString Then @@ -682,6 +733,10 @@ Namespace API.YouTube.Objects PostProcessing_AudioAC3 = True formats.StringAppend($"--audio-format {aac}", " ") atCodec = aac + ElseIf SelectedVideoIndex >= 0 And OutputAudioCodec.StringToLower = mp3 Then + PostProcessing_AudioMP3 = True + formats.StringAppend($"--audio-format {aac}", " ") + atCodec = aac Else formats.StringAppend($"--audio-format {OutputAudioCodec.StringToLower}", " ") atCodec = OutputAudioCodec.StringToLower @@ -716,7 +771,8 @@ Namespace API.YouTube.Objects If Not cmd.IsEmptyString Then 'URGENT: 2023.3.4 -> 2023.7.6 'cmd = $"yt-dlp -f ""{cmd}""" - cmd = $"yt-dlp -f {cmd}" + 'cmd = $"yt-dlp -f {cmd}" + cmd = $"{YTDLP_NAME} -f {cmd}" If Not MyYouTubeSettings.ReplaceModificationDate Then cmd &= " --no-mtime" cmd.StringAppend(formats, " ") cmd.StringAppend(subs, " ") @@ -738,7 +794,7 @@ Namespace API.YouTube.Objects _SubtitlesDelegated = New List(Of Subtitles) SubtitlesSelectedIndexes = New List(Of Integer) MediaObjects = New List(Of MediaObject) - Files = New List(Of SFile) + _Files = New List(Of SFile) PostProcessing_OutputSubtitlesFormats = New List(Of String) PostProcessing_OutputSubtitlesFormats.ListAddList(MyYouTubeSettings.DefaultSubtitlesFormatAddit) @@ -767,9 +823,19 @@ Namespace API.YouTube.Objects If RemoveFiles Then Dim fErr As New ErrorsDescriber(EDP.None) Dim dMode As SFODelete = SFODelete.DeleteToRecycleBin + Dim paths As New List(Of SFile) + Dim l As New ListAddParams(LAP.NotContainsOnly) With {.Comparer = New FComparer(Of SFile)(Function(x, y) x.PathNoSeparator = y.PathNoSeparator)} + Dim isArr As Boolean = ObjectType <> YouTubeMediaType.Single And ObjectType <> YouTubeMediaType.Undefined + If isArr And AbsolutePath Then paths.ListAddValue(File, l) File.Delete(SFO.File, dMode, fErr) + If isArr Then paths.ListAddValue(ThumbnailFile, l) ThumbnailFile.Delete(SFO.File, dMode, fErr) - If Files.Count > 0 Then Files.ForEach(Sub(f) f.Delete(SFO.File, dMode, fErr)) + If Files.Count > 0 Then + If isArr Then paths.ListAddList(Files, l) + Files.ForEach(Sub(f) f.Delete(SFO.File, dMode, fErr)) + End If + If paths.Count > 0 Then paths.ForEach(Sub(p) If SFile.GetFiles(p,, IO.SearchOption.AllDirectories, EDP.ReturnValue).Count = 0 Then _ + p.Delete(SFO.Path, dMode, EDP.SendToLog)) End If If HasElements Then Elements.ForEach(Sub(e) e.Delete(RemoveFiles)) End Sub @@ -854,17 +920,22 @@ Namespace API.YouTube.Objects If HasElements AndAlso Elements(0).ObjectType = YouTubeMediaType.Single AndAlso Elements(0).IsMusic Then Dim t As TextSaver = Nothing Try + Dim f As SFile If MyYouTubeSettings.MusicPlaylistCreate_M3U8 Then t = New TextSaver t.AppendLine("#EXTM3U") Elements.ForEach(Sub(e) t.AppendLine(GetPlaylistRow(e))) - t.SaveAs($"{Elements(0).File.PathWithSeparator}Playlist.m3u8", EDP.SendToLog) + f = $"{Elements(0).File.PathWithSeparator}Playlist.m3u8" + t.SaveAs(f, EDP.SendToLog) + If f.Exists Then AddFile(f) t.Dispose() End If If MyYouTubeSettings.MusicPlaylistCreate_M3U Then t = New TextSaver Elements.ForEach(Sub(e) t.AppendLine(e.File)) - t.SaveAs($"{Elements(0).File.PathWithSeparator}Playlist.m3u", EDP.SendToLog) + f = $"{Elements(0).File.PathWithSeparator}Playlist.m3u" + t.SaveAs(f, EDP.SendToLog) + If f.Exists Then AddFile(f) t.Dispose() End If Catch ex As Exception @@ -941,7 +1012,7 @@ Namespace API.YouTube.Objects ff.Name = "album" ff.Extension = "url" CreateUrlFile(url, ff) - If ff.Exists Then Files.Add(ff) + If ff.Exists Then AddFile(ff) End If If MyYouTubeSettings.CreateThumbnails_Music Then Using resp As New Responser @@ -967,7 +1038,7 @@ Namespace API.YouTube.Objects url = LinkFormatterSecure(u) f.Name = "cover" f.Extension = "jpg" - If resp.DownloadFile(url, f, EDP.ReturnValue) And f.Exists Then CoverDownloaded = True + If resp.DownloadFile(url, f, EDP.ReturnValue) And f.Exists Then CoverDownloaded = True : AddFile(f) End If End If End Using @@ -976,19 +1047,61 @@ Namespace API.YouTube.Objects ErrorsDescriber.Execute(EDP.SendToLog, ex, $"DownloadPlaylistCover({PlsId}, {f})") End Try End Sub + Private Structure TempFileConversion + Friend File As SFile + Friend Requested As Boolean + Friend ToReplace As Boolean + Friend ReadOnly Property Exists As Boolean + Get + Return File.Exists + End Get + End Property + Friend Sub Delete() + If Not Requested Then File.Delete() + End Sub + Private Sub New(ByVal f As SFile) + File = f + Requested = False + ToReplace = False + End Sub + Friend Sub New(ByVal f As SFile, ByVal Source As YouTubeMediaContainerBase) + Me.New(f) + Requested = Source.PostProcessing_OutputAudioFormats.Count > 0 AndAlso + Source.PostProcessing_OutputAudioFormats.Exists(Function(af) af.StringToLower = f.Extension) + End Sub + Public Shared Widening Operator CType(ByVal f As SFile) As TempFileConversion + Return New TempFileConversion(f) + End Operator + Public Shared Widening Operator CType(ByVal f As TempFileConversion) As SFile + Return f.File + End Operator + Public Overrides Function Equals(ByVal Obj As Object) As Boolean + If Not IsNothing(Obj) Then + If TypeOf Obj Is TempFileConversion Then + Return DirectCast(Obj, TempFileConversion).File = File + ElseIf TypeOf Obj Is SFile Then + Return DirectCast(Obj, SFile) = File + ElseIf TypeOf Obj Is String Then + Return New TempFileConversion(CStr(Obj)).File = File + End If + End If + Return False + End Function + End Structure Protected Sub DownloadCommand(ByVal UseCookies As Boolean, ByVal Token As CancellationToken) Dim dCommand$ = String.Empty Try ThrowAny(Token) If MediaState = UMStates.Downloaded Or Not Checked Then Exit Sub - Dim h As DataReceivedEventHandler = Sub(ByVal Sender As Object, ByVal e As DataReceivedEventArgs) - If Not e.Data.IsEmptyString Then - Dim v# = AConvert(Of Double)(RegexReplace(e.Data, DownloadProgressPattern), NumberProvider, -1) - If v >= 0 Then Progress.Value = v : Progress.Perform(0) - End If - End Sub RaiseEvent FileDownloadStarted(Me, Nothing) Using batch As New BatchExecutor(True) With {.Encoding = 65001} + Dim h As DataReceivedEventHandler = Sub(ByVal Sender As Object, ByVal e As DataReceivedEventArgs) + If Not e.Data.IsEmptyString Then + Dim v# = AConvert(Of Double)(RegexReplace(e.Data, DownloadProgressPattern), NumberProvider, -1) + If v >= 0 Then Progress.Value = v : Progress.Perform(0) + If Token.IsCancellationRequested Then batch.Kill() + End If + End Sub With batch Dim prExists As Boolean = Not Progress Is Nothing If prExists Then @@ -1001,7 +1114,7 @@ Namespace API.YouTube.Objects .Information = $"Download {MediaType}" End With End If - .MainProcessName = "yt-dlp" + .MainProcessName = MyYouTubeSettings.YTDLP.Name '"yt-dlp" .FileExchanger = MyCache.NewInstance(Of BatchFileExchanger)(CachePath, EDP.ReturnValue) .FileExchanger.DeleteCacheOnDispose = True .AddCommand("chcp 65001") @@ -1027,14 +1140,14 @@ Namespace API.YouTube.Objects Dim fileUrl As SFile = File fileUrl.Extension = "url" CreateUrlFile(URL, fileUrl) - If fileUrl.Exists Then Files.Add(fileUrl) + If fileUrl.Exists Then AddFile(fileUrl) End If If MyYouTubeSettings.CreateDescriptionFiles And Not Description.IsEmptyString Then Dim fileDesr As SFile = File fileDesr.Extension = "txt" TextSaver.SaveTextToFile(Description, fileDesr,,, EDP.None) - If fileDesr.Exists Then Files.Add(fileDesr) + If fileDesr.Exists Then AddFile(fileDesr) End If If PlaylistCount > 0 And Not CoverDownloaded And Not PlaylistID.IsEmptyString Then DownloadPlaylistCover(PlaylistID, File, UseCookies) @@ -1058,22 +1171,63 @@ Namespace API.YouTube.Objects Dim format$ Dim fPattern$ = $"{File.PathWithSeparator}{File.Name}." & "{0}" Dim fPatternFiles$ = $"{File.Name}*." & "{0}" - Dim fAacAudio As New SFile(String.Format(fPattern, aac)) - Dim fAc3Audio As New SFile(String.Format(fPattern, ac3)) - Dim aacRequested As Boolean = PostProcessing_OutputAudioFormats.Count > 0 AndAlso - PostProcessing_OutputAudioFormats.Exists(Function(af) af.StringToLower = aac) - Dim ac3Requested As Boolean = PostProcessing_OutputAudioFormats.Count > 0 AndAlso - PostProcessing_OutputAudioFormats.Exists(Function(af) af.StringToLower = ac3) + Dim fAacAudio As New TempFileConversion(New SFile(String.Format(fPattern, aac)), Me) + Dim mp3ThumbEmbedded As Boolean = False + + Dim tempFilesList As New List(Of TempFileConversion) + Dim ttFile As TempFileConversion + + Dim __updateBitrate As Boolean = OutputAudioBitrate > 0 AndAlso (SelectedAudioIndex = -1 OrElse SelectedAudio.Bitrate <> OutputAudioBitrate) + If __updateBitrate Then Bitrate = OutputAudioBitrate + Dim updateBitrate As Action(Of SFile) = + Sub(ByVal sourceFile As SFile) + If __updateBitrate AndAlso sourceFile.Exists Then + Dim destFile As SFile = sourceFile + destFile.Name &= "_new00" + .Execute($"ffmpeg -i ""{sourceFile}"" -crf {MyYouTubeSettings.DefaultAudioBitrate_crf.Value} -b:a {OutputAudioBitrate}k ""{destFile}""") + If destFile.Exists AndAlso sourceFile.Delete Then SFile.Rename(destFile, sourceFile) + End If + End Sub + Dim __getAAC_tried As Boolean = False + Dim AACExists As Func(Of Boolean) = Function() As Boolean + If Not __getAAC_tried Then + __getAAC_tried = True + .Execute($"ffmpeg -i ""{File}"" -vn -acodec {aac} ""{fAacAudio.File}""") + tempFilesList.Add(fAacAudio) + updateBitrate.Invoke(fAacAudio.File) + End If + Return fAacAudio.Exists + End Function + Dim tryToConvert As Action(Of String, SFile) = + Sub(ByVal codec As String, ByVal dFile As SFile) + ThrowAny(Token) + .Execute($"ffmpeg -i ""{File}"" -vn -acodec {codec} ""{dFile}""") + If Not codec = aac AndAlso Not dFile.Exists AndAlso AACExists.Invoke Then + ThrowAny(Token) + .Execute($"ffmpeg -i ""{fAacAudio.File}"" -f {codec} ""{dFile}""") + End If + End Sub + Dim embedThumbTo As Action(Of SFile) = + Sub(ByVal dFile As SFile) + If dFile.Exists And ThumbnailFile.Exists Then + Dim dFileNew As SFile = dFile + dFileNew.Name &= "_NEW" + .Execute($"ffmpeg -i ""{dFile}"" -i ""{ThumbnailFile}"" -map 0:0 -map 1:0 -c copy -id3v2_version 3 -metadata:s:v title=""Cover"" -metadata:s:v comment=""Cover"" ""{dFileNew}""") + If dFileNew.Exists AndAlso dFile.Delete(,, EDP.ReturnValue) Then SFile.Rename(dFileNew, dFile) + End If + End Sub + 'Subtitles ThrowAny(Token) If PostProcessing_OutputSubtitlesFormats.Count > 0 Then files = SFile.GetFiles(File, String.Format(fPatternFiles, OutputSubtitlesFormat.StringToLower),, EDP.ReturnValue) + AddFile(files) If files.ListExists Then For Each f In files For Each format In PostProcessing_OutputSubtitlesFormats format = format.StringToLower commandFile = $"{f.PathWithSeparator}{f.Name}.{format}" - Me.Files.Add(commandFile) + AddFile(commandFile) ThrowAny(Token) .Execute($"ffmpeg -i ""{f}"" ""{commandFile}""") Next @@ -1081,46 +1235,81 @@ Namespace API.YouTube.Objects End If End If + 'Audio ThrowAny(Token) - If PostProcessing_OutputAudioFormats.Count > 0 Or PostProcessing_AudioAC3 Then - If Not fAacAudio.Exists Then .Execute($"ffmpeg -i ""{File}"" -vn -acodec {aac} ""{fAacAudio}""") - If PostProcessing_AudioAC3 And Not fAc3Audio.Exists Then - ThrowAny(Token) - .Execute($"ffmpeg -i ""{File}"" -vn -acodec {ac3} ""{fAc3Audio}""") - If Not fAc3Audio.Exists And fAacAudio.Exists Then ThrowAny(Token) : .Execute($"ffmpeg -i ""{fAacAudio}"" -f {ac3} ""{fAc3Audio}""") + If PostProcessing_OutputAudioFormats.Count > 0 Or PostProcessing_AudioAC3 Or PostProcessing_AudioMP3 Or __updateBitrate Then + + If PostProcessing_AudioAC3 Then + ttFile = New TempFileConversion(New SFile(String.Format(fPattern, ac3)), Me) With {.ToReplace = True} + tempFilesList.Add(ttFile) + If Not ttFile.Exists Then tryToConvert.Invoke(ac3, ttFile.File) + updateBitrate.Invoke(ttFile.File) + End If + + If PostProcessing_AudioMP3 Then + ttFile = New TempFileConversion(New SFile(String.Format(fPattern, mp3)), Me) With {.ToReplace = True} + tempFilesList.Add(ttFile) + If Not ttFile.Requested Then ttFile.Requested = SelectedVideoIndex = -1 And OutputAudioCodec.StringToLower = mp3 + If Not ttFile.Exists Then tryToConvert.Invoke(mp3, ttFile.File) + updateBitrate.Invoke(ttFile.File) + embedThumbTo.Invoke(ttFile.File) + mp3ThumbEmbedded = True End If + + If __updateBitrate Then + format = OutputAudioCodec.StringToLower + If Not format.IsEmptyString Then + f = String.Format(fPattern, format) + ttFile = New TempFileConversion(f, Me) With {.ToReplace = True} + If Not ttFile.Requested Then ttFile.Requested = SelectedVideoIndex = -1 + If Not f.Exists Then + tempFilesList.ListAddValue(ttFile, LAP.NotContainsOnly) + tryToConvert.Invoke(format, f) + updateBitrate.Invoke(f) + ElseIf Not tempFilesList.Contains(ttFile) Then + tempFilesList.Add(ttFile) + updateBitrate.Invoke(f) + End If + End If + End If + If PostProcessing_OutputAudioFormats.Count > 0 Then For Each format In PostProcessing_OutputAudioFormats format = format.StringToLower f = String.Format(fPattern, format) - Me.Files.Add(f) - If Not format = ac3 Or Not f.Exists Then - ThrowAny(Token) - .Execute($"ffmpeg -i ""{fAacAudio}"" -f {format} ""{f}""") + AddFile(f) + If Not f.Exists Then + tryToConvert.Invoke(format, f) + updateBitrate(f) + If format = mp3 And Not mp3ThumbEmbedded And MyYouTubeSettings.DefaultAudioEmbedThumbnail_ExtractedFiles Then _ + embedThumbTo.Invoke(f) : mp3ThumbEmbedded = True If Not M3U8_PlaylistFiles.ListExists AndAlso f.Exists Then M3U8_Append(f) End If Next End If End If + 'Update video ThrowAny(Token) - If PostProcessing_AudioAC3 Then + If SelectedVideoIndex >= 0 AndAlso tempFilesList.Count > 0 AndAlso tempFilesList.Exists(Function(tf) tf.ToReplace) Then f = File - If SelectedVideoIndex >= 0 Then - f.Name &= "tmp00" - Else - f.Extension = ac3 + f.Name &= "tmp00" + Dim tfr As SFile = tempFilesList.FirstOrDefault(Function(tf) tf.ToReplace).File + If tfr.Exists And Not f.Exists Then + ThrowAny(Token) + .Execute($"ffmpeg -i ""{File}"" -i ""{tfr}"" -c:v copy -c copy -map 0:v:0 -map 1:a:0 ""{f}""") End If - If Not f.Exists Then ThrowAny(Token) : .Execute($"ffmpeg -i ""{File}"" -i ""{fAc3Audio}"" -c:v copy -c copy -map 0:v:0 -map 1:a:0 ""{f}""") If f.Exists Then File.Delete() If SelectedVideoIndex >= 0 Then SFile.Rename(f, File,, EDP.LogMessageValue) End If - If fAacAudio.Exists And Not aacRequested Then fAacAudio.Delete() - If fAc3Audio.Exists And Not ac3Requested And SelectedVideoIndex >= 0 Then fAc3Audio.Delete() End If - If SelectedVideoIndex >= 0 AndAlso OutputVideoFPS > 0 AndAlso SelectedVideo.Bitrate > OutputVideoFPS Then + 'Delete unrequsted files + If tempFilesList.Count > 0 Then tempFilesList.ForEach(Sub(tfr) If Not tfr.Requested Then tfr.File.Delete(,, EDP.None)) : tempFilesList.Clear() + + 'Update video FPS + If SelectedVideoIndex >= 0 AndAlso OutputVideoFPS > 0 AndAlso SelectedVideo.Bitrate <> OutputVideoFPS Then f = File f.Name &= "tmp00" .Execute($"ffmpeg -i ""{File}"" -filter:v fps={OutputVideoFPS.ToString.Replace(",", ".")} -c:a copy ""{f}""") @@ -1132,6 +1321,12 @@ Namespace API.YouTube.Objects End If End If End With + + Dim newSize# = 0 + If File.Exists Then newSize += File.Size + If Files.Count > 0 Then newSize += (From eFile As SFile In Files Where eFile.Exists Select eFile.Size).Sum + If ThumbnailFile.Exists Then newSize += ThumbnailFile.Size + If newSize > 0 Then newSize /= 1024 : Size = newSize : _SizeRecalculated = True End Using _MediaState = UMStates.Downloaded Catch oex As OperationCanceledException When Token.IsCancellationRequested @@ -1304,6 +1499,7 @@ Namespace API.YouTube.Objects End Sub #End Region #Region "Parse" + Friend Const DRC As String = "drc" Public Overridable Function Parse(ByVal Container As EContainer, ByVal Path As SFile, ByVal IsMusic As Boolean, Optional ByVal Token As CancellationToken = Nothing, Optional ByVal Progress As IMyProgress = Nothing) As Boolean Implements IYouTubeMediaContainer.Parse Try @@ -1366,6 +1562,7 @@ Namespace API.YouTube.Objects _File.Name = $"{ID}.{ext}" End If If Not MyYouTubeSettings.OutputPath.IsEmptyString Then _File.Path = MyYouTubeSettings.OutputPath.Value.Path + _File = CleanFileName(_File) File = _File If .Contains("duration") Then @@ -1455,7 +1652,8 @@ Namespace API.YouTube.Objects obj = New MediaObject With { .ID = ee.Value("format_id"), .URL = ee.Value("url"), - .Extension = ee.Value("ext") + .Extension = ee.Value("ext"), + .ID_DRC = Not .ID.IsEmptyString AndAlso .ID.StringToLower.Contains(DRC) } obj.Width = AConvert(Of Integer)(ee.Value("width"), NumberProvider, -1) obj.Height = AConvert(Of Integer)(ee.Value("height"), NumberProvider, -1) @@ -1510,6 +1708,14 @@ Namespace API.YouTube.Objects If MediaObjects.Count > 0 AndAlso MediaObjects.LongCount(CountAVC) > 0 Then MediaObjects.RemoveAll(RemoveAVC) Next End If + If t = UMTypes.Audio And MediaObjects.Count > 0 Then + Dim __audioComparerCount As Func(Of MediaObject, MediaObject, Boolean) = + Function(mo, mo2) (mo2.Type = t And mo2.Extension = mo.Extension And mo2.Bitrate = mo.Bitrate) AndAlso + mo2.Size.RoundDown = mo.Size.RoundDown AndAlso ACheck(Of Integer)(mo2.ID) + Dim RemoveDRC As Predicate(Of MediaObject) = Function(mo) mo.Type = t AndAlso Not ACheck(Of Integer)(mo.ID) AndAlso + MediaObjects.LongCount(Function(mo2) __audioComparerCount.Invoke(mo, mo2)) > 0 + MediaObjects.RemoveAll(RemoveDRC) + End If End Sub Dim protocolCleaner As Action = Sub() @@ -1691,7 +1897,7 @@ Namespace API.YouTube.Objects _SubtitlesDelegated.Clear() SubtitlesSelectedIndexes.Clear() MediaObjects.Clear() - Files.Clear() + _Files.Clear() PostProcessing_OutputAudioFormats.Clear() PostProcessing_OutputSubtitlesFormats.Clear() End If diff --git a/SCrawler/API/Base/Declarations.vb b/SCrawler/API/Base/Declarations.vb index 8dcd51a..af8c39d 100644 --- a/SCrawler/API/Base/Declarations.vb +++ b/SCrawler/API/Base/Declarations.vb @@ -49,7 +49,7 @@ Namespace API.Base End Sub Public Overrides Function Convert(ByVal Value As Object, ByVal DestinationType As Type, ByVal Provider As IFormatProvider, Optional ByVal NothingArg As Object = Nothing, Optional ByVal e As ErrorsDescriber = Nothing) As Object - Dim v% = AConvert(Of Integer)(Value, -1) + Dim v% = AConvert(Of Integer)(Value, -1, EDP.ReturnValue) If v > 0 Then Return Value ElseIf Not ACheck(Of Integer)(Value) Then diff --git a/SCrawler/API/Base/DeclaredNames.vb b/SCrawler/API/Base/DeclaredNames.vb index 0991bb7..2076ab4 100644 --- a/SCrawler/API/Base/DeclaredNames.vb +++ b/SCrawler/API/Base/DeclaredNames.vb @@ -11,8 +11,6 @@ Namespace API.Base Friend Const Header_Authorization As String = "authorization" Friend Const Header_CSRFToken As String = "x-csrf-token" - Friend Const Header_FB_FRIENDLY_NAME As String = "x-fb-friendly-name" - Friend Const ConcurrentDownloadsCaption As String = "Concurrent downloads" Friend Const ConcurrentDownloadsToolTip As String = "The number of concurrent downloads." Friend Const SavedPostsUserNameCaption As String = "Saved posts user" diff --git a/SCrawler/API/Base/M3U8Base.vb b/SCrawler/API/Base/M3U8Base.vb index c574d08..9189993 100644 --- a/SCrawler/API/Base/M3U8Base.vb +++ b/SCrawler/API/Base/M3U8Base.vb @@ -42,6 +42,13 @@ Namespace API.Base Friend NotInheritable Class M3U8Base Friend Const TempCacheFolderName As String = "tmpCache" Friend Const TempFilePrefix As String = "ConPart_" + Friend Const TempFileDefaultExtension As String = "ts" + ''' SFileNumbers.NumberProviderDefault + Friend Shared ReadOnly Property NumberProviderDefault As ANumbers + Get + Return SFileNumbers.NumberProviderDefault + End Get + End Property Private Sub New() End Sub Friend Shared Function CreateUrl(ByVal Appender As String, ByVal File As String) As String @@ -63,8 +70,7 @@ Namespace API.Base Friend Overloads Shared Function Download(ByVal URLs As List(Of M3U8URL), ByVal DestinationFile As SFile, Optional ByVal Responser As Responser = Nothing, Optional ByVal Token As CancellationToken = Nothing, Optional ByVal Progress As MyProgress = Nothing, Optional ByVal UsePreProgress As Boolean = True, Optional ByVal ExistingCache As CacheKeeper = Nothing, - Optional ByVal OnlyDownload As Boolean = False) As SFile - Const defaultExtension$ = "ts" + Optional ByVal OnlyDownload As Boolean = False, Optional ByVal SkipBroken As Boolean = False) As SFile Dim Cache As CacheKeeper = Nothing Using tmpPr As New PreProgress(Progress) Try @@ -89,13 +95,13 @@ Namespace API.Base End If End If Dim p As SFileNumbers = SFileNumbers.Default(ConcatFile.Name) - Dim pNum As ANumbers = SFileNumbers.NumberProviderDefault + Dim pNum As ANumbers = NumberProviderDefault p.NumberProvider = pNum DirectCast(p.NumberProvider, ANumbers).GroupSize = {URLs.Count.ToString.Length, 3}.Max ConcatFile = SFile.IndexReindex(ConcatFile,,, p, EDP.ReturnValue) Dim i% Dim dFile As SFile = cache2.RootDirectory - dFile.Extension = defaultExtension + dFile.Extension = TempFileDefaultExtension Using w As New DownloadObjects.WebClient2(Responser) For i = 0 To URLs.Count - 1 If progressExists Then @@ -107,9 +113,13 @@ Namespace API.Base End If Token.ThrowIfCancellationRequested() dFile.Name = $"{TempFilePrefix}{i.NumToString(pNum)}" - dFile.Extension = URLs(i).Extension.IfNullOrEmpty(defaultExtension) - w.DownloadFile(URLs(i).URL, dFile) - cache2.AddFile(dFile, True) + dFile.Extension = URLs(i).Extension.IfNullOrEmpty(TempFileDefaultExtension) + Try + w.DownloadFile(URLs(i).URL, dFile) + cache2.AddFile(dFile, True) + Catch ex As Exception + If Not SkipBroken Then Throw ex + End Try Next End Using If Not OnlyDownload Then _ diff --git a/SCrawler/API/Base/SiteSettingsBase.vb b/SCrawler/API/Base/SiteSettingsBase.vb index 08f767c..1e2b52e 100644 --- a/SCrawler/API/Base/SiteSettingsBase.vb +++ b/SCrawler/API/Base/SiteSettingsBase.vb @@ -17,6 +17,7 @@ Imports Download = SCrawler.Plugin.ISiteSettings.Download Namespace API.Base Friend MustInherit Class SiteSettingsBase : Implements ISiteSettings, IResponserContainer #Region "Declarations" + Protected ReadOnly Property SettingsVersion As PropertyValue Friend ReadOnly Property Site As String Implements ISiteSettings.Site Protected _Icon As Icon = Nothing Friend Overridable ReadOnly Property Icon As Icon Implements ISiteSettings.Icon @@ -61,6 +62,7 @@ Namespace API.Base End Sub #End Region #Region "Responser and cookies support" + Friend Const ResponserFilePrefix As String = "Responser_" Private _CookiesNetscapeFile As SFile = Nothing Friend ReadOnly Property CookiesNetscapeFile As SFile Get @@ -91,7 +93,7 @@ Namespace API.Base End Property Protected Sub UpdateResponserFile() Dim acc$ = If(AccountName.IsEmptyString OrElse AccountName = Hosts.SettingsHost.NameAccountNameDefault, String.Empty, $"_{AccountName}") - Responser.File = $"{SettingsFolderName}\Responser_{Site}{acc}.xml" + Responser.File = $"{SettingsFolderName}\{ResponserFilePrefix}{Site}{acc}.xml" _CookiesNetscapeFile = Responser.File _CookiesNetscapeFile.Name &= "_Cookies_Netscape" _CookiesNetscapeFile.Extension = "txt" @@ -106,6 +108,7 @@ Namespace API.Base _Icon = __Icon _Image = __Image Responser = New Responser With {.DeclaredError = EDP.ThrowException} + SettingsVersion = New PropertyValue(0) UpdateResponserFile() End Sub Friend Sub New(ByVal SiteName As String, ByVal CookiesDomain As String, ByVal AccName As String, ByVal Temp As Boolean, diff --git a/SCrawler/API/Base/UserDataBase.vb b/SCrawler/API/Base/UserDataBase.vb index d520226..9d806e8 100644 --- a/SCrawler/API/Base/UserDataBase.vb +++ b/SCrawler/API/Base/UserDataBase.vb @@ -1165,7 +1165,7 @@ BlockNullPicture: If Not Responser Is Nothing Then Responser.Dispose() Responser = New Responser If Not HOST.Responser Is Nothing Then Responser.Copy(HOST.Responser) - If Not Responser Is Nothing And _ResponserAutoUpdateCookies Then + If Not Responser Is Nothing And (_ResponserAutoUpdateCookies Or _ResponserAddResponseReceivedHandler) Then If _ResponserAutoUpdateCookies Then Responser.CookiesUpdateMode = CookieUpdateModes.ReplaceByNameAll Responser.CookiesExtractMode = Responser.CookiesExtractModes.Any @@ -1339,6 +1339,7 @@ BlockNullPicture: ResetHost() URL = Data.URL AccountName = Data.AccountName + TokenQueue = Token If HOST Is Nothing Then Throw New ExitException($"Host '{AccountName}' not found") Data.DownloadState = UserMediaStates.Tried Progress = Data.Progress @@ -1399,6 +1400,9 @@ BlockNullPicture: f = Web.FFMPEG.TakeSnapshot(f, ff, Settings.FfmpegFile, TimeSpan.FromSeconds(1),,, EDP.SendToLog + EDP.ReturnValue) If f.Exists Then DirectCast(Data, IDownloadableMedia).ThumbnailFile = f End If + Dim filesSize# = (From mm As UserMedia In _ContentNew Where mm.State = UStates.Downloaded AndAlso mm.File.Exists Select mm.File.Size).Sum + If filesSize > 0 Then filesSize /= 1024 + Data.Size = filesSize Else Data.DownloadState = UserMediaStates.Missing End If @@ -1771,6 +1775,7 @@ BlockNullPicture: Protected Overridable Function ValidateDownloadFile(ByVal URL As String, ByVal Media As UserMedia, ByRef Interrupt As Boolean) As Boolean Return True End Function + ''' MyFile.CutPath(IIf(IsSingleObjectDownload, 0, 1)).PathNoSeparator Protected Overridable Function DownloadContentDefault_GetRootDir() As String Return MyFile.CutPath(IIf(IsSingleObjectDownload, 0, 1)).PathNoSeparator End Function diff --git a/SCrawler/API/Base/YTDLP.vb b/SCrawler/API/Base/YTDLP.vb index 0369bbf..de95702 100644 --- a/SCrawler/API/Base/YTDLP.vb +++ b/SCrawler/API/Base/YTDLP.vb @@ -11,7 +11,7 @@ Namespace API.Base.YTDLP Friend Sub New(ByVal _Token As Threading.CancellationToken) MyBase.New(_Token) Commands.Clear() - MainProcessName = "yt-dlp" + MainProcessName = Settings.YtdlpFile.File.Name '"yt-dlp" ChangeDirectory(Settings.YtdlpFile.File) End Sub End Class diff --git a/SCrawler/API/Facebook/SiteSettings.vb b/SCrawler/API/Facebook/SiteSettings.vb index 35e5439..f1c5d43 100644 --- a/SCrawler/API/Facebook/SiteSettings.vb +++ b/SCrawler/API/Facebook/SiteSettings.vb @@ -46,7 +46,7 @@ Namespace API.Facebook With Responser.Headers .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.Authority, "www.facebook.com")) .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.Origin, "https://www.facebook.com")) - .Remove(DeclaredNames.Header_FB_FRIENDLY_NAME) + .Remove(Instagram.UserData.GQL_HEADER_FB_FRINDLY_NAME) End With Header_Accept = New PropertyValue(String.Empty, GetType(String)) ParsePhotoBlock = New PropertyValue(True) @@ -74,7 +74,7 @@ Namespace API.Facebook #End Region #Region "BaseAuthExists, GetUserUrl, GetUserPostUrl, IsMyUser, IsMyImageVideo" Friend Overrides Function BaseAuthExists() As Boolean - Return Responser.CookiesExists And ACheck(HH_IG_APP_ID.Value) + Return Responser.CookiesExists And ACheck(HH_IG_APP_ID.Value) And CBool(DownloadData_Impl.Value) End Function Friend Overrides Function GetUserUrl(ByVal User As IPluginContentProvider) As String Return DirectCast(User, UserData).GetProfileUrl diff --git a/SCrawler/API/Facebook/UserData.vb b/SCrawler/API/Facebook/UserData.vb index 2a27373..00057a6 100644 --- a/SCrawler/API/Facebook/UserData.vb +++ b/SCrawler/API/Facebook/UserData.vb @@ -124,26 +124,34 @@ Namespace API.Facebook .SendToLogOnlyMessage = True, .ReplaceMainMessage = True}) End Sub End Class - Private Token_dtsg As String = String.Empty - Private Token_lsd As String = String.Empty Private Token_Photosby As String = String.Empty Private Limit As Integer = -1 + Private Sub WaitTimer() + If CInt(MySettings.RequestsWaitTimer_Any.Value) > 0 Then Thread.Sleep(CInt(MySettings.RequestsWaitTimer_Any.Value)) + End Sub + Private Sub DisableDownload() + MySettings.DownloadData_Impl.Value = False + MyMainLOG = $"{Site} downloading is disabled until you update your credentials" + End Sub Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken) - Try - GetUserTokens(Token) - LoadSavePostsKV(True) - Limit = If(DownloadTopCount, -1) - If IsSavedPosts Then - DownloadData_SavedPosts(String.Empty, Token) - Else - If DownloadImages And ParsePhotoBlock Then DownloadData_Photo(String.Empty, Token) - If DownloadVideos And ParseVideoBlock Then DownloadData_Video(String.Empty, Token) - If (DownloadImages Or DownloadVideos) And ParseStoriesBlock Then DownloadData_Stories(Token) - End If - LoadSavePostsKV(False) - Finally - MySettings.UpdateResponserData(Responser) - End Try + If CBool(MySettings.DownloadData_Impl.Value) Then + Try + ResetBaseTokens() + GetUserTokens(Token) + LoadSavePostsKV(True) + Limit = If(DownloadTopCount, -1) + If IsSavedPosts Then + DownloadData_SavedPosts(String.Empty, Token) + Else + If DownloadImages And ParsePhotoBlock Then DownloadData_Photo(String.Empty, Token) + If DownloadVideos And ParseVideoBlock Then DownloadData_Video(String.Empty, Token) + If (DownloadImages Or DownloadVideos) And ParseStoriesBlock Then DownloadData_Stories(Token) + End If + LoadSavePostsKV(False) + Finally + MySettings.UpdateResponserData(Responser) + End Try + End If End Sub Private Const Header_fb_fr_name_Photo As String = "ProfileCometAppCollectionPhotosRendererPaginationQuery" Private Const Header_fb_fr_name_Video As String = "PagesCometChannelTabAllVideosCardImplPaginationQuery" @@ -167,13 +175,13 @@ Namespace API.Facebook ValidateBaseTokens() If Token_Photosby.IsEmptyString Then Throw New TokensException("Unable to obtain token 'Token_Photosby'", False) - URL = String.Format(Graphql_UrlPattern, Token_lsd, DocID_Photo, Header_fb_fr_name_Photo, - SymbolsConverter.ASCII.EncodeSymbolsOnly(Token_dtsg), + URL = String.Format(Graphql_UrlPattern, Token_lsd, DocID_Photo, Header_fb_fr_name_Photo, Token_dtsg_Var, SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & String.Format(VarPattern, Cursor, Token_Photosby) & "}")) ResponserApplyDefs(Header_fb_fr_name_Photo) ThrowAny(Token) + WaitTimer() Dim r$ = Responser.GetResponse(URL) If Not r.IsEmptyString Then Using j As EContainer = JsonDocument.Parse(r) @@ -233,13 +241,13 @@ Namespace API.Facebook If VideoPageID.IsEmptyString Then Throw New TokensException("Unable to obtain 'VideoPageID'", False) ValidateBaseTokens() - URL = String.Format(Graphql_UrlPattern, Token_lsd, DocID_Video, Header_fb_fr_name_Video, - SymbolsConverter.ASCII.EncodeSymbolsOnly(Token_dtsg), + URL = String.Format(Graphql_UrlPattern, Token_lsd, DocID_Video, Header_fb_fr_name_Video, Token_dtsg_Var, SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & String.Format(VarPattern, If(Cursor.IsEmptyString, "null", $"""{Cursor}"""), VideoPageID) & "}")) ResponserApplyDefs(Header_fb_fr_name_Video) ThrowAny(Token) + WaitTimer() Dim r$ = Responser.GetResponse(URL) If Not r.IsEmptyString Then Using j As EContainer = JsonDocument.Parse(r) @@ -288,13 +296,13 @@ Namespace API.Facebook ValidateBaseTokens() If StoryBucket.IsEmptyString Then Throw New TokensException("Unable to obtain 'StoryBucket'", False) - URL = String.Format(Graphql_UrlPattern, Token_lsd, DocID_Stories, Header_fb_fr_name_Stories, - SymbolsConverter.ASCII.EncodeSymbolsOnly(Token_dtsg), + URL = String.Format(Graphql_UrlPattern, Token_lsd, DocID_Stories, Header_fb_fr_name_Stories, Token_dtsg_Var, SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & String.Format(VarPattern, StoryBucket) & "}")) ResponserApplyDefs(Header_fb_fr_name_Stories) ThrowAny(Token) + WaitTimer() Dim r$ = Responser.GetResponse(URL) If Not r.IsEmptyString Then r = RegexReplace(r, RParams.DM("[^\r\n]+", 0, EDP.ReturnValue)) If Not r.IsEmptyString Then @@ -357,13 +365,13 @@ Namespace API.Facebook Dim pid As PostKV ValidateBaseTokens() - URL = String.Format(Graphql_UrlPattern, Token_lsd, DocID_SavedPosts, Header_fb_fr_name_SavedPosts, - SymbolsConverter.ASCII.EncodeSymbolsOnly(Token_dtsg), + URL = String.Format(Graphql_UrlPattern, Token_lsd, DocID_SavedPosts, Header_fb_fr_name_SavedPosts, Token_dtsg_Var, SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & String.Format(VarPattern, If(Cursor.IsEmptyString, "null", $"""{Cursor}""")) & "}")) ResponserApplyDefs(Header_fb_fr_name_SavedPosts) ThrowAny(Token) + WaitTimer() Dim r$ = Responser.GetResponse(URL) If Not r.IsEmptyString Then Using j As EContainer = JsonDocument.Parse(r) @@ -421,6 +429,7 @@ Namespace API.Facebook If Round > 0 Then ThrowAny(Token) Dim script$, newUrl$ Dim jNode As EContainer, jNode2 As EContainer + WaitTimer() Dim r$ = resp.GetResponse(PostUrl) If Not r.IsEmptyString Then @@ -488,16 +497,20 @@ Namespace API.Facebook #End Region #Region "ValidateBaseTokens, GetVideoPageID, GetUserTokens" ''' - Private Sub ValidateBaseTokens() + Protected Overrides Function ValidateBaseTokens() As Boolean Dim tokens$ = String.Empty - If Token_dtsg.IsEmptyString Then tokens.StringAppend("Token_dtsg") - If Token_lsd.IsEmptyString Then tokens.StringAppend("Token_lsd") - If Not tokens.IsEmptyString Then Throw New TokensException($"Unable to obtain token(s) ({tokens}){vbCr}Your credentials may have expired.", True) - End Sub + If Not ValidateBaseTokens(tokens) Then + DisableDownload() + Throw New TokensException($"Unable to obtain token(s) ({tokens}). Your credentials may have expired.", True) + Else + Return True + End If + End Function Private Sub GetVideoPageID(ByVal Token As CancellationToken) Dim URL$ = $"{GetProfileUrl()}\videos" Dim resp As Responser = HtmlResponserCreate() Try + WaitTimer() Dim r$ = resp.GetResponse(URL) If Not r.IsEmptyString Then VideoPageID = RegexReplace(r, Regex_VideoPageID) Catch ex As Exception @@ -510,9 +523,9 @@ Namespace API.Facebook Dim URL$ = If(IsSavedPosts, "https://www.facebook.com/saved", GetProfileUrl()) Dim resp As Responser = HtmlResponserCreate() Try - Token_dtsg = String.Empty - Token_lsd = String.Empty + ResetBaseTokens() Token_Photosby = String.Empty + WaitTimer() Dim r$ = resp.GetResponse(URL) If Not r.IsEmptyString Then If Responser.CookiesExists Then Responser.Cookies.Update(resp.Cookies) @@ -535,8 +548,7 @@ Namespace API.Facebook #Region "Responser options" Private Sub ResponserApplyDefs(ByVal __fb_friendly_name As String) With Responser - .Headers.Add(ThreadsNet.UserData.Header_FB_LSD, Token_lsd) - .Headers.Add(DeclaredNames.Header_FB_FRIENDLY_NAME, __fb_friendly_name) + UpdateHeadersGQL(__fb_friendly_name) .Method = "POST" .Accept = "*/*" .Referer = GetProfileUrl() @@ -655,6 +667,7 @@ Namespace API.Facebook Else URL = String.Format(VideoHtmlUrlPattern, m.Post.ID) End If + WaitTimer() r = resp.GetResponse(URL) If Not r.IsEmptyString Then re.Pattern = String.Format(pattern, nameHD) diff --git a/SCrawler/API/Instagram/Declarations.vb b/SCrawler/API/Instagram/Declarations.vb index 1545453..73958ff 100644 --- a/SCrawler/API/Instagram/Declarations.vb +++ b/SCrawler/API/Instagram/Declarations.vb @@ -18,7 +18,7 @@ Namespace API.Instagram Friend ReadOnly ObtainMedia_SizeFuncPic_RegexP As RParams = RParams.DMS("_p(\d+)x(\d+)", 1, EDP.ReturnValue) Friend ReadOnly ObtainMedia_SizeFuncPic_RegexS As RParams = RParams.DMS("_s(\d+)x(\d+)", 1, EDP.ReturnValue) Friend Const PageTokenRegexPatternDefault As String = "\[\],{""token"":""(.*?)""},\d+\]" - Friend Sub UpdateResponser(ByVal Source As IResponse, ByRef Destination As Responser) + Friend Sub UpdateResponser(ByVal Source As IResponse, ByRef Destination As Responser, ByVal UpdateWwwClaim As Boolean) Const r_wwwClaimName$ = "x-ig-set-www-claim" Const r_tokenName$ = SiteSettings.Header_CSRF_TOKEN_COOKIE If Not Source Is Nothing Then @@ -35,17 +35,17 @@ Namespace API.Instagram Dim token$ = String.Empty With Source If isInternal Then - If .HeadersExists Then wwwClaim = .Headers.Value(wwwClaimName) + If UpdateWwwClaim And .HeadersExists Then wwwClaim = .Headers.Value(wwwClaimName) If .CookiesExists Then token = If(.Cookies.FirstOrDefault(Function(c) c.Name = tokenName)?.Value, String.Empty) Else If .HeadersExists Then - wwwClaim = .Headers.Value(wwwClaimName) + If UpdateWwwClaim Then wwwClaim = .Headers.Value(wwwClaimName) token = .Headers.Value(tokenName) End If End If End With - If Not wwwClaim.IsEmptyString Then Destination.Headers.Add(SiteSettings.Header_IG_WWW_CLAIM, wwwClaim) + If UpdateWwwClaim And Not wwwClaim.IsEmptyString Then Destination.Headers.Add(SiteSettings.Header_IG_WWW_CLAIM, wwwClaim) If Not token.IsEmptyString Then Destination.Headers.Add(SiteSettings.Header_CSRF_TOKEN, token) If Not isInternal Then Destination.Cookies.Update(Source.Cookies, CookieKeeper.UpdateModes.ReplaceByNameAll, False, EDP.SendToLog) diff --git a/SCrawler/API/Instagram/SiteSettings.vb b/SCrawler/API/Instagram/SiteSettings.vb index 73217c2..b94f456 100644 --- a/SCrawler/API/Instagram/SiteSettings.vb +++ b/SCrawler/API/Instagram/SiteSettings.vb @@ -19,7 +19,7 @@ Namespace API.Instagram Friend Class SiteSettings : Inherits SiteSettingsBase #Region "Declarations" #Region "Providers" - Private Class TimersChecker : Inherits FieldsCheckerProviderBase + Friend Class TimersChecker : Inherits FieldsCheckerProviderBase Private ReadOnly LVProvider As New ANumbers With {.FormatOptions = ANumbers.Options.GroupIntegral} Private ReadOnly _LowestValue As Integer Friend Sub New(ByVal LowestValue As Integer) @@ -32,7 +32,7 @@ Namespace API.Instagram If Not ACheck(Of Integer)(Value) Then TypeError = True ElseIf CInt(Value) < _LowestValue Then - ErrorMessage = $"The value of [{Name}] field must be greater than or equal to {_LowestValue.NumToString(LVProvider)}" + ErrorMessage = $"The value of '{Name}' field must be greater than or equal to {_LowestValue.NumToString(LVProvider)}" HasError = True Else Return Value @@ -47,7 +47,7 @@ Namespace API.Instagram If v > 0 Or v = -1 Then Return Value Else - ErrorMessage = $"The value of [{Name}] field must be greater than 0 or equal to -1" + ErrorMessage = $"The value of '{Name}' field must be greater than 0 or equal to -1" HasError = True Return Nothing End If @@ -66,24 +66,30 @@ Namespace API.Instagram Friend ReadOnly Property HH_CSRF_TOKEN As PropertyValue - Friend Property HH_IG_APP_ID As PropertyValue + Friend ReadOnly Property HH_IG_APP_ID As PropertyValue - Friend Property HH_ASBD_ID As PropertyValue + Friend ReadOnly Property HH_ASBD_ID As PropertyValue 'PropertyOption(ControlText:="x-ig-www-claim", IsAuth:=True, AllowNull:=True) - Friend Property HH_IG_WWW_CLAIM As PropertyValue + Friend ReadOnly Property HH_IG_WWW_CLAIM As PropertyValue + Private ReadOnly Property HH_IG_WWW_CLAIM_IS_ZERO As Boolean + Get + Dim v$ = AConvert(Of String)(HH_IG_WWW_CLAIM.Value, String.Empty) + Return Not v.IsEmptyString AndAlso v = "0" + End Get + End Property - Private Property HH_BROWSER As PropertyValue + Private ReadOnly Property HH_BROWSER As PropertyValue - Private Property HH_BROWSER_EXT As PropertyValue + Private ReadOnly Property HH_BROWSER_EXT As PropertyValue - Private Property HH_PLATFORM As PropertyValue + Private ReadOnly Property HH_PLATFORM As PropertyValue - Private Property HH_USER_AGENT As PropertyValue + Private ReadOnly Property HH_USER_AGENT As PropertyValue Friend Overrides Function BaseAuthExists() As Boolean Return Responser.CookiesExists And ACheck(HH_IG_APP_ID.Value) And ACheck(HH_CSRF_TOKEN.Value) End Function @@ -110,17 +116,55 @@ Namespace API.Instagram End If End If End Sub +#Region "HH_IG_WWW_CLAIM" + + Private ReadOnly Property HH_IG_WWW_CLAIM_UPDATE_INTERVAL As PropertyValue + + Friend ReadOnly Property HH_IG_WWW_CLAIM_ALWAYS_ZERO As PropertyValue + + Friend ReadOnly Property HH_IG_WWW_CLAIM_RESET_EACH_SESSION As PropertyValue + + Friend ReadOnly Property HH_IG_WWW_CLAIM_RESET_EACH_TARGET As PropertyValue + + Friend ReadOnly Property HH_IG_WWW_CLAIM_USE As PropertyValue + + Friend ReadOnly Property HH_IG_WWW_CLAIM_USE_DEFAULT_ALGO As PropertyValue + + Private ReadOnly Property TokenUpdateIntervalProvider As IFormatProvider +#End Region + + Friend ReadOnly Property USE_GQL As PropertyValue #End Region #Region "Download properties" - + Friend Const TimersUrgentTip As String = vbCr & "It is highly recommended not to change the default value." + + Friend ReadOnly Property RequestsWaitTimer_Any As PropertyValue + + Private ReadOnly Property RequestsWaitTimer_AnyProvider As IFormatProvider + Friend ReadOnly Property RequestsWaitTimer As PropertyValue Private ReadOnly Property RequestsWaitTimerProvider As IFormatProvider - + Friend ReadOnly Property RequestsWaitTimerTaskCount As PropertyValue Private ReadOnly Property RequestsWaitTimerTaskCountProvider As IFormatProvider - + Friend ReadOnly Property SleepTimerOnPostsLimit As PropertyValue Private ReadOnly Property SleepTimerOnPostsLimitProvider As IFormatProvider @@ -161,6 +205,21 @@ Namespace API.Instagram #Region "429 bypass" Private ReadOnly Property DownloadingErrorDate As PropertyValue + + Private ReadOnly Property DownloadingErrorDateProvider As IFormatProvider = + New CustomProvider(Function(ByVal v As Object, ByVal d As Type) As Object + If d Is GetType(Date) Then + Return AConvert(Of Date)(v, AModes.Var, Nothing) + ElseIf d Is GetType(String) Then + If Not IsNothing(v) AndAlso TypeOf v Is Date AndAlso CDate(v) = Date.MinValue Then + Return String.Empty + Else + Return AConvert(Of String)(v, AModes.XML, String.Empty) + End If + Else + Return Nothing + End If + End Function) Friend Property LastApplyingValue As Integer? = Nothing Friend ReadOnly Property ReadyForDownload As Boolean Get @@ -174,11 +233,64 @@ Namespace API.Instagram End With End Get End Property + Private Const LastDownloadDateResetInterval As Integer = 60 Private ReadOnly Property LastDownloadDate As PropertyValue Private ReadOnly Property LastRequestsCount As PropertyValue + Private ReadOnly MyLastRequests As Dictionary(Of Date, Integer) + Private ReadOnly Property MyLastRequestsDate As Date + Get + Try + Return If(MyLastRequests.Count > 0, MyLastRequests.Keys.Max, Now.AddDays(-1)) + Catch ex As Exception + Return ErrorsDescriber.Execute(EDP.SendToLog + EDP.ReturnValue, ex, "[SiteSettings.Instagram.MyLastRequestsDate]", Now.AddDays(-1)) + End Try + End Get + End Property + Private Property MyLastRequestsCount As Integer + Get + Try + Return If(MyLastRequests.Count > 0, MyLastRequests.Values.Sum, 0) + Catch ex As Exception + Return ErrorsDescriber.Execute(EDP.SendToLog + EDP.ReturnValue, ex, "[SiteSettings.Instagram.MyLastRequestsCount]", 0) + End Try + End Get + Set(ByVal NewValue As Integer) + If Not MyLastRequests.ContainsKey(ActiveSessionDate) Then + MyLastRequests.Add(ActiveSessionDate, NewValue) + Else + MyLastRequests(ActiveSessionDate) += NewValue + End If + End Set + End Property + Private Sub RefreshMyLastRequests(Optional ByVal DateToReplace As Date? = Nothing) + Try + With MyLastRequests + If .Count > 0 Then + Dim d As Date + For i% = .Count - 1 To 0 Step -1 + d = .Keys(i) + If (Not DateToReplace.HasValue OrElse ActiveJobs < 1 OrElse d <> ActiveSessionDate) And + d.AddMinutes(LastDownloadDateResetInterval) < Now Then .Remove(d) + Next + End If + If .Count > 0 Then + If DateToReplace.HasValue Then + If .Keys.Contains(ActiveSessionDate) Then + Dim v% = .Item(ActiveSessionDate) + .Remove(ActiveSessionDate) + .Add(DateToReplace.Value, v) + End If + End If + LastDownloadDate.Value = .Keys.Max + LastRequestsCount.Value = .Values.Sum + End If + End With + Catch ex As Exception + ErrorsDescriber.Execute(EDP.SendToLog, ex, "[SiteSettings.Instagram.RefreshMyLastRequests]") + End Try + End Sub - Private Property LastRequestsCountLabel As PropertyValue - Private ReadOnly LastRequestsCountLabelStr As Func(Of Integer, String) = Function(r) $"Number of spent requests: {r.NumToGroupIntegral}" + Private ReadOnly Property LastRequestsCountLabel As PropertyValue Private TooManyRequestsReadyForCatch As Boolean = True Friend Function GetWaitDate() As Date With DownloadingErrorDate @@ -235,11 +347,14 @@ Namespace API.Instagram browserExt = .Value(Header_BrowserExt) platform = .Value(Header_Platform_Verion) End If + '.Add(Header_IG_WWW_CLAIM, 0) .Add("Dnt", 1) + .Add("Dpr", 1) .Add("Sec-Ch-Ua-Mobile", "?0") + .Add("Sec-Ch-Ua-Model", """""") .Add("Sec-Ch-Ua-Platform", """Windows""") - .Add("Sec-Fetch-Dest", "empty") - .Add("Sec-Fetch-Mode", "cors") + .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchDest, "empty")) + .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchMode, "cors")) .Add("Sec-Fetch-Site", "same-origin") .Add("X-Requested-With", "XMLHttpRequest") End With @@ -257,6 +372,15 @@ Namespace API.Instagram HH_PLATFORM = New PropertyValue(platform, GetType(String), Sub(v) ChangeResponserFields(NameOf(HH_PLATFORM), v)) HH_USER_AGENT = New PropertyValue(useragent, GetType(String), Sub(v) ChangeResponserFields(NameOf(HH_USER_AGENT), v)) + HH_IG_WWW_CLAIM_UPDATE_INTERVAL = New PropertyValue(120) + HH_IG_WWW_CLAIM_ALWAYS_ZERO = New PropertyValue(False) + HH_IG_WWW_CLAIM_RESET_EACH_SESSION = New PropertyValue(True) + HH_IG_WWW_CLAIM_RESET_EACH_TARGET = New PropertyValue(True) + HH_IG_WWW_CLAIM_USE = New PropertyValue(True) + HH_IG_WWW_CLAIM_USE_DEFAULT_ALGO = New PropertyValue(True) + TokenUpdateIntervalProvider = New TokenRefreshIntervalProvider + USE_GQL = New PropertyValue(False) + DownloadTimeline = New PropertyValue(True) DownloadTimeline_Def = New PropertyValue(DownloadTimeline.Value, GetType(Boolean)) DownloadReels = New PropertyValue(False) @@ -268,6 +392,8 @@ Namespace API.Instagram DownloadTagged = New PropertyValue(False) DownloadTagged_Def = New PropertyValue(DownloadTagged.Value, GetType(Boolean)) + RequestsWaitTimer_Any = New PropertyValue(1000) + RequestsWaitTimer_AnyProvider = New TimersChecker(0) RequestsWaitTimer = New PropertyValue(1000) RequestsWaitTimerProvider = New TimersChecker(100) RequestsWaitTimerTaskCount = New PropertyValue(1) @@ -283,17 +409,22 @@ Namespace API.Instagram TaggedNotifyLimit = New PropertyValue(200) TaggedNotifyLimitProvider = New TaggedNotifyLimitChecker - DownloadingErrorDate = New PropertyValue(Nothing, GetType(Date)) + DownloadingErrorDate = New PropertyValue(Now.AddYears(10), GetType(Date)) LastDownloadDate = New PropertyValue(Now.AddDays(-1)) LastRequestsCount = New PropertyValue(0) - LastRequestsCountLabel = New PropertyValue(LastRequestsCountLabelStr.Invoke(LastRequestsCount.Value)) - LastRequestsCount.OnChangeFunction = Sub(vv) LastRequestsCountLabel.Value = LastRequestsCountLabelStr.Invoke(vv) + LastRequestsCountLabel = New PropertyValue(String.Empty, GetType(String)) + MyLastRequests = New Dictionary(Of Date, Integer) _AllowUserAgentUpdate = False UrlPatternUser = "https://www.instagram.com/{0}/" UserRegex = RParams.DMS(String.Format(UserRegexDefaultPattern, "instagram.com/"), 1) ImageVideoContains = "instagram.com" End Sub + Friend Overrides Sub EndInit() + Try : MyLastRequests.Add(LastDownloadDate.Value, LastRequestsCount.Value) : Catch : End Try + If Not CBool(HH_IG_WWW_CLAIM_USE.Value) Then Responser.Headers.Remove(Header_IG_WWW_CLAIM) + MyBase.EndInit() + End Sub #End Region #Region "PropertiesDataChecker" @@ -326,11 +457,23 @@ Namespace API.Instagram Return ActiveJobs < 2 AndAlso Not SkipUntilNextSession AndAlso ReadyForDownload AndAlso BaseAuthExists() AndAlso DownloadTimeline.Value End Function Private ActiveJobs As Integer = 0 + Private ActiveSessionDate As Date Private _NextWNM As UserData.WNM = UserData.WNM.Notify Private _NextTagged As Boolean = True Friend Overrides Sub DownloadStarted(ByVal What As Download) ActiveJobs += 1 - If CDate(LastDownloadDate.Value).AddMinutes(120) < Now Or Not ACheck(HH_IG_WWW_CLAIM.Value) Then HH_IG_WWW_CLAIM.Value = "0" + If ActiveJobs = 1 Then ActiveSessionDate = Now + If Not HH_IG_WWW_CLAIM_IS_ZERO AndAlso + ( + (CBool(HH_IG_WWW_CLAIM_USE_DEFAULT_ALGO.Value) AndAlso MyLastRequestsDate.AddMinutes(HH_IG_WWW_CLAIM_UPDATE_INTERVAL.Value) < Now) Or + Not ACheck(HH_IG_WWW_CLAIM.Value) Or + ( + Not ( + CBool(HH_IG_WWW_CLAIM_USE_DEFAULT_ALGO.Value) And + (CBool(HH_IG_WWW_CLAIM_RESET_EACH_SESSION.Value) Or CBool(HH_IG_WWW_CLAIM_ALWAYS_ZERO.Value)) + ) + ) + ) Then HH_IG_WWW_CLAIM.Value = "0" End Sub Friend Overrides Sub BeforeStartDownload(ByVal User As Object, ByVal What As Download) With DirectCast(User, UserData) @@ -338,10 +481,9 @@ Namespace API.Instagram .WaitNotificationMode = _NextWNM .TaggedCheckSession = _NextTagged End If - If CDate(LastDownloadDate.Value).AddMinutes(60) > Now Then - .RequestsCount = LastRequestsCount.Value + If MyLastRequestsDate.AddMinutes(LastDownloadDateResetInterval) > Now Then + .RequestsCount = MyLastRequestsCount Else - LastRequestsCount.Value = 0 .RequestsCount = 0 End If End With @@ -351,8 +493,7 @@ Namespace API.Instagram _NextWNM = .WaitNotificationMode If _NextWNM = UserData.WNM.SkipTemp Or _NextWNM = UserData.WNM.SkipCurrent Then _NextWNM = UserData.WNM.Notify _NextTagged = .TaggedCheckSession - LastDownloadDate.Value = Now - LastRequestsCount.Value = .RequestsCount + MyLastRequestsCount = .RequestsCountSession _FieldsChangerSuspended = True HH_IG_WWW_CLAIM.Value = Responser.Headers.Value(Header_IG_WWW_CLAIM) HH_CSRF_TOKEN.Value = Responser.Headers.Value(Header_CSRF_TOKEN) @@ -362,7 +503,7 @@ Namespace API.Instagram Friend Overrides Sub DownloadDone(ByVal What As Download) _NextWNM = UserData.WNM.Notify _NextTagged = True - LastDownloadDate.Value = Now + RefreshMyLastRequests(Now) ActiveJobs -= 1 SkipUntilNextSession = False End Sub @@ -382,6 +523,11 @@ Namespace API.Instagram Private __DownloadStoriesUser As Boolean = False Private __DownloadTagged As Boolean = False Friend Overrides Sub BeginEdit() + RefreshMyLastRequests() + Dim v% = MyLastRequestsCount + Dim d$ = String.Empty + If v > 0 Then d = $" ({MyLastRequestsDate.ToStringDate(DateTimeDefaultProvider)})" + LastRequestsCountLabel.Value = $"Number of spent requests: {v.NumToGroupIntegral}{d}" ____HH_CSRF_TOKEN = AConvert(Of String)(HH_CSRF_TOKEN.Value, String.Empty) ____HH_IG_APP_ID = AConvert(Of String)(HH_IG_APP_ID.Value, String.Empty) ____HH_ASBD_ID = AConvert(Of String)(HH_ASBD_ID.Value, String.Empty) @@ -460,6 +606,12 @@ Namespace API.Instagram Return ErrorsDescriber.Execute(EDP.SendToLog, ex, "Can't open user's post", String.Empty) End Try End Function +#End Region +#Region "IDisposable Support" + Protected Overrides Sub Dispose(ByVal disposing As Boolean) + If Not disposedValue And disposing And Not MyLastRequests Is Nothing Then MyLastRequests.Clear() + MyBase.Dispose(disposing) + End Sub #End Region End Class End Namespace \ No newline at end of file diff --git a/SCrawler/API/Instagram/UserData.GQL.vb b/SCrawler/API/Instagram/UserData.GQL.vb new file mode 100644 index 0000000..472ea82 --- /dev/null +++ b/SCrawler/API/Instagram/UserData.GQL.vb @@ -0,0 +1,333 @@ +' Copyright (C) Andy https://github.com/AAndyProgram +' This program is free software: you can redistribute it and/or modify +' it under the terms of the GNU General Public License as published by +' the Free Software Foundation, either version 3 of the License, or +' (at your option) any later version. +' +' This program is distributed in the hope that it will be useful, +' but WITHOUT ANY WARRANTY +Imports System.Threading +Imports SCrawler.API.Base +Imports PersonalUtilities.Functions.XML +Imports PersonalUtilities.Functions.RegularExpressions +Imports PersonalUtilities.Tools.Web.Clients +Imports PersonalUtilities.Tools.Web.Documents.JSON +Namespace API.Instagram + Partial Friend Class UserData +#Region "Tokens" + Protected Property Token_dtsg As String = String.Empty + Protected ReadOnly Property Token_dtsg_Var As String + Get + Return If(Token_dtsg.IsEmptyString, String.Empty, SymbolsConverter.ASCII.EncodeSymbolsOnly(Token_dtsg)) + End Get + End Property + Protected Property Token_lsd As String = String.Empty + Protected Sub ResetBaseTokens() + Token_dtsg = String.Empty + Token_lsd = String.Empty + End Sub +#End Region +#Region "Headers" + Friend Const GQL_HEADER_FB_FRINDLY_NAME As String = "x-fb-friendly-name" + Friend Const GQL_HEADER_FB_LSD As String = "x-fb-lsd" +#End Region +#Region "Data constants" + Private Const GQL_UserData_DocId As String = "7381344031985950" + Private Const GQL_UserData_FbFriendlyName As String = "PolarisProfilePageContentQuery" + + Private Const GQL_Highlights_DocId As String = "8298007123561120" + Private Const GQL_Highlights_DocId_Second As String = "7559771384111300" + Private Const GQL_Highlights_FbFriendlyName As String = "PolarisProfileStoryHighlightsTrayContentQuery" + Private Const GQL_Highlights_FbFriendlyName_Second As String = "PolarisStoriesV3HighlightsPageQuery" + + Private Const GQL_UserStories_DocId As String = "25231722019806941" + Private Const GQL_UserStories_FbFriendlyName As String = "PolarisStoriesV3ReelPageStandaloneQuery" + + Private Const GQL_Timeline_DocId As String = "7268577773270422" + Private Const GQL_Timeline_FbFriendlyName As String = "PolarisProfilePostsQuery" + Private Const GQL_Timeline_DocId_Second As String = "7286316061475375" + Private Const GQL_Timeline_FbFriendlyName_Second As String = "PolarisProfilePostsTabContentQuery_connection" + + Private Const GQL_Reels_DocId As String = "7191572580905225" + Private Const GQL_Reels_FbFriendlyName As String = "PolarisProfileReelsTabContentQuery" + + Private Const GQL_Tagged_DocId As String = "7289408964443685" + Private Const GQL_Tagged_FbFriendlyName As String = "PolarisProfileTaggedTabContentQuery" +#End Region +#Region "Url & var constants" + Private Const GQL_URL_PATTERN_VARS As String = "doc_id={0}&lsd={1}&fb_dtsg={2}&fb_api_req_friendly_name={3}&variables={4}" + Private Const GQL_URL As String = "https://www.instagram.com/api/graphql" + Private Const GQL_URL_Q As String = "https://www.instagram.com/graphql/query" +#End Region +#Region "Download functions" + Protected Sub UpdateHeadersGQL(ByVal HeaderValue As String) + Responser.Headers.Add(GQL_HEADER_FB_FRINDLY_NAME, HeaderValue) + Responser.Headers.Add(GQL_HEADER_FB_LSD, Token_lsd) + End Sub + + Private Sub GetUserDataGQL(ByVal Token As CancellationToken) + Dim vars$ = String.Format(GQL_URL_PATTERN_VARS, GQL_UserData_DocId, Token_lsd, Token_dtsg_Var, GQL_UserData_FbFriendlyName, + SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & $"""id"":""{ID}"",""relay_header"":false,""render_surface"":""PROFILE""" & "}")) + UpdateRequestNumber() + ChangeResponserMode(True) + UpdateHeadersGQL(GQL_UserData_FbFriendlyName) + Dim r$ = Responser.GetResponse(GQL_URL, vars) + If Not r.IsEmptyString Then + Using j As EContainer = JsonDocument.Parse(r) + If j.ListExists Then + With j({"data", "user"}) + If .ListExists Then + UserSiteName = .Value("full_name").IfNullOrEmpty(UserSiteName) + Dim f As New SFile With {.Path = DownloadContentDefault_GetRootDir(), .Name = "ProfilePicture", .Extension = "jpg"} + Dim pic$ = .Value({"hd_profile_pic_url_info"}, "url").IfNullOrEmpty(.Value("profile_pic_url")) + If Not pic.IsEmptyString Then GetWebFile(pic, f, EDP.ReturnValue) + UserDescriptionUpdate(.Value("biography")) + End If + End With + End If + End Using + End If + End Sub + Private Function GetTimelineGQL(ByVal Cursor As String, ByVal Token As CancellationToken) As String + Const none_cursor$ = "none" + Dim nextCursor$ = String.Empty, hasNextPage$ = String.Empty + Dim vars$ + + ThrowAny(Token) + UpdateRequestNumber() + ChangeResponserMode(True) + + If Cursor.IsEmptyString Then + vars = "{""data"":{""count"":50,""include_relationship_info"":true,""latest_besties_reel_media"":true,""latest_reel_media"":true},""username"":""" & + NameTrue & """,""__relay_internal__pv__PolarisShareMenurelayprovider"":false}" + vars = String.Format(GQL_URL_PATTERN_VARS, GQL_Timeline_DocId, Token_lsd, Token_dtsg_Var, GQL_Timeline_FbFriendlyName, + SymbolsConverter.ASCII.EncodeSymbolsOnly(vars)) + UpdateHeadersGQL(GQL_Timeline_FbFriendlyName) + Else + vars = "{""after"":""" & Cursor & """,""before"":null,""data"":{""count"":50,""include_relationship_info"":true,""latest_besties_reel_media"":true,""latest_reel_media"":true},""first"":50,""last"":null,""username"":""" & + NameTrue & """,""__relay_internal__pv__PolarisShareMenurelayprovider"":false}" + vars = String.Format(GQL_URL_PATTERN_VARS, GQL_Timeline_DocId_Second, Token_lsd, Token_dtsg_Var, GQL_Timeline_FbFriendlyName_Second, + SymbolsConverter.ASCII.EncodeSymbolsOnly(vars)) + UpdateHeadersGQL(GQL_Timeline_FbFriendlyName_Second) + End If + + DefaultParser_ElemNode = {"node"} + + Dim r$ = Responser.GetResponse(GQL_URL, vars) + If Not r.IsEmptyString Then + Using j As EContainer = JsonDocument.Parse(r) + If j.ListExists Then + With j({"data", "xdt_api__v1__feed__user_timeline_graphql_connection"}) + If .ListExists Then + With .Item("page_info") + If .ListExists Then + nextCursor = .Value("end_cursor") + hasNextPage = .Value("has_next_page").FromXML(Of Boolean)(False) + End If + End With + With .Item("edges") + If .ListExists Then + If Not DefaultParser(.Self, Sections.Timeline, Token) Then Throw New ExitException + End If + End With + End If + End With + End If + End Using + End If + + Return If(hasNextPage And (Not nextCursor.IsEmptyString AndAlso Not nextCursor.StringToLower = none_cursor), nextCursor, String.Empty) + End Function + Private Function GetHighlightsGQL_List() As List(Of String) + + Dim nextCursor$ = String.Empty, hasNextPage$ = String.Empty + Dim i% = -1 + Dim hList As New List(Of String) + Dim tmpList As New List(Of String) + Dim vars$ = String.Format(GQL_URL_PATTERN_VARS, GQL_Highlights_DocId, Token_lsd, Token_dtsg_Var, GQL_Highlights_FbFriendlyName, + SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & $"""user_id"":""{ID}""" & "}")) + UpdateRequestNumber() + ChangeResponserMode(True) + UpdateHeadersGQL(GQL_Highlights_FbFriendlyName) + Dim r$ = Responser.GetResponse(GQL_URL_Q, vars) + + If Not r.IsEmptyString Then + Using j As EContainer = JsonDocument.Parse(r) + If j.ListExists Then + 'With j({"data"}) + With j({"data", "highlights"}) + If .ListExists Then + With .Item("page_info") + If .ListExists Then + nextCursor = .Value("end_cursor") + hasNextPage = .Value("has_next_page").FromXML(Of Boolean)(False) + End If + End With + With .Item({"edges"}) + If .ListExists Then hList.ListAddList(.Select(Function(jj) jj.Value({"node"}, "id")), LNC) + End With + End If + End With + End If + End Using + End If + Return hList + End Function + Private Sub GetHighlightsGQL(ByRef StoriesList As List(Of String), ByVal Token As CancellationToken) + Const highlightData$ = """first"":50,""initial_reel_id"":""{0}"",""last"":2,""reel_ids"":[{1}]" + Dim tmpList As New List(Of String) + Dim i% = -1 + If StoriesList.ListExists Then + tmpList.AddRange(StoriesList.Take(10)) + StoriesList.RemoveRange(0, tmpList.Count) + + Dim vars$ = String.Format(GQL_URL_PATTERN_VARS, GQL_Highlights_DocId_Second, Token_lsd, Token_dtsg_Var, GQL_Highlights_FbFriendlyName_Second, + SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & String.Format(highlightData, tmpList(0), tmpList.Select(Function(hl) $"""{hl}""").ListToString(",")) & "}")) + ThrowAny(Token) + UpdateRequestNumber() + ChangeResponserMode(True) + UpdateHeadersGQL(GQL_Highlights_FbFriendlyName_Second) + Dim r$ = Responser.GetResponse(GQL_URL_Q, vars) + If Not r.IsEmptyString Then + Using j As EContainer = JsonDocument.Parse(r) + If j.ListExists Then + With j({"data", "xdt_api__v1__feed__reels_media__connection", "edges"}) + If .ListExists Then + ProgressPre.ChangeMax(.Count) + For Each n As EContainer In .Self : GetStoriesData_ParseSingleHighlight(n("node"), i, False, Token) : Next + End If + End With + End If + End Using + End If + tmpList.Clear() + End If + + tmpList.Clear() + End Sub + Private Sub GetUserStoriesGQL(ByVal Token As CancellationToken) + '"{" & $"""user_id"":""{ID}""" & "}" + Dim vars$ = String.Format(GQL_URL_PATTERN_VARS, GQL_UserStories_DocId, Token_lsd, Token_dtsg_Var, GQL_UserStories_FbFriendlyName, + SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & $"""reel_ids_arr"":[""{ID}""]" & "}")) + UpdateRequestNumber() + ChangeResponserMode(True) + UpdateHeadersGQL(GQL_UserStories_FbFriendlyName) + Dim r$ = Responser.GetResponse(GQL_URL, vars) + If Not r.IsEmptyString Then + Using j As EContainer = JsonDocument.Parse(r) + If j.ListExists Then + Dim i% = -1 + GetStoriesData_ParseSingleHighlight(j.ItemF({"data", "xdt_api__v1__feed__reels_media", "reels_media", 0}), i, True, Token) + End If + End Using + End If + End Sub + Private WriteOnly Property GetReelsGQL_SetEnvir As Boolean + Set(ByVal init As Boolean) + If init Then + ObtainMedia_SetReelsFunc() + DefaultParser_PostUrlCreator = Function(post) $"{MySiteSettings.GetUserUrl(Me).TrimEnd("/")}/reel/{post.Code}" + Else + ObtainMedia_SizeFuncPic = Nothing + ObtainMedia_SizeFuncVid = Nothing + DefaultParser_PostUrlCreator = DefaultParser_PostUrlCreator_Default + End If + End Set + End Property + ''' Response + Private Function GetReelsGQL(ByVal Cursor As String) As String + GetReelsGQL_SetEnvir = True + + Dim errData$ = String.Empty + If Cursor.IsEmptyString And Not ValidateBaseTokens() Then GetPageTokens() + If Cursor.IsEmptyString And Not ValidateBaseTokens(errData) Then ValidateBaseTokens_Error(errData) + + Dim vars$ = """data"":{""include_feed_video"":true,""page_size"":50,""target_user_id"":""" & ID & """}" + If Not Cursor.IsEmptyString Then vars = $"""after"":""{Cursor}"",""before"":null,{vars},""first"":4,""last"":null" + vars = String.Format(GQL_URL_PATTERN_VARS, GQL_Reels_DocId, Token_lsd, Token_dtsg_Var, GQL_Reels_FbFriendlyName, + SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & vars & "}")) + UpdateRequestNumber() + ChangeResponserMode(True) + UpdateHeadersGQL(GQL_Reels_FbFriendlyName) + Return Responser.GetResponse(GQL_URL, vars) + End Function + ''' Response + Private Function GetTaggedGQL(ByVal Cursor As String) As String + 'default count = 12 + 'max count = 21 + Dim vars$ + If Cursor.IsEmptyString Then + vars = String.Format(GQL_URL_PATTERN_VARS, GQL_Tagged_DocId, Token_lsd, Token_dtsg_Var, GQL_Tagged_FbFriendlyName, + SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & $"""count"":50,""user_id"":""{ID}""" & "}")) + Else + vars = String.Format(GQL_URL_PATTERN_VARS, GQL_Tagged_DocId, Token_lsd, Token_dtsg_Var, GQL_Tagged_FbFriendlyName, + SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & $"""after"":""{Cursor}"",""before"":null,""count"":50,""first"":50,""last"":null,""user_id"":""{ID}""" & "}")) + End If + UpdateRequestNumber() + ChangeResponserMode(True) + UpdateHeadersGQL(GQL_Tagged_FbFriendlyName) + Return Responser.GetResponse(GQL_URL, vars) + End Function +#End Region +#Region "ValidateBaseTokens" + Protected Overridable Overloads Function ValidateBaseTokens() As Boolean + Return ValidateBaseTokens(Nothing) + End Function + Protected Overridable Overloads Function ValidateBaseTokens(ByRef ErrData As String) As Boolean + ErrData = String.Empty + If Token_dtsg.IsEmptyString Then ErrData.StringAppend("dtsg") + If Token_lsd.IsEmptyString Then ErrData.StringAppend("lsd") + Return ErrData.IsEmptyString + End Function + Protected Overridable Sub ValidateBaseTokens_Error(Optional ByVal ErrData As String = "") + If _UseGQL Then DisableSection(Sections.Timeline) + ExitException.ThrowTokens(Me, ErrData) + End Sub +#End Region +#Region "GetPageTokens" + Private Sub GetPageTokens() + ResetBaseTokens() + Try + UpdateRequestNumber() + ChangeResponserMode(False, Not _UseGQL) + With Responser + With .Headers + .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchDest, "document")) + .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchMode, "navigate")) + End With + End With + Dim r$ = Responser.GetResponse(MySiteSettings.GetUserUrl(Me)) + If Not r.IsEmptyString Then + Dim rr As RParams = RParams.DM(PageTokenRegexPatternDefault, 0, RegexReturn.List, EDP.ReturnValue) + Dim tokens As List(Of String) = RegexReplace(r, rr) + Dim tt$, ttVal$ + If tokens.ListExists Then + With rr + .Match = Nothing + .MatchSub = 1 + .WhatGet = RegexReturn.Value + End With + For Each tt In tokens + If Not Token_lsd.IsEmptyString And Not Token_dtsg.IsEmptyString Then + Exit For + Else + ttVal = RegexReplace(tt, rr) + If Not ttVal.IsEmptyString Then + If ttVal.Contains(":") Then + If Token_dtsg.IsEmptyString Then Token_dtsg = ttVal + Else + If Token_lsd.IsEmptyString Then Token_lsd = ttVal + End If + End If + End If + Next + End If + End If + Catch ex As Exception + Finally + ChangeResponserMode(_UseGQL, Not _UseGQL) + End Try + End Sub +#End Region + End Class +End Namespace \ No newline at end of file diff --git a/SCrawler/API/Instagram/UserData.vb b/SCrawler/API/Instagram/UserData.vb index 617ef69..4429044 100644 --- a/SCrawler/API/Instagram/UserData.vb +++ b/SCrawler/API/Instagram/UserData.vb @@ -69,7 +69,6 @@ Namespace API.Instagram Return New EContainer("Post", ID, {New EAttribute(Name_Section, CInt(Section)), New EAttribute(Name_Code, Code)}) End Function End Structure - Friend Const Header_FB_LSD As String = "x-fb-lsd" Private ReadOnly Property MySiteSettings As SiteSettings Get Return DirectCast(HOST.Source, SiteSettings) @@ -139,14 +138,19 @@ Namespace API.Instagram Friend Sub New() PostsKVIDs = New List(Of PostKV) PostsToReparse = New List(Of PostKV) - _ResponserAutoUpdateCookies = True End Sub #End Region #Region "Download data" + Private WwwClaimUpdate As Boolean = True + Private WwwClaimUpdate_R As Boolean = True + Private WwwClaimDefaultAlgo As Boolean = True + Private WwwClaimUse As Boolean = True Private E560Thrown As Boolean = False Friend Err5xx As Integer = -1 Private Class ExitException : Inherits Exception Friend Property Is560 As Boolean = False + Friend Property IsTokens As Boolean = False + Friend Property TokensData As String = String.Empty Friend Shared Sub Throw560(ByRef Source As UserData) If Not Source.E560Thrown Then MyMainLOG = $"{Source.ToStringForLog}: ({IIf(Source.Err5xx > 0, Source.Err5xx, 560)}) Download skipped until next session" @@ -154,6 +158,10 @@ Namespace API.Instagram End If Throw New ExitException With {.Is560 = True} End Sub + Friend Shared Sub ThrowTokens(ByRef Source As UserData, ByVal Data As String) + MyMainLOG = $"{Source.ToStringForLog}: failed to update some{IIf(Data.IsEmptyString, String.Empty, $" ({Data})")} credentials" + Throw New ExitException With {.IsTokens = True, .TokensData = Data} + End Sub End Class Private ReadOnly Property MyFilePostsKV As SFile Get @@ -235,8 +243,75 @@ Namespace API.Instagram Private _DownloadingInProgress As Boolean = False Private _Limit As Integer = -1 Private _TotalPostsParsed As Integer = 0 + Private _LastWwwClaim As String = String.Empty + Private _ResponserGQLMode As Boolean = False + Private _UseGQL As Boolean = False + Private Sub ChangeResponserMode(ByVal GQL As Boolean, Optional ByVal Force As Boolean = False) + If Not _ResponserGQLMode = GQL Or Force Then + _ResponserGQLMode = GQL + ChangeResponserMode_StoreWwwClaim() + Responser.Headers.Clear() + Responser.Headers.AddRange(MySiteSettings.Responser.Headers) + If GQL Then + WwwClaimUpdate = False + With Responser + .Method = "POST" + .ContentType = "application/x-www-form-urlencoded" + .Referer = MySiteSettings.GetUserUrl(Me) + .CookiesExtractMode = Responser.CookiesExtractModes.Any + With .Headers + .Remove(SiteSettings.Header_IG_WWW_CLAIM) + .Add("origin", "https://www.instagram.com") + .Add("authority", "www.instagram.com") + End With + End With + Else + WwwClaimUpdate = WwwClaimUpdate_R + With Responser + .Method = "GET" + .ContentType = Nothing + .Referer = Nothing + .CookiesExtractMode = MySiteSettings.Responser.CookiesExtractMode + With .Headers + .Remove("origin") + .Remove("authority") + .Remove(GQL_HEADER_FB_FRINDLY_NAME) + .Remove(GQL_HEADER_FB_LSD) + Dim hv$ = MySiteSettings.Responser.Headers.Value(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchDest)).IfNullOrEmpty("empty") + .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchDest, hv)) + hv = MySiteSettings.Responser.Headers.Value(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchMode)).IfNullOrEmpty("cors") + .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchMode, hv)) + If Not _UseGQL And WwwClaimUse Then .Add(SiteSettings.Header_IG_WWW_CLAIM, _LastWwwClaim) + End With + End With + End If + End If + End Sub + Private Sub ChangeResponserMode_StoreWwwClaim() + If Not _UseGQL Then + With Responser.Headers + If .Contains(SiteSettings.Header_IG_WWW_CLAIM) AndAlso Not .Value(SiteSettings.Header_IG_WWW_CLAIM).IsEmptyString Then _LastWwwClaim = .Value(SiteSettings.Header_IG_WWW_CLAIM) + End With + End If + End Sub Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken) + ResetBaseTokens() UserNameRequested = False + RequestsCountSession = 0 + _LastWwwClaim = String.Empty + _ResponserGQLMode = False + _UseGQL = MySiteSettings.USE_GQL.Value + WwwClaimUse = MySiteSettings.HH_IG_WWW_CLAIM_USE.Value + WwwClaimDefaultAlgo = MySiteSettings.HH_IG_WWW_CLAIM_USE_DEFAULT_ALGO.Value + With MySiteSettings : WwwClaimUpdate = (Not CBool(.HH_IG_WWW_CLAIM_ALWAYS_ZERO.Value) And CBool(.HH_IG_WWW_CLAIM_USE.Value)) Or + WwwClaimDefaultAlgo : End With + WwwClaimUpdate_R = WwwClaimUpdate + Dim upClaimRequest As Action = Sub() If WwwClaimUpdate And Not WwwClaimDefaultAlgo And CBool(MySiteSettings.HH_IG_WWW_CLAIM_RESET_EACH_TARGET.Value) Then _ + Responser.Headers.Add(SiteSettings.Header_IG_WWW_CLAIM, 0) + + DefaultParser_ElemNode = Nothing + ChangeResponserMode(_UseGQL) + Dim s As Sections = Sections.Timeline Dim errorFound As Boolean = False Try @@ -251,6 +326,7 @@ Namespace API.Instagram Dim dt As Func(Of Boolean) = Function() (CBool(MySiteSettings.DownloadTimeline.Value) And GetTimeline) Or IsSavedPosts If dt.Invoke And Not LastCursor.IsEmptyString Then s = IIf(IsSavedPosts, Sections.SavedPosts, Sections.Timeline) + upClaimRequest.Invoke DownloadData(LastCursor, s, Token) ProgressPre.Done() ThrowAny(Token) @@ -258,27 +334,51 @@ Namespace API.Instagram End If If dt.Invoke And Not HasError Then s = IIf(IsSavedPosts, Sections.SavedPosts, Sections.Timeline) + upClaimRequest.Invoke + ChangeResponserMode(_UseGQL) DownloadData(String.Empty, s, Token) ProgressPre.Done() ThrowAny(Token) If Not HasError Then FirstLoadingDone = True End If + DefaultParser_ElemNode = Nothing If FirstLoadingDone Then LastCursor = String.Empty If Not IsSavedPosts AndAlso MySiteSettings.BaseAuthExists() Then + DefaultParser_ElemNode = Nothing + ChangeResponserMode(_UseGQL) If CBool(MySiteSettings.DownloadReels.Value) And GetReels Then s = Sections.Reels DefaultParser_ElemNode = {"node", "media"} + upClaimRequest.Invoke + ChangeResponserMode(True) DownloadData(String.Empty, s, Token) - DefaultParser_ElemNode = Nothing - DownloadReels_SetEnvir = False + GetReelsGQL_SetEnvir = False ProgressPre.Done() End If - If CBool(MySiteSettings.DownloadStories.Value) And GetStories Then s = Sections.Stories : DownloadData(String.Empty, s, Token) : ProgressPre.Done() - If CBool(MySiteSettings.DownloadStoriesUser.Value) And GetStoriesUser Then s = Sections.UserStories : DownloadData(String.Empty, s, Token) : ProgressPre.Done() + DefaultParser_ElemNode = Nothing + ChangeResponserMode(_UseGQL) + If CBool(MySiteSettings.DownloadStories.Value) And GetStories Then + s = Sections.Stories + upClaimRequest.Invoke + DownloadData(String.Empty, s, Token) + ProgressPre.Done() + End If + DefaultParser_ElemNode = Nothing + ChangeResponserMode(_UseGQL) + If CBool(MySiteSettings.DownloadStoriesUser.Value) And GetStoriesUser Then + s = Sections.UserStories + upClaimRequest.Invoke + DownloadData(String.Empty, s, Token) + ProgressPre.Done() + End If + DefaultParser_ElemNode = Nothing + ChangeResponserMode(_UseGQL) If CBool(MySiteSettings.DownloadTagged.Value) And GetTaggedData Then s = Sections.Tagged + upClaimRequest.Invoke DownloadData(String.Empty, s, Token) ProgressPre.Done() + DefaultParser_ElemNode = Nothing If PostsToReparse.Count > 0 Then DownloadPosts(Token, True) End If End If @@ -289,7 +389,7 @@ Namespace API.Instagram Throw ex Finally DefaultParser_ElemNode = Nothing - DownloadReels_SetEnvir = False + GetReelsGQL_SetEnvir = False E560Thrown = False UpdateResponser() ValidateExtension() @@ -315,13 +415,13 @@ Namespace API.Instagram If _DownloadingInProgress AndAlso Not Responser Is Nothing AndAlso Not Responser.Disposed Then _DownloadingInProgress = False Responser_ResponseReceived_RemoveHandler() - Declarations.UpdateResponser(Responser, MySiteSettings.Responser) + Declarations.UpdateResponser(Responser, MySiteSettings.Responser, WwwClaimUpdate) End If Catch End Try End Sub Protected Overrides Sub Responser_ResponseReceived(ByVal Sender As Object, ByVal e As EventArguments.WebDataResponse) - Declarations.UpdateResponser(e, Responser) + Declarations.UpdateResponser(e, Responser, WwwClaimUpdate) End Sub Protected Enum Sections : Timeline : Reels : Tagged : Stories : UserStories : SavedPosts : End Enum Protected Const StoriesFolder As String = "Stories" @@ -329,6 +429,12 @@ Namespace API.Instagram #Region "429 bypass" Private Const MaxPostsCount As Integer = 200 Friend Property RequestsCount As Integer = 0 + Friend Property RequestsCountSession As Integer = 0 + Private Sub UpdateRequestNumber() + If CInt(MySiteSettings.RequestsWaitTimer_Any.Value) > 0 Then Thread.Sleep(CInt(MySiteSettings.RequestsWaitTimer_Any.Value)) + RequestsCount += 1 + RequestsCountSession += 1 + End Sub Friend Enum WNM As Integer Notify = 0 SkipCurrent = 1 @@ -468,46 +574,74 @@ Namespace API.Instagram Dim HasNextPage As Boolean = False Dim EndCursor$ = String.Empty Dim PostID$ = String.Empty, PostDate$ = String.Empty, SpecFolder$ = String.Empty + Dim TokensErrData$ = String.Empty Dim PostIDKV As PostKV Dim ENode() As Object = Nothing + Dim processGetResponse As Boolean = True NextRequest(True) 'Check environment If Not IsSavedPosts Then - If ID.IsEmptyString Then GetUserId() + If ID.IsEmptyString Then GetUserData() If ID.IsEmptyString Then Throw New Plugin.ExitException("can't get user ID") + If _UseGQL And Cursor.IsEmptyString And Not Section = Sections.SavedPosts Then + If Not ValidateBaseTokens() Then GetPageTokens() + If Not ValidateBaseTokens(TokensErrData) Then ValidateBaseTokens_Error(TokensErrData) + End If End If 'Create query Select Case Section Case Sections.Timeline - URL = $"https://www.instagram.com/api/v1/feed/user/{NameTrue}/username/?count=50" & - If(Cursor.IsEmptyString, String.Empty, $"&max_id={Cursor}") - ENode = Nothing + If _UseGQL Then + EndCursor = GetTimelineGQL(Cursor, Token) + HasNextPage = Not EndCursor.IsEmptyString + MySiteSettings.TooManyRequests(False) + GoTo NextPageBlock + Else + URL = $"https://www.instagram.com/api/v1/feed/user/{NameTrue}/username/?count=50" & + If(Cursor.IsEmptyString, String.Empty, $"&max_id={Cursor}") + ENode = Nothing + End If Case Sections.Reels - r = DownloadReels(Cursor, Token) + ChangeResponserMode(True) + r = GetReelsGQL(Cursor) ENode = {"data", "xdt_api__v1__clips__user__connection_v2"} + processGetResponse = False Case Sections.SavedPosts - SavedPostsDownload(String.Empty, Token) - Exit Sub + ChangeResponserMode(False) + EndCursor = SavedPostsDownload(String.Empty, Token) + HasNextPage = Not EndCursor.IsEmptyString + MySiteSettings.TooManyRequests(False) + ThrowAny(Token) + GoTo NextPageBlock Case Sections.Tagged - Dim vars$ = "{""id"":" & ID & ",""first"":50,""after"":""" & Cursor & """}" - vars = SymbolsConverter.ASCII.EncodeSymbolsOnly(vars) - URL = $"https://www.instagram.com/graphql/query/?doc_id=17946422347485809&variables={vars}" - ENode = {"data", "user", "edge_user_to_photos_of_you"} SpecFolder = TaggedFolder + If _UseGQL Then + r = GetTaggedGQL(Cursor) + ENode = {"data", "xdt_api__v1__usertags__user_id__feed_connection"} + processGetResponse = False + Else + Dim vars$ = "{""id"":" & ID & ",""first"":50,""after"":""" & Cursor & """}" + vars = SymbolsConverter.ASCII.EncodeSymbolsOnly(vars) + URL = $"https://www.instagram.com/graphql/query/?doc_id=17946422347485809&variables={vars}" + ENode = {"data", "user", "edge_user_to_photos_of_you"} + End If Case Sections.Stories If Not StoriesRequested Then - StoriesList = GetStoriesList() + StoriesList = If(_UseGQL, GetHighlightsGQL_List(), GetStoriesList()) StoriesRequested = True MySiteSettings.TooManyRequests(False) - RequestsCount += 1 ThrowAny(Token) + Continue Do End If If StoriesList.ListExists Then - GetStoriesData(StoriesList, False, Token) + If _UseGQL Then + GetHighlightsGQL(StoriesList, Token) + Else + GetStoriesData(StoriesList, False, Token) + End If MySiteSettings.TooManyRequests(False) - RequestsCount += 1 End If If StoriesList.ListExists Then Continue Do @@ -515,16 +649,17 @@ Namespace API.Instagram Throw New ExitException End If Case Sections.UserStories - GetStoriesData(Nothing, True, Token) + If _UseGQL Then GetUserStoriesGQL(Token) Else GetStoriesData(Nothing, True, Token) MySiteSettings.TooManyRequests(False) - RequestsCount += 1 Throw New ExitException End Select 'Get response - If Not Section = Sections.Reels Then r = Responser.GetResponse(URL,, EDP.ThrowException) + If processGetResponse Then + UpdateRequestNumber() + r = Responser.GetResponse(URL) + End If MySiteSettings.TooManyRequests(False) - RequestsCount += 1 ThrowAny(Token) 'Parsing @@ -608,6 +743,7 @@ Namespace API.Instagram Else Throw New ExitException End If +NextPageBlock: dValue = 0 If HasNextPage And Not EndCursor.IsEmptyString Then DownloadData(EndCursor, Section, Token) Catch jsonNull As JsonDocumentException When jsonNull.State = WebDocumentEventArgs.States.Error And @@ -625,6 +761,8 @@ Namespace API.Instagram Catch eex2 As ExitException If eex2.Is560 Then Throw New Plugin.ExitException With {.Silent = True} + ElseIf eex2.IsTokens And _UseGQL Then + Throw New Plugin.ExitException With {.Silent = True} Else If Not Section = Sections.Reels And (Section = Sections.Timeline Or Section = Sections.Tagged) And Not Cursor.IsEmptyString Then Throw eex2 End If @@ -641,6 +779,7 @@ Namespace API.Instagram Dim before% Dim specFolder$ = IIf(IsTagged, "Tagged", String.Empty) If PostsToReparse.Count > 0 Then ProgressPre.ChangeMax(PostsToReparse.Count) + ChangeResponserMode(False) Try Do While dValue = 1 ThrowAny(Token) @@ -660,9 +799,9 @@ Namespace API.Instagram ThrowAny(Token) NextRequest(((i + 1) Mod 5) = 0) ThrowAny(Token) + UpdateRequestNumber() r = Responser.GetResponse(URL,, e) MySiteSettings.TooManyRequests(False) - RequestsCount += 1 If Not r.IsEmptyString Then j = JsonDocument.Parse(r) If Not j Is Nothing Then @@ -695,27 +834,30 @@ Namespace API.Instagram ProcessException(DoEx, Token, $"downloading posts error [{URL}]",, Sections.Tagged) End Try End Sub - Private Sub SavedPostsDownload(ByVal Cursor As String, ByVal Token As CancellationToken) + ''' Cursor + Private Function SavedPostsDownload(ByVal Cursor As String, ByVal Token As CancellationToken) As String Dim URL$ = $"https://www.instagram.com/api/v1/feed/saved/posts/?max_id={Cursor}" Dim HasNextPage As Boolean = False Dim NextCursor$ = String.Empty - ThrowAny(Token) + Dim processNext As Boolean = False + UpdateRequestNumber() Dim r$ = Responser.GetResponse(URL) Dim nodes As IEnumerable(Of EContainer) = Nothing If Not r.IsEmptyString Then Using e As EContainer = JsonDocument.Parse(r) - If If(e?.Count, 0) > 0 Then + If e.ListExists Then With e HasNextPage = .Value("more_available").FromXML(Of Boolean)(False) NextCursor = .Value("next_max_id") If .Contains("items") Then nodes = (From ee As EContainer In .Item("items") Where ee.Count > 0 Select ee(0)) End With If nodes.ListExists AndAlso DefaultParser(nodes, Sections.SavedPosts, Token) AndAlso - HasNextPage AndAlso Not NextCursor.IsEmptyString Then SavedPostsDownload(NextCursor, Token) + HasNextPage AndAlso Not NextCursor.IsEmptyString Then processNext = True End If End Using End If - End Sub + Return If(processNext, NextCursor, String.Empty) + End Function Protected DefaultParser_ElemNode() As Object = Nothing Protected DefaultParser_IgnorePass As Boolean = False Private ReadOnly DefaultParser_PostUrlCreator_Default As Func(Of PostKV, String) = Function(post) $"https://www.instagram.com/p/{post.Code}/" @@ -740,25 +882,29 @@ Namespace API.Instagram For Each nn In Items ProgressPre.Perform() With If(Not DefaultParser_ElemNode Is Nothing, nn.ItemF(DefaultParser_ElemNode), nn) - PostIDKV = New PostKV(.Value("code"), .Value("id"), Section) - PostOriginUrl = DefaultParser_PostUrlCreator(PostIDKV) - Pinned = .Contains("timeline_pinned_user_ids") - If Not DefaultParser_IgnorePass AndAlso PostKvExists(PostIDKV) Then - If Not Pinned Then Return False - Else - _TempPostsList.Add(PostIDKV.ID) - PostsKVIDs.ListAddValue(PostIDKV, LNC) - PostDate = .Value("taken_at") - If Not DefaultParser_IgnorePass And Not IsSavedPosts Then - Select Case CheckDatesLimit(PostDate, UnixDate32Provider) - Case DateResult.Skip : Continue For - Case DateResult.Exit : If Not Pinned Then Return False - End Select + If .ListExists Then + PostIDKV = New PostKV(.Value("code"), .Value("id"), Section) + PostOriginUrl = DefaultParser_PostUrlCreator(PostIDKV) + Pinned = .Contains("timeline_pinned_user_ids") + If (Section = Sections.Timeline And Not DefaultParser_IgnorePass) AndAlso PostKvExists(PostIDKV) Then + If Not Pinned Then Return False + Else + _TempPostsList.Add(PostIDKV.ID) + PostsKVIDs.ListAddValue(PostIDKV, LNC) + PostDate = .Value("taken_at") + If Not DefaultParser_IgnorePass And Not IsSavedPosts Then + Select Case CheckDatesLimit(PostDate, UnixDate32Provider) + Case DateResult.Skip : Continue For + Case DateResult.Exit : If Not Pinned Then Return False + End Select + End If + before = _TempMediaList.Count + ObtainMedia(.Self, PostIDKV.ID, SpecFolder, PostDate,, PostOriginUrl, State, Attempts) + If Not before = _TempMediaList.Count Then _TotalPostsParsed += 1 + If _Limit > 0 And _TotalPostsParsed >= _Limit Then Return False End If - before = _TempMediaList.Count - ObtainMedia(.Self, PostIDKV.ID, SpecFolder, PostDate,, PostOriginUrl, State, Attempts) - If Not before = _TempMediaList.Count Then _TotalPostsParsed += 1 - If _Limit > 0 And _TotalPostsParsed >= _Limit Then Return False + Else + Return False End If End With Next @@ -768,106 +914,6 @@ Namespace API.Instagram End If End Function #End Region -#Region "Get reels" - Private _GetReels_LSD As String = String.Empty - Private _GetReels_dtsg As String = String.Empty - Private ReadOnly Property DownloadReels_Tokens_Valid As Boolean - Get - Return Not _GetReels_LSD.IsEmptyString And Not _GetReels_dtsg.IsEmptyString - End Get - End Property - Private WriteOnly Property DownloadReels_SetEnvir As Boolean - Set(ByVal init As Boolean) - If init Then - ObtainMedia_SetReelsFunc() - DefaultParser_PostUrlCreator = Function(post) $"{MySiteSettings.GetUserUrl(Me).TrimEnd("/")}/reel/{post.Code}" - Else - ObtainMedia_SizeFuncPic = Nothing - ObtainMedia_SizeFuncVid = Nothing - DefaultParser_PostUrlCreator = DefaultParser_PostUrlCreator_Default - End If - End Set - End Property - Private Class Responser2 : Inherits Responser - Friend Sub New(ByVal Source As Responser) - MyBase.New - Copy(Source) - ErrorProcessor = New ResponserErrorProcessor(Source) - End Sub - End Class - ''' Response - Private Function DownloadReels(ByVal Cursor As String, ByVal Token As CancellationToken) As String - Const requestPattern$ = "https://www.instagram.com/api/graphql?fb_dtsg={0}&fb_api_req_friendly_name=PolarisProfileReelsTabContentQuery&lsd={1}&doc_id=7191572580905225&variables={2}" - - DownloadReels_SetEnvir = True - - If Cursor.IsEmptyString And Not DownloadReels_Tokens_Valid Then GetPageTokens() - If Cursor.IsEmptyString And Not DownloadReels_Tokens_Valid Then Throw New ExitException - - Using resp As New Responser2(Responser) - Try - resp.Method = "POST" - AddHandler resp.ResponseReceived, AddressOf Responser_ResponseReceived - resp.Headers.Add(Header_FB_LSD, _GetReels_LSD) - - Dim vars$ = """data"":{""include_feed_video"":true,""page_size"":50,""target_user_id"":""" & ID & """}" - If Not Cursor.IsEmptyString Then vars = $"""after"":""{Cursor}"",""before"":null,{vars},""first"":4,""last"":null" - vars = "{" & vars & "}" - - Dim url$ = String.Format(requestPattern, _GetReels_dtsg, _GetReels_LSD, SymbolsConverter.ASCII.EncodeSymbolsOnly(vars)) - - Return resp.GetResponse(url,, EDP.ThrowException) - Finally - With resp - Responser.Cookies.Update(.Cookies) - With .Headers - If .Contains(SiteSettings.Header_IG_WWW_CLAIM) Then Responser.Headers.Add(SiteSettings.Header_IG_WWW_CLAIM, .Value(SiteSettings.Header_IG_WWW_CLAIM)) - If .Contains(SiteSettings.Header_CSRF_TOKEN) Then Responser.Headers.Add(SiteSettings.Header_CSRF_TOKEN, .Value(SiteSettings.Header_CSRF_TOKEN)) - End With - End With - End Try - End Using - End Function - Private Function GetPageTokens() As Boolean - _GetReels_LSD = String.Empty - _GetReels_dtsg = String.Empty - Try - Dim r$ = Responser.GetResponse(MySiteSettings.GetUserUrl(Me),, EDP.ThrowException) - If Not r.IsEmptyString Then - Dim rr As RParams = RParams.DM(PageTokenRegexPatternDefault, 0, RegexReturn.List, EDP.ReturnValue) - Dim tokens As List(Of String) = RegexReplace(r, rr) - Dim tt$, ttVal$ - If tokens.ListExists Then - With rr - .Match = Nothing - .MatchSub = 1 - .WhatGet = RegexReturn.Value - End With - For Each tt In tokens - If Not _GetReels_LSD.IsEmptyString And Not _GetReels_dtsg.IsEmptyString Then - Exit For - Else - ttVal = RegexReplace(tt, rr) - If Not ttVal.IsEmptyString Then - If ttVal.Contains(":") Then - If _GetReels_dtsg.IsEmptyString Then _GetReels_dtsg = ttVal - Else - If _GetReels_LSD.IsEmptyString Then _GetReels_LSD = ttVal - End If - End If - End If - Next - End If - End If - Catch ex As Exception - Dim notFound$ = String.Empty - If _GetReels_dtsg.IsEmptyString Then notFound.StringAppend(Header_FB_LSD) - If _GetReels_LSD.IsEmptyString Then notFound.StringAppend("lsd") - LogError(ex, $"failed to update some{IIf(notFound.IsEmptyString, String.Empty, $" ({notFound})")} credentials", EDP.SendToLog) - End Try - Return DownloadReels_Tokens_Valid - End Function -#End Region #Region "Code ID converters" Protected Function CodeToID(ByVal Code As String) As String Const CodeSymbols$ = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" @@ -1024,11 +1070,12 @@ Namespace API.Instagram End Sub #End Region #Region "GetUserId, GetUserName" - Private Sub GetUserId() + Private Sub GetUserData() Dim __idFound As Boolean = False Try - RequestsCount += 1 - Dim r$ = Responser.GetResponse($"https://i.instagram.com/api/v1/users/web_profile_info/?username={NameTrue}",, EDP.ThrowException) + ChangeResponserMode(False) + UpdateRequestNumber() + Dim r$ = Responser.GetResponse($"https://i.instagram.com/api/v1/users/web_profile_info/?username={NameTrue}") If Not r.IsEmptyString Then Using j As EContainer = JsonDocument.Parse(r) If Not j Is Nothing AndAlso j.Contains({"data", "user"}) Then @@ -1042,7 +1089,7 @@ Namespace API.Instagram Dim eUrl$ = .Value("external_url") If Not eUrl.IsEmptyString AndAlso (descr.IsEmptyString OrElse Not descr.Contains(eUrl)) Then descr.StringAppendLine(eUrl) UserDescriptionUpdate(descr) - Dim f As New SFile With {.Path = MyFile.CutPath.Path, .Name = "ProfilePicture", .Extension = "jpg"} + Dim f As New SFile With {.Path = DownloadContentDefault_GetRootDir(), .Name = "ProfilePicture", .Extension = "jpg"} If Not f.Exists Then Dim profilePicture$ = .Value("profile_pic_url_hd") If profilePicture.IsEmptyString OrElse Not GetWebFile(profilePicture, f, EDP.ReturnValue) Then @@ -1062,13 +1109,15 @@ Namespace API.Instagram LogError(ex, "get Instagram user ID") End If End If + Finally + ChangeResponserMode(_UseGQL) End Try End Sub Private Function GetUserNameById() As Boolean UserNameRequested = True Try If Not ID.IsEmptyString Then - RequestsCount += 1 + UpdateRequestNumber() Dim r$ = Responser.GetResponse($"https://i.instagram.com/api/v1/users/{ID}/info/",, EDP.ReturnValue) If Not r.IsEmptyString Then Using j As EContainer = JsonDocument.Parse(r, EDP.ReturnValue) @@ -1101,9 +1150,9 @@ Namespace API.Instagram Private Sub GetStoriesData(ByRef StoriesList As List(Of String), ByVal GetUserStory As Boolean, ByVal Token As CancellationToken) Const ReqUrl$ = "https://i.instagram.com/api/v1/feed/reels_media/?{0}" Dim tmpList As IEnumerable(Of String) = Nothing - Dim qStr$, r$, sFolder$, storyID$, pid$ + Dim qStr$, r$ Dim i% = -1 - Dim jj As EContainer, s As EContainer + Dim jj As EContainer ThrowAny(Token) If StoriesList.ListExists Or GetUserStory Then If Not GetUserStory Then tmpList = StoriesList.Take(5) @@ -1113,38 +1162,14 @@ Namespace API.Instagram Else qStr = String.Format(ReqUrl, tmpList.Select(Function(q) $"reel_ids=highlight:{q}").ListToString("&")) End If + UpdateRequestNumber() r = Responser.GetResponse(qStr,, EDP.ThrowException) ThrowAny(Token) If Not r.IsEmptyString Then Using j As EContainer = JsonDocument.Parse(r).XmlIfNothing If j.Contains("reels") Then ProgressPre.ChangeMax(j("reels").Count) - For Each jj In j("reels") - ProgressPre.Perform() - i += 1 - sFolder = jj.Value("title").StringRemoveWinForbiddenSymbols - storyID = jj.Value("id").Replace("highlight:", String.Empty) - If GetUserStory Then - sFolder = $"{StoriesFolder} (user)" - Else - If sFolder.IsEmptyString Then sFolder = $"Story_{storyID}" - If sFolder.IsEmptyString Then sFolder = $"Story_{i}" - sFolder = $"{StoriesFolder}\{sFolder}" - End If - If Not storyID.IsEmptyString Then storyID &= ":" - With jj("items").XmlIfNothing - If .Count > 0 Then - For Each s In .Self - pid = storyID & s.Value("id") - If Not _TempPostsList.Contains(pid) Then - ThrowAny(Token) - ObtainMedia(s, pid, sFolder) - _TempPostsList.Add(pid) - End If - Next - End If - End With - Next + For Each jj In j("reels") : GetStoriesData_ParseSingleHighlight(jj, i, GetUserStory, Token) : Next End If End Using End If @@ -1152,8 +1177,39 @@ Namespace API.Instagram End If End If End Sub + Private Sub GetStoriesData_ParseSingleHighlight(ByVal Node As EContainer, ByRef Index As Integer, ByVal GetUserStory As Boolean, ByVal Token As CancellationToken) + If Not Node Is Nothing Then + With Node + ProgressPre.Perform() + Index += 1 + Dim pid$ + Dim sFolder$ = .Value("title").StringRemoveWinForbiddenSymbols + Dim storyID$ = .Value("id").Replace("highlight:", String.Empty) + If GetUserStory Then + sFolder = $"{StoriesFolder} (user)" + Else + If sFolder.IsEmptyString Then sFolder = $"Story_{storyID.IfNullOrEmpty(Index)}" + sFolder = $"{StoriesFolder}\{sFolder}" + End If + If Not storyID.IsEmptyString Then storyID &= ":" + With .Item("items") + If .ListExists Then + For Each s As EContainer In .Self + pid = storyID & s.Value("id") + If Not _TempPostsList.Contains(pid) Then + ThrowAny(Token) + ObtainMedia(s, pid, sFolder) + _TempPostsList.Add(pid) + End If + Next + End If + End With + End With + End If + End Sub Private Function GetStoriesList() As List(Of String) Try + UpdateRequestNumber() Dim r$ = Responser.GetResponse($"https://i.instagram.com/api/v1/highlights/{ID}/highlights_tray/",, EDP.ThrowException) If Not r.IsEmptyString Then Dim ee As New ErrorsDescriber(EDP.ReturnValue) With {.DeclaredMessage = New MMessage($"{ToStringForLog()}:")} @@ -1223,15 +1279,26 @@ Namespace API.Instagram Private Sub DisableSection(ByVal Section As Object) If Not IsNothing(Section) AndAlso TypeOf Section Is Sections Then Dim s As Sections = DirectCast(Section, Sections) - Select Case s - Case Sections.Reels : MySiteSettings.DownloadReels.Value = False - Case Sections.Tagged : MySiteSettings.DownloadTagged.Value = False - Case Sections.Timeline, Sections.Stories, Sections.UserStories, Sections.SavedPosts - MySiteSettings.DownloadTimeline.Value = False - MySiteSettings.DownloadStories.Value = False - MySiteSettings.DownloadStoriesUser.Value = False - End Select - MyMainLOG = $"[{s}] downloading is disabled until you update your credentials".ToUpper + Dim ss As New List(Of Sections)([Enum].GetValues(GetType(Sections)).ToObjectsList(Of Sections)) + If s = Sections.Reels And Not _UseGQL Then + ss.Clear() + ss.Add(s) + ElseIf s = Sections.Tagged Then + ss.Clear() + ss.Add(s) + End If + If ss.Count > 0 Then + For Each s In ss + Select Case s + Case Sections.Reels : MySiteSettings.DownloadReels.Value = False + Case Sections.Tagged : MySiteSettings.DownloadTagged.Value = False + Case Sections.Timeline, Sections.SavedPosts : MySiteSettings.DownloadTimeline.Value = False + Case Sections.Stories : MySiteSettings.DownloadStories.Value = False + Case Sections.UserStories : MySiteSettings.DownloadStoriesUser.Value = False + End Select + Next + MyMainLOG = $"[{ss.ListToStringE(, New ANumbers.EnumToStringProvider(GetType(Sections)))}] downloading is disabled until you update your credentials".ToUpper + End If End If End Sub #End Region diff --git a/SCrawler/API/JustForFans/M3U8.vb b/SCrawler/API/JustForFans/M3U8.vb index b786f4f..cc4d1e4 100644 --- a/SCrawler/API/JustForFans/M3U8.vb +++ b/SCrawler/API/JustForFans/M3U8.vb @@ -6,7 +6,7 @@ ' ' This program is distributed in the hope that it will be useful, ' but WITHOUT ANY WARRANTY -Imports System.Threading +Imports System.Net Imports SCrawler.API.Base Imports PersonalUtilities.Tools Imports PersonalUtilities.Tools.Web @@ -18,8 +18,10 @@ Namespace API.JustForFans Friend NotInheritable Class M3U8 : Implements IDisposable #Region "Declarations" Friend Const AllVid As UTypes = UTypes.m3u8 + UTypes.VideoPre - Private ReadOnly DataVideo As List(Of String) - Private ReadOnly DataAudio As List(Of String) + Private Structure M3U8URL_Indexed + Friend Index As Integer + Friend File As SFile + End Structure Private Media As UserMedia Private DestinationFile As SFile Private ReadOnly Thrower As Plugin.IThrower @@ -32,31 +34,37 @@ Namespace API.JustForFans Private UrlAudio As String Private FileVideo As SFile Private FileAudio As SFile + Private FileVideo_M3U8 As SFile + Private FileAudio_M3U8 As SFile + Private ReadOnly FileVideo_IndexedParts As List(Of M3U8URL_Indexed) + Private ReadOnly FileAudio_IndexedParts As List(Of M3U8URL_Indexed) Private RootPlaylistUrl As String Private ReadOnly Cache As CacheKeeper Private ReadOnly Progress As MyProgress Private ReadOnly ProgressPre As PreProgress Private ReadOnly ProgressExists As Boolean Private ReadOnly UsePreProgress As Boolean - Private Property Token As CancellationToken + Private ReadOnly REGEX_FILE_EXT As RParams = RParams.DMS("[^\s""]+\.(\w+)([\?&]{1}.+|)", 1, EDP.ReturnValue) + Private ReadOnly REGEX_FILE_EXT_M4S As RParams = RParams.DM("[^\s""]+\.m4s([\?&]{1}.+|)", 0, EDP.ReturnValue) + Private ReadOnly MyFileNumberProvider As ANumbers #End Region #Region "Initializer" Private Sub New(ByVal m As UserMedia, ByVal Destination As SFile, ByVal Resp As Responser, ByVal _Thrower As Plugin.IThrower, - ByVal _Progress As MyProgress, ByVal _UsePreProgress As Boolean, ByVal _Token As CancellationToken) + ByVal _Progress As MyProgress, ByVal _UsePreProgress As Boolean) Media = m - DataVideo = New List(Of String) - DataAudio = New List(Of String) DestinationFile = Destination Thrower = _Thrower 'Responser = Resp Responser = New Responser ResponserInternal = True + FileVideo_IndexedParts = New List(Of M3U8URL_Indexed) + FileAudio_IndexedParts = New List(Of M3U8URL_Indexed) Progress = _Progress ProgressExists = Not Progress Is Nothing If ProgressExists Then ProgressPre = New PreProgress(Progress) UsePreProgress = _UsePreProgress - Token = _Token - Cache = New CacheKeeper($"{DestinationFile.PathWithSeparator}_{M3U8Base.TempCacheFolderName}\") + MyFileNumberProvider = M3U8Base.NumberProviderDefault + Cache = New CacheKeeper($"{DestinationFile.PathWithSeparator}_{M3U8Base.TempCacheFolderName}\") With {.DisposeSuspended = True} With Cache .CacheDeleteError = CacheDeletionError(Cache) .DisposeSuspended = True @@ -91,30 +99,138 @@ Namespace API.JustForFans UrlVideo = RegexReplace(r, RParams.DMS(R_VIDEO_REGEX_PATTERN, 6, EDP.ReturnValue)) UrlAudio = RegexReplace(r, REGEX_AUDIO_URL) If UrlVideo.IsEmptyString Then Throw New ArgumentException("Unable to identify m3u8 video track", "M3U8 video track") + Thrower.ThrowAny() - GetFiles(UrlVideo, FileVideo, False) + GetFileParts(UrlVideo, FileVideo_M3U8, FileVideo_IndexedParts, False) Thrower.ThrowAny() - If Not UrlAudio.IsEmptyString Then GetFiles(UrlAudio, FileAudio, True) + If Not UrlAudio.IsEmptyString Then GetFileParts(UrlAudio, FileAudio_M3U8, FileAudio_IndexedParts, True) + + If FileVideo_IndexedParts.Count > 0 Then _ + FileVideo = GetTempFile(FileVideo_M3U8, FileVideo_IndexedParts, False, FileAudio_IndexedParts, FileAudio_IndexedParts.Count = 0) + If FileAudio_IndexedParts.Count > 0 Then _ + FileAudio = GetTempFile(FileAudio_M3U8, FileAudio_IndexedParts, True, FileVideo_IndexedParts, False) Thrower.ThrowAny() MergeFiles() End If End If End Sub - Private Sub GetFiles(ByVal URL As String, ByRef File As SFile, ByVal IsAudio As Boolean) + Private Function GetTempFile(ByVal M3U8File As SFile, ByVal IndexedList As List(Of M3U8URL_Indexed), ByVal IsAudio As Boolean, + ByVal IndexedListOther As List(Of M3U8URL_Indexed), ByVal IgnoreAudio As Boolean) As SFile + Const mapStr$ = "#EXT-X-MAP:URI" + Const extinfStr$ = "#EXTINF:" + Const m4s$ = "m4s" + Dim M3U8FileLines$() = M3U8File.GetLines + If M3U8FileLines.ListExists AndAlso IndexedList.Count > 0 AndAlso (IndexedListOther.Count > 0 Or (Not IsAudio And IgnoreAudio)) Then + Dim outputFile As SFile = $"{Cache.RootDirectory.PathWithSeparator}{IIf(IsAudio, "AUDIO.aac", "VIDEO.mp4")}" + Dim M3U8FileNew As SFile = M3U8File + M3U8FileNew.Path = IndexedList(0).File.Path + Dim v$ + Dim i%, fIndx%, fIndx2% + Dim extIsm4s As Boolean + Dim LookingIndex% = -1 + Dim ignoreOtherList As Boolean = IndexedListOther.Count = 0 And (Not IsAudio And IgnoreAudio) + Dim fileFinder As Predicate(Of M3U8URL_Indexed) = Function(input) input.Index = LookingIndex + + Using m3u8Text As New TextSaver + For i = 0 To M3U8FileLines.Length - 1 + v = M3U8FileLines(i) + + If Not v.IsEmptyString Then + If v.StartsWith(mapStr) Then + LookingIndex += 1 + fIndx = IndexedList.FindIndex(fileFinder) + If fIndx >= 0 Then + extIsm4s = Not IndexedList(fIndx).File.Extension.IsEmptyString AndAlso IndexedList(fIndx).File.Extension = m4s + v = v.Replace(RegexReplace(v, If(extIsm4s, REGEX_FILE_EXT_M4S, REGEX_FILE_EXT)), IndexedList(fIndx).File.File) + m3u8Text.AppendLine(v) + Else + Throw New Exception($"The map file is missing ({IIf(IsAudio, "audio", "video")})") + End If + ElseIf v.StartsWith(extinfStr) Then + LookingIndex += 1 + If (i + 1) <= M3U8FileLines.Length - 1 Then + fIndx = IndexedList.FindIndex(fileFinder) + fIndx2 = If(ignoreOtherList, -1, IndexedListOther.FindIndex(fileFinder)) + If fIndx >= 0 And (fIndx2 >= 0 Or ignoreOtherList) Then + If ignoreOtherList OrElse IndexedListOther(fIndx2).Index = IndexedList(fIndx).Index Then + m3u8Text.AppendLine(v) + m3u8Text.AppendLine(IndexedList(fIndx).File.File) + End If + End If + i += 1 + Else + Throw New Exception($"Unexpected end of m3u8 file ({IIf(IsAudio, "audio", "video")})") + End If + Else + m3u8Text.AppendLine(v) + End If + End If + Next + + m3u8Text.SaveAs(M3U8FileNew) + End Using + + If M3U8FileNew.Exists Then + Using b As New BatchExecutor + AddHandler b.ErrorDataReceived, AddressOf Batch_OutputDataReceived + Thrower.ThrowAny() + ProgressChangeMax(IndexedList.Count) + b.ChangeDirectory(M3U8FileNew) + b.Execute($"""{Settings.FfmpegFile}"" -i {M3U8FileNew.File} -vcodec copy -strict -2 ""{outputFile}""") + End Using + If Not outputFile.Exists Then outputFile = Nothing + End If + + Return outputFile + Else + Return Nothing + End If + End Function + Private Sub GetFileParts(ByVal URL As String, ByRef M3U8File As SFile, ByRef IndexedList As List(Of M3U8URL_Indexed), ByVal IsAudio As Boolean) Try Dim r$ = Responser.GetResponse(URL) If Not r.IsEmptyString Then Dim data As List(Of RegexMatchStruct) = RegexFields(Of RegexMatchStruct)(r, {REGEX_PLS_FILES}, {1, 2}, EDP.ReturnValue) If data.ListExists Then - File = $"{Cache.RootDirectory.PathWithSeparator}{IIf(IsAudio, "AUDIO.aac", "VIDEO.mp4")}" - Using b As New TokenBatch(Token) With {.Encoding = Settings.CMDEncoding, .MainProcessName = "ffmpeg"} - AddHandler b.ErrorDataReceived, AddressOf Batch_OutputDataReceived - ProgressChangeMax(data.Count) - b.ChangeDirectory(Cache.RootDirectory) - b.Execute($"""{Settings.FfmpegFile}"" -i {URL} -vcodec copy -strict -2 ""{File}""") - Token.ThrowIfCancellationRequested() - If Not File.Exists Then File = Nothing - End Using + Dim appender$ = URL.Replace(URL.Split("/").LastOrDefault, String.Empty) + Dim createM3U8URL As Func(Of String, M3U8URL) = + Function(input) New M3U8URL(M3U8Base.CreateUrl(appender, input), RegexReplace(input, REGEX_FILE_EXT)) + With (From d As RegexMatchStruct In data + Where Not d.Arr(0).IfNullOrEmpty(d.Arr(1)).IsEmptyString + Select createM3U8URL.Invoke(d.Arr(0).IfNullOrEmpty(d.Arr(1)).StringTrim)) + If .ListExists Then + ProgressChangeMax(.Count) + M3U8File = $"{Cache.RootDirectory.PathWithSeparator}{IIf(IsAudio, "AUDIO", "VIDEO")}.m3u8" + M3U8File = TextSaver.SaveTextToFile(r, M3U8File, True) + + Dim tmpCache As CacheKeeper = Cache.NewInstance + Dim dFile As SFile = tmpCache.RootDirectory + dFile.Extension = .ElementAt(0).Extension.IfNullOrEmpty("m4s") + MyFileNumberProvider.GroupSize = { .Count.ToString.Length, 3}.Max + If tmpCache.Validate Then + Using w As New WebClient + For i% = 0 To .Count - 1 + Thrower.ThrowAny() + dFile.Name = $"{M3U8Base.TempFilePrefix}{i.NumToString(MyFileNumberProvider)}" + dFile.Extension = .ElementAt(i).Extension.IfNullOrEmpty(M3U8Base.TempFileDefaultExtension) + Try + ProgressPerform() + w.DownloadFile(.ElementAt(i).URL, dFile) + tmpCache.AddFile(dFile, True) + IndexedList.Add(New M3U8URL_Indexed With {.File = dFile, .Index = i}) + Catch down_oex As OperationCanceledException + Throw down_oex + Catch down_dex As ObjectDisposedException + Throw down_dex + Catch ex As Exception + End Try + Next + End Using + Else + Throw New Exception("Can't create cache directory") + End If + End If + End With End If End If Catch oex As OperationCanceledException @@ -123,7 +239,7 @@ Namespace API.JustForFans Throw dex Catch ex As Exception ErrorsDescriber.Execute(EDP.SendToLog + EDP.ThrowException, ex, - $"API.JustForFans.M3U8.GetFiles({IIf(IsAudio, "audio", "video")}):{vbCr}URL: {URL}{vbCr}File: {File}") + $"API.JustForFans.M3U8.GetFileParts({IIf(IsAudio, "audio", "video")}):{vbCr}URL: {URL}{vbCr}Post: {Media.URL_BASE}") End Try End Sub Private Async Sub Batch_OutputDataReceived(ByVal Sender As Object, ByVal e As DataReceivedEventArgs) @@ -135,8 +251,10 @@ Namespace API.JustForFans Dim f As SFile = SFile.IndexReindex(DestinationFile,,, p, EDP.ReturnValue).IfNullOrEmpty(DestinationFile) If Not FileVideo.IsEmptyString And Not FileAudio.IsEmptyString Then DestinationFile = FFMPEG.MergeFiles({FileVideo, FileAudio}, Settings.FfmpegFile, f, Settings.CMDEncoding, p, EDP.ThrowException) - Else + ElseIf FileVideo.Exists Then If Not SFile.Move(FileVideo, f) Then DestinationFile = FileVideo + Else + Throw New Exception($"Unable to download file ({Media.URL_BASE})") End If Catch ex As Exception ErrorsDescriber.Execute(EDP.SendToLog + EDP.ThrowException, ex, $"[M3U8.MergeFiles]") @@ -165,8 +283,8 @@ Namespace API.JustForFans #End Region #Region "Static Download" Friend Shared Function Download(ByVal Media As UserMedia, ByVal DestinationFile As SFile, ByVal Resp As Responser, ByVal Thrower As Plugin.IThrower, - ByVal Progress As MyProgress, ByVal UsePreProgress As Boolean, ByVal _Token As CancellationToken) As SFile - Using m As New M3U8(Media, DestinationFile, Resp, Thrower, Progress, UsePreProgress, _Token) + ByVal Progress As MyProgress, ByVal UsePreProgress As Boolean) As SFile + Using m As New M3U8(Media, DestinationFile, Resp, Thrower, Progress, UsePreProgress) m.Download() If m.DestinationFile.Exists Then Return m.DestinationFile Else Return Nothing End Using @@ -177,8 +295,8 @@ Namespace API.JustForFans Private Overloads Sub Dispose(ByVal disposing As Boolean) If Not disposedValue Then If disposing Then - DataVideo.Clear() - DataAudio.Clear() + FileVideo_IndexedParts.Clear() + FileAudio_IndexedParts.Clear() ProgressPre.DisposeIfReady Cache.Dispose() If ResponserInternal Then Responser.DisposeIfReady diff --git a/SCrawler/API/JustForFans/UserData.vb b/SCrawler/API/JustForFans/UserData.vb index 2041caf..9cd0cbf 100644 --- a/SCrawler/API/JustForFans/UserData.vb +++ b/SCrawler/API/JustForFans/UserData.vb @@ -336,7 +336,7 @@ Namespace API.JustForFans DownloadContentDefault(Token) End Sub Protected Overrides Function DownloadM3U8(ByVal URL As String, ByVal Media As UserMedia, ByVal DestinationFile As SFile, ByVal Token As CancellationToken) As SFile - Return M3U8.Download(Media, DestinationFile, ResponserNoHandlers, Me, Progress, Not IsSingleObjectDownload, Token) + Return M3U8.Download(Media, DestinationFile, ResponserNoHandlers, Me, Progress, Not IsSingleObjectDownload) End Function #End Region #Region "DownloadSingleObject" diff --git a/SCrawler/API/ThreadsNet/SiteSettings.vb b/SCrawler/API/ThreadsNet/SiteSettings.vb index ac06012..493de8e 100644 --- a/SCrawler/API/ThreadsNet/SiteSettings.vb +++ b/SCrawler/API/ThreadsNet/SiteSettings.vb @@ -68,6 +68,19 @@ Namespace API.ThreadsNet End If End Sub #End Region +#Region "Other properties" + + Friend ReadOnly Property RequestsWaitTimer_Any As PropertyValue + + Private ReadOnly Property RequestsWaitTimer_AnyProvider As IFormatProvider + + Friend ReadOnly Property DownloadData_Impl As PropertyValue +#End Region #End Region #Region "Initializer" Friend Sub New(ByVal AccName As String, ByVal Temp As Boolean) @@ -112,7 +125,7 @@ Namespace API.ThreadsNet .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchMode, "cors")) .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchSite, "same-origin")) .Add("Sec-Fetch-User", "?1") - .Add(DeclaredNames.Header_FB_FRIENDLY_NAME, "BarcelonaProfileThreadsTabRefetchableQuery") + .Add(Instagram.UserData.GQL_HEADER_FB_FRINDLY_NAME, "BarcelonaProfileThreadsTabRefetchableQuery") End With .CookiesExtractMode = Responser.CookiesExtractModes.Any .CookiesUpdateMode = CookieKeeper.UpdateModes.ReplaceByNameAll @@ -129,6 +142,10 @@ Namespace API.ThreadsNet HH_PLATFORM_VER = New PropertyValue(platform, GetType(String), Sub(v) ChangeResponserFields(NameOf(HH_PLATFORM_VER), v)) HH_USER_AGENT = New PropertyValue(useragent, GetType(String), Sub(v) ChangeResponserFields(NameOf(HH_USER_AGENT), v)) + RequestsWaitTimer_Any = New PropertyValue(1000) + RequestsWaitTimer_AnyProvider = New IG.TimersChecker(0) + DownloadData_Impl = New PropertyValue(True) + UrlPatternUser = "https://www.threads.net/@{0}" UserRegex = RParams.DMS(String.Format(UserRegexDefaultPattern, "threads.net/@"), 1) ImageVideoContains = "threads.net" @@ -155,7 +172,7 @@ Namespace API.ThreadsNet #End Region #Region "BaseAuthExists, GetUserUrl, GetUserPostUrl" Friend Overrides Function BaseAuthExists() As Boolean - Return Responser.CookiesExists And {HH_CSRF_TOKEN, HH_IG_APP_ID}.All(Function(v) ACheck(Of String)(v.Value)) + Return Responser.CookiesExists And {HH_CSRF_TOKEN, HH_IG_APP_ID}.All(Function(v) ACheck(Of String)(v.Value)) And CBool(DownloadData_Impl.Value) End Function Friend Overrides Function GetUserUrl(ByVal User As IPluginContentProvider) As String Return String.Format(UrlPatternUser, DirectCast(User, UserData).NameTrue) @@ -171,13 +188,23 @@ Namespace API.ThreadsNet End Function #End Region #Region "Update" + Private __Cookies As CookieKeeper = Nothing + Friend Overrides Sub BeginEdit() + __Cookies = Responser.Cookies.Copy + MyBase.BeginEdit() + End Sub Friend Overrides Sub Update() If _SiteEditorFormOpened And Responser.CookiesExists Then Dim csrf$ = If(Responser.Cookies.FirstOrDefault(Function(c) c.Name.StringToLower = IG.Header_CSRF_TOKEN_COOKIE)?.Value, String.Empty) If Not csrf.IsEmptyString Then HH_CSRF_TOKEN.Value = csrf + If Not __Cookies Is Nothing AndAlso Not __Cookies.ListEquals(Responser.Cookies) Then DownloadData_Impl.Value = True End If MyBase.Update() End Sub + Friend Overrides Sub EndEdit() + __Cookies.DisposeIfReady + MyBase.EndEdit() + End Sub #End Region End Class End Namespace \ No newline at end of file diff --git a/SCrawler/API/ThreadsNet/UserData.vb b/SCrawler/API/ThreadsNet/UserData.vb index b170f10..3a50282 100644 --- a/SCrawler/API/ThreadsNet/UserData.vb +++ b/SCrawler/API/ThreadsNet/UserData.vb @@ -24,11 +24,9 @@ Namespace API.ThreadsNet End Get End Property Private ReadOnly DefaultParser_ElemNode_Default() As Object = {"node", "thread_items", 0, "post"} - Private OPT_LSD As String = String.Empty - Private OPT_FB_DTSG As String = String.Empty Private ReadOnly Property Valid As Boolean Get - Return Not OPT_LSD.IsEmptyString And Not OPT_FB_DTSG.IsEmptyString And Not ID.IsEmptyString + Return ValidateBaseTokens() And Not ID.IsEmptyString End Get End Property #End Region @@ -54,23 +52,31 @@ Namespace API.ThreadsNet End Sub #End Region #Region "Download functions" + Private Sub WaitTimer() + If CInt(MySettings.RequestsWaitTimer_Any.Value) > 0 Then Thread.Sleep(CInt(MySettings.RequestsWaitTimer_Any.Value)) + End Sub + Private Sub DisableDownload() + MySettings.DownloadData_Impl.Value = False + MyMainLOG = $"{Site} downloading is disabled until you update your credentials" + End Sub Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken) - Dim errorFound As Boolean = False - Try - Responser.Method = "POST" - LoadSavePostsKV(True) - OPT_LSD = String.Empty - OPT_FB_DTSG = String.Empty - DownloadData(String.Empty, Token) - Catch ex As Exception - errorFound = True - Throw ex - Finally - Responser.Method = "POST" - UpdateResponser() - MySettings.UpdateResponserData(Responser) - If Not errorFound Then LoadSavePostsKV(False) - End Try + If CBool(MySettings.DownloadData_Impl.Value) Then + Dim errorFound As Boolean = False + Try + Responser.Method = "POST" + LoadSavePostsKV(True) + ResetBaseTokens() + DownloadData(String.Empty, Token) + Catch ex As Exception + errorFound = True + Throw ex + Finally + Responser.Method = "POST" + UpdateResponser() + MySettings.UpdateResponserData(Responser) + If Not errorFound Then LoadSavePostsKV(False) + End Try + End If End Sub Protected Overrides Sub UpdateResponser() If Not Responser Is Nothing AndAlso Not Responser.Disposed Then @@ -95,11 +101,11 @@ Namespace API.ThreadsNet UpdateCredentials() If idIsNull And Not ID.IsEmptyString Then _ForceSaveUserInfo = True End If - If Not Valid Then Throw New Plugin.ExitException("Some credentials are missing") + If Not Valid Then DisableDownload() : Throw New Plugin.ExitException("Some credentials are missing") Responser.Method = "POST" Responser.Referer = $"https://www.threads.net/@{NameTrue}" - Responser.Headers.Add(Header_FB_LSD, OPT_LSD) + Responser.Headers.Add(GQL_HEADER_FB_LSD, Token_lsd) Dim nextCursor$ = String.Empty Dim dataFound As Boolean = False @@ -112,7 +118,7 @@ Namespace API.ThreadsNet End If vars = SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & vars & "}") - URL = String.Format(urlPattern, OPT_LSD, vars, SymbolsConverter.ASCII.EncodeSymbolsOnly(OPT_FB_DTSG)) + URL = String.Format(urlPattern, Token_lsd, vars, Token_dtsg_Var) Using j As EContainer = GetDocument(URL, Token) If j.ListExists Then @@ -135,8 +141,9 @@ Namespace API.ThreadsNet Private Function GetDocument(ByVal URL As String, ByVal Token As CancellationToken, Optional ByVal Round As Integer = 0) As EContainer Try ThrowAny(Token) - If Round > 0 AndAlso Not UpdateCredentials() Then Throw New Exception("Failed to update credentials") + If Round > 0 AndAlso Not UpdateCredentials() Then DisableDownload() : Throw New Exception("Failed to update credentials") ThrowAny(Token) + WaitTimer() Dim r$ = Responser.GetResponse(URL) If Not r.IsEmptyString Then Return JsonDocument.Parse(r) Else Throw New Exception("Failed to get a response") Catch ex As Exception @@ -149,12 +156,12 @@ Namespace API.ThreadsNet End Function Private Function UpdateCredentials(Optional ByVal e As ErrorsDescriber = Nothing) As Boolean Dim URL$ = $"https://www.threads.net/@{NameTrue}" - OPT_LSD = String.Empty - OPT_FB_DTSG = String.Empty + ResetBaseTokens() Try Responser.Method = "GET" Responser.Referer = URL - Responser.Headers.Remove(Header_FB_LSD) + Responser.Headers.Remove(GQL_HEADER_FB_LSD) + WaitTimer() Dim r$ = Responser.GetResponse(URL,, EDP.ThrowException) Dim rr As RParams Dim tt$, ttVal$ @@ -168,15 +175,15 @@ Namespace API.ThreadsNet .WhatGet = RegexReturn.Value End With For Each tt In tokens - If Not OPT_FB_DTSG.IsEmptyString And Not OPT_LSD.IsEmptyString Then + If Not Token_dtsg.IsEmptyString And Not Token_lsd.IsEmptyString Then Exit For Else ttVal = RegexReplace(tt, rr) If Not ttVal.IsEmptyString Then If ttVal.Contains(":") Then - If OPT_FB_DTSG.IsEmptyString Then OPT_FB_DTSG = ttVal + If Token_dtsg.IsEmptyString Then Token_dtsg = ttVal Else - If OPT_LSD.IsEmptyString Then OPT_LSD = ttVal + If Token_lsd.IsEmptyString Then Token_lsd = ttVal End If End If End If @@ -187,9 +194,9 @@ Namespace API.ThreadsNet Return Valid Catch ex As Exception Dim notFound$ = String.Empty - If OPT_FB_DTSG.IsEmptyString Then notFound.StringAppend(Header_FB_LSD) - If OPT_LSD.IsEmptyString Then notFound.StringAppend("lsd") + ValidateBaseTokens(notFound) If ID.IsEmptyString Then notFound.StringAppend("User ID") + DisableDownload() Dim eex As New ErrorsDescriberException($"{ToStringForLog()}: failed to update some{IIf(notFound.IsEmptyString, String.Empty, $" ({notFound})")} credentials",,, ex) With { .ReplaceMainMessage = True, .SendToLogOnlyMessage = Responser.StatusCode = Net.HttpStatusCode.InternalServerError And Responser.Status = Net.WebExceptionStatus.ProtocolError @@ -224,7 +231,7 @@ Namespace API.ThreadsNet If m.State = UserMedia.States.Missing And Not m.Post.ID.IsEmptyString Then ThrowAny(Token) vars = SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & String.Format(varsPattern, m.Post.ID.Split("_").FirstOrDefault, ID) & "}") - URL = String.Format(urlPattern, OPT_LSD, vars, SymbolsConverter.ASCII.EncodeSymbolsOnly(OPT_FB_DTSG)) + URL = String.Format(urlPattern, Token_lsd, vars, Token_dtsg_Var) j = GetDocument(URL, Token) If j.ListExists Then diff --git a/SCrawler/Download/Groups/DownloadGroupCollection.vb b/SCrawler/Download/Groups/DownloadGroupCollection.vb index 7f52135..5db49f3 100644 --- a/SCrawler/Download/Groups/DownloadGroupCollection.vb +++ b/SCrawler/Download/Groups/DownloadGroupCollection.vb @@ -37,7 +37,7 @@ Namespace DownloadObjects.Groups End If End With End If - GroupsList.ListReindex + Reindex() End Sub #End Region #Region "Base properties" @@ -75,7 +75,10 @@ Namespace DownloadObjects.Groups GroupsList.Sort() End Sub Friend Sub Reindex() + Dim initUpValue As Boolean = _UpdateMode + BeginUpdate() GroupsList.ListReindex + If Not initUpValue Then EndUpdate() End Sub #End Region #Region "Group handlers" @@ -92,7 +95,7 @@ Namespace DownloadObjects.Groups If i >= 0 Then GroupsList(i).Dispose() GroupsList.RemoveAt(i) - GroupsList.ListReindex + Reindex() Update() End If End Sub @@ -133,9 +136,9 @@ Namespace DownloadObjects.Groups AddHandler .Deleted, AddressOf OnGroupDeleted AddHandler .Updated, AddressOf OnGroupUpdated If Not exists Then - GroupsList.ListReindex + Reindex() GroupsList.Sort() - GroupsList.ListReindex + Reindex() If Not Item.IsViewFilter And Not _UpdateMode Then RaiseEvent Added(.Self) Else If Not Item.IsViewFilter And Not _UpdateMode Then RaiseEvent Updated(.Self) diff --git a/SCrawler/Download/TDownloader.vb b/SCrawler/Download/TDownloader.vb index e538350..d19fb60 100644 --- a/SCrawler/Download/TDownloader.vb +++ b/SCrawler/Download/TDownloader.vb @@ -192,10 +192,6 @@ Namespace DownloadObjects _FilesSessionCleared = True Dim files As List(Of SFile) = SFile.GetFiles(SessionsPath.CSFileP, "*.xml",, EDP.ReturnValue) If files.ListExists Then files.RemoveAll(Settings.Feeds.FeedSpecialRemover) - If RenameOldFileNames(files) Then - files = SFile.GetFiles(SessionsPath.CSFileP, "*.xml",, EDP.ReturnValue) - If files.ListExists Then files.RemoveAll(Settings.Feeds.FeedSpecialRemover) - End If If files.ListExists Then Const ds$ = "yyyyMMdd" Dim nd$ = Now.ToString(ds), d1$ = Now.AddDays(-1).ToString(ds), d2$ = Now.AddDays(-2).ToString(ds) @@ -211,26 +207,6 @@ Namespace DownloadObjects ErrorsDescriber.Execute(EDP.SendToLog, ex, "[DownloadObjects.TDownloader.ClearSessions]") End Try End Sub - Private Function RenameOldFileNames(ByVal files As List(Of SFile)) As Boolean - Dim result As Boolean = False - Try - If files.ListExists AndAlso files.Exists(Function(ff) ff.Name.StringToLower.StartsWith("latest")) Then - Dim d As Date - Dim fileCurrent As SFile, fileNew As SFile - For Each fileCurrent In files - If fileCurrent.Name.StringToLower.StartsWith("latest") Then - d = IO.File.GetLastWriteTime(fileCurrent) - fileNew = fileCurrent - fileNew.Name = AConvert(Of String)(d, SessionDateTimeProvider) - SFile.Rename(fileCurrent, fileNew,, EDP.None) - result = True - End If - Next - End If - Catch - End Try - Return result - End Function #End Region Friend ReadOnly Property Downloaded As List(Of IUserData) Private ReadOnly NProv As IFormatProvider diff --git a/SCrawler/Editors/SiteEditorForm.vb b/SCrawler/Editors/SiteEditorForm.vb index 1197869..93ca302 100644 --- a/SCrawler/Editors/SiteEditorForm.vb +++ b/SCrawler/Editors/SiteEditorForm.vb @@ -138,6 +138,7 @@ Namespace Editors End Function End Class #End Region + Private ReadOnly PropertyValid As Predicate(Of PropertyValueHost) = Function(p) (Not p.IsHidden Or SiteSettingsShowHiddenControls) And Not p.Options Is Nothing Private ReadOnly Property Host As SettingsHost Private Property HostCollection As SettingsHostCollection Friend Sub New(ByVal h As SettingsHost) @@ -147,7 +148,7 @@ Namespace Editors If Not Host.Responser Is Nothing Then Cookies = Host.Responser.Cookies.Copy LBL_AUTH = New Label With {.Text = "Authorization", .TextAlign = ContentAlignment.MiddleCenter, .Dock = DockStyle.Fill} LBL_OTHER = New Label With {.Text = "Other Parameters", .TextAlign = ContentAlignment.MiddleCenter, .Dock = DockStyle.Fill} - Host.Source.BeginEdit() + Host.BeginEdit() End Sub Private Sub SiteEditorForm_Load(sender As Object, e As EventArgs) Handles Me.Load Try @@ -216,7 +217,7 @@ Namespace Editors If .PropList.Exists(Function(p) p.ControlNumber >= 0) Then .PropList.Sort() For Each pAuth As Boolean In pArr For Each prop As PropertyValueHost In .PropList - If Not prop.Options Is Nothing Then + If PropertyValid.Invoke(prop) Then With prop If .Options.IsAuth = pAuth Then @@ -286,7 +287,7 @@ Namespace Editors If Not SpecialButton Is Nothing Then SpecialButton.Dispose() LBL_AUTH.Dispose() LBL_OTHER.Dispose() - Host.Source.EndEdit() + Host.EndEdit() If Not Cookies Is Nothing Then Cookies.Dispose() End Sub Private Sub MyDefs_ButtonOkClick(ByVal Sender As Object, ByVal e As KeyHandleEventArgs) Handles MyDefs.ButtonOkClick @@ -311,8 +312,6 @@ Namespace Editors Next End If - Settings.BeginUpdate() - SiteDefaultsFunctions.SetPropByChecker(TP_SITE_PROPS, Host) If TXT_PATH.IsEmptyString Then .Path = Nothing Else .Path = TXT_PATH.Text .SavedPostsPath = TXT_PATH_SAVED_POSTS.Text @@ -327,13 +326,11 @@ Namespace Editors End With End If - If .PropList.Count > 0 Then .PropList.ForEach(Sub(p) If Not p.Options Is Nothing Then p.UpdateValueByControl()) + If .PropList.Count > 0 Then .PropList.ForEach(Sub(p) If PropertyValid.Invoke(p) Then p.UpdateValueByControl()) - .Source.Update() + .Update() End With - Settings.EndUpdate() - MyDefs.CloseForm() End If End Sub diff --git a/SCrawler/MainFrame.Designer.vb b/SCrawler/MainFrame.Designer.vb index 52c206a..4d96bab 100644 --- a/SCrawler/MainFrame.Designer.vb +++ b/SCrawler/MainFrame.Designer.vb @@ -420,9 +420,7 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form Me.BTT_DOWN_SPEC.Name = "BTT_DOWN_SPEC" Me.BTT_DOWN_SPEC.Size = New System.Drawing.Size(221, 22) Me.BTT_DOWN_SPEC.Text = "Download (advanced)" - Me.BTT_DOWN_SPEC.ToolTipText = "Filter the users you want to download and download them." & Global.Microsoft.VisualBasic.ChrW(13) & Global.Microsoft.VisualBasic.ChrW(10) & "Shift+Click to download" & - ", including non-existent users." & Global.Microsoft.VisualBasic.ChrW(13) & Global.Microsoft.VisualBasic.ChrW(10) & "Ctrl+Shift+Click to download, excluding from th" & - "e feed, including non-existent users." + Me.BTT_DOWN_SPEC.ToolTipText = "Filter the users you want to download and download them." ' 'BTT_DOWN_VIDEO ' diff --git a/SCrawler/MainMod.vb b/SCrawler/MainMod.vb index 7838c02..f95ee96 100644 --- a/SCrawler/MainMod.vb +++ b/SCrawler/MainMod.vb @@ -81,6 +81,7 @@ Friend Module MainMod Friend ReadOnly FeedVideoLengthProvider As New ADateTime("hh\:mm\:ss") With {.TimeParseMode = ADateTime.TimeModes.TimeSpan} Friend ReadOnly LogConnector As New LogHost Friend DefaultUserAgent As String = String.Empty + Friend SiteSettingsShowHiddenControls As Boolean = False #Region "NonExistingUsersLog" Friend ReadOnly NonExistingUsersLog As New TextSaver($"LOGs\NonExistingUsers.txt") With {.LogMode = True, .AutoSave = True} Friend Sub AddNonExistingUserToLog(ByVal Message As String) diff --git a/SCrawler/PluginsEnvironment/Attributes/Attributes.vb b/SCrawler/PluginsEnvironment/Attributes/Attributes.vb index ae63fb6..8e9737d 100644 --- a/SCrawler/PluginsEnvironment/Attributes/Attributes.vb +++ b/SCrawler/PluginsEnvironment/Attributes/Attributes.vb @@ -45,4 +45,12 @@ Namespace Plugin.Attributes Public Clone As Boolean = True Public Update As Boolean = True End Class + Public Class HiddenControlAttribute : Inherits Attribute + Public ReadOnly IsHidden As Boolean = True + Public Sub New() + End Sub + Public Sub New(ByVal _IsHidden As Boolean) + IsHidden = _IsHidden + End Sub + End Class End Namespace \ No newline at end of file diff --git a/SCrawler/PluginsEnvironment/Hosts/DownloadableMediaHost.vb b/SCrawler/PluginsEnvironment/Hosts/DownloadableMediaHost.vb index b148002..5e5cc12 100644 --- a/SCrawler/PluginsEnvironment/Hosts/DownloadableMediaHost.vb +++ b/SCrawler/PluginsEnvironment/Hosts/DownloadableMediaHost.vb @@ -125,7 +125,7 @@ Namespace Plugin.Hosts Dim __url$ = DirectCast(Me, IDownloadableMedia).URL_BASE.IfNullOrEmpty(URL) If File.Exists And Not __url.IsEmptyString And MyDownloaderSettings.CreateUrlFiles Then Dim urlFile As SFile = CreateUrlFile(__url, File) - If urlFile.Exists Then Files.Add(urlFile) + If urlFile.Exists Then AddFile(urlFile) End If If Not ExternalSource Is Nothing Then With ExternalSource : _HasError = .HasError : _Exists = .Exists : End With diff --git a/SCrawler/PluginsEnvironment/Hosts/PropertyValueHost.vb b/SCrawler/PluginsEnvironment/Hosts/PropertyValueHost.vb index fe7539c..07bb62c 100644 --- a/SCrawler/PluginsEnvironment/Hosts/PropertyValueHost.vb +++ b/SCrawler/PluginsEnvironment/Hosts/PropertyValueHost.vb @@ -46,6 +46,7 @@ Namespace Plugin.Hosts End Property Friend ReadOnly IsTaskCounter As Boolean Friend ReadOnly Exists As Boolean = False + Friend ReadOnly IsHidden As Boolean = False #Region "XML" Private ReadOnly _XmlName As String Private ReadOnly _XmlNameChecked As String @@ -309,6 +310,7 @@ Namespace Plugin.Hosts UpdateMember() Options = Member.GetCustomAttribute(Of PropertyOption)() IsTaskCounter = Not Member.GetCustomAttribute(Of TaskCounter)() Is Nothing + IsHidden = If(Member.GetCustomAttribute(Of HiddenControlAttribute)?.IsHidden, False) With Member.GetCustomAttribute(Of PXML) If Not .Self Is Nothing Then _XmlName = .ElementName diff --git a/SCrawler/PluginsEnvironment/Hosts/SettingsHost.vb b/SCrawler/PluginsEnvironment/Hosts/SettingsHost.vb index d35c986..2de05a0 100644 --- a/SCrawler/PluginsEnvironment/Hosts/SettingsHost.vb +++ b/SCrawler/PluginsEnvironment/Hosts/SettingsHost.vb @@ -23,6 +23,9 @@ Namespace Plugin.Hosts Friend Event Deleted As SettingsHostActionEventHandler Friend Event OkClick As SettingsHostActionEventHandler Friend Event CloneClick As SettingsHostActionEventHandler + Friend Event OnBeginEdit As SettingsHostActionEventHandler + Friend Event OnEndEdit As SettingsHostActionEventHandler + Friend Event OnUpdate As SettingsHostActionEventHandler #End Region #Region "Controls" Private WithEvents BTT_SETTINGS As ToolStripMenuItem @@ -258,7 +261,11 @@ Namespace Plugin.Hosts Source = Plugin Source.Logger = LogConnector [Default] = IsDef - If _XML Is Nothing Then IsAbstract = True + If _XML Is Nothing Then + IsAbstract = True + Else + _XML.BeginUpdate() + End If PropList = New List(Of PropertyValueHost) @@ -289,11 +296,11 @@ Namespace Plugin.Hosts End If Next End If + If _Key = API.PathPlugin.PluginKey And Not _XML Is Nothing Then _XML.XmlReadOnly = True Dim i% - Dim n() As String = {SettingsCLS.Name_Node_Sites, Name} - _AccountName = New XMLValue(Of String)(NameXML_AccountName,, _XML, n) + _AccountName = New XMLValue(Of String)(NameXML_AccountName,, _XML) Source.AccountName = _AccountName Source.BeginInit() @@ -373,34 +380,42 @@ Namespace Plugin.Hosts PropList.ForEach(Sub(p) p.SetDependents(PropList)) End If - _Path = New XMLValue(Of SFile)("Path",, _XML, n, New XMLToFilePathProvider) - _SavedPostsPath = New XMLValue(Of SFile)("SavedPostsPath",, _XML, n, New XMLToFilePathProvider) - DownloadSavedPosts = New XMLValue(Of Boolean)("DownloadSavedPosts", True, _XML, n) + _Path = New XMLValue(Of SFile)("Path",, _XML,, New XMLToFilePathProvider) + _SavedPostsPath = New XMLValue(Of SFile)("SavedPostsPath",, _XML,, New XMLToFilePathProvider) + DownloadSavedPosts = New XMLValue(Of Boolean)("DownloadSavedPosts", True, _XML) Temporary = New XMLValue(Of Boolean) - Temporary.SetExtended("Temporary", False, _XML, n) + Temporary.SetExtended("Temporary", False, _XML) Temporary.SetDefault(_Temp) + Temporary.Update() DownloadImages = New XMLValue(Of Boolean) - DownloadImages.SetExtended("DownloadImages", True, _XML, n) + DownloadImages.SetExtended("DownloadImages", True, _XML) DownloadImages.SetDefault(_Imgs) + DownloadImages.Update() DownloadVideos = New XMLValue(Of Boolean) - DownloadVideos.SetExtended("DownloadVideos", True, _XML, n) + DownloadVideos.SetExtended("DownloadVideos", True, _XML) DownloadVideos.SetDefault(_Vids) + DownloadVideos.Update() - DownloadSiteData = New XMLValue(Of Boolean)("DownloadSiteData", True, _XML, n) + DownloadSiteData = New XMLValue(Of Boolean)("DownloadSiteData", True, _XML) - GetUserMediaOnly = New XMLValue(Of Boolean)("GetUserMediaOnly", True, _XML, n) + GetUserMediaOnly = New XMLValue(Of Boolean)("GetUserMediaOnly", True, _XML) If PropList.Count > 0 Then Dim MaxOffset% = Math.Max(PropList.Max(Function(pp) pp.LeftOffset), PropertyValueHost.LeftOffsetDefault) For Each p As PropertyValueHost In PropList - If Not IsAbstract Then p.SetXmlEnvironment(_XML, n) + If Not IsAbstract Then p.SetXmlEnvironment(_XML) p.LeftOffset = MaxOffset Next End If Source.EndInit() + + If Not _XML Is Nothing Then + _XML.EndUpdate() + If _XML.ChangesDetected Then _XML.UpdateData(EDP.SendToLog) + End If End Sub Friend Function Apply(ByVal _XML As XmlFile, ByVal GlobalPath As SFile, ByRef _Temp As XMLValue(Of Boolean), ByRef _Imgs As XMLValue(Of Boolean), ByRef _Vids As XMLValue(Of Boolean)) As SettingsHost @@ -522,6 +537,20 @@ Namespace Plugin.Hosts Private Function ConvertUser(ByVal User As IUserData) As Object Return If(DirectCast(User, UserDataBase).ExternalPlugin, User) End Function +#Region "Edit" + Friend Sub BeginEdit() + Source.BeginEdit() + RaiseEvent OnBeginEdit(Me) + End Sub + Friend Sub Update() + Source.Update() + RaiseEvent OnUpdate(Me) + End Sub + Friend Sub EndEdit() + Source.EndEdit() + RaiseEvent OnEndEdit(Me) + End Sub +#End Region #End Region #Region "IEquatable Support" Friend Overloads Function Equals(ByVal Other As SettingsHost) As Boolean Implements IEquatable(Of SettingsHost).Equals diff --git a/SCrawler/PluginsEnvironment/Hosts/SettingsHostCollection.vb b/SCrawler/PluginsEnvironment/Hosts/SettingsHostCollection.vb index 3fe5558..18101ce 100644 --- a/SCrawler/PluginsEnvironment/Hosts/SettingsHostCollection.vb +++ b/SCrawler/PluginsEnvironment/Hosts/SettingsHostCollection.vb @@ -91,19 +91,49 @@ Namespace Plugin.Hosts End If End With HostsUnavailableIndexes = New List(Of Integer) - Hosts = New List(Of SettingsHost) From {New SettingsHost(CreateInstance(), True, _XML, GlobalPath, _Temp, _Imgs, _Vids)} - HostsXml = New List(Of XmlFile) + Dim defInstance As ISiteSettings = CreateInstance() + HostsXml = New List(Of XmlFile) From { + GetNewXmlFile($"{SettingsFolderName}\{SiteSettingsBase.ResponserFilePrefix}{defInstance.Site}_Settings.xml", defInstance.Site, _XML) + } + Hosts = New List(Of SettingsHost) From {New SettingsHost(defInstance, True, HostsXml(0), GlobalPath, _Temp, _Imgs, _Vids)} + Dim hostFiles As List(Of SFile) = SFile.GetFiles(SettingsFolderName.CSFileP, $"{String.Format(FileNamePattern, Key, Name)}*.xml",, EDP.ReturnValue) If hostFiles.ListExists Then For Each f As SFile In hostFiles - HostsXml.Add(New XmlFile(f) With {.AutoUpdateFile = True}) - Hosts.Add(New SettingsHost(CreateInstance(HostsXml.Last.Value({SettingsCLS.Name_Node_Sites, [Default].Name}, SettingsHost.NameXML_AccountName)), False, HostsXml.Last, + HostsXml.Add(GetNewXmlFile(f, [Default].Name)) + Hosts.Add(New SettingsHost(CreateInstance(HostsXml.Last.Value(SettingsHost.NameXML_AccountName)), False, HostsXml.Last, GlobalPath, _Temp, _Imgs, _Vids)) Next End If Hosts.ListReindex Hosts.ForEach(Sub(h) SetHostHandlers(h)) End Sub + Private Function GetNewXmlFile(ByVal f As SFile, ByVal SiteName As String, Optional ByVal SourceXml As XmlFile = Nothing) As XmlFile + Dim x As New XmlFile(f,, False) With {.AutoUpdateFile = True} + If Not f.Exists Then x.Name = "SiteSettings" + x.LoadData() + 'URGENT: reorganization of settings: remove the following code + Dim n$() = {SettingsCLS.Name_Node_Sites, SiteName} + Dim processed As Boolean = False + With If(SourceXml, x) + If .Count > 0 AndAlso .Contains(n) Then + With .Item(n) + If .ListExists Then + For Each container As EContainer In .Self : x.Add(container.Name, container.Value) : Next + processed = True + End If + End With + If processed Then + .Remove(n) + If SourceXml Is Nothing Then .Remove(SettingsCLS.Name_Node_Sites) + x.Name = "SiteSettings" + x.UpdateData() + End If + End If + End With + '-----END REMOVE----- + Return x + End Function #End Region #Region "CreateInstance" Private Function CreateInstance(Optional ByVal Name As String = Nothing, Optional ByVal Abstract As Boolean = False) As ISiteSettings @@ -121,15 +151,33 @@ Namespace Plugin.Hosts End Function #End Region #Region "Host handlers" +#Region "Edit" + Private Sub Hosts_OnBeginEdit(ByVal Obj As SettingsHost) + If Obj.Index.ValueBetween(0, HostsXml.Count - 1) Then HostsXml(Obj.Index).BeginUpdate() + End Sub + Private Sub Hosts_OnUpdate(ByVal Obj As SettingsHost) + End Sub + Private Sub Hosts_OnEndEdit(ByVal Obj As SettingsHost) + If Obj.Index.ValueBetween(0, HostsXml.Count - 1) Then + With HostsXml(Obj.Index) + .EndUpdate() + If .ChangesDetected Then .UpdateData(EDP.SendToLog) + End With + End If + End Sub +#End Region Private Sub SetHostHandlers(ByVal Host As SettingsHost) AddHandler Host.OkClick, AddressOf Hosts_OkClick AddHandler Host.Deleted, AddressOf Hosts_Deleted AddHandler Host.CloneClick, AddressOf Hosts_CloneClick + AddHandler Host.OnBeginEdit, AddressOf Hosts_OnBeginEdit + AddHandler Host.OnUpdate, AddressOf Hosts_OnUpdate + AddHandler Host.OnEndEdit, AddressOf Hosts_OnEndEdit If Host.Index > 0 Then Host.Source.DefaultInstance = [Default].Source : Host.DefaultInstanceChanged() End Sub Private Sub Hosts_OkClick(ByVal Obj As SettingsHost) If Obj.Index = -1 Then - HostsXml.Add(New XmlFile($"{SettingsFolderName}\{String.Format(FileNamePatternFull, Key, Name, Obj.AccountName)}.xml") With {.AutoUpdateFile = True}) + HostsXml.Add(GetNewXmlFile($"{SettingsFolderName}\{String.Format(FileNamePatternFull, Key, Name, Obj.AccountName)}.xml", Name)) With Settings : Hosts.Add(Obj.Apply(HostsXml.Last, .GlobalPath, .DefaultTemporary, .DefaultDownloadImages, .DefaultDownloadVideos)) : End With HostsXml.Last.UpdateData() @@ -157,11 +205,11 @@ Namespace Plugin.Hosts MsgBoxE({$"An error occurred while changing user accounts (see log for details).{vbCr}Operation canceled.", ChngUACC_MsgTitle}, vbCritical) Exit Sub End Select - With HostsXml(Obj.Index - 1) + With HostsXml(Obj.Index) .File.Delete(SFO.File, SFODelete.DeleteToRecycleBin, EDP.None) .Dispose() End With - HostsXml.RemoveAt(Obj.Index - 1) + HostsXml.RemoveAt(Obj.Index) Hosts.RemoveAt(Obj.Index) Hosts.ListReindex Obj.Source.Delete() diff --git a/SCrawler/SCrawler.vbproj b/SCrawler/SCrawler.vbproj index 3710022..4b5f446 100644 --- a/SCrawler/SCrawler.vbproj +++ b/SCrawler/SCrawler.vbproj @@ -181,6 +181,7 @@ + diff --git a/SCrawler/SettingsCLS.vb b/SCrawler/SettingsCLS.vb index 34c85e3..bc3c6e7 100644 --- a/SCrawler/SettingsCLS.vb +++ b/SCrawler/SettingsCLS.vb @@ -178,6 +178,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable Friend Property FeedViews As FeedViewCollection Private ReadOnly BlackListFile As SFile = $"{SettingsFolderName}\BlackList.txt" Private ReadOnly UsersSettingsFile As SFile = $"{SettingsFolderName}\Users.xml" + Private ReadOnly Property SettingsVersion As XMLValue(Of Integer) #End Region #Region "Initializer" Friend Sub New() @@ -201,6 +202,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable EnvironmentProgramsList = New List(Of String) AutomationFile = New XMLValue(Of String)("AutomationFile",, MyXML) + SiteSettingsShowHiddenControls = MyXML.Value("SiteSettingsShowHiddenControls").FromXML(Of Boolean)(False) Dim n() As String Dim n_old() As String 'URGENT: remove this line @@ -210,6 +212,8 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable Dim forceSaveXML As Boolean = Not SettingsReoranized 'URGENT: remove this line Dim forceSaveXML2 As Boolean = Not SettingsReoranized OrElse Not SettingsReoranized2 'URGENT: remove this line + SettingsVersion = New XMLValue(Of Integer)("SettingsVersion", 0, MyXML) + #Region "Properties: environment" 'Environment n = {"MediaEnvironment"}