diff --git a/Changelog.md b/Changelog.md index 9837595..bba3143 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,15 @@ +# 2024.1.26.0 + +*2024-01-26* + +- Added + - YouTube (standalone app): **the ability to reduce video FPS** + - TikTok: the ability to use a regex to clean the title + - YouTube (SCrawler): the ability to ignore community errors +- Fixed + - Instagram: stories (user) downloading with the wrong aspect ratio for some users + - Minor bugs + # 2024.1.20.0 *2024-01-20* diff --git a/SCrawler.YouTube/Base/YouTubeSettings.vb b/SCrawler.YouTube/Base/YouTubeSettings.vb index baa679e..107acb7 100644 --- a/SCrawler.YouTube/Base/YouTubeSettings.vb +++ b/SCrawler.YouTube/Base/YouTubeSettings.vb @@ -15,6 +15,7 @@ Imports PersonalUtilities.Forms Imports PersonalUtilities.Functions.XML Imports PersonalUtilities.Functions.XML.Base Imports PersonalUtilities.Functions.XML.Objects +Imports PersonalUtilities.Functions.XML.Attributes Imports PersonalUtilities.Functions.XML.Attributes.Specialized Imports PersonalUtilities.Tools Imports PersonalUtilities.Tools.Grid.Base @@ -275,6 +276,71 @@ Namespace API.YouTube.Base Public ReadOnly Property DefaultVideoIncludeNullSize As XMLValue(Of Boolean) + + Private ReadOnly Property DefaultVideoFPS_XML As XMLValue(Of Double) + + Public Property DefaultVideoFPS As Double + Get + Return DefaultVideoFPS_XML + End Get + Set(ByVal fps As Double) + DefaultVideoFPS_XML.Value = fps + End Set + End Property + Private Function ShouldSerializeDefaultVideoFPS() As Boolean + Return DefaultVideoFPS <> DefaultVideoFPS_XML.Value + End Function + Private Sub ResetDefaultVideoFPS() + DefaultVideoFPS = -1 + End Sub + Friend Class FpsFormatProvider : Implements IGridConversionProvider + Private Property Converter As TypeConverter Implements IGridConversionProvider.Converter + Private Property Context As ITypeDescriptorContext Implements IGridConversionProvider.Context + Private Property DataType As Type Implements IGridConversionProvider.DataType + Private Property Instance As Object Implements IGridConversionProvider.Instance + Friend Shared ReadOnly Property MyProviderDefault As ANumbers + Get + Return New ANumbers(ANumbers.Cultures.Primitive) With {.DecimalDigits = 5, .TrimDecimalDigits = True} + End Get + End Property + Friend Const ErrorMessageDefault As String = "The fps value must be a number" + Private ReadOnly MyProvider As ANumbers = MyProviderDefault + Friend Function ToObject(ByVal Context As ITypeDescriptorContext, ByVal Culture As CultureInfo, ByVal Value As Object) As Object Implements IGridConversionProvider.ToObject + Return AConvert(Of Double)(Value, MyProvider, -1) + End Function + Friend Overloads Function ToString(ByVal Context As ITypeDescriptorContext, ByVal Culture As CultureInfo, ByVal Value As Object, + ByVal DestinationType As Type) As Object Implements IGridConversionProvider.ToString + If ACheck(Of Double)(Value, AModes.Var, MyProvider) Then + Return Value.ToString + Else + Return -1 + End If + End Function + Friend Function CreateInstance(ByVal Context As ITypeDescriptorContext, ByVal NewValue As Object, ByRef RefreshGrid As Boolean) As Object Implements IGridConversionProvider.CreateInstance + If ACheck(Of Double)(NewValue, AModes.Var, MyProvider) Then + Return NewValue + Else + RefreshGrid = True + Return -1 + End If + End Function + Friend Function Convert(ByVal Value As Object, ByVal DestinationType As Type, ByVal Provider As IFormatProvider, + Optional ByVal NothingArg As Object = Nothing, Optional ByVal e As ErrorsDescriber = Nothing) As Object Implements ICustomProvider.Convert + Return AConvert(Value, AModes.Var, DestinationType,, True, -1, MyProvider, EDP.ReturnValue) + End Function + Friend Function IsValid(ByVal Context As ITypeDescriptorContext, ByVal Value As Object, ByVal DestinationType As Type) As Boolean Implements IGridValidator.IsValid + If ACheck(Of Double)(Value, AModes.Var, MyProvider) Then + Return True + Else + Throw New FormatException(ErrorMessageDefault) + End If + End Function + Private Function GetFormat(ByVal FormatType As Type) As Object Implements IFormatProvider.GetFormat + Throw New NotImplementedException("'GetFormat' is not available in 'FpsFormatProvider'") + End Function + End Class #End Region #Region "Defaults Audio" + + iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO + xAAADsQBlSsOGwAAAIZJREFUOE+1j10KwCAMgz2b755xl/IsvnaL2K20UfbDAmEako+ZROSTafjE12Go + tbbB43rK5xSAQq1VYFtmeQBoqZTSreVZvgTknM8yyyjA/qodsDF9gspD2Bj6B+DH+NqzhQQAG+POMnSX + AFuc5QFgn6ClHh5iOQVAKNixyucB8NY0vG9JOzzyhrdq5IRgAAAAAElFTkSuQmCC + + + iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO wwAADsMBx2+oZAAAAR5JREFUOE+VkjFqwzAUhn2D9iShRyi+QhYbGujg3ZATZPKYdC6FQhPwlAMkg3dP @@ -299,7 +307,7 @@ cMaRN0UdBBkAAAAASUVORK5CYII= - + iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGOfPtRkwAAACBjSFJNAAB6 JQAAgIMAAPn/AACA6QAAdTAAAOpgAAA6mAAAF2+SX8VGAAAACXBIWXMAAAsTAAALEwEAmpwYAAACOElE @@ -315,7 +323,7 @@ VnR1MIwhwMTCyqEQ37qEmZVDFF0OE/9nAACtFF4Ey6OP+wAAAABJRU5ErkJggg== - + iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO xAAADsQBlSsOGwAAAIZJREFUOE+1j10KwCAMgz2b755xl/IsvnaL2K20UfbDAmEako+ZROSTafjE12Go @@ -323,7 +331,7 @@ AFuc5QFgn6ClHh5iOQVAKNixyucB8NY0vG9JOzzyhrdq5IRgAAAAAElFTkSuQmCC - + iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO wwAADsMBx2+oZAAAAR5JREFUOE+VkjFqwzAUhn2D9iShRyi+QhYbGujg3ZATZPKYdC6FQhPwlAMkg3dP @@ -334,7 +342,7 @@ cMaRN0UdBBkAAAAASUVORK5CYII= - + iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGOfPtRkwAAACBjSFJNAAB6 JQAAgIMAAPn/AACA6QAAdTAAAOpgAAA6mAAAF2+SX8VGAAAACXBIWXMAAAsTAAALEwEAmpwYAAACOElE @@ -350,7 +358,7 @@ VnR1MIwhwMTCyqEQ37qEmZVDFF0OE/9nAACtFF4Ey6OP+wAAAABJRU5ErkJggg== - + iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO xAAADsQBlSsOGwAAAIZJREFUOE+1j10KwCAMgz2b755xl/IsvnaL2K20UfbDAmEako+ZROSTafjE12Go diff --git a/SCrawler.YouTube/Controls/VideoOptionsForm.vb b/SCrawler.YouTube/Controls/VideoOptionsForm.vb index 6f9fcb2..1b267ca 100644 --- a/SCrawler.YouTube/Controls/VideoOptionsForm.vb +++ b/SCrawler.YouTube/Controls/VideoOptionsForm.vb @@ -22,6 +22,7 @@ Namespace API.YouTube.Controls Friend Class VideoOptionsForm : Implements IDesignXMLContainer #Region "Declarations" Private MyView As FormView + Private ReadOnly MyFieldsChecker As FieldsChecker Friend Property DesignXML As EContainer Implements IDesignXMLContainer.DesignXML Private Property DesignXMLNodes As String() Implements IDesignXMLContainer.DesignXMLNodes Private Property DesignXMLNodeName As String Implements IDesignXMLContainer.DesignXMLNodeName @@ -30,6 +31,25 @@ Namespace API.YouTube.Controls Friend Property MyContainer As YouTubeMediaContainerBase Private Initialization As Boolean = True Private ReadOnly IsSavedObject As Boolean + Private Class FpsFieldChecker : Inherits FieldsCheckerProviderBase + Private ReadOnly MyProvider As ANumbers = YouTubeSettings.FpsFormatProvider.MyProviderDefault + Public Overrides Property ErrorMessage As String + Get + Return IIf(HasError, YouTubeSettings.FpsFormatProvider.ErrorMessageDefault, String.Empty) + End Get + Set : End Set + End Property + Public Overrides Function Convert(ByVal Value As Object, ByVal DestinationType As Type, ByVal Provider As IFormatProvider, + Optional ByVal NothingArg As Object = Nothing, Optional ByVal e As ErrorsDescriber = Nothing) As Object + If ACheck(Of Double)(Value, AModes.Var, MyProvider) Then + Return Value + Else + HasError = True + TypeError = True + Return Nothing + End If + End Function + End Class #End Region #Region "Initializers" Friend Sub New(ByVal Container As YouTubeMediaContainerBase, Optional ByVal IsSavedObject As Boolean = False) @@ -37,6 +57,7 @@ Namespace API.YouTube.Controls MyContainer = Container CNT_PROCESSOR = New TableControlsProcessor(TP_CONTROLS) Me.IsSavedObject = IsSavedObject + MyFieldsChecker = New FieldsChecker End Sub #End Region #Region "Form handlers" @@ -132,6 +153,9 @@ Namespace API.YouTube.Controls End If setDef(CMB_SUBS_FORMAT, __optionValue) + If MyYouTubeSettings.DefaultVideoFPS > 0 Then TXT_FPS.Text = MyYouTubeSettings.DefaultVideoFPS + MyFieldsChecker.AddControl(Of Double)(TXT_FPS, TXT_FPS.CaptionText, True, New FpsFieldChecker) + MyFieldsChecker.EndLoaderOperations() TP_SUBS.Enabled = .Subtitles.Count > 0 TXT_SUBS_ADDIT.Enabled = .Subtitles.Count > 0 RefillTextBoxes() @@ -275,6 +299,7 @@ Namespace API.YouTube.Controls #Region "OK, Cancel" Private Sub BTT_DOWN_Click(sender As Object, e As EventArgs) Handles BTT_DOWN.Click Try + If Not MyFieldsChecker.AllParamsOK Then Exit Sub Dim f As SFile If MyContainer.HasElements Then f = TXT_FILE.Text.CSFileP @@ -284,6 +309,7 @@ Namespace API.YouTube.Controls If f.IsEmptyString Then Throw New ArgumentNullException("File", "The output file cannot be null") With MyContainer .OutputVideoExtension = CMB_FORMAT.Text.StringToLower + .OutputVideoFPS = AConvert(Of Double)(TXT_FPS.Text, YouTubeSettings.FpsFormatProvider.MyProviderDefault, -1) .OutputAudioCodec = CMB_AUDIO_CODEC.Text.StringToLower .OutputSubtitlesFormat = CMB_SUBS_FORMAT.Text.StringToLower diff --git a/SCrawler.YouTube/My Project/AssemblyInfo.vb b/SCrawler.YouTube/My Project/AssemblyInfo.vb index c75c616..22452b9 100644 --- a/SCrawler.YouTube/My Project/AssemblyInfo.vb +++ b/SCrawler.YouTube/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + + diff --git a/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb b/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb index da5d4b9..0f12868 100644 --- a/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb +++ b/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb @@ -217,6 +217,16 @@ Namespace API.YouTube.Objects If HasElements Then Elements.ForEach(Sub(e) e.OutputVideoExtension = _OutputVideoExtension) End Set End Property + Protected _OutputVideoFPS As Double = -1 + Friend Property OutputVideoFPS As Double + Get + Return _OutputVideoFPS + End Get + Set(ByVal fps As Double) + _OutputVideoFPS = fps + If HasElements Then Elements.ForEach(Sub(elem) DirectCast(elem, YouTubeMediaContainerBase).OutputVideoFPS = fps) + End Set + End Property #End Region #Region "Audio" Friend Property SelectedAudioIndex As Integer = -1 Implements IYouTubeMediaContainer.SelectedAudioIndex @@ -1026,6 +1036,16 @@ Namespace API.YouTube.Objects If fAacAudio.Exists And Not aacRequested Then fAacAudio.Delete() If fAc3Audio.Exists And Not ac3Requested And SelectedVideoIndex >= 0 Then fAc3Audio.Delete() End If + + If SelectedVideoIndex >= 0 AndAlso OutputVideoFPS > 0 AndAlso SelectedVideo.Bitrate > OutputVideoFPS Then + f = File + f.Name &= "tmp00" + .Execute($"ffmpeg -i ""{File}"" -filter:v fps={OutputVideoFPS.ToString.Replace(",", ".")} -c:a copy ""{f}""") + If f.Exists Then + File.Delete() + SFile.Rename(f, File,, EDP.LogMessageValue) + End If + End If End If End If End With diff --git a/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb b/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb index 90f385c..01c2761 100644 --- a/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb +++ b/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + + diff --git a/SCrawler/API/Instagram/EditorExchangeOptions.vb b/SCrawler/API/Instagram/EditorExchangeOptions.vb index f539a0c..6171a0e 100644 --- a/SCrawler/API/Instagram/EditorExchangeOptions.vb +++ b/SCrawler/API/Instagram/EditorExchangeOptions.vb @@ -11,7 +11,7 @@ Namespace API.Instagram Friend Class EditorExchangeOptions Friend Property GetTimeline As Boolean - + Friend Property GetReels As Boolean Friend Property GetStories As Boolean diff --git a/SCrawler/API/Instagram/SiteSettings.vb b/SCrawler/API/Instagram/SiteSettings.vb index 82b8533..0aba8c6 100644 --- a/SCrawler/API/Instagram/SiteSettings.vb +++ b/SCrawler/API/Instagram/SiteSettings.vb @@ -121,7 +121,7 @@ Namespace API.Instagram Private ReadOnly Property SleepTimerOnPostsLimitProvider As IFormatProvider Friend ReadOnly Property GetTimeline As PropertyValue - + Friend ReadOnly Property GetReels As PropertyValue Friend ReadOnly Property GetStories As PropertyValue @@ -139,7 +139,7 @@ Namespace API.Instagram #Region "Download ready" Friend ReadOnly Property DownloadTimeline As PropertyValue - + Friend ReadOnly Property DownloadReels As PropertyValue Friend ReadOnly Property DownloadStories As PropertyValue diff --git a/SCrawler/API/Instagram/UserData.vb b/SCrawler/API/Instagram/UserData.vb index e5e5498..8903041 100644 --- a/SCrawler/API/Instagram/UserData.vb +++ b/SCrawler/API/Instagram/UserData.vb @@ -902,8 +902,8 @@ Namespace API.Instagram Dim maxSize As Func(Of EContainer, Integer) = Function(ByVal _ss As EContainer) As Integer Dim w% = AConvert(Of Integer)(_ss.Value("width"), 0) Dim h% = AConvert(Of Integer)(_ss.Value("height"), 0) - Return w + h - 'Return Math.Max(w, h) + 'Return w + h + Return Math.Max(w, h) End Function Dim wrongData As Predicate(Of Sizes) = Function(_ss) _ss.HasError Or _ss.Data.IsEmptyString Dim img As Predicate(Of EContainer) = Function(_img) Not _img.Name.IsEmptyString AndAlso _img.Name.StartsWith("image_versions") AndAlso _img.Count > 0 diff --git a/SCrawler/API/TikTok/SiteSettings.vb b/SCrawler/API/TikTok/SiteSettings.vb index 785464d..260dc62 100644 --- a/SCrawler/API/TikTok/SiteSettings.vb +++ b/SCrawler/API/TikTok/SiteSettings.vb @@ -22,6 +22,10 @@ Namespace API.TikTok Friend ReadOnly Property TitleUseNativeSTD As PropertyValue Friend ReadOnly Property TitleAddVideoID As PropertyValue + + Friend ReadOnly Property TitleUseRegexForTitle As PropertyValue + + Friend ReadOnly Property TitleUseRegexForTitle_Value As PropertyValue Friend ReadOnly Property UseParsedVideoDate As PropertyValue @@ -31,6 +35,8 @@ Namespace API.TikTok TitleUseNative = New PropertyValue(True) TitleUseNativeSTD = New PropertyValue(False) TitleAddVideoID = New PropertyValue(True) + TitleUseRegexForTitle = New PropertyValue(False) + TitleUseRegexForTitle_Value = New PropertyValue(String.Empty, GetType(String)) UseParsedVideoDate = New PropertyValue(True) UseNetscapeCookies = True UrlPatternUser = "https://www.tiktok.com/@{0}/" diff --git a/SCrawler/API/TikTok/UserData.vb b/SCrawler/API/TikTok/UserData.vb index 8ca3e96..4d2bd0c 100644 --- a/SCrawler/API/TikTok/UserData.vb +++ b/SCrawler/API/TikTok/UserData.vb @@ -20,6 +20,9 @@ Namespace API.TikTok Private Const Name_TitleUseNative As String = "TitleUseNative" Private Const Name_TitleAddVideoID As String = "TitleAddVideoID" Private Const Name_LastDownloadDate As String = "LastDownloadDate" + Private Const Name_TitleUseRegexForTitle As String = "TitleUseRegexForTitle" + Private Const Name_TitleUseRegexForTitle_Value As String = "TitleUseRegexForTitle_Value" + Private Const Name_TitleUseGlobalRegexOptions As String = "TitleUseGlobalRegexOptions" #End Region #Region "Declarations" Private ReadOnly Property MySettings As SiteSettings @@ -27,26 +30,37 @@ Namespace API.TikTok Return HOST.Source End Get End Property + Private UserCache As CacheKeeper = Nothing Private ReadOnly Property RootCacheTikTok As ICacheKeeper Get - With Settings.Cache - Dim f As SFile = $"{Settings.Cache.RootDirectory.PathWithSeparator}TikTokCache\" - If .ContainsFolder(f) Then - Return .GetInstance(f) - Else - f.Exists(SFO.Path, True) - With .NewInstance(Of BatchFileExchanger)(f) - .DeleteCacheOnDispose = False - .DeleteRootOnDispose = False - Return .Self - End With - End If - End With + If Not UserCache Is Nothing AndAlso Not UserCache.Disposed Then + With DirectCast(UserCache.NewInstance(Of BatchFileExchanger), BatchFileExchanger) + .Validate() + Return .Self + End With + Else + With Settings.Cache + Dim f As SFile = $"{Settings.Cache.RootDirectory.PathWithSeparator}TikTokCache\" + If .ContainsFolder(f) Then + Return .GetInstance(f) + Else + f.Exists(SFO.Path, True) + With .NewInstance(Of BatchFileExchanger)(f) + .DeleteCacheOnDispose = False + .DeleteRootOnDispose = False + Return .Self + End With + End If + End With + End If End Get End Property Friend Property RemoveTagsFromTitle As Boolean = False Friend Property TitleUseNative As Boolean = True Friend Property TitleAddVideoID As Boolean = True + Friend Property TitleUseRegexForTitle As Boolean = False + Friend Property TitleUseRegexForTitle_Value As String = String.Empty + Friend Property TitleUseGlobalRegexOptions As Boolean = True Private Property LastDownloadDate As Date? = Nothing Private _TrueName As String = String.Empty Friend Property TrueName As String @@ -68,6 +82,9 @@ Namespace API.TikTok RemoveTagsFromTitle = .RemoveTagsFromTitle TitleUseNative = .TitleUseNative TitleAddVideoID = .TitleAddVideoID + TitleUseRegexForTitle = .TitleUseRegexForTitle + TitleUseRegexForTitle_Value = .TitleUseRegexForTitle_Value + TitleUseGlobalRegexOptions = .TitleUseGlobalRegexOptions End With End If End Sub @@ -82,12 +99,18 @@ Namespace API.TikTok LastDownloadDate = AConvert(Of Date)(.Value(Name_LastDownloadDate), ADateTime.Formats.BaseDateTime, Nothing) If Not LastDownloadDate.HasValue Then LastDownloadDate = LastUpdated _TrueName = .Value(Name_TrueName) + TitleUseRegexForTitle = .Value(Name_TitleUseRegexForTitle).FromXML(Of Boolean)(False) + TitleUseRegexForTitle_Value = .Value(Name_TitleUseRegexForTitle_Value) + TitleUseGlobalRegexOptions = .Value(Name_TitleUseGlobalRegexOptions).FromXML(Of Boolean)(True) Else .Add(Name_RemoveTagsFromTitle, RemoveTagsFromTitle.BoolToInteger) .Add(Name_TitleUseNative, TitleUseNative.BoolToInteger) .Add(Name_TitleAddVideoID, TitleAddVideoID.BoolToInteger) .Add(Name_LastDownloadDate, AConvert(Of String)(LastDownloadDate, AModes.XML, ADateTime.Formats.BaseDateTime, String.Empty)) .Add(Name_TrueName, _TrueName) + .Add(Name_TitleUseRegexForTitle, TitleUseRegexForTitle.BoolToInteger) + .Add(Name_TitleUseRegexForTitle_Value, TitleUseRegexForTitle_Value) + .Add(Name_TitleUseGlobalRegexOptions, TitleUseGlobalRegexOptions.BoolToInteger) End If End With End Sub @@ -99,110 +122,145 @@ Namespace API.TikTok End Sub #End Region #Region "Download functions" + Private Function GetTitleRegex() As RParams + Dim titleRegex As RParams = Nothing + If TitleUseGlobalRegexOptions Then + If CBool(MySettings.TitleUseRegexForTitle.Value) AndAlso Not CStr(MySettings.TitleUseRegexForTitle_Value.Value).IsEmptyString Then _ + titleRegex = RParams.DM(MySettings.TitleUseRegexForTitle_Value.Value, 0, RegexReturn.List, EDP.ReturnValue) + ElseIf TitleUseRegexForTitle And Not TitleUseRegexForTitle_Value.IsEmptyString Then + titleRegex = RParams.DM(TitleUseRegexForTitle_Value, 0, RegexReturn.List, EDP.ReturnValue) + End If + If Not titleRegex Is Nothing Then + titleRegex.NothingExists = True + titleRegex.Nothing = New List(Of String) + titleRegex.Converter = Function(input) input.StringTrim + End If + Return titleRegex + End Function + Private Function ChangeTitleRegex(ByVal Title As String, ByVal Regex As RParams) As String + Try + If Not Regex Is Nothing Then + With DirectCast(RegexReplace(Title, Regex), List(Of String)) + If .ListExists Then + Dim newTitle$ = .ListToString(String.Empty, EDP.ReturnValue).StringTrim + If Not newTitle.IsEmptyString Then Return newTitle + End If + End With + End If + Catch ex As Exception + End Try + Return Title + End Function + Friend Overrides Sub DownloadData(ByVal Token As CancellationToken) + MyBase.DownloadData(Token) + UserCache.DisposeIfReady(False) + UserCache = Nothing + End Sub Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken) Dim URL$ = $"https://www.tiktok.com/@{TrueName}" - Using cache As CacheKeeper = CreateCache() - Try - Dim postID$, title$, postUrl$, newName$ - Dim postDate As Date? - Dim dateAfterC As Date? = Nothing - Dim dateBefore As Date? = DownloadDateTo - Dim dateAfter As Date? = DownloadDateFrom - Dim baseDataObtained As Boolean = False - - If _ContentList.Count > 0 Then - With (From d In _ContentList Where d.Post.Date.HasValue Select d.Post.Date.Value) - If .ListExists Then dateAfterC = .Min - End With - End If + UserCache = CreateCache() + Try + Dim postID$, title$, postUrl$, newName$ + Dim postDate As Date? + Dim dateAfterC As Date? = Nothing + Dim dateBefore As Date? = DownloadDateTo + Dim dateAfter As Date? = DownloadDateFrom + Dim baseDataObtained As Boolean = False + Dim titleRegex As RParams = GetTitleRegex() - With {CStr(AConvert(Of String)(dateAfter, SimpleDateConverter, String.Empty)).FromXML(Of Integer)(-1), - CStr(AConvert(Of String)(dateAfterC, SimpleDateConverter, String.Empty)).FromXML(Of Integer)(-1)}.ListWithRemove(Function(d) d = -1) - If .ListExists Then dateAfter = AConvert(Of Date)(CStr(.Min), SimpleDateConverter, Nothing) + If _ContentList.Count > 0 Then + With (From d In _ContentList Where d.Post.Date.HasValue Select d.Post.Date.Value) + If .ListExists Then dateAfterC = .Min End With + End If - If LastDownloadDate.HasValue Then - If Not DownloadDateTo.HasValue And Not DownloadDateFrom.HasValue Then - If LastDownloadDate.Value.AddDays(1) <= Now Then - dateAfter = LastDownloadDate.Value - Else - dateAfter = LastDownloadDate.Value.AddDays(-1) - End If - dateBefore = Nothing - ElseIf dateAfter.HasValue And Not DownloadDateFrom.HasValue Then - If (LastDownloadDate.Value - dateAfter.Value).TotalDays > 1 Then dateAfter = dateAfter.Value.AddDays(1) + With {CStr(AConvert(Of String)(dateAfter, SimpleDateConverter, String.Empty)).FromXML(Of Integer)(-1), + CStr(AConvert(Of String)(dateAfterC, SimpleDateConverter, String.Empty)).FromXML(Of Integer)(-1)}.ListWithRemove(Function(d) d = -1) + If .ListExists Then dateAfter = AConvert(Of Date)(CStr(.Min), SimpleDateConverter, Nothing) + End With + + If LastDownloadDate.HasValue Then + If Not DownloadDateTo.HasValue And Not DownloadDateFrom.HasValue Then + If LastDownloadDate.Value.AddDays(1) <= Now Then + dateAfter = LastDownloadDate.Value + Else + dateAfter = LastDownloadDate.Value.AddDays(-1) End If + dateBefore = Nothing + ElseIf dateAfter.HasValue And Not DownloadDateFrom.HasValue Then + If (LastDownloadDate.Value - dateAfter.Value).TotalDays > 1 Then dateAfter = dateAfter.Value.AddDays(1) End If + End If - Using b As New YTDLP.YTDLPBatch(Token) With {.TempPostsList = _TempPostsList} - b.Commands.Clear() - b.ChangeDirectory(cache) - b.Encoding = BatchExecutor.UnicodeEncoding - b.Execute(CreateYTCommand(cache.RootDirectory, URL, False, dateBefore, dateAfter)) - End Using + Using b As New YTDLP.YTDLPBatch(Token) With {.TempPostsList = _TempPostsList} + b.Commands.Clear() + b.ChangeDirectory(UserCache) + b.Encoding = BatchExecutor.UnicodeEncoding + b.Execute(CreateYTCommand(UserCache.RootDirectory, URL, False, dateBefore, dateAfter)) + End Using - ThrowAny(Token) + ThrowAny(Token) - Dim files As List(Of SFile) = SFile.GetFiles(cache, "*.json",, EDP.ReturnValue) - If files.ListExists Then - Dim j As EContainer - For Each file As SFile In files - j = JsonDocument.Parse(file.GetText, EDP.ReturnValue) - If j.ListExists Then - If j.Value("_type").StringToLower = "video" Then - If Not baseDataObtained Then - baseDataObtained = True - If ID.IsEmptyString Then - ID = j.Value("uploader_id") - If Not ID.IsEmptyString Then _ForceSaveUserInfo = True - End If - newName = j.Value("uploader") - If Not newName.IsEmptyString Then - If Not _TrueName = newName Then _ForceSaveUserInfo = True - _TrueName = newName - End If - newName = j.Value("creator") - If Not newName.IsEmptyString Then UserSiteName = newName + Dim files As List(Of SFile) = SFile.GetFiles(UserCache, "*.json",, EDP.ReturnValue) + If files.ListExists Then + Dim j As EContainer + For Each file As SFile In files + j = JsonDocument.Parse(file.GetText, EDP.ReturnValue) + If j.ListExists Then + If j.Value("_type").StringToLower = "video" Then + If Not baseDataObtained Then + baseDataObtained = True + If ID.IsEmptyString Then + ID = j.Value("uploader_id") + If Not ID.IsEmptyString Then _ForceSaveUserInfo = True End If - postID = j.Value("id") - If Not _TempPostsList.Contains(postID) Then - _TempPostsList.Add(postID) - Else - Exit Sub + newName = j.Value("uploader") + If Not newName.IsEmptyString Then + If Not _TrueName = newName Then _ForceSaveUserInfo = True + _TrueName = newName End If - title = j.Value("title").StringRemoveWinForbiddenSymbols - If title.IsEmptyString Or Not TitleUseNative Then + newName = j.Value("creator") + If Not newName.IsEmptyString Then UserSiteName = newName + End If + postID = j.Value("id") + If Not _TempPostsList.Contains(postID) Then + _TempPostsList.Add(postID) + Else + Exit Sub + End If + title = j.Value("title").StringRemoveWinForbiddenSymbols + If title.IsEmptyString Or Not TitleUseNative Then + title = postID + Else + If RemoveTagsFromTitle Then title = RegexReplace(title, RegexTagsReplacer) + title = title.StringTrim + If title.IsEmptyString Then title = postID - Else - If RemoveTagsFromTitle Then title = RegexReplace(title, RegexTagsReplacer) - title = title.StringTrim - If title.IsEmptyString Then - title = postID - ElseIf TitleAddVideoID Then - title &= $" ({postID})" - End If + ElseIf TitleAddVideoID Then + title &= $" ({postID})" End If - postDate = AConvert(Of Date)(j.Value("timestamp"), UnixDate32Provider, Nothing) - If Not postDate.HasValue Then postDate = AConvert(Of Date)(j.Value("upload_date"), SimpleDateConverter, Nothing) - Select Case CheckDatesLimit(postDate, SimpleDateConverter) - Case DateResult.Skip : Continue For - Case DateResult.Exit : Exit Sub - End Select - - postUrl = j.Value("webpage_url") - If postUrl.IsEmptyString Then postUrl = $"https://www.tiktok.com/@{Name}/video/{postID}" - _TempMediaList.Add(New UserMedia(postUrl, UserMedia.Types.Video) With { - .File = $"{title}.mp4", .Post = New UserPost(postID, postDate)}) + title = ChangeTitleRegex(title, titleRegex) End If - j.Dispose() + postDate = AConvert(Of Date)(j.Value("timestamp"), UnixDate32Provider, Nothing) + If Not postDate.HasValue Then postDate = AConvert(Of Date)(j.Value("upload_date"), SimpleDateConverter, Nothing) + Select Case CheckDatesLimit(postDate, SimpleDateConverter) + Case DateResult.Skip : Continue For + Case DateResult.Exit : Exit Sub + End Select + + postUrl = j.Value("webpage_url") + If postUrl.IsEmptyString Then postUrl = $"https://www.tiktok.com/@{Name}/video/{postID}" + _TempMediaList.Add(New UserMedia(postUrl, UserMedia.Types.Video) With { + .File = $"{title}.mp4", .Post = New UserPost(postID, postDate)}) End If - Next - End If - If _TempMediaList.Count > 0 Then LastDownloadDate = Now - Catch ex As Exception - ProcessException(ex, Token, $"data downloading error [{URL}]") - End Try - End Using + j.Dispose() + End If + Next + End If + If _TempMediaList.Count > 0 Then LastDownloadDate = Now + Catch ex As Exception + ProcessException(ex, Token, $"data downloading error [{URL}]") + End Try End Sub #End Region #Region "ReparseMissing" @@ -292,6 +350,7 @@ Namespace API.TikTok f = f.StringTrim If Not f.IsEmptyString Then If CBool(MySettings.TitleAddVideoID.Value) Then f &= $" ({m.File.Name})" + f = ChangeTitleRegex(f, GetTitleRegex) m.File.Name = f End If End If @@ -303,6 +362,15 @@ Namespace API.TikTok Optional ByVal EObj As Object = Nothing) As Integer Return 0 End Function +#End Region +#Region "IDisposable Support" + Protected Overrides Sub Dispose(ByVal disposing As Boolean) + If Not disposedValue And disposing Then + UserCache.DisposeIfReady(False) + UserCache = Nothing + End If + MyBase.Dispose(disposing) + End Sub #End Region End Class End Namespace \ No newline at end of file diff --git a/SCrawler/API/TikTok/UserExchangeOptions.vb b/SCrawler/API/TikTok/UserExchangeOptions.vb index 35b2f1f..456d0bc 100644 --- a/SCrawler/API/TikTok/UserExchangeOptions.vb +++ b/SCrawler/API/TikTok/UserExchangeOptions.vb @@ -15,18 +15,29 @@ Namespace API.TikTok Friend Property TitleUseNative As Boolean Friend Property TitleAddVideoID As Boolean + + Friend Property TitleUseRegexForTitle As Boolean + + Friend Property TitleUseRegexForTitle_Value As String + + Friend Property TitleUseGlobalRegexOptions As Boolean = True Private ReadOnly MySettings As SiteSettings Friend Sub New(ByVal u As UserData) MySettings = u.HOST.Source RemoveTagsFromTitle = u.RemoveTagsFromTitle TitleUseNative = u.TitleUseNative TitleAddVideoID = u.TitleAddVideoID + TitleUseRegexForTitle = u.TitleUseRegexForTitle + TitleUseRegexForTitle_Value = u.TitleUseRegexForTitle_Value + TitleUseGlobalRegexOptions = u.TitleUseGlobalRegexOptions End Sub Friend Sub New(ByVal s As SiteSettings) MySettings = s RemoveTagsFromTitle = s.RemoveTagsFromTitle.Value TitleUseNative = s.TitleUseNative.Value TitleAddVideoID = s.TitleAddVideoID.Value + TitleUseRegexForTitle = s.TitleUseRegexForTitle.Value + TitleUseRegexForTitle_Value = s.TitleUseRegexForTitle_Value.Value End Sub End Class End Namespace diff --git a/SCrawler/API/YouTube/SiteSettings.vb b/SCrawler/API/YouTube/SiteSettings.vb index 234eac6..02b3657 100644 --- a/SCrawler/API/YouTube/SiteSettings.vb +++ b/SCrawler/API/YouTube/SiteSettings.vb @@ -24,6 +24,8 @@ Namespace API.YouTube Friend ReadOnly Property DownloadCommunityImages As PropertyValue Friend ReadOnly Property DownloadCommunityVideos As PropertyValue + + Friend ReadOnly Property IgnoreCommunityErrors As PropertyValue Friend ReadOnly Property UseCookies As PropertyValue #End Region @@ -36,6 +38,7 @@ Namespace API.YouTube DownloadPlaylists = New PropertyValue(False) DownloadCommunityImages = New PropertyValue(False) DownloadCommunityVideos = New PropertyValue(False) + IgnoreCommunityErrors = New PropertyValue(False) UseCookies = New PropertyValue(False) _SubscriptionsAllowed = True UseNetscapeCookies = True diff --git a/SCrawler/API/YouTube/UserData.vb b/SCrawler/API/YouTube/UserData.vb index cae6ff9..b0e5006 100644 --- a/SCrawler/API/YouTube/UserData.vb +++ b/SCrawler/API/YouTube/UserData.vb @@ -333,7 +333,7 @@ Namespace API.YouTube Next End If End With - Else + ElseIf Not CBool(DirectCast(HOST.Source, SiteSettings).IgnoreCommunityErrors.Value) Then With j({"error"}) If .ListExists Then MyMainLOG = $"{ToStringForLog()} {errMsg} [{ .Value("code")}]: { .Value("message")}" End With diff --git a/SCrawler/My Project/AssemblyInfo.vb b/SCrawler/My Project/AssemblyInfo.vb index 613b694..3021810 100644 --- a/SCrawler/My Project/AssemblyInfo.vb +++ b/SCrawler/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + +