diff --git a/Changelog.md b/Changelog.md index 7724506..8f0cbfd 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,18 @@ +# 2024.5.4.0 + +*2024-05-04* + +- Added + - YouTube (standalone app): setting to remove specific characters (`Defaults` - `Remove characters`) + - Instagram: simplify the `Connection closed` error + - Users search: add 'FriendlyName' to search results +- Fixed + - YouTube (standalone app): incorrect download processing when the file name ends with a dot (Issue #188) + - The program is freezes when editing users in some cases + - Sites + - Reddit: token update error + - Threads: unable to obtain credentials (ID) + # 2024.4.26.0 *2024-04-26* diff --git a/SCrawler.YouTube/Base/YouTubeSettings.vb b/SCrawler.YouTube/Base/YouTubeSettings.vb index b238c97..12504af 100644 --- a/SCrawler.YouTube/Base/YouTubeSettings.vb +++ b/SCrawler.YouTube/Base/YouTubeSettings.vb @@ -243,6 +243,9 @@ Namespace API.YouTube.Base Friend ReadOnly Property ProgramDescription As XMLValue(Of String) + + Friend ReadOnly Property FileRemoveCharacters As XMLValue(Of String) #End Region #Region "Defaults ChannelsDownload" - - + + diff --git a/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb b/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb index f9483d7..ef02e8f 100644 --- a/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb +++ b/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + + diff --git a/SCrawler/API/Instagram/UserData.vb b/SCrawler/API/Instagram/UserData.vb index 9d6500c..a71b593 100644 --- a/SCrawler/API/Instagram/UserData.vb +++ b/SCrawler/API/Instagram/UserData.vb @@ -1269,6 +1269,9 @@ NextPageBlock: ElseIf Responser.StatusCode = 560 Or Responser.StatusCode = HttpStatusCode.InternalServerError Then '560, 500 MySiteSettings.SkipUntilNextSession = True Err5xx = Responser.StatusCode + ElseIf Responser.StatusCode = -1 And Responser.Status = -1 Then + MySiteSettings.SkipUntilNextSession = True + Err5xx = Responser.StatusCode Else MyMainLOG = $"Something is wrong. Your credentials may have expired [{CInt(Responser.StatusCode)}/{CInt(Responser.Status)}]: {ToString()} [{s}]" DisableSection(s) diff --git a/SCrawler/API/Reddit/SiteSettings.vb b/SCrawler/API/Reddit/SiteSettings.vb index be8e845..fff9d0c 100644 --- a/SCrawler/API/Reddit/SiteSettings.vb +++ b/SCrawler/API/Reddit/SiteSettings.vb @@ -10,6 +10,7 @@ Imports SCrawler.API.Base Imports SCrawler.Plugin Imports SCrawler.Plugin.Attributes Imports PersonalUtilities.Tools.Web.Clients +Imports PersonalUtilities.Tools.Web.Clients.Base Imports PersonalUtilities.Tools.Web.Documents.JSON Imports PersonalUtilities.Functions.XML Imports PersonalUtilities.Functions.RegularExpressions @@ -34,6 +35,8 @@ Namespace API.Reddit "You can find different tokens in the responses. Make sure that bearer token belongs to Reddit and not RedGifs." & vbCr & "There is not need to add a token if you are not using cookies to download the timeline.", IsAuth:=True)> Friend ReadOnly Property BearerToken As PropertyValue + + Private ReadOnly Property BearerTokenUseCurl As PropertyValue #Region "TokenUpdateInterval" @@ -82,6 +85,7 @@ Namespace API.Reddit ApiClientID = New PropertyValue(String.Empty, GetType(String)) ApiClientSecret = New PropertyValue(String.Empty, GetType(String)) BearerToken = New PropertyValue(token, GetType(String), Sub(v) Responser.Headers.Add(DeclaredNames.Header_Authorization, v)) + BearerTokenUseCurl = New PropertyValue(True) TokenUpdateInterval = New PropertyValue(60 * 12) TokenUpdateIntervalProvider = New TokenRefreshIntervalProvider BearerTokenDateUpdate = New PropertyValue(Now.AddYears(-1)) @@ -269,29 +273,51 @@ Namespace API.Reddit result = False Dim r$ = String.Empty Dim c% = 0 - Dim _found As Boolean + Dim useCurl As Boolean = Settings.CurlFile.Exists And CBool(BearerTokenUseCurl.Value) + Dim curlUsed As Boolean = useCurl Do c += 1 Using resp As New Responser With { .Method = "POST", - .ProcessExceptionDecision = Function(status, obj, ee) If(status.StatusCode = 429, EDP.ReturnValue, ee) + .ProcessExceptionDecision = Function(ByVal status As IResponserStatus, ByVal nullArg As Object, ByVal currErr As ErrorsDescriber) As ErrorsDescriber + If status.StatusCode = 429 Then + useCurl = False + Return EDP.ReturnValue + ElseIf status.StatusCode = Net.HttpStatusCode.Forbidden And Not useCurl And Settings.CurlFile.Exists Then + useCurl = True + Return EDP.ReturnValue + Else + Return currErr + End If + End Function } With resp - With .PayLoadValues - .Add("grant_type", "password") - .Add("username", UserName) - .Add("password", Password) - End With - .CredentialsUserName = ClientID - .CredentialsPassword = ClientSecret - .PreAuthenticate = True + If useCurl Then + If Settings.CurlFile.Exists Then + curlUsed = True + .Mode = Responser.Modes.Curl + .CurlPath = Settings.CurlFile + .CurlArgumentsLeft = $"-d ""grant_type=password&username={UserName}&password={Password}"" --user ""{ClientID}:{ClientSecret}""" + Else + Throw New ArgumentNullException("cUrl file", "The path to the cUrl file is not specified") + End If + Else + .Mode = Responser.Modes.Default + With .PayLoadValues + .Add("grant_type", "password") + .Add("username", UserName) + .Add("password", Password) + End With + .CredentialsUserName = ClientID + .CredentialsPassword = ClientSecret + .PreAuthenticate = True + End If End With r = resp.GetResponse("https://www.reddit.com/api/v1/access_token",, EDP.ThrowException) End Using If Not r.IsEmptyString Then Using j As EContainer = JsonDocument.Parse(r) If j.ListExists Then - _found = True Dim newToken$ = j.Value("access_token") If Not newToken.IsEmptyString Then BearerToken.Value = $"Bearer {newToken}" @@ -302,7 +328,7 @@ Namespace API.Reddit End If End Using End If - Loop While c < 5 And Not _found + Loop While c < 5 And Not result End If Return result Catch ex As Exception diff --git a/SCrawler/API/ThreadsNet/SiteSettings.vb b/SCrawler/API/ThreadsNet/SiteSettings.vb index a5df0db..acd7017 100644 --- a/SCrawler/API/ThreadsNet/SiteSettings.vb +++ b/SCrawler/API/ThreadsNet/SiteSettings.vb @@ -40,10 +40,10 @@ Namespace API.ThreadsNet Friend ReadOnly Property HH_ASBD_ID As PropertyValue - Private ReadOnly Property HH_BROWSER As PropertyValue + Friend ReadOnly Property HH_BROWSER As PropertyValue - Private ReadOnly Property HH_BROWSER_EXT As PropertyValue + Friend ReadOnly Property HH_BROWSER_EXT As PropertyValue Friend ReadOnly Property HH_PLATFORM_VER As PropertyValue @@ -127,9 +127,10 @@ Namespace API.ThreadsNet .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchMode, "cors")) .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchSite, "same-origin")) .Add("Sec-Fetch-User", "?1") - .Add("dht", 1) + .Add("dnt", 1) .Add("drp", 1) .Add(Instagram.UserData.GQL_HEADER_FB_FRINDLY_NAME, "BarcelonaProfileThreadsTabRefetchableQuery") + .Remove("dht") End With .CookiesExtractMode = Responser.CookiesExtractModes.Any .CookiesUpdateMode = CookieKeeper.UpdateModes.ReplaceByNameAll diff --git a/SCrawler/API/ThreadsNet/UserData.vb b/SCrawler/API/ThreadsNet/UserData.vb index 08b963c..1009f97 100644 --- a/SCrawler/API/ThreadsNet/UserData.vb +++ b/SCrawler/API/ThreadsNet/UserData.vb @@ -164,16 +164,28 @@ Namespace API.ThreadsNet .Method = "GET" .Referer = URL With .Headers - .Remove(GQL_HEADER_FB_LSD) + .Clear() + .Add("dnt", 1) + .Add("drp", 1) + .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.Authority, "www.threads.net")) + .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.Origin, "https://www.threads.net")) + .Add("Sec-Ch-Ua-Model", "") + .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaMobile, "?0")) + .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatform, """Windows""")) .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchDest, "document")) .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchMode, "navigate")) + .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchSite, "none")) + .Add("Upgrade-Insecure-Requests", 1) + .Add("Sec-Fetch-User", "?1") + .Add(IGS.Header_Browser, MySettings.HH_BROWSER.Value) + .Add(IGS.Header_BrowserExt, MySettings.HH_BROWSER_EXT.Value) End With End With WaitTimer() Dim r$ = Responser.GetResponse(URL,, EDP.ThrowException) If Not r.IsEmptyString Then ParseTokens(r, 0) - If ID.IsEmptyString Then ID = RegexReplace(r, RParams.DMS("""props"":\{""user_id"":""(\d+)""\},", 1, EDP.ReturnValue)) + If ID.IsEmptyString Then ID = RegexReplace(r, RParams.DMS("""props"":\{""user_id"":""(\d+)""", 1, EDP.ReturnValue)) End If Return Valid Catch ex As Exception diff --git a/SCrawler/Download/Automation/AutoDownloader.vb b/SCrawler/Download/Automation/AutoDownloader.vb index 2f5eadb..fa4cbe4 100644 --- a/SCrawler/Download/Automation/AutoDownloader.vb +++ b/SCrawler/Download/Automation/AutoDownloader.vb @@ -572,7 +572,19 @@ Namespace DownloadObjects With Downloader .AutoDownloaderWorking = True If .Downloaded.Count > 0 Then .Downloaded.RemoveAll(Function(u) Keys.Contains(u.Key)) : .InvokeDownloadsChangeEvent() - Do : Try : doRound += 1 : .AddRange(users, True) : Exit Do : Catch iex As IndexOutOfRangeException : Thread.Sleep(200) : End Try : Loop While doRound < doLim + Do + Try + doRound += 1 + .AddRange(users, True) + Exit Do + Catch iex As Exception + If doRound = doLim Then + Throw iex + Else + Thread.Sleep(200) + End If + End Try + Loop While doRound <= doLim While .Working Or .Count > 0 : notify.Invoke() : Thread.Sleep(200) : End While .AutoDownloaderWorking = False notify.Invoke @@ -586,7 +598,7 @@ Namespace DownloadObjects End With End If Catch ex As Exception - ErrorsDescriber.Execute(EDP.SendToLog, ex, "[AutoDownloader.Download]") + ErrorsDescriber.Execute(EDP.SendToLog, ex, $"[AutoDownloader.Download({Name})]") Finally Keys.Clear() LastDownloadDate = Now diff --git a/SCrawler/Download/Feed/FeedSpecialCollection.vb b/SCrawler/Download/Feed/FeedSpecialCollection.vb index 71e1d6e..417ba28 100644 --- a/SCrawler/Download/Feed/FeedSpecialCollection.vb +++ b/SCrawler/Download/Feed/FeedSpecialCollection.vb @@ -236,7 +236,7 @@ Namespace DownloadObjects Friend Sub UpdateUsers(ByVal InitialUser As UserInfo, ByVal NewUser As UserInfo) Try Load() - If Count > 0 Then + If Count > 0 AndAlso Not UserInfo.ExactEquals(InitialUser, NewUser) Then Feeds.ForEach(Sub(f) f.UpdateUsers(InitialUser, NewUser)) If Downloader.Files.Count > 0 Then PendingUsersToUpdate.Add(New KeyValuePair(Of UserInfo, UserInfo)(InitialUser, NewUser)) diff --git a/SCrawler/Download/TDownloader.vb b/SCrawler/Download/TDownloader.vb index 372e749..d3aa598 100644 --- a/SCrawler/Download/TDownloader.vb +++ b/SCrawler/Download/TDownloader.vb @@ -191,9 +191,9 @@ Namespace DownloadObjects End Sub Private _FilesUpdating As Boolean = False Friend Sub FilesUpdatePendingUsers() - _FilesUpdating = True Try If Files.Count > 0 Then + _FilesUpdating = True With Settings.Feeds Dim pUsers As List(Of KeyValuePair(Of UserInfo, UserInfo)) Dim pendingUser As KeyValuePair(Of UserInfo, UserInfo) @@ -214,19 +214,21 @@ Namespace DownloadObjects Next End If End With - If changed Then FilesSave() + If changed Then _FilesUpdating = False : FilesSave() Next pUsers.Clear() End While End With + _FilesUpdating = False End If Catch aex As ArgumentOutOfRangeException + _FilesUpdating = False Catch iex As IndexOutOfRangeException + _FilesUpdating = False Catch ex As Exception + _FilesUpdating = False ErrorsDescriber.Execute(EDP.SendToLog, ex, "[TDownloader.FilesUpdatePendingUsers]") MainFrameObj.UpdateLogButton() - Finally - _FilesUpdating = False End Try End Sub Friend Sub ClearSessions() diff --git a/SCrawler/My Project/AssemblyInfo.vb b/SCrawler/My Project/AssemblyInfo.vb index 3912a5e..c9017bd 100644 --- a/SCrawler/My Project/AssemblyInfo.vb +++ b/SCrawler/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + + diff --git a/SCrawler/UserSearchForm.vb b/SCrawler/UserSearchForm.vb index ae80e63..fc54458 100644 --- a/SCrawler/UserSearchForm.vb +++ b/SCrawler/UserSearchForm.vb @@ -31,6 +31,7 @@ Friend Class UserSearchForm Else If User.IncludedInCollection Then Text &= $"[{User.CollectionName}] " Text &= $"[{User.Site}] [{User.Name}]" + If Not User.FriendlyName.IsEmptyString Then Text &= $" ({User.FriendlyName})" End If End Sub Private Function CompareTo(ByVal Other As SearchResult) As Integer Implements IComparable(Of SearchResult).CompareTo