diff --git a/Changelog.md b/Changelog.md index b5cc29b..5484d6b 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,10 @@ +# 2024.4.14.0 + +*2024-04-14* + +- Fixed + - Facebook: can't get tokens + # 2024.4.13.0 *2024-04-13* diff --git a/SCrawler.YouTube/My Project/AssemblyInfo.vb b/SCrawler.YouTube/My Project/AssemblyInfo.vb index 6cfe06a..f823ea0 100644 --- a/SCrawler.YouTube/My Project/AssemblyInfo.vb +++ b/SCrawler.YouTube/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + + diff --git a/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb b/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb index fee7244..5226a6c 100644 --- a/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb +++ b/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb @@ -709,7 +709,7 @@ Namespace API.YouTube.Objects Bitrate = 0 _MediaType = UMTypes.Undefined If SelectedVideoIndex >= 0 Then - 'URGENT: 2023.3.4 -> 2023.7.6 + '2023.3.4 -> 2023.7.6 'cmd.StringAppend($"bv*[format_id={SelectedVideo.ID}]") cmd.StringAppend(SelectedVideo.ID) _Size = SelectedVideo.Size @@ -726,7 +726,7 @@ Namespace API.YouTube.Objects End If If SelectedAudioIndex >= 0 Then Dim atCodec$ - 'URGENT: 2023.3.4 -> 2023.7.6 + '2023.3.4 -> 2023.7.6 'cmd.StringAppend($"ba*[format_id={SelectedAudio.ID}]", "+") cmd.StringAppend(SelectedAudio.ID, "+") If OutputAudioCodec.StringToLower = ac3 Then @@ -769,7 +769,7 @@ Namespace API.YouTube.Objects subs = $"--write-subs --write-auto-subs --sub-format {OutputSubtitlesFormat.StringToLower} --sub-langs ""{subs}"" --convert-subs {OutputSubtitlesFormat.StringToLower}" End If If Not cmd.IsEmptyString Then - 'URGENT: 2023.3.4 -> 2023.7.6 + '2023.3.4 -> 2023.7.6 'cmd = $"yt-dlp -f ""{cmd}""" 'cmd = $"yt-dlp -f {cmd}" cmd = $"{YTDLP_NAME} -f {cmd}" diff --git a/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb b/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb index 71c317b..f9483d7 100644 --- a/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb +++ b/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + + diff --git a/SCrawler/API/Facebook/Declarations.vb b/SCrawler/API/Facebook/Declarations.vb index de6d84e..fbeaf13 100644 --- a/SCrawler/API/Facebook/Declarations.vb +++ b/SCrawler/API/Facebook/Declarations.vb @@ -11,9 +11,8 @@ Imports PersonalUtilities.Functions.XML.Base Imports PersonalUtilities.Functions.RegularExpressions Namespace API.Facebook Friend Module Declarations - Friend ReadOnly Regex_UserToken_dtsg As RParams = RParams.DMS("DTSGInitialData.:.?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue) - Friend ReadOnly Regex_UserToken_lsd As RParams = RParams.DMS("LSD.:.?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue) Friend ReadOnly Regex_UserID As RParams = RParams.DMS("userid.:.(\d+)", 1, RegexOptions.IgnoreCase, EDP.ReturnValue) + Friend ReadOnly Regex_AppID As RParams = RParams.DMS("APP_ID.:.(\d+)", 1, RegexOptions.IgnoreCase, EDP.ReturnValue) Friend ReadOnly Regex_Photos_by As RParams = RParams.DMS("photos_by"",""id"":""([^""]+)", 1, EDP.ReturnValue) Friend ReadOnly Regex_FileName As RParams = RParams.DM("([^/\?]+\..{3,4})(?=(\?|\Z))", 0, EDP.ReturnValue) diff --git a/SCrawler/API/Facebook/SiteSettings.vb b/SCrawler/API/Facebook/SiteSettings.vb index f1c5d43..0439de1 100644 --- a/SCrawler/API/Facebook/SiteSettings.vb +++ b/SCrawler/API/Facebook/SiteSettings.vb @@ -18,7 +18,7 @@ Namespace API.Facebook #Region "Auth" Friend ReadOnly Property Header_Accept As PropertyValue - + Friend Overrides ReadOnly Property HH_IG_APP_ID As PropertyValue Get Return __HH_IG_APP_ID @@ -74,7 +74,7 @@ Namespace API.Facebook #End Region #Region "BaseAuthExists, GetUserUrl, GetUserPostUrl, IsMyUser, IsMyImageVideo" Friend Overrides Function BaseAuthExists() As Boolean - Return Responser.CookiesExists And ACheck(HH_IG_APP_ID.Value) And CBool(DownloadData_Impl.Value) + Return Responser.CookiesExists And CBool(DownloadData_Impl.Value) 'And ACheck(HH_IG_APP_ID.Value) End Function Friend Overrides Function GetUserUrl(ByVal User As IPluginContentProvider) As String Return DirectCast(User, UserData).GetProfileUrl diff --git a/SCrawler/API/Facebook/UserData.vb b/SCrawler/API/Facebook/UserData.vb index 00057a6..5c5eff0 100644 --- a/SCrawler/API/Facebook/UserData.vb +++ b/SCrawler/API/Facebook/UserData.vb @@ -136,6 +136,7 @@ Namespace API.Facebook Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken) If CBool(MySettings.DownloadData_Impl.Value) Then Try + If Responser.Headers.Value(IG.Header_IG_APP_ID).IsEmptyString Then Responser.Headers.Remove(IG.Header_IG_APP_ID) ResetBaseTokens() GetUserTokens(Token) LoadSavePostsKV(True) @@ -529,8 +530,14 @@ Namespace API.Facebook Dim r$ = resp.GetResponse(URL) If Not r.IsEmptyString Then If Responser.CookiesExists Then Responser.Cookies.Update(resp.Cookies) - Token_dtsg = RegexReplace(r, Regex_UserToken_dtsg) - Token_lsd = RegexReplace(r, Regex_UserToken_lsd) + ParseTokens(r, 0) + Dim app_id$ = RegexReplace(r, Regex_AppID) + If Not app_id.IsEmptyString Then + If Not AEquals(Of String)(MySettings.HH_IG_APP_ID.Value, app_id) Then + MySettings.HH_IG_APP_ID.Value = app_id + Responser.Headers.Add(IG.Header_IG_APP_ID, app_id) + End If + End If Token_Photosby = RegexReplace(r, Regex_Photos_by) If StoryBucket.IsEmptyString Then StoryBucket = RegexReplace(r, Regex_StoryBucket) If ID.IsEmptyString Then @@ -568,14 +575,14 @@ Namespace API.Facebook .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchSite, "none")) .Add("Sec-Fetch-User", "?1") .Add("Upgrade-Insecure-Requests", 1) - Dim h$ = Responser.Headers.Value(IG.Header_Browser) - If Not h.IsEmptyString Then .Add(IG.Header_Browser, h) - h = Responser.Headers.Value(IG.Header_BrowserExt) - If Not h.IsEmptyString Then .Add(IG.Header_BrowserExt, h) - h = Responser.Headers.Value(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatform)) - If Not h.IsEmptyString Then .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatform, h)) - h = Responser.Headers.Value(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatformVersion)) - If Not h.IsEmptyString Then .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatformVersion, h)) + Dim cloneHeader As Action(Of String) = Sub(ByVal hName As String) + Dim hValue$ = Responser.Headers.Value(hName) + If Not hValue.IsEmptyString Then .Add(hName, hValue) + End Sub + cloneHeader.Invoke(IG.Header_Browser) + cloneHeader.Invoke(IG.Header_BrowserExt) + cloneHeader.Invoke(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatform).Name) + cloneHeader.Invoke(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatformVersion).Name) .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaMobile, "?0")) .Add("Sec-Ch-Ua-Model", "") End With diff --git a/SCrawler/API/Instagram/Declarations.vb b/SCrawler/API/Instagram/Declarations.vb index 73958ff..00c5fbf 100644 --- a/SCrawler/API/Instagram/Declarations.vb +++ b/SCrawler/API/Instagram/Declarations.vb @@ -18,6 +18,8 @@ Namespace API.Instagram Friend ReadOnly ObtainMedia_SizeFuncPic_RegexP As RParams = RParams.DMS("_p(\d+)x(\d+)", 1, EDP.ReturnValue) Friend ReadOnly ObtainMedia_SizeFuncPic_RegexS As RParams = RParams.DMS("_s(\d+)x(\d+)", 1, EDP.ReturnValue) Friend Const PageTokenRegexPatternDefault As String = "\[\],{""token"":""(.*?)""},\d+\]" + Friend ReadOnly Regex_UserToken_dtsg As RParams = RParams.DMS("DTSGInitialData["":,.\[\]]*?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue) + Friend ReadOnly Regex_UserToken_lsd As RParams = RParams.DMS("LSD["":,.\[\]]*?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue) Friend Sub UpdateResponser(ByVal Source As IResponse, ByRef Destination As Responser, ByVal UpdateWwwClaim As Boolean) Const r_wwwClaimName$ = "x-ig-set-www-claim" Const r_tokenName$ = SiteSettings.Header_CSRF_TOKEN_COOKIE diff --git a/SCrawler/API/Instagram/UserData.GQL.vb b/SCrawler/API/Instagram/UserData.GQL.vb index 472ea82..a436501 100644 --- a/SCrawler/API/Instagram/UserData.GQL.vb +++ b/SCrawler/API/Instagram/UserData.GQL.vb @@ -297,35 +297,49 @@ Namespace API.Instagram End With End With Dim r$ = Responser.GetResponse(MySiteSettings.GetUserUrl(Me)) + ParseTokens(r, 0) + Catch ex As Exception + Finally + ChangeResponserMode(_UseGQL, Not _UseGQL) + End Try + End Sub + Protected Sub ParseTokens(ByVal r As String, ByVal Attempt As Integer) + Try If Not r.IsEmptyString Then - Dim rr As RParams = RParams.DM(PageTokenRegexPatternDefault, 0, RegexReturn.List, EDP.ReturnValue) - Dim tokens As List(Of String) = RegexReplace(r, rr) - Dim tt$, ttVal$ - If tokens.ListExists Then - With rr - .Match = Nothing - .MatchSub = 1 - .WhatGet = RegexReturn.Value - End With - For Each tt In tokens - If Not Token_lsd.IsEmptyString And Not Token_dtsg.IsEmptyString Then - Exit For - Else - ttVal = RegexReplace(tt, rr) - If Not ttVal.IsEmptyString Then - If ttVal.Contains(":") Then - If Token_dtsg.IsEmptyString Then Token_dtsg = ttVal + ResetBaseTokens() + Select Case Attempt + Case 0 + Dim rr As RParams = RParams.DM(PageTokenRegexPatternDefault, 0, RegexReturn.List, EDP.ReturnValue) + Dim tokens As List(Of String) = RegexReplace(r, rr) + Dim tt$, ttVal$ + If tokens.ListExists Then + With rr + .Match = Nothing + .MatchSub = 1 + .WhatGet = RegexReturn.Value + End With + For Each tt In tokens + If Not Token_lsd.IsEmptyString And Not Token_dtsg.IsEmptyString Then + Exit For Else - If Token_lsd.IsEmptyString Then Token_lsd = ttVal + ttVal = RegexReplace(tt, rr) + If Not ttVal.IsEmptyString Then + If ttVal.Contains(":") Then + If Token_dtsg.IsEmptyString Then Token_dtsg = ttVal + Else + If Token_lsd.IsEmptyString Then Token_lsd = ttVal + End If + End If End If - End If + Next End If - Next - End If + Case 1 + Token_dtsg = RegexReplace(r, Regex_UserToken_dtsg) + Token_lsd = RegexReplace(r, Regex_UserToken_lsd) + End Select + If Not ValidateBaseTokens() And Attempt = 0 Then ParseTokens(r, Attempt + 1) End If - Catch ex As Exception - Finally - ChangeResponserMode(_UseGQL, Not _UseGQL) + Catch End Try End Sub #End Region diff --git a/SCrawler/API/PornHub/UserData.vb b/SCrawler/API/PornHub/UserData.vb index f0e71ab..d529a18 100644 --- a/SCrawler/API/PornHub/UserData.vb +++ b/SCrawler/API/PornHub/UserData.vb @@ -29,7 +29,6 @@ Namespace API.PornHub Private Const Name_DownloadFavorite As String = "DownloadFavorite" Private Const Name_DownloadGifs As String = "DownloadGifs" Private Const Name_DownloadPhotoOnlyFromModelHub As String = "DownloadPhotoOnlyFromModelHub" - Private Const Name_IsUser As String = "IsUser" #End Region #Region "Structures" Private Structure FlashVar : Implements IRegExCreator @@ -254,14 +253,7 @@ Namespace API.PornHub DownloadFavorite = .Value(Name_DownloadFavorite).FromXML(Of Boolean)(False) DownloadGifs = .Value(Name_DownloadGifs).FromXML(Of Integer)(False) DownloadPhotoOnlyFromModelHub = .Value(Name_DownloadPhotoOnlyFromModelHub).FromXML(Of Boolean)(True) - If .Contains(Name_SiteMode) Then - SiteMode = .Value(Name_SiteMode).FromXML(Of Integer)(SiteModes.User) - Else - 'TODELETE: PornHub 'IsUser' 20231113 -#Disable Warning BC40008 - SiteMode = IIf(.Value(Name_IsUser).FromXML(Of Boolean)(True), SiteModes.User, SiteModes.Search) -#Enable Warning - End If + SiteMode = .Value(Name_SiteMode).FromXML(Of Integer)(SiteModes.User) UpdateUserOptions() Else If UpdateUserOptions() Then .Value(Name_LabelsName) = LabelsString @@ -404,7 +396,6 @@ Namespace API.PornHub Dim r$ = Responser.GetResponse(URL) If Not r.IsEmptyString Then Dim l As List(Of UserVideo) = RegexFields(Of UserVideo)(r, {RegexUserVideos}, {6, 7, 3, 10}) - 'URGENT: PornHub: changed list trimming 'If l.ListExists And Not SiteMode = SiteModes.Playlists Then l = l.ListTake(3, l.Count).ToList If l.ListExists And Not SiteMode = SiteModes.Playlists Then l = l.ListTake(1, l.Count).ToList If l.ListExists Then diff --git a/SCrawler/API/ThreadsNet/SiteSettings.vb b/SCrawler/API/ThreadsNet/SiteSettings.vb index 493de8e..8bbfa18 100644 --- a/SCrawler/API/ThreadsNet/SiteSettings.vb +++ b/SCrawler/API/ThreadsNet/SiteSettings.vb @@ -103,8 +103,6 @@ Namespace API.ThreadsNet With Responser .Accept = "*/*" - 'URGENT: remove after debug - .DeclaredError = EDP.SendToLog + EDP.ThrowException If .UserAgentExists Then useragent = .UserAgent With .Headers If .Count > 0 Then @@ -125,6 +123,8 @@ Namespace API.ThreadsNet .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchMode, "cors")) .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchSite, "same-origin")) .Add("Sec-Fetch-User", "?1") + .Add("dht", 1) + .Add("drp", 1) .Add(Instagram.UserData.GQL_HEADER_FB_FRINDLY_NAME, "BarcelonaProfileThreadsTabRefetchableQuery") End With .CookiesExtractMode = Responser.CookiesExtractModes.Any diff --git a/SCrawler/API/ThreadsNet/UserData.vb b/SCrawler/API/ThreadsNet/UserData.vb index 3a50282..08b963c 100644 --- a/SCrawler/API/ThreadsNet/UserData.vb +++ b/SCrawler/API/ThreadsNet/UserData.vb @@ -157,38 +157,22 @@ Namespace API.ThreadsNet Private Function UpdateCredentials(Optional ByVal e As ErrorsDescriber = Nothing) As Boolean Dim URL$ = $"https://www.threads.net/@{NameTrue}" ResetBaseTokens() + Dim headers As New HttpHeaderCollection + headers.AddRange(Responser.Headers) Try - Responser.Method = "GET" - Responser.Referer = URL - Responser.Headers.Remove(GQL_HEADER_FB_LSD) + With Responser + .Method = "GET" + .Referer = URL + With .Headers + .Remove(GQL_HEADER_FB_LSD) + .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchDest, "document")) + .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchMode, "navigate")) + End With + End With WaitTimer() Dim r$ = Responser.GetResponse(URL,, EDP.ThrowException) - Dim rr As RParams - Dim tt$, ttVal$ If Not r.IsEmptyString Then - rr = RParams.DM(Instagram.PageTokenRegexPatternDefault, 0, RegexReturn.List, EDP.ReturnValue) - Dim tokens As List(Of String) = RegexReplace(r, rr) - If tokens.ListExists Then - With rr - .Match = Nothing - .MatchSub = 1 - .WhatGet = RegexReturn.Value - End With - For Each tt In tokens - If Not Token_dtsg.IsEmptyString And Not Token_lsd.IsEmptyString Then - Exit For - Else - ttVal = RegexReplace(tt, rr) - If Not ttVal.IsEmptyString Then - If ttVal.Contains(":") Then - If Token_dtsg.IsEmptyString Then Token_dtsg = ttVal - Else - If Token_lsd.IsEmptyString Then Token_lsd = ttVal - End If - End If - End If - Next - End If + ParseTokens(r, 0) If ID.IsEmptyString Then ID = RegexReplace(r, RParams.DMS("""props"":\{""user_id"":""(\d+)""\},", 1, EDP.ReturnValue)) End If Return Valid @@ -204,6 +188,12 @@ Namespace API.ThreadsNet 'LogError(ex, $"failed to update some{IIf(notFound.IsEmptyString, String.Empty, $" ({notFound})")} credentials", e) LogError(eex, String.Empty, e) Return False + Finally + If headers.ListExists Then + Responser.Headers.Clear() + Responser.Headers.AddRange(headers) + headers.Dispose() + End If End Try End Function #End Region diff --git a/SCrawler/API/Xhamster/UserData.vb b/SCrawler/API/Xhamster/UserData.vb index 9d2cf8e..8a210d0 100644 --- a/SCrawler/API/Xhamster/UserData.vb +++ b/SCrawler/API/Xhamster/UserData.vb @@ -280,32 +280,18 @@ Namespace API.Xhamster End If End If - 'TODELETE: xHamster remove old container nodes attachments - If IsSavedPosts Then URL = $"https://xhamster.com/my/favorites/{IIf(IsVideo, "videos", "photos-and-galleries")}{IIf(Page = 1, String.Empty, $"/{Page}")}" containerNodes.Add(If(IsVideo, {"favoriteVideoListComponent", "models"}, {"favoritesGalleriesAndPhotosCollection"})) ElseIf IsChannel Then URL = $"https://xhamster.com/channels/{TrueName}/newest{IIf(Page = 1, String.Empty, $"/{Page}")}" - 'containerNodes.Add({"trendingVideoListComponent", "models"}) - 'containerNodes.Add({"pagesCategoryComponent", "trendingVideoListProps", "models"}) ElseIf SiteMode = SiteModes.Search Then URL = GetNonUserUrl(Page) containerNodes.Add({"searchResult", "models"}) ElseIf IsCreator Or SiteMode = SiteModes.Tags Or SiteMode = SiteModes.Categories Or SiteMode = SiteModes.Pornstars Then URL = GetNonUserUrl(Page) - 'If SiteMode = SiteModes.Pornstars Then - ' containerNodes.Add({"trendingVideoListComponent", "models"}) - ' containerNodes.Add({"pagesCategoryComponent", "trendingVideoListProps", "models"}) - 'Else - ' containerNodes.Add({"pagesCategoryComponent", "trendingVideoListProps", "models"}) - ' containerNodes.Add({"trendingVideoListComponent", "models"}) - 'End If - 'containerNodes.Add({"trendingVideoSectionComponent", "videoModels"}) Else URL = $"https://xhamster.com/users/{TrueName}/{IIf(IsVideo, "videos", "photos")}{IIf(Page = 1, String.Empty, $"/{Page}")}" - 'containerNodes.Add({If(IsVideo, "userVideoCollection", "userGalleriesCollection")}) - 'containerNodes.Add(If(IsVideo, {"videoListComponent", "models"}, {"userGalleriesCollection"})) End If ThrowAny(Token) diff --git a/SCrawler/My Project/AssemblyInfo.vb b/SCrawler/My Project/AssemblyInfo.vb index 55df751..d9f7fca 100644 --- a/SCrawler/My Project/AssemblyInfo.vb +++ b/SCrawler/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + + diff --git a/Tools/NET.FrameworkVersion.ps1 b/Tools/NET.FrameworkVersion.ps1 index f291775..afebac3 100644 --- a/Tools/NET.FrameworkVersion.ps1 +++ b/Tools/NET.FrameworkVersion.ps1 @@ -1,4 +1,5 @@ -$arr=Get-ChildItem -Path Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\.NETFramework\|Select-Object Name +try{Set-ExecutionPolicy Unrestricted CurrentUser -ErrorAction SilentlyContinue}catch{} +$arr=Get-ChildItem -Path Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\.NETFramework\|Select-Object Name $mv=0 $found=0 foreach($v in $arr){if($v.Name -match "v4.0."){$mv=$v.Name}}