2024.4.14.0

Delete old notes and comments API.Facebook: add app-id extraction from page; remove app-id from site requirements; update tokens parsing; update tokens regex API.Instagram: add default function to parse tokens
AAndyProgram · Apr 14, 2024 · 5b5857e · 5b5857e
1 parent 46372ec
commit 5b5857e
Show file tree

Hide file tree

Showing 15 changed files with 99 additions and 102 deletions.
diff --git a/Changelog.md b/Changelog.md
@@ -1,3 +1,10 @@
+# 2024.4.14.0
+
+*2024-04-14*
+
+- Fixed
+ - Facebook: can't get tokens
+
 # 2024.4.13.0
 
 *2024-04-13*

diff --git a/SCrawler.YouTube/My Project/AssemblyInfo.vb b/SCrawler.YouTube/My Project/AssemblyInfo.vb
@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2024.4.13.0")>
-<Assembly: AssemblyFileVersion("2024.4.13.0")>
+<Assembly: AssemblyVersion("2024.4.14.0")>
+<Assembly: AssemblyFileVersion("2024.4.14.0")>
 <Assembly: NeutralResourcesLanguage("en")>
diff --git a/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb b/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb
@@ -709,7 +709,7 @@ Namespace API.YouTube.Objects
  Bitrate = 0
  _MediaType = UMTypes.Undefined
  If SelectedVideoIndex >= 0 Then
- 'URGENT: 2023.3.4 -> 2023.7.6
+ '2023.3.4 -> 2023.7.6
  'cmd.StringAppend($"bv*[format_id={SelectedVideo.ID}]")
  cmd.StringAppend(SelectedVideo.ID)
  _Size = SelectedVideo.Size
@@ -726,7 +726,7 @@ Namespace API.YouTube.Objects
  End If
  If SelectedAudioIndex >= 0 Then
  Dim atCodec$
- 'URGENT: 2023.3.4 -> 2023.7.6
+ '2023.3.4 -> 2023.7.6
  'cmd.StringAppend($"ba*[format_id={SelectedAudio.ID}]", "+")
  cmd.StringAppend(SelectedAudio.ID, "+")
  If OutputAudioCodec.StringToLower = ac3 Then
@@ -769,7 +769,7 @@ Namespace API.YouTube.Objects
  subs = $"--write-subs --write-auto-subs --sub-format {OutputSubtitlesFormat.StringToLower} --sub-langs ""{subs}"" --convert-subs {OutputSubtitlesFormat.StringToLower}"
  End If
  If Not cmd.IsEmptyString Then
- 'URGENT: 2023.3.4 -> 2023.7.6
+ '2023.3.4 -> 2023.7.6
  'cmd = $"yt-dlp -f ""{cmd}"""
  'cmd = $"yt-dlp -f {cmd}"
  cmd = $"{YTDLP_NAME} -f {cmd}"

diff --git a/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb b/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb
@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2024.4.13.0")>
-<Assembly: AssemblyFileVersion("2024.4.13.0")>
+<Assembly: AssemblyVersion("2024.4.14.0")>
+<Assembly: AssemblyFileVersion("2024.4.14.0")>
 <Assembly: NeutralResourcesLanguage("en")>
diff --git a/SCrawler/API/Facebook/Declarations.vb b/SCrawler/API/Facebook/Declarations.vb
@@ -11,9 +11,8 @@ Imports PersonalUtilities.Functions.XML.Base
 Imports PersonalUtilities.Functions.RegularExpressions
 Namespace API.Facebook
  Friend Module Declarations
- Friend ReadOnly Regex_UserToken_dtsg As RParams = RParams.DMS("DTSGInitialData.:.?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue)
- Friend ReadOnly Regex_UserToken_lsd As RParams = RParams.DMS("LSD.:.?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue)
  Friend ReadOnly Regex_UserID As RParams = RParams.DMS("userid.:.(\d+)", 1, RegexOptions.IgnoreCase, EDP.ReturnValue)
+ Friend ReadOnly Regex_AppID As RParams = RParams.DMS("APP_ID.:.(\d+)", 1, RegexOptions.IgnoreCase, EDP.ReturnValue)
 
  Friend ReadOnly Regex_Photos_by As RParams = RParams.DMS("photos_by"",""id"":""([^""]+)", 1, EDP.ReturnValue)
  Friend ReadOnly Regex_FileName As RParams = RParams.DM("([^/\?]+\..{3,4})(?=(\?|\Z))", 0, EDP.ReturnValue)

diff --git a/SCrawler/API/Facebook/SiteSettings.vb b/SCrawler/API/Facebook/SiteSettings.vb
@@ -18,7 +18,7 @@ Namespace API.Facebook
 #Region "Auth"
  <PropertyOption(AllowNull:=False, ControlText:="Accept", ControlToolTip:="Header 'Accept'", IsAuth:=True), ControlNumber(21), PXML, PClonable>
  Friend ReadOnly Property Header_Accept As PropertyValue
- <PropertyOption(ControlText:="x-ig-app-id", AllowNull:=True, IsAuth:=True)>
+ <PropertyOption(ControlText:="x-ig-app-id", AllowNull:=True, IsAuth:=True), HiddenControl>
  Friend Overrides ReadOnly Property HH_IG_APP_ID As PropertyValue
  Get
  Return __HH_IG_APP_ID
@@ -74,7 +74,7 @@ Namespace API.Facebook
 #End Region
 #Region "BaseAuthExists, GetUserUrl, GetUserPostUrl, IsMyUser, IsMyImageVideo"
  Friend Overrides Function BaseAuthExists() As Boolean
- Return Responser.CookiesExists And ACheck(HH_IG_APP_ID.Value) And CBool(DownloadData_Impl.Value)
+ Return Responser.CookiesExists And CBool(DownloadData_Impl.Value) 'And ACheck(HH_IG_APP_ID.Value)
  End Function
  Friend Overrides Function GetUserUrl(ByVal User As IPluginContentProvider) As String
  Return DirectCast(User, UserData).GetProfileUrl

diff --git a/SCrawler/API/Facebook/UserData.vb b/SCrawler/API/Facebook/UserData.vb
@@ -136,6 +136,7 @@ Namespace API.Facebook
  Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
  If CBool(MySettings.DownloadData_Impl.Value) Then
  Try
+ If Responser.Headers.Value(IG.Header_IG_APP_ID).IsEmptyString Then Responser.Headers.Remove(IG.Header_IG_APP_ID)
  ResetBaseTokens()
  GetUserTokens(Token)
  LoadSavePostsKV(True)
@@ -529,8 +530,14 @@ Namespace API.Facebook
  Dim r$ = resp.GetResponse(URL)
  If Not r.IsEmptyString Then
  If Responser.CookiesExists Then Responser.Cookies.Update(resp.Cookies)
- Token_dtsg = RegexReplace(r, Regex_UserToken_dtsg)
- Token_lsd = RegexReplace(r, Regex_UserToken_lsd)
+ ParseTokens(r, 0)
+ Dim app_id$ = RegexReplace(r, Regex_AppID)
+ If Not app_id.IsEmptyString Then
+ If Not AEquals(Of String)(MySettings.HH_IG_APP_ID.Value, app_id) Then
+ MySettings.HH_IG_APP_ID.Value = app_id
+ Responser.Headers.Add(IG.Header_IG_APP_ID, app_id)
+ End If
+ End If
  Token_Photosby = RegexReplace(r, Regex_Photos_by)
  If StoryBucket.IsEmptyString Then StoryBucket = RegexReplace(r, Regex_StoryBucket)
  If ID.IsEmptyString Then
@@ -568,14 +575,14 @@ Namespace API.Facebook
  .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchSite, "none"))
  .Add("Sec-Fetch-User", "?1")
  .Add("Upgrade-Insecure-Requests", 1)
- Dim h$ = Responser.Headers.Value(IG.Header_Browser)
- If Not h.IsEmptyString Then .Add(IG.Header_Browser, h)
- h = Responser.Headers.Value(IG.Header_BrowserExt)
- If Not h.IsEmptyString Then .Add(IG.Header_BrowserExt, h)
- h = Responser.Headers.Value(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatform))
- If Not h.IsEmptyString Then .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatform, h))
- h = Responser.Headers.Value(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatformVersion))
- If Not h.IsEmptyString Then .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatformVersion, h))
+ Dim cloneHeader As Action(Of String) = Sub(ByVal hName As String)
+  Dim hValue$ = Responser.Headers.Value(hName)
+  If Not hValue.IsEmptyString Then .Add(hName, hValue)
+  End Sub
+ cloneHeader.Invoke(IG.Header_Browser)
+ cloneHeader.Invoke(IG.Header_BrowserExt)
+ cloneHeader.Invoke(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatform).Name)
+ cloneHeader.Invoke(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatformVersion).Name)
  .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaMobile, "?0"))
  .Add("Sec-Ch-Ua-Model", "")
  End With

diff --git a/SCrawler/API/Instagram/Declarations.vb b/SCrawler/API/Instagram/Declarations.vb
@@ -18,6 +18,8 @@ Namespace API.Instagram
  Friend ReadOnly ObtainMedia_SizeFuncPic_RegexP As RParams = RParams.DMS("_p(\d+)x(\d+)", 1, EDP.ReturnValue)
  Friend ReadOnly ObtainMedia_SizeFuncPic_RegexS As RParams = RParams.DMS("_s(\d+)x(\d+)", 1, EDP.ReturnValue)
  Friend Const PageTokenRegexPatternDefault As String = "\[\],{""token"":""(.*?)""},\d+\]"
+ Friend ReadOnly Regex_UserToken_dtsg As RParams = RParams.DMS("DTSGInitialData["":,.\[\]]*?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue)
+ Friend ReadOnly Regex_UserToken_lsd As RParams = RParams.DMS("LSD["":,.\[\]]*?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue)
  Friend Sub UpdateResponser(ByVal Source As IResponse, ByRef Destination As Responser, ByVal UpdateWwwClaim As Boolean)
  Const r_wwwClaimName$ = "x-ig-set-www-claim"
  Const r_tokenName$ = SiteSettings.Header_CSRF_TOKEN_COOKIE

diff --git a/SCrawler/API/Instagram/UserData.GQL.vb b/SCrawler/API/Instagram/UserData.GQL.vb
@@ -297,35 +297,49 @@ Namespace API.Instagram
  End With
  End With
  Dim r$ = Responser.GetResponse(MySiteSettings.GetUserUrl(Me))
+ ParseTokens(r, 0)
+ Catch ex As Exception
+ Finally
+ ChangeResponserMode(_UseGQL, Not _UseGQL)
+ End Try
+ End Sub
+ Protected Sub ParseTokens(ByVal r As String, ByVal Attempt As Integer)
+ Try
  If Not r.IsEmptyString Then
- Dim rr As RParams = RParams.DM(PageTokenRegexPatternDefault, 0, RegexReturn.List, EDP.ReturnValue)
- Dim tokens As List(Of String) = RegexReplace(r, rr)
- Dim tt$, ttVal$
- If tokens.ListExists Then
- With rr
- .Match = Nothing
- .MatchSub = 1
- .WhatGet = RegexReturn.Value
- End With
- For Each tt In tokens
- If Not Token_lsd.IsEmptyString And Not Token_dtsg.IsEmptyString Then
- Exit For
- Else
- ttVal = RegexReplace(tt, rr)
- If Not ttVal.IsEmptyString Then
- If ttVal.Contains(":") Then
- If Token_dtsg.IsEmptyString Then Token_dtsg = ttVal
+ ResetBaseTokens()
+ Select Case Attempt
+ Case 0
+ Dim rr As RParams = RParams.DM(PageTokenRegexPatternDefault, 0, RegexReturn.List, EDP.ReturnValue)
+ Dim tokens As List(Of String) = RegexReplace(r, rr)
+ Dim tt$, ttVal$
+ If tokens.ListExists Then
+ With rr
+ .Match = Nothing
+ .MatchSub = 1
+ .WhatGet = RegexReturn.Value
+ End With
+ For Each tt In tokens
+ If Not Token_lsd.IsEmptyString And Not Token_dtsg.IsEmptyString Then
+ Exit For
  Else
- If Token_lsd.IsEmptyString Then Token_lsd = ttVal
+ ttVal = RegexReplace(tt, rr)
+ If Not ttVal.IsEmptyString Then
+ If ttVal.Contains(":") Then
+ If Token_dtsg.IsEmptyString Then Token_dtsg = ttVal
+ Else
+ If Token_lsd.IsEmptyString Then Token_lsd = ttVal
+ End If
+ End If
  End If
- End If
+ Next
  End If
- Next
- End If
+ Case 1
+ Token_dtsg = RegexReplace(r, Regex_UserToken_dtsg)
+ Token_lsd = RegexReplace(r, Regex_UserToken_lsd)
+ End Select
+ If Not ValidateBaseTokens() And Attempt = 0 Then ParseTokens(r, Attempt + 1)
  End If
- Catch ex As Exception
- Finally
- ChangeResponserMode(_UseGQL, Not _UseGQL)
+ Catch
  End Try
  End Sub
 #End Region

diff --git a/SCrawler/API/PornHub/UserData.vb b/SCrawler/API/PornHub/UserData.vb
@@ -29,7 +29,6 @@ Namespace API.PornHub
  Private Const Name_DownloadFavorite As String = "DownloadFavorite"
  Private Const Name_DownloadGifs As String = "DownloadGifs"
  Private Const Name_DownloadPhotoOnlyFromModelHub As String = "DownloadPhotoOnlyFromModelHub"
- <Obsolete> Private Const Name_IsUser As String = "IsUser"
 #End Region
 #Region "Structures"
  Private Structure FlashVar : Implements IRegExCreator
@@ -254,14 +253,7 @@ Namespace API.PornHub
  DownloadFavorite = .Value(Name_DownloadFavorite).FromXML(Of Boolean)(False)
  DownloadGifs = .Value(Name_DownloadGifs).FromXML(Of Integer)(False)
  DownloadPhotoOnlyFromModelHub = .Value(Name_DownloadPhotoOnlyFromModelHub).FromXML(Of Boolean)(True)
- If .Contains(Name_SiteMode) Then
- SiteMode = .Value(Name_SiteMode).FromXML(Of Integer)(SiteModes.User)
- Else
- 'TODELETE: PornHub 'IsUser' 20231113
-#Disable Warning BC40008
- SiteMode = IIf(.Value(Name_IsUser).FromXML(Of Boolean)(True), SiteModes.User, SiteModes.Search)
-#Enable Warning
- End If
+ SiteMode = .Value(Name_SiteMode).FromXML(Of Integer)(SiteModes.User)
  UpdateUserOptions()
  Else
  If UpdateUserOptions() Then .Value(Name_LabelsName) = LabelsString
@@ -404,7 +396,6 @@ Namespace API.PornHub
  Dim r$ = Responser.GetResponse(URL)
  If Not r.IsEmptyString Then
  Dim l As List(Of UserVideo) = RegexFields(Of UserVideo)(r, {RegexUserVideos}, {6, 7, 3, 10})
- 'URGENT: PornHub: changed list trimming
  'If l.ListExists And Not SiteMode = SiteModes.Playlists Then l = l.ListTake(3, l.Count).ToList
  If l.ListExists And Not SiteMode = SiteModes.Playlists Then l = l.ListTake(1, l.Count).ToList
  If l.ListExists Then

diff --git a/SCrawler/API/ThreadsNet/SiteSettings.vb b/SCrawler/API/ThreadsNet/SiteSettings.vb
@@ -103,8 +103,6 @@ Namespace API.ThreadsNet
 
  With Responser
  .Accept = "*/*"
- 'URGENT: remove after debug
- .DeclaredError = EDP.SendToLog + EDP.ThrowException
  If .UserAgentExists Then useragent = .UserAgent
  With .Headers
  If .Count > 0 Then
@@ -125,6 +123,8 @@ Namespace API.ThreadsNet
  .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchMode, "cors"))
  .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchSite, "same-origin"))
  .Add("Sec-Fetch-User", "?1")
+ .Add("dht", 1)
+ .Add("drp", 1)
  .Add(Instagram.UserData.GQL_HEADER_FB_FRINDLY_NAME, "BarcelonaProfileThreadsTabRefetchableQuery")
  End With
  .CookiesExtractMode = Responser.CookiesExtractModes.Any

diff --git a/SCrawler/API/ThreadsNet/UserData.vb b/SCrawler/API/ThreadsNet/UserData.vb
@@ -157,38 +157,22 @@ Namespace API.ThreadsNet
  Private Function UpdateCredentials(Optional ByVal e As ErrorsDescriber = Nothing) As Boolean
  Dim URL$ = $"https://www.threads.net/@{NameTrue}"
  ResetBaseTokens()
+ Dim headers As New HttpHeaderCollection
+ headers.AddRange(Responser.Headers)
  Try
- Responser.Method = "GET"
- Responser.Referer = URL
- Responser.Headers.Remove(GQL_HEADER_FB_LSD)
+ With Responser
+ .Method = "GET"
+ .Referer = URL
+ With .Headers
+ .Remove(GQL_HEADER_FB_LSD)
+ .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchDest, "document"))
+ .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchMode, "navigate"))
+ End With
+ End With
  WaitTimer()
  Dim r$ = Responser.GetResponse(URL,, EDP.ThrowException)
- Dim rr As RParams
- Dim tt$, ttVal$
  If Not r.IsEmptyString Then
- rr = RParams.DM(Instagram.PageTokenRegexPatternDefault, 0, RegexReturn.List, EDP.ReturnValue)
- Dim tokens As List(Of String) = RegexReplace(r, rr)
- If tokens.ListExists Then
- With rr
- .Match = Nothing
- .MatchSub = 1
- .WhatGet = RegexReturn.Value
- End With
- For Each tt In tokens
- If Not Token_dtsg.IsEmptyString And Not Token_lsd.IsEmptyString Then
- Exit For
- Else
- ttVal = RegexReplace(tt, rr)
- If Not ttVal.IsEmptyString Then
- If ttVal.Contains(":") Then
- If Token_dtsg.IsEmptyString Then Token_dtsg = ttVal
- Else
- If Token_lsd.IsEmptyString Then Token_lsd = ttVal
- End If
- End If
- End If
- Next
- End If
+ ParseTokens(r, 0)
  If ID.IsEmptyString Then ID = RegexReplace(r, RParams.DMS("""props"":\{""user_id"":""(\d+)""\},", 1, EDP.ReturnValue))
  End If
  Return Valid
@@ -204,6 +188,12 @@ Namespace API.ThreadsNet
  'LogError(ex, $"failed to update some{IIf(notFound.IsEmptyString, String.Empty, $" ({notFound})")} credentials", e)
  LogError(eex, String.Empty, e)
  Return False
+ Finally
+ If headers.ListExists Then
+ Responser.Headers.Clear()
+ Responser.Headers.AddRange(headers)
+ headers.Dispose()
+ End If
  End Try
  End Function
 #End Region