2024.4.26.0

Add 'CookieValueExtractorAttribute' and the ability to immediately populate fields with values that can be extracted from cookies
Feed: add the ability to load the last session of the current day (if it exists) as the current session after restarting SCrawler
UserSearchForm: include friendly name matches in search result
API.Xhamster: saved posts aren't downloading

Changelog.md

@@ -1,3 +1,16 @@
+# 2024.4.26.0
+
+*2024-04-26*
+
+- Added
+ - Site settings: the values that can be extracted from cookies immediately populate fields
+ - Feed: ability to load the last session of the current day (if it exists) as the current session after restarting SCrawler
+ - Users search: include friendly name matches in search result
+- Updated
+ - gallery-dl up to version **1.26.9**
+- Fixed
+ - xHamster: **saved posts aren't downloading**
+
 # 2024.4.14.0
 
 *2024-04-14*

SCrawler/API/Base/Declarations.vb

@@ -6,6 +6,7 @@
 '
 ' This program is distributed in the hope that it will be useful,
 ' but WITHOUT ANY WARRANTY
+Imports System.Runtime.CompilerServices
 Imports PersonalUtilities.Forms
 Imports PersonalUtilities.Functions.RegularExpressions
 Namespace API.Base
@@ -72,5 +73,12 @@ Namespace API.Base
  $"Current query: [{CurrentQuery}]{vbCr}New query: [{NewQuery}]",
  "Changing a query"}, vbExclamation,,, {"Process", "Cancel"}) = 0
  End Function
+ <Extension> Friend Function GetCookieValue(ByVal Cookies As IEnumerable(Of System.Net.Cookie), ByVal CookieName As String) As String
+ If Cookies.ListExists Then Return If(Cookies.FirstOrDefault(Function(c) c.Name.ToLower = CookieName.ToLower)?.Value, String.Empty) Else Return String.Empty
+ End Function
+ <Extension> Friend Function GetCookieValue(ByVal Cookies As IEnumerable(Of System.Net.Cookie), ByVal CookieName As String,
+ ByVal PropName As String, ByVal PropNameComp As String) As String
+ Return If(PropName = PropNameComp, Cookies.GetCookieValue(CookieName), String.Empty)
+ End Function
  End Module
 End Namespace

SCrawler/API/Instagram/SiteSettings.vb

@@ -65,6 +65,10 @@ Namespace API.Instagram
  Friend Const Header_Platform_Verion As String = "Sec-Ch-Ua-Platform-Version"
  <PropertyOption(ControlText:="x-csrftoken", ControlToolTip:="Can be automatically extracted from cookies", IsAuth:=True, AllowNull:=True), ControlNumber(2), PClonable(Clone:=False)>
  Friend ReadOnly Property HH_CSRF_TOKEN As PropertyValue
+ <CookieValueExtractor(NameOf(HH_CSRF_TOKEN))>
+ Private Function GetValueFromCookies(ByVal PropName As String, ByVal c As CookieKeeper) As String
+ Return c.GetCookieValue(Header_CSRF_TOKEN_COOKIE, PropName, NameOf(HH_CSRF_TOKEN))
+ End Function
  <PropertyOption(ControlText:="x-ig-app-id", IsAuth:=True, AllowNull:=False), ControlNumber(3), PClonable(Clone:=False)>
  Friend ReadOnly Property HH_IG_APP_ID As PropertyValue
  <PropertyOption(ControlText:="x-asbd-id", IsAuth:=True, AllowNull:=True), ControlNumber(4), PClonable(Clone:=False)>
@@ -557,7 +561,7 @@ Namespace API.Instagram
  If vals.Any(Function(v) Not v.ValueOld = v.ValueNew) OrElse
  Not Responser.Cookies.ListEquals(____Cookies) Then HH_IG_WWW_CLAIM.Value = 0 : credentialsUpdated = True
  If Responser.CookiesExists Then
- Dim csrf$ = If(Responser.Cookies.FirstOrDefault(Function(c) c.Name.StringToLower = Header_CSRF_TOKEN_COOKIE)?.Value, String.Empty)
+ Dim csrf$ = GetValueFromCookies(NameOf(HH_CSRF_TOKEN), Responser.Cookies)
  If Not csrf.IsEmptyString Then
  If Not AEquals(Of String)(CStr(HH_CSRF_TOKEN.Value), csrf) Then credentialsUpdated = True
  HH_CSRF_TOKEN.Value = csrf

SCrawler/API/JustForFans/SiteSettings.vb

@@ -20,6 +20,10 @@ Namespace API.JustForFans
  Friend ReadOnly Property UserID As PropertyValue
  <PropertyOption, PXML, PClonable(Clone:=False)>
  Friend ReadOnly Property UserHash4 As PropertyValue
+ <CookieValueExtractor(NameOf(UserHash4))>
+ Private Function GetValueFromCookies(ByVal PropName As String, ByVal c As CookieKeeper) As String
+ Return c.GetCookieValue(UserHash4_CookieName, PropName, NameOf(UserHash4))
+ End Function
  <PropertyOption(ControlText:="Accept", ControlToolTip:="Header 'Accept'"), PClonable>
  Friend ReadOnly Property HeaderAccept As PropertyValue
  <PropertyOption(InheritanceName:=SettingsCLS.HEADER_DEF_UserAgent), PClonable, PXML(OnlyForChecked:=True)>
@@ -61,7 +65,7 @@ Namespace API.JustForFans
  Private Sub UpdateUserHash4()
  If Responser.CookiesExists Then
  Dim hv_current$ = UserHash4.Value
- Dim hv_cookie$ = If(Responser.Cookies.FirstOrDefault(Function(cc) cc.Name.ToLower = UserHash4_CookieName)?.Value, String.Empty)
+ Dim hv_cookie$ = GetValueFromCookies(NameOf(UserHash4), Responser.Cookies)
  If Not hv_cookie.IsEmptyString And Not hv_cookie = hv_current And Responser.Cookies.Changed Then UserHash4.Value = hv_cookie
  End If
  End Sub

SCrawler/API/OnlyFans/SiteSettings.vb

@@ -59,6 +59,16 @@ Namespace API.OnlyFans
  Responser.UserAgent = Value
  End If
  End Sub
+ <CookieValueExtractor(NameOf(HH_USER_ID)), CookieValueExtractor(NameOf(HH_X_BC))>
+ Private Function GetValueFromCookies(ByVal PropName As String, ByVal c As CookieKeeper) As String
+ If c.ListExists Then
+ Select Case PropName
+ Case NameOf(HH_USER_ID) : Return c.GetCookieValue("auth_id")
+ Case NameOf(HH_X_BC) : Return c.GetCookieValue("fp")
+ End Select
+ End If
+ Return String.Empty
+ End Function
 #End Region
 #Region "Rules"
  <PXML("LastDateUpdated")> Private ReadOnly Property LastDateUpdated_XML As PropertyValue

SCrawler/API/Pinterest/UserData.vb

@@ -287,7 +287,7 @@ Namespace API.Pinterest
  End Function
  End Class
  Private Function GetDataFromGalleryDL(ByVal URL As String, ByVal IsBoardsRequested As Boolean, ByVal Token As CancellationToken) As List(Of String)
- Dim command$ = $"gallery-dl --verbose --simulate "
+ Dim command$ = $"""{Settings.GalleryDLFile.File}"" --verbose --simulate "
  Try
  If Not URL.IsEmptyString Then
  If MySettings.CookiesNetscapeFile.Exists Then command &= $"--cookies ""{MySettings.CookiesNetscapeFile}"" "

SCrawler/API/ThreadsNet/SiteSettings.vb

@@ -25,6 +25,10 @@ Namespace API.ThreadsNet
  Return __HH_CSRF_TOKEN
  End Get
  End Property
+ <CookieValueExtractor(NameOf(HH_CSRF_TOKEN))>
+ Private Function GetValueFromCookies(ByVal PropName As String, ByVal c As CookieKeeper) As String
+ Return c.GetCookieValue(IG.Header_CSRF_TOKEN_COOKIE, PropName, NameOf(HH_CSRF_TOKEN))
+ End Function
  <PClonable> Protected ReadOnly __HH_IG_APP_ID As PropertyValue
  <PropertyOption(ControlText:="x-ig-app-id", AllowNull:=False, IsAuth:=True), ControlNumber(10)>
  Friend Overridable ReadOnly Property HH_IG_APP_ID As PropertyValue
@@ -195,7 +199,7 @@ Namespace API.ThreadsNet
  End Sub
  Friend Overrides Sub Update()
  If _SiteEditorFormOpened And Responser.CookiesExists Then
- Dim csrf$ = If(Responser.Cookies.FirstOrDefault(Function(c) c.Name.StringToLower = IG.Header_CSRF_TOKEN_COOKIE)?.Value, String.Empty)
+ Dim csrf$ = GetValueFromCookies(NameOf(HH_CSRF_TOKEN), Responser.Cookies)
  If Not csrf.IsEmptyString Then HH_CSRF_TOKEN.Value = csrf
  If Not __Cookies Is Nothing AndAlso Not __Cookies.ListEquals(Responser.Cookies) Then DownloadData_Impl.Value = True
  End If

SCrawler/API/Xhamster/UserData.vb

@@ -265,7 +265,12 @@ Namespace API.Xhamster
  Dim checkLimit As Func(Of Boolean) = Function() limit > 0 And SearchPostsCount >= limit And IsVideo
 
  If IsSavedPosts Then
- containerNodes.Add(If(IsVideo, {"favoriteVideoListComponent", "models"}, {"favoritesGalleriesAndPhotosCollection"}))
+ If IsVideo Then
+ containerNodes.Add({"favoriteVideoListComponent", "models"})
+ containerNodes.Add({"favoriteVideoListComponent", "videoThumbProps"})
+ Else
+ containerNodes.Add({"favoritesGalleriesAndPhotosCollection"})
+ End If
  ElseIf Not SiteMode = SiteModes.Search Then
  If IsVideo Then
  containerNodes.Add({"trendingVideoListComponent", "models"})

SCrawler/Download/Feed/DownloadFeedForm.vb

@@ -173,6 +173,7 @@ Namespace DownloadObjects
  MENU_DOWN.Visible = OPT_SUBSCRIPTIONS.Checked
  UpdateSettings()
  FeedChangeMode(FeedModes.Current)
+ Downloader.FilesLoadLastSession()
  RefillList(True, False)
  .EndLoaderOperations(False)
  End With

SCrawler/Download/TDownloader.vb

@@ -116,8 +116,10 @@ Namespace DownloadObjects
  Friend Const SessionsPath As String = "Settings\Sessions\"
  Private _FilesSessionCleared As Boolean = False
  Private _FilesSessionActual As SFile = Nothing
+ Private _FilesSessionChecked As Boolean = False
  Friend ReadOnly Property FilesSessionActual(Optional ByVal GenerateFileName As Boolean = True) As SFile
  Get
+ FilesLoadLastSession()
  If _FilesSessionActual.IsEmptyString And GenerateFileName Then _
  _FilesSessionActual = $"{SessionsPath}{AConvert(Of String)(Now, SessionDateTimeProvider)}.xml"
  Return _FilesSessionActual
@@ -146,6 +148,47 @@ Namespace DownloadObjects
  _FilesSaving = False
  End Try
  End Function
+ Private _FilesSessionChecked_Impl As Boolean = False
+ Friend Sub FilesLoadLastSession()
+ Try
+ If Not _FilesSessionChecked And Not _FilesSessionChecked_Impl And _FilesSessionActual.IsEmptyString Then
+ _FilesSessionChecked = True
+ _FilesSessionChecked_Impl = True
+ Dim settingValue% = Settings.FeedCurrentTryLoadLastSession
+ If settingValue >= 0 Then
+ Dim startTime As Date = Process.GetCurrentProcess.StartTime
+ Dim files As List(Of SFile) = SFile.GetFiles(SessionsPath.CSFileP, "*.xml",, EDP.ReturnValue)
+ If files.ListExists Then files.RemoveAll(Settings.Feeds.FeedSpecialRemover)
+ If files.ListExists Then
+ Dim nd$ = Now.ToString("yyyyMMdd")
+ files.RemoveAll(Function(f) Not f.Name.StartsWith(nd))
+ End If
+ If files.ListExists Then
+ files.Sort()
+ Dim lastDate As Date = AConvert(Of Date)(files.Last.Name, SessionDateTimeProvider)
+ If lastDate.Date = startTime.Date Then
+ Dim __files As New List(Of UserMediaD)
+ Using x As New XmlFile(files.Last, Protector.Modes.All, False) With {.AllowSameNames = True, .XmlReadOnly = True}
+ x.LoadData()
+ If x.Count > 0 Then __files.ListAddList(x, LAP.IgnoreICopier)
+ If __files.Count > 0 AndAlso (settingValue = 0 OrElse
+ (startTime - {lastDate, __files.Max(Function(f) f.Date)}.Max).TotalMinutes <= settingValue) Then
+ _Session = __files.Max(Function(f) f.Session)
+ Me.Files.AddRange(__files)
+ _FilesSessionActual = files.Last
+ End If
+ __files.Clear()
+ End Using
+ End If
+ End If
+ End If
+ _FilesSessionChecked_Impl = False
+ End If
+ Catch ex As Exception
+ _FilesSessionChecked_Impl = False
+ ErrorsDescriber.Execute(EDP.SendToLog, ex, "[TDownloader.FilesLoadLastSession]")
+ End Try
+ End Sub
  Private _FilesUpdating As Boolean = False
  Friend Sub FilesUpdatePendingUsers()
  _FilesUpdating = True
@@ -444,7 +487,16 @@ Namespace DownloadObjects
  End If
  End Set
  End Property
- Private Session As Integer = 0
+ Private _Session As Integer = 0
+ Private Property Session As Integer
+ Get
+ FilesLoadLastSession()
+ Return _Session
+ End Get
+ Set(ByVal _Session As Integer)
+ Me._Session = _Session
+ End Set
+ End Property
  Private Sub [Start]()
  If Not AutoDownloaderWorking AndAlso MyProgressForm.ReadyToOpen AndAlso Pool.LongCount(Function(p) p.Count > 0) > 1 Then MyProgressForm.Show() : MainFrameObj.Focus()
  If Not If(CheckerThread?.IsAlive, False) Then