3.0.0.7

Added script usage
Fixed  downloading of LPSG images
Fixed Instagram Stories
Fixed date/time file pattern

Changelog.md

@@ -1,3 +1,13 @@
+# 3.0.0.7
+
+- Added
+  - Ability to run a script after the user download if complete
+  - Hotkey ```F2``` for additional options in the user creation form
+- Fixed
+  - (Issue #32) In some cases, Date and Time are still not added for Stories and Tagged Photos
+  - (Issue #33) Instagram Stories downloading error
+  - LPSG downloader does not download all content
+
 # 3.0.0.6
 
 - Added

SCrawler.Plugin.LPSG/Declarations.vb

@@ -9,10 +9,15 @@
 Imports PersonalUtilities.Functions.RegularExpressions
 Friend Module Declarations
     Friend ReadOnly Property PhotoRegEx As RParams = RParams.DM("(https://www.lpsg.com/attachments)(.+?)(?="")", 0, RegexReturn.List)
+    Friend ReadOnly Property PhotoRegExExt As New RParams("img.data.src=""(/proxy[^""]+?)""", Nothing, 1, RegexReturn.List) With {
+        .Converter = Function(Input) $"https://www.lpsg.com/{SymbolsConverter.HTML.Decode(Input)}"}
     Friend ReadOnly Property NextPageRegex As RParams = RParams.DMS("<link rel=""next"" href=""(.+?/page-(\d+))""", 2)
-    Private Const FileUrlRegexDefault As String = "(.+[^/]+?)(jpg|jpeg|gif|png)"
+    Private Const FileUrlRegexDefault As String = "([^/]+?)(jpg|jpeg|gif|png|webm)"
+    Private ReadOnly InputFReplacer As New ErrorsDescriber(EDP.ReturnValue)
+    Private ReadOnly InputForbidRemover As Func(Of String, String) = Function(Input) If(Input.IsEmptyString, Input, Input.StringRemoveWinForbiddenSymbols(, InputFReplacer))
     Friend ReadOnly Property FileRegEx As New RParams(FileUrlRegexDefault, Nothing, 0) With {
         .Converter = Function(ByVal Input As String) As String
+                         Input = InputForbidRemover.Invoke(Input)
                          If Not Input.IsEmptyString Then
                              Dim lv$ = Input.Split("-").LastOrDefault
                              If Not lv.IsEmptyString Then
@@ -22,5 +27,22 @@ Friend Module Declarations
                          End If
                          Return Input
                      End Function}
+    Friend ReadOnly Property FileRegExExt As New RParams(FileUrlRegexDefault, 0, Nothing, InputForbidRemover)
+    Friend ReadOnly Property FileRegExExt2 As New RParams("([^/]+?)(?=(\Z|&))", 0, Nothing, InputForbidRemover)
     Friend ReadOnly Property FileExistsRegEx As RParams = RParams.DMS(FileUrlRegexDefault, 2)
+    Private Class PUMComparer : Implements IEqualityComparer, IEqualityComparer(Of PluginUserMedia)
+        Private Overloads Function Equals(ByVal x As PluginUserMedia, ByVal y As PluginUserMedia) As Boolean Implements IEqualityComparer(Of PluginUserMedia).Equals
+            Return x.URL = y.URL
+        End Function
+        Private Function IEqualityComparer_Equals(ByVal x As Object, ByVal y As Object) As Boolean Implements IEqualityComparer.Equals
+            Return DirectCast(x, PluginUserMedia).URL = DirectCast(y, PluginUserMedia).URL
+        End Function
+        Private Overloads Function GetHashCode(ByVal Obj As Object) As Integer Implements IEqualityComparer.GetHashCode
+            Throw New NotImplementedException()
+        End Function
+        Private Overloads Function GetHashCode(ByVal Obj As PluginUserMedia) As Integer Implements IEqualityComparer(Of PluginUserMedia).GetHashCode
+            Throw New NotImplementedException()
+        End Function
+    End Class
+    Friend ReadOnly TempListAddParams As New ListAddParams(LAP.NotContainsOnly) With {.Comparer = New PUMComparer}
 End Module

SCrawler.Plugin.LPSG/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("1.0.0.1")>
-<Assembly: AssemblyFileVersion("1.0.0.1")>
+<Assembly: AssemblyVersion("1.0.0.2")>
+<Assembly: AssemblyFileVersion("1.0.0.2")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler.Plugin.LPSG/UserData.vb

@@ -9,6 +9,7 @@
 Imports PersonalUtilities.Functions.RegularExpressions
 Imports UStates = SCrawler.Plugin.PluginUserMedia.States
 Imports UTypes = SCrawler.Plugin.PluginUserMedia.Types
+Imports Converters = PersonalUtilities.Functions.SymbolsConverter.Converters
 Public Class UserData : Implements IPluginContentProvider
 #Region "XML names"
     Private Const Name_LatestPage As String = "LatestPage"
@@ -60,13 +61,13 @@ Public Class UserData : Implements IPluginContentProvider
 #End Region
     Private Property LatestPage As String = String.Empty
     Private Property Responser As Response = Nothing
+    Private Enum Mode : Internal : External : End Enum
     Public Sub GetMedia() Implements IPluginContentProvider.GetMedia
         Try
             If Not Responser Is Nothing Then Responser.Dispose()
             Responser = New Response
             With Responser : .Copy(Settings.Responser) : .Error = EDP.ThrowException : End With
 
-            Dim l As List(Of String) = Nothing
             Dim NextPage$
             Dim r$
             Dim _LPage As Func(Of String) = Function() If(LatestPage.IsEmptyString, String.Empty, $"page-{LatestPage}")
@@ -78,35 +79,60 @@ Public Class UserData : Implements IPluginContentProvider
                 Thrower.ThrowAny()
                 If Not r.IsEmptyString Then
                     NextPage = RegexReplace(r, NextPageRegex)
-                    l.ListAddList(RegexReplace(r, PhotoRegEx), LAP.NotContainsOnly)
+                    UpdateMediaList(RegexReplace(r, PhotoRegEx), Mode.Internal)
+                    UpdateMediaList(RegexReplace(r, PhotoRegExExt), Mode.External)
                     If NextPage = LatestPage Or NextPage.IsEmptyString Then Exit Do Else LatestPage = NextPage
                 Else
                     Exit Do
                 End If
             Loop
 
-            If l.ListExists Then
-                Dim f As SFile
-                For Each u$ In l
-                    If Not IsEmptyString(RegexReplace(u, FileExistsRegEx)) Then
-                        f = CStr(RegexReplace(u, FileRegEx))
-                        f.Path = DataPath.CSFilePSN
-                        f.Separator = "\"
-                        TempMediaList.Add(New PluginUserMedia With {.ContentType = UTypes.Picture, .URL = u, .File = f})
-                    End If
-                Next
-                If TempMediaList.ListExists And ExistingContentList.ListExists Then _
-                   TempMediaList.RemoveAll(Function(m) ExistingContentList.Exists(Function(mm) mm.URL = m.URL))
-            End If
+            If TempMediaList.ListExists And ExistingContentList.ListExists Then _
+               TempMediaList.RemoveAll(Function(m) ExistingContentList.Exists(Function(mm) mm.URL = m.URL))
         Catch oex As OperationCanceledException
         Catch dex As ObjectDisposedException
         Catch ex As Exception
-            LogProvider.Add(ex, "[LPSG.UserData.GetMedia]")
+            If Responser.StatusCode = Net.HttpStatusCode.ServiceUnavailable Then
+                LogProvider.Add("LPSG not available")
+            Else
+                LogProvider.Add(ex, "[LPSG.UserData.GetMedia]")
+            End If
         End Try
     End Sub
+    Private Sub UpdateMediaList(ByVal l As List(Of String), ByVal m As Mode)
+        If l.ListExists Then
+            Dim f As SFile
+            Dim u$
+            Dim exists As Boolean
+            Dim r As RParams
+            Dim ude As New ErrorsDescriber(EDP.ReturnValue)
+            For Each url$ In l
+                If Not url.IsEmptyString Then u = SymbolsConverter.Decode(url, {Converters.HTML, Converters.ASCII}, ude) Else u = String.Empty
+                If Not u.IsEmptyString Then
+                    exists = Not IsEmptyString(RegexReplace(u, FileExistsRegEx))
+                    If m = Mode.Internal Then
+                        r = FileRegEx
+                    Else
+                        r = FileRegExExt
+                        If Not exists Then
+                            r = FileRegExExt2
+                            exists = Not IsEmptyString(RegexReplace(u, FileRegExExt2))
+                        End If
+                    End If
+                    If exists Then
+                        f = CStr(RegexReplace(u, r))
+                        f.Path = DataPath.CSFilePSN
+                        f.Separator = "\"
+                        If f.Extension.IsEmptyString Then f.Extension = "jpg"
+                        TempMediaList.ListAddValue(New PluginUserMedia With {.ContentType = UTypes.Picture, .URL = url, .File = f}, TempListAddParams)
+                    End If
+                End If
+            Next
+        End If
+    End Sub
     Public Sub Download() Implements IPluginContentProvider.Download
         Try
-            With Responser : .UseWebClient = True : .UseWebClientCookies = True : End With
+            With Responser : .UseWebClient = True : .UseWebClientCookies = True : .ResetError() : End With
             If TempMediaList.ListExists Then
                 Dim m As PluginUserMedia
                 Dim eweb As ErrorsDescriber = EDP.ThrowException
@@ -122,8 +148,12 @@ Public Class UserData : Implements IPluginContentProvider
                         Else
                             m.DownloadState = UStates.Skipped
                         End If
-                    Catch ex As Exception
-                        m.DownloadState = UStates.Skipped
+                    Catch wex As Exception
+                        If Responser.Client.StatusCode = Net.HttpStatusCode.ServiceUnavailable Then
+                            LogProvider.Add("LPSG not available")
+                        Else
+                            m.DownloadState = UStates.Skipped
+                        End If
                     End Try
                     RaiseEvent ProgressChanged(1)
                     TempMediaList(i) = m

SCrawler/API/Base/UserDataBase.vb

@@ -10,6 +10,7 @@ Imports PersonalUtilities.Functions.XML
 Imports PersonalUtilities.Functions.RegularExpressions
 Imports PersonalUtilities.Forms.Toolbars
 Imports PersonalUtilities.Tools.WEB
+Imports PersonalUtilities.Tools
 Imports System.IO
 Imports System.Net
 Imports System.Threading
@@ -115,6 +116,9 @@ Namespace API.Base
         Private Const Name_PicturesCount As String = "PicturesCount"
         Private Const Name_LastUpdated As String = "LastUpdated"
 
+        Private Const Name_ScriptUse As String = "ScriptUse"
+        Private Const Name_ScriptFile As String = "ScriptFile"
+
         Private Const Name_DataMerging As String = "DataMerging"
 #Region "Downloaded data"
         Private Const Name_MediaType As String = "Type"
@@ -405,6 +409,10 @@ BlockNullPicture:
             End Get
         End Property
 #End Region
+#Region "Script"
+        Friend Overridable Property ScriptUse As Boolean = False Implements IUserData.ScriptUse
+        Friend Overridable Property ScriptFile As SFile Implements IUserData.ScriptFile
+#End Region
 #End Region
 #Region "Plugins Support"
         Protected Event ProgressChanged As IPluginContentProvider.ProgressChangedEventHandler Implements IPluginContentProvider.ProgressChanged
@@ -576,6 +584,17 @@ BlockNullPicture:
                         DownloadedVideos(True) = x.Value(Name_VideoCount).FromXML(Of Integer)(0)
                         DownloadedPictures(True) = x.Value(Name_PicturesCount).FromXML(Of Integer)(0)
                         LastUpdated = AConvert(Of Date)(x.Value(Name_LastUpdated), ADateTime.Formats.BaseDateTime, Nothing)
+                        ScriptUse = x.Value(Name_ScriptUse).FromXML(Of Boolean)(False)
+                        Dim s$ = x.Value(Name_ScriptFile)
+                        If Not s.IsEmptyString Then
+                            If SFile.IsDirectory(s) Then
+                                ScriptFile = s
+                            Else
+                                ScriptFile = New SFile(s) With {.Path = MyFile.Path}
+                            End If
+                        Else
+                            ScriptFile = Nothing
+                        End If
                         DataMerging = x.Value(Name_DataMerging).FromXML(Of Boolean)(False)
                         ChangeCollectionName(x.Value(Name_CollectionName), False)
                         Labels.ListAddList(x.Value(Name_LabelsName).StringToList(Of String, List(Of String))("|", EDP.ReturnValue), LAP.NotContainsOnly, LAP.ClearBeforeAdd)
@@ -615,6 +634,16 @@ BlockNullPicture:
                     x.Add(Name_VideoCount, DownloadedVideos(True))
                     x.Add(Name_PicturesCount, DownloadedPictures(True))
                     x.Add(Name_LastUpdated, AConvert(Of String)(LastUpdated, ADateTime.Formats.BaseDateTime, String.Empty))
+                    x.Add(Name_ScriptUse, ScriptUse.BoolToInteger)
+                    If Not ScriptFile.IsEmptyString Then
+                        If ScriptFile.Path = MyFile.Path Then
+                            x.Add(Name_ScriptFile, ScriptFile.File)
+                        Else
+                            x.Add(Name_ScriptFile, ScriptFile)
+                        End If
+                    Else
+                        x.Add(Name_ScriptFile, String.Empty)
+                    End If
                     x.Add(Name_CollectionName, CollectionName)
                     x.Add(Name_LabelsName, Labels.ListToString(, "|", EDP.ReturnValue))
                     x.Add(Name_DataMerging, DataMerging.BoolToInteger)
@@ -760,6 +789,7 @@ BlockNullPicture:
                 If DownloadedTotal(False) > 0 Or EnvirChanged.Invoke Then
                     If __SaveData Then
                         LastUpdated = Now
+                        RunScript()
                         DownloadedPictures(True) = SFile.GetFiles(User.File.CutPath, "*.jpg|*.jpeg|*.png|*.gif|*.webm",, EDP.ReturnValue).Count
                         DownloadedVideos(True) = SFile.GetFiles(User.File.CutPath, "*.mp4|*.mkv|*.mov", SearchOption.AllDirectories, EDP.ReturnValue).Count
                         If Labels.Contains(LabelsKeeper.NoParsedUser) Then Labels.Remove(LabelsKeeper.NoParsedUser)
@@ -919,16 +949,10 @@ BlockNullPicture:
             Dim ff As SFile = Nothing
             Try
                 If Not f.IsEmptyString AndAlso f.Exists Then
-                    Dim d As Date? = m.Post.Date
                     If Settings.FileReplaceNameByDate Then
-                        Dim dd$ = AConvert(Of String)(If(d, Now), FileDateAppenderProvider, String.Empty)
                         ff = f
-                        ff.Name = dd
+                        ff.Name = String.Format(FileDateAppenderPattern, f.Name, CStr(AConvert(Of String)(If(m.Post.Date, Now), FileDateAppenderProvider, String.Empty)))
                         ff = SFile.Indexed_IndexFile(ff,, New NumberedFile(ff))
-                    ElseIf d.HasValue AndAlso (Settings.FileAddDateToFileName Or Settings.FileAddTimeToFileName) AndAlso
-                                              (Not FileDateAppenderProvider Is Nothing And Not FileDateAppenderPattern.IsEmptyString) Then
-                        ff = f
-                        ff.Name = String.Format(FileDateAppenderPattern, f.Name, CStr(AConvert(Of String)(d.Value, FileDateAppenderProvider, String.Empty)))
                     End If
                     If Not ff.Name.IsEmptyString Then My.Computer.FileSystem.RenameFile(f, ff.File) : Return ff
                 End If
@@ -938,6 +962,27 @@ BlockNullPicture:
                 Return f
             End Try
         End Function
+        Private Sub RunScript()
+            Try
+                If ScriptUse Then
+                    Dim ScriptPattern$
+                    If Not ScriptFile.IsEmptyString Then
+                        ScriptPattern = ScriptFile
+                    Else
+                        ScriptPattern = Settings.ScriptFile.Value
+                    End If
+                    If Not ScriptPattern.IsEmptyString Then
+                        ScriptPattern &= " {0}"
+                        Using b As New BatchExecutor With {.RedirectStandardError = True}
+                            b.Execute({String.Format(ScriptPattern, MyFile.CutPath(1).ToString)}, EDP.SendInLog + EDP.ThrowException)
+                            If b.HasError Or Not b.ErrorOutput.IsEmptyString Then Throw New Exception(b.ErrorOutput, b.ErrorException)
+                        End Using
+                    End If
+                End If
+            Catch ex As Exception
+                LogError(ex, "script execution error")
+            End Try
+        End Sub
 #End Region
 #Region "Delete, Move, Merge"
         Friend Overridable Function Delete() As Integer Implements IUserData.Delete
@@ -990,6 +1035,11 @@ BlockNullPicture:
                 End If
                 f.CutPath.Exists(SFO.Path)
                 Directory.Move(UserBefore.File.CutPath(, EDP.ThrowException).Path, f.Path)
+                If Not ScriptFile.IsEmptyString AndAlso ScriptFile.Path = UserBefore.File.Path Then
+                    Dim ff As SFile = ScriptFile
+                    f.Path = MyFile.Path
+                    ScriptFile = ff
+                End If
                 Settings.UsersList.Remove(UserBefore)
                 Settings.UpdateUsersList(User)
                 UpdateUserInformation()
@@ -1043,6 +1093,11 @@ BlockNullPicture:
                                       New ErrorsDescriber(False, False, False, New List(Of SFile))).Count = 0 Then
                         UserBefore.File.CutPath.Delete(SFO.Path, Settings.DeleteMode, EDP.SendInLog)
                     End If
+                    If Not ScriptFile.IsEmptyString AndAlso ScriptFile.Path = UserBefore.File.Path Then
+                        Dim f As SFile = ScriptFile
+                        f.Path = MyFile.Path
+                        ScriptFile = f
+                    End If
                     UpdateUserInformation()
                 End If
             Catch ioex As InvalidOperationException When ioex.HelpLink = 1
@@ -1207,6 +1262,8 @@ BlockNullPicture:
         ReadOnly Property Key As String
         Property DownloadImages As Boolean
         Property DownloadVideos As Boolean
+        Property ScriptUse As Boolean
+        Property ScriptFile As SFile
         Function GetLVI(ByVal Destination As ListView) As ListViewItem
         Function GetLVIGroup(ByVal Destination As ListView) As ListViewGroup
         Sub LoadUserInformation()

SCrawler/API/Instagram/UserData.vb

@@ -193,6 +193,7 @@ Namespace API.Instagram
                             Case Sections.Stories
                                 If Not StoriesRequested Then
                                     StoriesList = GetStoriesList()
+                                    StoriesRequested = True
                                     MySiteSettings.TooManyRequests(False)
                                     RequestsCount += 1
                                     ThrowAny(Token)