2.0.0.4

Fixed network paths bug Updated classes for new library structures and functions Fixed some typos Fixed minor bugs SitePaths
AAndyProgram · Feb 7, 2022 · 5b64e55 · 5b64e55
1 parent 1bc0485
commit 5b64e55
Show file tree

Hide file tree

Showing 24 changed files with 256 additions and 81 deletions.
diff --git a/Changelog.md b/Changelog.md
@@ -1,8 +1,22 @@
+# 2.0.0.4
+
+**Removed compatibility of program settings with version 1.0.0.4 and lower.**
+
+**If your program version is 1.0.0.4 and lower, it is strongly recommended that you upgrade to release 2.0.0.1 to update the program settings (and run the program). Then update to this release. Otherwise, you will have to configure the program settings again**
+
+**If your program version is 1.0.1.0 or higher, you should not pay attention to this message.**
+
+- Added
+  - Ability to specify the path to store saved posts
+- Fixed
+  - **Error when specifying network paths**
+  - Minor bugs
+
 # 2.0.0.3
 
 **Removed compatibility of program settings with version 1.0.0.4 and lower.**
 
-**If your program version is 1.0.0.4 and lower, it is strongly recommended that you upgrade to release 2.0.0.2 to update the program settings (and run the program). Then update to this release. Otherwise, you will have to configure the program settings again**
+**If your program version is 1.0.0.4 and lower, it is strongly recommended that you upgrade to release 2.0.0.1 to update the program settings (and run the program). Then update to this release. Otherwise, you will have to configure the program settings again**
 
 **If your program version is 1.0.1.0 or higher, you should not pay attention to this message.**
 
@@ -31,7 +45,7 @@
   - Clear information about downloaded profiles of the current session in the "Download info form"
   - Increased the number of Instagram posts (from 12 to 50) received per request
   - Channels' statistics
-  - **RedGisf profiles support**
+  - **RedGifs profiles support**
 - Fixed
   - The program was showing incorrect information about the total numbers of images and videos downloaded when a Reddit user was created from a channel
 

diff --git a/README.md b/README.md
@@ -88,6 +88,8 @@ You can add users by patterns:
 
 Read more about adding users and subreddits [here](https://github.com/AAndyProgram/SCrawler/wiki/Users)
 
+# Guide
+
 **Full guide you can find [here](https://github.com/AAndyProgram/SCrawler/wiki)**
 
 ## Using program as just video downloader

diff --git a/SCrawler/API/Base/SiteSettings.vb b/SCrawler/API/Base/SiteSettings.vb
@@ -9,20 +9,41 @@
 Imports PersonalUtilities.Tools
 Imports PersonalUtilities.Functions.XML
 Imports PersonalUtilities.Functions.XML.Base
+Imports PersonalUtilities.Functions.RegularExpressions
 Namespace API.Base
     Friend Class SiteSettings : Implements IDisposable
         Friend Const Header_Twitter_Authorization As String = "authorization"
         Friend Const Header_Twitter_Token As String = "x-csrf-token"
         Friend ReadOnly Site As Sites
         Friend ReadOnly Responser As WEB.Response
         Private ReadOnly _Path As XMLValue(Of SFile)
-        Friend Property Path As SFile
+        Friend Property Path(Optional ByVal SetProp As Boolean = True) As SFile
             Get
-                If _Path.IsEmptyString Then _Path.Value = SFile.GetPath($"{Settings.GlobalPath.Value.PathWithSeparator}{Site}")
+                If _Path.IsEmptyString Then
+                    Dim tmpPath As SFile = SFile.GetPath($"{Settings.GlobalPath.Value.PathWithSeparator}{Site}")
+                    If SetProp Then _Path.Value = tmpPath Else Return tmpPath
+                End If
                 Return _Path.Value
             End Get
-            Set(ByVal NewFile As SFile)
-                _Path.Value = NewFile
+            Set(ByVal NewPath As SFile)
+                _Path.Value = NewPath
+            End Set
+        End Property
+        Private ReadOnly _SavedPostsPath As XMLValue(Of SFile)
+        Friend Property SavedPostsPath(Optional ByVal GetAny As Boolean = True) As SFile
+            Get
+                If Not _SavedPostsPath.Value.IsEmptyString Then
+                    Return _SavedPostsPath.Value
+                Else
+                    If GetAny Then
+                        Return $"{Path.PathNoSeparator}\!Saved\"
+                    Else
+                        Return Nothing
+                    End If
+                End If
+            End Get
+            Set(ByVal NewPath As SFile)
+                _SavedPostsPath.Value = NewPath
             End Set
         End Property
 #Region "Instagram"
@@ -122,7 +143,7 @@ Namespace API.Base
             End If
 
             Dim n() As String = {SettingsCLS.Name_Node_Sites, Site.ToString}
-            _Path = New XMLValue(Of SFile)("Path", SFile.GetPath($"{GlobalPath.PathWithSeparator}{Site}"), _XML, n, XMLValue(Of SFile).ToFilePath)
+            _Path = New XMLValue(Of SFile)("Path",, _XML, n, XMLValue(Of SFile).ToFilePath)
             _XML.Remove(Site.ToString)
 
             Temporary = New XMLValue(Of Boolean)
@@ -138,6 +159,7 @@ Namespace API.Base
             DownloadVideos.SetDefault(_Vids)
 
             GetUserMediaOnly = New XMLValue(Of Boolean)("GetUserMediaOnly", True, _XML, n)
+            _SavedPostsPath = New XMLValue(Of SFile)("SavedPostsPath",, _XML, n, XMLValue(Of SFile).ToFilePath)
 
             CreateProp(InstaHashUpdateRequired, Sites.Instagram, "InstaHashUpdateRequired", True, _XML, n)
             CreateProp(InstaHash, Sites.Instagram, "InstaHash", String.Empty, _XML, n)
@@ -149,7 +171,7 @@ Namespace API.Base
             CreateProp(RequestsWaitTimerTaskCount, Sites.Instagram, "RequestsWaitTimerTaskCount", 1, _XML, n)
             CreateProp(SleepTimerOnPostsLimit, Sites.Instagram, "SleepTimerOnPostsLimit", 60000, _XML, n)
             If Site = Sites.Instagram Then
-                InstagramDownloadingErrorDate = New XMLValue(Of Date) With {.ToStringFunction = Function(ss, vv) AConvert(Of String)(vv, Nothing)}
+                InstagramDownloadingErrorDate = New XMLValue(Of Date) With {.ToStringFunction = Function(ss, vv) AConvert(Of String)(vv, AModes.Var, Nothing)}
                 InstagramDownloadingErrorDate.SetExtended("InstagramDownloadingErrorDate", Now.AddYears(-10), _XML, n)
             Else
                 InstagramDownloadingErrorDate = New XMLValue(Of Date)
@@ -170,10 +192,7 @@ Namespace API.Base
         End Sub
         Friend Function GatherInstaHash() As Boolean
             Try
-                Dim rs As New RegexStructure("=" & Chr(34) & "([^" & Chr(34) & "]+?ConsumerLibCommons[^" & Chr(34) & "]+?.js)" & Chr(34), 1) With {
-                    .UseTimeOut = True,
-                    .MatchTimeOutSeconds = 10
-                }
+                Dim rs As New RParams("=""([^""]+?ConsumerLibCommons[^""]+?.js)""", Nothing, 1) With {.MatchTimeOut = 10}
                 Dim r$ = Responser.GetResponse("https://instagram.com",, EDP.ThrowException)
                 If Not r.IsEmptyString Then
                     Dim hStr$ = RegexReplace(r, rs)
@@ -182,10 +201,7 @@ Namespace API.Base
                         hStr = $"https://instagram.com/{hStr}"
                         r = Responser.GetResponse(hStr,, EDP.ThrowException)
                         If Not r.IsEmptyString Then
-                            rs = New RegexStructure("generatePaginationActionCreators.+?.profilePosts.byUserId.get.+?queryId:.([\d\w\S]+?)" & Chr(34), 1) With {
-                                .UseTimeOut = True,
-                                .MatchTimeOutSeconds = 10
-                            }
+                            rs = New RParams("generatePaginationActionCreators.+?.profilePosts.byUserId.get.+?queryId:.([\d\w\S]+?)""", Nothing, 1) With {.MatchTimeOut = 10}
                             Dim h$ = RegexReplace(r, rs)
                             If Not h.IsEmptyString Then
                                 InstaHash.Value = h

diff --git a/SCrawler/API/Base/UserDataBase.vb b/SCrawler/API/Base/UserDataBase.vb
@@ -7,6 +7,7 @@
 ' This program is distributed in the hope that it will be useful,
 ' but WITHOUT ANY WARRANTY
 Imports PersonalUtilities.Functions.XML
+Imports PersonalUtilities.Functions.RegularExpressions
 Imports PersonalUtilities.Forms.Toolbars
 Imports System.IO
 Imports System.Net
@@ -470,7 +471,7 @@ BlockNullPicture:
                         Temporary = x.Value(Name_Temporary).FromXML(Of Boolean)(False)
                         Favorite = x.Value(Name_Favorite).FromXML(Of Boolean)(False)
                         CreatedByChannel = x.Value(Name_CreatedByChannel).FromXML(Of Boolean)(False)
-                        SeparateVideoFolder = AConvert(Of Boolean)(x.Value(Name_SeparateVideoFolder), Nothing)
+                        SeparateVideoFolder = AConvert(Of Boolean)(x.Value(Name_SeparateVideoFolder), AModes.Var, Nothing)
                         ReadyForDownload = x.Value(Name_ReadyForDownload).FromXML(Of Boolean)(True)
                         DownloadImages = x.Value(Name_DownloadImages).FromXML(Of Boolean)(True)
                         DownloadVideos = x.Value(Name_DownloadVideos).FromXML(Of Boolean)(True)
@@ -944,7 +945,7 @@ BlockNullPicture:
         End Sub
         Private Function CheckFile(ByVal f As SFile, ByRef List As IEnumerable(Of SFile)) As SFile
             If List.ListExists Then
-                Dim p As New RegexStructure(".+?\s{0,1}\((\d+)\)|.+",,,,,,, String.Empty, EDP.ReturnValue)
+                Dim p As RParams = RParams.DMS(".+?\s{0,1}\((\d+)\)|.+", 0, EDP.ReturnValue)
                 Dim i% = List.Where(Function(ff) CStr(RegexReplace(ff.Name, p)).Trim.ToLower = f.Name.Trim.ToLower).Count
                 If i > 0 Then f.Name &= $" ({i + 1})"
             End If

diff --git a/SCrawler/API/Imgur/Envir.vb b/SCrawler/API/Imgur/Envir.vb
@@ -7,16 +7,17 @@
 ' This program is distributed in the hope that it will be useful,
 ' but WITHOUT ANY WARRANTY
 Imports PersonalUtilities.Functions.XML
+Imports PersonalUtilities.Functions.RegularExpressions
 Imports PersonalUtilities.Tools.WebDocuments.JSON
 Imports System.Net
 Imports SCrawler.API.Imgur.Declarations
 Imports SCrawler.API.Base
-Namespace API.Imgur.Declarations
-    Friend Module Imgur_Declarations
-        Friend ReadOnly PostRegex As New RegexStructure("/([\w\d]+?)(|\.[\w]{0,4})\Z", 1)
-    End Module
-End Namespace
 Namespace API.Imgur
+    Namespace Declarations
+        Friend Module Imgur_Declarations
+            Friend ReadOnly PostRegex As RParams = RParams.DMS("/([\w\d]+?)(|\.[\w]{0,4})\Z", 1)
+        End Module
+    End Namespace
     Friend NotInheritable Class Envir
         Private Sub New()
         End Sub

diff --git a/SCrawler/API/Instagram/Declarations.vb b/SCrawler/API/Instagram/Declarations.vb
@@ -6,9 +6,10 @@
 '
 ' This program is distributed in the hope that it will be useful,
 ' but WITHOUT ANY WARRANTY
+Imports PersonalUtilities.Functions.RegularExpressions
 Namespace API.Instagram
     Friend Module Declarations
-        Friend ReadOnly FilesPattern As New RegexStructure(".+?([^/\?]+?\.[\w\d]{3,4})(?=(\?|\Z))",,,, 1,,, String.Empty, EDP.ReturnValue)
+        Friend ReadOnly FilesPattern As RParams = RParams.DMS(".+?([^/\?]+?\.[\w\d]{3,4})(?=(\?|\Z))", 1, EDP.ReturnValue)
         Friend ReadOnly Property DateProvider As New JsonDate
         Friend Class JsonDate : Implements ICustomProvider
             Friend Function Convert(ByVal Value As Object, ByVal DestinationType As Type, ByVal Provider As IFormatProvider,

diff --git a/SCrawler/API/Instagram/ProfileSaved.vb b/SCrawler/API/Instagram/ProfileSaved.vb
@@ -11,7 +11,7 @@ Imports System.Threading
 Imports PersonalUtilities.Forms.Toolbars
 Namespace API.Instagram
     Friend NotInheritable Class ProfileSaved
-        Friend Shared ReadOnly Property DataPath As SFile = $"{Settings(Sites.Instagram).Path.PathNoSeparator}\!Saved\"
+        Friend Shared ReadOnly Property DataPath As SFile = Settings(Sites.Instagram).SavedPostsPath
         Private Sub New()
         End Sub
         Friend Shared Sub Download(ByRef Bar As MyProgress, ByVal Token As CancellationToken)

diff --git a/SCrawler/API/Instagram/UserData.vb b/SCrawler/API/Instagram/UserData.vb
@@ -8,6 +8,7 @@
 ' but WITHOUT ANY WARRANTY
 Imports PersonalUtilities.Functions.XML
 Imports PersonalUtilities.Functions.Messaging
+Imports PersonalUtilities.Functions.RegularExpressions
 Imports PersonalUtilities.Tools.WebDocuments.JSON
 Imports PersonalUtilities.Forms.Toolbars
 Imports SCrawler.API.Base
@@ -414,7 +415,7 @@ Namespace API.Instagram
         Friend Shared Function GetVideoInfo(ByVal URL As String) As IEnumerable(Of UserMedia)
             Try
                 If Not URL.IsEmptyString AndAlso URL.Contains("instagram.com") Then
-                    Dim PID$ = RegexReplace(URL, New RegexStructure(".*?instagram.com/p/([_\w\d]+)", 1))
+                    Dim PID$ = RegexReplace(URL, RParams.DMS(".*?instagram.com/p/([_\w\d]+)", 1))
                     If Not PID.IsEmptyString Then
                         Using t As New UserData
                             t.Responser = New PersonalUtilities.Tools.WEB.Response

diff --git a/SCrawler/API/Reddit/Declarations.vb b/SCrawler/API/Reddit/Declarations.vb
@@ -7,13 +7,14 @@
 ' This program is distributed in the hope that it will be useful,
 ' but WITHOUT ANY WARRANTY
 Imports PersonalUtilities.Functions.XML.Base
+Imports PersonalUtilities.Functions.RegularExpressions
 Namespace API.Reddit
     Friend Module Declarations
         Friend ReadOnly JsonNodesJson() As NodeParams = {New NodeParams("posts", True, True, True, True, 3)}
         Friend ReadOnly ChannelJsonNodes() As NodeParams = {New NodeParams("data", True, True, True, True, 1),
                                                             New NodeParams("children", True, True, True)}
-        Friend ReadOnly UrlBasePattern As New RegexStructure("(?<=/)([^/]+?\.[\w]{3,4})(?=(\?|\Z))", True, False)
-        Friend ReadOnly VideoRegEx As New RegexStructure("http.{0,1}://[^" & Chr(34) & "]+?mp4", True, False)
+        Friend ReadOnly UrlBasePattern As RParams = RParams.DM("(?<=/)([^/]+?\.[\w]{3,4})(?=(\?|\Z))", 0)
+        Friend ReadOnly VideoRegEx As RParams = RParams.DM("http.{0,1}://[^" & Chr(34) & "]+?mp4", 0)
         Friend ReadOnly DateProvider As New JsonDate
         Friend ReadOnly DateProviderChannel As New JsonDateChannel
         Private ReadOnly EUR_PROVIDER As New ANumbers(ANumbers.Cultures.EUR)

diff --git a/SCrawler/API/Reddit/M3U8.vb b/SCrawler/API/Reddit/M3U8.vb
@@ -9,26 +9,25 @@
 Imports System.Net
 Imports SCrawler.API.Reddit.M3U8_Declarations
 Imports PersonalUtilities.Tools.WEB
-Namespace API.Reddit.M3U8_Declarations
-    Friend Module M3U8_Declarations
-        Friend ReadOnly BaseUrlPattern As New RegexStructure("([htps:/]{7,8}.+?/.+?)(?=/)", True, False,,,,,, EDP.ReturnValue)
-        Friend ReadOnly PlayListRegEx_1 As New RegexStructure("(#EXT-X-STREAM-INF)(.+)(RESOLUTION=)(\d+)(.+?[\r\n]{1,2})(.+?)([\r\n]{1,2})", True, False,,,
-                                                              RegexReturn.List,, New List(Of String),
-                                                              New ErrorsDescriber(False, False, True, New List(Of String)))
-        Friend ReadOnly PlayListRegEx_2 As New RegexStructure("(?<=#EXT-X-BYTERANGE.+?[\r\n]{1,2})(.+)(?=[\r\n]{0,2})", True, False,,, RegexReturn.List,,
-                                                              New List(Of String),
-                                                              New ErrorsDescriber(False, False, True, New List(Of String)))
-        Friend ReadOnly DPED As New ErrorsDescriber(EDP.SendInLog + EDP.ReturnValue)
-    End Module
-End Namespace
+Imports PersonalUtilities.Functions.RegularExpressions
 Namespace API.Reddit
+    Namespace M3U8_Declarations
+        Friend Module M3U8_Declarations
+            Friend ReadOnly BaseUrlPattern As RParams = RParams.DM("([htps:/]{7,8}.+?/.+?)(?=/)", 0, EDP.ReturnValue)
+            Friend ReadOnly PlayListRegEx_1 As RParams = RParams.DM("(#EXT-X-STREAM-INF)(.+)(RESOLUTION=)(\d+)(.+?[\r\n]{1,2})(.+?)([\r\n]{1,2})", 0,
+                                                                    RegexReturn.List, EDP.SendInLog, EDP.ReturnValue)
+            Friend ReadOnly PlayListRegEx_2 As RParams = RParams.DM("(?<=#EXT-X-BYTERANGE.+?[\r\n]{1,2})(.+)(?=[\r\n]{0,2})", 0,
+                                                                    RegexReturn.List, EDP.SendInLog, EDP.ReturnValue)
+            Friend ReadOnly DPED As New ErrorsDescriber(EDP.SendInLog + EDP.ReturnValue)
+        End Module
+    End Namespace
     Friend NotInheritable Class M3U8
         Private Sub New()
         End Sub
         Private Structure Resolution : Implements IRegExCreator, IComparable(Of Resolution)
             Friend File As String
             Friend Resolution As Integer
-            Friend Function CreateFromArray(ByVal ParamsArray() As String) As IRegExCreator Implements IRegExCreator.CreateFromArray
+            Friend Function CreateFromArray(ByVal ParamsArray() As String) As Object Implements IRegExCreator.CreateFromArray
                 If ParamsArray.ArrayExists Then
                     File = ParamsArray(0)
                     If ParamsArray.Length > 1 Then Resolution = AConvert(Of Integer)(ParamsArray(1), 0)
@@ -45,7 +44,7 @@ Namespace API.Reddit
                     Using w As New WebClient
                         Dim r$ = w.DownloadString(PlayListURL)
                         If Not r.IsEmptyString Then
-                            Dim l As List(Of Resolution) = RegexFields(Of Resolution)(r, {PlayListRegEx_1}, {6, 4})
+                            Dim l As List(Of Resolution) = FNF.RegexFields(Of Resolution)(r, {PlayListRegEx_1}, {6, 4})
                             If l.ListExists Then
                                 l.Sort()
                                 Dim pls$ = $"{BaseUrl}/{l.First.File}"

diff --git a/SCrawler/API/Reddit/ProfileSaved.vb b/SCrawler/API/Reddit/ProfileSaved.vb
@@ -11,7 +11,7 @@ Imports System.Threading
 Imports PersonalUtilities.Forms.Toolbars
 Namespace API.Reddit
     Friend NotInheritable Class ProfileSaved
-        Friend Shared ReadOnly Property DataPath As SFile = $"{Settings(Sites.Reddit).Path.PathNoSeparator}\!Saved\"
+        Friend Shared ReadOnly Property DataPath As SFile = Settings(Sites.Reddit).SavedPostsPath
         Private Sub New()
         End Sub
         Friend Shared Sub Download(ByRef Bar As MyProgress, ByVal Token As CancellationToken)