1.0.0.1

Added: limited downloads, remove active downloads, simulation tasks management, copy channel image, x86
Removed reparse function
Fixed some errors, typo and Reddit parsing algo

Changelog.md

@@ -0,0 +1,25 @@
+# 1.0.0.1
+
+- Added
+  - Limited download if user added from the channel
+  - Forced limited download for any user
+  - x86 compatibility
+  - Coping user image posted in the channel (if user added from the channel)
+  - Check for new version at start setting
+  - Removing currently downloading user
+  - Change maximum count of along downloading tasks of users
+  - Change maximum count of along downloading tasks of channels
+- Removed
+  - Reparse not downloaded content (left from the older versions)
+- Fixed
+  - ```No Label``` and ```No Parsed``` labels does not shown in the labels list
+  - User list does not refresh by labels change in the main window
+  - Disabled collection editing
+  - Collection name does not show in some operations
+  - Error (in some cases) on add to collection
+  - Wrong some Reddit videos parsing
+  - Wrong some Reddit images parsing
+
+# 1.0.0.0
+
+Initial release

SCrawler.sln

@@ -10,8 +10,10 @@ EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{FE5CE5B9-DDBE-4399-A17C-880893635307}"
 	ProjectSection(SolutionItems) = preProject
 		.gitignore = .gitignore
+		Changelog.md = Changelog.md
 		README.md = README.md
 		Info\RedditUrlsInfo.txt = Info\RedditUrlsInfo.txt
+		ToDo.txt = ToDo.txt
 		Info\TwitterNewAlgo.txt = Info\TwitterNewAlgo.txt
 		Info\TwitterUrlsInfo.txt = Info\TwitterUrlsInfo.txt
 	EndProjectSection

SCrawler/API/Base/UserDataBase.vb

@@ -40,6 +40,7 @@ Namespace API.Base
         Private Const Name_ParseUserMediaOnly As String = "ParseUserMediaOnly"
         Private Const Name_Temporary As String = "Temporary"
         Private Const Name_Favorite As String = "Favorite"
+        Private Const Name_CreatedByChannel As String = "CreatedByChannel"
 
         Private Const Name_SeparateVideoFolder As String = "SeparateVideoFolder"
         Private Const Name_CollectionName As String = "Collection"
@@ -103,6 +104,7 @@ Namespace API.Base
                 Return User.IsChannel
             End Get
         End Property
+        Friend Property CreatedByChannel As Boolean = False
         Friend ReadOnly Property Self As IUserData Implements IUserData.Self
             Get
                 Return Me
@@ -235,7 +237,6 @@ BlockNullPicture:
 #Region "Content"
         Protected ReadOnly _ContentList As List(Of UserMedia)
         Protected ReadOnly _ContentNew As List(Of UserMedia)
-        Protected ReadOnly _ContentForReparse As List(Of UserMedia)
         Protected ReadOnly _TempMediaList As List(Of UserMedia)
         Protected ReadOnly _TempPostsList As List(Of String)
 #End Region
@@ -250,8 +251,6 @@ BlockNullPicture:
             End Set
         End Property
         Protected MyFileData As SFile
-        Protected MyFileDataR As SFile
-        Protected MyFileDataRV As SFile
         Protected MyFilePosts As SFile
         Friend Overridable Property FileExists As Boolean = False Implements IUserData.FileExists
         Friend Overridable Property DataMerging As Boolean
@@ -398,7 +397,6 @@ BlockNullPicture:
             _InvokeImageHandler = InvokeImageHandler
             _ContentList = New List(Of UserMedia)
             _ContentNew = New List(Of UserMedia)
-            _ContentForReparse = New List(Of UserMedia)
             _TempMediaList = New List(Of UserMedia)
             _TempPostsList = New List(Of String)
             Labels = New List(Of String)
@@ -430,6 +428,7 @@ BlockNullPicture:
                         ParseUserMediaOnly = x.Value(Name_ParseUserMediaOnly).FromXML(Of Boolean)(False)
                         Temporary = x.Value(Name_Temporary).FromXML(Of Boolean)(False)
                         Favorite = x.Value(Name_Favorite).FromXML(Of Boolean)(False)
+                        CreatedByChannel = x.Value(Name_CreatedByChannel).FromXML(Of Boolean)(False)
                         SeparateVideoFolder = AConvert(Of Boolean)(x.Value(Name_SeparateVideoFolder), Nothing)
                         ReadyForDownload = x.Value(Name_ReadyForDownload).FromXML(Of Boolean)(True)
                         _CountVideo = x.Value(Name_VideoCount).FromXML(Of Integer)(0)
@@ -440,7 +439,6 @@ BlockNullPicture:
                         Labels.ListAddList(x.Value(Name_LabelsName).StringToList(Of String, List(Of String))("|", EDP.ReturnValue), LAP.NotContainsOnly, LAP.ClearBeforeAdd)
                     End Using
                     UpdateDataFiles()
-                    _DataForReparseExists = MyFileDataR.Exists
                 End If
             Catch ex As Exception
                 LogError(ex, "user information loading error")
@@ -458,6 +456,7 @@ BlockNullPicture:
                     x.Add(Name_ParseUserMediaOnly, ParseUserMediaOnly.BoolToInteger)
                     x.Add(Name_Temporary, Temporary.BoolToInteger)
                     x.Add(Name_Favorite, Favorite.BoolToInteger)
+                    x.Add(Name_CreatedByChannel, CreatedByChannel.BoolToInteger)
                     If SeparateVideoFolder.HasValue Then
                         x.Add(Name_SeparateVideoFolder, SeparateVideoFolder.Value.BoolToInteger)
                     Else
@@ -483,9 +482,6 @@ BlockNullPicture:
         Friend Overridable Overloads Sub LoadContentInformation()
             UpdateDataFiles()
             LoadContentInformation(_ContentList, MyFileData)
-            LoadContentInformation(_ContentForReparse, MyFileDataR)
-            LoadContentInformation(_TempMediaList, MyFileDataRV)
-            _DataForReparseExists = False
         End Sub
         Private Overloads Sub LoadContentInformation(ByRef _CLIST As List(Of UserMedia), ByVal f As SFile)
             Try
@@ -570,13 +566,7 @@ BlockNullPicture:
         End Sub
 #End Region
 #Region "Download functions and options"
-        Friend Overridable Property DownloadReparseOnly As Boolean = False Implements IUserData.DownloadReparseOnly
-        Private _DataForReparseExists As Boolean = False
-        Friend Overridable ReadOnly Property DataForReparseExists As Boolean Implements IUserData.DataForReparseExists
-            Get
-                Return _ContentForReparse.Count > 0 Or _DataForReparseExists
-            End Get
-        End Property
+        Friend Overridable Property DownloadTopCount As Integer? = Nothing Implements IUserData.DownloadTopCount
         Friend Overridable Sub DownloadData(ByVal Token As CancellationToken) Implements IContentProvider.DownloadData
             Dim Canceled As Boolean = False
             Try
@@ -586,79 +576,60 @@ BlockNullPicture:
                 _DownloadedVideosSession = 0
                 _TempMediaList.Clear()
                 _TempPostsList.Clear()
+                Dim __SaveData As Boolean = Not CreatedByChannel Or Not Settings.FromChannelDownloadTopUse
 
                 If Not _DataLoaded Then LoadContentInformation()
 
-                If Not DownloadReparseOnly Then
-                    If MyFilePosts.Exists Then _TempPostsList.ListAddList(File.ReadAllLines(MyFilePosts))
-                    If _ContentList.Count > 0 Then _TempPostsList.ListAddList(_ContentList.Select(Function(u) u.Post.ID), LNC)
-                    Token.ThrowIfCancellationRequested()
-                    DownloadDataF(Token)
-                    Token.ThrowIfCancellationRequested()
-                End If
+                If MyFilePosts.Exists Then _TempPostsList.ListAddList(File.ReadAllLines(MyFilePosts))
+                If _ContentList.Count > 0 Then _TempPostsList.ListAddList(_ContentList.Select(Function(u) u.Post.ID), LNC)
+                ThrowAny(Token)
+                DownloadDataF(Token)
+                ThrowAny(Token)
 
                 ReparseVideo(Token)
-                If Token.IsCancellationRequested Then
-                    If Not DownloadReparseOnly Then
-                        If _TempMediaList.Count > 0 AndAlso _TempMediaList.Exists(Function(c) c.Type = UserMedia.Types.VideoPre) Then
-                            TextSaver.SaveTextToFile((From c As UserMedia In _TempMediaList
-                                                      Where c.Type = UserMedia.Types.VideoPre
-                                                      Select c.URL).ListToString(, Environment.NewLine), MyFileDataRV, True,, EDP.SendInLog)
-                        Else
-                            If MyFileDataRV.Exists Then MyFileDataRV.Delete(,,, EDP.SendInLog)
-                        End If
-                    End If
-                Else
-                    If Not DownloadReparseOnly And _TempPostsList.Count > 0 Then TextSaver.SaveTextToFile(_TempPostsList.ListToString(, Environment.NewLine), MyFilePosts, True,, EDP.None)
-                    _ContentNew.ListAddList(_TempMediaList, LAP.ClearBeforeAdd)
-                End If
-
-                Dim r% = 0
-                Do While r <= 2 And (_ContentNew.Count > 0 Or _ContentForReparse.Count > 0) And Not Token.IsCancellationRequested
-                    DownloadContent(Token)
-                    If _ContentNew.Count > 0 Then
-                        _ContentForReparse.ListAddList(_ContentNew.Where(Function(c) c.State = UState.Tried Or c.State = UState.Unknown), LNC)
-                        _ContentList.ListAddList(_ContentNew.Where(Function(c) c.State = UState.Downloaded), LNC)
-                    End If
-                    If _ContentForReparse.Count > 0 Then _ContentForReparse.RemoveAll(Function(c) _ContentList.Contains(c))
-                    _ContentNew.Clear()
-                    If _ContentForReparse.Count > 0 Then _ContentNew.ListAddList(_ContentForReparse, LNC)
-                    r += 1
-                Loop
-
+                ThrowAny(Token)
+                If _TempPostsList.Count > 0 And __SaveData Then TextSaver.SaveTextToFile(_TempPostsList.ListToString(, Environment.NewLine), MyFilePosts, True,, EDP.None)
+                _ContentNew.ListAddList(_TempMediaList, LAP.ClearBeforeAdd)
+                DownloadContent(Token)
+                ThrowIfDisposed()
+                _ContentList.ListAddList(_ContentNew.Where(Function(c) c.State = UState.Downloaded), LNC)
                 _CountPictures = _ContentList.LongCount(Function(c) c.Type = UserMedia.Types.Picture)
                 _CountVideo = _ContentList.LongCount(Function(c) c.Type = UserMedia.Types.Video)
                 If DownloadedPictures + DownloadedVideos > 0 Then
-                    LastUpdated = Now
-                    If Labels.Contains(LabelsKeeper.NoParsedUser) Then Labels.Remove(LabelsKeeper.NoParsedUser)
-                    UpdateContentInformation(_ContentList, MyFileData)
+                    If __SaveData Then
+                        LastUpdated = Now
+                        If Labels.Contains(LabelsKeeper.NoParsedUser) Then Labels.Remove(LabelsKeeper.NoParsedUser)
+                        UpdateContentInformation(_ContentList, MyFileData)
+                    Else
+                        _CountVideo = 0
+                        _CountPictures = 0
+                        _ContentList.Clear()
+                        CreatedByChannel = False
+                    End If
                     UpdateUserInformation()
                 End If
-                If _ContentForReparse.Count > 0 Then UpdateContentInformation(_ContentForReparse, MyFileDataR)
+                ThrowIfDisposed()
                 _DownloadedPicturesTotal += _DownloadedPicturesSession
                 _DownloadedVideosTotal += _DownloadedVideosSession
                 If UpPic Then Raise_OnPictureUpdated()
             Catch oex As OperationCanceledException When Token.IsCancellationRequested
                 MyMainLOG = $"{Site} - {Name}: downloading canceled"
                 Canceled = True
+            Catch dex As ObjectDisposedException When Disposed
+                Canceled = True
             Catch ex As Exception
                 LogError(ex, "downloading data error")
                 HasError = True
             Finally
                 If Not Canceled Then _DataParsed = True ': LastUpdated = Now
                 _ContentNew.Clear()
-                DownloadReparseOnly = False
-                If _ContentForReparse.Count = 0 And MyFileDataR.Exists Then MyFileDataR.Delete(,,, EDP.SendInLog)
+                DownloadTopCount = Nothing
             End Try
         End Sub
         Private Sub UpdateDataFiles()
             If Not User.File.IsEmptyString Then
                 MyFileData = User.File
                 MyFileData.Name &= "_Data"
-                MyFileDataR = MyFileData
-                MyFileDataR.Name &= "_REPARSE"
-                MyFileDataRV = MyFileData
-                MyFileDataRV.Name &= "_RVideo"
                 MyFilePosts = User.File
                 MyFilePosts.Name &= "_Posts"
                 MyFilePosts.Extension = "txt"
@@ -687,9 +658,10 @@ BlockNullPicture:
         End Function
         Friend Overridable Function MoveFiles(ByVal __CollectionName As String, ByVal _MergeData As Boolean) As Boolean Implements IUserData.MoveFiles
             Dim UserBefore As UserInfo = User
+            Dim Removed As Boolean = True
+            Dim _TurnBack As Boolean = False
             Try
                 Dim f As SFile
-                Dim Removed As Boolean
                 If IncludedInCollection Then
                     Settings.Users.Add(Me)
                     Removed = False
@@ -701,6 +673,7 @@ BlockNullPicture:
                     User.CollectionName = __CollectionName
                     User.IncludedInCollection = True
                 End If
+                _TurnBack = True
                 User.UpdateUserFile()
                 f = User.File.CutPath(, EDP.ThrowException)
                 If f.Exists(SFO.Path, False) Then
@@ -711,6 +684,7 @@ BlockNullPicture:
                         MsgBoxE("Operation canceled", MsgBoxStyle.Exclamation)
                         User = UserBefore
                         If Removed Then Settings.Users.Add(Me) Else Settings.Users.Remove(Me)
+                        _TurnBack = False
                         Return False
                     End If
                     f.Delete(SFO.Path, False, False, EDP.ThrowException)
@@ -724,6 +698,9 @@ BlockNullPicture:
             Catch ex As Exception
                 ErrorsDescriber.Execute(EDP.LogMessageValue, ex, "Files moving error")
                 User = UserBefore
+                If _TurnBack Then
+                    If Removed Then Settings.Users.Add(Me) Else Settings.Users.Remove(Me)
+                End If
                 Return False
             End Try
         End Function
@@ -789,6 +766,16 @@ BlockNullPicture:
         Protected Sub ErrorDownloading(ByVal f As SFile, ByVal URL As String)
             If Not f.Exists Then MyMainLOG = $"Error downloading from [{URL}] to [{f}]"
         End Sub
+        ''' <exception cref="ObjectDisposedException"></exception>
+        Protected Sub ThrowIfDisposed()
+            If Disposed Then Throw New ObjectDisposedException(ToString(), "Object disposed")
+        End Sub
+        ''' <exception cref="OperationCanceledException"></exception>
+        ''' <exception cref="ObjectDisposedException"></exception>
+        Protected Sub ThrowAny(ByVal Token As CancellationToken)
+            Token.ThrowIfCancellationRequested()
+            ThrowIfDisposed()
+        End Sub
 #End Region
         Public Overrides Function ToString() As String
             If Settings.ViewModeIsPicture Then
@@ -861,7 +848,7 @@ BlockNullPicture:
 #End Region
 #Region "IDisposable Support"
         Protected disposedValue As Boolean = False
-        Friend ReadOnly Property Disposed As Boolean
+        Friend ReadOnly Property Disposed As Boolean Implements IUserData.Disposed
             Get
                 Return disposedValue
             End Get
@@ -942,9 +929,9 @@ BlockNullPicture:
         Function Delete() As Integer
         Function MoveFiles(ByVal CollectionName As String, ByVal MergeData As Boolean) As Boolean
         Sub OpenFolder()
-        Property DownloadReparseOnly As Boolean
-        ReadOnly Property DataForReparseExists As Boolean
         ReadOnly Property Self As IUserData
+        Property DownloadTopCount As Integer?
+        ReadOnly Property Disposed As Boolean
     End Interface
     Friend Interface IChannelLimits
         Property AutoGetLimits As Boolean

SCrawler/API/Reddit/Channel.vb

@@ -222,7 +222,7 @@ Namespace API.Reddit
             End If
         End Function
 #End Region
-#Region "IXMLContainer Support"
+#Region "ILoaderSaver Support"
         Friend Overloads Function LoadData(Optional ByVal f As SFile = Nothing, Optional ByVal e As ErrorsDescriber = Nothing) As Boolean Implements ILoaderSaver.Load
             Return LoadData(File, False, e)
         End Function

SCrawler/API/Reddit/ChannelsCollection.vb

@@ -72,16 +72,18 @@ Namespace API.Reddit
         Friend Sub DownloadData(ByVal Token As CancellationToken, Optional ByVal SkipExists As Boolean = True,
                                 Optional ByVal p As MyProgress = Nothing)
             Try
+                Dim m% = Settings.ChannelsMaxJobsCount
                 If Count > 0 Then
                     Dim t As New List(Of Task)
+                    Dim i% = 0
                     For Each c As Channel In Channels
                         If Not c.Downloading Then t.Add(Task.Run(Sub()
                                                                      c.SetLimit(Me)
                                                                      c.DownloadData(Token, SkipExists, p)
-                                                                 End Sub))
+                                                                 End Sub)) : i += 1
+                        If t.Count > 0 And i >= m Then Task.WaitAll(t.ToArray, Token) : t.Clear() : i = 0
                     Next
-                    If t.Count > 0 Then Task.WaitAll(t.ToArray)
-                    Token.ThrowIfCancellationRequested()
+                    If t.Count > 0 Then Task.WaitAll(t.ToArray, Token) : t.Clear()
                 End If
             Catch oex As OperationCanceledException When Token.IsCancellationRequested
             End Try

SCrawler/API/Reddit/M3U8.vb

@@ -80,7 +80,7 @@ Namespace API.Reddit
                                 eFiles.Add(dFile)
                             Next
                         End Using
-                        f = FFMPEG.ConcatenateFiles(eFiles, "ffmpeg.exe", ConcatFile, p, DPED)
+                        f = FFMPEG.ConcatenateFiles(eFiles, Settings.FfmpegFile, ConcatFile, p, DPED)
                         eFiles.Clear()
                         Return f
                     End If