Skip to content

Commit

Permalink
1.0.0.2
Browse files Browse the repository at this point in the history 
Add: downloading certain types of media, name files by date
Fix: Stop button is not activated after download start
  • Loading branch information
@AAndyProgram
AAndyProgram committed Dec 10, 2021
1 parent c7dc0ec commit 55a28a7
Show file tree
Hide file tree
Showing 16 changed files with 597 additions and 124 deletions.
8 changes: 8 additions & 0 deletions Changelog.md
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,11 @@
# 1.0.0.2

- Added
- Ability to choose what types of media you want to download (images only, videos only, both)
- Ability to name files by date
- Fixed
- In some cases, the "Stop" button is not activated after download start

# 1.0.0.1

- Added
Expand Down
1 change: 1 addition & 0 deletions SCrawler/API/Base/Structures.vb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
''' <summary>Post ID</summary>
Friend ID As String
Friend [Date] As Date?
Friend Title As String
#Region "Channel compatible fields"
Friend UserID As String
Friend CachedFile As SFile
Expand Down
39 changes: 39 additions & 0 deletions SCrawler/API/Base/UserDataBase.vb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ Namespace API.Base
Private Const Name_LabelsName As String = "Labels"

Private Const Name_ReadyForDownload As String = "ReadyForDownload"
Private Const Name_DownloadImages As String = "DownloadImages"
Private Const Name_DownloadVideos As String = "DownloadVideos"

Private Const Name_VideoCount As String = "VideoCount"
Private Const Name_PicturesCount As String = "PicturesCount"
Expand Down Expand Up @@ -233,6 +235,8 @@ BlockNullPicture:
Protected _DataLoaded As Boolean = False
Protected _DataParsed As Boolean = False
Friend Property ReadyForDownload As Boolean = True Implements IUserData.ReadyForDownload
Friend Property DownloadImages As Boolean = True Implements IUserData.DownloadImages
Friend Property DownloadVideos As Boolean = True Implements IUserData.DownloadVideos
#End Region
#Region "Content"
Protected ReadOnly _ContentList As List(Of UserMedia)
Expand Down Expand Up @@ -431,6 +435,8 @@ BlockNullPicture:
CreatedByChannel = x.Value(Name_CreatedByChannel).FromXML(Of Boolean)(False)
SeparateVideoFolder = AConvert(Of Boolean)(x.Value(Name_SeparateVideoFolder), Nothing)
ReadyForDownload = x.Value(Name_ReadyForDownload).FromXML(Of Boolean)(True)
DownloadImages = x.Value(Name_DownloadImages).FromXML(Of Boolean)(True)
DownloadedVideos = x.Value(Name_DownloadVideos).FromXML(Of Boolean)(True)
_CountVideo = x.Value(Name_VideoCount).FromXML(Of Integer)(0)
_CountPictures = x.Value(Name_PicturesCount).FromXML(Of Integer)(0)
LastUpdated = AConvert(Of Date)(x.Value(Name_LastUpdated), ADateTime.Formats.BaseDateTime, Nothing)
Expand Down Expand Up @@ -463,6 +469,8 @@ BlockNullPicture:
x.Add(Name_SeparateVideoFolder, String.Empty)
End If
x.Add(Name_ReadyForDownload, ReadyForDownload.BoolToInteger)
x.Add(Name_DownloadImages, DownloadImages.BoolToInteger)
x.Add(Name_DownloadVideos, DownloadVideos.BoolToInteger)
x.Add(Name_VideoCount, _CountVideo)
x.Add(Name_PicturesCount, _CountPictures)
x.Add(Name_LastUpdated, AConvert(Of String)(LastUpdated, ADateTime.Formats.BaseDateTime, String.Empty))
Expand Down Expand Up @@ -586,6 +594,12 @@ BlockNullPicture:
DownloadDataF(Token)
ThrowAny(Token)

If _TempMediaList.Count > 0 Then
If Not DownloadImages Then _TempMediaList.RemoveAll(Function(m) m.Type = UserMedia.Types.GIF Or m.Type = UserMedia.Types.Picture)
If Not DownloadVideos Then _TempMediaList.RemoveAll(Function(m) m.Type = UserMedia.Types.Video Or
m.Type = UserMedia.Types.VideoPre Or m.Type = UserMedia.Types.m3u8)
End If

ReparseVideo(Token)
ThrowAny(Token)
If _TempPostsList.Count > 0 And __SaveData Then TextSaver.SaveTextToFile(_TempPostsList.ListToString(, Environment.NewLine), MyFilePosts, True,, EDP.None)
Expand Down Expand Up @@ -640,6 +654,29 @@ BlockNullPicture:
Protected MustOverride Sub DownloadDataF(ByVal Token As CancellationToken)
Protected MustOverride Sub ReparseVideo(ByVal Token As CancellationToken)
Protected MustOverride Sub DownloadContent(ByVal Token As CancellationToken)
Protected Function ChangeFileNameByProvider(ByVal f As SFile, ByVal m As UserMedia) As SFile
Dim ff As SFile = Nothing
Try
If Not f.IsEmptyString AndAlso f.Exists Then
Dim d As Date? = m.Post.Date
If Settings.FileReplaceNameByDate Then
Dim dd$ = AConvert(Of String)(If(d, Now), FileDateAppenderProvider, String.Empty)
ff = f
ff.Name = dd
ff = SFile.Indexed_IndexFile(ff,, New NumberedFile(ff))
ElseIf d.HasValue AndAlso (Settings.FileAddDateToFileName Or Settings.FileAddTimeToFileName) AndAlso
(Not FileDateAppenderProvider Is Nothing And Not FileDateAppenderPattern.IsEmptyString) Then
ff = f
ff.Name = String.Format(FileDateAppenderPattern, f.Name, CStr(AConvert(Of String)(d.Value, FileDateAppenderProvider, String.Empty)))
End If
If Not ff.Name.IsEmptyString Then My.Computer.FileSystem.RenameFile(f, ff.File) : Return ff
End If
Return f
Catch ex As Exception
LogError(ex, $"change file name from [{f}] to [{ff}]")
Return f
End Try
End Function
#End Region
#Region "Delete, Move, Merge"
Friend Overridable Function Delete() As Integer Implements IUserData.Delete
Expand Down Expand Up @@ -916,6 +953,8 @@ BlockNullPicture:
ReadOnly Property FitToAddParams As Boolean
ReadOnly Property LVIKey As String
ReadOnly Property LVIIndex As Integer
Property DownloadImages As Boolean
Property DownloadVideos As Boolean
Function GetLVI(ByVal Destination As ListView) As ListViewItem
Function GetLVIGroup(ByVal Destination As ListView) As ListViewGroup
Sub LoadUserInformation()
Expand Down
47 changes: 30 additions & 17 deletions SCrawler/API/Reddit/UserData.vb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ Namespace API.Reddit
Dim URL$ = String.Empty
Try
Dim PostID$ = String.Empty
Dim PostDate$
Dim PostDate$, PostTitle$
Dim n As EContainer, nn As EContainer, s As EContainer
Dim NewPostDetected As Boolean = False
Dim ExistsDetected As Boolean = False
Expand All @@ -107,7 +107,7 @@ Namespace API.Reddit
ThrowAny(Token)
Dim r$ = GetSiteResponse(URL)
If Not r.IsEmptyString Then
Using w As EContainer = JsonDocument.Parse(r)
Using w As EContainer = JsonDocument.Parse(r).XmlIfNothing
If w.Count > 0 Then
n = w.GetNode(JsonNodesJson)
If Not n Is Nothing AndAlso n.Count > 0 Then
Expand All @@ -124,29 +124,32 @@ Namespace API.Reddit
ExistsDetected = True
Continue For
End If
PostTitle = nn.Value("title")

If CheckNode(nn) Then
_ItemsBefore = _TempMediaList.Count
added = True
s = nn.ItemF({"source", "url"})
If s.XmlIfNothingValue("/").Contains("redgifs.com") Then
_TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre, s.Value, PostID, PostDate,, IsChannel), LNC)
_TotalPostsDownloaded += 1
_TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre, s.Value, PostID, PostDate,, IsChannel, PostTitle), LNC)
Else
s = nn.ItemF({"media"}).XmlIfNothing
__ItemType = s("type").XmlIfNothingValue
Select Case __ItemType
Case "gallery" : DownloadGallery(s, PostID, PostDate) : _TotalPostsDownloaded += 1
Case "gallery" : If Not DownloadGallery(s, PostID, PostDate,,, PostTitle) Then added = False
Case "image", "gifvideo"
If s.Contains("content") Then
_TempMediaList.ListAddValue(MediaFromData(UPicType(__ItemType), s.Value("content"),
PostID, PostDate,, IsChannel), LNC)
_TotalPostsDownloaded += 1
PostID, PostDate,, IsChannel, PostTitle), LNC)
Else
added = False
End If
Case "video"
If Settings.UseM3U8 AndAlso s("hlsUrl").XmlIfNothingValue("/").ToLower.Contains("m3u8") Then
_TempMediaList.ListAddValue(MediaFromData(UTypes.m3u8, s.Value("hlsUrl"),
PostID, PostDate,, IsChannel), LNC)
_TotalPostsDownloaded += 1
PostID, PostDate,, IsChannel, PostTitle), LNC)
Else
added = False
End If
Case Else : added = False
End Select
Expand All @@ -164,8 +167,7 @@ Namespace API.Reddit
End Select
End With
If Not tmpType = UTypes.Undefined Then
_TempMediaList.ListAddValue(MediaFromData(tmpType, s.Value, PostID, PostDate,, IsChannel), LNC)
_TotalPostsDownloaded += 1
_TempMediaList.ListAddValue(MediaFromData(tmpType, s.Value, PostID, PostDate,, IsChannel, PostTitle), LNC)
End If
End If
End If
Expand Down Expand Up @@ -267,25 +269,30 @@ Namespace API.Reddit
End Sub
#End Region
#Region "Download Base Functions"
Private Sub DownloadGallery(ByVal w As EContainer, ByVal PostID As String, ByVal PostDate As String,
Optional ByVal _UserID As String = Nothing, Optional ByVal FirstOnly As Boolean = False)
Private Function DownloadGallery(ByVal w As EContainer, ByVal PostID As String, ByVal PostDate As String,
Optional ByVal _UserID As String = Nothing, Optional ByVal FirstOnly As Boolean = False,
Optional ByVal Title As String = Nothing) As Boolean
Try
Dim added As Boolean = False
Dim cn$ = IIf(IsChannel, "media_metadata", "mediaMetadata")
If Not w Is Nothing AndAlso w(cn).XmlIfNothing.Count > 0 Then
Dim t As EContainer
For Each n As EContainer In w(cn)
t = n.ItemF({"s", "u"})
If Not t Is Nothing AndAlso Not t.Value.IsEmptyString Then
_TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, t.Value, PostID, PostDate, _UserID, IsChannel), LNC)
_TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, t.Value, PostID, PostDate, _UserID, IsChannel, Title), LNC)
added = True
If FirstOnly Then Exit For
End If
Next
End If
Return added
Catch ex As Exception
LogError(ex, "gallery parsing error")
HasError = True
Return False
End Try
End Sub
End Function
Protected Overrides Sub ReparseVideo(ByVal Token As CancellationToken)
Try
ThrowAny(Token)
Expand Down Expand Up @@ -333,13 +340,15 @@ Namespace API.Reddit
#End Region
#Region "Structure creator"
Protected Shared Function MediaFromData(ByVal t As UTypes, ByVal _URL As String, ByVal PostID As String, ByVal PostDate As String,
Optional ByVal _UserID As String = "", Optional ByVal IsChannel As Boolean = False) As UserMedia
Optional ByVal _UserID As String = "", Optional ByVal IsChannel As Boolean = False,
Optional ByVal Title As String = Nothing) As UserMedia
If _URL.IsEmptyString And t = UTypes.Picture Then Return Nothing
_URL = LinkFormatterSecure(RegexReplace(_URL.Replace("\", String.Empty), LinkPattern))
Dim m As New UserMedia(_URL, t) With {.Post = New UserPost With {.ID = PostID, .UserID = _UserID}}
If t = UTypes.Picture Or t = UTypes.GIF Then m.File = UrlToFile(m.URL) Else m.File = Nothing
If m.URL.Contains("preview") Then m.URL = $"https://i.redd.it/{m.File.File}"
If Not PostDate.IsEmptyString Then m.Post.Date = AConvert(Of Date)(PostDate, If(IsChannel, DateProviderChannel, DateProvider), Nothing) Else m.Post.Date = Nothing
If Not Title.IsEmptyString Then m.Post.Title = Title
Return m
End Function
Private Function TryFile(ByVal URL As String) As Boolean
Expand Down Expand Up @@ -436,7 +445,11 @@ Namespace API.Reddit
Case UTypes.Picture : DownloadedPictures += 1 : _CountPictures += 1
Case UTypes.Video, UTypes.m3u8 : DownloadedVideos += 1 : _CountVideo += 1
End Select
v.File = f
If Not IsChannel Or Not SaveToCache Then
v.File = ChangeFileNameByProvider(f, v)
Else
v.File = f
End If
v.Post.CachedFile = f
v.State = UStates.Downloaded
dCount += 1
Expand Down
Loading

0 comments on commit 55a28a7

Please sign in to comment.