diff --git a/Changelog.md b/Changelog.md index fe94de6..92b66d8 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,22 @@ +# 2024.6.4.0 + +*2024-06-04* + +**If you were using the [`yt-dlp-TTUser`](https://github.com/bashonly/yt-dlp-TTUser) plugin, you should remove it because this plugin was added to yt-dlp itself! Read more [here](https://github.com/AAndyProgram/SCrawler/wiki/Settings#tiktok-requirements).** + +- Added + - Added highlighting of scheduler plans (working, stopped, pending, etc.) + - YouTube (standalone app): add option to add the video upload date before/after the file name (`Settings` - `Defaults` - `Add date to file name`) + - Twitter: **`Communities` downloading** + - Feed: ability to select one of the download sessions and set it as the current session + - Minor improvements +- Updated + - yt-dlp up to version **2024.05.27** + - gallery-dl up to version **1.27.0** +- Fixed + - Twitter: deleting user directory when redownloading missing posts + - Minor bugs + # 2024.5.19.0 *2024-05-19* diff --git a/ProgramScreenshots/SettingsGlobalDownloading.png b/ProgramScreenshots/SettingsGlobalDownloading.png index 7eb466f..a4a9843 100644 Binary files a/ProgramScreenshots/SettingsGlobalDownloading.png and b/ProgramScreenshots/SettingsGlobalDownloading.png differ diff --git a/README.md b/README.md index b834f0f..1b7fe77 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ - +# 🏳️‍🌈 Happy LGBT Pride Month 🎉 + # 🏳️‍🌈 Social networks crawler 🏳️‍🌈 [![GitHub release (latest by date)](https://img.shields.io/github/v/release/AAndyProgram/SCrawler)](https://github.com/AAndyProgram/SCrawler/releases/latest) @@ -37,8 +37,8 @@ A program to download photo and video from [any site](#supported-sites) (e.g. Yo - YouTube videos, shorts, community feeds, users, artists, playlists, music, tracks; - Reddit images, galleries of images, videos, saved posts; - Redgifs videos (https://www.redgifs.com/); - - Twitter images and videos, saved (bookmarked) posts; - - OnlyFans images and videos, saved (bookmarked) posts; + - Twitter images and videos, saved (bookmarked) posts, likes, communities; + - OnlyFans images and videos, saved (bookmarked) posts, stories; - JustForFans images and videos, saved (bookmarked) posts; - Mastodon images and videos, saved (bookmarked) posts; - Instagram images and videos, tagged posts, stories, saved posts; @@ -79,11 +79,11 @@ A program to download photo and video from [any site](#supported-sites) (e.g. Yo - **Reddit** - **Twitter** - **OnlyFans** *(partial support)*[^1] -- **Mastodon** - **Instagram** - **Threads** - **Facebook** - JustForFans *(partial support)*[^1] +- Mastodon *(out of support)* - TikTok - RedGifs - Pinterest diff --git a/SCrawler.YouTube/My Project/AssemblyInfo.vb b/SCrawler.YouTube/My Project/AssemblyInfo.vb index 47e54e6..9c8577a 100644 --- a/SCrawler.YouTube/My Project/AssemblyInfo.vb +++ b/SCrawler.YouTube/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + + diff --git a/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb b/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb index 272d70b..8940716 100644 --- a/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb +++ b/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + + diff --git a/SCrawler/API/Base/UserDataBase.vb b/SCrawler/API/Base/UserDataBase.vb index 154f7e8..1bd1d2e 100644 --- a/SCrawler/API/Base/UserDataBase.vb +++ b/SCrawler/API/Base/UserDataBase.vb @@ -143,7 +143,7 @@ Namespace API.Base Protected Const Name_UserID As String = "UserID" Protected Const Name_Options As String = "Options" Protected Const Name_Description As String = "Description" - Private Const Name_ParseUserMediaOnly As String = "ParseUserMediaOnly" + Protected Const Name_ParseUserMediaOnly As String = "ParseUserMediaOnly" Private Const Name_IsSubscription As String = UserInfo.Name_IsSubscription Private Const Name_Temporary As String = "Temporary" Private Const Name_Favorite As String = "Favorite" diff --git a/SCrawler/API/Twitter/SiteSettings.vb b/SCrawler/API/Twitter/SiteSettings.vb index 28f1a92..1b06d23 100644 --- a/SCrawler/API/Twitter/SiteSettings.vb +++ b/SCrawler/API/Twitter/SiteSettings.vb @@ -101,7 +101,7 @@ Namespace API.Twitter ConcurrentDownloads = New PropertyValue(1) MyConcurrentDownloadsProvider = New ConcurrentDownloadsProvider - UserRegex = RParams.DMS(String.Format(UserRegexDefaultPattern, "/(twitter|x).com/"), 2) + UserRegex = RParams.DMS(String.Format(UserRegexDefaultPattern, $"/(twitter|x).com({CommunitiesUser}|)/"), 3) UrlPatternUser = "https://x.com/{0}" ImageVideoContains = "twitter" CheckNetscapeCookiesOnEndInit = True @@ -110,8 +110,9 @@ Namespace API.Twitter Friend Overrides Function GetInstance(ByVal What As ISiteSettings.Download) As IPluginContentProvider Return New UserData End Function + Friend Const SinglePostPattern As String = "https://x.com/i/web/status/{0}" Friend Overrides Function GetUserPostUrl(ByVal User As UserDataBase, ByVal Media As UserMedia) As String - Return $"https://x.com/{User.Name}/status/{Media.Post.ID}" + Return String.Format(SinglePostPattern, Media.Post.ID) End Function Friend Overrides Function BaseAuthExists() As Boolean Return Responser.CookiesExists @@ -151,5 +152,18 @@ Namespace API.Twitter End If MyBase.Update() End Sub + Friend Const CommunitiesUser As String = "/i/communities" + Friend Overrides Function IsMyUser(ByVal UserURL As String) As ExchangeOptions + Dim e As ExchangeOptions = MyBase.IsMyUser(UserURL) + If Not e.UserName.IsEmptyString Then + If UserURL.Contains(CommunitiesUser) Then e.Options = CommunitiesUser : e.UserName &= "@c" + Return e + Else + Return Nothing + End If + End Function + Friend Overrides Function GetUserUrl(ByVal User As IPluginContentProvider) As String + Return DirectCast(User, UserData).GetUserUrl + End Function End Class End Namespace \ No newline at end of file diff --git a/SCrawler/API/Twitter/UserData.vb b/SCrawler/API/Twitter/UserData.vb index 1a6eafc..d93f7af 100644 --- a/SCrawler/API/Twitter/UserData.vb +++ b/SCrawler/API/Twitter/UserData.vb @@ -26,8 +26,24 @@ Namespace API.Twitter Private Const Name_GifsDownload As String = "GifsDownload" Private Const Name_GifsSpecialFolder As String = "GifsSpecialFolder" Private Const Name_GifsPrefix As String = "GifsPrefix" + Private Const Name_IsCommunity As String = "IsCommunity" #End Region #Region "Declarations" + Private Const Label_Community As String = "Community" + Private _NameTrue As String = String.Empty + Friend Property NameTrue As String + Get + Return _NameTrue.IfNullOrEmpty(Name) + End Get + Set(ByVal NewName As String) + _NameTrue = NewName + End Set + End Property + Friend Overrides ReadOnly Property SpecialLabels As IEnumerable(Of String) + Get + Return {Label_Community} + End Get + End Property Friend Enum DownloadModels As Integer Undefined = 0 Media = 1 @@ -42,6 +58,7 @@ Namespace API.Twitter Friend Property GifsDownload As Boolean = True Friend Property GifsSpecialFolder As String = String.Empty Friend Property GifsPrefix As String = String.Empty + Friend Property IsCommunity As Boolean = False Private ReadOnly LikesPosts As List(Of String) Private ReadOnly _DataNames As List(Of String) Private ReadOnly Property MySettings As SiteSettings @@ -57,6 +74,9 @@ Namespace API.Twitter Private Function RenameGdlFile(ByVal Input As SFile, ByVal i As Integer) As SFile Return SFile.Rename(Input, $"{Input.PathWithSeparator}{i.NumToString(FileNameProvider)}.{Input.Extension}",, EDP.ThrowException) End Function + Friend Function GetUserUrl() As String + Return $"https://x.com{IIf(IsCommunity, SiteSettings.CommunitiesUser, String.Empty)}/{NameTrue}" + End Function #End Region #Region "Exchange options" Friend Overrides Function ExchangeOptionsGet() As Object @@ -121,7 +141,20 @@ Namespace API.Twitter RemoveExistingDuplicates = .Value(Name_RemoveExistingDuplicates).FromXML(Of Boolean)(False) StartMD5Checked = .Value(Name_StartMD5Checked).FromXML(Of Boolean)(False) MediaModelAllowNonUserTweets = .Value(Name_MediaModelAllowNonUserTweets).FromXML(Of Boolean)(False) + IsCommunity = .Value(Name_IsCommunity).FromXML(Of Boolean)(False) + _NameTrue = .Value(Name_TrueName) Else + If Name.Contains("@") And Not IsCommunity Then + IsCommunity = True + _NameTrue = Name.Split("@")(0) + ID = _NameTrue + ParseUserMediaOnly = False + Labels.ListAddValue(Label_Community, LNC) + Labels.Sort() + .Add(Name_UserID, ID) + .Add(Name_LabelsName, LabelsString) + .Add(Name_ParseUserMediaOnly, ParseUserMediaOnly.BoolToInteger) + End If .Add(Name_FirstDownloadComplete, FirstDownloadComplete.BoolToInteger) .Add(Name_DownloadModelForceApply, DownloadModelForceApply.BoolToInteger) .Add(Name_DownloadModel, CInt(DownloadModel)) @@ -132,6 +165,8 @@ Namespace API.Twitter .Add(Name_RemoveExistingDuplicates, RemoveExistingDuplicates.BoolToInteger) .Add(Name_StartMD5Checked, StartMD5Checked.BoolToInteger) .Add(Name_MediaModelAllowNonUserTweets, MediaModelAllowNonUserTweets.BoolToInteger) + .Add(Name_IsCommunity, IsCommunity.BoolToInteger) + .Add(Name_TrueName, _NameTrue) End If End With End Sub @@ -188,14 +223,15 @@ Namespace API.Twitter Dim newTwitterNodes() As Object = {0, "content", "items"} Dim p As Predicate(Of EContainer) Dim pIndx% + Dim indxChanged As Boolean = False Dim isOneNode As Boolean, isPins As Boolean, ExistsDetected As Boolean, userInfoParsed As Boolean = False Dim j As EContainer, rootNode As EContainer, optionalNode As EContainer, workingNode As EContainer, tmpNode As EContainer, nn As EContainer = Nothing Dim __parseContainer As Func(Of EContainer, Boolean) = Function(ByVal ee As EContainer) As Boolean nn = Nothing - If dirIndx > 1 Then nn = ee - If Not nn.ListExists Then + If dirIndx > 1 Or IsCommunity Then nn = ee + If Not nn.ListExists Or IsCommunity Then For Each node In nodes nn = ee(node) If nn.ListExists Then Exit For @@ -269,10 +305,22 @@ Namespace API.Twitter For i = 0 To timelineFiles.Count - 1 j = JsonDocument.Parse(timelineFiles(i).GetText) If Not j Is Nothing Then - If i = 0 Then + If i = 0 And Not indxChanged Then If Not userInfoParsed Then userInfoParsed = True - Dim resValue$ = j.Value({"data", "user", "result"}, "__typename").StringTrim.StringToLower + Dim resValue$ = j.Value({"data", IIf(IsCommunity, "communityResults", "user"), "result"}, "__typename").StringTrim.StringToLower + Dim icon$ + Dim __getImage As Action(Of String) = Sub(ByVal img As String) + If Not img.IsEmptyString Then + Dim __imgFile As SFile = UrlFile(img, True) + If Not __imgFile.Name.IsEmptyString Then + If __imgFile.Extension.IsEmptyString Then __imgFile.Extension = "jpg" + __imgFile.Path = MyFile.CutPath.Path + If Not __imgFile.Exists Then GetWebFile(img, __imgFile, EDP.None) + If __imgFile.Exists Then IconBannerDownloaded = True + End If + End If + End Sub If resValue.IsEmptyString Then UserExists = False j.Dispose() @@ -281,6 +329,29 @@ Namespace API.Twitter UserSuspended = True j.Dispose() Exit Sub + ElseIf IsCommunity Then + With j({"data", "communityResults", "result", "community_media_timeline", "timeline", "instructions"}) + If .ListExists Then + With .Find(entriesNode, True) + If .ListExists Then + With .ItemF({0, "content", "items", 0, "item", "itemContent", "tweet_results", "result", "tweet", "community_results", "result"}) + If .ListExists Then + If ID = .Value("id_str") Then + UserSiteNameUpdate(.Value("name")) + UserDescriptionUpdate(.Value("description")) + + icon = .Value({"custom_banner_media", "media_info"}, "original_img_url"). + IfNullOrEmpty(.Value({"default_banner_media", "media_info"}, "original_img_url")) + If Not icon.IsEmptyString And DownloadIconBanner Then __getImage.Invoke(icon) + End If + End If + End With + End If + End With + End If + End With + i = -1 + indxChanged = True Else With j({"data", "user", "result"}) If .ListExists Then @@ -290,21 +361,11 @@ Namespace API.Twitter End If With .Item({"legacy"}) If .ListExists Then - If .Value("screen_name").StringToLower = Name.ToLower Then + If .Value("screen_name").StringToLower = NameTrue.ToLower Then UserSiteNameUpdate(.Value("name")) UserDescriptionUpdate(.Value("description")) - Dim __getImage As Action(Of String) = Sub(ByVal img As String) - If Not img.IsEmptyString Then - Dim __imgFile As SFile = UrlFile(img, True) - If Not __imgFile.Name.IsEmptyString Then - If __imgFile.Extension.IsEmptyString Then __imgFile.Extension = "jpg" - __imgFile.Path = MyFile.CutPath.Path - If Not __imgFile.Exists Then GetWebFile(img, __imgFile, EDP.None) - If __imgFile.Exists Then IconBannerDownloaded = True - End If - End If - End Sub - Dim icon$ = .Value("profile_image_url_https") + + icon = .Value("profile_image_url_https") If Not icon.IsEmptyString Then icon = icon.Replace("_normal", String.Empty) If DownloadIconBanner Then __getImage.Invoke(.Value("profile_banner_url")) @@ -316,34 +377,55 @@ Namespace API.Twitter End If End With End If + ElseIf IsCommunity Then + i = -1 + indxChanged = True End If Else For pIndx = 0 To IIf(dirIndx < 2 Or dirIndx = 3, 1, 0) optionalNode = Nothing - Select Case dirIndx - Case 0, 1, 3 - rootNode = j({"data", "user", "result", "timeline_v2", "timeline", "instructions"}) - If rootNode.ListExists Then - If dirIndx = 3 Then - p = entriesNode - isPins = False + rootNode = Nothing + If IsCommunity Then + With j({"data", "communityResults", "result", "community_media_timeline", "timeline", "instructions"}) + If .ListExists Then + If i = 0 Then + rootNode = .Find(entriesNode, True) Else - p = If(pIndx = 0, pinNode, timelineNode) - isPins = pIndx = 0 + rootNode = .Find(moduleItemsPredicate, True) End If optionalNode = rootNode - rootNode = rootNode.Find(p, dirIndx = 3) - If dirIndx <> 3 And rootNode.ListExists Then rootNode = rootNode.Find(entriesNode, dirIndx = 3) End If - Case Else - isPins = False - rootNode = j({"globalObjects", "tweets"}) - optionalNode = rootNode - End Select + End With + Else + Select Case dirIndx + Case 0, 1, 3 + rootNode = j({"data", "user", "result", "timeline_v2", "timeline", "instructions"}) + If rootNode.ListExists Then + If dirIndx = 3 Then + p = entriesNode + isPins = False + Else + p = If(pIndx = 0, pinNode, timelineNode) + isPins = pIndx = 0 + End If + optionalNode = rootNode + rootNode = rootNode.Find(p, dirIndx = 3) + If dirIndx <> 3 And rootNode.ListExists Then rootNode = rootNode.Find(entriesNode, dirIndx = 3) + End If + Case Else + isPins = False + rootNode = j({"globalObjects", "tweets"}) + optionalNode = rootNode + End Select + End If If rootNode.ListExists Then With rootNode - isOneNode = dirIndx < 2 AndAlso .Name = entry + If IsCommunity Then + isOneNode = pIndx = 0 + Else + isOneNode = dirIndx < 2 AndAlso .Name = entry + End If ProgressPre.ChangeMax(If(isOneNode, 1, .Count)) If isOneNode Then ProgressPre.Perform() @@ -660,6 +742,7 @@ Namespace API.Twitter Dim dir As SFile Dim dm As List(Of DownloadModels) = EnumExtract(Of DownloadModels)(DownloadModel).ListIfNothing Dim process As Boolean + Dim urlPrePattern$ = $"https://x.com{IIf(IsCommunity, SiteSettings.CommunitiesUser, String.Empty)}/" Using tgdl As New TwitterGDL(Nothing, Token, MySettings.AbortOnLimit.Value) With { .TempPostsList = _TempPostsList, @@ -670,7 +753,7 @@ Namespace API.Twitter } tgdl.FileExchanger.DeleteCacheOnDispose = False tgdl.FileExchanger.DeleteRootOnDispose = False - For i As Byte = 0 To 3 + For i As Byte = 0 To IIf(IsCommunity, 0, 3) dir = rootDir.NewPath dir.Exists(SFO.Path, True, EDP.ThrowException) outList.Add(dir) @@ -678,10 +761,10 @@ Namespace API.Twitter command = $"""{Settings.GalleryDLFile}"" --verbose --no-download --no-skip --config ""{conf}"" --write-pages " command &= GdlGetIdFilterString() Select Case i - Case 0 : command &= $"https://x.com/{Name}/media" : process = dm.Contains(DownloadModels.Media) - Case 1 : command &= $"https://x.com/{Name}" : process = dm.Contains(DownloadModels.Profile) - Case 2 : command &= $"-o search-endpoint=graphql https://x.com/search?q=from:{Name}+include:nativeretweets" : process = dm.Contains(DownloadModels.Search) - Case 3 : command &= $"https://x.com/{Name}/likes" : process = dm.Contains(DownloadModels.Likes) + Case 0 : command &= $"{urlPrePattern}{NameTrue}/media" : process = dm.Contains(DownloadModels.Media) Or IsCommunity + Case 1 : command &= $"{urlPrePattern}{NameTrue}" : process = dm.Contains(DownloadModels.Profile) + Case 2 : command &= $"-o search-endpoint=graphql https://x.com/search?q=from:{NameTrue}+include:nativeretweets" : process = dm.Contains(DownloadModels.Search) And Not IsCommunity + Case 3 : command &= $"{urlPrePattern}{NameTrue}/likes" : process = dm.Contains(DownloadModels.Likes) Case Else : process = False End Select '#If DEBUG Then @@ -735,7 +818,6 @@ Namespace API.Twitter #Region "ReparseMissing" Private _ReparseLikes As Boolean = False Protected Overrides Sub ReparseMissing(ByVal Token As CancellationToken) - Const SinglePostPattern$ = "https://x.com/{0}/status/{1}" Dim rList As New List(Of Integer) Dim URL$ = String.Empty Dim cache As CacheKeeper = Nothing @@ -766,7 +848,7 @@ Namespace API.Twitter ElseIf _ReparseLikes Then URL = LikesPosts(i) Else - URL = String.Format(SinglePostPattern, Name, m.Post.ID) + URL = String.Format(SiteSettings.SinglePostPattern, m.Post.ID) End If f = GetDataFromGalleryDL(URL, cache, False, Token) If Not f.IsEmptyString Then diff --git a/SCrawler/Download/Feed/DownloadFeedForm.Designer.vb b/SCrawler/Download/Feed/DownloadFeedForm.Designer.vb index c07a588..63af54e 100644 --- a/SCrawler/Download/Feed/DownloadFeedForm.Designer.vb +++ b/SCrawler/Download/Feed/DownloadFeedForm.Designer.vb @@ -62,14 +62,15 @@ Namespace DownloadObjects Me.BTT_MERGE_FEEDS = New System.Windows.Forms.ToolStripMenuItem() Me.BTT_CHECK_ALL = New System.Windows.Forms.ToolStripMenuItem() Me.BTT_CHECK_NONE = New System.Windows.Forms.ToolStripMenuItem() + Me.BTT_VIEW_SAVE = New System.Windows.Forms.ToolStripMenuItem() + Me.BTT_VIEW_LOAD = New System.Windows.Forms.ToolStripMenuItem() Me.SEP_0 = New System.Windows.Forms.ToolStripSeparator() Me.MENU_DOWN = New System.Windows.Forms.ToolStripDropDownButton() Me.BTT_DOWN_ALL = New System.Windows.Forms.ToolStripMenuItem() Me.BTT_DOWN_SELECTED = New System.Windows.Forms.ToolStripMenuItem() Me.BTT_REFRESH = New System.Windows.Forms.ToolStripButton() Me.TP_DATA = New System.Windows.Forms.TableLayoutPanel() - Me.BTT_VIEW_SAVE = New System.Windows.Forms.ToolStripMenuItem() - Me.BTT_VIEW_LOAD = New System.Windows.Forms.ToolStripMenuItem() + Me.BTT_CURR_SESSION_SET = New System.Windows.Forms.ToolStripMenuItem() SEP_1 = New System.Windows.Forms.ToolStripSeparator() SEP_2 = New System.Windows.Forms.ToolStripSeparator() MENU_VIEW = New System.Windows.Forms.ToolStripDropDownButton() @@ -157,6 +158,11 @@ Namespace DownloadObjects MENU_LOAD_SEP_0.Name = "MENU_LOAD_SEP_0" MENU_LOAD_SEP_0.Size = New System.Drawing.Size(349, 6) ' + 'MENU_LOAD_SEP_8 + ' + MENU_LOAD_SEP_8.Name = "MENU_LOAD_SEP_8" + MENU_LOAD_SEP_8.Size = New System.Drawing.Size(349, 6) + ' 'ToolbarTOP ' Me.ToolbarTOP.GripStyle = System.Windows.Forms.ToolStripGripStyle.Hidden @@ -170,7 +176,7 @@ Namespace DownloadObjects ' Me.MENU_LOAD_SESSION.AutoToolTip = False Me.MENU_LOAD_SESSION.DisplayStyle = System.Windows.Forms.ToolStripItemDisplayStyle.Image - Me.MENU_LOAD_SESSION.DropDownItems.AddRange(New System.Windows.Forms.ToolStripItem() {Me.BTT_LOAD_SESSION_CURRENT, Me.BTT_LOAD_SESSION_LAST, Me.BTT_LOAD_SESSION_CHOOSE, MENU_LOAD_SEP_0, Me.BTT_COPY_TO, Me.BTT_MOVE_TO, MENU_LOAD_SEP_1, Me.BTT_LOAD_FAV, Me.BTT_LOAD_SPEC, MENU_LOAD_SEP_2, Me.BTT_FEED_ADD_FAV, Me.BTT_FEED_ADD_FAV_REMOVE, Me.BTT_FEED_REMOVE_FAV, MENU_LOAD_SEP_3, Me.BTT_FEED_ADD_SPEC, Me.BTT_FEED_ADD_SPEC_REMOVE, Me.BTT_FEED_REMOVE_SPEC, MENU_LOAD_SEP_4, Me.BTT_FEED_CLEAR_FAV, Me.BTT_FEED_CLEAR_SPEC, Me.BTT_FEED_DELETE_SPEC, Me.BTT_FEED_DELETE_DAILY_LIST, Me.BTT_FEED_DELETE_DAILY_DATE, MENU_LOAD_SEP_5, Me.BTT_MERGE_SESSIONS, Me.BTT_CLEAR_DAILY, MENU_LOAD_SEP_6, Me.BTT_MERGE_FEEDS, MENU_LOAD_SEP_7, Me.BTT_CHECK_ALL, Me.BTT_CHECK_NONE, MENU_LOAD_SEP_8, Me.BTT_VIEW_SAVE, Me.BTT_VIEW_LOAD}) + Me.MENU_LOAD_SESSION.DropDownItems.AddRange(New System.Windows.Forms.ToolStripItem() {Me.BTT_LOAD_SESSION_CURRENT, Me.BTT_LOAD_SESSION_LAST, Me.BTT_LOAD_SESSION_CHOOSE, MENU_LOAD_SEP_0, Me.BTT_COPY_TO, Me.BTT_MOVE_TO, MENU_LOAD_SEP_1, Me.BTT_LOAD_FAV, Me.BTT_LOAD_SPEC, MENU_LOAD_SEP_2, Me.BTT_FEED_ADD_FAV, Me.BTT_FEED_ADD_FAV_REMOVE, Me.BTT_FEED_REMOVE_FAV, MENU_LOAD_SEP_3, Me.BTT_FEED_ADD_SPEC, Me.BTT_FEED_ADD_SPEC_REMOVE, Me.BTT_FEED_REMOVE_SPEC, MENU_LOAD_SEP_4, Me.BTT_FEED_CLEAR_FAV, Me.BTT_FEED_CLEAR_SPEC, Me.BTT_FEED_DELETE_SPEC, Me.BTT_FEED_DELETE_DAILY_LIST, Me.BTT_FEED_DELETE_DAILY_DATE, MENU_LOAD_SEP_5, Me.BTT_CURR_SESSION_SET, Me.BTT_MERGE_SESSIONS, Me.BTT_CLEAR_DAILY, MENU_LOAD_SEP_6, Me.BTT_MERGE_FEEDS, MENU_LOAD_SEP_7, Me.BTT_CHECK_ALL, Me.BTT_CHECK_NONE, MENU_LOAD_SEP_8, Me.BTT_VIEW_SAVE, Me.BTT_VIEW_LOAD}) Me.MENU_LOAD_SESSION.Image = Global.SCrawler.My.Resources.Resources.ArrowDownPic_Blue_24 Me.MENU_LOAD_SESSION.ImageTransparentColor = System.Drawing.Color.Magenta Me.MENU_LOAD_SESSION.Name = "MENU_LOAD_SESSION" @@ -343,6 +349,20 @@ Namespace DownloadObjects Me.BTT_CHECK_NONE.Size = New System.Drawing.Size(352, 22) Me.BTT_CHECK_NONE.Text = "Select none" ' + 'BTT_VIEW_SAVE + ' + Me.BTT_VIEW_SAVE.Name = "BTT_VIEW_SAVE" + Me.BTT_VIEW_SAVE.Size = New System.Drawing.Size(352, 22) + Me.BTT_VIEW_SAVE.Text = "Save current view" + ' + 'BTT_VIEW_LOAD + ' + Me.BTT_VIEW_LOAD.AutoToolTip = True + Me.BTT_VIEW_LOAD.Name = "BTT_VIEW_LOAD" + Me.BTT_VIEW_LOAD.Size = New System.Drawing.Size(352, 22) + Me.BTT_VIEW_LOAD.Text = "Load view (from saved)" + Me.BTT_VIEW_LOAD.ToolTipText = "Load one of your previously saved views" + ' 'SEP_0 ' Me.SEP_0.Name = "SEP_0" @@ -409,24 +429,14 @@ Namespace DownloadObjects Me.TP_DATA.Size = New System.Drawing.Size(484, 436) Me.TP_DATA.TabIndex = 1 ' - 'MENU_LOAD_SEP_8 - ' - MENU_LOAD_SEP_8.Name = "MENU_LOAD_SEP_8" - MENU_LOAD_SEP_8.Size = New System.Drawing.Size(349, 6) - ' - 'BTT_VIEW_SAVE - ' - Me.BTT_VIEW_SAVE.Name = "BTT_VIEW_SAVE" - Me.BTT_VIEW_SAVE.Size = New System.Drawing.Size(352, 22) - Me.BTT_VIEW_SAVE.Text = "Save current view" - ' - 'BTT_VIEW_LOAD + 'BTT_CURR_SESSION_SET ' - Me.BTT_VIEW_LOAD.AutoToolTip = True - Me.BTT_VIEW_LOAD.Name = "BTT_VIEW_LOAD" - Me.BTT_VIEW_LOAD.Size = New System.Drawing.Size(352, 22) - Me.BTT_VIEW_LOAD.Text = "Load view (from saved)" - Me.BTT_VIEW_LOAD.ToolTipText = "Load one of your previously saved views" + Me.BTT_CURR_SESSION_SET.AutoToolTip = True + Me.BTT_CURR_SESSION_SET.Image = Global.SCrawler.My.Resources.Resources.ArrowDownPic_Blue_24 + Me.BTT_CURR_SESSION_SET.Name = "BTT_CURR_SESSION_SET" + Me.BTT_CURR_SESSION_SET.Size = New System.Drawing.Size(352, 22) + Me.BTT_CURR_SESSION_SET.Text = "Set current session..." + Me.BTT_CURR_SESSION_SET.ToolTipText = "Select one of the download sessions and set it as the current session" ' 'DownloadFeedForm ' @@ -483,5 +493,6 @@ Namespace DownloadObjects Private WithEvents BTT_MOVE_TO As ToolStripMenuItem Private WithEvents BTT_VIEW_SAVE As ToolStripMenuItem Private WithEvents BTT_VIEW_LOAD As ToolStripMenuItem + Private WithEvents BTT_CURR_SESSION_SET As ToolStripMenuItem End Class End Namespace \ No newline at end of file diff --git a/SCrawler/Download/Feed/DownloadFeedForm.resx b/SCrawler/Download/Feed/DownloadFeedForm.resx index d63cc5b..5d446a2 100644 --- a/SCrawler/Download/Feed/DownloadFeedForm.resx +++ b/SCrawler/Download/Feed/DownloadFeedForm.resx @@ -159,10 +159,10 @@ False - - 17, 17 - False + + 17, 17 + \ No newline at end of file diff --git a/SCrawler/Download/Feed/DownloadFeedForm.vb b/SCrawler/Download/Feed/DownloadFeedForm.vb index 8b7b439..22ce038 100644 --- a/SCrawler/Download/Feed/DownloadFeedForm.vb +++ b/SCrawler/Download/Feed/DownloadFeedForm.vb @@ -439,7 +439,9 @@ Namespace DownloadObjects End Sub Private Sub SessionChooser(ByVal GetLast As Boolean, Optional ByVal GetFilesOnly As Boolean = False, Optional ByRef ResultFilesList As List(Of SFile) = Nothing, - Optional ByVal SelectedMode As FeedModes = -1) + Optional ByVal SelectedMode As FeedModes = -1, + Optional ByVal GetSessionFile As Boolean = False, + Optional ByRef SessionFile As SFile = Nothing) Try LoadedSessionName = String.Empty Downloader.ClearSessions() @@ -474,6 +476,12 @@ Namespace DownloadObjects If fList.ListExists Then If GetFilesOnly Then ResultFilesList.AddRange(fList) + ElseIf GetSessionFile Then + If fList.Count > 1 Then + MsgBoxE({"You must select one session file", "Get session file"}, vbExclamation) + Else + SessionFile = fList(0) + End If Else DataList.Clear() If SelectedMode >= 0 Then @@ -900,16 +908,31 @@ Namespace DownloadObjects End Try End Sub #End Region -#Region "Clear session" - Private Sub BTT_CLEAR_DAILY_Click(sender As Object, e As EventArgs) Handles BTT_CLEAR_DAILY.Click - If MsgBoxE({"Are you sure you want to clear this session data?", "Clear session"}, vbExclamation,,, {"Process", "Cancel"}) = 0 Then - Downloader.Files.Clear() - ClearTable() - RefillList() - End If +#Region "Sessions set, merge, clear" + Private Sub BTT_CURR_SESSION_SET_Click(sender As Object, e As EventArgs) Handles BTT_CURR_SESSION_SET.Click + Try + Dim f As SFile = Nothing + SessionChooser(False,,,, True, f) + If f.Exists Then + Using x As New XmlFile(f, Protector.Modes.All, False) With {.AllowSameNames = True, .XmlReadOnly = True} + x.LoadData() + If x.Count > 0 Then + With Downloader + .Files.Clear() + .Files.ListAddList(x, LAP.NotContainsOnly, LAP.IgnoreICopier) + .FilesLoadLastSession(f) + End With + FeedChangeMode(FeedModes.Current) + RefillList(True, False) + Else + MsgBoxE({"There is no data in the selected session", "Replace current session"}, vbCritical) + End If + End Using + End If + Catch ex As Exception + ErrorsDescriber.Execute(EDP.LogMessageValue, ex, "Replace current session") + End Try End Sub -#End Region -#Region "Merge feeds" Private Sub BTT_MERGE_SESSIONS_Click(sender As Object, e As EventArgs) Handles BTT_MERGE_SESSIONS.Click Try Const msgTitle$ = "Merge feeds" @@ -970,6 +993,15 @@ Namespace DownloadObjects ErrorsDescriber.Execute(EDP.SendToLog, ex, "[DownloadFeedForm.MergeSessions]") End Try End Sub + Private Sub BTT_CLEAR_DAILY_Click(sender As Object, e As EventArgs) Handles BTT_CLEAR_DAILY.Click + If MsgBoxE({"Are you sure you want to clear this session data?", "Clear session"}, vbExclamation,,, {"Process", "Cancel"}) = 0 Then + Downloader.Files.Clear() + ClearTable() + RefillList() + End If + End Sub +#End Region +#Region "Merge feeds" Private Sub BTT_MERGE_FEEDS_Click(sender As Object, e As EventArgs) Handles BTT_MERGE_FEEDS.Click Try Const msgTitle$ = "Merge feeds" diff --git a/SCrawler/Download/TDownloader.vb b/SCrawler/Download/TDownloader.vb index f3233ed..c4a5802 100644 --- a/SCrawler/Download/TDownloader.vb +++ b/SCrawler/Download/TDownloader.vb @@ -149,24 +149,30 @@ Namespace DownloadObjects End Try End Function Private _FilesSessionChecked_Impl As Boolean = False - Friend Sub FilesLoadLastSession() + Friend Sub FilesLoadLastSession(Optional ByVal SelectedSessionFile As SFile = Nothing) Try - If Not _FilesSessionChecked And Not _FilesSessionChecked_Impl And _FilesSessionActual.IsEmptyString Then + If Not SelectedSessionFile.IsEmptyString Or (Not _FilesSessionChecked And Not _FilesSessionChecked_Impl And _FilesSessionActual.IsEmptyString) Then _FilesSessionChecked = True _FilesSessionChecked_Impl = True Dim settingValue% = Settings.FeedCurrentTryLoadLastSession - If settingValue >= 0 Then + Dim ssfExists As Boolean = Not SelectedSessionFile.IsEmptyString AndAlso SelectedSessionFile.Exists + If settingValue >= 0 Or ssfExists Then Dim startTime As Date = Process.GetCurrentProcess.StartTime - Dim files As List(Of SFile) = SFile.GetFiles(SessionsPath.CSFileP, "*.xml",, EDP.ReturnValue) - If files.ListExists Then files.RemoveAll(Settings.Feeds.FeedSpecialRemover) - If files.ListExists Then - Dim nd$ = Now.ToString("yyyyMMdd") - files.RemoveAll(Function(f) Not f.Name.StartsWith(nd)) + Dim files As List(Of SFile) + If ssfExists Then + files = New List(Of SFile) From {SelectedSessionFile} + Else + files = SFile.GetFiles(SessionsPath.CSFileP, "*.xml",, EDP.ReturnValue) + If files.ListExists Then files.RemoveAll(Settings.Feeds.FeedSpecialRemover) + If files.ListExists Then + Dim nd$ = Now.ToString("yyyyMMdd") + files.RemoveAll(Function(f) Not f.Name.StartsWith(nd)) + End If End If If files.ListExists Then files.Sort() Dim lastDate As Date = AConvert(Of Date)(files.Last.Name, SessionDateTimeProvider) - If lastDate.Date = startTime.Date Then + If ssfExists Or lastDate.Date = startTime.Date Then Dim __files As New List(Of UserMediaD) Using x As New XmlFile(files.Last, Protector.Modes.All, False) With {.AllowSameNames = True, .XmlReadOnly = True} x.LoadData() diff --git a/SCrawler/My Project/AssemblyInfo.vb b/SCrawler/My Project/AssemblyInfo.vb index 4936e74..95ece00 100644 --- a/SCrawler/My Project/AssemblyInfo.vb +++ b/SCrawler/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + +