Skip to content

Commit

Permalink
Support for tumblr.com hosted videos.
Browse files Browse the repository at this point in the history 
- Added functionality to download video files which are hosted at
tumblr.com. Customizable in the settings window (size of videos,
download pictures/videos: on/off).
  • Loading branch information
@johanneszab
johanneszab committed Jun 10, 2016
1 parent b0f1f01 commit cf61201
Show file tree
Hide file tree
Showing 6 changed files with 264 additions and 60 deletions.
9 changes: 9 additions & 0 deletions TumblTwo/App.config
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,15 @@
<setting name="configCheckMirror" serializeAs="String">
<value>False</value>
</setting>
<setting name="configVideoSize" serializeAs="String">
<value>1080</value>
</setting>
<setting name="configDownloadImage" serializeAs="String">
<value>True</value>
</setting>
<setting name="configDownloadVideo" serializeAs="String">
<value>False</value>
</setting>
</TumblTwo.Properties.Settings>
</userSettings>
<system.net>
Expand Down
127 changes: 95 additions & 32 deletions TumblTwo/CrawlerForm.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -923,21 +923,52 @@ public Tuple<int, List<string>> GetImageUrls(TumblrBlog blog, CancellationToken
{
XDocument document = null;
document = XDocument.Load(GetApiUrl(blog.Url) + (i * 50).ToString() + "&num=50");
newUrls = (from n in document.Descendants("post")
where

// Identify Posts
n.Elements("photo-url").Where(x => x.Attribute("max-width").Value == Properties.Settings.Default.configImageSize.ToString()).Any() &&
!n.Elements("photo-url").Where(x => x.Value == "www.tumblr.com").Any() ||

// Identify Photosets
n.Elements("photoset").Where(photoset => photoset.Descendants("photo-url")
.Any(photourl => (string)photourl.Attribute("max-width").Value
== Properties.Settings.Default.configImageSize.ToString() &&
photourl.Value != "www.tumblr.com")).Any()
from m in n.Descendants("photo-url")
where m.Attribute("max-width").Value == Properties.Settings.Default.configImageSize.ToString()
select (string)m).ToList();

if (Properties.Settings.Default.configDownloadImage)
{

newUrls = (from n in document.Descendants("post")
where

// Identify Posts
n.Elements("photo-url").Where(x => x.Attribute("max-width").Value == Properties.Settings.Default.configImageSize.ToString()).Any() &&
!n.Elements("photo-url").Where(x => x.Value == "www.tumblr.com").Any() ||

// Identify Photosets
n.Elements("photoset").Where(photoset => photoset.Descendants("photo-url")
.Any(photourl => (string)photourl.Attribute("max-width").Value
== Properties.Settings.Default.configImageSize.ToString() &&
photourl.Value != "www.tumblr.com")).Any()
from m in n.Descendants("photo-url")
where m.Attribute("max-width").Value == Properties.Settings.Default.configImageSize.ToString()
select (string)m).ToList();
}
if (Properties.Settings.Default.configDownloadVideo)
{
foreach (var post in (from data in document.Descendants("post") where data.Attribute("type").Value == "video" select data))
{
var videoUrl = post.Descendants("video-player").Where(x => x.Value.Contains("<source src=")).Select(result =>
System.Text.RegularExpressions.Regex.Match(
result.Value, "<source src=\"(.*)\" type=\"video/mp4\">").Groups[1].Value).ToList();

foreach (string video in videoUrl)
{
if (Properties.Settings.Default.configVideoSize == 1080)
{
Monitor.Enter(urlList);
urlList.Add(video.Replace("/480", "") + ".mp4");
Monitor.Exit(urlList);
}
else if (Properties.Settings.Default.configVideoSize == 480)
{
Monitor.Enter(urlList);
urlList.Add("http://vt.tumblr.com/" + video.Replace("/480", "").Split('/').Last() + "_480.mp4");
Monitor.Exit(urlList);
}
}
}
}

}
catch (Exception e)
{
Expand All @@ -952,23 +983,55 @@ where m.Attribute("max-width").Value == Properties.Settings.Default.configImageS
{
XDocument document = null;
document = XDocument.Load(GetApiUrl(blog.Url) + (i * 50).ToString() + "&num=50");
newUrls = (from n in document.Descendants("post")

// Identify Posts
where n.Elements("photo-url").Where(x => x.Attribute("max-width").Value == Properties.Settings.Default.configImageSize.ToString()).Any() &&
!n.Elements("photo-url").Where(x => x.Value == "www.tumblr.com").Any() &&
n.Elements("tag").Where(x => tags.Contains(x.Value)).Any() ||

// Identify Photosets
n.Elements("photoset").Where(photoset => photoset.Descendants("photo-url")
.Any(photourl => (string)photourl.Attribute("max-width").Value
== Properties.Settings.Default.configImageSize.ToString() &&
photourl.Value != "www.tumblr.com")).Any() &&
n.Elements("tag").Where(x => tags.Contains(x.Value)).Any()

from m in n.Descendants("photo-url")
where m.Attribute("max-width").Value == Properties.Settings.Default.configImageSize.ToString()
select (string)m).ToList();

if (Properties.Settings.Default.configDownloadImage)
{
newUrls = (from n in document.Descendants("post")

// Identify Posts
where n.Elements("photo-url").Where(x => x.Attribute("max-width").Value == Properties.Settings.Default.configImageSize.ToString()).Any() &&
!n.Elements("photo-url").Where(x => x.Value == "www.tumblr.com").Any() &&
n.Elements("tag").Where(x => tags.Contains(x.Value)).Any() ||

// Identify Photosets
n.Elements("photoset").Where(photoset => photoset.Descendants("photo-url")
.Any(photourl => (string)photourl.Attribute("max-width").Value
== Properties.Settings.Default.configImageSize.ToString() &&
photourl.Value != "www.tumblr.com")).Any() &&
n.Elements("tag").Where(x => tags.Contains(x.Value)).Any()

from m in n.Descendants("photo-url")
where m.Attribute("max-width").Value == Properties.Settings.Default.configImageSize.ToString()
select (string)m).ToList();
}
if (Properties.Settings.Default.configDownloadVideo)
{
foreach (var post in (from data in document.Descendants("post")
where data.Attribute("type").Value == "video" &&
data.Descendants("tag").Where(x => tags.Contains(x.Value, StringComparer.OrdinalIgnoreCase)).Any()
select data))
{
var videoUrl = post.Descendants("video-player").Where(x => x.Value.Contains("<source src=")).Select(result =>
System.Text.RegularExpressions.Regex.Match(
result.Value, "<source src=\"(.*)\" type=\"video/mp4\">").Groups[1].Value).ToList();

foreach (string video in videoUrl)
{
if (Properties.Settings.Default.configVideoSize == 1080)
{
Monitor.Enter(urlList);
urlList.Add(video.Replace("/480", "") + ".mp4");
Monitor.Exit(urlList);
}
else if (Properties.Settings.Default.configVideoSize == 480)
{
Monitor.Enter(urlList);
urlList.Add("http://vt.tumblr.com/" + video.Replace("/480", "").Split('/').Last() + "_480.mp4");
Monitor.Exit(urlList);
}
}
}
}
}
catch (Exception e)
{
Expand Down
36 changes: 36 additions & 0 deletions TumblTwo/Properties/Settings.Designer.cs
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 9 additions & 0 deletions TumblTwo/Properties/Settings.settings
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,5 +50,14 @@
<Setting Name="configCheckMirror" Type="System.Boolean" Scope="User">
<Value Profile="(Default)">False</Value>
</Setting>
<Setting Name="configVideoSize" Type="System.Int32" Scope="User">
<Value Profile="(Default)">1080</Value>
</Setting>
<Setting Name="configDownloadImage" Type="System.Boolean" Scope="User">
<Value Profile="(Default)">True</Value>
</Setting>
<Setting Name="configDownloadVideo" Type="System.Boolean" Scope="User">
<Value Profile="(Default)">False</Value>
</Setting>
</Settings>
</SettingsFile>
Loading

0 comments on commit cf61201

Please sign in to comment.