diff --git a/src/TumblThree/SharedAssemblyInfo.cs b/src/TumblThree/SharedAssemblyInfo.cs index 15acc671..44508ec2 100644 --- a/src/TumblThree/SharedAssemblyInfo.cs +++ b/src/TumblThree/SharedAssemblyInfo.cs @@ -12,5 +12,5 @@ [assembly: ComVisible(false)] [assembly: NeutralResourcesLanguage("en-US", UltimateResourceFallbackLocation.MainAssembly)] -[assembly: AssemblyVersion("1.6.5.0")] -[assembly: AssemblyFileVersion("1.6.5.0")] +[assembly: AssemblyVersion("2.0.0.0")] +[assembly: AssemblyFileVersion("2.0.0.0")] diff --git a/src/TumblThree/TumblThree.Applications/Controllers/CrawlerController.cs b/src/TumblThree/TumblThree.Applications/Controllers/CrawlerController.cs index 40b9de10..947a7f19 100644 --- a/src/TumblThree/TumblThree.Applications/Controllers/CrawlerController.cs +++ b/src/TumblThree/TumblThree.Applications/Controllers/CrawlerController.cs @@ -210,7 +210,7 @@ private async Task RunCrawlerTasksAsync(PauseToken pt, CancellationToken ct) ICrawler crawler = _crawlerFactory.GetCrawler(blog, new Progress(), pt, ct); try { - crawler.IsBlogOnlineAsync().Wait(4000); + crawler.IsBlogOnlineAsync().Wait(4000, ct); } catch (AggregateException ex) { diff --git a/src/TumblThree/TumblThree.Applications/Controllers/ManagerController.cs b/src/TumblThree/TumblThree.Applications/Controllers/ManagerController.cs index 5494f29b..91656f68 100644 --- a/src/TumblThree/TumblThree.Applications/Controllers/ManagerController.cs +++ b/src/TumblThree/TumblThree.Applications/Controllers/ManagerController.cs @@ -278,6 +278,11 @@ private IReadOnlyList GetIBlogsCore(string directory) { blogs.Add(new TumblrTagSearchBlog().Load(filename)); } + + if (filename.EndsWith(BlogTypes.twitter.ToString())) + { + blogs.Add(new TwitterBlog().Load(filename)); + } } catch (SerializationException ex) { @@ -509,7 +514,7 @@ private void EnqueueAutoDownload() } } - private bool CanAddBlog() => _blogFactory.IsValidTumblrBlogUrl(_crawlerService.NewBlogUrl) || _blogFactory.IsValidUrl(_crawlerService.NewBlogUrl); + private bool CanAddBlog() => _blogFactory.IsValidBlogUrl(_crawlerService.NewBlogUrl) || _blogFactory.IsValidUrl(_crawlerService.NewBlogUrl); private async Task AddBlog() { @@ -796,11 +801,11 @@ private async Task UpdateMetaInformationAsync(IBlog blog) private async Task CheckIfCrawlableBlog(string blogUrl) { - if (!_blogFactory.IsValidTumblrBlogUrl(blogUrl) && _blogFactory.IsValidUrl(blogUrl)) + if (!_blogFactory.IsValidBlogUrl(blogUrl) && _blogFactory.IsValidUrl(blogUrl)) { if ( await _tumblrBlogDetector.IsTumblrBlogWithCustomDomainAsync(blogUrl)) return TumblrBlog.Create(blogUrl, Path.Combine(_shellService.Settings.DownloadLocation, "Index"), _shellService.Settings.FilenameTemplate, true); - throw new Exception($"The url '{blogUrl}' cannot be recognized as Tumblr blog!"); + throw new Exception($"The url '{blogUrl}' cannot be recognized as valid blog!"); } return _blogFactory.GetBlog(blogUrl, Path.Combine(_shellService.Settings.DownloadLocation, "Index"), _shellService.Settings.FilenameTemplate); } diff --git a/src/TumblThree/TumblThree.Applications/Crawler/AbstractCrawler.cs b/src/TumblThree/TumblThree.Applications/Crawler/AbstractCrawler.cs index 07cf5280..9d85ad97 100644 --- a/src/TumblThree/TumblThree.Applications/Crawler/AbstractCrawler.cs +++ b/src/TumblThree/TumblThree.Applications/Crawler/AbstractCrawler.cs @@ -36,14 +36,14 @@ public abstract class AbstractCrawler protected IShellService ShellService { get; } protected PauseToken Pt { get; } protected CancellationToken Ct { get; } - protected IPostQueue PostQueue { get; } + protected IPostQueue PostQueue { get; } protected ConcurrentBag StatisticsBag { get; set; } = new ConcurrentBag(); protected List Tags { get; set; } = new List(); protected IDownloader Downloader; protected AbstractCrawler(IShellService shellService, ICrawlerService crawlerService, IProgress progress, IWebRequestFactory webRequestFactory, - ISharedCookieService cookieService, IPostQueue postQueue, IBlog blog, IDownloader downloader, + ISharedCookieService cookieService, IPostQueue postQueue, IBlog blog, IDownloader downloader, PauseToken pt, CancellationToken ct) { ShellService = shellService; @@ -292,13 +292,19 @@ protected void AddToDownloadList(TumblrPost addToList) protected ulong GetLastPostId() { - ulong lastId = Blog.LastId; if (Blog.ForceRescan) { return 0; } + return !string.IsNullOrEmpty(Blog.DownloadPages) ? 0 : Blog.LastId; + } - return !string.IsNullOrEmpty(Blog.DownloadPages) ? 0 : lastId; + protected void GenerateTags() + { + if (!string.IsNullOrWhiteSpace(Blog.Tags)) + { + Tags = Blog.Tags.Split(',').Select(x => x.Trim()).ToList(); + } } protected void UpdateBlogStats(bool add) @@ -393,7 +399,7 @@ protected bool HandleLimitExceededWebException(WebException webException) return false; } - Logger.Error("{0}, {1}", string.Format(CultureInfo.CurrentCulture, Resources.LimitExceeded, Blog.Name), webException); + Logger.Error("{0}, {1}", string.Format(CultureInfo.CurrentCulture, Resources.LimitExceeded, Blog.Name), webException); //TODO: 2nd resource ShellService.ShowError(webException, Resources.LimitExceeded, Blog.Name); return true; } diff --git a/src/TumblThree/TumblThree.Applications/Crawler/AbstractTumblrCrawler.cs b/src/TumblThree/TumblThree.Applications/Crawler/AbstractTumblrCrawler.cs index 0ea825a0..364919b4 100644 --- a/src/TumblThree/TumblThree.Applications/Crawler/AbstractTumblrCrawler.cs +++ b/src/TumblThree/TumblThree.Applications/Crawler/AbstractTumblrCrawler.cs @@ -48,7 +48,7 @@ public abstract class AbstractTumblrCrawler : AbstractCrawler protected AbstractTumblrCrawler(IShellService shellService, ICrawlerService crawlerService, IWebRequestFactory webRequestFactory, ISharedCookieService cookieService, ITumblrParser tumblrParser, IImgurParser imgurParser, IGfycatParser gfycatParser, IWebmshareParser webmshareParser, IMixtapeParser mixtapeParser, IUguuParser uguuParser, ISafeMoeParser safemoeParser, ILoliSafeParser lolisafeParser, - ICatBoxParser catboxParser, IPostQueue postQueue, IBlog blog, IDownloader downloader, IProgress progress, PauseToken pt, CancellationToken ct) + ICatBoxParser catboxParser, IPostQueue postQueue, IBlog blog, IDownloader downloader, IProgress progress, PauseToken pt, CancellationToken ct) : base(shellService, crawlerService, progress, webRequestFactory, cookieService, postQueue, blog, downloader, pt, ct) { this.TumblrParser = tumblrParser; @@ -124,14 +124,6 @@ protected string ResizeTumblrImageUrl(string imageUrl) .ToString(); } - protected void GenerateTags() - { - if (!string.IsNullOrWhiteSpace(Blog.Tags)) - { - Tags = Blog.Tags.Split(',').Select(x => x.Trim()).ToList(); - } - } - protected bool CheckIfSkipGif(string imageUrl) { return Blog.SkipGif && imageUrl.EndsWith(".gif") || imageUrl.EndsWith(".gifv"); diff --git a/src/TumblThree/TumblThree.Applications/Crawler/CrawlerFactory.cs b/src/TumblThree/TumblThree.Applications/Crawler/CrawlerFactory.cs index 8a74bb64..007e53b8 100644 --- a/src/TumblThree/TumblThree.Applications/Crawler/CrawlerFactory.cs +++ b/src/TumblThree/TumblThree.Applications/Crawler/CrawlerFactory.cs @@ -8,8 +8,7 @@ using TumblThree.Applications.DataModels; using TumblThree.Applications.DataModels.TumblrApiJson; -using TumblThree.Applications.DataModels.TumblrCrawlerData; -using TumblThree.Applications.DataModels.TumblrPosts; +using TumblThree.Applications.DataModels.CrawlerData; using TumblThree.Applications.Downloader; using TumblThree.Applications.Parser; using TumblThree.Applications.Properties; @@ -18,6 +17,7 @@ using TumblThree.Domain.Models; using TumblThree.Domain.Models.Blogs; using TumblThree.Domain.Models.Files; +using TumblThree.Applications.DataModels.Twitter.TimelineTweets; namespace TumblThree.Applications.Crawler { @@ -59,7 +59,7 @@ public ICrawler GetCrawler(IBlog blog) public ICrawler GetCrawler(IBlog blog, IProgress progress, PauseToken pt, CancellationToken ct) { - IPostQueue postQueue = GetProducerConsumerCollection(); + IPostQueue postQueue = GetProducerConsumerCollection(); IFiles files = LoadFiles(blog); IWebRequestFactory webRequestFactory = GetWebRequestFactory(); IImgurParser imgurParser = GetImgurParser(webRequestFactory, ct); @@ -67,18 +67,18 @@ public ICrawler GetCrawler(IBlog blog, IProgress progress, Pau switch (blog.BlogType) { case BlogTypes.tumblr: - IPostQueue> jsonApiQueue = GetJsonQueue(); + IPostQueue> jsonApiQueue = GetJsonQueue(); return new TumblrBlogCrawler(shellService, crawlerService, webRequestFactory, cookieService, - GetTumblrDownloader(progress, blog, files, postQueue, pt, ct), GetTumblrJsonDownloader(jsonApiQueue, blog, pt, ct), + GetTumblrDownloader(progress, blog, files, postQueue, pt, ct), GetJsonDownloader(jsonApiQueue, blog, pt, ct), GetTumblrApiJsonToTextParser(blog), GetTumblrParser(), imgurParser, gfycatParser, GetWebmshareParser(), GetMixtapeParser(), GetUguuParser(), GetSafeMoeParser(), GetLoliSafeParser(), GetCatBoxParser(), postQueue, jsonApiQueue, blog, progress, pt, ct); case BlogTypes.tmblrpriv: - IPostQueue> jsonSvcQueue = + IPostQueue> jsonSvcQueue = GetJsonQueue(); return new TumblrHiddenCrawler(shellService, crawlerService, webRequestFactory, cookieService, GetTumblrDownloader(progress, blog, files, postQueue, pt, ct), - GetTumblrJsonDownloader(jsonSvcQueue, blog, pt, ct), GetTumblrSvcJsonToTextParser(blog), GetTumblrParser(), + GetJsonDownloader(jsonSvcQueue, blog, pt, ct), GetTumblrSvcJsonToTextParser(blog), GetTumblrParser(), imgurParser, gfycatParser, GetWebmshareParser(), GetMixtapeParser(), GetUguuParser(), GetSafeMoeParser(), GetLoliSafeParser(), GetCatBoxParser(), postQueue, jsonSvcQueue, blog, progress, pt, ct); case BlogTypes.tlb: @@ -87,19 +87,24 @@ public ICrawler GetCrawler(IBlog blog, IProgress progress, Pau imgurParser, gfycatParser, GetWebmshareParser(), GetMixtapeParser(), GetUguuParser(), GetSafeMoeParser(), GetLoliSafeParser(), GetCatBoxParser(), postQueue, blog, progress, pt, ct); case BlogTypes.tumblrsearch: - IPostQueue> jsonQueue = GetJsonQueue(); + IPostQueue> jsonQueue = GetJsonQueue(); return new TumblrSearchCrawler(shellService, crawlerService, webRequestFactory, - cookieService, GetTumblrDownloader(progress, blog, files, postQueue, pt, ct), GetTumblrJsonDownloader(jsonQueue, blog, pt, ct), + cookieService, GetTumblrDownloader(progress, blog, files, postQueue, pt, ct), GetJsonDownloader(jsonQueue, blog, pt, ct), GetTumblrParser(), imgurParser, gfycatParser, GetWebmshareParser(), GetMixtapeParser(), GetUguuParser(), GetSafeMoeParser(), GetLoliSafeParser(), GetCatBoxParser(), postQueue, jsonQueue, blog, progress, pt, ct); case BlogTypes.tumblrtagsearch: - IPostQueue> jsonTagSearchQueue = + IPostQueue> jsonTagSearchQueue = GetJsonQueue(); return new TumblrTagSearchCrawler(shellService, crawlerService, webRequestFactory, cookieService, GetTumblrDownloader(progress, blog, files, postQueue, pt, ct), - GetTumblrJsonDownloader(jsonTagSearchQueue, blog, pt, ct), GetTumblrParser(), + GetJsonDownloader(jsonTagSearchQueue, blog, pt, ct), GetTumblrParser(), imgurParser, gfycatParser, GetWebmshareParser(), GetMixtapeParser(), GetUguuParser(), GetSafeMoeParser(), GetLoliSafeParser(), GetCatBoxParser(), postQueue, jsonTagSearchQueue, blog, progress, pt, ct); + case BlogTypes.twitter: + IPostQueue> jsonTwitterQueue = GetJsonQueue(); + return new TwitterCrawler(shellService, crawlerService, progress, webRequestFactory, + cookieService, postQueue, jsonTwitterQueue, blog, GetTwitterDownloader(progress, blog, files, postQueue, pt, ct), + GetJsonDownloader(jsonTwitterQueue, blog, pt, ct), pt, ct); default: throw new ArgumentException("Website is not supported!", nameof(blog)); } @@ -183,28 +188,35 @@ private static IBlogService GetBlogService(IBlog blog, IFiles files) return new BlogService(blog, files); } + private TwitterDownloader GetTwitterDownloader(IProgress progress, IBlog blog, IFiles files, + IPostQueue postQueue, PauseToken pt, CancellationToken ct) + { + return new TwitterDownloader(shellService, managerService, ct, pt, progress, postQueue, GetFileDownloader(ct), + crawlerService, blog, files); + } + private TumblrDownloader GetTumblrDownloader(IProgress progress, IBlog blog, IFiles files, - IPostQueue postQueue, PauseToken pt, CancellationToken ct) + IPostQueue postQueue, PauseToken pt, CancellationToken ct) { return new TumblrDownloader(shellService, managerService, pt, progress, postQueue, GetFileDownloader(ct), crawlerService, blog, files, ct); } - private TumblrXmlDownloader GetTumblrXmlDownloader(IPostQueue> xmlQueue, IBlog blog, + private TumblrXmlDownloader GetTumblrXmlDownloader(IPostQueue> xmlQueue, IBlog blog, PauseToken pt, CancellationToken ct) { return new TumblrXmlDownloader(shellService, pt, xmlQueue, crawlerService, blog, ct); } - private TumblrJsonDownloader GetTumblrJsonDownloader(IPostQueue> jsonQueue, IBlog blog, + private JsonDownloader GetJsonDownloader(IPostQueue> jsonQueue, IBlog blog, PauseToken pt, CancellationToken ct) { - return new TumblrJsonDownloader(shellService, pt, jsonQueue, crawlerService, blog, ct); + return new JsonDownloader(shellService, pt, jsonQueue, crawlerService, blog, ct); } - private IPostQueue GetProducerConsumerCollection() + private IPostQueue GetProducerConsumerCollection() { - return new PostQueue(new ConcurrentQueue()); + return new PostQueue(new ConcurrentQueue()); } private ITumblrApiXmlToTextParser GetTumblrApiXmlToTextParser() @@ -238,14 +250,14 @@ private ITumblrToTextParser GetTumblrApiJsonToTextParser(IBlog blog) } } - private IPostQueue> GetApiXmlQueue() + private IPostQueue> GetApiXmlQueue() { - return new PostQueue>(new ConcurrentQueue>()); + return new PostQueue>(new ConcurrentQueue>()); } - private IPostQueue> GetJsonQueue() + private IPostQueue> GetJsonQueue() { - return new PostQueue>(new ConcurrentQueue>()); + return new PostQueue>(new ConcurrentQueue>()); } } } diff --git a/src/TumblThree/TumblThree.Applications/Crawler/TumblrBlogCrawler.cs b/src/TumblThree/TumblThree.Applications/Crawler/TumblrBlogCrawler.cs index c55d7c16..b0cb4c91 100644 --- a/src/TumblThree/TumblThree.Applications/Crawler/TumblrBlogCrawler.cs +++ b/src/TumblThree/TumblThree.Applications/Crawler/TumblrBlogCrawler.cs @@ -12,7 +12,7 @@ using TumblThree.Applications.DataModels; using TumblThree.Applications.DataModels.TumblrApiJson; -using TumblThree.Applications.DataModels.TumblrCrawlerData; +using TumblThree.Applications.DataModels.CrawlerData; using TumblThree.Applications.DataModels.TumblrPosts; using TumblThree.Applications.Downloader; using TumblThree.Applications.Parser; @@ -30,7 +30,7 @@ public class TumblrBlogCrawler : AbstractTumblrCrawler, ICrawler, IDisposable { private readonly IDownloader downloader; private readonly ITumblrToTextParser tumblrJsonParser; - private readonly IPostQueue> jsonQueue; + private readonly IPostQueue> jsonQueue; private readonly ICrawlerDataDownloader crawlerDataDownloader; private bool completeGrab = true; @@ -46,7 +46,7 @@ public TumblrBlogCrawler(IShellService shellService, ICrawlerService crawlerServ ITumblrToTextParser tumblrJsonParser, ITumblrParser tumblrParser, IImgurParser imgurParser, IGfycatParser gfycatParser, IWebmshareParser webmshareParser, IMixtapeParser mixtapeParser, IUguuParser uguuParser, ISafeMoeParser safemoeParser, ILoliSafeParser lolisafeParser, ICatBoxParser catboxParser, - IPostQueue postQueue, IPostQueue> jsonQueue, IBlog blog, + IPostQueue postQueue, IPostQueue> jsonQueue, IBlog blog, IProgress progress, PauseToken pt, CancellationToken ct) : base(shellService, crawlerService, webRequestFactory, cookieService, tumblrParser, imgurParser, gfycatParser, webmshareParser, mixtapeParser, uguuParser, safemoeParser, lolisafeParser, catboxParser, postQueue, blog, downloader, progress, pt, ct) @@ -427,7 +427,7 @@ private bool CheckPostAge(TumblrApiJson response) return highestPostId >= GetLastPostId(); } - private void AddToJsonQueue(TumblrCrawlerData addToList) + private void AddToJsonQueue(CrawlerData addToList) { if (Blog.DumpCrawlerData) { @@ -551,7 +551,7 @@ private void AddTextUrlToDownloadList(Post post) string textBody = tumblrJsonParser.ParseText(post); AddToDownloadList(new TextPost(textBody, post.Id)); - AddToJsonQueue(new TumblrCrawlerData(Path.ChangeExtension(post.Id, ".json"), post)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(post.Id, ".json"), post)); } private void AddQuoteUrlToDownloadList(Post post) @@ -568,7 +568,7 @@ private void AddQuoteUrlToDownloadList(Post post) string textBody = tumblrJsonParser.ParseQuote(post); AddToDownloadList(new QuotePost(textBody, post.Id)); - AddToJsonQueue(new TumblrCrawlerData(Path.ChangeExtension(post.Id, ".json"), post)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(post.Id, ".json"), post)); } private void AddLinkUrlToDownloadList(Post post) @@ -585,7 +585,7 @@ private void AddLinkUrlToDownloadList(Post post) string textBody = tumblrJsonParser.ParseLink(post); AddToDownloadList(new LinkPost(textBody, post.Id)); - AddToJsonQueue(new TumblrCrawlerData(Path.ChangeExtension(post.Id, ".json"), post)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(post.Id, ".json"), post)); } private void AddConversationUrlToDownloadList(Post post) @@ -602,7 +602,7 @@ private void AddConversationUrlToDownloadList(Post post) string textBody = tumblrJsonParser.ParseConversation(post); AddToDownloadList(new ConversationPost(textBody, post.Id)); - AddToJsonQueue(new TumblrCrawlerData(Path.ChangeExtension(post.Id, ".json"), post)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(post.Id, ".json"), post)); } private void AddAnswerUrlToDownloadList(Post post) @@ -619,7 +619,7 @@ private void AddAnswerUrlToDownloadList(Post post) string textBody = tumblrJsonParser.ParseAnswer(post); AddToDownloadList(new AnswerPost(textBody, post.Id)); - AddToJsonQueue(new TumblrCrawlerData(Path.ChangeExtension(post.Id, ".json"), post)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(post.Id, ".json"), post)); } private void AddPhotoMetaUrlToDownloadList(Post post) @@ -636,7 +636,7 @@ private void AddPhotoMetaUrlToDownloadList(Post post) string textBody = tumblrJsonParser.ParsePhotoMeta(post); AddToDownloadList(new PhotoMetaPost(textBody, post.Id)); - AddToJsonQueue(new TumblrCrawlerData(Path.ChangeExtension(post.Id, ".json"), post)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(post.Id, ".json"), post)); } private void AddVideoMetaUrlToDownloadList(Post post) @@ -653,7 +653,7 @@ private void AddVideoMetaUrlToDownloadList(Post post) string textBody = tumblrJsonParser.ParseVideoMeta(post); AddToDownloadList(new VideoMetaPost(textBody, post.Id)); - AddToJsonQueue(new TumblrCrawlerData(Path.ChangeExtension(post.Id, ".json"), post)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(post.Id, ".json"), post)); } private void AddAudioMetaUrlToDownloadList(Post post) @@ -670,7 +670,7 @@ private void AddAudioMetaUrlToDownloadList(Post post) string textBody = tumblrJsonParser.ParseAudioMeta(post); AddToDownloadList(new AudioMetaPost(textBody, post.Id)); - AddToJsonQueue(new TumblrCrawlerData(Path.ChangeExtension(post.Id, ".json"), post)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(post.Id, ".json"), post)); } private string ParseImageUrl(Post post) @@ -726,7 +726,7 @@ private void AddPhotoUrl(Post post) if (post.Photos?.Count > 0 && post.PhotoUrl1280 == post.Photos[0].PhotoUrl1280 && !post.Photos[0].PhotoUrl1280.Split('/').Last().StartsWith("tumblr_")) index = 1; AddToDownloadList(new PhotoPost(imageUrl, post.Id, post.UnixTimestamp.ToString(), BuildFileName(imageUrl, post, index))); - AddToJsonQueue(new TumblrCrawlerData(Path.ChangeExtension(imageUrl.Split('/').Last(), ".json"), post)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(imageUrl.Split('/').Last(), ".json"), post)); } private void AddPhotoSetUrl(Post post) @@ -741,7 +741,7 @@ private void AddPhotoSetUrl(Post post) foreach (string imageUrl in post.Photos.Select(ParseImageUrl).Where(imgUrl => !CheckIfSkipGif(imgUrl))) { AddToDownloadList(new PhotoPost(imageUrl, post.Id, post.UnixTimestamp.ToString(), BuildFileName(imageUrl, post, i))); - AddToJsonQueue(new TumblrCrawlerData(Path.ChangeExtension(imageUrl.Split('/').Last(), ".json"), post)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(imageUrl.Split('/').Last(), ".json"), post)); if (i != -1) i++; } } @@ -761,7 +761,7 @@ private void AddVideoUrl(Post post) } AddToDownloadList(new VideoPost("https://vtt.tumblr.com/" + videoUrl + ".mp4", post.Id, post.UnixTimestamp.ToString(), BuildFileName("https://vtt.tumblr.com/" + videoUrl + ".mp4", post, -1))); - AddToJsonQueue(new TumblrCrawlerData(videoUrl + ".json", post)); + AddToJsonQueue(new CrawlerData(videoUrl + ".json", post)); } private void AddAudioUrl(Post post) @@ -774,7 +774,7 @@ private void AddAudioUrl(Post post) } AddToDownloadList(new AudioPost(WebUtility.UrlDecode(audioUrl), post.Id, post.UnixTimestamp.ToString())); - AddToJsonQueue(new TumblrCrawlerData(Path.ChangeExtension(audioUrl.Split('/').Last(), ".json"), post)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(audioUrl.Split('/').Last(), ".json"), post)); } private async Task AddExternalPhotoUrlToDownloadListAsync(Post post) diff --git a/src/TumblThree/TumblThree.Applications/Crawler/TumblrHiddenCrawler.cs b/src/TumblThree/TumblThree.Applications/Crawler/TumblrHiddenCrawler.cs index 7531fc73..96add9eb 100644 --- a/src/TumblThree/TumblThree.Applications/Crawler/TumblrHiddenCrawler.cs +++ b/src/TumblThree/TumblThree.Applications/Crawler/TumblrHiddenCrawler.cs @@ -9,7 +9,7 @@ using System.Threading.Tasks; using TumblThree.Applications.DataModels; -using TumblThree.Applications.DataModels.TumblrCrawlerData; +using TumblThree.Applications.DataModels.CrawlerData; using TumblThree.Applications.DataModels.TumblrPosts; using TumblThree.Applications.DataModels.TumblrSvcJson; using TumblThree.Applications.Downloader; @@ -28,7 +28,7 @@ public class TumblrHiddenCrawler : AbstractTumblrCrawler, ICrawler, IDisposable { private readonly IDownloader downloader; private readonly ITumblrToTextParser tumblrJsonParser; - private readonly IPostQueue> jsonQueue; + private readonly IPostQueue> jsonQueue; private readonly ICrawlerDataDownloader crawlerDataDownloader; private string tumblrKey = string.Empty; @@ -45,7 +45,7 @@ public TumblrHiddenCrawler(IShellService shellService, ICrawlerService crawlerSe ITumblrToTextParser tumblrJsonParser, ITumblrParser tumblrParser, IImgurParser imgurParser, IGfycatParser gfycatParser, IWebmshareParser webmshareParser, IMixtapeParser mixtapeParser, IUguuParser uguuParser, ISafeMoeParser safemoeParser, ILoliSafeParser lolisafeParser, ICatBoxParser catboxParser, - IPostQueue postQueue, IPostQueue> jsonQueue, IBlog blog, IProgress progress, PauseToken pt, CancellationToken ct) + IPostQueue postQueue, IPostQueue> jsonQueue, IBlog blog, IProgress progress, PauseToken pt, CancellationToken ct) : base(shellService, crawlerService, webRequestFactory, cookieService, tumblrParser, imgurParser, gfycatParser, webmshareParser, mixtapeParser, uguuParser, safemoeParser, lolisafeParser, catboxParser, postQueue, blog, downloader, progress, pt, ct) @@ -419,7 +419,7 @@ private bool CheckIfDownloadRebloggedPosts(Post post) return Blog.DownloadRebloggedPosts || string.IsNullOrEmpty(post.RebloggedFromName) || post.RebloggedFromName == Blog.Name; } - private void AddToJsonQueue(TumblrCrawlerData addToList) + private void AddToJsonQueue(CrawlerData addToList) { if (Blog.DumpCrawlerData) { @@ -471,7 +471,7 @@ private void AddPhotoUrl(Post post) if (CheckIfSkipGif(imageUrl)) { continue; } AddToDownloadList(new PhotoPost(imageUrl, postId, post.Timestamp.ToString(), BuildFileName(imageUrl, post, i))); - AddToJsonQueue(new TumblrCrawlerData(Path.ChangeExtension(imageUrl.Split('/').Last(), ".json"), post)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(imageUrl.Split('/').Last(), ".json"), post)); if (i != -1) i++; } } @@ -527,7 +527,7 @@ private void AddVideoUrl(Post post) } AddToDownloadList(new VideoPost(videoUrl, postId, post.Timestamp.ToString(), BuildFileName(videoUrl, post, -1))); - AddToJsonQueue(new TumblrCrawlerData(Path.ChangeExtension(videoUrl.Split('/').Last(), ".json"), post)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(videoUrl.Split('/').Last(), ".json"), post)); } private void AddGenericInlineVideoUrl(Post post) @@ -556,7 +556,7 @@ private void AddAudioUrlToDownloadList(Post post) } AddToDownloadList(new AudioPost(audioUrl, postId, post.Timestamp.ToString())); - AddToJsonQueue(new TumblrCrawlerData(Path.ChangeExtension(audioUrl.Split('/').Last(), ".json"), post)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(audioUrl.Split('/').Last(), ".json"), post)); } private void AddTextUrlToDownloadList(Post post) @@ -567,7 +567,7 @@ private void AddTextUrlToDownloadList(Post post) string postId = post.Id; string textBody = tumblrJsonParser.ParseText(post); AddToDownloadList(new TextPost(textBody, postId, post.Timestamp.ToString())); - AddToJsonQueue(new TumblrCrawlerData(Path.ChangeExtension(postId, ".json"), post)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(postId, ".json"), post)); } private void AddQuoteUrlToDownloadList(Post post) @@ -578,7 +578,7 @@ private void AddQuoteUrlToDownloadList(Post post) string postId = post.Id; string textBody = tumblrJsonParser.ParseQuote(post); AddToDownloadList(new QuotePost(textBody, postId, post.Timestamp.ToString())); - AddToJsonQueue(new TumblrCrawlerData(Path.ChangeExtension(postId, ".json"), post)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(postId, ".json"), post)); } private void AddLinkUrlToDownloadList(Post post) @@ -589,7 +589,7 @@ private void AddLinkUrlToDownloadList(Post post) string postId = post.Id; string textBody = tumblrJsonParser.ParseLink(post); AddToDownloadList(new LinkPost(textBody, postId, post.Timestamp.ToString())); - AddToJsonQueue(new TumblrCrawlerData(Path.ChangeExtension(postId, ".json"), post)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(postId, ".json"), post)); } private void AddConversationUrlToDownloadList(Post post) @@ -600,7 +600,7 @@ private void AddConversationUrlToDownloadList(Post post) string postId = post.Id; string textBody = tumblrJsonParser.ParseConversation(post); AddToDownloadList(new ConversationPost(textBody, postId, post.Timestamp.ToString())); - AddToJsonQueue(new TumblrCrawlerData(Path.ChangeExtension(postId, ".json"), post)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(postId, ".json"), post)); } private void AddAnswerUrlToDownloadList(Post post) @@ -611,7 +611,7 @@ private void AddAnswerUrlToDownloadList(Post post) string postId = post.Id; string textBody = tumblrJsonParser.ParseAnswer(post); AddToDownloadList(new AnswerPost(textBody, postId, post.Timestamp.ToString())); - AddToJsonQueue(new TumblrCrawlerData(Path.ChangeExtension(postId, ".json"), post)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(postId, ".json"), post)); } private void AddPhotoMetaUrlToDownloadList(Post post) @@ -622,7 +622,7 @@ private void AddPhotoMetaUrlToDownloadList(Post post) string postId = post.Id; string textBody = tumblrJsonParser.ParsePhotoMeta(post); AddToDownloadList(new PhotoMetaPost(textBody, postId)); - AddToJsonQueue(new TumblrCrawlerData(Path.ChangeExtension(postId, ".json"), post)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(postId, ".json"), post)); } private void AddVideoMetaUrlToDownloadList(Post post) @@ -633,7 +633,7 @@ private void AddVideoMetaUrlToDownloadList(Post post) string postId = post.Id; string textBody = tumblrJsonParser.ParseVideoMeta(post); AddToDownloadList(new VideoMetaPost(textBody, postId)); - AddToJsonQueue(new TumblrCrawlerData(Path.ChangeExtension(postId, ".json"), post)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(postId, ".json"), post)); } private void AddAudioMetaUrlToDownloadList(Post post) @@ -644,7 +644,7 @@ private void AddAudioMetaUrlToDownloadList(Post post) string postId = post.Id; string textBody = tumblrJsonParser.ParseAudioMeta(post); AddToDownloadList(new AudioMetaPost(textBody, postId)); - AddToJsonQueue(new TumblrCrawlerData(Path.ChangeExtension(postId, ".json"), post)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(postId, ".json"), post)); } private static string InlineSearch(Post post) diff --git a/src/TumblThree/TumblThree.Applications/Crawler/TumblrLikedByCrawler.cs b/src/TumblThree/TumblThree.Applications/Crawler/TumblrLikedByCrawler.cs index b657c90d..d1474475 100644 --- a/src/TumblThree/TumblThree.Applications/Crawler/TumblrLikedByCrawler.cs +++ b/src/TumblThree/TumblThree.Applications/Crawler/TumblrLikedByCrawler.cs @@ -35,7 +35,7 @@ public TumblrLikedByCrawler(IShellService shellService, ICrawlerService crawlerS ISharedCookieService cookieService, IDownloader downloader, ITumblrParser tumblrParser, IImgurParser imgurParser, IGfycatParser gfycatParser, IWebmshareParser webmshareParser, IMixtapeParser mixtapeParser, IUguuParser uguuParser, ISafeMoeParser safemoeParser, ILoliSafeParser lolisafeParser, ICatBoxParser catboxParser, - IPostQueue postQueue, IBlog blog, IProgress progress, PauseToken pt, CancellationToken ct) + IPostQueue postQueue, IBlog blog, IProgress progress, PauseToken pt, CancellationToken ct) : base(shellService, crawlerService, webRequestFactory, cookieService, tumblrParser, imgurParser, gfycatParser, webmshareParser, mixtapeParser, uguuParser, safemoeParser, lolisafeParser, catboxParser, postQueue, blog, downloader, progress, pt, ct) diff --git a/src/TumblThree/TumblThree.Applications/Crawler/TumblrSearchCrawler.cs b/src/TumblThree/TumblThree.Applications/Crawler/TumblrSearchCrawler.cs index 9f62131a..76dd422f 100644 --- a/src/TumblThree/TumblThree.Applications/Crawler/TumblrSearchCrawler.cs +++ b/src/TumblThree/TumblThree.Applications/Crawler/TumblrSearchCrawler.cs @@ -9,7 +9,7 @@ using TumblThree.Applications.Converter; using TumblThree.Applications.DataModels; using TumblThree.Applications.DataModels.TumblrApiJson; -using TumblThree.Applications.DataModels.TumblrCrawlerData; +using TumblThree.Applications.DataModels.CrawlerData; using TumblThree.Applications.DataModels.TumblrPosts; using TumblThree.Applications.DataModels.TumblrSearchJson; using TumblThree.Applications.Downloader; @@ -29,7 +29,7 @@ public class TumblrSearchCrawler : AbstractTumblrCrawler, ICrawler, IDisposable private static readonly Regex extractJsonFromSearch = new Regex("window\\['___INITIAL_STATE___'\\] = (.*);"); private readonly IDownloader downloader; - private readonly IPostQueue> jsonQueue; + private readonly IPostQueue> jsonQueue; private readonly ICrawlerDataDownloader crawlerDataDownloader; private SemaphoreSlim semaphoreSlim; @@ -38,8 +38,8 @@ public class TumblrSearchCrawler : AbstractTumblrCrawler, ICrawler, IDisposable public TumblrSearchCrawler(IShellService shellService, ICrawlerService crawlerService, IWebRequestFactory webRequestFactory, ISharedCookieService cookieService, IDownloader downloader, ICrawlerDataDownloader crawlerDataDownloader, ITumblrParser tumblrParser, IImgurParser imgurParser, IGfycatParser gfycatParser, IWebmshareParser webmshareParser, IMixtapeParser mixtapeParser, IUguuParser uguuParser, - ISafeMoeParser safemoeParser, ILoliSafeParser lolisafeParser, ICatBoxParser catboxParser, IPostQueue postQueue, - IPostQueue> jsonQueue, IBlog blog, IProgress progress, PauseToken pt, CancellationToken ct) + ISafeMoeParser safemoeParser, ILoliSafeParser lolisafeParser, ICatBoxParser catboxParser, IPostQueue postQueue, + IPostQueue> jsonQueue, IBlog blog, IProgress progress, PauseToken pt, CancellationToken ct) : base(shellService, crawlerService, webRequestFactory, cookieService, tumblrParser, imgurParser, gfycatParser, webmshareParser, mixtapeParser, uguuParser, safemoeParser, lolisafeParser, catboxParser, postQueue, blog, downloader, progress, pt, ct) @@ -182,7 +182,7 @@ private void DownloadMedia(TumblrSearchApi page) index += (post.Content.Count > 1) ? 1 : 0; DownloadMedia(content, data, index); } - AddToJsonQueue(new TumblrCrawlerData(Path.ChangeExtension(post.Id, ".json"), post)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(post.Id, ".json"), post)); } catch (Exception ex) { @@ -226,7 +226,7 @@ private void DownloadMedia(SearchJson page) index += (post.Content.Count > 1) ? 1 : 0; DownloadMedia(content, data, index); } - AddToJsonQueue(new TumblrCrawlerData(Path.ChangeExtension(post.Id, ".json"), post)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(post.Id, ".json"), post)); } } catch (TimeoutException timeoutException) @@ -305,17 +305,17 @@ private async Task GetSearchPageAsync() return await RequestDataAsync(Blog.Url, headers, cookieHosts); } - private void AddToJsonQueue(TumblrCrawlerData addToList) + private void AddToJsonQueue(CrawlerData addToList) { if (Blog.DumpCrawlerData) { var datum = new Datum(); PropertyCopier.Copy(addToList.Data, datum); - jsonQueue.Add(new TumblrCrawlerData(addToList.Filename, datum)); + jsonQueue.Add(new CrawlerData(addToList.Filename, datum)); } } - private void AddToJsonQueue(TumblrCrawlerData addToList) + private void AddToJsonQueue(CrawlerData addToList) { if (Blog.DumpCrawlerData) jsonQueue.Add(addToList); diff --git a/src/TumblThree/TumblThree.Applications/Crawler/TumblrTagSearchCrawler.cs b/src/TumblThree/TumblThree.Applications/Crawler/TumblrTagSearchCrawler.cs index 76e0b5f7..8b64de9f 100644 --- a/src/TumblThree/TumblThree.Applications/Crawler/TumblrTagSearchCrawler.cs +++ b/src/TumblThree/TumblThree.Applications/Crawler/TumblrTagSearchCrawler.cs @@ -11,7 +11,7 @@ using TumblThree.Applications.DataModels; using TumblThree.Applications.DataModels.TumblrApiJson; -using TumblThree.Applications.DataModels.TumblrCrawlerData; +using TumblThree.Applications.DataModels.CrawlerData; using TumblThree.Applications.DataModels.TumblrPosts; using TumblThree.Applications.DataModels.TumblrTaggedSearchJson; using TumblThree.Applications.Downloader; @@ -31,7 +31,7 @@ public class TumblrTagSearchCrawler : AbstractTumblrCrawler, ICrawler, IDisposab private static readonly Regex extractJsonFromSearch = new Regex("window\\['___INITIAL_STATE___'\\] = (.*);"); private readonly IDownloader downloader; - private readonly IPostQueue> jsonQueue; + private readonly IPostQueue> jsonQueue; private readonly ICrawlerDataDownloader crawlerDataDownloader; private SemaphoreSlim semaphoreSlim; @@ -43,7 +43,7 @@ public TumblrTagSearchCrawler(IShellService shellService, ICrawlerService crawle ISharedCookieService cookieService, IDownloader downloader, ICrawlerDataDownloader crawlerDataDownloader, ITumblrParser tumblrParser, IImgurParser imgurParser, IGfycatParser gfycatParser, IWebmshareParser webmshareParser, IMixtapeParser mixtapeParser, IUguuParser uguuParser, ISafeMoeParser safemoeParser, ILoliSafeParser lolisafeParser, ICatBoxParser catboxParser, - IPostQueue postQueue, IPostQueue> jsonQueue, IBlog blog, IProgress progress, PauseToken pt, CancellationToken ct) + IPostQueue postQueue, IPostQueue> jsonQueue, IBlog blog, IProgress progress, PauseToken pt, CancellationToken ct) : base(shellService, crawlerService, webRequestFactory, cookieService, tumblrParser, imgurParser, gfycatParser, webmshareParser, mixtapeParser, uguuParser, safemoeParser, lolisafeParser, catboxParser, postQueue, blog, downloader, progress, pt, ct) { @@ -196,7 +196,7 @@ private void DownloadMedia(TumblrTaggedSearchApi page) index += (post.Content.Count > 1) ? 1 : 0; DownloadMedia(content, data, index); } - AddToJsonQueue(new TumblrCrawlerData(Path.ChangeExtension(post.Id, ".json"), post)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(post.Id, ".json"), post)); } catch (TimeoutException timeoutException) { @@ -311,7 +311,7 @@ private bool CheckIfContainsTaggedPost(IList tags) return !Tags.Any() || tags.Any(x => Tags.Contains(x, StringComparer.OrdinalIgnoreCase)); } - private void AddToJsonQueue(TumblrCrawlerData addToList) + private void AddToJsonQueue(CrawlerData addToList) { if (Blog.DumpCrawlerData) jsonQueue.Add(addToList); diff --git a/src/TumblThree/TumblThree.Applications/Crawler/TwitterCrawler.cs b/src/TumblThree/TumblThree.Applications/Crawler/TwitterCrawler.cs new file mode 100644 index 00000000..93e37a77 --- /dev/null +++ b/src/TumblThree/TumblThree.Applications/Crawler/TwitterCrawler.cs @@ -0,0 +1,972 @@ +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.ComponentModel.Composition; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using TumblThree.Applications.DataModels; +using TumblThree.Applications.DataModels.CrawlerData; +using TumblThree.Applications.DataModels.TumblrPosts; +using TumblThree.Applications.DataModels.Twitter.TwitterUser; +using TumblThree.Applications.DataModels.Twitter.TimelineTweets; +using TumblThree.Applications.Downloader; +using TumblThree.Applications.Properties; +using TumblThree.Applications.Services; +using TumblThree.Domain; +using TumblThree.Domain.Models.Blogs; +using Newtonsoft.Json.Linq; +using System.Diagnostics; + +namespace TumblThree.Applications.Crawler +{ + [Export(typeof(ICrawler))] + [ExportMetadata("BlogType", typeof(TwitterCrawler))] + [PartCreationPolicy(CreationPolicy.NonShared)] + public class TwitterCrawler : AbstractCrawler, ICrawler, IDisposable + { + private const string twitterDateTemplate = "ddd MMM dd HH:mm:ss +ffff yyyy"; + private const string graphQlTokenUserByScreenName = "esn6mjj-y68fNAj45x5IYA"; + private const string BearerToken = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"; + + private readonly IDownloader downloader; + private readonly IPostQueue> jsonQueue; + private readonly ICrawlerDataDownloader crawlerDataDownloader; + + private bool completeGrab = true; + private bool incompleteCrawl; + + private SemaphoreSlim semaphoreSlim; + private List trackedTasks; + + private int numberOfPagesCrawled; + + private Dictionary Users; + private TwitterUser twUser; + private string guestToken; + private ulong highestId; + private string cursor; + private string oldestApiPost; + + public TwitterCrawler(IShellService shellService, ICrawlerService crawlerService, IProgress progress, IWebRequestFactory webRequestFactory, + ISharedCookieService cookieService, IPostQueue postQueue, IPostQueue> jsonQueue, IBlog blog, IDownloader downloader, + ICrawlerDataDownloader crawlerDataDownloader, PauseToken pt, CancellationToken ct) + : base(shellService, crawlerService, progress, webRequestFactory, cookieService, postQueue, blog, downloader, pt, ct) + { + this.downloader = downloader; + this.jsonQueue = jsonQueue; + this.crawlerDataDownloader = crawlerDataDownloader; + } + + public override async Task IsBlogOnlineAsync() + { + try + { + twUser = await GetTwUser(); + if (!string.IsNullOrEmpty(twUser.Errors?[0]?.Message)) + { + Logger.Warning("TwitterCrawler.IsBlogOnlineAsync: {0}: {1}", Blog.Name, twUser.Errors?[0]?.Message); + ShellService.ShowError(null, twUser.Errors?[0]?.Message); + Blog.Online = false; + } + else + { + Blog.Online = true; + } + } + catch (WebException webException) + { + if (webException.Status == WebExceptionStatus.RequestCanceled) + { + return; + } + + if (HandleUnauthorizedWebException(webException)) + { + Blog.Online = true; + } + else if (HandleLimitExceededWebException(webException)) + { + Blog.Online = true; + } + else + { + Logger.Error("TwitterCrawler:IsBlogOnlineAsync:WebException {0}", webException); + ShellService.ShowError(webException, Resources.BlogIsOffline, Blog.Name); + Blog.Online = false; + } + } + catch (TimeoutException timeoutException) + { + HandleTimeoutException(timeoutException, Resources.OnlineChecking); + Blog.Online = false; + } + } + + public override async Task UpdateMetaInformationAsync() + { + try + { + await UpdateMetaInformationCoreAsync(); + } + catch (WebException webException) + { + if (webException.Status == WebExceptionStatus.RequestCanceled) + { + return; + } + + HandleLimitExceededWebException(webException); + } + } + + private async Task UpdateMetaInformationCoreAsync() + { + if (!Blog.Online) + { + return; + } + + twUser = await GetTwUser(); + + Blog.Title = twUser.Data.User.Legacy.ScreenName; + Blog.Description = twUser.Data.User.Legacy.Description; + Blog.TotalCount = twUser.Data.User.Legacy.StatusesCount; + Blog.Posts = twUser.Data.User.Legacy.StatusesCount; + } + + private void UpdateBlogDuplicates() + { + if (GetLastPostId() == 0) + { + Blog.DuplicatePhotos = DetermineDuplicates(); + Blog.DuplicateVideos = DetermineDuplicates(); + Blog.DuplicateAudios = DetermineDuplicates(); + Blog.TotalCount = Blog.TotalCount - Blog.DuplicatePhotos - Blog.DuplicateAudios - Blog.DuplicateVideos; + } + else + { + var dupPhoto = DetermineDuplicates(); + var dupVideo = DetermineDuplicates(); + var dupAudio = DetermineDuplicates(); + Blog.DuplicatePhotos += dupPhoto; + Blog.DuplicateVideos += dupVideo; + Blog.DuplicateAudios += dupAudio; + Blog.TotalCount = Blog.TotalCount - dupPhoto - dupVideo - dupAudio; + } + } + + public async Task CrawlAsync() + { + Logger.Verbose("TwitterCrawler.Crawl:Start"); + + Task grabber = GetUrlsAsync(); + + Task download = downloader.DownloadBlogAsync(); + + Task crawlerDownloader = Task.CompletedTask; + if (Blog.DumpCrawlerData) + { + crawlerDownloader = crawlerDataDownloader.DownloadCrawlerDataAsync(); + } + + bool apiLimitHit = await grabber; + + UpdateProgressQueueInformation(Resources.ProgressUniqueDownloads); + + UpdateBlogDuplicates(); + + CleanCollectedBlogStatistics(); + + await crawlerDownloader; + bool finishedDownloading = await download; + + if (!Ct.IsCancellationRequested) + { + Blog.LastCompleteCrawl = DateTime.Now; + if (finishedDownloading && !apiLimitHit) + { + Blog.LastId = highestId; + } + } + + Blog.Save(); + + UpdateProgressQueueInformation(string.Empty); + } + + private async Task GetApiUrl(string url, byte type, string cursor, int pageSize) + { + switch (type) + { + case 0: + url = "https://api.twitter.com/1.1/guest/activate.json"; + break; + case 1: + url = string.Format("https://api.twitter.com/graphql/{0}/UserByScreenName" + + "?variables=%7B%22screen_name%22%3A%22{1}%22%2C%22withHighlightedLabel%22%3Atrue%7D", + graphQlTokenUserByScreenName, url.Split('/').Last()); + break; + case 2: + if (!string.IsNullOrEmpty(cursor)) cursor = string.Format("&cursor={0}", cursor.Replace("+", "%2B")); //HttpUtility.UrlEncode(cursor) + var restId = (await GetTwUser()).Data.User.RestId; + url = string.Format("https://api.twitter.com/2/timeline/profile/{0}.json" + + "?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cards_platform=Web-12&include_cards=1&include_ext_alt_text=true&include_quote_count=true&include_reply_count=1&tweet_mode=extended&include_entities=true&include_user_entities=true&include_ext_media_color=true&include_ext_media_availability=true&send_error_codes=true&simple_quoted_tweets=true&include_tweet_replies=true&userId={0}&count={1}{2}&ext=mediaStats%2ChighlightedLabel%2CcameraMoment", + restId, pageSize, cursor); + break; + case 3: + if (!string.IsNullOrEmpty(cursor)) cursor = string.Format("&cursor={0}", cursor.Replace("+", "%2B")); + url = string.Format("https://twitter.com/i/api/2/search/adaptive.json?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cards_platform=Web-12&include_cards=1&include_ext_alt_text=true&include_quote_count=true&include_reply_count=1&tweet_mode=extended&include_entities=true&include_user_entities=true&include_ext_media_color=true&include_ext_media_availability=true&send_error_codes=true&simple_quoted_tweet=true&q=(from%3A{0})%20until%3A{1}%20since%3A2006-01-01&tweet_search_mode=live&count={2}&query_source=typed_query{3}&pc=1&spelling_corrections=1&ext=mediaStats%2ChighlightedLabel", Blog.Name, oldestApiPost, pageSize, cursor); + break; + } + return url; + } + + private async Task GetRequestAsync(string url, string referer = "", Dictionary headers = null) + { + string[] cookieHosts = { "https://twitter.com/" }; + return await RequestApiDataAsync(url, BearerToken, referer, headers, cookieHosts); + } + + private async Task RequestApiDataAsync(string url, string bearerToken, string referer = "", + Dictionary headers = null, IEnumerable cookieHosts = null) + { + var requestRegistration = new CancellationTokenRegistration(); + try + { + HttpWebRequest request = WebRequestFactory.CreateGetXhrRequest(url, referer, headers); + cookieHosts = cookieHosts ?? new List(); + foreach (string cookieHost in cookieHosts) + { + CookieService.GetUriCookie(request.CookieContainer, new Uri(cookieHost)); + } + + request.PreAuthenticate = true; + request.Headers.Add("Authorization", "Bearer " + bearerToken); + request.Accept = "application/json"; + + requestRegistration = Ct.Register(() => request.Abort()); + return await WebRequestFactory.ReadRequestToEndAsync(request, true); + } + finally + { + requestRegistration.Dispose(); + } + } + + private async Task GetTwUser() + { + if (twUser == null) + { + var data = await GetUserByScreenNameAsync(); + try + { + twUser = JsonConvert.DeserializeObject(data); + } + catch (Exception e) + { + Logger.Error("TwitterCrawler.GetTwUser: {0}", e); + throw; + } + } + return twUser; + } + + private async Task GetGuestToken() + { + if (guestToken == null) + { + var requestRegistration = new CancellationTokenRegistration(); + try + { + string url = await GetApiUrl(Blog.Url, 0, "", 0); + if (ShellService.Settings.LimitConnectionsApi) + { + CrawlerService.TimeconstraintApi.Acquire(); + } + var headers = new Dictionary(); + headers.Add("Origin", "https://twitter.com"); + headers.Add("Authorization", "Bearer " + BearerToken); + HttpWebRequest request = WebRequestFactory.CreatePostRequest(url, "https://twitter.com", headers); + CookieService.GetUriCookie(request.CookieContainer, new Uri("https://twitter.com")); + requestRegistration = Ct.Register(() => request.Abort()); + var content = await WebRequestFactory.ReadRequestToEndAsync(request); + guestToken = ((JValue)((JObject)JsonConvert.DeserializeObject(content))["guest_token"]).Value(); + } + catch (Exception e) + { + Logger.Error("GetGuestToken: {0}", e); + } + finally + { + requestRegistration.Dispose(); + } + } + return guestToken; + } + + private async Task GetApiPageAsync(byte type, string cursor) + { + string url = await GetApiUrl(Blog.Url, type, cursor, Blog.PageSize == 0 ? 50 : Blog.PageSize); + + if (ShellService.Settings.LimitConnectionsApi) + { + CrawlerService.TimeconstraintApi.Acquire(); + } + + var referer = Blog.Url; + + var headers = new Dictionary(); + headers.Add("Origin", "https://twitter.com"); + if (type > 0) + { + var token = await GetGuestToken(); + headers.Add("x-guest-token", token); + headers.Add("x-twitter-active-user", "yes"); + headers.Add("x-twitter-client-language", "en"); + var cookie = CookieService.GetAllCookies().FirstOrDefault(c => c.Name == "ct0"); + if (cookie != null) headers.Add("x-csrf-token", cookie.Value); + } + + return await GetRequestAsync(url, referer, headers); + } + + private async Task GetUserByScreenNameAsync() + { + string page; + var attemptCount = 0; + + do + { + page = await GetApiPageAsync(1, ""); + attemptCount++; + } + while (string.IsNullOrEmpty(page) && (attemptCount < ShellService.Settings.MaxNumberOfRetries)); + + return page; + } + + private async Task GetUserTweetsAsync(byte type, string cursor) + { + string page; + var attemptCount = 0; + + do + { + page = await GetApiPageAsync(type, cursor); + attemptCount++; + } + while (string.IsNullOrEmpty(page) && (attemptCount < ShellService.Settings.MaxNumberOfRetries)); + + return page; + } + + protected override IEnumerable GetPageNumbers() + { + if (string.IsNullOrEmpty(Blog.DownloadPages)) + { + int totalPosts = Blog.Posts; + if (!TestRange(Blog.PageSize, 1, 50)) + { + Blog.PageSize = 50; + } + + int totalPages = (totalPosts / Blog.PageSize) + 1; + + return Enumerable.Range(0, totalPages); + } + + return RangeToSequence(Blog.DownloadPages); + } + + private async Task GetUrlsAsync() + { + trackedTasks = new List(); + semaphoreSlim = new SemaphoreSlim(ShellService.Settings.ConcurrentScans); + + GenerateTags(); + + await IsBlogOnlineAsync(); + if (!Blog.Online) + { + PostQueue.CompleteAdding(); + jsonQueue.CompleteAdding(); + return true; + } + + Blog.Posts = twUser.Data.User.Legacy.StatusesCount; + if (Blog.PageSize == 0) Blog.PageSize = 50; + + int currentPage = Blog.Posts / Blog.PageSize + 1; + if (Blog.Posts > 3200) currentPage += 50; + int pageNo = 1; + + while (true) + { + await semaphoreSlim.WaitAsync(); + + if (!completeGrab) + { + break; + } + if (CheckIfShouldStop()) + { + break; + } + CheckIfShouldPause(); + + await CrawlPageAsync(pageNo); + + if (currentPage > 0) + { + currentPage--; + pageNo++; + } + else + { + break; + } + } + + PostQueue.CompleteAdding(); + jsonQueue.CompleteAdding(); + + UpdateBlogStats(GetLastPostId() != 0); + + return incompleteCrawl; + } + + private async Task CrawlPageAsync(int pageNo) + { + const int maxRetries = 2; + int retries = 0; + do + { + string handle429 = null; + try + { + string document = await GetUserTweetsAsync((byte)(oldestApiPost == null ? 2 : 3), cursor); + if (string.IsNullOrEmpty(document)) Debug.WriteLine(""); + + var response = ConvertJsonToClassNew(document); + + if (highestId == 0) + { + highestId = ulong.Parse(response.Timeline.Instructions[0].AddEntries.Entries.First().Content.Item.Content.Tweet.Id); + if (response.GlobalObjects.Tweets.ContainsKey(highestId.ToString())) + Blog.LatestPost = DateTime.ParseExact(response.GlobalObjects.Tweets[highestId.ToString()].CreatedAt, twitterDateTemplate, new CultureInfo("en-US")); + else + highestId = 0; + } + + bool noNewCursor = false; + if (response.GlobalObjects.Tweets.Count == 1 || + (oldestApiPost == null && pageNo * Blog.PageSize >= 3200 && response.GlobalObjects.Tweets.Count < Blog.PageSize + 2)) + { + DateTime createdAt = response.GlobalObjects.Tweets.Count == 1 + ? DateTime.ParseExact(response.GlobalObjects.Tweets.First().Value.CreatedAt, twitterDateTemplate, new CultureInfo("en-US")) + : DateTime.ParseExact(response.GlobalObjects.Tweets.OrderBy(x => x.Key).First().Value.CreatedAt, twitterDateTemplate, new CultureInfo("en-US")); + oldestApiPost = createdAt.ToString("yyyy-MM-dd", new CultureInfo("en-US")); + cursor = null; + noNewCursor = response.GlobalObjects.Tweets.Count != 1; + if (response.GlobalObjects.Tweets.Count == 1) + { + document = await GetUserTweetsAsync(3, cursor); + response = ConvertJsonToClassNew(document); + } + } + + completeGrab = CheckPostAge(response); + + Entry entry = (response.Timeline.Instructions.Last().ReplaceEntry != null) ? response.Timeline.Instructions.Last().ReplaceEntry.Entry : response.Timeline.Instructions[0].AddEntries.Entries.Last(); + var cursorNew = entry.Content.Operation.Cursor.Value; + if (cursor == cursorNew || response.GlobalObjects.Tweets.Count == 0) completeGrab = false; + if (!noNewCursor) cursor = cursorNew; + + await AddUrlsToDownloadListAsync(response); + + numberOfPagesCrawled += Blog.PageSize; + UpdateProgressQueueInformation(Resources.ProgressGetUrlLong, numberOfPagesCrawled, Blog.Posts); + retries = 200; + } + catch (WebException webException) when (webException.Response != null) + { + if (HandleLimitExceededWebException(webException)) + { + //incompleteCrawl = true; + retries++; + handle429 = ((HttpWebResponse)webException?.Response).Headers["x-rate-limit-reset"]; + } + if (((HttpWebResponse)webException?.Response).StatusCode == HttpStatusCode.Forbidden) + { + Logger.Error("TwitterCrawler.CrawlPageAsync: {0}", string.Format(CultureInfo.CurrentCulture, Resources.ProtectedBlog, Blog.Name)); + ShellService.ShowError(webException, Resources.ProtectedBlog, Blog.Name); + completeGrab = false; + retries = 403; + } + } + catch (TimeoutException timeoutException) + { + //incompleteCrawl = true; + retries++; + HandleTimeoutException(timeoutException, Resources.Crawling); + Thread.Sleep(3000); + } + catch (Exception e) + { + Debug.WriteLine(e.ToString()); + retries = 400; + } + finally + { + semaphoreSlim.Release(); + } + if (!string.IsNullOrEmpty(handle429)) + { + try + { + DateTimeOffset dto = DateTimeOffset.FromUnixTimeSeconds(long.Parse(handle429)); + Progress.Report(new DownloadProgress() { Progress = string.Format("waiting until {0}", dto.ToLocalTime().ToString()) }); + var cancelled = Ct.WaitHandle.WaitOne((int)dto.Subtract(DateTime.Now).TotalMilliseconds); + if (cancelled) retries = 400; + } + catch (Exception e) + { + Logger.Error("TwitterCrawler.CrawlPageAsync: error while handling 429: {0}", e); + retries = 400; + } + } + } while (retries < maxRetries); + + if (retries <= maxRetries || retries >= 400) + { + incompleteCrawl = true; + } + } + + private bool PostWithinTimeSpan(Tweet post) + { + if (string.IsNullOrEmpty(Blog.DownloadFrom) && string.IsNullOrEmpty(Blog.DownloadTo)) + { + return true; + } + + long downloadFromUnixTime = 0; + long downloadToUnixTime = long.MaxValue; + if (!string.IsNullOrEmpty(Blog.DownloadFrom)) + { + DateTime downloadFrom = DateTime.ParseExact(Blog.DownloadFrom, "yyyyMMdd", CultureInfo.InvariantCulture, + DateTimeStyles.None); + downloadFromUnixTime = new DateTimeOffset(downloadFrom).ToUnixTimeSeconds(); + } + + if (!string.IsNullOrEmpty(Blog.DownloadTo)) + { + DateTime downloadTo = DateTime.ParseExact(Blog.DownloadTo, "yyyyMMdd", CultureInfo.InvariantCulture, + DateTimeStyles.None); + downloadToUnixTime = new DateTimeOffset(downloadTo).ToUnixTimeSeconds(); + } + + DateTime createdAt = DateTime.ParseExact(post.CreatedAt, twitterDateTemplate, new CultureInfo("en-US")); + long postTime = ((DateTimeOffset)createdAt).ToUnixTimeSeconds(); + return downloadFromUnixTime < postTime && postTime < downloadToUnixTime; + } + + private bool CheckPostAge(TimelineTweets response) + { + var entries = response?.Timeline?.Instructions?[0]?.AddEntries?.Entries; + if (entries == null || entries.Count == 0) return false; + var id = entries[0]?.SortIndex; + if (id == null) return false; + ulong highestPostId; + _ = ulong.TryParse(id, out highestPostId); + + return highestPostId >= GetLastPostId(); + } + + private void AddToJsonQueue(CrawlerData addToList) + { + if (Blog.DumpCrawlerData) + { + jsonQueue.Add(addToList); + } + } + + private async Task AddUrlsToDownloadListAsync(TimelineTweets document) + { + Users = document.GlobalObjects.Users; + var lastPostId = GetLastPostId(); + foreach (Entry entry in document.Timeline.Instructions[0].AddEntries.Entries) + { + var cursorType = entry.Content.Operation?.Cursor.CursorType; + if (cursorType != null) continue; + if (!entry.EntryId.ToLower().StartsWith("tweet-", StringComparison.InvariantCultureIgnoreCase) && + !entry.EntryId.ToLower().StartsWith("sq-i-t-", StringComparison.InvariantCultureIgnoreCase)) continue; + if (!document.GlobalObjects.Tweets.ContainsKey(entry.Content.Item.Content.Tweet.Id)) + { + Logger.Warning("tweet-id {0} of blog {1} not found", entry.Content.Item.Content.Tweet.Id, twUser.Data.User.Id); + continue; + } + Tweet post = document.GlobalObjects.Tweets[entry.Content.Item.Content.Tweet.Id]; + try + { + if (lastPostId > 0 && ulong.TryParse(post.IdStr, out var postId) && postId < lastPostId) { continue; } + if (!PostWithinTimeSpan(post)) { continue; } + if (!CheckIfContainsTaggedPost(post)) { continue; } + if (!CheckIfDownloadRebloggedPosts(post)) { continue; } + + try + { + AddPhotoUrlToDownloadList(post); + AddVideoUrlToDownloadList(post); + AddGifUrlToDownloadList(post); + AddTextUrlToDownloadList(post); + } + catch (NullReferenceException) + { + } + } + catch (Exception e) + { + Logger.Error("TwitterCrawler.AddUrlsToDownloadListAsync: {0}", e); + ShellService.ShowError(e, "Error parsing tweet!"); + } + } + await Task.CompletedTask; + } + + private bool CheckIfDownloadRebloggedPosts(Tweet post) + { + var rsis = post.RetweetedStatusIdStr; + return Blog.DownloadRebloggedPosts || rsis == null || rsis == post.IdStr; + } + + private bool CheckIfContainsTaggedPost(Tweet post) + { + return Tags.Count == 0 || post.Entities.Hashtags.Any(x => Tags.Contains(x.Text)); + } + + private string GetUrlForPreferredImageSize(string url) + { + // ?format=&name= + // https://pbs.twimg.com/media/abcdefghi?format=jpg&name=large + + var filename = url.Split('/').Last(); + var path = url.Replace(filename, ""); + var ext = Path.GetExtension(filename).Replace(".", ""); + if (ext.Length == 0) return url; + return string.Format("{0}{1}?format={2}&name={3}", path, Path.GetFileNameWithoutExtension(filename), ext, ShellService.Settings.ImageSizeCategory); + } + + private static List GetMedia(Tweet post) + { + if (post.ExtendedEntities != null) + { + foreach (var item in post.ExtendedEntities.Media) + { + if (!(item.Type == "photo" || item.Type == "video" || item.Type == "animated_gif")) + throw new Exception("unknown new media type: " + item.Type); + } + return post.ExtendedEntities.Media; + } + return post.Entities.Media; + } + + private void AddPhotoUrlToDownloadList(Tweet post) + { + if (!Blog.DownloadPhoto) return; + + var media = GetMedia(post); + + if (media?[0].Type == "photo") + { + AddPhotoUrl(post, media); + } + } + + private void AddVideoUrlToDownloadList(Tweet post) + { + if (!Blog.DownloadVideo) return; + + var media = GetMedia(post); + + if (media?[0].Type == "video") + { + AddVideoUrl(post, media); + } + } + + private Tweet TweetToSave(Tweet post) + { + Tweet postCopy = (Tweet)post.Clone(); + if (Users.ContainsKey(postCopy.UserIdStr)) + { + postCopy.User = Users[postCopy.UserIdStr]; + } + return postCopy; + } + + private void AddTextUrlToDownloadList(Tweet post) + { + if (!Blog.DownloadText) return; + if (!(post.Entities == null || post.Entities.Media == null || post.Entities.Media.Count == 0)) return; + + AddToDownloadList(new TextPost(post.FullText, post.IdStr)); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(post.IdStr, ".json"), TweetToSave(post))); + } + + private void AddGifUrlToDownloadList(Tweet post) + { + if (Blog.SkipGif) return; + + var media = GetMedia(post); + + if (media?[0].Type == "animated_gif") + { + AddGifUrl(post, media[0]); + } + } + + private void AddGifUrl(Tweet post, Media media) + { + var imageUrl = media.MediaUrlHttps; + var imageUrlConverted = GetUrlForPreferredImageSize(imageUrl); + var filename = BuildFileName(imageUrl, post, "photo", -1); + AddToDownloadList(new PhotoPost(imageUrlConverted, post.IdStr, UnixTimestamp(post).ToString(), filename)); + + var item = media.VideoInfo.Variants[0]; + var urlPrepared = item.Url.IndexOf('?') > 0 ? item.Url.Substring(0, item.Url.IndexOf('?')) : item.Url; + AddToDownloadList(new VideoPost(item.Url, post.IdStr, UnixTimestamp(post).ToString(), BuildFileName(urlPrepared, post, "gif", -1))); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(urlPrepared.Split('/').Last(), ".json"), TweetToSave(post))); + } + + private static int UnixTimestamp(Tweet post) + { + long postTime = ((DateTimeOffset)GetDate(post)).ToUnixTimeSeconds(); + return (int)postTime; + } + + private static DateTime GetDate(Tweet post) + { + return DateTime.ParseExact(post.CreatedAt, twitterDateTemplate, new CultureInfo("en-US")); + } + + private void AddPhotoUrl(Tweet post, List media) + { + for (int i = 0; i < media.Count; i++) + { + var imageUrl = media[i].MediaUrlHttps; + var imageUrlConverted = GetUrlForPreferredImageSize(imageUrl); + var index = media.Count > 1 ? i + 1 : -1; + var filename = BuildFileName(imageUrl, post, "photo", index); + AddToDownloadList(new PhotoPost(imageUrlConverted, post.IdStr, UnixTimestamp(post).ToString(), filename)); + } + var imageUrl2 = media[0].MediaUrlHttps; + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(imageUrl2.Split('/').Last(), ".json"), TweetToSave(post))); + } + + private void AddVideoUrl(Tweet post, List media) + { + var size = ShellService.Settings.VideoSize; + + var imageUrl = media[0].MediaUrlHttps; + var imageUrlConverted = GetUrlForPreferredImageSize(imageUrl); + var filename = BuildFileName(imageUrl, post, "photo", -1); + AddToDownloadList(new PhotoPost(imageUrlConverted, post.IdStr, UnixTimestamp(post).ToString(), filename)); + + int max = media[0].VideoInfo.Variants.Where(v => v.ContentType == "video/mp4").Max(v => v.Bitrate.GetValueOrDefault()); + var item = media[0].VideoInfo.Variants.First(v => v.Bitrate == max); + var urlPrepared = item.Url.IndexOf('?') > 0 ? item.Url.Substring(0, item.Url.IndexOf('?')) : item.Url; + AddToDownloadList(new VideoPost(item.Url, post.IdStr, UnixTimestamp( post).ToString(), BuildFileName(urlPrepared, post, "video", -1))); + AddToJsonQueue(new CrawlerData(Path.ChangeExtension(urlPrepared.Split('/').Last(), ".json"), TweetToSave(post))); + } + + private string GetUserOfPost(string userId) + { + DataModels.Twitter.TimelineTweets.User usr; + if (Users.TryGetValue(userId, out usr)) + return usr.ScreenName; + return ""; + } + + private static List GetTags(Tweet post) + { + var ht = post.Entities.Hashtags; + return ht == null ? new List() : ht.Select(h => h.Text).ToList(); + } + + private string BuildFileName(string url, Tweet post, string type, int index) + { + var reblogged = !string.IsNullOrEmpty(post.RetweetedStatusIdStr) && post.RetweetedStatusIdStr != post.IdStr; + var userId = post.Entities.Media[0].SourceUserIdStr; + var reblogName = ""; + var reblogId = ""; + if (reblogged && !Users.ContainsKey(userId)) + { + if (post.FullText.StartsWith("RT @", StringComparison.InvariantCulture) && post.FullText.Contains(':')) + { + reblogName = post.FullText.Substring(4, post.FullText.IndexOf(':', 4) - 4); + } + reblogId = "123"; + } + else if (reblogged) + { + reblogName = Users[userId].ScreenName; + reblogId = Users[userId].IdStr; + } + return BuildFileNameCore(url, GetUserOfPost(post.UserIdStr), GetDate(post), UnixTimestamp(post), index, type, post.IdStr, + GetTags(post), "", post.FullText, reblogName, reblogId); + } + + protected static string FileName(string url) + { + return url.Split('/').Last(); + } + + private static string Sanitize(string filename) + { + var invalids = System.IO.Path.GetInvalidFileNameChars(); + return string.Join("-", filename.Split(invalids, StringSplitOptions.RemoveEmptyEntries)).TrimEnd('.'); + } + + private static string ReplaceCI(string input, string search, string replacement) + { + string result = Regex.Replace( + input, + Regex.Escape(search), + replacement.Replace("$", "$$"), + RegexOptions.IgnoreCase + ); + return result; + } + + private static bool ContainsCI(string input, string search) + { + return input.IndexOf(search, StringComparison.OrdinalIgnoreCase) >= 0; + } + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA1305:Specify IFormatProvider", Justification = "")] + private string BuildFileNameCore(string url, string blogName, DateTime date, int timestamp, int index, string type, string id, List tags, string slug, string title, string rebloggedFromName, string reblog_key) + { + /* + * Replaced are: + * %f original filename (default) + %b blog name + %d post date (yyyyMMddHHmmss) + %u post timestamp (number) + %p post title (shorted if needed…) + %i post id + %n image index (of photo sets) + %t for all tags (cute+cats,big+dogs) + %r for reblog ("" / "reblog") + %s slug (last part of a post's url) + %k reblog-key + Tokens to make filenames unique: + %x "_{number}" ({number}: 2..n) + %y " ({number})" ({number}: 2..n) + */ + string filename = Blog.FilenameTemplate; + + filename += Path.GetExtension(FileName(url)); + if (ContainsCI(filename, "%f")) filename = ReplaceCI(filename, "%f", Path.GetFileNameWithoutExtension(FileName(url))); + if (ContainsCI(filename, "%d")) filename = ReplaceCI(filename, "%d", date.ToString("yyyyMMdd")); + if (ContainsCI(filename, "%u")) filename = ReplaceCI(filename, "%u", timestamp.ToString()); + if (ContainsCI(filename, "%b")) filename = ReplaceCI(filename, "%b", blogName); + if (ContainsCI(filename, "%i")) + { + if (type == "photo" && Blog.GroupPhotoSets && index != -1) id = $"{id}_{index}"; + filename = ReplaceCI(filename, "%i", id); + } + else if (type == "photo" && Blog.GroupPhotoSets && index != -1) + { + filename = $"{id}_{index}_{filename}"; + } + if (ContainsCI(filename, "%n")) + { + if (type != "photo" || index == -1) + { + string charBefore = ""; + string charAfter = ""; + if (filename.IndexOf("%n", StringComparison.OrdinalIgnoreCase) > 0) + charBefore = filename.Substring(filename.IndexOf("%n", StringComparison.OrdinalIgnoreCase) - 1, 1); + if (filename.IndexOf("%n", StringComparison.OrdinalIgnoreCase) < filename.Length - 2) + charAfter = filename.Substring(filename.IndexOf("%n", StringComparison.OrdinalIgnoreCase) + 2, 1); + if (charBefore == charAfter) + filename = filename.Remove(filename.IndexOf("%n", StringComparison.OrdinalIgnoreCase) - 1, 1); + filename = ReplaceCI(filename, "%n", ""); + } + else + { + filename = ReplaceCI(filename, "%n", index.ToString()); + } + } + if (ContainsCI(filename, "%t")) filename = ReplaceCI(filename, "%t", string.Join(",", tags).Replace(" ", "+")); + if (ContainsCI(filename, "%r")) + { + if (rebloggedFromName.Length == 0 && filename.IndexOf("%r", StringComparison.OrdinalIgnoreCase) > 0 && + filename.IndexOf("%r", StringComparison.OrdinalIgnoreCase) < filename.Length - 2 && + filename.Substring(filename.IndexOf("%r", StringComparison.OrdinalIgnoreCase) - 1, 1) == filename.Substring(filename.IndexOf("%r", StringComparison.OrdinalIgnoreCase) + 2, 1)) + { + filename = filename.Remove(filename.IndexOf("%r", StringComparison.OrdinalIgnoreCase), 3); + } + filename = ReplaceCI(filename, "%r", (rebloggedFromName.Length == 0 ? "" : "reblog")); + } + if (ContainsCI(filename, "%s")) filename = ReplaceCI(filename, "%s", slug); + if (ContainsCI(filename, "%k")) filename = ReplaceCI(filename, "%k", reblog_key); + int neededChars = 0; + if (ContainsCI(filename, "%x")) + { + neededChars = 6; + Downloader.AppendTemplate = "_<0>"; + filename = ReplaceCI(filename, "%x", ""); + } + if (ContainsCI(filename, "%y")) + { + neededChars = 8; + Downloader.AppendTemplate = " (<0>)"; + filename = ReplaceCI(filename, "%y", ""); + } + if (ContainsCI(filename, "%p")) + { + string _title = title; + if (!ShellService.IsLongPathSupported) + { + string filepath = Path.Combine(Blog.DownloadLocation(), filename); + // 260 (max path minus NULL) - current filename length + 2 chars (%p) - chars for numbering + int charactersLeft = 259 - filepath.Length + 2 - neededChars; + if (charactersLeft < 0) throw new PathTooLongException($"{Blog.Name}: filename for post id {id} is too long"); + if (charactersLeft < _title.Length) _title = _title.Substring(0, charactersLeft - 1) + "…"; + } + filename = ReplaceCI(filename, "%p", _title); + } + else if (!ShellService.IsLongPathSupported) + { + string filepath = Path.Combine(Blog.DownloadLocation(), filename); + // 260 (max path minus NULL) - current filename length - chars for numbering + int charactersLeft = 259 - filepath.Length - neededChars; + if (charactersLeft < 0) throw new PathTooLongException($"{Blog.Name}: filename for post id {id} is too long"); + } + + return Sanitize(filename); + } + + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + semaphoreSlim?.Dispose(); + downloader.Dispose(); + } + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + } +} diff --git a/src/TumblThree/TumblThree.Applications/DataModels/AbstractPost.cs b/src/TumblThree/TumblThree.Applications/DataModels/AbstractPost.cs new file mode 100644 index 00000000..0160c196 --- /dev/null +++ b/src/TumblThree/TumblThree.Applications/DataModels/AbstractPost.cs @@ -0,0 +1,33 @@ +namespace TumblThree.Applications.DataModels +{ + public enum PostType + { + Binary, + Text + } + + public abstract class AbstractPost + { + public PostType PostType { get; protected set; } + + public string Url { get; } + + public string Id { get; } + + public string Date { get; } + + public string Filename { get; } + + public string DbType { get; protected set; } + + public string TextFileLocation { get; protected set; } + + protected AbstractPost(string url, string id, string date, string filename) + { + Url = url; + Id = id; + Date = date; + Filename = filename; + } + } +} diff --git a/src/TumblThree/TumblThree.Applications/DataModels/TumblrCrawlerData/TumblrCrawlerData.cs b/src/TumblThree/TumblThree.Applications/DataModels/CrawlerData/CrawlerData.cs similarity index 54% rename from src/TumblThree/TumblThree.Applications/DataModels/TumblrCrawlerData/TumblrCrawlerData.cs rename to src/TumblThree/TumblThree.Applications/DataModels/CrawlerData/CrawlerData.cs index 045147b7..9279bbe4 100644 --- a/src/TumblThree/TumblThree.Applications/DataModels/TumblrCrawlerData/TumblrCrawlerData.cs +++ b/src/TumblThree/TumblThree.Applications/DataModels/CrawlerData/CrawlerData.cs @@ -1,12 +1,12 @@ -namespace TumblThree.Applications.DataModels.TumblrCrawlerData +namespace TumblThree.Applications.DataModels.CrawlerData { - public class TumblrCrawlerData + public class CrawlerData { public T Data { get; protected set; } public string Filename { get; protected set; } - public TumblrCrawlerData(string filename, T data) + public CrawlerData(string filename, T data) { Filename = filename; Data = data; diff --git a/src/TumblThree/TumblThree.Applications/DataModels/TumblrPosts/TumblrPost.cs b/src/TumblThree/TumblThree.Applications/DataModels/TumblrPosts/TumblrPost.cs index 615cec4c..1c829e08 100644 --- a/src/TumblThree/TumblThree.Applications/DataModels/TumblrPosts/TumblrPost.cs +++ b/src/TumblThree/TumblThree.Applications/DataModels/TumblrPosts/TumblrPost.cs @@ -1,33 +1,10 @@ namespace TumblThree.Applications.DataModels.TumblrPosts { - public enum PostType + public abstract class TumblrPost : AbstractPost { - Binary, - Text - } - - public abstract class TumblrPost - { - public PostType PostType { get; protected set; } - - public string Url { get; } - - public string Id { get; } - - public string Date { get; } - - public string Filename { get; } - - public string DbType { get; protected set; } - - public string TextFileLocation { get; protected set; } - protected TumblrPost(string url, string id, string date, string filename) + : base(url, id, date, filename) { - Url = url; - Id = id; - Date = date; - Filename = filename; } } } diff --git a/src/TumblThree/TumblThree.Applications/DataModels/Twitter/TimelineTweets.cs b/src/TumblThree/TumblThree.Applications/DataModels/Twitter/TimelineTweets.cs new file mode 100644 index 00000000..f2e147ce --- /dev/null +++ b/src/TumblThree/TumblThree.Applications/DataModels/Twitter/TimelineTweets.cs @@ -0,0 +1,912 @@ +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System.Collections.Generic; + +namespace TumblThree.Applications.DataModels.Twitter.TimelineTweets +{ + public class TimelineTweets + { + [JsonProperty("globalObjects")] + public GlobalObjects GlobalObjects { get; set; } + + [JsonProperty("timeline")] + public Timeline Timeline { get; set; } + } + + public class Tweet + { + [JsonProperty("created_at")] + public string CreatedAt { get; set; } + + [JsonProperty("id_str")] + public string IdStr { get; set; } + + [JsonProperty("full_text")] + public string FullText { get; set; } + + [JsonProperty("display_text_range")] + public List DisplayTextRange { get; set; } + + [JsonProperty("entities")] + public Entities Entities { get; set; } + + [JsonProperty("extended_entities")] + public ExtendedEntities ExtendedEntities { get; set; } + + [JsonProperty("source")] + public string Source { get; set; } + + [JsonProperty("user_id_str")] + public string UserIdStr { get; set; } + + [JsonProperty("user")] + public User User { get; set; } + + [JsonProperty("retweeted_status_id_str")] + public string RetweetedStatusIdStr { get; set; } + + [JsonProperty("retweet_count")] + public int RetweetCount { get; set; } + + [JsonProperty("favorite_count")] + public int FavoriteCount { get; set; } + + [JsonProperty("reply_count")] + public int ReplyCount { get; set; } + + [JsonProperty("quote_count")] + public int QuoteCount { get; set; } + + [JsonProperty("conversation_id_str")] + public string ConversationIdStr { get; set; } + + [JsonProperty("possibly_sensitive_editable")] + public bool PossiblySensitiveEditable { get; set; } + + [JsonProperty("lang")] + public string Lang { get; set; } + + public object Clone() + { + return MemberwiseClone(); + } + } + + public class FocusRect + { + [JsonProperty("x")] + public int X { get; set; } + + [JsonProperty("y")] + public int Y { get; set; } + + [JsonProperty("h")] + public int H { get; set; } + + [JsonProperty("w")] + public int W { get; set; } + } + + public class OriginalInfo + { + [JsonProperty("width")] + public int Width { get; set; } + + [JsonProperty("height")] + public int Height { get; set; } + + [JsonProperty("focus_rects")] + public List FocusRects { get; set; } + } + + public class Thumb + { + [JsonProperty("w")] + public int W { get; set; } + + [JsonProperty("h")] + public int H { get; set; } + + [JsonProperty("resize")] + public string Resize { get; set; } + } + + public class Medium + { + [JsonProperty("w")] + public int W { get; set; } + + [JsonProperty("h")] + public int H { get; set; } + + [JsonProperty("resize")] + public string Resize { get; set; } + } + + public class Large + { + [JsonProperty("w")] + public int W { get; set; } + + [JsonProperty("h")] + public int H { get; set; } + + [JsonProperty("resize")] + public string Resize { get; set; } + } + + public class Small + { + [JsonProperty("w")] + public int W { get; set; } + + [JsonProperty("h")] + public int H { get; set; } + + [JsonProperty("resize")] + public string Resize { get; set; } + } + + public class Sizes + { + [JsonProperty("thumb")] + public Thumb Thumb { get; set; } + + [JsonProperty("medium")] + public Medium Medium { get; set; } + + [JsonProperty("large")] + public Large Large { get; set; } + + [JsonProperty("small")] + public Small Small { get; set; } + } + + public class Media + { + [JsonProperty("id_str")] + public string IdStr { get; set; } + + [JsonProperty("indices")] + public List Indices { get; set; } + + [JsonProperty("media_url")] + public string MediaUrl { get; set; } + + [JsonProperty("media_url_https")] + public string MediaUrlHttps { get; set; } + + [JsonProperty("url")] + public string Url { get; set; } + + [JsonProperty("display_url")] + public string DisplayUrl { get; set; } + + [JsonProperty("expanded_url")] + public string ExpandedUrl { get; set; } + + [JsonProperty("type")] + public string Type { get; set; } + + [JsonProperty("original_info")] + public OriginalInfo OriginalInfo { get; set; } + + [JsonProperty("sizes")] + public Sizes Sizes { get; set; } + + [JsonProperty("source_status_id_str")] + public string SourceStatusIdStr { get; set; } + + [JsonProperty("source_user_id_str")] + public string SourceUserIdStr { get; set; } + + [JsonProperty("video_info")] + public VideoInfo VideoInfo { get; set; } + + [JsonProperty("media_key")] + public string MediaKey { get; set; } + + [JsonProperty("ext_alt_text")] + public object ExtAltText { get; set; } + + [JsonProperty("ext_media_availability")] + public ExtMediaAvailability ExtMediaAvailability { get; set; } + + [JsonProperty("ext_media_color")] + public ExtMediaColor ExtMediaColor { get; set; } + + [JsonProperty("ext")] + public Ext Ext { get; set; } + + [JsonProperty("additional_media_info")] + public AdditionalMediaInfo AdditionalMediaInfo { get; set; } + } + + public class ProfileImageExtensions + { + [JsonProperty("mediaStats")] + public MediaStats MediaStats { get; set; } + } + + public class ProfileBannerExtensionsMediaColor + { + [JsonProperty("palette")] + public List Palette { get; set; } + } + + public class ProfileBannerExtensions + { + [JsonProperty("mediaStats")] + public MediaStats MediaStats { get; set; } + } + + public class SourceUser + { + [JsonProperty("id_str")] + public string IdStr { get; set; } + + [JsonProperty("name")] + public string Name { get; set; } + + [JsonProperty("screen_name")] + public string ScreenName { get; set; } + + [JsonProperty("location")] + public string Location { get; set; } + + [JsonProperty("description")] + public string Description { get; set; } + + [JsonProperty("entities")] + public Entities Entities { get; set; } + + [JsonProperty("followers_count")] + public int FollowersCount { get; set; } + + [JsonProperty("fast_followers_count")] + public int FastFollowersCount { get; set; } + + [JsonProperty("normal_followers_count")] + public int NormalFollowersCount { get; set; } + + [JsonProperty("friends_count")] + public int FriendsCount { get; set; } + + [JsonProperty("listed_count")] + public int ListedCount { get; set; } + + [JsonProperty("created_at")] + public string CreatedAt { get; set; } + + [JsonProperty("favourites_count")] + public int FavouritesCount { get; set; } + + [JsonProperty("statuses_count")] + public int StatusesCount { get; set; } + + [JsonProperty("media_count")] + public int MediaCount { get; set; } + + [JsonProperty("profile_image_url_https")] + public string ProfileImageUrlHttps { get; set; } + + [JsonProperty("profile_banner_url")] + public string ProfileBannerUrl { get; set; } + + [JsonProperty("profile_image_extensions_alt_text")] + public object ProfileImageExtensionsAltText { get; set; } + + [JsonProperty("profile_image_extensions_media_availability")] + public object ProfileImageExtensionsMediaAvailability { get; set; } + + [JsonProperty("profile_image_extensions_media_color")] + public ProfileImageExtensionsMediaColor ProfileImageExtensionsMediaColor { get; set; } + + [JsonProperty("profile_image_extensions")] + public ProfileImageExtensions ProfileImageExtensions { get; set; } + + [JsonProperty("profile_banner_extensions_media_color")] + public ProfileBannerExtensionsMediaColor ProfileBannerExtensionsMediaColor { get; set; } + + [JsonProperty("profile_banner_extensions_alt_text")] + public object ProfileBannerExtensionsAltText { get; set; } + + [JsonProperty("profile_banner_extensions_media_availability")] + public object ProfileBannerExtensionsMediaAvailability { get; set; } + + [JsonProperty("profile_banner_extensions")] + public ProfileBannerExtensions ProfileBannerExtensions { get; set; } + + [JsonProperty("profile_link_color")] + public string ProfileLinkColor { get; set; } + + [JsonProperty("default_profile")] + public bool DefaultProfile { get; set; } + + [JsonProperty("pinned_tweet_ids")] + public List PinnedTweetIds { get; set; } + + [JsonProperty("pinned_tweet_ids_str")] + public List PinnedTweetIdsStr { get; set; } + + [JsonProperty("advertiser_account_type")] + public string AdvertiserAccountType { get; set; } + + [JsonProperty("advertiser_account_service_levels")] + public List AdvertiserAccountServiceLevels { get; set; } + + [JsonProperty("profile_interstitial_type")] + public string ProfileInterstitialType { get; set; } + + [JsonProperty("business_profile_state")] + public string BusinessProfileState { get; set; } + + [JsonProperty("translator_type")] + public string TranslatorType { get; set; } + + [JsonProperty("withheld_in_countries")] + public List WithheldInCountries { get; set; } + + [JsonProperty("ext")] + public Ext Ext { get; set; } + } + + public class VisitSite + { + [JsonProperty("url")] + public string Url { get; set; } + } + + public class CallToActions + { + [JsonProperty("visit_site")] + public VisitSite VisitSite { get; set; } + } + + public class AdditionalMediaInfo + { + [JsonProperty("title")] + public string Title { get; set; } + + [JsonProperty("description")] + public string Description { get; set; } + + [JsonProperty("call_to_actions")] + public CallToActions CallToActions { get; set; } + + [JsonProperty("embeddable")] + public bool Embeddable { get; set; } + + [JsonProperty("monetizable")] + public bool Monetizable { get; set; } + + [JsonProperty("source_user")] + public SourceUser SourceUser { get; set; } + } + + public class Entities + { + [JsonProperty("hashtags")] + public List Hashtags { get; set; } + + [JsonProperty("symbols")] + public List Symbols { get; set; } + + [JsonProperty("user_mentions")] + public List UserMentions { get; set; } + + [JsonProperty("urls")] + public List Urls { get; set; } + + [JsonProperty("media")] + public List Media { get; set; } + } + + public class Symbol + { + [JsonProperty("text")] + public string Text { get; set; } + + [JsonProperty("indices")] + public List Indices { get; set; } + } + + public class Url2 + { + [JsonProperty("url")] + public string Url { get; set; } + + [JsonProperty("expanded_url")] + public string ExpandedUrl { get; set; } + + [JsonProperty("display_url")] + public string DisplayUrl { get; set; } + + [JsonProperty("indices")] + public List Indices { get; set; } + } + + public class UserMention + { + [JsonProperty("screen_name")] + public string ScreenName { get; set; } + + [JsonProperty("name")] + public string Name { get; set; } + + [JsonProperty("id")] + public long Id { get; set; } + + [JsonProperty("id_str")] + public string IdStr { get; set; } + + [JsonProperty("indices")] + public List Indices { get; set; } + } + + public class UserEntities + { + [JsonProperty("description")] + public Description Description { get; set; } + } + + public class ExtMediaAvailability + { + [JsonProperty("status")] + public string Status { get; set; } + } + + public class Rgb + { + [JsonProperty("red")] + public int Red { get; set; } + + [JsonProperty("green")] + public int Green { get; set; } + + [JsonProperty("blue")] + public int Blue { get; set; } + } + + public class Palette + { + [JsonProperty("rgb")] + public Rgb Rgb { get; set; } + + [JsonProperty("percentage")] + public double Percentage { get; set; } + } + + public class ExtMediaColor + { + [JsonProperty("palette")] + public List Palette { get; set; } + } + + public class MediaStats + { + [JsonProperty("r")] + public object R { get; set; } + + [JsonProperty("ttl")] + public int Ttl { get; set; } + } + + public class MediaStatsR + { + [JsonProperty("missing")] + public string Missing { get; set; } + } + + public class Ext + { + [JsonProperty("mediaStats")] + public MediaStats MediaStats { get; set; } + + [JsonProperty("highlightedLabel")] + public HighlightedLabel HighlightedLabel { get; set; } + } + + public class ExtendedEntities + { + [JsonProperty("media")] + public List Media { get; set; } + } + + public class Description + { + } + + public class ProfileImageExtensionsMediaColor + { + [JsonProperty("palette")] + public List Palette { get; set; } + } + + public class R + { + [JsonProperty("missing")] + public object Missing { get; set; } + + [JsonProperty("ok")] + public Ok Ok { get; set; } + } + + public class VideoInfo + { + [JsonProperty("aspect_ratio")] + public List AspectRatio { get; set; } + + [JsonProperty("duration_millis")] + public int DurationMillis { get; set; } + + [JsonProperty("variants")] + public List Variants { get; set; } + } + + public class Variant + { + [JsonProperty("content_type")] + public string ContentType { get; set; } + + [JsonProperty("url")] + public string Url { get; set; } + + [JsonProperty("bitrate")] + public int? Bitrate { get; set; } + } + + public class Ok + { + } + + public class HighlightedLabel + { + [JsonProperty("r")] + public R R { get; set; } + + [JsonProperty("ttl")] + public int Ttl { get; set; } + } + + public class User + { + [JsonProperty("id_str")] + public string IdStr { get; set; } + + [JsonProperty("name")] + public string Name { get; set; } + + [JsonProperty("screen_name")] + public string ScreenName { get; set; } + + [JsonProperty("location")] + public string Location { get; set; } + + [JsonProperty("description")] + public string Description { get; set; } + + [JsonProperty("entities")] + public UserEntities Entities { get; set; } + + [JsonProperty("followers_count")] + public int FollowersCount { get; set; } + + [JsonProperty("fast_followers_count")] + public int FastFollowersCount { get; set; } + + [JsonProperty("normal_followers_count")] + public int NormalFollowersCount { get; set; } + + [JsonProperty("friends_count")] + public int FriendsCount { get; set; } + + [JsonProperty("listed_count")] + public int ListedCount { get; set; } + + [JsonProperty("created_at")] + public string CreatedAt { get; set; } + + [JsonProperty("favourites_count")] + public int FavouritesCount { get; set; } + + [JsonProperty("statuses_count")] + public int StatusesCount { get; set; } + + [JsonProperty("media_count")] + public int MediaCount { get; set; } + + [JsonProperty("profile_image_url_https")] + public string ProfileImageUrlHttps { get; set; } + + [JsonProperty("profile_image_extensions_alt_text")] + public object ProfileImageExtensionsAltText { get; set; } + + [JsonProperty("profile_image_extensions_media_availability")] + public object ProfileImageExtensionsMediaAvailability { get; set; } + + [JsonProperty("profile_image_extensions_media_color")] + public ProfileImageExtensionsMediaColor ProfileImageExtensionsMediaColor { get; set; } + + [JsonProperty("profile_image_extensions")] + public ProfileImageExtensions ProfileImageExtensions { get; set; } + + [JsonProperty("profile_link_color")] + public string ProfileLinkColor { get; set; } + + [JsonProperty("has_extended_profile")] + public bool HasExtendedProfile { get; set; } + + [JsonProperty("default_profile")] + public bool DefaultProfile { get; set; } + + [JsonProperty("pinned_tweet_ids")] + public List PinnedTweetIds { get; set; } + + [JsonProperty("pinned_tweet_ids_str")] + public List PinnedTweetIdsStr { get; set; } + + [JsonProperty("advertiser_account_type")] + public string AdvertiserAccountType { get; set; } + + [JsonProperty("advertiser_account_service_levels")] + public List AdvertiserAccountServiceLevels { get; set; } + + [JsonProperty("profile_interstitial_type")] + public string ProfileInterstitialType { get; set; } + + [JsonProperty("business_profile_state")] + public string BusinessProfileState { get; set; } + + [JsonProperty("translator_type")] + public string TranslatorType { get; set; } + + [JsonProperty("withheld_in_countries")] + public List WithheldInCountries { get; set; } + + [JsonProperty("ext")] + public Ext Ext { get; set; } + } + + public class Moments + { + } + + public class Cards + { + } + + public class Places + { + } + + public class Broadcasts + { + } + + public class Topics + { + } + + public class Lists + { + } + + public class GlobalObjects + { + [JsonProperty("tweets")] + public Dictionary Tweets { get; set; } + + [JsonProperty("users")] + public Dictionary Users { get; set; } + + [JsonProperty("moments")] + public Moments Moments { get; set; } + + [JsonProperty("cards")] + public Cards Cards { get; set; } + + [JsonProperty("places")] + public Places Places { get; set; } + + [JsonProperty("media")] + public Media2 Media { get; set; } + + [JsonProperty("broadcasts")] + public Broadcasts Broadcasts { get; set; } + + [JsonProperty("topics")] + public Topics Topics { get; set; } + + [JsonProperty("lists")] + public Lists Lists { get; set; } + } + + public class Media2 + { + } + + public class ContentTweet + { + [JsonProperty("id")] + public string Id { get; set; } + + [JsonProperty("displayType")] + public string DisplayType { get; set; } + + [JsonProperty("socialContext", NullValueHandling = NullValueHandling.Ignore)] + public SocialContext SocialContext { get; set; } + } + + public class ItemContent + { + [JsonProperty("tweet")] + public ContentTweet Tweet { get; set; } + } + + public class Item + { + [JsonProperty("content")] + public ItemContent Content { get; set; } + + [JsonProperty("clientEventInfo", NullValueHandling = NullValueHandling.Ignore)] + public ClientEventInfo ClientEventInfo { get; set; } + } + + public class Cursor + { + [JsonProperty("value")] + public string Value { get; set; } + + [JsonProperty("cursorType")] + public string CursorType { get; set; } + + [JsonProperty("stopOnEmptyResponse")] + public bool? StopOnEmptyResponse { get; set; } + } + + public class Operation + { + [JsonProperty("cursor")] + public Cursor Cursor { get; set; } + } + + public class EntryContent + { + [JsonProperty("item")] + public Item Item { get; set; } + + [JsonProperty("operation")] + public Operation Operation { get; set; } + } + + public class Entry + { + [JsonProperty("entryId")] + public string EntryId { get; set; } + + [JsonProperty("sortIndex")] + public string SortIndex { get; set; } + + [JsonProperty("content")] + public EntryContent Content { get; set; } + } + + public class AddEntries + { + [JsonProperty("entries")] + public List Entries { get; set; } + } + + public class GeneralContext + { + [JsonProperty("contextType")] + public string ContextType { get; set; } + + [JsonProperty("text")] + public string Text { get; set; } + } + + public class SocialContext + { + [JsonProperty("generalContext")] + public GeneralContext GeneralContext { get; set; } + } + + public class TimelinesDetails + { + [JsonProperty("injectionType")] + public string InjectionType { get; set; } + } + + public class Details + { + [JsonProperty("timelinesDetails")] + public TimelinesDetails TimelinesDetails { get; set; } + } + + public class ClientEventInfo + { + [JsonProperty("component")] + public string Component { get; set; } + + [JsonProperty("details")] + public Details Details { get; set; } + } + + public class PinEntryContent + { + [JsonProperty("item")] + public Item Item { get; set; } + } + + public class PinEntryEntry + { + [JsonProperty("entryId")] + public string EntryId { get; set; } + + [JsonProperty("sortIndex")] + public string SortIndex { get; set; } + + [JsonProperty("content")] + public PinEntryContent Content { get; set; } + } + + public class PinEntry + { + [JsonProperty("entry")] + public PinEntryEntry Entry { get; set; } + } + + public class Instruction + { + [JsonProperty("addEntries")] + public AddEntries AddEntries { get; set; } + + [JsonProperty("pinEntry")] + public PinEntry PinEntry { get; set; } + + [JsonProperty("replaceEntry")] + public ReplaceEntry ReplaceEntry { get; set; } + } + + public class ReplaceEntry + { + [JsonProperty("entryIdToReplace")] + public string EntryIdToReplace { get; set; } + + [JsonProperty("entry")] + public Entry Entry { get; set; } + } + + public class FeedbackActions + { + } + + public class ResponseObjects + { + [JsonProperty("feedbackActions")] + public FeedbackActions FeedbackActions { get; set; } + } + + public class Timeline + { + [JsonProperty("id")] + public string Id { get; set; } + + [JsonProperty("instructions")] + public List Instructions { get; set; } + + [JsonProperty("responseObjects")] + public ResponseObjects ResponseObjects { get; set; } + } + + public class Hashtag + { + [JsonProperty("text")] + public string Text { get; set; } + + [JsonProperty("indices")] + public List Indices { get; set; } + } +} diff --git a/src/TumblThree/TumblThree.Applications/DataModels/Twitter/TwitterPost.cs b/src/TumblThree/TumblThree.Applications/DataModels/Twitter/TwitterPost.cs new file mode 100644 index 00000000..f7d4c033 --- /dev/null +++ b/src/TumblThree/TumblThree.Applications/DataModels/Twitter/TwitterPost.cs @@ -0,0 +1,10 @@ +namespace TumblThree.Applications.DataModels.Twitter +{ + public class TwitterPost : AbstractPost + { + public TwitterPost(string url, string id, string date, string filename) + : base(url, id, date, filename) + { + } + } +} diff --git a/src/TumblThree/TumblThree.Applications/DataModels/Twitter/TwitterUser.cs b/src/TumblThree/TumblThree.Applications/DataModels/Twitter/TwitterUser.cs new file mode 100644 index 00000000..0103cba2 --- /dev/null +++ b/src/TumblThree/TumblThree.Applications/DataModels/Twitter/TwitterUser.cs @@ -0,0 +1,304 @@ +using Newtonsoft.Json; +using System.Collections.Generic; + +namespace TumblThree.Applications.DataModels.Twitter.TwitterUser +{ + public class TwitterUser + { + [JsonProperty("data")] + public Data Data { get; set; } + + [JsonProperty("errors")] + public List Errors { get; set; } + } + + public class Location + { + [JsonProperty("line")] + public int Line { get; set; } + + [JsonProperty("column")] + public int Column { get; set; } + } + + public class Tracing + { + [JsonProperty("trace_id")] + public string TraceId { get; set; } + } + + public class Extensions + { + [JsonProperty("name")] + public string Name { get; set; } + + [JsonProperty("source")] + public string Source { get; set; } + + [JsonProperty("code")] + public int Code { get; set; } + + [JsonProperty("kind")] + public string Kind { get; set; } + + [JsonProperty("tracing")] + public Tracing Tracing { get; set; } + } + + public class Error + { + [JsonProperty("message")] + public string Message { get; set; } + + [JsonProperty("path")] + public List Path { get; set; } + + [JsonProperty("locations")] + public List Locations { get; set; } + + [JsonProperty("name")] + public string Name { get; set; } + + [JsonProperty("source")] + public string Source { get; set; } + + [JsonProperty("code")] + public int Code { get; set; } + + [JsonProperty("kind")] + public string Kind { get; set; } + + [JsonProperty("tracing")] + public Tracing Tracing { get; set; } + + [JsonProperty("extensions")] + public Extensions Extensions { get; set; } + } + + public class AffiliatesHighlightedLabel + { + } + + public class Description + { + [JsonProperty("urls")] + public List Urls { get; set; } + } + + public class Entities + { + [JsonProperty("description")] + public Description Description { get; set; } + + [JsonProperty("url")] + public Url Url { get; set; } + } + + public class Url + { + [JsonProperty("urls")] + public List Urls { get; set; } + } + + public class Url2 + { + [JsonProperty("display_url")] + public string DisplayUrl { get; set; } + + [JsonProperty("expanded_url")] + public string ExpandedUrl { get; set; } + + [JsonProperty("url")] + public string Url { get; set; } + + [JsonProperty("indices")] + public List Indices { get; set; } + } + + public class Rgb + { + [JsonProperty("blue")] + public int Blue { get; set; } + + [JsonProperty("green")] + public int Green { get; set; } + + [JsonProperty("red")] + public int Red { get; set; } + } + + public class Palette + { + [JsonProperty("percentage")] + public double Percentage { get; set; } + + [JsonProperty("rgb")] + public Rgb Rgb { get; set; } + } + + public class Ok + { + [JsonProperty("palette")] + public List Palette { get; set; } + } + + public class R + { + [JsonProperty("ok")] + public Ok Ok { get; set; } + } + + public class MediaColor + { + [JsonProperty("r")] + public R R { get; set; } + } + + public class ProfileImageExtensions + { + [JsonProperty("mediaColor")] + public MediaColor MediaColor { get; set; } + } + + public class Legacy + { + [JsonProperty("blocked_by")] + public bool BlockedBy { get; set; } + + [JsonProperty("blocking")] + public bool Blocking { get; set; } + + [JsonProperty("can_dm")] + public bool CanDm { get; set; } + + [JsonProperty("can_media_tag")] + public bool CanMediaTag { get; set; } + + [JsonProperty("created_at")] + public string CreatedAt { get; set; } + + [JsonProperty("default_profile")] + public bool DefaultProfile { get; set; } + + [JsonProperty("default_profile_image")] + public bool DefaultProfileImage { get; set; } + + [JsonProperty("description")] + public string Description { get; set; } + + [JsonProperty("entities")] + public Entities Entities { get; set; } + + [JsonProperty("fast_followers_count")] + public int FastFollowersCount { get; set; } + + [JsonProperty("favourites_count")] + public int FavouritesCount { get; set; } + + [JsonProperty("follow_request_sent")] + public bool FollowRequestSent { get; set; } + + [JsonProperty("followed_by")] + public bool FollowedBy { get; set; } + + [JsonProperty("followers_count")] + public int FollowersCount { get; set; } + + [JsonProperty("following")] + public bool Following { get; set; } + + [JsonProperty("friends_count")] + public int FriendsCount { get; set; } + + [JsonProperty("has_custom_timelines")] + public bool HasCustomTimelines { get; set; } + + [JsonProperty("is_translator")] + public bool IsTranslator { get; set; } + + [JsonProperty("listed_count")] + public int ListedCount { get; set; } + + [JsonProperty("location")] + public string Location { get; set; } + + [JsonProperty("media_count")] + public int MediaCount { get; set; } + + [JsonProperty("muting")] + public bool Muting { get; set; } + + [JsonProperty("name")] + public string Name { get; set; } + + [JsonProperty("normal_followers_count")] + public int NormalFollowersCount { get; set; } + + [JsonProperty("notifications")] + public bool Notifications { get; set; } + + [JsonProperty("pinned_tweet_ids_str")] + public List PinnedTweetIdsStr { get; set; } + + [JsonProperty("profile_image_extensions")] + public ProfileImageExtensions ProfileImageExtensions { get; set; } + + [JsonProperty("profile_image_url_https")] + public string ProfileImageUrlHttps { get; set; } + + [JsonProperty("profile_interstitial_type")] + public string ProfileInterstitialType { get; set; } + + [JsonProperty("protected")] + public bool Protected { get; set; } + + [JsonProperty("screen_name")] + public string ScreenName { get; set; } + + [JsonProperty("statuses_count")] + public int StatusesCount { get; set; } + + [JsonProperty("translator_type")] + public string TranslatorType { get; set; } + + [JsonProperty("verified")] + public bool Verified { get; set; } + + [JsonProperty("want_retweets")] + public bool WantRetweets { get; set; } + + [JsonProperty("withheld_in_countries")] + public List WithheldInCountries { get; set; } + } + + public class LegacyExtendedProfile + { + } + + public class User + { + [JsonProperty("id")] + public string Id { get; set; } + + [JsonProperty("rest_id")] + public string RestId { get; set; } + + [JsonProperty("affiliates_highlighted_label")] + public AffiliatesHighlightedLabel AffiliatesHighlightedLabel { get; set; } + + [JsonProperty("legacy")] + public Legacy Legacy { get; set; } + + [JsonProperty("legacy_extended_profile")] + public LegacyExtendedProfile LegacyExtendedProfile { get; set; } + + [JsonProperty("is_profile_translatable")] + public bool IsProfileTranslatable { get; set; } + } + + public class Data + { + [JsonProperty("user")] + public User User { get; set; } + } +} diff --git a/src/TumblThree/TumblThree.Applications/Downloader/AbstractDownloader.cs b/src/TumblThree/TumblThree.Applications/Downloader/AbstractDownloader.cs index 8d60d4a1..eb4c8264 100644 --- a/src/TumblThree/TumblThree.Applications/Downloader/AbstractDownloader.cs +++ b/src/TumblThree/TumblThree.Applications/Downloader/AbstractDownloader.cs @@ -25,7 +25,7 @@ public abstract class AbstractDownloader : IDownloader, IDisposable private readonly IManagerService managerService; protected readonly IProgress progress; protected readonly object lockObjectDownload = new object(); - protected readonly IPostQueue postQueue; + protected readonly IPostQueue postQueue; protected readonly IShellService shellService; protected readonly CancellationToken ct; protected readonly PauseToken pt; @@ -41,7 +41,7 @@ public abstract class AbstractDownloader : IDownloader, IDisposable private readonly object diskFilesLock = new object(); private HashSet diskFiles; - protected AbstractDownloader(IShellService shellService, IManagerService managerService, CancellationToken ct, PauseToken pt, IProgress progress, IPostQueue postQueue, FileDownloader fileDownloader, ICrawlerService crawlerService = null, IBlog blog = null, IFiles files = null) + protected AbstractDownloader(IShellService shellService, IManagerService managerService, CancellationToken ct, PauseToken pt, IProgress progress, IPostQueue postQueue, FileDownloader fileDownloader, ICrawlerService crawlerService = null, IBlog blog = null, IFiles files = null) { this.shellService = shellService; this.crawlerService = crawlerService; @@ -171,6 +171,8 @@ public virtual async Task DownloadBlogAsync() blog.CreateDataFolder(); + await Task.Run(() => Task.CompletedTask); + try { foreach (TumblrPost downloadItem in postQueue.GetConsumingEnumerable(ct)) @@ -254,7 +256,7 @@ public virtual async Task DownloadPageAsync(string url) } } - private bool CheckIfLinkRestored(TumblrPost downloadItem) + protected bool CheckIfLinkRestored(TumblrPost downloadItem) { if (!blog.ForceRescan || blog.FilenameTemplate != "%f") return false; lock (diskFilesLock) @@ -332,7 +334,7 @@ private void AddTextToDb(TumblrPost downloadItem) files.AddFileToDb(PostId(downloadItem), downloadItem.Filename); } - private string AddFileToDb(TumblrPost downloadItem) + protected string AddFileToDb(TumblrPost downloadItem) { if (AppendTemplate == null) { @@ -347,7 +349,7 @@ public bool CheckIfFileExistsInDB(string filenameUrl) return files.CheckIfFileExistsInDB(filenameUrl); } - private bool CheckIfFileExistsInDB(TumblrPost downloadItem) + protected bool CheckIfFileExistsInDB(TumblrPost downloadItem) { string filename = FileName(downloadItem); if (shellService.Settings.LoadAllDatabases) @@ -379,7 +381,7 @@ private void DownloadTextPost(TumblrPost downloadItem) } } - private void UpdateBlogDB(string postType) + protected void UpdateBlogDB(string postType) { blog.UpdatePostCount(postType); blog.UpdateProgress(false); @@ -400,12 +402,12 @@ protected static string Url(TumblrPost downloadItem) return downloadItem.Url; } - private static string FileName(TumblrPost downloadItem) + protected virtual string FileName(TumblrPost downloadItem) { return downloadItem.Url.Split('/').Last(); } - private static string FileNameNew(TumblrPost downloadItem) + protected static string FileNameNew(TumblrPost downloadItem) { return downloadItem.Filename; } diff --git a/src/TumblThree/TumblThree.Applications/Downloader/TumblrJsonDownloader.cs b/src/TumblThree/TumblThree.Applications/Downloader/JsonDownloader.cs similarity index 87% rename from src/TumblThree/TumblThree.Applications/Downloader/TumblrJsonDownloader.cs rename to src/TumblThree/TumblThree.Applications/Downloader/JsonDownloader.cs index baf3c8c1..30bfd372 100644 --- a/src/TumblThree/TumblThree.Applications/Downloader/TumblrJsonDownloader.cs +++ b/src/TumblThree/TumblThree.Applications/Downloader/JsonDownloader.cs @@ -9,7 +9,7 @@ using System.Xml; using TumblThree.Applications.DataModels; -using TumblThree.Applications.DataModels.TumblrCrawlerData; +using TumblThree.Applications.DataModels.CrawlerData; using TumblThree.Applications.Properties; using TumblThree.Applications.Services; using TumblThree.Domain; @@ -17,16 +17,16 @@ namespace TumblThree.Applications.Downloader { - public class TumblrJsonDownloader : ICrawlerDataDownloader + public class JsonDownloader : ICrawlerDataDownloader { private readonly IBlog blog; private readonly ICrawlerService crawlerService; - private readonly IPostQueue> jsonQueue; + private readonly IPostQueue> jsonQueue; private readonly IShellService shellService; private readonly CancellationToken ct; private readonly PauseToken pt; - public TumblrJsonDownloader(IShellService shellService, PauseToken pt, IPostQueue> jsonQueue, + public JsonDownloader(IShellService shellService, PauseToken pt, IPostQueue> jsonQueue, ICrawlerService crawlerService, IBlog blog, CancellationToken ct) { this.shellService = shellService; @@ -44,7 +44,7 @@ public virtual async Task DownloadCrawlerDataAsync() try { - foreach (TumblrCrawlerData downloadItem in jsonQueue.GetConsumingEnumerable(ct)) + foreach (CrawlerData downloadItem in jsonQueue.GetConsumingEnumerable(ct)) { if (ct.IsCancellationRequested) { @@ -67,7 +67,7 @@ public virtual async Task DownloadCrawlerDataAsync() await Task.WhenAll(trackedTasks); } - private async Task DownloadPostAsync(TumblrCrawlerData downloadItem) + private async Task DownloadPostAsync(CrawlerData downloadItem) { try { @@ -78,7 +78,7 @@ private async Task DownloadPostAsync(TumblrCrawlerData downloadItem) } } - private async Task DownloadTextPostAsync(TumblrCrawlerData crawlerData) + private async Task DownloadTextPostAsync(CrawlerData crawlerData) { string blogDownloadLocation = blog.DownloadLocation(); string fileLocation = FileLocation(blogDownloadLocation, crawlerData.Filename); @@ -89,7 +89,7 @@ private async Task AppendToTextFileAsync(string fileLocation, T data) { try { - if (typeof(T) == typeof(DataModels.TumblrSearchJson.Datum)) + if (typeof(T) == typeof(DataModels.TumblrSearchJson.Datum) || typeof(T) == typeof(DataModels.Twitter.TimelineTweets.Tweet)) { var serializer = new JsonSerializer(); using (StreamWriter sw = new StreamWriter(fileLocation, false)) diff --git a/src/TumblThree/TumblThree.Applications/Downloader/TumblrDownloader.cs b/src/TumblThree/TumblThree.Applications/Downloader/TumblrDownloader.cs index 7449bcc3..dd814595 100644 --- a/src/TumblThree/TumblThree.Applications/Downloader/TumblrDownloader.cs +++ b/src/TumblThree/TumblThree.Applications/Downloader/TumblrDownloader.cs @@ -18,7 +18,7 @@ public class TumblrDownloader : AbstractDownloader private int numberOfPagesCrawled = 0; public TumblrDownloader(IShellService shellService, IManagerService managerService, PauseToken pt, - IProgress progress, IPostQueue postQueue, FileDownloader fileDownloader, + IProgress progress, IPostQueue postQueue, FileDownloader fileDownloader, ICrawlerService crawlerService, IBlog blog, IFiles files, CancellationToken ct) : base(shellService, managerService, ct, pt, progress, postQueue, fileDownloader, crawlerService, blog, files) { diff --git a/src/TumblThree/TumblThree.Applications/Downloader/TumblrXmlDownloader.cs b/src/TumblThree/TumblThree.Applications/Downloader/TumblrXmlDownloader.cs index 1d04a5de..895ca92a 100644 --- a/src/TumblThree/TumblThree.Applications/Downloader/TumblrXmlDownloader.cs +++ b/src/TumblThree/TumblThree.Applications/Downloader/TumblrXmlDownloader.cs @@ -7,7 +7,7 @@ using System.Xml; using System.Xml.Linq; using TumblThree.Applications.DataModels; -using TumblThree.Applications.DataModels.TumblrCrawlerData; +using TumblThree.Applications.DataModels.CrawlerData; using TumblThree.Applications.Properties; using TumblThree.Applications.Services; using TumblThree.Domain; @@ -19,12 +19,12 @@ public sealed class TumblrXmlDownloader : ICrawlerDataDownloader { private readonly IBlog _blog; private readonly ICrawlerService _crawlerService; - private readonly IPostQueue> _xmlQueue; + private readonly IPostQueue> _xmlQueue; private readonly IShellService _shellService; private readonly CancellationToken _ct; private readonly PauseToken _pt; - public TumblrXmlDownloader(IShellService shellService, PauseToken pt, IPostQueue> xmlQueue, ICrawlerService crawlerService, IBlog blog, CancellationToken ct) + public TumblrXmlDownloader(IShellService shellService, PauseToken pt, IPostQueue> xmlQueue, ICrawlerService crawlerService, IBlog blog, CancellationToken ct) { _shellService = shellService; _crawlerService = crawlerService; @@ -41,7 +41,7 @@ public async Task DownloadCrawlerDataAsync() try { - foreach (TumblrCrawlerData downloadItem in _xmlQueue.GetConsumingEnumerable(_ct)) + foreach (CrawlerData downloadItem in _xmlQueue.GetConsumingEnumerable(_ct)) { if (_ct.IsCancellationRequested) { @@ -64,7 +64,7 @@ public async Task DownloadCrawlerDataAsync() await Task.WhenAll(trackedTasks); } - private async Task DownloadPostAsync(TumblrCrawlerData downloadItem) + private async Task DownloadPostAsync(CrawlerData downloadItem) { try { @@ -75,7 +75,7 @@ private async Task DownloadPostAsync(TumblrCrawlerData downloadItem) } } - private async Task DownloadTextPostAsync(TumblrCrawlerData crawlerData) + private async Task DownloadTextPostAsync(CrawlerData crawlerData) { string blogDownloadLocation = _blog.DownloadLocation(); string fileLocation = FileLocation(blogDownloadLocation, crawlerData.Filename); diff --git a/src/TumblThree/TumblThree.Applications/Downloader/TwitterDownloader.cs b/src/TumblThree/TumblThree.Applications/Downloader/TwitterDownloader.cs new file mode 100644 index 00000000..fac01653 --- /dev/null +++ b/src/TumblThree/TumblThree.Applications/Downloader/TwitterDownloader.cs @@ -0,0 +1,32 @@ +using System; +using System.Linq; +using System.Threading; +using TumblThree.Applications.DataModels; +using TumblThree.Applications.DataModels.TumblrPosts; +using TumblThree.Applications.Services; +using TumblThree.Domain.Models.Blogs; +using TumblThree.Domain.Models.Files; + +namespace TumblThree.Applications.Downloader +{ + public class TwitterDownloader : AbstractDownloader + { + public TwitterDownloader(IShellService shellService, IManagerService managerService, CancellationToken ct, PauseToken pt, IProgress progress, + IPostQueue postQueue, FileDownloader fileDownloader, ICrawlerService crawlerService = null, IBlog blog = null, IFiles files = null) + : base(shellService, managerService, ct, pt, progress, postQueue, fileDownloader, crawlerService, blog, files) + { + } + + protected override string FileName(TumblrPost downloadItem) + { + var url = downloadItem.Url.Split('/').Last(); + if (url.Contains("?format=") && url.Contains("&name=")) + { + var ext = url.Substring(url.IndexOf('?') + 1).Replace("format=", ""); + ext = ext.Substring(0, ext.IndexOf('&')); + url = url.Substring(0, url.IndexOf('?')) + "." + ext; + } + return url; + } + } +} diff --git a/src/TumblThree/TumblThree.Applications/Properties/AppSettings.cs b/src/TumblThree/TumblThree.Applications/Properties/AppSettings.cs index 00f9af2b..0a0c5847 100644 --- a/src/TumblThree/TumblThree.Applications/Properties/AppSettings.cs +++ b/src/TumblThree/TumblThree.Applications/Properties/AppSettings.cs @@ -24,11 +24,21 @@ public sealed class AppSettings : IExtensibleDataObject "best", "1280", "500", "400", "250", "100", "75" }; + private static readonly string[] imageSizeCategories = + { + "large", "medium", "small", "thumb" + }; + private static readonly string[] videoSizes = { "1080", "480" }; + private static readonly string[] videoSizeCategories = + { + "large", "medium", "small" + }; + private static string[] tumblrHosts = { "data.tumblr.com" @@ -152,9 +162,15 @@ public AppSettings() [DataMember] public string ImageSize { get; set; } + [DataMember] + public string ImageSizeCategory { get; set; } + [DataMember] public int VideoSize { get; set; } + [DataMember] + public string VideoSizeCategory { get; set; } + [DataMember] public string BlogType { get; set; } @@ -361,8 +377,12 @@ public AppSettings() public ObservableCollection ImageSizes => new ObservableCollection(imageSizes); + public ObservableCollection ImageSizeCategories => new ObservableCollection(imageSizeCategories); + public ObservableCollection VideoSizes => new ObservableCollection(videoSizes); + public ObservableCollection VideoSizeCategories => new ObservableCollection(videoSizeCategories); + public ObservableCollection BlogTypes => new ObservableCollection(blogTypes); public ObservableCollection LogLevels => new ObservableCollection(logLevels); @@ -458,7 +478,7 @@ private void Initialize() OAuthCallbackUrl = @"https://github.com/TumblThreeApp/TumblThree"; ApiKey = "x8pd1InspmnuLSFKT4jNxe8kQUkbRXPNkAffntAFSk01UjRsLV"; SecretKey = "Mul4BviRQgPLuhN1xzEqmXzwvoWicEoc4w6ftWBGWtioEvexmM"; - UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36"; + UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"; OAuthToken = string.Empty; OAuthTokenSecret = string.Empty; Left = 50; @@ -490,6 +510,8 @@ private void Initialize() BufferSize = 512; ImageSize = "best"; VideoSize = 1080; + ImageSizeCategory = ""; + VideoSizeCategory = ""; BlogType = "None"; CheckClipboard = true; ShowPicturePreview = true; diff --git a/src/TumblThree/TumblThree.Applications/Properties/Resources.Designer.cs b/src/TumblThree/TumblThree.Applications/Properties/Resources.Designer.cs index d9edf062..0534d4d8 100644 --- a/src/TumblThree/TumblThree.Applications/Properties/Resources.Designer.cs +++ b/src/TumblThree/TumblThree.Applications/Properties/Resources.Designer.cs @@ -729,6 +729,15 @@ public static string ProgressUniqueDownloads { } } + /// + /// Looks up a localized string similar to Protected or offline blog detected: {0}.. + /// + public static string ProtectedBlog { + get { + return ResourceManager.GetString("ProtectedBlog", resourceCulture); + } + } + /// /// Looks up a localized string similar to Queuelist. /// diff --git a/src/TumblThree/TumblThree.Applications/Properties/Resources.resx b/src/TumblThree/TumblThree.Applications/Properties/Resources.resx index c7e3ce78..5c84b67c 100644 --- a/src/TumblThree/TumblThree.Applications/Properties/Resources.resx +++ b/src/TumblThree/TumblThree.Applications/Properties/Resources.resx @@ -393,4 +393,7 @@ TumblThree requires the "Microsoft Visual C++ 2015-2019 Redistributable". You need to download and install it first. Do you want to download it now? + + Protected or offline blog detected: {0}. + \ No newline at end of file diff --git a/src/TumblThree/TumblThree.Applications/Services/IWebRequestFactory.cs b/src/TumblThree/TumblThree.Applications/Services/IWebRequestFactory.cs index 2117f26e..db7d7add 100644 --- a/src/TumblThree/TumblThree.Applications/Services/IWebRequestFactory.cs +++ b/src/TumblThree/TumblThree.Applications/Services/IWebRequestFactory.cs @@ -21,7 +21,7 @@ public interface IWebRequestFactory Task RemotePageIsValidAsync(string url); - Task ReadRequestToEndAsync(HttpWebRequest request); + Task ReadRequestToEndAsync(HttpWebRequest request, bool storeCookies = false); Task ReadRequestToEnd2Async(HttpWebRequest request); diff --git a/src/TumblThree/TumblThree.Applications/Services/WebRequestFactory.cs b/src/TumblThree/TumblThree.Applications/Services/WebRequestFactory.cs index 752ddf8b..d72db999 100644 --- a/src/TumblThree/TumblThree.Applications/Services/WebRequestFactory.cs +++ b/src/TumblThree/TumblThree.Applications/Services/WebRequestFactory.cs @@ -29,7 +29,7 @@ public WebRequestFactory(IShellService shellService, ISharedCookieService cookie private HttpWebRequest CreateStubRequest(string url, string referer = "", Dictionary headers = null, bool allowAutoRedirect = true) { - var request = (HttpWebRequest)WebRequest.Create(HttpUtility.UrlDecode(url)); + var request = (HttpWebRequest)WebRequest.Create(url); //HttpUtility.UrlDecode(url) what was the use case!? request.ProtocolVersion = HttpVersion.Version11; request.UserAgent = settings.UserAgent; request.AllowAutoRedirect = allowAutoRedirect; @@ -124,10 +124,14 @@ public async Task RemotePageIsValidAsync(string url) return (response.StatusCode == HttpStatusCode.OK); } - public async Task ReadRequestToEndAsync(HttpWebRequest request) + public async Task ReadRequestToEndAsync(HttpWebRequest request, bool storeCookies = false) { using (var response = await request.GetResponseAsync().TimeoutAfter(shellService.Settings.TimeOut) as HttpWebResponse) { + if (storeCookies) + { + cookieService.SetUriCookie(response.Cookies); + } using (Stream stream = GetStreamForApiRequest(response.GetResponseStream())) { using (var buffer = new BufferedStream(stream)) diff --git a/src/TumblThree/TumblThree.Applications/TumblThree.Applications.csproj b/src/TumblThree/TumblThree.Applications/TumblThree.Applications.csproj index 28aaff07..f011b170 100644 --- a/src/TumblThree/TumblThree.Applications/TumblThree.Applications.csproj +++ b/src/TumblThree/TumblThree.Applications/TumblThree.Applications.csproj @@ -124,18 +124,24 @@ + + - + - + + + + + @@ -252,6 +258,7 @@ + diff --git a/src/TumblThree/TumblThree.Applications/ViewModels/DetailsViewModels/DetailsTwitterBlogViewModel.cs b/src/TumblThree/TumblThree.Applications/ViewModels/DetailsViewModels/DetailsTwitterBlogViewModel.cs new file mode 100644 index 00000000..d2a7a9c4 --- /dev/null +++ b/src/TumblThree/TumblThree.Applications/ViewModels/DetailsViewModels/DetailsTwitterBlogViewModel.cs @@ -0,0 +1,78 @@ +using System.ComponentModel.Composition; +using System.Waf.Applications; +using System.Windows.Forms; +using System.Windows.Input; +using TumblThree.Applications.Services; +using TumblThree.Applications.Views; +using TumblThree.Domain.Models.Blogs; + +namespace TumblThree.Applications.ViewModels.DetailsViewModels +{ + [Export(typeof(IDetailsViewModel))] + [ExportMetadata("BlogType", typeof(TwitterBlog))] + public class DetailsTwitterBlogViewModel : ViewModel, IDetailsViewModel + { + private readonly DelegateCommand _browseFileDownloadLocationCommand; + private readonly DelegateCommand _copyUrlCommand; + + private readonly IClipboardService _clipboardService; + private readonly IDetailsService _detailsService; + private IBlog _blogFile; + private int _count = 0; + + [ImportingConstructor] + public DetailsTwitterBlogViewModel([Import("TwitterBlogView", typeof(IDetailsView))] IDetailsView view, IClipboardService clipboardService, IDetailsService detailsService) + : base(view) + { + _clipboardService = clipboardService; + _detailsService = detailsService; + _copyUrlCommand = new DelegateCommand(CopyUrlToClipboard); + _browseFileDownloadLocationCommand = new DelegateCommand(BrowseFileDownloadLocation); + } + + public ICommand CopyUrlCommand => _copyUrlCommand; + + public ICommand BrowseFileDownloadLocationCommand => _browseFileDownloadLocationCommand; + + public void ViewLostFocus() + { + if (Count == 1) BlogFile?.Save(); + } + + public bool FilenameTemplateValidate(string enteredFilenameTemplate) + { + return _detailsService.FilenameTemplateValidate(enteredFilenameTemplate); + } + + public IBlog BlogFile + { + get => _blogFile; + set => SetProperty(ref _blogFile, value); + } + + public int Count + { + get => _count; + set => SetProperty(ref _count, value); + } + + private void CopyUrlToClipboard() + { + if (BlogFile != null) + { + _clipboardService.SetText(BlogFile.Url); + } + } + + private void BrowseFileDownloadLocation() + { + using (var dialog = new FolderBrowserDialog { SelectedPath = BlogFile?.FileDownloadLocation }) + { + if (dialog.ShowDialog() == DialogResult.OK && BlogFile != null) + { + BlogFile.FileDownloadLocation = dialog.SelectedPath; + } + } + } + } +} diff --git a/src/TumblThree/TumblThree.Applications/ViewModels/SettingsViewModel.cs b/src/TumblThree/TumblThree.Applications/ViewModels/SettingsViewModel.cs index abcb8121..8ff785d1 100644 --- a/src/TumblThree/TumblThree.Applications/ViewModels/SettingsViewModel.cs +++ b/src/TumblThree/TumblThree.Applications/ViewModels/SettingsViewModel.cs @@ -78,6 +78,7 @@ public class SettingsViewModel : ViewModel private bool _forceSize; private bool _forceRescan; private string _imageSize; + private string _imageSizeCategory; private bool _limitConnectionsApi; private bool _limitConnectionsSvc; private bool _limitScanBandwidth; @@ -123,6 +124,7 @@ public class SettingsViewModel : ViewModel private int _timeOut; private string _timerInterval; private int _videoSize; + private string _videoSizeCategory; private int _settingsTabIndex; private string _userAgent; private string _tumblrUser = string.Empty; @@ -333,6 +335,18 @@ public int VideoSize set => SetProperty(ref _videoSize, value); } + public string ImageSizeCategory + { + get => _imageSizeCategory; + set => SetProperty(ref _imageSizeCategory, value); + } + + public string VideoSizeCategory + { + get => _videoSizeCategory; + set => SetProperty(ref _videoSizeCategory, value); + } + public string BlogType { get => _blogType; @@ -1001,6 +1015,8 @@ private void LoadSettings() LimitScanBandwidth = _settings.LimitScanBandwidth; ImageSize = _settings.ImageSize; VideoSize = _settings.VideoSize; + ImageSizeCategory = _settings.ImageSizeCategory; + VideoSizeCategory = _settings.VideoSizeCategory; BlogType = _settings.BlogType; TimeOut = _settings.TimeOut; LimitConnectionsApi = _settings.LimitConnectionsApi; @@ -1165,7 +1181,7 @@ private void LoadSettings() ProxyPort = string.Empty; TimerInterval = "22:40:00"; SettingsTabIndex = 0; - UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36"; + UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"; LogLevel = nameof(System.Diagnostics.TraceLevel.Verbose); GroupPhotoSets = false; FilenameTemplate = "%f"; @@ -1277,6 +1293,8 @@ private void SaveSettings() _settings.Bandwidth = Bandwidth; _settings.ImageSize = ImageSize; _settings.VideoSize = VideoSize; + _settings.ImageSizeCategory = ImageSizeCategory; + _settings.VideoSizeCategory = VideoSizeCategory; _settings.BlogType = BlogType; _settings.CheckClipboard = CheckClipboard; _settings.ShowPicturePreview = ShowPicturePreview; diff --git a/src/TumblThree/TumblThree.Domain/Models/BlogFactory.cs b/src/TumblThree/TumblThree.Domain/Models/BlogFactory.cs index 12fe0b0a..6aa80187 100644 --- a/src/TumblThree/TumblThree.Domain/Models/BlogFactory.cs +++ b/src/TumblThree/TumblThree.Domain/Models/BlogFactory.cs @@ -17,7 +17,7 @@ internal BlogFactory(IUrlValidator urlValidator) _urlValidator = urlValidator; } - public bool IsValidTumblrBlogUrl(string blogUrl) + public bool IsValidBlogUrl(string blogUrl) { blogUrl = _urlValidator.AddHttpsProtocol(blogUrl); return _urlValidator.IsValidTumblrUrl(blogUrl) @@ -26,7 +26,8 @@ public bool IsValidTumblrBlogUrl(string blogUrl) || _urlValidator.IsValidTumblrLikesUrl(blogUrl) || _urlValidator.IsValidTumblrSearchUrl(blogUrl) || _urlValidator.IsValidTumblrTagSearchUrl(blogUrl) - || _urlValidator.IsTumbexUrl(blogUrl); + || _urlValidator.IsTumbexUrl(blogUrl) + || _urlValidator.IsValidTwitterUrl(blogUrl); } public bool IsValidUrl(string url) @@ -74,6 +75,11 @@ public IBlog GetBlog(string blogUrl, string path, string filenameTemplate) return TumblrTagSearchBlog.Create(blogUrl, path, filenameTemplate); } + if (_urlValidator.IsValidTwitterUrl(blogUrl)) + { + return TwitterBlog.Create(blogUrl, path, filenameTemplate); + } + throw new ArgumentException("Website is not supported!", nameof(blogUrl)); } diff --git a/src/TumblThree/TumblThree.Domain/Models/Blogs/TwitterBlog.cs b/src/TumblThree/TumblThree.Domain/Models/Blogs/TwitterBlog.cs new file mode 100644 index 00000000..047961ee --- /dev/null +++ b/src/TumblThree/TumblThree.Domain/Models/Blogs/TwitterBlog.cs @@ -0,0 +1,43 @@ +using System; +using System.IO; +using System.Runtime.Serialization; +using TumblThree.Domain.Models.Files; + +namespace TumblThree.Domain.Models.Blogs +{ + [DataContract] + public class TwitterBlog : Blog + { + public static Blog Create(string url, string location, string filenameTemplate) + { + var blog = new TwitterBlog() + { + Url = ExtractUrl(url), + Name = ExtractName(url), + BlogType = BlogTypes.twitter, + OriginalBlogType = BlogTypes.twitter, + Location = location, + Online = true, + Version = "4", + DateAdded = DateTime.Now, + FilenameTemplate = filenameTemplate + }; + + Directory.CreateDirectory(location); + Directory.CreateDirectory(Path.Combine(Directory.GetParent(location).FullName, blog.Name)); + + blog.ChildId = Path.Combine(location, blog.Name + "_files." + blog.BlogType); + if (!File.Exists(blog.ChildId)) + { + IFiles files = new TwitterBlogFiles(blog.Name, blog.Location); + files.Save(); + } + + return blog; + } + + protected static new string ExtractName(string url) => url.Split('/')[3]; + + protected static new string ExtractUrl(string url) => "https://twitter.com/" + ExtractName(url); + } +} diff --git a/src/TumblThree/TumblThree.Domain/Models/Files/TwitterBlogFiles.cs b/src/TumblThree/TumblThree.Domain/Models/Files/TwitterBlogFiles.cs new file mode 100644 index 00000000..fdb30101 --- /dev/null +++ b/src/TumblThree/TumblThree.Domain/Models/Files/TwitterBlogFiles.cs @@ -0,0 +1,14 @@ +using System.Runtime.Serialization; + +namespace TumblThree.Domain.Models.Files +{ + [DataContract] + public class TwitterBlogFiles : Files + { + public TwitterBlogFiles(string name, string location) + : base(name, location) + { + BlogType = BlogTypes.twitter; + } + } +} diff --git a/src/TumblThree/TumblThree.Domain/Models/IBlogFactory.cs b/src/TumblThree/TumblThree.Domain/Models/IBlogFactory.cs index 62ed2e75..baa25147 100644 --- a/src/TumblThree/TumblThree.Domain/Models/IBlogFactory.cs +++ b/src/TumblThree/TumblThree.Domain/Models/IBlogFactory.cs @@ -4,7 +4,7 @@ namespace TumblThree.Domain.Models { public interface IBlogFactory { - bool IsValidTumblrBlogUrl(string blogUrl); + bool IsValidBlogUrl(string blogUrl); bool IsValidUrl(string blogUrl); diff --git a/src/TumblThree/TumblThree.Domain/Models/IUrlValidator.cs b/src/TumblThree/TumblThree.Domain/Models/IUrlValidator.cs index ddec16bb..33e1a666 100644 --- a/src/TumblThree/TumblThree.Domain/Models/IUrlValidator.cs +++ b/src/TumblThree/TumblThree.Domain/Models/IUrlValidator.cs @@ -19,5 +19,7 @@ public interface IUrlValidator bool IsValidTumblrLikesUrl(string url); bool IsValidUrl(string url); + + bool IsValidTwitterUrl(string url); } } diff --git a/src/TumblThree/TumblThree.Domain/Models/UrlValidator.cs b/src/TumblThree/TumblThree.Domain/Models/UrlValidator.cs index eafc833f..827a053b 100644 --- a/src/TumblThree/TumblThree.Domain/Models/UrlValidator.cs +++ b/src/TumblThree/TumblThree.Domain/Models/UrlValidator.cs @@ -10,6 +10,7 @@ public class UrlValidator : IUrlValidator { private readonly Regex tumbexRegex = new Regex("(http[A-Za-z0-9_/:.]*www.tumbex.com[A-Za-z0-9_/:.-]*tumblr/)"); private readonly Regex urlRegex = new Regex("(^https?://[A-Za-z0-9_.]*[/]?$)"); + private readonly Regex twitterRegex = new Regex("(^https?://twitter.com/[A-Za-z0-9_]+$)"); public bool IsValidTumblrUrl(string url) { @@ -59,6 +60,11 @@ public bool IsValidUrl(string url) urlRegex.IsMatch(url); } + public bool IsValidTwitterUrl(string url) + { + return url != null && twitterRegex.IsMatch(url) && !url.EndsWith("/home"); + } + public string AddHttpsProtocol(string url) { if (url == null) diff --git a/src/TumblThree/TumblThree.Domain/TumblThree.Domain.csproj b/src/TumblThree/TumblThree.Domain/TumblThree.Domain.csproj index e72b9b2f..6b931283 100644 --- a/src/TumblThree/TumblThree.Domain/TumblThree.Domain.csproj +++ b/src/TumblThree/TumblThree.Domain/TumblThree.Domain.csproj @@ -112,6 +112,7 @@ + @@ -125,6 +126,7 @@ + diff --git a/src/TumblThree/TumblThree.Presentation/Properties/Resources.Designer.cs b/src/TumblThree/TumblThree.Presentation/Properties/Resources.Designer.cs index 0811518c..23ba31b6 100644 --- a/src/TumblThree/TumblThree.Presentation/Properties/Resources.Designer.cs +++ b/src/TumblThree/TumblThree.Presentation/Properties/Resources.Designer.cs @@ -879,6 +879,15 @@ public static string ImageSize { } } + /// + /// Looks up a localized string similar to Image size (category). + /// + public static string ImageSizeCategory { + get { + return ResourceManager.GetString("ImageSizeCategory", resourceCulture); + } + } + /// /// Looks up a localized string similar to Import Blog List. /// @@ -2100,7 +2109,7 @@ public static string ToolTipFilenameTemplate { /// %k reblog-key ///Mandatory tokens (if not using token %f) to make filenames unique: /// %x "_{number}" ({number}: 2..n) appended to filename - /// %y " ({number})" ({number} [rest of string was truncated]";. + /// %y " ({nu [rest of string was truncated]";. /// public static string ToolTipFilenameTemplateDescription { get { @@ -2193,6 +2202,26 @@ public static string ToolTipImageSize { } } + /// + /// Looks up a localized string similar to Photo size to download (Twitter). + /// + public static string ToolTipImageSizeCategory { + get { + return ResourceManager.GetString("ToolTipImageSizeCategory", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Photo size category to download. + /// + ///Possible categories: large, medium, small, thumb. + /// + public static string ToolTipImageSizeCategoryDescription { + get { + return ResourceManager.GetString("ToolTipImageSizeCategoryDescription", resourceCulture); + } + } + /// /// Looks up a localized string similar to Photo size to download in pixels. /// @@ -2770,6 +2799,26 @@ public static string ToolTipVideoConnectionsDescription { } } + /// + /// Looks up a localized string similar to Video size to download (Twitter). + /// + public static string ToolTipVideoSizeCategory { + get { + return ResourceManager.GetString("ToolTipVideoSizeCategory", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Video size category to download. + /// + ///Possible categories: large, medium, small. + /// + public static string ToolTipVideoSizeCategoryDescription { + get { + return ResourceManager.GetString("ToolTipVideoSizeCategoryDescription", resourceCulture); + } + } + /// /// Looks up a localized string similar to File type to download. /// @@ -2851,6 +2900,15 @@ public static string VideoSize { } } + /// + /// Looks up a localized string similar to Video size (category). + /// + public static string VideoSizeCategory { + get { + return ResourceManager.GetString("VideoSizeCategory", resourceCulture); + } + } + /// /// Looks up a localized string similar to Open the site in the web browser. /// diff --git a/src/TumblThree/TumblThree.Presentation/Properties/Resources.de.resx b/src/TumblThree/TumblThree.Presentation/Properties/Resources.de.resx index 76fd741a..20e0fe2c 100644 --- a/src/TumblThree/TumblThree.Presentation/Properties/Resources.de.resx +++ b/src/TumblThree/TumblThree.Presentation/Properties/Resources.de.resx @@ -843,4 +843,7 @@ Durch die Aktivierung wird die Speichernutzung während des Betriebs basierend a Wenn diese Option aktiviert ist, werden die Indexdateien nicht gelöscht, sondern in einen Unterordner Archive im Indexordner verschoben. Alle Indexdateien in diesem Unterordner können bei Aktivierung für die globale Überprüfung auf Dateiduplikate verwendet werden. + + Bildergröße (Kategorie) + \ No newline at end of file diff --git a/src/TumblThree/TumblThree.Presentation/Properties/Resources.es.resx b/src/TumblThree/TumblThree.Presentation/Properties/Resources.es.resx index 22c49df8..cbce240f 100644 --- a/src/TumblThree/TumblThree.Presentation/Properties/Resources.es.resx +++ b/src/TumblThree/TumblThree.Presentation/Properties/Resources.es.resx @@ -840,4 +840,7 @@ La activación aumentará el uso de memoria durante la operación en función de Si esta opción está activada, los archivos de índice no se eliminan, sino que se mueven a un archivo de subcarpeta dentro de la carpeta de índice. Todos los archivos de índice dentro de esta subcarpeta se pueden usar para la verificación global de archivos duplicados, si están activados. + + Tamaño de la imagen (categoría) + \ No newline at end of file diff --git a/src/TumblThree/TumblThree.Presentation/Properties/Resources.fr.resx b/src/TumblThree/TumblThree.Presentation/Properties/Resources.fr.resx index da2a2c84..feede303 100644 --- a/src/TumblThree/TumblThree.Presentation/Properties/Resources.fr.resx +++ b/src/TumblThree/TumblThree.Presentation/Properties/Resources.fr.resx @@ -843,4 +843,7 @@ L'activation augmentera l'utilisation de la mémoire pendant le fonctionnement e Si cette option est activée, les fichiers d'index ne sont pas supprimés, mais déplacés vers un sous-dossier Archive dans le dossier Index. Tous les fichiers d'index de ce sous-dossier peuvent être utilisés pour la vérification globale des doublons de fichiers, s'ils sont activés. + + Taille de l'image (catégorie) + \ No newline at end of file diff --git a/src/TumblThree/TumblThree.Presentation/Properties/Resources.resx b/src/TumblThree/TumblThree.Presentation/Properties/Resources.resx index 23f739fc..26b246ee 100644 --- a/src/TumblThree/TumblThree.Presentation/Properties/Resources.resx +++ b/src/TumblThree/TumblThree.Presentation/Properties/Resources.resx @@ -1097,4 +1097,26 @@ Activation will increase the memory usage during operation based on the amount o If this option is activated, the index files are not deleted, but moved to a subfolder Archive inside the Index folder. All index files inside this subfolder can be used for the global check for file duplicates, if activated. + + Image size (category) + + + Photo size category to download. + +Possible categories: large, medium, small, thumb + + + Photo size to download (Twitter) + + + Video size (category) + + + Video size to download (Twitter) + + + Video size category to download. + +Possible categories: large, medium, small + \ No newline at end of file diff --git a/src/TumblThree/TumblThree.Presentation/TumblThree.Presentation.csproj b/src/TumblThree/TumblThree.Presentation/TumblThree.Presentation.csproj index 0581b5d0..82f4fc4b 100644 --- a/src/TumblThree/TumblThree.Presentation/TumblThree.Presentation.csproj +++ b/src/TumblThree/TumblThree.Presentation/TumblThree.Presentation.csproj @@ -189,6 +189,10 @@ MSBuild:Compile Designer + + Designer + MSBuild:Compile + Designer MSBuild:Compile @@ -305,6 +309,9 @@ DetailsTumblrTagSearchView.xaml + + DetailsTwitterBlogView.xaml + ExceptionWindow.xaml diff --git a/src/TumblThree/TumblThree.Presentation/Views/DetailsViews/DetailsTwitterBlogView.xaml b/src/TumblThree/TumblThree.Presentation/Views/DetailsViews/DetailsTwitterBlogView.xaml new file mode 100644 index 00000000..39d35029 --- /dev/null +++ b/src/TumblThree/TumblThree.Presentation/Views/DetailsViews/DetailsTwitterBlogView.xaml @@ -0,0 +1,694 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/TumblThree/TumblThree.Presentation/Views/DetailsViews/DetailsTwitterBlogView.xaml.cs b/src/TumblThree/TumblThree.Presentation/Views/DetailsViews/DetailsTwitterBlogView.xaml.cs new file mode 100644 index 00000000..ec8bc9d2 --- /dev/null +++ b/src/TumblThree/TumblThree.Presentation/Views/DetailsViews/DetailsTwitterBlogView.xaml.cs @@ -0,0 +1,51 @@ +using System; +using System.ComponentModel.Composition; +using System.Waf.Applications; +using System.Windows; +using System.Windows.Controls; +using System.Windows.Input; +using TumblThree.Applications.ViewModels.DetailsViewModels; +using TumblThree.Applications.Views; + +namespace TumblThree.Presentation.Views +{ + /// + /// Interaction logic for QueueView.xaml. + /// + [Export("TwitterBlogView", typeof(IDetailsView))] + public partial class DetailsTwitterBlogView : IDetailsView + { + private readonly Lazy viewModel; + + public DetailsTwitterBlogView() + { + InitializeComponent(); + viewModel = new Lazy(() => ViewHelper.GetViewModel(this)); + } + + private DetailsTwitterBlogViewModel ViewModel + { + get { return viewModel.Value; } + } + + public int TabsCount => this.Tabs.Items.Count; + + // FIXME: Implement in proper MVVM. + private void Preview_OnMouseDown(object sender, MouseButtonEventArgs e) + { + var fullScreenMediaView = new FullScreenMediaView { DataContext = viewModel.Value.BlogFile }; + fullScreenMediaView.ShowDialog(); + } + + private void View_LostFocus(object sender, RoutedEventArgs e) + { + if (!((UserControl)sender).IsKeyboardFocusWithin) + ViewModel?.ViewLostFocus(); + } + + private void FilenameTemplate_PreviewLostKeyboardFocus(object sender, KeyboardFocusChangedEventArgs e) + { + e.Handled = !ViewModel.FilenameTemplateValidate(((TextBox)e.Source).Text); + } + } +} diff --git a/src/TumblThree/TumblThree.Presentation/Views/SettingsView.xaml b/src/TumblThree/TumblThree.Presentation/Views/SettingsView.xaml index e9f1d5c5..3bf3bf62 100644 --- a/src/TumblThree/TumblThree.Presentation/Views/SettingsView.xaml +++ b/src/TumblThree/TumblThree.Presentation/Views/SettingsView.xaml @@ -12,7 +12,7 @@ xmlns:vr="clr-namespace:TumblThree.Presentation.ValidationRules" Title="Settings" Width="760" - Height="785" + Height="790" d:DataContext="{d:DesignInstance vm:SettingsViewModel}" ContentRendered="Window_ContentRendered" ResizeMode="CanResize" @@ -667,41 +667,7 @@ - - - - - - - - - - - +