vrr/VOffline
2
0
Fork 0
mirror of https://github.com/Rast1234/VOffline.git synced 2026-04-28 03:49:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

httpclient fix, refactor namespaces, WIP video/playlist/album attachments

Browse source
This commit is contained in:
Rostislav Kirillov 2019-01-15 06:53:55 +05:00
parent 17330ece9a
commit cce04fcf8e
19 changed files with 445 additions and 262 deletions
Download patch file Download diff file Expand all files Collapse all files

55
VOffline/Models/Storage/AlbumWithPhoto.cs Normal file
View file

@ -0,0 +1,55 @@
using System;
using System.Collections.Generic;
using System.Linq;
using VkNet.Model;
using VkNet.Model.Attachments;
namespace VOffline.Models.Storage
{
public class AlbumWithPhoto
{
public PhotoAlbum Album { get; }
public IReadOnlyList<Photo> Photo { get; }
public AlbumWithPhoto(PhotoAlbum album, IReadOnlyList<Photo> photo)
{
Album = album;
Photo = photo
.OrderBy(a => a.CreateTime)
.ToList();
}
public AlbumWithPhoto(Album album, IReadOnlyList<Photo> photo)
{
this.Album = new PhotoAlbum
{
Id = album.Id.Value,
Title = album.Title,
Description = album.Description,
Created = album.CreateTime,
ThumbId = album.Thumb?.Id,
OwnerId = album.OwnerId,
Updated = album.UpdateTime
};
this.Photo = photo
.OrderBy(a => a.CreateTime)
.ToList();
}
public AlbumWithPhoto(IReadOnlyList<Photo> photo)
{
this.Album = new PhotoAlbum
{
Id = long.MinValue,
Title = "__default",
Description = "Photos without album",
Created = DateTime.MinValue
};
this.Photo = photo
.OrderBy(a => a.CreateTime)
.ToList();
}
}
}

1
VOffline/Models/Storage/Download.cs
View file

@ -5,7 +5,6 @@ using System.Threading;
using System.Threading.Tasks;
using RestSharp;
using VOffline.Services;
using VOffline.Services.Handlers;
namespace VOffline.Models.Storage
{

2
VOffline/Models/Storage/NetworkException.cs
View file

@ -1,6 +1,6 @@
using System;
namespace VOffline.Services
namespace VOffline.Models.Storage
{
public class NetworkException : ApplicationException
{

2
VOffline/Program.cs
View file

@ -69,6 +69,8 @@ namespace VOffline
serviceCollection.AddSingleton<CommentHandler>();
serviceCollection.AddSingleton<AudioHandler>();
serviceCollection.AddSingleton<PlaylistHandler>();
serviceCollection.AddSingleton<PhotoHandler>();
serviceCollection.AddSingleton<AlbumHandler>();
serviceCollection.AddSingleton<AttachmentProcessor>();
serviceCollection.AddTransient(provider => LogManager.GetLogger(Assembly.GetEntryAssembly(), typeof(Program)));

16
VOffline/Models/Storage/Extensions.cs → VOffline/Services/Extensions.cs
View file

@ -1,5 +1,8 @@
using System.Net;
using System.Collections.Generic;
using System.Net;
using log4net;
using RestSharp;
using VOffline.Models.Storage;
namespace VOffline.Services
{
@ -22,5 +25,16 @@ namespace VOffline.Services
throw new NetworkException($"Null response bytes");
}
}
public static void AddUnique<T>(this HashSet<T> result, IEnumerable<T> items, ILog log)
{
foreach (var item in items)
{
if (!result.Add(item))
{
log.Warn($"Duplicate item [{item}]");
}
}
}
}
}

43
VOffline/Services/Handlers/AlbumHandler.cs Normal file
View file

@ -0,0 +1,43 @@
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using log4net;
using VOffline.Models.Storage;
using VOffline.Services.Storage;
namespace VOffline.Services.Handlers
{
public class AlbumHandler : HandlerBase<AlbumWithPhoto>
{
private readonly AttachmentProcessor attachmentProcessor;
public AlbumHandler(FilesystemTools filesystemTools, AttachmentProcessor attachmentProcessor) : base(filesystemTools)
{
this.attachmentProcessor = attachmentProcessor;
}
public override async Task ProcessInternal(AlbumWithPhoto albumWithPhoto, DirectoryInfo workDir, CancellationToken token, ILog log)
{
if (!string.IsNullOrWhiteSpace(albumWithPhoto.Album.Description))
{
var text = filesystemTools.CreateFile(workDir, $"__description.txt", CreateMode.MergeWithExisting);
File.WriteAllText(text.FullName, albumWithPhoto.Album.Description);
}
if (albumWithPhoto.Album.ThumbId != null)
{
var cover = albumWithPhoto.Photo.FirstOrDefault(x => x.Id == albumWithPhoto.Album.ThumbId);
if (cover != null)
{
await attachmentProcessor.ProcessAttachment(cover, -1, workDir, token, log);
}
}
var attachmentTasks = albumWithPhoto.Photo.Select((a, i) => attachmentProcessor.ProcessAttachment(a, i, workDir, token, log));
await Task.WhenAll(attachmentTasks);
}
public override DirectoryInfo GetWorkingDirectory(AlbumWithPhoto albumWithPhoto, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, $"{albumWithPhoto.Album.Title}", CreateMode.MergeWithExisting);
}
}

14
VOffline/Services/Handlers/AudioHandler.cs
View file

@ -3,34 +3,32 @@ using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using log4net;
using VkNet;
using VOffline.Models.Storage;
using VOffline.Services.Storage;
using VOffline.Services.Vk;
using VOffline.Services.VkNetHacks;
namespace VOffline.Services.Handlers
{
public class AudioHandler : HandlerBase<long>
{
private readonly VkApi vkApi;
private readonly VkApiUtils vkApiUtils;
private readonly PlaylistHandler playlistHandler;
public AudioHandler(VkApi vkApi, FilesystemTools filesystemTools, PlaylistHandler playlistHandler) : base(filesystemTools)
public AudioHandler(VkApiUtils vkApiUtils, FilesystemTools filesystemTools, PlaylistHandler playlistHandler) : base(filesystemTools)
{
this.vkApi = vkApi;
this.vkApiUtils = vkApiUtils;
this.playlistHandler = playlistHandler;
}
public override async Task ProcessInternal(long id, DirectoryInfo workDir, CancellationToken token, ILog log)
{
var vkPlaylists = await vkApi.Audio.GetAllPlaylistsAsync(id, token, log);
var vkPlaylists = await vkApiUtils.GetAllPagesAsync(vkApiUtils.Playlists(id), 200, token, log);
log.Debug($"Audio: {vkPlaylists.Count} playlists");
var expandTasks = vkPlaylists.Select(p => vkApi.Audio.ExpandPlaylist(p, token, log));
var expandTasks = vkPlaylists.Select(p => vkApiUtils.ExpandPlaylist(p, token, log));
var playlists = await Task.WhenAll(expandTasks);
log.Debug($"Audio: {playlists.Sum(p => p.Audio.Count)} in {playlists.Length} playlists");
var allAudios = await vkApi.Audio.GetAllAudios(id, token, log);
var allAudios = await vkApiUtils.GetAllPagesAsync(vkApiUtils.Audios(id), long.MaxValue, token, log);
var audioInPlaylists = playlists
.SelectMany(p => p.Audio.Select(t => t.Id))
.ToHashSet();

13
VOffline/Services/Handlers/CommentsHandler.cs
View file

@ -3,29 +3,26 @@ using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using log4net;
using VkNet;
using VkNet.Model.Attachments;
using VOffline.Models.Storage;
using VOffline.Services.Storage;
using VOffline.Services.Vk;
using VOffline.Services.VkNetHacks;
namespace VOffline.Services.Handlers
{
public class CommentsHandler : HandlerBase<Post>
{
private readonly VkApi vkApi;
private readonly VkApiUtils vkApiUtils;
private readonly CommentHandler commentHandler;
public CommentsHandler(VkApi vkApi, FilesystemTools filesystemTools, CommentHandler commentHandler) : base(filesystemTools)
public CommentsHandler(VkApiUtils vkApiUtils, FilesystemTools filesystemTools, CommentHandler commentHandler) : base(filesystemTools)
{
this.vkApi = vkApi;
this.vkApiUtils = vkApiUtils;
this.commentHandler = commentHandler;
}
public override async Task ProcessInternal(Post post, DirectoryInfo workDir, CancellationToken token, ILog log)
{
var allComments = await vkApi.Wall.GetAllCommentsAsync(post.OwnerId.Value, post.Id.Value, token, log);
var allComments = await vkApiUtils.GetAllPagesAsync(vkApiUtils.Comments(post), 100, token, log);
log.Debug($"Post {post.Id} has {allComments.Count} comments");
var byDate = allComments
.OrderBy(c => c.Date)
@ -37,6 +34,6 @@ namespace VOffline.Services.Handlers
await Task.WhenAll(commentTasks);
}
public override DirectoryInfo GetWorkingDirectory(Post post, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, "comments", CreateMode.MergeWithExisting);
public override DirectoryInfo GetWorkingDirectory(Post post, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, "Comments", CreateMode.MergeWithExisting);
}
}

52
VOffline/Services/Handlers/PhotoHandler.cs Normal file
View file

@ -0,0 +1,52 @@
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using log4net;
using VOffline.Models.Storage;
using VOffline.Services.Storage;
using VOffline.Services.Vk;
namespace VOffline.Services.Handlers
{
public class PhotoHandler : HandlerBase<long>
{
private readonly VkApiUtils vkApiUtils;
private readonly AlbumHandler albumHandler;
public PhotoHandler(VkApiUtils vkApiUtils, AlbumHandler albumHandler, FilesystemTools filesystemTools) : base(filesystemTools)
{
this.vkApiUtils = vkApiUtils;
this.albumHandler = albumHandler;
}
public override async Task ProcessInternal(long id, DirectoryInfo workDir, CancellationToken token, ILog log)
{
var vkAlbums = await vkApiUtils.GetAllPagesAsync(vkApiUtils.PhotoAlbums(id), int.MaxValue, token, log);
var expandTasks = vkAlbums.Select(album => vkApiUtils.ExpandAlbum(album, token, log));
var albums = await Task.WhenAll(expandTasks);
var allPhotos = await vkApiUtils.GetAllPagesAsync(vkApiUtils.Photos(id), 100, token, log);
var photosInAlbums = albums
.SelectMany(awp => awp.Photo.Select(photo => photo.Id.Value))
.ToHashSet();
var uniquePhotos = allPhotos
.Where(photo => !photosInAlbums.Contains(photo.Id.Value))
.ToList();
var defaultAlbum = new AlbumWithPhoto(uniquePhotos);
var allAlbums = albums.ToList();
if (defaultAlbum.Photo.Any())
{
allAlbums.Add(defaultAlbum);
}
var allTasks = allAlbums
.OrderBy(p => p.Album.Created)
.Select(p => albumHandler.Process(p, workDir, token, log));
await Task.WhenAll(allTasks);
}
public override DirectoryInfo GetWorkingDirectory(long id, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, "Photo", CreateMode.MergeWithExisting);
}
}

10
VOffline/Services/Handlers/WallHandler.cs
View file

@ -3,27 +3,25 @@ using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using log4net;
using VkNet;
using VOffline.Services.Storage;
using VOffline.Services.Vk;
using VOffline.Services.VkNetHacks;
namespace VOffline.Services.Handlers
{
public class WallHandler : HandlerBase<long>
{
private readonly VkApi vkApi;
private readonly VkApiUtils vkApiUtils;
private readonly PostHandler postHandler;
public WallHandler(VkApi vkApi, FilesystemTools filesystemTools, PostHandler postHandler):base(filesystemTools)
public WallHandler(VkApiUtils vkApiUtils, FilesystemTools filesystemTools, PostHandler postHandler):base(filesystemTools)
{
this.vkApi = vkApi;
this.vkApiUtils = vkApiUtils;
this.postHandler = postHandler;
}
public override async Task ProcessInternal(long id, DirectoryInfo workDir, CancellationToken token, ILog log)
{
var allPosts = await vkApi.Wall.GetAllPostsAsync(id, token, log);
var allPosts = await vkApiUtils.GetAllPagesAsync(vkApiUtils.Posts(id), 100, token, log);
log.Debug($"Wall has {allPosts.Count} posts");
var allTasks = allPosts
.OrderBy(x => x.Date)

7
VOffline/Services/Logic.cs
View file

@ -28,8 +28,9 @@ namespace VOffline.Services
private readonly DownloadQueueProvider queueProvider;
private readonly WallHandler wallHandler;
private readonly AudioHandler audioHandler;
private readonly PhotoHandler photoHandler;
public Logic(TokenMagic tokenMagic, VkApi vkApi, VkApiUtils vkApiUtils, BackgroundDownloader downloader, FilesystemTools filesystemTools, DownloadQueueProvider queueProvider, WallHandler wallHandler, AudioHandler audioHandler, IOptionsSnapshot<Settings> settings)
public Logic(TokenMagic tokenMagic, VkApi vkApi, VkApiUtils vkApiUtils, BackgroundDownloader downloader, FilesystemTools filesystemTools, DownloadQueueProvider queueProvider, WallHandler wallHandler, AudioHandler audioHandler, PhotoHandler photoHandler, IOptionsSnapshot<Settings> settings)
{
this.settings = settings.Value;
this.tokenMagic = tokenMagic;
@ -40,6 +41,7 @@ namespace VOffline.Services
this.queueProvider = queueProvider;
this.wallHandler = wallHandler;
this.audioHandler = audioHandler;
this.photoHandler = photoHandler;
}
public async Task Run(CancellationToken token, ILog log)
@ -101,6 +103,9 @@ namespace VOffline.Services
case Mode.Audio:
await audioHandler.Process(id, dir, token, log);
break;
case Mode.Photos:
await photoHandler.Process(id, dir, token, log);
break;
case Mode.All:
throw new ArgumentOutOfRangeException(nameof(mode), mode, "This mode should have been replaced before processing");
default:

29
VOffline/Services/Storage/AttachmentProcessor.cs
View file

@ -1,5 +1,4 @@
using System;
using System.IO;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using log4net;
@ -7,20 +6,21 @@ using Newtonsoft.Json;
using VkNet;
using VkNet.Model;
using VkNet.Model.Attachments;
using VOffline.Services.Storage;
using VOffline.Services.Vk;
namespace VOffline.Services
namespace VOffline.Services.Storage
{
public class AttachmentProcessor
{
private readonly VkApi vkApi;
private readonly VkApiUtils vkApiUtils;
private readonly FilesystemTools filesystemTools;
private readonly DownloadQueueProvider downloadQueueProvider;
public AttachmentProcessor(VkApi vkApi, FilesystemTools filesystemTools, DownloadQueueProvider downloadQueueProvider)
public AttachmentProcessor(VkApi vkApi, VkApiUtils vkApiUtils, FilesystemTools filesystemTools, DownloadQueueProvider downloadQueueProvider)
{
this.vkApi = vkApi;
this.vkApiUtils = vkApiUtils;
this.filesystemTools = filesystemTools;
this.downloadQueueProvider = downloadQueueProvider;
}
@ -53,16 +53,25 @@ namespace VOffline.Services
await downloadQueueProvider.EnqueueAll(link.ToDownloads(number, filesystemTools, workDir, log), token);
await link.SaveHumanReadableText(number, filesystemTools, workDir, token, log);
break;
case VkNet.Model.Attachments.Video video: // vlc же как-то получает MP4 поток. а что делать с видосами на хостингах?
case VkNet.Model.Attachments.AudioPlaylist audioPlaylist:
var playlistWithAudio = await vkApiUtils.ExpandPlaylist(audioPlaylist, token, log);
//await playlistHandler.Process(p, workDir, token, log)
log.Warn($"TODO: playlist attachment");
break;
case VkNet.Model.Attachments.Album album:
var albumWithPhoto = await vkApiUtils.ExpandAlbum(album, token, log);
//await albumHandler.Process(p, workDir, token, log)
log.Warn($"TODO: photoalbum attachment");
break;
case VkNet.Model.Attachments.Video video:
await downloadQueueProvider.EnqueueAll(video.ToDownloads(number, filesystemTools, workDir, log), token);
break;
case VkNet.Model.Attachments.Note note: // note и page похожи
case VkNet.Model.Attachments.Page page:
case VkNet.Model.Attachments.Album album: // это к фотографиям
case VkNet.Model.Attachments.AudioPlaylist audioPlaylist: // это к аудиозаписям. так вообще бывает?
// остальное похоже на хлам
case VkNet.Model.Attachments.ApplicationContent applicationContent:
case VkNet.Model.Attachments.AudioMessage audioMessage:

2
VOffline/Services/Storage/BackgroundDownloader.cs
View file

@ -27,7 +27,7 @@ namespace VOffline.Services.Storage
IDownload item;
int success;
for(success=0; (item = await GetItem(token)) != null; success++)
for (success = 0; (item = await GetItem(token)) != null;)
{
// TODO: add second queue, semaphore and support for retries
try

1
VOffline/Services/Storage/DownloadQueueProvider.cs
View file

@ -6,7 +6,6 @@ using Microsoft.Extensions.Options;
using Nito.AsyncEx;
using VOffline.Models;
using VOffline.Models.Storage;
using VOffline.Services.Vk;
namespace VOffline.Services.Storage
{

42
VOffline/Services/Vk/AttachmentExtensions.cs
View file

@ -8,6 +8,7 @@ using System.Threading.Tasks;
using log4net;
using Newtonsoft.Json;
using VkNet;
using VkNet.Enums.SafetyEnums;
using VkNet.Model;
using VkNet.Model.Attachments;
using VOffline.Models.Storage;
@ -32,7 +33,7 @@ namespace VOffline.Services.Vk
public static IEnumerable<IDownload> ToDownloads(this Photo photo, int i, FilesystemTools filesystemTools, DirectoryInfo dir, ILog log)
{
// TODO: not sure about order
// TODO: not sure about ANYTHING here
var url = photo.BigPhotoSrc
?? photo.Photo2560
?? photo.PhotoSrc
@ -45,7 +46,22 @@ namespace VOffline.Services.Vk
?? photo.Photo100
?? photo.Photo75
?? photo.Photo50
?? photo.Sizes.Select(s => (square:s.Width*s.Height, size:s)).OrderByDescending(x => x.square).FirstOrDefault().size.Url;
?? photo.Sizes
.Select(s => (square:s.Width*s.Height, size:s))
.OrderByDescending(x => x.square)
.FirstOrDefault(s => s.square > 0) // width/height can be null
.size?.Url
?? photo.Sizes.FirstOrDefault(s => s.Type == PhotoSizeType.W)?.Url
?? photo.Sizes.FirstOrDefault(s => s.Type == PhotoSizeType.Z)?.Url
?? photo.Sizes.FirstOrDefault(s => s.Type == PhotoSizeType.Y)?.Url
?? photo.Sizes.FirstOrDefault(s => s.Type == PhotoSizeType.X)?.Url
?? photo.Sizes.FirstOrDefault(s => s.Type == PhotoSizeType.R)?.Url
?? photo.Sizes.FirstOrDefault(s => s.Type == PhotoSizeType.Q)?.Url
?? photo.Sizes.FirstOrDefault(s => s.Type == PhotoSizeType.P)?.Url
?? photo.Sizes.FirstOrDefault(s => s.Type == PhotoSizeType.O)?.Url
?? photo.Sizes.FirstOrDefault(s => s.Type == PhotoSizeType.M)?.Url
?? photo.Sizes.FirstOrDefault(s => s.Type == PhotoSizeType.S)?.Url
;
// TODO: i guess it's always jpeg?
var ext = Path.HasExtension(url?.AbsoluteUri) ? Path.GetExtension(url?.AbsoluteUri) : ".jpg";
@ -60,6 +76,28 @@ namespace VOffline.Services.Vk
}
}
public static IEnumerable<IDownload> ToDownloads(this Video video, int i, FilesystemTools filesystemTools, DirectoryInfo dir, ILog log)
{
var vkUrl = video.Files?.Mp4_1080
?? video.Files?.Mp4_720
?? video.Files?.Mp4_480
?? video.Files?.Mp4_360
?? video.Files?.Mp4_240;
if (vkUrl != null)
{
yield return new Download(vkUrl, dir, video.Title);
}
else if(video.Files?.External != null)
{
log.Warn($"Video {video.Id} [{video.Title}] is external");
yield return new Download(video.Files.External, dir, video.Title);
}
else
{
log.Warn($"Video {video.Id} [{video.Title}] has no links");
}
}
public static IEnumerable<IDownload> ToDownloads(this Document document, int i, FilesystemTools filesystemTools, DirectoryInfo dir, ILog log)
{
// TODO: looks like Title already has Extension

184
VOffline/Services/Vk/VkApiUtils.cs
View file

@ -1,9 +1,19 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using log4net;
using VkNet;
using VkNet.Enums;
using VkNet.Enums.Filters;
using VkNet.Enums.SafetyEnums;
using VkNet.Model;
using VkNet.Model.Attachments;
using VkNet.Model.RequestParams;
using VkNet.Utils;
using VOffline.Models.Storage;
namespace VOffline.Services.Vk
{
@ -64,7 +74,169 @@ namespace VOffline.Services.Vk
var group = groups[0];
return String.Join(" ", group.Name, GroupOrEmpty(" - ", group.Id.ToString(), group.ScreenName, group.Type.ToString()));
}
public PageGetter<PhotoAlbum> PhotoAlbums(long id) => async (count, offset) => await vkApi.Photo.GetAlbumsAsync(new PhotoGetAlbumsParams()
{
OwnerId = id,
NeedCovers = true,
NeedSystem = true,
Count = (uint)count,
Offset = (uint)offset
});
public PageGetter<Photo> Photos(long id) => async (count, offset) => await vkApi.Photo.GetAllAsync(new PhotoGetAllParams()
{
OwnerId = id,
Extended = true,
SkipHidden = false,
Count = (ulong)count,
Offset = (ulong)offset
});
public PageGetter<Photo> PhotosInAlbum(long id, PhotoAlbumType albumIdType) => async (count, offset) => await vkApi.Photo.GetAsync(new PhotoGetParams()
{
OwnerId = id,
AlbumId = albumIdType,
Extended = true,
PhotoSizes = true,
Count = (ulong)count,
Offset = (ulong)offset
});
public PageGetter<Post> Posts(long id) => async (count, offset) =>
{
var response = await vkApi.Wall.GetAsync(new WallGetParams
{
OwnerId = id,
Count = (ulong)count,
Offset = (ulong)offset
});
return new VkCollection<Post>(response.TotalCount, response.WallPosts);
};
public PageGetter<AudioPlaylist> Playlists(long id) => async (count, offset) => await vkApi.Audio.GetPlaylistsAsync(id, (uint)count, (uint)offset);
public PageGetter<Audio> Audios(long id) => async (count, offset) => await vkApi.Audio.GetAsync(new AudioGetParams()
{
OwnerId = id,
Count = (long)count,
Offset = (long)offset
});
public PageGetter<Audio> AudiosInPlaylist(AudioPlaylist playlist) => async (count, offset) => await vkApi.Audio.GetAsync(new AudioGetParams()
{
OwnerId = playlist.OwnerId.Value,
AlbumId = playlist.Id.Value,
Count = (long)count,
Offset = (long)offset
});
public PageGetter<Comment> Comments(Post post) => async (count, offset) => await vkApi.Wall.GetCommentsAsync(new WallGetCommentsParams()
{
Count = (uint)count,
Offset = (uint)offset,
OwnerId = post.OwnerId.Value,
PostId = post.Id.Value,
});
public async Task<IReadOnlyList<T>> GetAllPagesAsync<T>(PageGetter<T> pageGetter, decimal pageSize, CancellationToken token, ILog log, bool throwIfCountMismatch=false)
{
var firstPage = await pageGetter(pageSize, 0);
var total = firstPage.TotalCount;
log.Debug($"{0}/{total}, {firstPage.Count} items");
var result = new HashSet<T>((int)total);
result.AddUnique(firstPage, log);
var remainingPages = total / pageSize;
var pageTasks = Enumerable.Range(1, (int)remainingPages)
.Select(pageNumber => (ulong)pageNumber * pageSize)
.Select(async offset =>
{
var page = await pageGetter(pageSize, offset);
log.Debug($"{0}/{total}, {page.Count} items");
token.ThrowIfCancellationRequested();
return page;
});
var pages = await Task.WhenAll(pageTasks);
foreach (var page in pages)
{
result.AddUnique(page, log);
}
if ((int)total != result.Count)
{
var message = $"Expected {total} items, got {result.Count}. Maybe they were created/deleted, or it's VK bugs again.";
log.Warn(message);
if (throwIfCountMismatch)
{
throw new InvalidOperationException(message);
}
}
return result.ToList();
}
public async Task<PlaylistWithAudio> ExpandPlaylist(AudioPlaylist playlist, CancellationToken token, ILog log)
{
var audios = await GetAllPagesAsync(AudiosInPlaylist(playlist), long.MaxValue, token, log, true);
log.Debug($"Expanded playlist {playlist.Title}: {audios.Count} audios");
return new PlaylistWithAudio(playlist, audios);
}
public async Task<int> GetPhotoAlbumsSimpleCountAsync(long id, CancellationToken token, ILog log)
{
throw new NotImplementedException();
var errors = new Lazy<List<Exception>>();
try
{
return await vkApi.Photo.GetAlbumsCountAsync(id, null);
}
catch (Exception e)
{
errors.Value.Add(e);
}
token.ThrowIfCancellationRequested();
try
{
return await vkApi.Photo.GetAlbumsCountAsync(null, id);
}
catch (Exception e)
{
errors.Value.Add(e);
}
throw new Exception(string.Join("\n---------------------------------\n", errors.Value.Select(x => x.ToString())));
}
public async Task<AlbumWithPhoto> ExpandAlbum(Album album, CancellationToken token, ILog log)
{
var photos = await GetAllPagesAsync(PhotosInAlbum(album.OwnerId.Value, GetAlbumType(album.Id.Value)), 1000, token, log, true);
return new AlbumWithPhoto(album, photos);
}
public async Task<AlbumWithPhoto> ExpandAlbum(PhotoAlbum album, CancellationToken token, ILog log)
{
var photos = await GetAllPagesAsync(PhotosInAlbum(album.OwnerId.Value, GetAlbumType(album.Id)), 1000, token, log, true);
return new AlbumWithPhoto(album, photos);
}
private static PhotoAlbumType GetAlbumType(long albumId)
{
switch (albumId)
{
case -6:
return PhotoAlbumType.Profile;
case -7:
return PhotoAlbumType.Wall;
case -15:
return PhotoAlbumType.Saved;
default:
return PhotoAlbumType.Id(albumId);
}
}
private static string GroupOrEmpty(string separator, params string[] parts)
{
var all = String.Join(separator, parts);
@ -76,13 +248,7 @@ namespace VOffline.Services.Vk
private static readonly Regex CommunityPattern = new Regex(@"^(public|club|event)(\d+)$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
private static readonly Regex PersonalPattern = new Regex(@"^(id)(\d+)$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
private static readonly Regex DigitalPattern = new Regex(@"^(-?\d+)$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
public static void ThrowIfCountMismatch(decimal expectedTotal, decimal resultCount)
{
if (resultCount != expectedTotal)
{
throw new InvalidOperationException($"Expected {expectedTotal} items, got {resultCount}. Maybe they were created/deleted, try again.");
}
}
}
public delegate Task<VkCollection<T>> PageGetter<T>(decimal count, decimal offset);
}

204
VOffline/Services/Vk/VkNetExtensions.cs
View file

@ -1,204 +0,0 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using log4net;
using VkNet.Abstractions;
using VkNet.Model;
using VkNet.Model.Attachments;
using VkNet.Model.RequestParams;
using VOffline.Models.Storage;
namespace VOffline.Services.Vk
{
public static class VkNetExtensions
{
public static async Task<IReadOnlyList<Post>> GetAllPostsAsync(this IWallCategory wall, long id, CancellationToken token, ILog log)
{
var pageSize = 100u;
var wallResponse = await wall.GetAsync(new WallGetParams
{
OwnerId = id,
Count = pageSize
});
log.Debug($"Wall posts [{id}]: {wallResponse.WallPosts.Count}, offset {0}/{wallResponse.TotalCount}");
var total = wallResponse.TotalCount;
var result = new HashSet<Post>((int) total);
void AddToResult(IEnumerable<Post> posts)
{
foreach (var item in posts)
{
if (!result.Add(item))
{
log.Warn($"Duplicate item [{item}]");
}
}
}
AddToResult(wallResponse.WallPosts);
var remainingPages = total / pageSize;
var pageTasks = Enumerable.Range(1, (int) remainingPages)
.Select(pageNumber => (ulong) pageNumber * pageSize)
.Select(async offset =>
{
var page = await wall.GetAsync(new WallGetParams
{
OwnerId = id,
Count = pageSize,
Offset = offset
});
log.Debug($"Wall posts [{id}]: {page.WallPosts.Count}, offset {offset}/{page.TotalCount}");
token.ThrowIfCancellationRequested();
return page;
});
var pages = await Task.WhenAll(pageTasks);
foreach (var page in pages)
{
AddToResult(page.WallPosts);
}
if ((int) total != result.Count)
{
log.Warn($"Expected {total} items, got {result.Count}. Maybe they were created/deleted, or it's VK bugs again.");
}
return result
.ToList();
}
public static async Task<IReadOnlyList<Comment>> GetAllCommentsAsync(this IWallCategory wall, long id, long postId, CancellationToken token, ILog log)
{
var pageSize = 100u;
var commentResponse = await wall.GetCommentsAsync(new WallGetCommentsParams()
{
Count = pageSize,
Offset = 0,
OwnerId = id,
PostId = postId,
});
log.Debug($"Post comments [{id} {postId}]: {commentResponse.Count}, offset {0}/{commentResponse.TotalCount}");
var total = commentResponse.TotalCount;
var result = new HashSet<Comment>((int)total);
void AddToResult(IEnumerable<Comment> posts)
{
foreach (var item in posts)
{
if (!result.Add(item))
{
log.Warn($"Duplicate item [{item}]");
}
}
}
AddToResult(commentResponse);
var remainingPages = total / pageSize;
var pageTasks = Enumerable.Range(1, (int)remainingPages)
.Select(pageNumber => (long)pageNumber * pageSize)
.Select(async offset =>
{
var page = await wall.GetCommentsAsync(new WallGetCommentsParams()
{
Count = pageSize,
Offset = offset,
OwnerId = id,
PostId = postId,
});
log.Debug($"Post comments [{id} {postId}]: {page.Count}, offset {0}/{page.TotalCount}");
token.ThrowIfCancellationRequested();
return page;
});
var pages = await Task.WhenAll(pageTasks);
foreach (var page in pages)
{
AddToResult(page);
}
if ((int)total != result.Count)
{
log.Warn($"Expected {total} items, got {result.Count}. Maybe they were created/deleted, or it's VK bugs again.");
}
return result
.ToList();
}
public static async Task<IReadOnlyList<AudioPlaylist>> GetAllPlaylistsAsync(this IAudioCategory audio, long id, CancellationToken token, ILog log)
{
var pageSize = 200u;
var playlistResponse = await audio.GetPlaylistsAsync(id, pageSize);
//log.Debug($"Got {playlistResponse.Count}/{playlistResponse.TotalCount} tracks in playlist {id}");
log.Debug($"Playlists [{id}]: {playlistResponse.Count}, offset {0}/{playlistResponse.TotalCount}");
var total = playlistResponse.TotalCount;
var result = new HashSet<AudioPlaylist>((int) total);
void AddToResult(IEnumerable<AudioPlaylist> playlists)
{
foreach (var item in playlists)
{
if (!result.Add(item))
{
log.Warn($"Duplicate item [{item}]");
}
}
}
AddToResult(playlistResponse);
var remainingPages = total / pageSize;
var pageTasks = Enumerable.Range(1, (int) remainingPages)
.Select(pageNumber => (ulong) pageNumber * pageSize)
.Select(async offset =>
{
var page = await audio.GetPlaylistsAsync(id, pageSize, (uint) offset);
log.Debug($"Playlists [{id}]: {page.Count}, offset {offset}/{page.TotalCount}");
token.ThrowIfCancellationRequested();
return page;
});
var pages = await Task.WhenAll(pageTasks);
foreach (var page in pages)
{
AddToResult(page);
}
if ((int) total != result.Count)
{
log.Warn($"Expected {total} items, got {result.Count}. Maybe they were created/deleted, or it's VK bugs again.");
}
return result
.ToList();
}
public static async Task<PlaylistWithAudio> ExpandPlaylist(this IAudioCategory audio, AudioPlaylist playlist, CancellationToken token, ILog log)
{
var audios = await audio.GetAsync(new AudioGetParams
{
AlbumId = playlist.Id,
OwnerId = playlist.OwnerId,
});
log.Debug($"Expanded playlist {playlist.Title}: {audios.TotalCount} audios");
VkApiUtils.ThrowIfCountMismatch(audios.TotalCount, audios.Count);
return new PlaylistWithAudio(playlist, audios);
}
public static async Task<IReadOnlyList<Audio>> GetAllAudios(this IAudioCategory audio, long id, CancellationToken token, ILog log)
{
// TODO: handle paging if api returns partial result
var audios = await audio.GetAsync(new AudioGetParams()
{
OwnerId = id
});
log.Debug($"Audios [{id}]: {audios.Count}/{audios.TotalCount}");
VkApiUtils.ThrowIfCountMismatch(audios.TotalCount, audios.Count);
return audios;
}
}
}

24
VOffline/Services/VkNetHacks/CustomRestClient.cs
View file

@ -11,7 +11,6 @@ using Newtonsoft.Json;
using VkNet.Abstractions.Utils;
using VkNet.Utils;
using VOffline.Models;
using VOffline.Services.Vk;
namespace VOffline.Services.VkNetHacks
{
@ -55,16 +54,27 @@ namespace VOffline.Services.VkNetHacks
Query = string.Join("&", queries)
};
logger?.LogDebug($"GET request: {url.Uri}");
var request = new HttpRequestMessage(HttpMethod.Get, uri);
return CallWithRetry(httpClient => httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token));
Task<HttpResponseMessage> HttpFunc(HttpClient httpClient)
{
var request = new HttpRequestMessage(HttpMethod.Get, uri);
return httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
}
return CallWithRetry(HttpFunc);
}
public Task<HttpResponse<string>> PostAsync(Uri uri,IEnumerable<KeyValuePair<string, string>> parameters)
{
logger?.LogDebug($"POST request: {uri}{Environment.NewLine}{Utilities.PrettyPrintJson(JsonConvert.SerializeObject(parameters))}");
var content = new FormUrlEncodedContent(parameters);
var request = new HttpRequestMessage(HttpMethod.Post, uri) { Content = content };
return CallWithRetry(httpClient => httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token));
var parametersList = parameters.ToList();
logger?.LogDebug($"POST request: {uri}{Environment.NewLine}{Utilities.PrettyPrintJson(JsonConvert.SerializeObject(parametersList))}");
Task<HttpResponseMessage> HttpFunc(HttpClient httpClient)
{
var content = new FormUrlEncodedContent(parametersList);
var request = new HttpRequestMessage(HttpMethod.Post, uri) {Content = content};
return httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
}
return CallWithRetry(HttpFunc);
}
private async Task<HttpResponse<string>> CallWithRetry(Func<HttpClient, Task<HttpResponseMessage>> method)

6
VOffline/appsettings.json
View file

@ -2,8 +2,10 @@
"Settings": {
//"Targets": [ "id1", "durov", "public147415323", "tech" ],
//"Targets": [ "retrowave" ],
"Targets": ["rast1234", "retrowave"],
"Modes": [ "Audio", "Wall" ],
//"Targets": ["rast1234", "retrowave"],
//"Modes": [ "Audio", "Wall" ],
//"Modes": [ "Photos" ],
//"Modes": [ "All" ],
"OutputPath": "."
},
"VkCredentials": {