vrr/VOffline
2
0
Fork 0
mirror of https://github.com/Rast1234/VOffline.git synced 2026-04-28 03:49:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

rename and secrets

Browse source
This commit is contained in:
Rostislav Kirillov 2019-01-14 03:26:07 +05:00
parent 0a4178f1dd
commit 8522fde29f
39 changed files with 304 additions and 391 deletions
Download patch file Download diff file Expand all files Collapse all files

5
.gitignore vendored
View file

@ -331,4 +331,7 @@ ASALocalRun/
.mfractor/
# Local History for Visual Studio
.localhistory/
.localhistory/
**/FileCache.*.json
**logs/

7
VOffline/Models/Settings.cs
View file

@ -1,8 +1,6 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text.RegularExpressions;
namespace VOffline.Models
{
@ -28,5 +26,10 @@ namespace VOffline.Models
.ToList();
}
public string UserAgent { get; set; } = "KateMobileAndroid/51.2 lite-443 (Android 4.4.2; SDK 19; x86; unknown Android SDK built for x86; en)";
public int RequestRetryCount { get; set; } = 3;
public TimeSpan RequestRetryDelay { get; set; } = TimeSpan.FromSeconds(3);
public int DownloadQueueLimit { get; set; } = 100;
}
}

13
VOffline/Models/Storage/Download.cs
View file

@ -1,25 +1,14 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using RestSharp;
using VkNet;
using VOffline.Services;
using VOffline.Services.Handlers;
namespace VOffline.Models.Storage
{
public interface IDownload
{
DirectoryInfo Location { get; }
string DesiredName { get; }
int RetryCount { get; }
IReadOnlyList<Exception> Errors { get; }
void AddError(Exception e);
Task<byte[]> GetContent(CancellationToken token);
}
public class Download : IDownload
{
private readonly List<Exception> errors;

9
VOffline/Services/Extensions.cs → VOffline/Models/Storage/Extensions.cs
View file

@ -1,12 +1,7 @@
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using Nito.AsyncEx;
using System.Net;
using RestSharp;
namespace VOffline.Services.Handlers
namespace VOffline.Services
{
public static class Extensions
{

18
VOffline/Models/Storage/IDownload.cs Normal file
View file

@ -0,0 +1,18 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
namespace VOffline.Models.Storage
{
public interface IDownload
{
DirectoryInfo Location { get; }
string DesiredName { get; }
int RetryCount { get; }
IReadOnlyList<Exception> Errors { get; }
void AddError(Exception e);
Task<byte[]> GetContent(CancellationToken token);
}
}

2
VOffline/Services/Handlers/NetworkException.cs → VOffline/Models/Storage/NetworkException.cs
View file

@ -1,6 +1,6 @@
using System;
namespace VOffline.Services.Handlers
namespace VOffline.Services
{
public class NetworkException : ApplicationException
{

3
VOffline/Models/Storage/PlaylistWithAudio.cs
View file

@ -1,10 +1,7 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using VkNet.Model;
using VkNet.Model.Attachments;
using VOffline.Services.Storage;
namespace VOffline.Models.Storage
{

32
VOffline/Program.cs
View file

@ -8,7 +8,6 @@ using log4net;
using log4net.Config;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.FileSystemGlobbing.Internal;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Newtonsoft.Json;
@ -16,16 +15,15 @@ using Newtonsoft.Json.Converters;
using VkNet;
using VkNet.Abstractions.Core;
using VkNet.Abstractions.Utils;
using VkNet.Infrastructure;
using VkNet.Model;
using VkNet.Utils;
using VOffline.Models;
using VOffline.Models.Google;
using VOffline.Models.Vk;
using VOffline.Services;
using VOffline.Services.Google;
using VOffline.Services.Handlers;
using VOffline.Services.Storage;
using VOffline.Services.Token;
using VOffline.Services.Token.Google;
using VOffline.Services.Token.Vk;
using VOffline.Services.Vk;
using VOffline.Services.VkNetHacks;
@ -37,7 +35,7 @@ namespace VOffline
public static async Task<int> Main(string[] args)
{
var log = ConfigureLog4Net();
try
{
var cts = CreateCancellationTokenSource();
@ -47,10 +45,12 @@ namespace VOffline
var configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.AddUserSecrets<Settings>()
.AddUserSecrets<VkCredentials>()
.Build();
serviceCollection.Configure<Settings>(configuration.GetSection("Settings"));
serviceCollection.Configure<VkCredentials>(configuration.GetSection("VkCredentials"));
serviceCollection.AddSingleton<FileCache<VkToken>>();
serviceCollection.AddSingleton<FileCache<GoogleCredentials>>();
serviceCollection.AddSingleton<FileCache<GoogleCheckIn>>();
@ -58,14 +58,15 @@ namespace VOffline
serviceCollection.AddSingleton<GoogleHttpRequests>();
serviceCollection.AddSingleton<VkHttpRequests>();
serviceCollection.AddSingleton<VkApiUtils>();
serviceCollection.AddSingleton<ConstantsProvider>();
serviceCollection.AddSingleton<VkApi>(_ => CreateVkApi(cts, log));
serviceCollection.AddSingleton<VkApi>(s => CreateVkApi(s, cts, log));
serviceCollection.AddSingleton<FilesystemTools>();
serviceCollection.AddSingleton<DownloadQueueProvider>();
serviceCollection.AddSingleton<BackgroundDownloader>();
serviceCollection.AddSingleton<WallHandler>();
serviceCollection.AddSingleton<PostHandler>();
serviceCollection.AddSingleton<CommentsHandler>();
serviceCollection.AddSingleton<CommentHandler>();
serviceCollection.AddSingleton<AudioHandler>();
serviceCollection.AddSingleton<PlaylistHandler>();
serviceCollection.AddSingleton<AttachmentProcessor>();
@ -79,9 +80,13 @@ namespace VOffline
var services = serviceCollection.BuildServiceProvider();
await services.GetRequiredService<Logic>().Run(cts.Token, log);
return 0;
}
catch (TaskCanceledException e)
{
log.Warn($"Canceled by user");
return -2;
}
catch (Exception e)
{
log.Fatal(e);
@ -89,11 +94,11 @@ namespace VOffline
}
}
private static VkApi CreateVkApi(CancellationTokenSource cancellationTokenSource, ILog log)
private static VkApi CreateVkApi(IServiceProvider services, CancellationTokenSource cancellationTokenSource, ILog log)
{
// VkNet uses its own DI for internal services
var sc = new ServiceCollection();
sc.AddSingleton<ConstantsProvider>();
sc.AddSingleton<CancellationTokenSource>(cancellationTokenSource);
sc.AddSingleton<IRestClient, CustomRestClient>();
sc.AddSingleton<IAwaitableConstraint, CancellableConstraint>(_ => new CancellableConstraint(3, TimeSpan.FromSeconds(1), cancellationTokenSource.Token));
@ -107,6 +112,9 @@ namespace VOffline
//builder.Services.AddSingleton<ILoggerProvider>((ILoggerProvider)new Log4NetProvider(options));
builder.AddProvider(new SimpleLoggerProvider(log));
});
sc.AddTransient<Settings>(_ => services.GetRequiredService<IOptionsSnapshot<Settings>>().Value);
var vkApi = new VkApi(sc);
vkApi.VkApiVersion.SetVersion(5,90); // hack for linear comments without threads
return vkApi;

13
VOffline/Services/Handlers/AudioHandler.cs
View file

@ -1,24 +1,13 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using log4net;
using Newtonsoft.Json;
using Nito.AsyncEx;
using RestSharp;
using VkNet;
using VkNet.Model;
using VkNet.Model.Attachments;
using VkNet.Model.RequestParams;
using VkNet.Utils;
using VOffline.Models.Storage;
using VOffline.Services.Storage;
using VOffline.Services.Vk;
using VOffline.Services.VkNetHacks;
using RestClient = RestSharp.RestClient;
namespace VOffline.Services.Handlers
{

29
VOffline/Services/Handlers/CommentHandler.cs Normal file
View file

@ -0,0 +1,29 @@
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using log4net;
using VkNet.Model;
using VkNet.Model.Attachments;
using VOffline.Services.Storage;
namespace VOffline.Services.Handlers
{
public class CommentHandler : HandlerBase<Comment>
{
private readonly AttachmentProcessor attachmentProcessor;
public CommentHandler(FilesystemTools filesystemTools, AttachmentProcessor attachmentProcessor) : base(filesystemTools)
{
this.attachmentProcessor = attachmentProcessor;
}
public override async Task ProcessInternal(Comment comment, DirectoryInfo workDir, CancellationToken token, ILog log)
{
var attachmentTasks = comment.Attachments.Select((a, i) => attachmentProcessor.ProcessAttachment((Attachment) a, i, workDir, token, log));
await Task.WhenAll(attachmentTasks);
}
public override DirectoryInfo GetWorkingDirectory(Comment comment, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, $"{comment.Date.Value:s} {comment.Id}", CreateMode.MergeWithExisting);
}
}

42
VOffline/Services/Handlers/CommentsHandler.cs Normal file
View file

@ -0,0 +1,42 @@
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using log4net;
using VkNet;
using VkNet.Model.Attachments;
using VOffline.Models.Storage;
using VOffline.Services.Storage;
using VOffline.Services.Vk;
using VOffline.Services.VkNetHacks;
namespace VOffline.Services.Handlers
{
public class CommentsHandler : HandlerBase<Post>
{
private readonly VkApi vkApi;
private readonly CommentHandler commentHandler;
public CommentsHandler(VkApi vkApi, FilesystemTools filesystemTools, CommentHandler commentHandler) : base(filesystemTools)
{
this.vkApi = vkApi;
this.commentHandler = commentHandler;
}
public override async Task ProcessInternal(Post post, DirectoryInfo workDir, CancellationToken token, ILog log)
{
var allComments = await vkApi.Wall.GetAllCommentsAsync(post.OwnerId.Value, post.Id.Value, token, log);
log.Debug($"Post {post.Id} has {allComments.Count} comments");
var byDate = allComments
.OrderBy(c => c.Date)
.ToList();
await byDate.SaveHumanReadableText(filesystemTools, workDir, token, log);
var commentTasks = byDate
.Where(c => c.Attachments.Count > 0)
.Select(c => commentHandler.Process(c, workDir, token, log));
await Task.WhenAll(commentTasks);
}
public override DirectoryInfo GetWorkingDirectory(Post post, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, "comments", CreateMode.MergeWithExisting);
}
}

1
VOffline/Services/Handlers/PlaylistHandler.cs
View file

@ -3,7 +3,6 @@ using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using log4net;
using VkNet.Model.Attachments;
using VOffline.Models.Storage;
using VOffline.Services.Storage;

54
VOffline/Services/Handlers/PostHandler.cs
View file

@ -1,16 +1,10 @@
using System.Collections.Generic;
using System.IO;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using log4net;
using VkNet;
using VkNet.Model;
using VkNet.Model.Attachments;
using VkNet.Model.RequestParams;
using VOffline.Models.Storage;
using VOffline.Services.Storage;
using VOffline.Services.VkNetHacks;
namespace VOffline.Services.Handlers
{
@ -50,50 +44,4 @@ namespace VOffline.Services.Handlers
public override DirectoryInfo GetWorkingDirectory(Post post, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, $"{post.Date.Value:s} {post.Id}", CreateMode.MergeWithExisting);
}
public class CommentsHandler : HandlerBase<Post>
{
private readonly VkApi vkApi;
private readonly CommentHandler commentHandler;
public CommentsHandler(VkApi vkApi, FilesystemTools filesystemTools, CommentHandler commentHandler) : base(filesystemTools)
{
this.vkApi = vkApi;
this.commentHandler = commentHandler;
}
public override async Task ProcessInternal(Post post, DirectoryInfo workDir, CancellationToken token, ILog log)
{
var allComments = await vkApi.Wall.GetAllCommentsAsync(post.OwnerId.Value, post.Id.Value, token, log);
log.Debug($"Post {post.Id} has {allComments.Count} comments");
var byDate = allComments
.OrderBy(c => c.Date)
.ToList();
await byDate.SaveHumanReadableText(filesystemTools, workDir, token, log);
var commentTasks = byDate
.Where(c => c.Attachments.Count > 0)
.Select(c => commentHandler.Process(c, workDir, token, log));
await Task.WhenAll(commentTasks);
}
public override DirectoryInfo GetWorkingDirectory(Post post, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, "comments", CreateMode.MergeWithExisting);
}
public class CommentHandler : HandlerBase<Comment>
{
private readonly AttachmentProcessor attachmentProcessor;
public CommentHandler(FilesystemTools filesystemTools, AttachmentProcessor attachmentProcessor) : base(filesystemTools)
{
this.attachmentProcessor = attachmentProcessor;
}
public override async Task ProcessInternal(Comment comment, DirectoryInfo workDir, CancellationToken token, ILog log)
{
var attachmentTasks = comment.Attachments.Select((a, i) => attachmentProcessor.ProcessAttachment(a, i, workDir, token, log));
await Task.WhenAll(attachmentTasks);
}
public override DirectoryInfo GetWorkingDirectory(Comment comment, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, $"{comment.Date.Value:s} {comment.Id}", CreateMode.MergeWithExisting);
}
}

2
VOffline/Services/Handlers/WallHandler.cs
View file

@ -4,8 +4,6 @@ using System.Threading;
using System.Threading.Tasks;
using log4net;
using VkNet;
using VkNet.Model;
using VkNet.Model.RequestParams;
using VOffline.Services.Storage;
using VOffline.Services.Vk;
using VOffline.Services.VkNetHacks;

28
VOffline/Services/Logic.cs
View file

@ -1,5 +1,4 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Linq;
@ -10,11 +9,10 @@ using Microsoft.Extensions.Options;
using Newtonsoft.Json;
using VkNet;
using VkNet.Model;
using VkNet.Model.RequestParams;
using VOffline.Models;
using VOffline.Models.Storage;
using VOffline.Services.Handlers;
using VOffline.Services.Storage;
using VOffline.Services.Token;
using VOffline.Services.Vk;
namespace VOffline.Services
@ -58,26 +56,32 @@ namespace VOffline.Services
var modes = settings.GetWorkingModes();
var ids = settings.Targets
.Select(async x => await vkApiUtils.ResolveId(x))
.Select(x => x.Result)
var idTasks = settings.Targets
.Select(async x =>
{
var id = await vkApiUtils.ResolveId(x);
return (name: x, id: id);
});
var allIds = await Task.WhenAll(idTasks);
var identifiers = allIds
.Distinct()
.ToImmutableList();
log.Debug($"Processing {JsonConvert.SerializeObject(modes)} for {JsonConvert.SerializeObject(ids)}");
log.Debug($"Processing {JsonConvert.SerializeObject(modes)} for {string.Join(", ", identifiers.Select(x => $"[{x.name} {x.id}]"))}");
var downloaderTask = downloader.Process(token, log);
var rootDir = filesystemTools.MkDir(settings.OutputPath);
foreach (var id in ids)
foreach (var identifier in identifiers)
{
var name = await vkApiUtils.GetName(id);
var name = await vkApiUtils.GetName(identifier.id);
var workDir = filesystemTools.CreateSubdir(rootDir, name, CreateMode.MergeWithExisting);
log.Info($"id [{id}], name [{name}], path [{workDir.FullName}]");
log.Info($"id [{identifier.id}], name [{name}], path [{workDir.FullName}]");
foreach (var mode in modes)
{
await ProcessTarget(id, workDir, mode, token, log);
await ProcessTarget(identifier.id, workDir, mode, token, log);
}
}
queueProvider.Pending.CompleteAdding();
var downloadErrors = await downloaderTask;
foreach (var downloadError in downloadErrors)

36
VOffline/Services/Retrier.cs
View file

@ -1,36 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using log4net;
namespace VOffline.Services
{
public class Retrier
{
public static async Task<T> Retry<T>(Func<Task<T>> taskFactory, int limit, TimeSpan delay, CancellationToken token, ILog log)
{
var exceptions = new List<Exception>();
for (var i = 1; i <= limit; i++)
{
token.ThrowIfCancellationRequested();
try
{
return await taskFactory();
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e)
{
exceptions.Add(e);
}
log.Warn($"Attempt {i}/{limit}, last error was [{exceptions.LastOrDefault()?.GetType()}]");
await Task.Delay(delay, token);
}
throw new AggregateException(exceptions);
}
}
}

7
VOffline/Services/Handlers/AttachmentProcessor.cs → VOffline/Services/Storage/AttachmentProcessor.cs
View file

@ -1,5 +1,4 @@
using System;
using System.IO;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using log4net;
@ -7,10 +6,10 @@ using Newtonsoft.Json;
using VkNet;
using VkNet.Model;
using VkNet.Model.Attachments;
using VOffline.Models.Storage;
using VOffline.Services.Storage;
using VOffline.Services.Vk;
namespace VOffline.Services.Handlers
namespace VOffline.Services
{
public class AttachmentProcessor
{

4
VOffline/Services/Storage/BackgroundDownloader.cs
View file

@ -1,15 +1,11 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using log4net;
using Nito.AsyncEx;
using RestSharp;
using VkNet;
using VOffline.Models.Storage;
using VOffline.Services.Handlers;
namespace VOffline.Services.Storage
{

9
VOffline/Services/Storage/CreateMode.cs Normal file
View file

@ -0,0 +1,9 @@
namespace VOffline.Services.Storage
{
public enum CreateMode
{
AutoRenameCollisions,
ThrowIfExists,
MergeWithExisting
}
}

6
VOffline/Services/Storage/DownloadQueueProvider.cs
View file

@ -2,7 +2,9 @@
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Options;
using Nito.AsyncEx;
using VOffline.Models;
using VOffline.Models.Storage;
using VOffline.Services.Vk;
@ -10,9 +12,9 @@ namespace VOffline.Services.Storage
{
public class DownloadQueueProvider
{
public DownloadQueueProvider(ConstantsProvider constantsProvider)
public DownloadQueueProvider(IOptionsSnapshot<Settings> settings)
{
Pending = new AsyncProducerConsumerQueue<IDownload>(constantsProvider.DownloadQueueLimit);
Pending = new AsyncProducerConsumerQueue<IDownload>(settings.Value.DownloadQueueLimit);
}
public async Task EnqueueAll(IEnumerable<IDownload> items, CancellationToken token)

9
VOffline/Services/Storage/FilesystemTools.cs
View file

@ -87,8 +87,6 @@ namespace VOffline.Services.Storage
}
}
public async Task WriteFileWithCompletionMark(DirectoryInfo parent, string desiredName, Func<Task<string>> contentTaskFunc, CancellationToken token, ILog log)
{
var validName = MakeValidName(desiredName);
@ -234,11 +232,4 @@ namespace VOffline.Services.Storage
private static readonly string CompletedFilename = MakeValidName(".done.voffline");
}
public enum CreateMode
{
AutoRenameCollisions,
ThrowIfExists,
MergeWithExisting
}
}

51
VOffline/Services/Throttler.cs
View file

@ -1,51 +0,0 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace VOffline.Services
{
public class Throttler
{
public async Task<IReadOnlyList<T>> ProcessWithThrottling<T>(Task<T>[] tasks, int limit, CancellationToken token)
{
using (var semaphore = new SemaphoreSlim(limit))
{
var newTasks = tasks.Select(async t =>
{
await semaphore.WaitAsync(token);
try
{
var data = await t;
return data;
}
finally
{
semaphore.Release();
}
});
return await Task.WhenAll(newTasks);
}
}
public async Task ProcessWithThrottling(Task[] tasks, int limit, CancellationToken token)
{
using (var semaphore = new SemaphoreSlim(limit))
{
var newTasks = tasks.Select(async t =>
{
await semaphore.WaitAsync(token);
try
{
await t;
}
finally
{
semaphore.Release();
}
});
await Task.WhenAll(newTasks);
}
}
}
}

2
VOffline/Services/Token/Google/AndroidAuth.cs
View file

@ -1,7 +1,7 @@
using log4net;
using VOffline.Models.Google;
namespace VOffline.Services.Google
namespace VOffline.Services.Token.Google
{
public class AndroidAuth
{

2
VOffline/Services/Token/Google/GoogleHttpRequests.cs
View file

@ -5,7 +5,7 @@ using log4net;
using RestSharp;
using VOffline.Models.Google;
namespace VOffline.Services.Google
namespace VOffline.Services.Token.Google
{
public class GoogleHttpRequests
{

2
VOffline/Services/Token/Google/MTalk.cs
View file

@ -6,7 +6,7 @@ using System.Text;
using log4net;
using VOffline.Models.Google;
namespace VOffline.Services.Google
namespace VOffline.Services.Token.Google
{
public class MTalk
{

2
VOffline/Services/Token/Google/ProtobufParser.cs
View file

@ -3,7 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using VOffline.Models.Google;
namespace VOffline.Services.Google
namespace VOffline.Services.Token.Google
{
public class ProtobufParser
{

2
VOffline/Services/Token/Google/VarInt.cs
View file

@ -1,7 +1,7 @@
using System;
using System.Collections.Generic;
namespace VOffline.Services.Google
namespace VOffline.Services.Token.Google
{
public static class VarInt
{

6
VOffline/Services/Token/TokenMagic.cs
View file

@ -1,9 +1,9 @@
using log4net;
using VOffline.Models.Vk;
using VOffline.Services.Google;
using VOffline.Services.Vk;
using VOffline.Services.Token.Google;
using VOffline.Services.Token.Vk;
namespace VOffline.Services
namespace VOffline.Services.Token
{
public class TokenMagic
{

7
VOffline/Services/Token/Vk/VkHttpRequests.cs
View file

@ -3,19 +3,20 @@ using log4net;
using Microsoft.Extensions.Options;
using Newtonsoft.Json;
using RestSharp;
using VOffline.Models;
using VOffline.Models.Vk;
using RestClient = RestSharp.RestClient;
namespace VOffline.Services.Vk
namespace VOffline.Services.Token.Vk
{
public class VkHttpRequests
{
private readonly string userAgent;
private VkCredentials VkCredentials { get; }
public VkHttpRequests(IOptionsSnapshot<VkCredentials> vkCredentials, ConstantsProvider constantsProvider)
public VkHttpRequests(IOptionsSnapshot<VkCredentials> vkCredentials, IOptionsSnapshot<Settings> settings)
{
userAgent = constantsProvider.UserAgent;
userAgent = settings.Value.UserAgent;
VkCredentials = vkCredentials.Value;
}

4
VOffline/Services/Token/Vk/VkTokenReceiver.cs
View file

@ -1,8 +1,8 @@
using log4net;
using VOffline.Models.Google;
using VOffline.Services.Google;
using VOffline.Services.Token.Google;
namespace VOffline.Services.Vk
namespace VOffline.Services.Token.Vk
{
public class VkTokenReceiver
{

15
VOffline/Services/Utils.cs
View file

@ -1,15 +0,0 @@
using System;
namespace VOffline.Services
{
public static class Utils
{
public static void ThrowIfCountMismatch(decimal expectedTotal, decimal resultCount)
{
if (resultCount != expectedTotal)
{
throw new InvalidOperationException($"Expected {expectedTotal} items, got {resultCount}. Maybe they were created/deleted, try again.");
}
}
}
}

4
VOffline/Models/Storage/AttachmentExtensions.cs → VOffline/Services/Vk/AttachmentExtensions.cs
View file

@ -10,9 +10,10 @@ using Newtonsoft.Json;
using VkNet;
using VkNet.Model;
using VkNet.Model.Attachments;
using VOffline.Models.Storage;
using VOffline.Services.Storage;
namespace VOffline.Models.Storage
namespace VOffline.Services.Vk
{
public static class AttachmentExtensions
{
@ -169,7 +170,6 @@ namespace VOffline.Models.Storage
public static string GetName(this Poll poll) => $"poll {poll.Id}";
public static async Task SaveText(this Photo photo, int i, FilesystemTools filesystemTools, DirectoryInfo dir, CancellationToken token, ILog log)
{
if (!string.IsNullOrWhiteSpace(photo.Text))

44
VOffline/Services/Vk/VkApiUtils.cs
View file

@ -1,20 +1,9 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography.X509Certificates;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using log4net;
using Newtonsoft.Json;
using VkNet;
using VkNet.Enums;
using VkNet.Enums.Filters;
using VkNet.Model;
using VkNet.Model.Attachments;
using VkNet.Model.RequestParams;
using VOffline.Models.Storage;
namespace VOffline.Services.Vk
{
@ -33,21 +22,21 @@ namespace VOffline.Services.Vk
var communityMatch = CommunityPattern.Match(target);
if (communityMatch.Success)
{
return -1 * long.Parse(communityMatch.Groups[2].Value);
return -1 * Int64.Parse(communityMatch.Groups[2].Value);
}
// any user eg. id123
var personalMatch = PersonalPattern.Match(target);
if (personalMatch.Success)
{
return long.Parse(personalMatch.Groups[2].Value);
return Int64.Parse(personalMatch.Groups[2].Value);
}
// any id eg. 123 or -123
var digitalMatch = DigitalPattern.Match(target);
if (digitalMatch.Success)
{
return long.Parse(digitalMatch.Groups[1].Value);
return Int64.Parse(digitalMatch.Groups[1].Value);
}
// any screen name
@ -69,18 +58,18 @@ namespace VOffline.Services.Vk
{
var users = await vkApi.Users.GetAsync(new []{id}, ProfileFields.All);
var user = users[0];
return string.Join(" ", user.LastName, user.FirstName, GroupOrEmpty(" - ", user.Id.ToString(), user.ScreenName, user.Domain));
return String.Join(" ", user.LastName, user.FirstName, GroupOrEmpty(" - ", user.Id.ToString(), user.ScreenName, user.Domain));
}
var groups = await vkApi.Groups.GetByIdAsync(null, (-1*id).ToString(), GroupsFields.All);
var group = groups[0];
return string.Join(" ", group.Name, GroupOrEmpty(" - ", group.Id.ToString(), group.ScreenName, group.Type.ToString()));
return String.Join(" ", group.Name, GroupOrEmpty(" - ", group.Id.ToString(), group.ScreenName, group.Type.ToString()));
}
private static string GroupOrEmpty(string separator, params string[] parts)
{
var all = string.Join(separator, parts);
return string.IsNullOrEmpty(all)
? string.Empty
var all = String.Join(separator, parts);
return String.IsNullOrEmpty(all)
? String.Empty
: $"({all})";
}
@ -88,15 +77,12 @@ namespace VOffline.Services.Vk
private static readonly Regex PersonalPattern = new Regex(@"^(id)(\d+)$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
private static readonly Regex DigitalPattern = new Regex(@"^(-?\d+)$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
}
public class ConstantsProvider
{
public readonly string UserAgent = "KateMobileAndroid/51.2 lite-443 (Android 4.4.2; SDK 19; x86; unknown Android SDK built for x86; en)";
public readonly int RequestRetryCount = 3;
public readonly TimeSpan RequestRetryDelay = TimeSpan.FromSeconds(3);
public readonly int DownloadQueueLimit = 100;
public static void ThrowIfCountMismatch(decimal expectedTotal, decimal resultCount)
{
if (resultCount != expectedTotal)
{
throw new InvalidOperationException($"Expected {expectedTotal} items, got {resultCount}. Maybe they were created/deleted, try again.");
}
}
}
}

10
VOffline/Services/VkNetHacks/Class1.cs → VOffline/Services/Vk/VkNetExtensions.cs
View file

@ -1,7 +1,5 @@
using System;
using System.Collections.Generic;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using log4net;
@ -11,7 +9,7 @@ using VkNet.Model.Attachments;
using VkNet.Model.RequestParams;
using VOffline.Models.Storage;
namespace VOffline.Services.VkNetHacks
namespace VOffline.Services.Vk
{
public static class VkNetExtensions
{
@ -187,7 +185,7 @@ namespace VOffline.Services.VkNetHacks
OwnerId = playlist.OwnerId,
});
log.Debug($"Expanded playlist {playlist.Title}: {audios.TotalCount} audios");
Utils.ThrowIfCountMismatch(audios.TotalCount, audios.Count);
VkApiUtils.ThrowIfCountMismatch(audios.TotalCount, audios.Count);
return new PlaylistWithAudio(playlist, audios);
}
@ -199,7 +197,7 @@ namespace VOffline.Services.VkNetHacks
OwnerId = id
});
log.Debug($"Audios [{id}]: {audios.Count}/{audios.TotalCount}");
Utils.ThrowIfCountMismatch(audios.TotalCount, audios.Count);
VkApiUtils.ThrowIfCountMismatch(audios.TotalCount, audios.Count);
return audios;
}
}

10
VOffline/Services/VkNetHacks/CustomRestClient.cs
View file

@ -6,9 +6,11 @@ using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Newtonsoft.Json;
using VkNet.Abstractions.Utils;
using VkNet.Utils;
using VOffline.Models;
using VOffline.Services.Vk;
namespace VOffline.Services.VkNetHacks
@ -25,14 +27,14 @@ namespace VOffline.Services.VkNetHacks
private readonly int retryMaxCount;
private readonly TimeSpan retryDelay;
public CustomRestClient(ILogger<RestClient> logger, IWebProxy proxy, ConstantsProvider constantsProvider, CancellationTokenSource cancellationTokenSource)
public CustomRestClient(ILogger<RestClient> logger, IWebProxy proxy, IOptionsSnapshot<Settings> settings, CancellationTokenSource cancellationTokenSource)
{
this.logger = logger;
this.token = cancellationTokenSource.Token;
Proxy = proxy;
userAgent = constantsProvider.UserAgent;
retryMaxCount = constantsProvider.RequestRetryCount;
retryDelay = constantsProvider.RequestRetryDelay;
userAgent = settings.Value.UserAgent;
retryMaxCount = settings.Value.RequestRetryCount;
retryDelay = settings.Value.RequestRetryDelay;
}
public IWebProxy Proxy { get; set; }

99
VOffline/Services/VkNetHacks/LimitedSizeStack.cs
View file

@ -1,9 +1,4 @@
using System;
using System.Collections.Generic;
using log4net;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Log4Net.AspNetCore.Extensions;
using Microsoft.Extensions.Logging.Scope;
using System.Collections.Generic;
namespace VOffline.Services.VkNetHacks
{
@ -24,96 +19,4 @@ namespace VOffline.Services.VkNetHacks
this.RemoveLast();
}
}
public class SimpleLoggerProvider : ILoggerProvider
{
private readonly SimpleLogger logger;
public SimpleLoggerProvider(ILog log)
{
logger = new SimpleLogger(log);
}
public void Dispose()
{
}
public ILogger CreateLogger(string categoryName)
{
return logger;
}
}
public class SimpleLogger : ILogger
{
private readonly ILog log;
public SimpleLogger(ILog log)
{
this.log = log;
}
public string Name => log.Logger.Name;
public IDisposable BeginScope<TState>(TState state)
{
return null;
}
public bool IsEnabled(LogLevel logLevel)
{
switch (logLevel)
{
case LogLevel.Trace:
case LogLevel.Debug:
return this.log.IsDebugEnabled;
case LogLevel.Information:
return this.log.IsInfoEnabled;
case LogLevel.Warning:
return this.log.IsWarnEnabled;
case LogLevel.Error:
return this.log.IsErrorEnabled;
case LogLevel.Critical:
return this.log.IsFatalEnabled;
default:
throw new ArgumentOutOfRangeException(nameof(logLevel));
}
}
public void Log<TState>(LogLevel logLevel,EventId eventId,TState state,Exception exception,Func<TState, Exception, string> formatter)
{
if (!this.IsEnabled(logLevel))
return;
if (formatter == null)
throw new ArgumentNullException(nameof(formatter));
string str = formatter(state, exception);
if (string.IsNullOrEmpty(str) && exception == null)
return;
switch (logLevel)
{
case LogLevel.Trace:
this.log.Trace((object)str, exception);
break;
case LogLevel.Debug:
this.log.Debug((object)str, exception);
break;
case LogLevel.Information:
this.log.Info((object)str, exception);
break;
case LogLevel.Warning:
this.log.Warn((object)str, exception);
break;
case LogLevel.Error:
this.log.Error((object)str, exception);
break;
case LogLevel.Critical:
this.log.Fatal((object)str, exception);
break;
default:
this.log.Warn((object)string.Format("Encountered unknown log level {0}, writing out as Info.", (object)logLevel));
this.log.Info((object)str, exception);
break;
}
}
}
}

80
VOffline/Services/VkNetHacks/SimpleLogger.cs Normal file
View file

@ -0,0 +1,80 @@
using System;
using log4net;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Log4Net.AspNetCore.Extensions;
namespace VOffline.Services.VkNetHacks
{
public class SimpleLogger : ILogger
{
private readonly ILog log;
public SimpleLogger(ILog log)
{
this.log = log;
}
public string Name => log.Logger.Name;
public IDisposable BeginScope<TState>(TState state)
{
return null;
}
public bool IsEnabled(LogLevel logLevel)
{
switch (logLevel)
{
case LogLevel.Trace:
case LogLevel.Debug:
return this.log.IsDebugEnabled;
case LogLevel.Information:
return this.log.IsInfoEnabled;
case LogLevel.Warning:
return this.log.IsWarnEnabled;
case LogLevel.Error:
return this.log.IsErrorEnabled;
case LogLevel.Critical:
return this.log.IsFatalEnabled;
default:
throw new ArgumentOutOfRangeException(nameof(logLevel));
}
}
public void Log<TState>(LogLevel logLevel,EventId eventId,TState state,Exception exception,Func<TState, Exception, string> formatter)
{
if (!this.IsEnabled(logLevel))
return;
if (formatter == null)
throw new ArgumentNullException(nameof(formatter));
string str = formatter(state, exception);
if (string.IsNullOrEmpty(str) && exception == null)
return;
switch (logLevel)
{
case LogLevel.Trace:
this.log.Trace((object)str, exception);
break;
case LogLevel.Debug:
this.log.Debug((object)str, exception);
break;
case LogLevel.Information:
this.log.Info((object)str, exception);
break;
case LogLevel.Warning:
this.log.Warn((object)str, exception);
break;
case LogLevel.Error:
this.log.Error((object)str, exception);
break;
case LogLevel.Critical:
this.log.Fatal((object)str, exception);
break;
default:
this.log.Warn((object)string.Format("Encountered unknown log level {0}, writing out as Info.", (object)logLevel));
this.log.Info((object)str, exception);
break;
}
}
}
}

24
VOffline/Services/VkNetHacks/SimpleLoggerProvider.cs Normal file
View file

@ -0,0 +1,24 @@
using log4net;
using Microsoft.Extensions.Logging;
namespace VOffline.Services.VkNetHacks
{
public class SimpleLoggerProvider : ILoggerProvider
{
private readonly SimpleLogger logger;
public SimpleLoggerProvider(ILog log)
{
logger = new SimpleLogger(log);
}
public void Dispose()
{
}
public ILogger CreateLogger(string categoryName)
{
return logger;
}
}
}

2
VOffline/VOffline.csproj
View file

@ -3,6 +3,7 @@
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp2.1</TargetFramework>
<UsersecretsId>26938AF3-7336-429C-A3AF-2D3048ACB226</UsersecretsId>
<LangVersion>latest</LangVersion>
</PropertyGroup>
@ -25,6 +26,7 @@
<PackageReference Include="Microsoft.Extensions.Configuration" Version="2.2.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.2.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.2.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="2.2.0" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="2.2.0" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="2.2.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Log4Net.AspNetCore" Version="2.2.9" />