vrr/VOffline
2
0
Fork 0
mirror of https://github.com/Rast1234/VOffline.git synced 2026-04-28 03:49:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

audio, playlists, texts work. downloader needs polishing

Browse source
This commit is contained in:
Rostislav 2019-01-10 00:28:18 +05:00
parent f0b5fa92d2
commit a602981f71
26 changed files with 1050 additions and 59 deletions
Download patch file Download diff file Expand all files Collapse all files

12
VOffline/Models/Mode.cs Normal file
View file

@ -0,0 +1,12 @@
namespace VOffline.Models
{
public enum Mode
{
Wall,
Audio,
Photos,
Video,
Docs,
All
}
}

30
VOffline/Models/Settings.cs Normal file
View file

@ -0,0 +1,30 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text.RegularExpressions;
namespace VOffline.Models
{
public class Settings
{
public List<string> Targets { get; set; }
public List<Mode> Modes { get; set; }
public ImmutableHashSet<Mode> GetWorkingModes()
{
var unique = Modes.ToImmutableHashSet();
if (!unique.Contains(Mode.All))
{
return unique;
}
var goodEnumValues = Enum.GetValues(typeof(Mode))
.Cast<Mode>()
.Except(Enumerable.Repeat(Mode.All, 1));
return goodEnumValues.ToImmutableHashSet();
}
}
}

42
VOffline/Models/Storage/Playlist.cs Normal file
View file

@ -0,0 +1,42 @@
using System;
using System.Collections.Generic;
using VkNet.Model;
namespace VOffline.Models.Storage
{
public class Playlist
{
public long Id { get; set; }
public string Name { get; set; }
public string Description { get; set; }
public Uri Cover { get; set; }
public Playlist(AudioPlaylist playlist, IReadOnlyList<Track> tracks)
{
Id = playlist.Id;
Name = playlist.Title;
Description = playlist.Description;
Cover = GetBestImage(playlist.Cover);
Tracks = tracks;
}
public Playlist(IReadOnlyList<Track> tracks)
{
Id = -1;
Name = "__default";
Description = "Tracks without playlist";
Cover = null;
Tracks = tracks;
}
public IReadOnlyList<Track> Tracks { get; set; }
private static Uri GetBestImage(AudioCover cover)
{
// TODO: theoretically images could be different or missing, but looks like this works fine
return string.IsNullOrEmpty(cover?.Photo600)
? null
: new Uri(cover?.Photo600);
}
}
}

26
VOffline/Models/Storage/Track.cs Normal file
View file

@ -0,0 +1,26 @@
using System;
using VkNet.Model;
using VkNet.Model.Attachments;
namespace VOffline.Models.Storage
{
public class Track
{
public long Id { get; set; }
public string Name { get; set; }
public string Artist { get; set; }
public Uri Url { get; set; }
public bool? Hq { get; set; }
public string Lyrics { get; set; }
public Track(Audio audio, Lyrics lyrics)
{
Id = audio.Id.Value;
Name = audio.Title;
Artist = audio.Artist;
Url = audio.Url;
Hq = audio.IsHq;
Lyrics = lyrics?.Text;
}
}
}

78
VOffline/Program.cs
View file

@ -1,51 +1,117 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
using log4net;
using log4net.Config;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using VkNet;
using VkNet.Abstractions.Core;
using VkNet.Abstractions.Utils;
using VkNet.Model;
using VkNet.Utils;
using VOffline.Models;
using VOffline.Models.Google;
using VOffline.Models.Vk;
using VOffline.Services;
using VOffline.Services.Google;
using VOffline.Services.Handlers;
using VOffline.Services.Vk;
namespace VOffline
{
public class Program
{
static void Main(string[] args)
public static async Task<int> Main(string[] args)
{
var log = ConfigureLog4Net();
try
{
var token = CreateCancellationToken();
ConfigureJsonSerializer();
var serviceCollection = new ServiceCollection();
serviceCollection.AddTransient(provider => LogManager.GetLogger(Assembly.GetEntryAssembly(), typeof(Program)));
serviceCollection.AddSingleton<FileCache<VkCredentials>>();
var configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.Build();
serviceCollection.Configure<Settings>(configuration.GetSection("Settings"));
serviceCollection.Configure<VkCredentials>(configuration.GetSection("VkCredentials"));
serviceCollection.AddSingleton<FileCache<VkToken>>();
serviceCollection.AddSingleton<FileCache<GoogleCredentials>>();
serviceCollection.AddSingleton<FileCache<GoogleCheckIn>>();
serviceCollection.AddSingleton<Random>();
serviceCollection.AddSingleton<GoogleHttpRequests>();
serviceCollection.AddSingleton<VkHttpRequests>();
serviceCollection.AddSingleton<VkApiUtils>();
serviceCollection.AddSingleton<UserAgentProvider>();
serviceCollection.AddSingleton<VkApi>(_ => VkApiFactory(token));
serviceCollection.AddTransient(provider => LogManager.GetLogger(Assembly.GetEntryAssembly(), typeof(Program)));
serviceCollection.AddTransient<AndroidAuth>();
serviceCollection.AddTransient<VkTokenReceiver>();
serviceCollection.AddTransient<MTalk>();
serviceCollection.AddTransient<TokenMagic>();
serviceCollection.AddTransient<Logic>();
serviceCollection.AddTransient<AudioHandler>();
var services = serviceCollection.BuildServiceProvider();
services.GetRequiredService<Logic>().Run(log);
await services.GetRequiredService<Logic>().Run(token, log);
return 0;
}
catch (Exception e)
{
log.Fatal(e);
return -1;
}
Console.ReadLine();
}
private static VkApi VkApiFactory(CancellationToken token)
{
// VkNet uses its own DI for internal services
var sc = new ServiceCollection();
sc.AddSingleton<UserAgentProvider>();
sc.AddSingleton<IRestClient, RestClientWithUserAgent>();
sc.AddSingleton<IAwaitableConstraint, CancellableConstraint>(_ => new CancellableConstraint(3, TimeSpan.FromSeconds(1.3), token));
sc.AddSingleton<ILoggerFactory, LoggerFactory>();
sc.AddSingleton(typeof(ILogger<>), typeof(Logger<>));
sc.AddLogging(builder =>
{
builder.ClearProviders();
builder.SetMinimumLevel(LogLevel.Debug);
builder.AddLog4Net();
});
return new VkApi(sc);
}
private static void ConfigureJsonSerializer()
{
JsonConvert.DefaultSettings = () => new JsonSerializerSettings()
{
Converters = new List<JsonConverter>() {new StringEnumConverter()}
};
}
private static CancellationToken CreateCancellationToken()
{
var cts = new CancellationTokenSource();
Console.CancelKeyPress += (sender, a) =>
{
a.Cancel = true;
Console.WriteLine("Ctrl-C pressed, trying to stop gracefully...");
cts.Cancel(true);
};
return cts.Token;
}
private static ILog ConfigureLog4Net()

117
VOffline/Services/Filesystem/Storage.cs Normal file
View file

@ -0,0 +1,117 @@
using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using VkNet.Model.RequestParams;
namespace VOffline.Services
{
public class Storage
{
private readonly DirectoryInfo rootDir;
public string FullPath => rootDir.FullName;
public Storage(string rootName)
{
rootDir = new DirectoryInfo(FilterFilename(rootName));
lock (LockObject)
{
if (rootDir.Exists)
{
throw new InvalidOperationException($"Storage directory [{rootDir.Name}] already exists");
}
rootDir.Create();
}
}
private Storage(string rootName, bool useExisting)
{
rootDir = new DirectoryInfo(rootName);
lock (LockObject)
{
if (!useExisting && rootDir.Exists)
{
throw new InvalidOperationException($"Storage directory [{rootDir.Name}] already exists");
}
rootDir.Create();
}
}
public Storage Descend(string subName, bool makeUniqueName)
{
var filteredName = FilterFilename(subName);
if (Path.IsPathRooted(filteredName))
{
throw new InvalidOperationException($"Expected relative path, got absolute: [{filteredName}]");
}
// TODO: looks like cancer. also probable race conditions?
if (makeUniqueName)
{
var uniqueName = GetUniqueDirectory(filteredName);
return new Storage(Path.Combine(rootDir.FullName, uniqueName.Name), true);
}
return new Storage(Path.Combine(rootDir.FullName, filteredName), false);
}
/// <summary>
/// Creates unique file
/// </summary>
/// <param name="name"></param>
/// <returns></returns>
public FileInfo GetFile(string name)
{
var filename = FilterFilename(name);
var file = GetUniqueFile(filename);
return file;
}
private FileInfo GetUniqueFile(string filteredName)
{
var name = Path.GetFileNameWithoutExtension(filteredName);
var extension = Path.GetExtension(filteredName);
var fileInfo = new FileInfo(Path.Combine(rootDir.FullName, filteredName));
for (var i = 1;; i++)
{
lock (LockObject)
{
if (!fileInfo.Exists)
{
fileInfo.Create().Close();
return fileInfo;
}
}
fileInfo = new FileInfo(Path.Combine(rootDir.FullName, $"{name} ({i}){extension}"));
}
}
private DirectoryInfo GetUniqueDirectory(string filteredName)
{
var directoryInfo = new DirectoryInfo(Path.Combine(rootDir.FullName, filteredName));
for (var i = 1; ; i++)
{
lock (LockObject)
{
if (!directoryInfo.Exists)
{
directoryInfo.Create();
return directoryInfo;
}
}
directoryInfo = new DirectoryInfo(Path.Combine(rootDir.FullName, $"{filteredName} ({i})"));
}
}
private static string FilterFilename(string value) => string.Join("_", value.Split(AllBadChars)).Trim();
private static readonly char[] AllBadChars =
Path.GetInvalidFileNameChars()
.Concat(Path.GetInvalidPathChars())
.Concat(new[] {'\\', '/', ':'})
.Distinct()
.ToArray();
private static readonly object LockObject = new object();
}
}

192
VOffline/Services/Handlers/Audio.cs Normal file
View file

@ -0,0 +1,192 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using log4net;
using Newtonsoft.Json;
using RestSharp;
using VkNet;
using VkNet.Model;
using VkNet.Model.Attachments;
using VkNet.Model.RequestParams;
using VkNet.Utils;
using VOffline.Models.Storage;
using VOffline.Services.Vk;
using RestClient = RestSharp.RestClient;
namespace VOffline.Services.Handlers
{
public class AudioHandler
{
private readonly VkApi vkApi;
public AudioHandler(VkApi vkApi)
{
this.vkApi = vkApi;
}
public async Task ProcessAudio(long id, Storage storage, CancellationToken token, ILog log)
{
var allPlaylists = await GetAllPlaylists(id, token, log);
var throttler = new Throttler();
foreach (var playlist in allPlaylists)
{
var playlistStorage = storage.Descend(playlist.Name, true);
var downloadTasks = playlist.Tracks
.Select(async track => await Download(track, playlistStorage, token, log))
.ToArray();
await throttler.ProcessWithThrottling(downloadTasks, 3, token);
}
}
private async Task Download(Track track, Storage playlistStorage, CancellationToken token, ILog log)
{
return;
var trackName = string.Join(" - ", new[] { track.Artist, track.Name }.Where(x => !string.IsNullOrEmpty(x)));
if (track.Url != null)
{
var content = await Retrier.Retry(async () =>
{
var client = new RestClient($"{track.Url.Scheme}://{track.Url.Authority}");
var response = await client.ExecuteGetTaskAsync(new RestRequest(track.Url.PathAndQuery), token);
response.ThrowIfSomethingWrong();
return response.RawBytes;
}, 3, TimeSpan.FromSeconds(5), token, log);
var file = playlistStorage.GetFile($"{trackName}.mp3").FullName;
log.Debug($"Saving {file}");
await File.WriteAllBytesAsync(file, content, token);
}
else
{
playlistStorage.GetFile($"{trackName}.mp3.deleted");
}
if (track.Lyrics != null)
{
var lyricsFile = playlistStorage.GetFile($"{trackName}.txt").FullName;
await File.WriteAllTextAsync(lyricsFile, track.Lyrics, token);
}
}
private async Task<IReadOnlyList<Playlist>> GetAllPlaylists(long id, CancellationToken token, ILog log)
{
var playlists = await GetPlaylists(id, token, log);
token.ThrowIfCancellationRequested();
var tracksInPlaylists = playlists
.SelectMany(p => p.Tracks.Select(t => t.Id))
.ToHashSet();
var uniqueTracks = await GetTracks(id, tracksInPlaylists, token, log);
log.Info($"Found {uniqueTracks.Count} tracks and {playlists.Sum(x => x.Tracks.Count)} in {playlists.Count} playlists");
var defaultPlaylist = new Playlist(uniqueTracks);
var allPlaylists = playlists.ToList();
allPlaylists.Add(defaultPlaylist);
return allPlaylists;
}
public async Task<IReadOnlyList<Playlist>> GetPlaylists(long id, CancellationToken token, ILog log)
{
var vkPlaylists = await GetAllVkPlaylists(id, token, log);
var playlistTasks = vkPlaylists.Select(async playlist =>
{
var vkAudios = await ExpandPlaylist(playlist, log);
var audioTasks = vkAudios.Select(async audio =>
{
token.ThrowIfCancellationRequested();
var lyrics = await GetLyrics(audio, log);
return new Track(audio, lyrics);
});
var tracks = await Task.WhenAll(audioTasks);
return new Playlist(playlist, tracks);
});
return await Task.WhenAll(playlistTasks);
}
public async Task<IReadOnlyList<Track>> GetTracks(long id, HashSet<long> tracksInPlaylists, CancellationToken token, ILog log)
{
// TODO: handle paging if api returns partial result
log.Debug("Getting all tracks not in playlists");
var vkAudios = await vkApi.Audio.GetAsync(new AudioGetParams()
{
OwnerId = id
});
ThrowIfCountMismatch(vkAudios.TotalCount, vkAudios.Count);
var trackTasks = vkAudios
.Where(t => !tracksInPlaylists.Contains(t.Id.Value))
.Select(async audio =>
{
// TODO: filter out extra tracks here
var lyrics = await GetLyrics(audio, log);
token.ThrowIfCancellationRequested();
return new Track(audio, lyrics);
});
return await Task.WhenAll(trackTasks);
}
private async Task<IReadOnlyList<AudioPlaylist>> GetAllVkPlaylists(long id, CancellationToken token, ILog log)
{
var pageSize = 200;
var playlistResponse = await vkApi.Audio.GetPlaylistsAsync(id, (uint)pageSize);
var total = (int)playlistResponse.TotalCount;
var result = new List<AudioPlaylist>(total);
result.AddRange(playlistResponse);
var remainingPages = total / pageSize;
var pageTasks = Enumerable.Range(1, remainingPages)
.Select(pageNumber => pageNumber * pageSize)
.Select(async offset =>
{
log.Debug($"Playlists at offset {offset}");
var page = await vkApi.Audio.GetPlaylistsAsync(id, (uint)pageSize, (uint)offset);
token.ThrowIfCancellationRequested(); // not sure if this is needed here
return page;
});
var pages = await Task.WhenAll(pageTasks);
foreach (var page in pages)
{
result.AddRange(page);
}
ThrowIfCountMismatch(total, result.Count);
return result;
}
private async Task<IReadOnlyList<Audio>> ExpandPlaylist(AudioPlaylist playlist, ILog log)
{
log.Debug($"Expanding {playlist.Title}");
var vkAudios = await vkApi.Audio.GetAsync(new AudioGetParams()
{
AlbumId = playlist.Id,
OwnerId = playlist.OwnerId,
});
ThrowIfCountMismatch(vkAudios.TotalCount, vkAudios.Count);
return vkAudios;
}
private async Task<Lyrics> GetLyrics(Audio audio, ILog log)
{
if (audio.LyricsId == null)
{
return null;
}
log.Debug($"Lyrics for {audio.Title}");
return await vkApi.Audio.GetLyricsAsync(audio.LyricsId.Value);
}
private static void ThrowIfCountMismatch(decimal expectedTotal, decimal resultCount)
{
if (resultCount != expectedTotal)
{
throw new InvalidOperationException($"Expected {expectedTotal} items, got {resultCount}. Maybe they were created/deleted, try again.");
}
}
}
}

26
VOffline/Services/Handlers/Extensions.cs Normal file
View file

@ -0,0 +1,26 @@
using System.Net;
using RestSharp;
namespace VOffline.Services.Handlers
{
public static class Extensions
{
public static void ThrowIfSomethingWrong(this IRestResponse response)
{
if (response.ErrorException != null)
{
throw new NetworkException("RestSharp failed", response.ErrorException);
}
if (response.StatusCode != HttpStatusCode.OK)
{
throw new NetworkException($"Bad response code: [{response.StatusCode}]");
}
if (response.RawBytes == null)
{
throw new NetworkException($"Null response bytes");
}
}
}
}

15
VOffline/Services/Handlers/NetworkException.cs Normal file
View file

@ -0,0 +1,15 @@
using System;
namespace VOffline.Services.Handlers
{
public class NetworkException : ApplicationException
{
public NetworkException(string message, Exception innerException) : base(message, innerException)
{
}
public NetworkException(string message) : base(message)
{
}
}
}

93
VOffline/Services/Logic.cs
View file

@ -1,41 +1,96 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using log4net;
using Microsoft.Extensions.Options;
using Newtonsoft.Json;
using VkNet;
using VkNet.Model;
using VkNet.Model.RequestParams;
using VOffline.Models;
using VOffline.Models.Vk;
using VOffline.Services.Google;
using VOffline.Models.Storage;
using VOffline.Services.Handlers;
using VOffline.Services.Vk;
namespace VOffline.Services
{
public class Logic
{
private FileCache<VkCredentials> VkCredentials { get; }
public TokenMagic TokenMagic { get; }
private readonly Settings settings;
private readonly TokenMagic tokenMagic;
private readonly VkApi vkApi;
private readonly VkApiUtils vkApiUtils;
private readonly AudioHandler audioHandler;
public Logic(FileCache<VkCredentials> vkCredentials, TokenMagic tokenMagic)
public Logic(TokenMagic tokenMagic, VkApi vkApi, VkApiUtils vkApiUtils, AudioHandler audioHandler, IOptionsSnapshot<Settings> settings)
{
VkCredentials = vkCredentials;
TokenMagic = tokenMagic;
this.settings = settings.Value;
this.tokenMagic = tokenMagic;
this.vkApi = vkApi;
this.vkApiUtils = vkApiUtils;
this.audioHandler = audioHandler;
}
public void Run(ILog log)
public async Task Run(CancellationToken token, ILog log)
{
log.Debug("started");
log.Debug("Started");
if (VkCredentials.Value == null)
// TODO: async token retrieval
var vkToken = tokenMagic.GetTokenFromScratch(log);
await vkApi.AuthorizeAsync(new ApiAuthParams()
{
log.Debug("Enter login");
var login = Console.ReadLine();
log.Debug("Enter password");
var password = Console.ReadLine();
VkCredentials.Value = new VkCredentials
AccessToken = vkToken.Token
});
var modes = settings.GetWorkingModes();
var ids = settings.Targets
.Select(async x => await vkApiUtils.ResolveId(x))
.Select(x => x.Result)
.Distinct()
.ToImmutableHashSet();
log.Debug($"Processing {JsonConvert.SerializeObject(modes)} for {JsonConvert.SerializeObject(ids)}");
foreach (var id in ids)
{
var name = await vkApiUtils.GetName(id);
var storage = new Storage(name);
log.Info($"id [{id}], name [{name}], path [{storage.FullPath}]");
foreach (var mode in modes)
{
Login = login,
Password = password
};
var subStorage = storage.Descend(mode.ToString(), false);
await ProcessTarget(id, subStorage, mode, token, log);
}
}
var vkToken = TokenMagic.GetTokenFromScratch(log);
//var audio = vkApi.Audio.Get(new AudioGetParams()
//{
// OwnerId = 1
//});
//log.Debug(JsonConvert.SerializeObject(audio, Formatting.Indented));
}
private async Task ProcessTarget(long id, Storage storage, Mode mode, CancellationToken token, ILog log)
{
switch (mode)
{
case Mode.Wall:
break;
case Mode.Audio:
await audioHandler.ProcessAudio(id, storage, token, log);
break;
case Mode.All:
throw new ArgumentOutOfRangeException(nameof(mode), mode, "This mode should have been replaced before processing");
default:
throw new NotImplementedException();
}
}
}
}

36
VOffline/Services/Retrier.cs Normal file
View file

@ -0,0 +1,36 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using log4net;
namespace VOffline.Services
{
public class Retrier
{
public static async Task<T> Retry<T>(Func<Task<T>> taskFactory, int limit, TimeSpan delay, CancellationToken token, ILog log)
{
var exceptions = new List<Exception>();
for (var i = 1; i <= limit; i++)
{
token.ThrowIfCancellationRequested();
try
{
return await taskFactory();
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e)
{
exceptions.Add(e);
}
log.Warn($"Attempt {i}/{limit}, last error was [{exceptions.LastOrDefault()?.GetType()}]");
await Task.Delay(delay, token);
}
throw new AggregateException(exceptions);
}
}
}

51
VOffline/Services/Throttler.cs Normal file
View file

@ -0,0 +1,51 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace VOffline.Services
{
public class Throttler
{
public async Task<IReadOnlyList<T>> ProcessWithThrottling<T>(Task<T>[] tasks, int limit, CancellationToken token)
{
using (var semaphore = new SemaphoreSlim(limit))
{
var newTasks = tasks.Select(async t =>
{
await semaphore.WaitAsync(token);
try
{
var data = await t;
return data;
}
finally
{
semaphore.Release();
}
});
return await Task.WhenAll(newTasks);
}
}
public async Task ProcessWithThrottling(Task[] tasks, int limit, CancellationToken token)
{
using (var semaphore = new SemaphoreSlim(limit))
{
var newTasks = tasks.Select(async t =>
{
await semaphore.WaitAsync(token);
try
{
await t;
}
finally
{
semaphore.Release();
}
});
await Task.WhenAll(newTasks);
}
}
}
}

7
VOffline/Services/Google/AndroidAuth.cs → VOffline/Services/Token/Google/AndroidAuth.cs
View file

@ -1,9 +1,4 @@
using System;
using System.Collections.Generic;
using System.Text;
using log4net;
using RestSharp;
using VOffline.Models;
using log4net;
using VOffline.Models.Google;
namespace VOffline.Services.Google

0
VOffline/Services/Google/GoogleHttpRequests.cs → VOffline/Services/Token/Google/GoogleHttpRequests.cs
View file

2
VOffline/Services/Google/MTalk.cs → VOffline/Services/Token/Google/MTalk.cs
View file

@ -1,11 +1,9 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Security;
using System.Net.Sockets;
using System.Text;
using log4net;
using VOffline.Models;
using VOffline.Models.Google;
namespace VOffline.Services.Google

2
VOffline/Services/Google/ProtobufParser.cs → VOffline/Services/Token/Google/ProtobufParser.cs
View file

@ -1,8 +1,6 @@
using System;
using System.Collections.Generic;
using System.Data.Common;
using System.Linq;
using VOffline.Models;
using VOffline.Models.Google;
namespace VOffline.Services.Google

0
VOffline/Services/Google/VarInt.cs → VOffline/Services/Token/Google/VarInt.cs
View file

5
VOffline/Services/TokenMagic.cs → VOffline/Services/Token/TokenMagic.cs
View file

@ -1,5 +1,4 @@
using log4net;
using VOffline.Models;
using VOffline.Models.Vk;
using VOffline.Services.Google;
using VOffline.Services.Vk;
@ -10,14 +9,12 @@ namespace VOffline.Services
{
private AndroidAuth AndroidAuth { get; }
private VkTokenReceiver VkTokenReceiver { get; }
private FileCache<VkCredentials> VkCredentials { get; }
public FileCache<VkToken> VkToken { get; }
public TokenMagic(AndroidAuth androidAuth, VkTokenReceiver vkTokenReceiver, FileCache<VkCredentials> vkCredentials, FileCache<VkToken> vkToken)
public TokenMagic(AndroidAuth androidAuth, VkTokenReceiver vkTokenReceiver, FileCache<VkToken> vkToken)
{
AndroidAuth = androidAuth;
VkTokenReceiver = vkTokenReceiver;
VkCredentials = vkCredentials;
VkToken = vkToken;
}

28
VOffline/Services/Vk/VkHttpRequests.cs → VOffline/Services/Token/Vk/VkHttpRequests.cs
View file

@ -1,19 +1,22 @@
using System;
using log4net;
using Microsoft.Extensions.Options;
using Newtonsoft.Json;
using RestSharp;
using VOffline.Models;
using VOffline.Models.Vk;
using RestClient = RestSharp.RestClient;
namespace VOffline.Services.Vk
{
public class VkHttpRequests
{
private FileCache<VkCredentials> VkCredentialsCache { get; }
private readonly string userAgent;
private VkCredentials VkCredentials { get; }
public VkHttpRequests(FileCache<VkCredentials> vkCredentialsCache)
public VkHttpRequests(IOptionsSnapshot<VkCredentials> vkCredentials, UserAgentProvider userAgentProvider)
{
VkCredentialsCache = vkCredentialsCache;
userAgent = userAgentProvider.UserAgent;
VkCredentials = vkCredentials.Value;
}
public string GetNonRefreshedToken(ILog log)
@ -21,12 +24,12 @@ namespace VOffline.Services.Vk
var client = GetClient("https://oauth.vk.com/token");
var request = new RestRequest(Method.GET);
request.AddQueryParameter("grant_type", "password");
request.AddQueryParameter("client_id", "2685278");
request.AddQueryParameter("client_id", ClientId.ToString());
request.AddQueryParameter("client_secret", "lxhD8OD7dMsqtXIm5IUY");
request.AddQueryParameter("username", VkCredentialsCache.Value.Login);
request.AddQueryParameter("password", VkCredentialsCache.Value.Password);
request.AddQueryParameter("username", VkCredentials.Login);
request.AddQueryParameter("password", VkCredentials.Password);
request.AddQueryParameter("v", VkApiVersion);
request.AddQueryParameter("scope", "audio,offline");
request.AddQueryParameter("scope", Scope);
log.Debug($"request non-refreshed token");
var response = client.Execute(request);
if (!response.IsSuccessful)
@ -71,15 +74,16 @@ namespace VOffline.Services.Vk
{
var client = new RestClient(url)
{
UserAgent = VkUserAgent
UserAgent = userAgent
};
client.RemoteCertificateValidationCallback += (sender, certificate, chain, errors) => true;
return client;
}
private const string VkUserAgent = "KateMobileAndroid/51.2 lite-443 (Android 4.4.2; SDK 19; x86; unknown Android SDK built for x86; en)";
private const string VkApiVersion = "5.72";
public const string VkApiVersion = "5.72";
public const ulong ClientId = 2685278;
public const string Scope = "audio,offline,notify,friends,photos,video,stories,pages,status,notes,messages,wall,ads,docs,groups,notifications,stats,email,market";
}
}

6
VOffline/Services/Vk/VkTokenReceiver.cs → VOffline/Services/Token/Vk/VkTokenReceiver.cs
View file

@ -1,21 +1,17 @@
using log4net;
using VOffline.Models;
using VOffline.Models.Google;
using VOffline.Models.Vk;
using VOffline.Services.Google;
namespace VOffline.Services.Vk
{
public class VkTokenReceiver
{
private FileCache<VkCredentials> VkCredentialsCache { get; }
private FileCache<GoogleCredentials> CredentialsCache { get; }
private VkHttpRequests VkHttpRequests { get; }
private GoogleHttpRequests GoogleHttpRequests { get; }
public VkTokenReceiver(FileCache<VkCredentials> vkCredentialsCache, FileCache<GoogleCredentials> credentialsCache, VkHttpRequests vkHttpRequests, GoogleHttpRequests googleHttpRequests)
public VkTokenReceiver(FileCache<GoogleCredentials> credentialsCache, VkHttpRequests vkHttpRequests, GoogleHttpRequests googleHttpRequests)
{
VkCredentialsCache = vkCredentialsCache;
CredentialsCache = credentialsCache;
VkHttpRequests = vkHttpRequests;
GoogleHttpRequests = googleHttpRequests;

98
VOffline/Services/Vk/VkApiUtils.cs Normal file
View file

@ -0,0 +1,98 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography.X509Certificates;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using log4net;
using Newtonsoft.Json;
using VkNet;
using VkNet.Enums;
using VkNet.Enums.Filters;
using VkNet.Model;
using VkNet.Model.Attachments;
using VkNet.Model.RequestParams;
using VOffline.Models.Storage;
namespace VOffline.Services.Vk
{
public class VkApiUtils
{
private readonly VkApi vkApi;
public VkApiUtils(VkApi vkApi)
{
this.vkApi = vkApi;
}
public async Task<long> ResolveId(string target)
{
// any community type with id, eg. club123 or event123
var communityMatch = CommunityPattern.Match(target);
if (communityMatch.Success)
{
return -1 * long.Parse(communityMatch.Groups[2].Value);
}
// any user eg. id123
var personalMatch = PersonalPattern.Match(target);
if (personalMatch.Success)
{
return long.Parse(personalMatch.Groups[2].Value);
}
// any id eg. 123 or -123
var digitalMatch = DigitalPattern.Match(target);
if (digitalMatch.Success)
{
return long.Parse(digitalMatch.Groups[2].Value);
}
// any screen name
var vkObj = await vkApi.Utils.ResolveScreenNameAsync(target);
switch (vkObj.Type)
{
case VkObjectType.User:
return vkObj.Id.Value;
case VkObjectType.Group:
return -1 * vkObj.Id.Value;
default:
throw new ArgumentOutOfRangeException();
}
}
public async Task<string> GetName(long id)
{
if (id >= 0)
{
var users = await vkApi.Users.GetAsync(new []{id}, ProfileFields.All);
var user = users[0];
return string.Join(" ", user.LastName, user.FirstName, GroupOrEmpty(" - ", user.Id.ToString(), user.ScreenName, user.Domain));
}
var groups = await vkApi.Groups.GetByIdAsync(null, (-1*id).ToString(), GroupsFields.All);
var group = groups[0];
return string.Join(" ", group.Name, GroupOrEmpty(" - ", group.Id.ToString(), group.ScreenName, group.Type.ToString()));
}
private static string GroupOrEmpty(string separator, params string[] parts)
{
var all = string.Join(separator, parts);
return string.IsNullOrEmpty(all)
? string.Empty
: $"({all})";
}
private static readonly Regex CommunityPattern = new Regex(@"^(public|club|event)(\d+)$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
private static readonly Regex PersonalPattern = new Regex(@"^(id)(\d+)$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
private static readonly Regex DigitalPattern = new Regex(@"^(-?\d+)$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
}
public class UserAgentProvider
{
public string UserAgent => UserAgentValue;
private const string UserAgentValue = "KateMobileAndroid/51.2 lite-443 (Android 4.4.2; SDK 19; x86; unknown Android SDK built for x86; en)";
}
}

84
VOffline/Services/VkNet/CancellableConstraint.cs Normal file
View file

@ -0,0 +1,84 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using VkNet.Abstractions.Core;
using VkNet.Utils;
namespace VOffline.Services.Vk
{
public class CancellableConstraint : IAwaitableConstraint
{
private readonly SemaphoreSlim _semaphore = new SemaphoreSlim(1, 1);
private readonly object _sync = new object();
private readonly LimitedSizeStack<DateTime> _timeStamps;
private int _count;
private TimeSpan _timeSpan;
private readonly CancellationToken token;
public CancellableConstraint(int number, TimeSpan timeSpan, CancellationToken token)
{
if (number <= 0)
throw new ArgumentException("count should be strictly positive", nameof(number));
if (timeSpan.TotalMilliseconds <= 0.0)
throw new ArgumentException("timeSpan should be strictly positive", nameof(timeSpan));
this._count = number;
this._timeSpan = timeSpan;
this.token = token;
this._timeStamps = new LimitedSizeStack<DateTime>(this._count);
}
public async Task<IDisposable> WaitForReadiness(CancellationToken cancellationToken)
{
using (var cts = CancellationTokenSource.CreateLinkedTokenSource(token, cancellationToken))
{
CancellableConstraint awaitableConstraint = this;
await awaitableConstraint._semaphore.WaitAsync(cts.Token);
int num = 0;
DateTime now = DateTime.Now;
DateTime dateTime = now - awaitableConstraint._timeSpan;
LinkedListNode<DateTime> linkedListNode1 = awaitableConstraint._timeStamps.First;
LinkedListNode<DateTime> linkedListNode2 = (LinkedListNode<DateTime>) null;
while (linkedListNode1 != null && linkedListNode1.Value > dateTime)
{
linkedListNode2 = linkedListNode1;
linkedListNode1 = linkedListNode1.Next;
++num;
}
if (num < awaitableConstraint._count)
return (IDisposable) new DisposableAction(new Action(awaitableConstraint.OnEnded));
TimeSpan delay = linkedListNode2.Value.Add(awaitableConstraint._timeSpan) - now;
try
{
await Task.Delay(delay, cts.Token);
}
catch (Exception ex)
{
awaitableConstraint._semaphore.Release();
throw;
}
return (IDisposable) new DisposableAction(new Action(awaitableConstraint.OnEnded));
}
}
/// <inheritdoc />
public void SetRate(int number, TimeSpan timeSpan)
{
lock (this._sync)
{
this._count = number;
this._timeSpan = timeSpan;
}
}
private void OnEnded()
{
this._timeStamps.Push(DateTime.Now);
this._semaphore.Release();
}
}
}

22
VOffline/Services/VkNet/LimitedSizeStack.cs Normal file
View file

@ -0,0 +1,22 @@
using System.Collections.Generic;
namespace VOffline.Services.Vk
{
public class LimitedSizeStack<T> : LinkedList<T>
{
private readonly int _maxSize;
public LimitedSizeStack(int maxSize)
{
this._maxSize = maxSize;
}
public void Push(T item)
{
this.AddFirst(item);
if (this.Count <= this._maxSize)
return;
this.RemoveLast();
}
}
}

114
VOffline/Services/VkNet/RestClientWithUserAgent.cs Normal file
View file

@ -0,0 +1,114 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using VkNet.Abstractions.Utils;
using VkNet.Utils;
namespace VOffline.Services.Vk
{
public class RestClientWithUserAgent : IRestClient
{
/// <summary>The log</summary>
private readonly ILogger<RestClient> _logger;
private TimeSpan _timeoutSeconds;
private string userAgent;
/// <inheritdoc />
public RestClientWithUserAgent(ILogger<RestClient> logger, IWebProxy proxy, UserAgentProvider userAgentProvider)
{
_logger = logger;
Proxy = proxy;
userAgent = userAgentProvider.UserAgent;
}
/// <inheritdoc />
public IWebProxy Proxy { get; set; }
/// <inheritdoc />
public TimeSpan Timeout
{
get
{
if (!(this._timeoutSeconds == TimeSpan.Zero))
return this._timeoutSeconds;
return TimeSpan.FromSeconds(300.0);
}
set
{
this._timeoutSeconds = value;
}
}
public Task<HttpResponse<string>> GetAsync(Uri uri,IEnumerable<KeyValuePair<string, string>> parameters)
{
IEnumerable<string> values = parameters.Where<KeyValuePair<string, string>>((Func<KeyValuePair<string, string>, bool>)(parameter => !string.IsNullOrWhiteSpace(parameter.Value))).Select<KeyValuePair<string, string>, string>((Func<KeyValuePair<string, string>, string>)(parameter => parameter.Key.ToLowerInvariant() + "=" + parameter.Value));
UriBuilder uriBuilder = new UriBuilder(uri)
{
Query = string.Join("&", values)
};
ILogger<RestClient> logger = this._logger;
if (logger != null)
logger.LogDebug(string.Format("GET request: {0}", (object)uriBuilder.Uri));
HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, uri);
return this.Call((Func<HttpClient, Task<HttpResponseMessage>>)(httpClient => httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead)));
}
public Task<HttpResponse<string>> PostAsync(Uri uri,IEnumerable<KeyValuePair<string, string>> parameters)
{
if (this._logger != null)
{
string json = JsonConvert.SerializeObject((object)parameters);
this._logger.LogDebug(string.Format("POST request: {0}{1}{2}", (object)uri, (object)Environment.NewLine, (object)Utilities.PreetyPrintJson(json)));
}
FormUrlEncodedContent urlEncodedContent = new FormUrlEncodedContent(parameters);
HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, uri)
{
Content = (HttpContent)urlEncodedContent
};
return this.Call((Func<HttpClient, Task<HttpResponseMessage>>)(httpClient => httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead)));
}
private async Task<HttpResponse<string>> Call(Func<HttpClient, Task<HttpResponseMessage>> method)
{
var httpClientHandler = new HttpClientHandler
{
UseProxy = false
};
if (Proxy != null)
{
httpClientHandler = new HttpClientHandler
{
Proxy = Proxy,
UseProxy = true
};
var logger = _logger;
logger?.LogDebug($"Use Proxy: {(object) Proxy}");
}
using (var client = new HttpClient(httpClientHandler))
{
client.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent);
if (Timeout != TimeSpan.Zero)
client.Timeout = Timeout;
var response = await method(client).ConfigureAwait(false);
var requestUri = response.RequestMessage.RequestUri.ToString();
if (!response.IsSuccessStatusCode)
return HttpResponse<string>.Fail(response.StatusCode,
await response.Content.ReadAsStringAsync().ConfigureAwait(false), requestUri);
var json = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
var logger = _logger;
logger?.LogDebug("Response:" + Environment.NewLine + Utilities.PreetyPrintJson(json));
return HttpResponse<string>.Success(response.StatusCode, json, requestUri);
}
}
}
}

11
VOffline/VOffline.csproj
View file

@ -8,17 +8,22 @@
<ItemGroup>
<PackageReference Include="log4net" Version="2.0.8" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="2.1.1" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="2.2.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.1.1" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.1.1" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="2.1.1" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="2.1.1" />
<PackageReference Include="Microsoft.Extensions.Logging.Log4Net.AspNetCore" Version="2.2.8" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="2.1.1" />
<PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
<PackageReference Include="RestSharp" Version="106.5.0" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.1" />
<PackageReference Include="RestSharp" Version="106.6.3" />
<PackageReference Include="VkNet" Version="1.40.0" />
</ItemGroup>
<ItemGroup>
<None Update="appsettings.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="log4net.config">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>

12
VOffline/appsettings.json Normal file
View file

@ -0,0 +1,12 @@
{
"Settings": {
//"Targets": [ "id1", "durov", "public147415323", "tech" ],
//"Targets": [ "retrowave" ],
"Targets": [ "rast1234" ],
"Modes": [ "Audio" ]
},
"VkCredentials": {
//"Login": "username@gmail.com",
//"Password": "your_vk_password"
}
}