vrr/VOffline
2
0
Fork 0
mirror of https://github.com/Rast1234/VOffline.git synced 2026-04-28 03:49:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

rewrite downloading

Browse source
This commit is contained in:
Rostislav Kirillov 2019-01-10 07:33:07 +05:00
parent a602981f71
commit 80fb2c9e9e
19 changed files with 437 additions and 220 deletions
Download patch file Download diff file Expand all files Collapse all files

2
VOffline.sln
View file

@ -3,7 +3,7 @@ Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 15
VisualStudioVersion = 15.0.28010.2036
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VOffline", "VOffline\VOffline.csproj", "{4EBB18FF-457C-4F6D-98E4-A44AEAF59CDD}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "VOffline", "VOffline\VOffline.csproj", "{4EBB18FF-457C-4F6D-98E4-A44AEAF59CDD}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution

1
VOffline/Models/Settings.cs
View file

@ -10,6 +10,7 @@ namespace VOffline.Models
{
public List<string> Targets { get; set; }
public List<Mode> Modes { get; set; }
public string OutputPath { get; set; }
public ImmutableHashSet<Mode> GetWorkingModes()
{

90
VOffline/Models/Storage/Download.cs Normal file
View file

@ -0,0 +1,90 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using RestSharp;
using VkNet;
using VOffline.Services.Handlers;
namespace VOffline.Models.Storage
{
public interface IDownload
{
DirectoryInfo Location { get; }
string DesiredName { get; }
int RetryCount { get; }
IReadOnlyList<Exception> Errors { get; }
void AddError(Exception e);
Task<byte[]> GetContent(VkApi vkApi, CancellationToken token);
}
public class Download : IDownload
{
private readonly List<Exception> errors;
public Download(Uri uri, DirectoryInfo location, string desiredName)
{
errors = new List<Exception>();
Uri = uri ?? throw new ArgumentNullException(nameof(uri));
Location = location ?? throw new ArgumentNullException(nameof(location));
DesiredName = desiredName ?? throw new ArgumentNullException(nameof(desiredName));
}
public Uri Uri { get; }
public DirectoryInfo Location { get; }
public string DesiredName { get; }
public int RetryCount => errors.Count;
public IReadOnlyList<Exception> Errors => errors;
public void AddError(Exception e)
{
errors.Add(e);
}
public async Task<byte[]> GetContent(VkApi vkApi, CancellationToken token)
{
var client = new RestClient($"{Uri.Scheme}://{Uri.Authority}");
var response = await client.ExecuteGetTaskAsync(new RestRequest(Uri.PathAndQuery), token);
response.ThrowIfSomethingWrong();
return response.RawBytes;
}
}
public class LyricsVkDownload : IDownload
{
private readonly long lyricsId;
private readonly List<Exception> errors;
public LyricsVkDownload(long lyricsId, DirectoryInfo location, string desiredName)
{
this.lyricsId = lyricsId;
errors = new List<Exception>();
Location = location ?? throw new ArgumentNullException(nameof(location));
DesiredName = desiredName ?? throw new ArgumentNullException(nameof(desiredName));
}
public DirectoryInfo Location { get; }
public string DesiredName { get; }
public int RetryCount => errors.Count;
public IReadOnlyList<Exception> Errors => errors;
public void AddError(Exception e)
{
errors.Add(e);
}
public async Task<byte[]> GetContent(VkApi vkApi, CancellationToken token)
{
var lyrics = await vkApi.Audio.GetLyricsAsync(lyricsId);
return !string.IsNullOrWhiteSpace(lyrics?.Text)
? new byte[] { } // TODO: fix this to provide a no-value without exceptions
: Encoding.UTF8.GetBytes(lyrics.Text);
}
}
}

19
VOffline/Models/Storage/Playlist.cs
View file

@ -1,6 +1,9 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using VkNet.Model;
using VOffline.Services.Storage;
namespace VOffline.Models.Storage
{
@ -31,6 +34,22 @@ namespace VOffline.Models.Storage
public IReadOnlyList<Track> Tracks { get; set; }
public IEnumerable<IDownload> ToDownloads(FilesystemTools filesystemTools, DirectoryInfo dir)
{
var playlistDir = filesystemTools.CreateSubdir(dir, Name, true);
if (Cover != null)
{
// TODO: i guess it's always jpeg?
var ext = Path.HasExtension(Cover.AbsoluteUri) ? Path.GetExtension(Cover.AbsoluteUri) : ".jpg";
yield return new Download(Cover, playlistDir, $"playlist_cover{ext}");
}
foreach (var download in Tracks.SelectMany(t => t.ToDownloads(filesystemTools, playlistDir)))
{
yield return download;
}
}
private static Uri GetBestImage(AudioCover cover)
{
// TODO: theoretically images could be different or missing, but looks like this works fine

38
VOffline/Models/Storage/Track.cs
View file

@ -1,26 +1,48 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using VkNet.Model;
using VkNet.Model.Attachments;
using VOffline.Services.Storage;
namespace VOffline.Models.Storage
{
public class Track
{
public long Id { get; set; }
public string Name { get; set; }
public string Artist { get; set; }
public Uri Url { get; set; }
public bool? Hq { get; set; }
public string Lyrics { get; set; }
public long Id { get; }
public string Name { get; }
public string Artist { get; }
public Uri Url { get; }
public bool? Hq { get; }
public long? LyricsId { get; }
public Track(Audio audio, Lyrics lyrics)
public Track(Audio audio)
{
Id = audio.Id.Value;
Name = audio.Title;
Artist = audio.Artist;
Url = audio.Url;
Hq = audio.IsHq;
Lyrics = lyrics?.Text;
LyricsId = audio.LyricsId;
}
public IEnumerable<IDownload> ToDownloads(FilesystemTools filesystemTools, DirectoryInfo dir)
{
var trackName = string.Join(" - ", new[] { Artist, Name }.Where(x => !string.IsNullOrEmpty(x)));
if (Url != null)
{
yield return new Download(Url, dir, $"{trackName}.mp3");
}
else
{
filesystemTools.CreateUniqueFile(dir, $"{trackName}.mp3.deleted");
}
if (LyricsId != null)
{
yield return new LyricsVkDownload(LyricsId.Value, dir, $"{trackName}.txt");
}
}
}
}

11
VOffline/Program.cs
View file

@ -23,6 +23,7 @@ using VOffline.Models.Vk;
using VOffline.Services;
using VOffline.Services.Google;
using VOffline.Services.Handlers;
using VOffline.Services.Storage;
using VOffline.Services.Vk;
namespace VOffline
@ -54,8 +55,11 @@ namespace VOffline
serviceCollection.AddSingleton<GoogleHttpRequests>();
serviceCollection.AddSingleton<VkHttpRequests>();
serviceCollection.AddSingleton<VkApiUtils>();
serviceCollection.AddSingleton<UserAgentProvider>();
serviceCollection.AddSingleton<ConstantsProvider>();
serviceCollection.AddSingleton<VkApi>(_ => VkApiFactory(token));
serviceCollection.AddSingleton<FilesystemTools>();
serviceCollection.AddSingleton<DownloadQueueProvider>();
serviceCollection.AddSingleton<BackgroundDownloader>();
serviceCollection.AddTransient(provider => LogManager.GetLogger(Assembly.GetEntryAssembly(), typeof(Program)));
serviceCollection.AddTransient<AndroidAuth>();
@ -67,6 +71,7 @@ namespace VOffline
var services = serviceCollection.BuildServiceProvider();
await services.GetRequiredService<Logic>().Run(token, log);
return 0;
}
catch (Exception e)
@ -80,7 +85,7 @@ namespace VOffline
{
// VkNet uses its own DI for internal services
var sc = new ServiceCollection();
sc.AddSingleton<UserAgentProvider>();
sc.AddSingleton<ConstantsProvider>();
sc.AddSingleton<IRestClient, RestClientWithUserAgent>();
sc.AddSingleton<IAwaitableConstraint, CancellableConstraint>(_ => new CancellableConstraint(3, TimeSpan.FromSeconds(1.3), token));
sc.AddSingleton<ILoggerFactory, LoggerFactory>();
@ -89,7 +94,7 @@ namespace VOffline
{
builder.ClearProviders();
builder.SetMinimumLevel(LogLevel.Debug);
builder.AddLog4Net();
//builder.AddLog4Net();
});
return new VkApi(sc);
}

117
VOffline/Services/Filesystem/Storage.cs
View file

@ -1,117 +0,0 @@
using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using VkNet.Model.RequestParams;
namespace VOffline.Services
{
public class Storage
{
private readonly DirectoryInfo rootDir;
public string FullPath => rootDir.FullName;
public Storage(string rootName)
{
rootDir = new DirectoryInfo(FilterFilename(rootName));
lock (LockObject)
{
if (rootDir.Exists)
{
throw new InvalidOperationException($"Storage directory [{rootDir.Name}] already exists");
}
rootDir.Create();
}
}
private Storage(string rootName, bool useExisting)
{
rootDir = new DirectoryInfo(rootName);
lock (LockObject)
{
if (!useExisting && rootDir.Exists)
{
throw new InvalidOperationException($"Storage directory [{rootDir.Name}] already exists");
}
rootDir.Create();
}
}
public Storage Descend(string subName, bool makeUniqueName)
{
var filteredName = FilterFilename(subName);
if (Path.IsPathRooted(filteredName))
{
throw new InvalidOperationException($"Expected relative path, got absolute: [{filteredName}]");
}
// TODO: looks like cancer. also probable race conditions?
if (makeUniqueName)
{
var uniqueName = GetUniqueDirectory(filteredName);
return new Storage(Path.Combine(rootDir.FullName, uniqueName.Name), true);
}
return new Storage(Path.Combine(rootDir.FullName, filteredName), false);
}
/// <summary>
/// Creates unique file
/// </summary>
/// <param name="name"></param>
/// <returns></returns>
public FileInfo GetFile(string name)
{
var filename = FilterFilename(name);
var file = GetUniqueFile(filename);
return file;
}
private FileInfo GetUniqueFile(string filteredName)
{
var name = Path.GetFileNameWithoutExtension(filteredName);
var extension = Path.GetExtension(filteredName);
var fileInfo = new FileInfo(Path.Combine(rootDir.FullName, filteredName));
for (var i = 1;; i++)
{
lock (LockObject)
{
if (!fileInfo.Exists)
{
fileInfo.Create().Close();
return fileInfo;
}
}
fileInfo = new FileInfo(Path.Combine(rootDir.FullName, $"{name} ({i}){extension}"));
}
}
private DirectoryInfo GetUniqueDirectory(string filteredName)
{
var directoryInfo = new DirectoryInfo(Path.Combine(rootDir.FullName, filteredName));
for (var i = 1; ; i++)
{
lock (LockObject)
{
if (!directoryInfo.Exists)
{
directoryInfo.Create();
return directoryInfo;
}
}
directoryInfo = new DirectoryInfo(Path.Combine(rootDir.FullName, $"{filteredName} ({i})"));
}
}
private static string FilterFilename(string value) => string.Join("_", value.Split(AllBadChars)).Trim();
private static readonly char[] AllBadChars =
Path.GetInvalidFileNameChars()
.Concat(Path.GetInvalidPathChars())
.Concat(new[] {'\\', '/', ':'})
.Distinct()
.ToArray();
private static readonly object LockObject = new object();
}
}

82
VOffline/Services/Handlers/Audio.cs
View file

@ -7,6 +7,7 @@ using System.Threading;
using System.Threading.Tasks;
using log4net;
using Newtonsoft.Json;
using Nito.AsyncEx;
using RestSharp;
using VkNet;
using VkNet.Model;
@ -14,6 +15,7 @@ using VkNet.Model.Attachments;
using VkNet.Model.RequestParams;
using VkNet.Utils;
using VOffline.Models.Storage;
using VOffline.Services.Storage;
using VOffline.Services.Vk;
using RestClient = RestSharp.RestClient;
@ -22,56 +24,25 @@ namespace VOffline.Services.Handlers
public class AudioHandler
{
private readonly VkApi vkApi;
private readonly FilesystemTools filesystemTools;
private readonly DownloadQueueProvider downloadQueueProvider;
public AudioHandler(VkApi vkApi)
public AudioHandler(VkApi vkApi, FilesystemTools filesystemTools, DownloadQueueProvider downloadQueueProvider)
{
this.vkApi = vkApi;
this.filesystemTools = filesystemTools;
this.downloadQueueProvider = downloadQueueProvider;
}
public async Task ProcessAudio(long id, Storage storage, CancellationToken token, ILog log)
public async Task ProcessAudio(long id, DirectoryInfo dir, CancellationToken token, ILog log)
{
// TODO: continuation/recovery? at least on album level? maybe create and detect some 'completeness' file?
var allPlaylists = await GetAllPlaylists(id, token, log);
var throttler = new Throttler();
foreach (var playlist in allPlaylists)
{
var playlistStorage = storage.Descend(playlist.Name, true);
var downloadTasks = playlist.Tracks
.Select(async track => await Download(track, playlistStorage, token, log))
.ToArray();
await throttler.ProcessWithThrottling(downloadTasks, 3, token);
}
}
private async Task Download(Track track, Storage playlistStorage, CancellationToken token, ILog log)
{
return;
var trackName = string.Join(" - ", new[] { track.Artist, track.Name }.Where(x => !string.IsNullOrEmpty(x)));
if (track.Url != null)
{
var content = await Retrier.Retry(async () =>
{
var client = new RestClient($"{track.Url.Scheme}://{track.Url.Authority}");
var response = await client.ExecuteGetTaskAsync(new RestRequest(track.Url.PathAndQuery), token);
response.ThrowIfSomethingWrong();
return response.RawBytes;
}, 3, TimeSpan.FromSeconds(5), token, log);
var file = playlistStorage.GetFile($"{trackName}.mp3").FullName;
log.Debug($"Saving {file}");
await File.WriteAllBytesAsync(file, content, token);
}
else
{
playlistStorage.GetFile($"{trackName}.mp3.deleted");
}
if (track.Lyrics != null)
{
var lyricsFile = playlistStorage.GetFile($"{trackName}.txt").FullName;
await File.WriteAllTextAsync(lyricsFile, track.Lyrics, token);
}
var allDownloadTasks = allPlaylists
.SelectMany(p => p.ToDownloads(filesystemTools, dir))
.Select(d => downloadQueueProvider.Pending.EnqueueAsync(d, token));
await Task.WhenAll(allDownloadTasks);
}
private async Task<IReadOnlyList<Playlist>> GetAllPlaylists(long id, CancellationToken token, ILog log)
@ -101,8 +72,7 @@ namespace VOffline.Services.Handlers
var audioTasks = vkAudios.Select(async audio =>
{
token.ThrowIfCancellationRequested();
var lyrics = await GetLyrics(audio, log);
return new Track(audio, lyrics);
return new Track(audio);
});
var tracks = await Task.WhenAll(audioTasks);
return new Playlist(playlist, tracks);
@ -113,20 +83,18 @@ namespace VOffline.Services.Handlers
public async Task<IReadOnlyList<Track>> GetTracks(long id, HashSet<long> tracksInPlaylists, CancellationToken token, ILog log)
{
// TODO: handle paging if api returns partial result
log.Debug("Getting all tracks not in playlists");
var vkAudios = await vkApi.Audio.GetAsync(new AudioGetParams()
{
OwnerId = id
});
log.Debug($"Got {vkAudios.Count} tracks not in playlists for {id}");
ThrowIfCountMismatch(vkAudios.TotalCount, vkAudios.Count);
var trackTasks = vkAudios
.Where(t => !tracksInPlaylists.Contains(t.Id.Value))
.Select(async audio =>
{
// TODO: filter out extra tracks here
var lyrics = await GetLyrics(audio, log);
token.ThrowIfCancellationRequested();
return new Track(audio, lyrics);
return new Track(audio);
});
return await Task.WhenAll(trackTasks);
}
@ -135,6 +103,7 @@ namespace VOffline.Services.Handlers
{
var pageSize = 200;
var playlistResponse = await vkApi.Audio.GetPlaylistsAsync(id, (uint)pageSize);
log.Debug($"Got {playlistResponse.Count}/{playlistResponse.TotalCount} tracks in playlist {id}");
var total = (int)playlistResponse.TotalCount;
var result = new List<AudioPlaylist>(total);
@ -145,8 +114,9 @@ namespace VOffline.Services.Handlers
.Select(pageNumber => pageNumber * pageSize)
.Select(async offset =>
{
log.Debug($"Playlists at offset {offset}");
//log.Debug($"Playlists at offset {offset}");
var page = await vkApi.Audio.GetPlaylistsAsync(id, (uint)pageSize, (uint)offset);
log.Debug($"Got {page.Count}/{playlistResponse.TotalCount} tracks in playlist {id} at offset {offset}");
token.ThrowIfCancellationRequested(); // not sure if this is needed here
return page;
});
@ -161,24 +131,22 @@ namespace VOffline.Services.Handlers
private async Task<IReadOnlyList<Audio>> ExpandPlaylist(AudioPlaylist playlist, ILog log)
{
log.Debug($"Expanding {playlist.Title}");
var vkAudios = await vkApi.Audio.GetAsync(new AudioGetParams()
{
AlbumId = playlist.Id,
OwnerId = playlist.OwnerId,
});
log.Debug($"Expanded {playlist.Title}: {vkAudios.TotalCount}");
ThrowIfCountMismatch(vkAudios.TotalCount, vkAudios.Count);
return vkAudios;
}
private async Task<Lyrics> GetLyrics(Audio audio, ILog log)
private async Task<Lyrics> GetLyrics(long lyricsId, ILog log)
{
if (audio.LyricsId == null)
{
return null;
}
log.Debug($"Lyrics for {audio.Title}");
return await vkApi.Audio.GetLyricsAsync(audio.LyricsId.Value);
var lyrics = await vkApi.Audio.GetLyricsAsync(lyricsId);
log.Debug($"Got lyrics for {lyricsId}");
return lyrics;
}
private static void ThrowIfCountMismatch(decimal expectedTotal, decimal resultCount)

34
VOffline/Services/Logic.cs
View file

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
@ -13,6 +14,7 @@ using VkNet.Model.RequestParams;
using VOffline.Models;
using VOffline.Models.Storage;
using VOffline.Services.Handlers;
using VOffline.Services.Storage;
using VOffline.Services.Vk;
namespace VOffline.Services
@ -23,14 +25,18 @@ namespace VOffline.Services
private readonly TokenMagic tokenMagic;
private readonly VkApi vkApi;
private readonly VkApiUtils vkApiUtils;
private readonly BackgroundDownloader downloader;
private readonly FilesystemTools filesystemTools;
private readonly AudioHandler audioHandler;
public Logic(TokenMagic tokenMagic, VkApi vkApi, VkApiUtils vkApiUtils, AudioHandler audioHandler, IOptionsSnapshot<Settings> settings)
public Logic(TokenMagic tokenMagic, VkApi vkApi, VkApiUtils vkApiUtils, BackgroundDownloader downloader, FilesystemTools filesystemTools, AudioHandler audioHandler, IOptionsSnapshot<Settings> settings)
{
this.settings = settings.Value;
this.tokenMagic = tokenMagic;
this.vkApi = vkApi;
this.vkApiUtils = vkApiUtils;
this.downloader = downloader;
this.filesystemTools = filesystemTools;
this.audioHandler = audioHandler;
}
@ -54,35 +60,37 @@ namespace VOffline.Services
.Distinct()
.ToImmutableHashSet();
log.Debug($"Processing {JsonConvert.SerializeObject(modes)} for {JsonConvert.SerializeObject(ids)}");
var downloaderTask = downloader.Process(token, log);
var rootDir = filesystemTools.MkDir(settings.OutputPath);
foreach (var id in ids)
{
var name = await vkApiUtils.GetName(id);
var storage = new Storage(name);
log.Info($"id [{id}], name [{name}], path [{storage.FullPath}]");
var workDir = filesystemTools.CreateSubdir(rootDir, name, false);
log.Info($"id [{id}], name [{name}], path [{workDir.FullName}]");
foreach (var mode in modes)
{
var subStorage = storage.Descend(mode.ToString(), false);
await ProcessTarget(id, subStorage, mode, token, log);
var modeDir = filesystemTools.CreateSubdir(workDir, mode.ToString(), false);
await ProcessTarget(id, modeDir, mode, token, log);
}
}
//var audio = vkApi.Audio.Get(new AudioGetParams()
//{
// OwnerId = 1
//});
//log.Debug(JsonConvert.SerializeObject(audio, Formatting.Indented));
var downloadErrors = await downloaderTask;
foreach (var downloadError in downloadErrors)
{
log.Warn($"Failed {downloadError.DesiredName}", downloadError.Errors.LastOrDefault());
}
}
private async Task ProcessTarget(long id, Storage storage, Mode mode, CancellationToken token, ILog log)
private async Task ProcessTarget(long id, DirectoryInfo dir, Mode mode, CancellationToken token, ILog log)
{
switch (mode)
{
case Mode.Wall:
break;
case Mode.Audio:
await audioHandler.ProcessAudio(id, storage, token, log);
await audioHandler.ProcessAudio(id, dir, token, log);
break;
case Mode.All:
throw new ArgumentOutOfRangeException(nameof(mode), mode, "This mode should have been replaced before processing");

80
VOffline/Services/Storage/BackgroundDownloader.cs Normal file
View file

@ -0,0 +1,80 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using log4net;
using Nito.AsyncEx;
using RestSharp;
using VkNet;
using VOffline.Models.Storage;
using VOffline.Services.Handlers;
namespace VOffline.Services.Storage
{
public class BackgroundDownloader
{
private readonly DownloadQueueProvider queueProvider;
private readonly FilesystemTools filesystemTools;
private readonly VkApi vkApi;
//private readonly int retryLimit;
public BackgroundDownloader(DownloadQueueProvider queueProvider, FilesystemTools filesystemTools, VkApi vkApi)
{
this.queueProvider = queueProvider;
this.filesystemTools = filesystemTools;
this.vkApi = vkApi;
}
public async Task<List<IDownload>> Process(CancellationToken token, ILog log)
{
log.Debug($"{nameof(BackgroundDownloader)} started");
var errors = new AsyncProducerConsumerQueue<IDownload>();
IDownload item;
int success;
for(success=0; (item = await GetItem(token)) != null; success++)
{
// TODO: add second queue, semaphore and support for retries
try
{
var content = await item.GetContent(vkApi, token);
var file = filesystemTools.CreateUniqueFile(item.Location, item.DesiredName);
await File.WriteAllBytesAsync(file.FullName, content, token);
success++;
log.Info($"Saved [{item.DesiredName}] as [{file.FullName}] with [{content.Length}] bytes");
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e)
{
item.AddError(e);
await errors.EnqueueAsync(item, token);
log.Warn($"Error downloading {item.DesiredName}", e);
}
}
errors.CompleteAdding();
var errorList = errors.GetConsumingEnumerable(token).ToList();
log.Info($"All downloads completed. {success} successful, {errorList.Count} failed");
return errorList;
}
private async Task<IDownload> GetItem(CancellationToken token)
{
try
{
return await queueProvider.Pending.DequeueAsync(token);
}
catch (InvalidOperationException)
{
}
return null;
}
}
}

16
VOffline/Services/Storage/DownloadQueueProvider.cs Normal file
View file

@ -0,0 +1,16 @@
using Nito.AsyncEx;
using VOffline.Models.Storage;
using VOffline.Services.Vk;
namespace VOffline.Services.Storage
{
public class DownloadQueueProvider
{
public DownloadQueueProvider(ConstantsProvider constantsProvider)
{
Pending = new AsyncProducerConsumerQueue<IDownload>(constantsProvider.DownloadQueueLimit);
}
public AsyncProducerConsumerQueue<IDownload> Pending { get; }
}
}

116
VOffline/Services/Storage/FilesystemTools.cs Normal file
View file

@ -0,0 +1,116 @@
using System;
using System.IO;
using System.Linq;
namespace VOffline.Services.Storage
{
public class FilesystemTools
{
public DirectoryInfo MkDir(string path)
{
var dir = new DirectoryInfo(path);
dir.Create();
dir.Refresh();
return dir;
}
public DirectoryInfo CreateSubdir(DirectoryInfo parent, string desiredName, bool resolveCollisions)
{
if (!parent.Exists)
{
throw new DirectoryNotFoundException($"Parent {parent.FullName} does not exist, can not create dir {desiredName}");
}
var validName = MakeValidName(desiredName);
lock (LockObject)
{
var newDir = resolveCollisions
? GetUniqueDirectory(parent, validName)
: new DirectoryInfo(Path.Combine(parent.FullName, validName));
if (newDir.Exists)
{
throw new InvalidOperationException($"Dir already exists: {newDir.FullName}");
}
newDir.Create();
newDir.Refresh();
return newDir;
}
}
public FileInfo CreateUniqueFile(DirectoryInfo parent, string desiredName)
{
if (!parent.Exists)
{
throw new DirectoryNotFoundException($"Parent {parent.FullName} does not exist, can not create file {desiredName}");
}
var validName = MakeValidName(desiredName);
lock (LockObject)
{
var newFile = GetUniqueFile(parent, validName);
if (newFile.Exists)
{
throw new InvalidOperationException($"File already exists: {newFile.FullName}");
}
newFile.Create().Close();
newFile.Refresh();
return newFile;
}
}
/// <summary>
/// Appends number to name in case of collision. Has no side effects. Should be used under lock.
/// </summary>
/// <param name="parent"></param>
/// <param name="validName"></param>
/// <returns></returns>
private static FileInfo GetUniqueFile(DirectoryInfo parent, string validName)
{
var name = Path.GetFileNameWithoutExtension(validName);
var extension = Path.GetExtension(validName);
var fileInfo = new FileInfo(Path.Combine(parent.FullName, validName));
for (var i = 1;; i++)
{
if (!fileInfo.Exists)
{
return fileInfo;
}
fileInfo = new FileInfo(Path.Combine(parent.FullName, $"{name} ({i}){extension}"));
}
}
/// <summary>
/// Appends number to name in case of collision. Has no side effects. Should be used under lock.
/// </summary>
/// <param name="parent"></param>
/// <param name="validName"></param>
/// <returns></returns>
private static DirectoryInfo GetUniqueDirectory(DirectoryInfo parent, string validName)
{
var directoryInfo = new DirectoryInfo(Path.Combine(parent.FullName, validName));
for (var i = 1;; i++)
{
if (!directoryInfo.Exists)
{
return directoryInfo;
}
directoryInfo = new DirectoryInfo(Path.Combine(parent.FullName, $"{validName} ({i})"));
}
}
private static string MakeValidName(string value) => string
.Join("_", value.Split(AllBadChars))
.Trim();
private static readonly char[] AllBadChars =
Path.GetInvalidFileNameChars()
.Concat(Path.GetInvalidPathChars())
.Concat(new[] { '\\', '/', ':' })
.Distinct()
.ToArray();
private static readonly object LockObject = new object();
}
}

4
VOffline/Services/Token/Vk/VkHttpRequests.cs
View file

@ -13,9 +13,9 @@ namespace VOffline.Services.Vk
private readonly string userAgent;
private VkCredentials VkCredentials { get; }
public VkHttpRequests(IOptionsSnapshot<VkCredentials> vkCredentials, UserAgentProvider userAgentProvider)
public VkHttpRequests(IOptionsSnapshot<VkCredentials> vkCredentials, ConstantsProvider constantsProvider)
{
userAgent = userAgentProvider.UserAgent;
userAgent = constantsProvider.UserAgent;
VkCredentials = vkCredentials.Value;
}

8
VOffline/Services/Vk/VkApiUtils.cs
View file

@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography.X509Certificates;
using System.Text.RegularExpressions;
@ -90,9 +91,10 @@ namespace VOffline.Services.Vk
}
public class UserAgentProvider
public class ConstantsProvider
{
public string UserAgent => UserAgentValue;
private const string UserAgentValue = "KateMobileAndroid/51.2 lite-443 (Android 4.4.2; SDK 19; x86; unknown Android SDK built for x86; en)";
public readonly string UserAgent = "KateMobileAndroid/51.2 lite-443 (Android 4.4.2; SDK 19; x86; unknown Android SDK built for x86; en)";
public readonly int DownloadQueueLimit = 5;
//public readonly int DownloadRetryLimit = 3;
}
}

2
VOffline/Services/VkNet/CancellableConstraint.cs
View file

@ -55,7 +55,7 @@ namespace VOffline.Services.Vk
{
await Task.Delay(delay, cts.Token);
}
catch (Exception ex)
catch (Exception)
{
awaitableConstraint._semaphore.Release();
throw;

4
VOffline/Services/VkNet/RestClientWithUserAgent.cs
View file

@ -20,11 +20,11 @@ namespace VOffline.Services.Vk
private string userAgent;
/// <inheritdoc />
public RestClientWithUserAgent(ILogger<RestClient> logger, IWebProxy proxy, UserAgentProvider userAgentProvider)
public RestClientWithUserAgent(ILogger<RestClient> logger, IWebProxy proxy, ConstantsProvider constantsProvider)
{
_logger = logger;
Proxy = proxy;
userAgent = userAgentProvider.UserAgent;
userAgent = constantsProvider.UserAgent;
}
/// <inheritdoc />

24
VOffline/VOffline.csproj
View file

@ -6,6 +6,20 @@
<LangVersion>latest</LangVersion>
</PropertyGroup>
<ItemGroup>
<None Remove="appsettings.json" />
<None Remove="log4net.config" />
</ItemGroup>
<ItemGroup>
<EmbeddedResource Include="appsettings.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</EmbeddedResource>
<EmbeddedResource Include="log4net.config">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</EmbeddedResource>
</ItemGroup>
<ItemGroup>
<PackageReference Include="log4net" Version="2.0.8" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="2.2.0" />
@ -16,17 +30,9 @@
<PackageReference Include="Microsoft.Extensions.Logging.Log4Net.AspNetCore" Version="2.2.8" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="2.1.1" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.1" />
<PackageReference Include="Nito.AsyncEx" Version="5.0.0-pre-05" />
<PackageReference Include="RestSharp" Version="106.6.3" />
<PackageReference Include="VkNet" Version="1.40.0" />
</ItemGroup>
<ItemGroup>
<None Update="appsettings.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="log4net.config">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>

3
VOffline/appsettings.json
View file

@ -3,7 +3,8 @@
//"Targets": [ "id1", "durov", "public147415323", "tech" ],
//"Targets": [ "retrowave" ],
"Targets": [ "rast1234" ],
"Modes": [ "Audio" ]
"Modes": [ "Audio" ],
"OutputPath": "."
},
"VkCredentials": {
//"Login": "username@gmail.com",

6
VOffline/log4net.config
View file

@ -2,7 +2,7 @@
<log4net>
<appender name="ConsoleAppender" type="log4net.Appender.ConsoleAppender" >
<layout type="log4net.Layout.PatternLayout">
<conversionPattern value="%date %-5level %message%newline" />
<conversionPattern value="%date [%3thread] %-5level %message%newline" />
</layout>
</appender>
<appender name="RollingLogFileAppender" type="log4net.Appender.RollingFileAppender">
@ -14,12 +14,12 @@
<maxSizeRollBackups value="100"/>
<maximumFileSize value="15MB"/>
<layout type="log4net.Layout.PatternLayout">
<conversionPattern value="%date [%thread] %-5level App %newline %message %newline %newline"/>
<conversionPattern value="%date [%3thread] %-5level %message%newline"/>
</layout>
</appender>
<root>
<level value="ALL"/>
<!--<appender-ref ref="RollingLogFileAppender"/>-->
<appender-ref ref="RollingLogFileAppender"/>
<appender-ref ref="ConsoleAppender"/>
</root>
</log4net>