diff --git a/VOffline.sln b/VOffline.sln index c589120..c3af502 100644 --- a/VOffline.sln +++ b/VOffline.sln @@ -3,7 +3,7 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio 15 VisualStudioVersion = 15.0.28010.2036 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VOffline", "VOffline\VOffline.csproj", "{4EBB18FF-457C-4F6D-98E4-A44AEAF59CDD}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "VOffline", "VOffline\VOffline.csproj", "{4EBB18FF-457C-4F6D-98E4-A44AEAF59CDD}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/VOffline/Models/Settings.cs b/VOffline/Models/Settings.cs index 875d27f..21b2e27 100644 --- a/VOffline/Models/Settings.cs +++ b/VOffline/Models/Settings.cs @@ -10,6 +10,7 @@ namespace VOffline.Models { public List Targets { get; set; } public List Modes { get; set; } + public string OutputPath { get; set; } public ImmutableHashSet GetWorkingModes() { diff --git a/VOffline/Models/Storage/Download.cs b/VOffline/Models/Storage/Download.cs new file mode 100644 index 0000000..6d134fa --- /dev/null +++ b/VOffline/Models/Storage/Download.cs @@ -0,0 +1,90 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using RestSharp; +using VkNet; +using VOffline.Services.Handlers; + +namespace VOffline.Models.Storage +{ + public interface IDownload + { + DirectoryInfo Location { get; } + string DesiredName { get; } + int RetryCount { get; } + IReadOnlyList Errors { get; } + void AddError(Exception e); + Task GetContent(VkApi vkApi, CancellationToken token); + } + + public class Download : IDownload + { + private readonly List errors; + + public Download(Uri uri, DirectoryInfo location, string desiredName) + { + errors = new List(); + Uri = uri ?? throw new ArgumentNullException(nameof(uri)); + Location = location ?? throw new ArgumentNullException(nameof(location)); + DesiredName = desiredName ?? throw new ArgumentNullException(nameof(desiredName)); + } + + public Uri Uri { get; } + public DirectoryInfo Location { get; } + public string DesiredName { get; } + + public int RetryCount => errors.Count; + + public IReadOnlyList Errors => errors; + + public void AddError(Exception e) + { + errors.Add(e); + } + + public async Task GetContent(VkApi vkApi, CancellationToken token) + { + var client = new RestClient($"{Uri.Scheme}://{Uri.Authority}"); + var response = await client.ExecuteGetTaskAsync(new RestRequest(Uri.PathAndQuery), token); + response.ThrowIfSomethingWrong(); + return response.RawBytes; + } + } + + public class LyricsVkDownload : IDownload + { + private readonly long lyricsId; + private readonly List errors; + + public LyricsVkDownload(long lyricsId, DirectoryInfo location, string desiredName) + { + this.lyricsId = lyricsId; + errors = new List(); + Location = location ?? throw new ArgumentNullException(nameof(location)); + DesiredName = desiredName ?? throw new ArgumentNullException(nameof(desiredName)); + } + + public DirectoryInfo Location { get; } + public string DesiredName { get; } + + public int RetryCount => errors.Count; + + public IReadOnlyList Errors => errors; + + public void AddError(Exception e) + { + errors.Add(e); + } + + public async Task GetContent(VkApi vkApi, CancellationToken token) + { + var lyrics = await vkApi.Audio.GetLyricsAsync(lyricsId); + return !string.IsNullOrWhiteSpace(lyrics?.Text) + ? new byte[] { } // TODO: fix this to provide a no-value without exceptions + : Encoding.UTF8.GetBytes(lyrics.Text); + } + } +} \ No newline at end of file diff --git a/VOffline/Models/Storage/Playlist.cs b/VOffline/Models/Storage/Playlist.cs index 3e2d84c..3580876 100644 --- a/VOffline/Models/Storage/Playlist.cs +++ b/VOffline/Models/Storage/Playlist.cs @@ -1,6 +1,9 @@ using System; using System.Collections.Generic; +using System.IO; +using System.Linq; using VkNet.Model; +using VOffline.Services.Storage; namespace VOffline.Models.Storage { @@ -31,6 +34,22 @@ namespace VOffline.Models.Storage public IReadOnlyList Tracks { get; set; } + public IEnumerable ToDownloads(FilesystemTools filesystemTools, DirectoryInfo dir) + { + var playlistDir = filesystemTools.CreateSubdir(dir, Name, true); + if (Cover != null) + { + // TODO: i guess it's always jpeg? + var ext = Path.HasExtension(Cover.AbsoluteUri) ? Path.GetExtension(Cover.AbsoluteUri) : ".jpg"; + yield return new Download(Cover, playlistDir, $"playlist_cover{ext}"); + } + + foreach (var download in Tracks.SelectMany(t => t.ToDownloads(filesystemTools, playlistDir))) + { + yield return download; + } + } + private static Uri GetBestImage(AudioCover cover) { // TODO: theoretically images could be different or missing, but looks like this works fine diff --git a/VOffline/Models/Storage/Track.cs b/VOffline/Models/Storage/Track.cs index 87c2733..80ea05f 100644 --- a/VOffline/Models/Storage/Track.cs +++ b/VOffline/Models/Storage/Track.cs @@ -1,26 +1,48 @@ using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; using VkNet.Model; using VkNet.Model.Attachments; +using VOffline.Services.Storage; namespace VOffline.Models.Storage { public class Track { - public long Id { get; set; } - public string Name { get; set; } - public string Artist { get; set; } - public Uri Url { get; set; } - public bool? Hq { get; set; } - public string Lyrics { get; set; } + public long Id { get; } + public string Name { get; } + public string Artist { get; } + public Uri Url { get; } + public bool? Hq { get; } + public long? LyricsId { get; } - public Track(Audio audio, Lyrics lyrics) + public Track(Audio audio) { Id = audio.Id.Value; Name = audio.Title; Artist = audio.Artist; Url = audio.Url; Hq = audio.IsHq; - Lyrics = lyrics?.Text; + LyricsId = audio.LyricsId; + } + + public IEnumerable ToDownloads(FilesystemTools filesystemTools, DirectoryInfo dir) + { + var trackName = string.Join(" - ", new[] { Artist, Name }.Where(x => !string.IsNullOrEmpty(x))); + if (Url != null) + { + yield return new Download(Url, dir, $"{trackName}.mp3"); + } + else + { + filesystemTools.CreateUniqueFile(dir, $"{trackName}.mp3.deleted"); + } + + if (LyricsId != null) + { + yield return new LyricsVkDownload(LyricsId.Value, dir, $"{trackName}.txt"); + } } } } \ No newline at end of file diff --git a/VOffline/Program.cs b/VOffline/Program.cs index 9a82dd3..e54181c 100644 --- a/VOffline/Program.cs +++ b/VOffline/Program.cs @@ -23,6 +23,7 @@ using VOffline.Models.Vk; using VOffline.Services; using VOffline.Services.Google; using VOffline.Services.Handlers; +using VOffline.Services.Storage; using VOffline.Services.Vk; namespace VOffline @@ -54,8 +55,11 @@ namespace VOffline serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); - serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); serviceCollection.AddSingleton(_ => VkApiFactory(token)); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); serviceCollection.AddTransient(provider => LogManager.GetLogger(Assembly.GetEntryAssembly(), typeof(Program))); serviceCollection.AddTransient(); @@ -67,6 +71,7 @@ namespace VOffline var services = serviceCollection.BuildServiceProvider(); await services.GetRequiredService().Run(token, log); + return 0; } catch (Exception e) @@ -80,7 +85,7 @@ namespace VOffline { // VkNet uses its own DI for internal services var sc = new ServiceCollection(); - sc.AddSingleton(); + sc.AddSingleton(); sc.AddSingleton(); sc.AddSingleton(_ => new CancellableConstraint(3, TimeSpan.FromSeconds(1.3), token)); sc.AddSingleton(); @@ -89,7 +94,7 @@ namespace VOffline { builder.ClearProviders(); builder.SetMinimumLevel(LogLevel.Debug); - builder.AddLog4Net(); + //builder.AddLog4Net(); }); return new VkApi(sc); } diff --git a/VOffline/Services/Filesystem/Storage.cs b/VOffline/Services/Filesystem/Storage.cs deleted file mode 100644 index ee74428..0000000 --- a/VOffline/Services/Filesystem/Storage.cs +++ /dev/null @@ -1,117 +0,0 @@ -using System; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using VkNet.Model.RequestParams; - -namespace VOffline.Services -{ - public class Storage - { - private readonly DirectoryInfo rootDir; - - public string FullPath => rootDir.FullName; - - public Storage(string rootName) - { - rootDir = new DirectoryInfo(FilterFilename(rootName)); - lock (LockObject) - { - if (rootDir.Exists) - { - throw new InvalidOperationException($"Storage directory [{rootDir.Name}] already exists"); - } - rootDir.Create(); - } - } - - private Storage(string rootName, bool useExisting) - { - rootDir = new DirectoryInfo(rootName); - lock (LockObject) - { - if (!useExisting && rootDir.Exists) - { - throw new InvalidOperationException($"Storage directory [{rootDir.Name}] already exists"); - } - rootDir.Create(); - } - } - - public Storage Descend(string subName, bool makeUniqueName) - { - var filteredName = FilterFilename(subName); - if (Path.IsPathRooted(filteredName)) - { - throw new InvalidOperationException($"Expected relative path, got absolute: [{filteredName}]"); - } - - // TODO: looks like cancer. also probable race conditions? - if (makeUniqueName) - { - var uniqueName = GetUniqueDirectory(filteredName); - return new Storage(Path.Combine(rootDir.FullName, uniqueName.Name), true); - } - return new Storage(Path.Combine(rootDir.FullName, filteredName), false); - } - - /// - /// Creates unique file - /// - /// - /// - public FileInfo GetFile(string name) - { - var filename = FilterFilename(name); - var file = GetUniqueFile(filename); - return file; - } - - private FileInfo GetUniqueFile(string filteredName) - { - var name = Path.GetFileNameWithoutExtension(filteredName); - var extension = Path.GetExtension(filteredName); - var fileInfo = new FileInfo(Path.Combine(rootDir.FullName, filteredName)); - for (var i = 1;; i++) - { - lock (LockObject) - { - if (!fileInfo.Exists) - { - fileInfo.Create().Close(); - return fileInfo; - } - } - fileInfo = new FileInfo(Path.Combine(rootDir.FullName, $"{name} ({i}){extension}")); - } - } - - private DirectoryInfo GetUniqueDirectory(string filteredName) - { - var directoryInfo = new DirectoryInfo(Path.Combine(rootDir.FullName, filteredName)); - for (var i = 1; ; i++) - { - lock (LockObject) - { - if (!directoryInfo.Exists) - { - directoryInfo.Create(); - return directoryInfo; - } - } - directoryInfo = new DirectoryInfo(Path.Combine(rootDir.FullName, $"{filteredName} ({i})")); - } - } - - private static string FilterFilename(string value) => string.Join("_", value.Split(AllBadChars)).Trim(); - - private static readonly char[] AllBadChars = - Path.GetInvalidFileNameChars() - .Concat(Path.GetInvalidPathChars()) - .Concat(new[] {'\\', '/', ':'}) - .Distinct() - .ToArray(); - - private static readonly object LockObject = new object(); - } -} \ No newline at end of file diff --git a/VOffline/Services/Handlers/Audio.cs b/VOffline/Services/Handlers/Audio.cs index ba8dbd8..f81ac31 100644 --- a/VOffline/Services/Handlers/Audio.cs +++ b/VOffline/Services/Handlers/Audio.cs @@ -7,6 +7,7 @@ using System.Threading; using System.Threading.Tasks; using log4net; using Newtonsoft.Json; +using Nito.AsyncEx; using RestSharp; using VkNet; using VkNet.Model; @@ -14,6 +15,7 @@ using VkNet.Model.Attachments; using VkNet.Model.RequestParams; using VkNet.Utils; using VOffline.Models.Storage; +using VOffline.Services.Storage; using VOffline.Services.Vk; using RestClient = RestSharp.RestClient; @@ -22,56 +24,25 @@ namespace VOffline.Services.Handlers public class AudioHandler { private readonly VkApi vkApi; + private readonly FilesystemTools filesystemTools; + private readonly DownloadQueueProvider downloadQueueProvider; - public AudioHandler(VkApi vkApi) + public AudioHandler(VkApi vkApi, FilesystemTools filesystemTools, DownloadQueueProvider downloadQueueProvider) { this.vkApi = vkApi; + this.filesystemTools = filesystemTools; + this.downloadQueueProvider = downloadQueueProvider; } - public async Task ProcessAudio(long id, Storage storage, CancellationToken token, ILog log) + public async Task ProcessAudio(long id, DirectoryInfo dir, CancellationToken token, ILog log) { + // TODO: continuation/recovery? at least on album level? maybe create and detect some 'completeness' file? var allPlaylists = await GetAllPlaylists(id, token, log); - var throttler = new Throttler(); - foreach (var playlist in allPlaylists) - { - var playlistStorage = storage.Descend(playlist.Name, true); - var downloadTasks = playlist.Tracks - .Select(async track => await Download(track, playlistStorage, token, log)) - .ToArray(); - await throttler.ProcessWithThrottling(downloadTasks, 3, token); - } - } - - private async Task Download(Track track, Storage playlistStorage, CancellationToken token, ILog log) - { - return; - var trackName = string.Join(" - ", new[] { track.Artist, track.Name }.Where(x => !string.IsNullOrEmpty(x))); - if (track.Url != null) - { - var content = await Retrier.Retry(async () => - { - var client = new RestClient($"{track.Url.Scheme}://{track.Url.Authority}"); - var response = await client.ExecuteGetTaskAsync(new RestRequest(track.Url.PathAndQuery), token); - response.ThrowIfSomethingWrong(); - - return response.RawBytes; - }, 3, TimeSpan.FromSeconds(5), token, log); - - var file = playlistStorage.GetFile($"{trackName}.mp3").FullName; - log.Debug($"Saving {file}"); - await File.WriteAllBytesAsync(file, content, token); - } - else - { - playlistStorage.GetFile($"{trackName}.mp3.deleted"); - } - - if (track.Lyrics != null) - { - var lyricsFile = playlistStorage.GetFile($"{trackName}.txt").FullName; - await File.WriteAllTextAsync(lyricsFile, track.Lyrics, token); - } + var allDownloadTasks = allPlaylists + .SelectMany(p => p.ToDownloads(filesystemTools, dir)) + .Select(d => downloadQueueProvider.Pending.EnqueueAsync(d, token)); + await Task.WhenAll(allDownloadTasks); } private async Task> GetAllPlaylists(long id, CancellationToken token, ILog log) @@ -101,8 +72,7 @@ namespace VOffline.Services.Handlers var audioTasks = vkAudios.Select(async audio => { token.ThrowIfCancellationRequested(); - var lyrics = await GetLyrics(audio, log); - return new Track(audio, lyrics); + return new Track(audio); }); var tracks = await Task.WhenAll(audioTasks); return new Playlist(playlist, tracks); @@ -113,20 +83,18 @@ namespace VOffline.Services.Handlers public async Task> GetTracks(long id, HashSet tracksInPlaylists, CancellationToken token, ILog log) { // TODO: handle paging if api returns partial result - log.Debug("Getting all tracks not in playlists"); var vkAudios = await vkApi.Audio.GetAsync(new AudioGetParams() { OwnerId = id }); + log.Debug($"Got {vkAudios.Count} tracks not in playlists for {id}"); ThrowIfCountMismatch(vkAudios.TotalCount, vkAudios.Count); var trackTasks = vkAudios .Where(t => !tracksInPlaylists.Contains(t.Id.Value)) .Select(async audio => { - // TODO: filter out extra tracks here - var lyrics = await GetLyrics(audio, log); token.ThrowIfCancellationRequested(); - return new Track(audio, lyrics); + return new Track(audio); }); return await Task.WhenAll(trackTasks); } @@ -135,6 +103,7 @@ namespace VOffline.Services.Handlers { var pageSize = 200; var playlistResponse = await vkApi.Audio.GetPlaylistsAsync(id, (uint)pageSize); + log.Debug($"Got {playlistResponse.Count}/{playlistResponse.TotalCount} tracks in playlist {id}"); var total = (int)playlistResponse.TotalCount; var result = new List(total); @@ -145,8 +114,9 @@ namespace VOffline.Services.Handlers .Select(pageNumber => pageNumber * pageSize) .Select(async offset => { - log.Debug($"Playlists at offset {offset}"); + //log.Debug($"Playlists at offset {offset}"); var page = await vkApi.Audio.GetPlaylistsAsync(id, (uint)pageSize, (uint)offset); + log.Debug($"Got {page.Count}/{playlistResponse.TotalCount} tracks in playlist {id} at offset {offset}"); token.ThrowIfCancellationRequested(); // not sure if this is needed here return page; }); @@ -161,24 +131,22 @@ namespace VOffline.Services.Handlers private async Task> ExpandPlaylist(AudioPlaylist playlist, ILog log) { - log.Debug($"Expanding {playlist.Title}"); + var vkAudios = await vkApi.Audio.GetAsync(new AudioGetParams() { AlbumId = playlist.Id, OwnerId = playlist.OwnerId, }); + log.Debug($"Expanded {playlist.Title}: {vkAudios.TotalCount}"); ThrowIfCountMismatch(vkAudios.TotalCount, vkAudios.Count); return vkAudios; } - private async Task GetLyrics(Audio audio, ILog log) + private async Task GetLyrics(long lyricsId, ILog log) { - if (audio.LyricsId == null) - { - return null; - } - log.Debug($"Lyrics for {audio.Title}"); - return await vkApi.Audio.GetLyricsAsync(audio.LyricsId.Value); + var lyrics = await vkApi.Audio.GetLyricsAsync(lyricsId); + log.Debug($"Got lyrics for {lyricsId}"); + return lyrics; } private static void ThrowIfCountMismatch(decimal expectedTotal, decimal resultCount) diff --git a/VOffline/Services/Logic.cs b/VOffline/Services/Logic.cs index e45a5aa..e592cef 100644 --- a/VOffline/Services/Logic.cs +++ b/VOffline/Services/Logic.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; +using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -13,6 +14,7 @@ using VkNet.Model.RequestParams; using VOffline.Models; using VOffline.Models.Storage; using VOffline.Services.Handlers; +using VOffline.Services.Storage; using VOffline.Services.Vk; namespace VOffline.Services @@ -23,14 +25,18 @@ namespace VOffline.Services private readonly TokenMagic tokenMagic; private readonly VkApi vkApi; private readonly VkApiUtils vkApiUtils; + private readonly BackgroundDownloader downloader; + private readonly FilesystemTools filesystemTools; private readonly AudioHandler audioHandler; - public Logic(TokenMagic tokenMagic, VkApi vkApi, VkApiUtils vkApiUtils, AudioHandler audioHandler, IOptionsSnapshot settings) + public Logic(TokenMagic tokenMagic, VkApi vkApi, VkApiUtils vkApiUtils, BackgroundDownloader downloader, FilesystemTools filesystemTools, AudioHandler audioHandler, IOptionsSnapshot settings) { this.settings = settings.Value; this.tokenMagic = tokenMagic; this.vkApi = vkApi; this.vkApiUtils = vkApiUtils; + this.downloader = downloader; + this.filesystemTools = filesystemTools; this.audioHandler = audioHandler; } @@ -54,35 +60,37 @@ namespace VOffline.Services .Distinct() .ToImmutableHashSet(); log.Debug($"Processing {JsonConvert.SerializeObject(modes)} for {JsonConvert.SerializeObject(ids)}"); + var downloaderTask = downloader.Process(token, log); + var rootDir = filesystemTools.MkDir(settings.OutputPath); foreach (var id in ids) { + var name = await vkApiUtils.GetName(id); - var storage = new Storage(name); - log.Info($"id [{id}], name [{name}], path [{storage.FullPath}]"); + var workDir = filesystemTools.CreateSubdir(rootDir, name, false); + log.Info($"id [{id}], name [{name}], path [{workDir.FullName}]"); foreach (var mode in modes) { - var subStorage = storage.Descend(mode.ToString(), false); - await ProcessTarget(id, subStorage, mode, token, log); + var modeDir = filesystemTools.CreateSubdir(workDir, mode.ToString(), false); + await ProcessTarget(id, modeDir, mode, token, log); } } - //var audio = vkApi.Audio.Get(new AudioGetParams() - //{ - // OwnerId = 1 - //}); - //log.Debug(JsonConvert.SerializeObject(audio, Formatting.Indented)); - + var downloadErrors = await downloaderTask; + foreach (var downloadError in downloadErrors) + { + log.Warn($"Failed {downloadError.DesiredName}", downloadError.Errors.LastOrDefault()); + } } - private async Task ProcessTarget(long id, Storage storage, Mode mode, CancellationToken token, ILog log) + private async Task ProcessTarget(long id, DirectoryInfo dir, Mode mode, CancellationToken token, ILog log) { switch (mode) { case Mode.Wall: break; case Mode.Audio: - await audioHandler.ProcessAudio(id, storage, token, log); + await audioHandler.ProcessAudio(id, dir, token, log); break; case Mode.All: throw new ArgumentOutOfRangeException(nameof(mode), mode, "This mode should have been replaced before processing"); diff --git a/VOffline/Services/Storage/BackgroundDownloader.cs b/VOffline/Services/Storage/BackgroundDownloader.cs new file mode 100644 index 0000000..4605d81 --- /dev/null +++ b/VOffline/Services/Storage/BackgroundDownloader.cs @@ -0,0 +1,80 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using log4net; +using Nito.AsyncEx; +using RestSharp; +using VkNet; +using VOffline.Models.Storage; +using VOffline.Services.Handlers; + +namespace VOffline.Services.Storage +{ + public class BackgroundDownloader + { + private readonly DownloadQueueProvider queueProvider; + private readonly FilesystemTools filesystemTools; + + private readonly VkApi vkApi; + //private readonly int retryLimit; + + public BackgroundDownloader(DownloadQueueProvider queueProvider, FilesystemTools filesystemTools, VkApi vkApi) + { + this.queueProvider = queueProvider; + this.filesystemTools = filesystemTools; + this.vkApi = vkApi; + } + + public async Task> Process(CancellationToken token, ILog log) + { + log.Debug($"{nameof(BackgroundDownloader)} started"); + var errors = new AsyncProducerConsumerQueue(); + IDownload item; + int success; + + for(success=0; (item = await GetItem(token)) != null; success++) + { + // TODO: add second queue, semaphore and support for retries + try + { + var content = await item.GetContent(vkApi, token); + var file = filesystemTools.CreateUniqueFile(item.Location, item.DesiredName); + await File.WriteAllBytesAsync(file.FullName, content, token); + success++; + log.Info($"Saved [{item.DesiredName}] as [{file.FullName}] with [{content.Length}] bytes"); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception e) + { + item.AddError(e); + await errors.EnqueueAsync(item, token); + log.Warn($"Error downloading {item.DesiredName}", e); + } + } + + errors.CompleteAdding(); + var errorList = errors.GetConsumingEnumerable(token).ToList(); + log.Info($"All downloads completed. {success} successful, {errorList.Count} failed"); + return errorList; + } + + private async Task GetItem(CancellationToken token) + { + try + { + return await queueProvider.Pending.DequeueAsync(token); + } + catch (InvalidOperationException) + { + } + + return null; + } + } +} \ No newline at end of file diff --git a/VOffline/Services/Storage/DownloadQueueProvider.cs b/VOffline/Services/Storage/DownloadQueueProvider.cs new file mode 100644 index 0000000..5b854d9 --- /dev/null +++ b/VOffline/Services/Storage/DownloadQueueProvider.cs @@ -0,0 +1,16 @@ +using Nito.AsyncEx; +using VOffline.Models.Storage; +using VOffline.Services.Vk; + +namespace VOffline.Services.Storage +{ + public class DownloadQueueProvider + { + public DownloadQueueProvider(ConstantsProvider constantsProvider) + { + Pending = new AsyncProducerConsumerQueue(constantsProvider.DownloadQueueLimit); + } + + public AsyncProducerConsumerQueue Pending { get; } + } +} \ No newline at end of file diff --git a/VOffline/Services/Storage/FilesystemTools.cs b/VOffline/Services/Storage/FilesystemTools.cs new file mode 100644 index 0000000..fc0a6a7 --- /dev/null +++ b/VOffline/Services/Storage/FilesystemTools.cs @@ -0,0 +1,116 @@ +using System; +using System.IO; +using System.Linq; + +namespace VOffline.Services.Storage +{ + public class FilesystemTools + { + public DirectoryInfo MkDir(string path) + { + var dir = new DirectoryInfo(path); + dir.Create(); + dir.Refresh(); + return dir; + } + + public DirectoryInfo CreateSubdir(DirectoryInfo parent, string desiredName, bool resolveCollisions) + { + if (!parent.Exists) + { + throw new DirectoryNotFoundException($"Parent {parent.FullName} does not exist, can not create dir {desiredName}"); + } + + var validName = MakeValidName(desiredName); + lock (LockObject) + { + var newDir = resolveCollisions + ? GetUniqueDirectory(parent, validName) + : new DirectoryInfo(Path.Combine(parent.FullName, validName)); + if (newDir.Exists) + { + throw new InvalidOperationException($"Dir already exists: {newDir.FullName}"); + } + newDir.Create(); + newDir.Refresh(); + return newDir; + } + } + + public FileInfo CreateUniqueFile(DirectoryInfo parent, string desiredName) + { + if (!parent.Exists) + { + throw new DirectoryNotFoundException($"Parent {parent.FullName} does not exist, can not create file {desiredName}"); + } + + var validName = MakeValidName(desiredName); + lock (LockObject) + { + var newFile = GetUniqueFile(parent, validName); + if (newFile.Exists) + { + throw new InvalidOperationException($"File already exists: {newFile.FullName}"); + } + newFile.Create().Close(); + newFile.Refresh(); + return newFile; + } + } + + /// + /// Appends number to name in case of collision. Has no side effects. Should be used under lock. + /// + /// + /// + /// + private static FileInfo GetUniqueFile(DirectoryInfo parent, string validName) + { + var name = Path.GetFileNameWithoutExtension(validName); + var extension = Path.GetExtension(validName); + var fileInfo = new FileInfo(Path.Combine(parent.FullName, validName)); + for (var i = 1;; i++) + { + if (!fileInfo.Exists) + { + return fileInfo; + } + + fileInfo = new FileInfo(Path.Combine(parent.FullName, $"{name} ({i}){extension}")); + } + } + + /// + /// Appends number to name in case of collision. Has no side effects. Should be used under lock. + /// + /// + /// + /// + private static DirectoryInfo GetUniqueDirectory(DirectoryInfo parent, string validName) + { + var directoryInfo = new DirectoryInfo(Path.Combine(parent.FullName, validName)); + for (var i = 1;; i++) + { + if (!directoryInfo.Exists) + { + return directoryInfo; + } + + directoryInfo = new DirectoryInfo(Path.Combine(parent.FullName, $"{validName} ({i})")); + } + } + + private static string MakeValidName(string value) => string + .Join("_", value.Split(AllBadChars)) + .Trim(); + + private static readonly char[] AllBadChars = + Path.GetInvalidFileNameChars() + .Concat(Path.GetInvalidPathChars()) + .Concat(new[] { '\\', '/', ':' }) + .Distinct() + .ToArray(); + + private static readonly object LockObject = new object(); + } +} \ No newline at end of file diff --git a/VOffline/Services/Token/Vk/VkHttpRequests.cs b/VOffline/Services/Token/Vk/VkHttpRequests.cs index ab74996..41dd97c 100644 --- a/VOffline/Services/Token/Vk/VkHttpRequests.cs +++ b/VOffline/Services/Token/Vk/VkHttpRequests.cs @@ -13,9 +13,9 @@ namespace VOffline.Services.Vk private readonly string userAgent; private VkCredentials VkCredentials { get; } - public VkHttpRequests(IOptionsSnapshot vkCredentials, UserAgentProvider userAgentProvider) + public VkHttpRequests(IOptionsSnapshot vkCredentials, ConstantsProvider constantsProvider) { - userAgent = userAgentProvider.UserAgent; + userAgent = constantsProvider.UserAgent; VkCredentials = vkCredentials.Value; } diff --git a/VOffline/Services/Vk/VkApiUtils.cs b/VOffline/Services/Vk/VkApiUtils.cs index e77e44a..697be1f 100644 --- a/VOffline/Services/Vk/VkApiUtils.cs +++ b/VOffline/Services/Vk/VkApiUtils.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Security.Cryptography.X509Certificates; using System.Text.RegularExpressions; @@ -90,9 +91,10 @@ namespace VOffline.Services.Vk } - public class UserAgentProvider + public class ConstantsProvider { - public string UserAgent => UserAgentValue; - private const string UserAgentValue = "KateMobileAndroid/51.2 lite-443 (Android 4.4.2; SDK 19; x86; unknown Android SDK built for x86; en)"; + public readonly string UserAgent = "KateMobileAndroid/51.2 lite-443 (Android 4.4.2; SDK 19; x86; unknown Android SDK built for x86; en)"; + public readonly int DownloadQueueLimit = 5; + //public readonly int DownloadRetryLimit = 3; } } \ No newline at end of file diff --git a/VOffline/Services/VkNet/CancellableConstraint.cs b/VOffline/Services/VkNet/CancellableConstraint.cs index 20c37ca..ff908be 100644 --- a/VOffline/Services/VkNet/CancellableConstraint.cs +++ b/VOffline/Services/VkNet/CancellableConstraint.cs @@ -55,7 +55,7 @@ namespace VOffline.Services.Vk { await Task.Delay(delay, cts.Token); } - catch (Exception ex) + catch (Exception) { awaitableConstraint._semaphore.Release(); throw; diff --git a/VOffline/Services/VkNet/RestClientWithUserAgent.cs b/VOffline/Services/VkNet/RestClientWithUserAgent.cs index 10d4c7c..f6e17e0 100644 --- a/VOffline/Services/VkNet/RestClientWithUserAgent.cs +++ b/VOffline/Services/VkNet/RestClientWithUserAgent.cs @@ -20,11 +20,11 @@ namespace VOffline.Services.Vk private string userAgent; /// - public RestClientWithUserAgent(ILogger logger, IWebProxy proxy, UserAgentProvider userAgentProvider) + public RestClientWithUserAgent(ILogger logger, IWebProxy proxy, ConstantsProvider constantsProvider) { _logger = logger; Proxy = proxy; - userAgent = userAgentProvider.UserAgent; + userAgent = constantsProvider.UserAgent; } /// diff --git a/VOffline/VOffline.csproj b/VOffline/VOffline.csproj index a21ff49..fb46557 100644 --- a/VOffline/VOffline.csproj +++ b/VOffline/VOffline.csproj @@ -6,6 +6,20 @@ latest + + + + + + + + Always + + + Always + + + @@ -16,17 +30,9 @@ + - - - Always - - - Always - - - diff --git a/VOffline/appsettings.json b/VOffline/appsettings.json index 3291750..d77ffba 100644 --- a/VOffline/appsettings.json +++ b/VOffline/appsettings.json @@ -3,7 +3,8 @@ //"Targets": [ "id1", "durov", "public147415323", "tech" ], //"Targets": [ "retrowave" ], "Targets": [ "rast1234" ], - "Modes": [ "Audio" ] + "Modes": [ "Audio" ], + "OutputPath": "." }, "VkCredentials": { //"Login": "username@gmail.com", diff --git a/VOffline/log4net.config b/VOffline/log4net.config index 6521fcd..55dbfb1 100644 --- a/VOffline/log4net.config +++ b/VOffline/log4net.config @@ -2,7 +2,7 @@ - + @@ -14,12 +14,12 @@ - + - + \ No newline at end of file