vrr/VOffline
2
0
Fork 0
mirror of https://github.com/Rast1234/VOffline.git synced 2026-04-28 03:49:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

WIP handlers DI recursion break

Browse source
This commit is contained in:
Rostislav Kirillov 2019-01-17 02:16:50 +05:00
parent cce04fcf8e
commit 2f63d801c2
18 changed files with 168 additions and 149 deletions
Download patch file Download diff file Expand all files Collapse all files

2
VOffline/Models/Settings.cs
View file

@ -30,6 +30,6 @@ namespace VOffline.Models
public string UserAgent { get; set; } = "KateMobileAndroid/51.2 lite-443 (Android 4.4.2; SDK 19; x86; unknown Android SDK built for x86; en)";
public int RequestRetryCount { get; set; } = 3;
public TimeSpan RequestRetryDelay { get; set; } = TimeSpan.FromSeconds(3);
public int DownloadQueueLimit { get; set; } = 100;
public int DownloadQueueLimit { get; set; } = 0;
}
}

21
VOffline/Program.cs
View file

@ -15,8 +15,11 @@ using Newtonsoft.Json.Converters;
using VkNet;
using VkNet.Abstractions.Core;
using VkNet.Abstractions.Utils;
using VkNet.Model;
using VkNet.Model.Attachments;
using VOffline.Models;
using VOffline.Models.Google;
using VOffline.Models.Storage;
using VOffline.Models.Vk;
using VOffline.Services;
using VOffline.Services.Handlers;
@ -63,15 +66,16 @@ namespace VOffline
serviceCollection.AddSingleton<DownloadQueueProvider>();
serviceCollection.AddSingleton<BackgroundDownloader>();
serviceCollection.AddSingleton<WallHandler>();
serviceCollection.AddSingleton<PostHandler>();
serviceCollection.AddSingleton<CommentsHandler>();
serviceCollection.AddSingleton<CommentHandler>();
serviceCollection.AddSingleton<AudioHandler>();
serviceCollection.AddSingleton<PlaylistHandler>();
serviceCollection.AddSingleton<PhotoHandler>();
serviceCollection.AddSingleton<AlbumHandler>();
serviceCollection.AddSingleton<WallHandler>();
serviceCollection.AddSingleton<CommentsHandler>();
serviceCollection.AddSingleton<IHandler<Post>, PostHandler>();
serviceCollection.AddSingleton<IHandler<Comment>, CommentHandler>();
serviceCollection.AddSingleton<IHandler<PlaylistWithAudio>, PlaylistHandler>();
serviceCollection.AddSingleton<IHandler<AlbumWithPhoto>, AlbumHandler>();
serviceCollection.AddSingleton<AttachmentProcessor>();
serviceCollection.AddSingleton<IServiceProvider>(s => s); // hack to avoid circular deps between AttachmentProcessor and Handlers
serviceCollection.AddTransient(provider => LogManager.GetLogger(Assembly.GetEntryAssembly(), typeof(Program)));
serviceCollection.AddTransient<AndroidAuth>();
@ -84,11 +88,6 @@ namespace VOffline
await services.GetRequiredService<Logic>().Run(cts.Token, log);
return 0;
}
catch (TaskCanceledException)
{
log.Warn($"Canceled by user");
return -2;
}
catch (Exception e)
{
log.Fatal(e);

8
VOffline/Services/Handlers/AlbumHandler.cs
View file

@ -21,8 +21,8 @@ namespace VOffline.Services.Handlers
{
if (!string.IsNullOrWhiteSpace(albumWithPhoto.Album.Description))
{
var text = filesystemTools.CreateFile(workDir, $"__description.txt", CreateMode.MergeWithExisting);
File.WriteAllText(text.FullName, albumWithPhoto.Album.Description);
var text = filesystemTools.CreateFile(workDir, $"__description.txt", CreateMode.OverwriteExisting);
await File.WriteAllTextAsync(text.FullName, albumWithPhoto.Album.Description, token);
}
if (albumWithPhoto.Album.ThumbId != null)
@ -34,10 +34,10 @@ namespace VOffline.Services.Handlers
}
}
var attachmentTasks = albumWithPhoto.Photo.Select((a, i) => attachmentProcessor.ProcessAttachment(a, i, workDir, token, log));
var attachmentTasks = albumWithPhoto.Photo.Select((a, i) => attachmentProcessor.ProcessAttachment(a, i+1, workDir, token, log));
await Task.WhenAll(attachmentTasks);
}
public override DirectoryInfo GetWorkingDirectory(AlbumWithPhoto albumWithPhoto, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, $"{albumWithPhoto.Album.Title}", CreateMode.MergeWithExisting);
public override DirectoryInfo GetWorkingDirectory(AlbumWithPhoto albumWithPhoto, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, $"{albumWithPhoto.Album.Title}", CreateMode.OverwriteExisting);
}
}

6
VOffline/Services/Handlers/AudioHandler.cs
View file

@ -12,9 +12,9 @@ namespace VOffline.Services.Handlers
public class AudioHandler : HandlerBase<long>
{
private readonly VkApiUtils vkApiUtils;
private readonly PlaylistHandler playlistHandler;
private readonly IHandler<PlaylistWithAudio> playlistHandler;
public AudioHandler(VkApiUtils vkApiUtils, FilesystemTools filesystemTools, PlaylistHandler playlistHandler) : base(filesystemTools)
public AudioHandler(VkApiUtils vkApiUtils, FilesystemTools filesystemTools, IHandler<PlaylistWithAudio> playlistHandler) : base(filesystemTools)
{
this.vkApiUtils = vkApiUtils;
this.playlistHandler = playlistHandler;
@ -47,6 +47,6 @@ namespace VOffline.Services.Handlers
await Task.WhenAll(allTasks);
}
public override DirectoryInfo GetWorkingDirectory(long id, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, "Audio", CreateMode.MergeWithExisting);
public override DirectoryInfo GetWorkingDirectory(long id, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, "Audio", CreateMode.OverwriteExisting);
}
}

4
VOffline/Services/Handlers/CommentHandler.cs
View file

@ -20,10 +20,10 @@ namespace VOffline.Services.Handlers
public override async Task ProcessInternal(Comment comment, DirectoryInfo workDir, CancellationToken token, ILog log)
{
var attachmentTasks = comment.Attachments.Select((a, i) => attachmentProcessor.ProcessAttachment((Attachment) a, i, workDir, token, log));
var attachmentTasks = comment.Attachments.Select((a, i) => attachmentProcessor.ProcessAttachment(a, i+1, workDir, token, log));
await Task.WhenAll(attachmentTasks);
}
public override DirectoryInfo GetWorkingDirectory(Comment comment, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, $"{comment.Date.Value:s} {comment.Id}", CreateMode.MergeWithExisting);
public override DirectoryInfo GetWorkingDirectory(Comment comment, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, $"{comment.Date.Value:s} {comment.Id}", CreateMode.OverwriteExisting);
}
}

7
VOffline/Services/Handlers/CommentsHandler.cs
View file

@ -3,6 +3,7 @@ using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using log4net;
using VkNet.Model;
using VkNet.Model.Attachments;
using VOffline.Services.Storage;
using VOffline.Services.Vk;
@ -12,9 +13,9 @@ namespace VOffline.Services.Handlers
public class CommentsHandler : HandlerBase<Post>
{
private readonly VkApiUtils vkApiUtils;
private readonly CommentHandler commentHandler;
private readonly IHandler<Comment> commentHandler;
public CommentsHandler(VkApiUtils vkApiUtils, FilesystemTools filesystemTools, CommentHandler commentHandler) : base(filesystemTools)
public CommentsHandler(VkApiUtils vkApiUtils, FilesystemTools filesystemTools, IHandler<Comment> commentHandler) : base(filesystemTools)
{
this.vkApiUtils = vkApiUtils;
this.commentHandler = commentHandler;
@ -34,6 +35,6 @@ namespace VOffline.Services.Handlers
await Task.WhenAll(commentTasks);
}
public override DirectoryInfo GetWorkingDirectory(Post post, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, "Comments", CreateMode.MergeWithExisting);
public override DirectoryInfo GetWorkingDirectory(Post post, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, "Comments", CreateMode.OverwriteExisting);
}
}

24
VOffline/Services/Handlers/HandlerBase.cs
View file

@ -8,7 +8,7 @@ using VOffline.Services.Storage;
namespace VOffline.Services.Handlers
{
public abstract class HandlerBase<T>
public abstract class HandlerBase<T> : IHandler<T>
{
protected readonly FilesystemTools filesystemTools;
@ -24,26 +24,8 @@ namespace VOffline.Services.Handlers
{
token.ThrowIfCancellationRequested(); // this helps stop synchronous stuff inside long nested loops
workDir = GetWorkingDirectory(data, parentDir);
if (IsCompletable)
{
if (filesystemTools.IsCompleted(workDir))
{
log.Info($"Skipping [{workDir.FullName}] because marked as competed");
return;
}
}
await ProcessInternal(data, workDir, token, log);
if (IsCompletable)
{
filesystemTools.MarkAsCompleted(workDir);
}
var completedText = IsCompletable
? ", marked"
: "";
log.Info($"Completed [{workDir.FullName}]{completedText}");
log.Info($"Completed crawling data for [{workDir.FullName}]");
}
catch (OperationCanceledException)
@ -59,7 +41,5 @@ namespace VOffline.Services.Handlers
public abstract Task ProcessInternal(T data, DirectoryInfo workDir, CancellationToken token, ILog log);
public abstract DirectoryInfo GetWorkingDirectory(T data, DirectoryInfo parentDir);
protected virtual bool IsCompletable { get; } = false;
}
}

5
VOffline/Services/Handlers/IHandler.cs
View file

@ -5,8 +5,9 @@ using log4net;
namespace VOffline.Services.Handlers
{
public interface IHandler
public interface IHandler<T>
{
Task Process(DirectoryInfo parentDir, CancellationToken token, ILog log);
Task Process(T data, DirectoryInfo parentDir, CancellationToken token, ILog log);
DirectoryInfo GetWorkingDirectory(T data, DirectoryInfo parentDir);
}
}

6
VOffline/Services/Handlers/PhotoHandler.cs
View file

@ -12,9 +12,9 @@ namespace VOffline.Services.Handlers
public class PhotoHandler : HandlerBase<long>
{
private readonly VkApiUtils vkApiUtils;
private readonly AlbumHandler albumHandler;
private readonly IHandler<AlbumWithPhoto> albumHandler;
public PhotoHandler(VkApiUtils vkApiUtils, AlbumHandler albumHandler, FilesystemTools filesystemTools) : base(filesystemTools)
public PhotoHandler(VkApiUtils vkApiUtils, FilesystemTools filesystemTools, IHandler<AlbumWithPhoto> albumHandler) : base(filesystemTools)
{
this.vkApiUtils = vkApiUtils;
this.albumHandler = albumHandler;
@ -47,6 +47,6 @@ namespace VOffline.Services.Handlers
await Task.WhenAll(allTasks);
}
public override DirectoryInfo GetWorkingDirectory(long id, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, "Photo", CreateMode.MergeWithExisting);
public override DirectoryInfo GetWorkingDirectory(long id, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, "Photo", CreateMode.OverwriteExisting);
}
}

8
VOffline/Services/Handlers/PlaylistHandler.cs
View file

@ -21,8 +21,8 @@ namespace VOffline.Services.Handlers
{
if (!string.IsNullOrWhiteSpace(playlistWithAudio.Playlist.Description))
{
var text = filesystemTools.CreateFile(workDir, $"__description.txt", CreateMode.MergeWithExisting);
File.WriteAllText(text.FullName, playlistWithAudio.Playlist.Description);
var text = filesystemTools.CreateFile(workDir, $"__description.txt", CreateMode.OverwriteExisting);
await File.WriteAllTextAsync(text.FullName, playlistWithAudio.Playlist.Description, token);
}
if (playlistWithAudio.Playlist.Cover != null)
@ -30,10 +30,10 @@ namespace VOffline.Services.Handlers
await attachmentProcessor.ProcessAttachment(playlistWithAudio.Playlist.Cover, workDir, token, log);
}
var attachmentTasks = playlistWithAudio.Audio.Select((a, i) => attachmentProcessor.ProcessAttachment(a, i, workDir, token, log));
var attachmentTasks = playlistWithAudio.Audio.Select((a, i) => attachmentProcessor.ProcessAttachment(a, i+1, workDir, token, log));
await Task.WhenAll(attachmentTasks);
}
public override DirectoryInfo GetWorkingDirectory(PlaylistWithAudio playlistWithAudio, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, $"{playlistWithAudio.Playlist.Title}", CreateMode.MergeWithExisting);
public override DirectoryInfo GetWorkingDirectory(PlaylistWithAudio playlistWithAudio, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, $"{playlistWithAudio.Playlist.Title}", CreateMode.OverwriteExisting);
}
}

9
VOffline/Services/Handlers/PostHandler.cs
View file

@ -3,6 +3,7 @@ using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using log4net;
using VkNet.Model;
using VkNet.Model.Attachments;
using VOffline.Services.Storage;
@ -23,11 +24,11 @@ namespace VOffline.Services.Handlers
{
if (!string.IsNullOrWhiteSpace(post.Text))
{
var postText = filesystemTools.CreateFile(workDir, $"text.txt", CreateMode.MergeWithExisting);
File.WriteAllText(postText.FullName, post.Text);
var postText = filesystemTools.CreateFile(workDir, $"text.txt", CreateMode.OverwriteExisting);
await File.WriteAllTextAsync(postText.FullName, post.Text, token);
}
var attachmentTasks = post.Attachments.Select((a, i) => attachmentProcessor.ProcessAttachment(a, i, workDir, token, log));
var attachmentTasks = post.Attachments.Select((a, i) => attachmentProcessor.ProcessAttachment(a, i+1, workDir, token, log));
await Task.WhenAll(attachmentTasks);
if (post.Comments?.Count > 0)
@ -47,7 +48,7 @@ namespace VOffline.Services.Handlers
var dateMaybe = post.Date != null
? $"{post.Date.Value:s}"
: "no_date";
return filesystemTools.CreateSubdir(parentDir, $"{dateMaybe} {post.Id}", CreateMode.MergeWithExisting);
return filesystemTools.CreateSubdir(parentDir, $"{dateMaybe} {post.Id}", CreateMode.OverwriteExisting);
}
}
}

7
VOffline/Services/Handlers/WallHandler.cs
View file

@ -3,6 +3,7 @@ using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using log4net;
using VkNet.Model.Attachments;
using VOffline.Services.Storage;
using VOffline.Services.Vk;
@ -11,9 +12,9 @@ namespace VOffline.Services.Handlers
public class WallHandler : HandlerBase<long>
{
private readonly VkApiUtils vkApiUtils;
private readonly PostHandler postHandler;
private readonly IHandler<Post> postHandler;
public WallHandler(VkApiUtils vkApiUtils, FilesystemTools filesystemTools, PostHandler postHandler):base(filesystemTools)
public WallHandler(VkApiUtils vkApiUtils, FilesystemTools filesystemTools, IHandler<Post> postHandler):base(filesystemTools)
{
this.vkApiUtils = vkApiUtils;
this.postHandler = postHandler;
@ -29,6 +30,6 @@ namespace VOffline.Services.Handlers
await Task.WhenAll(allTasks);
}
public override DirectoryInfo GetWorkingDirectory(long id, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, "Wall", CreateMode.MergeWithExisting);
public override DirectoryInfo GetWorkingDirectory(long id, DirectoryInfo parentDir) => filesystemTools.CreateSubdir(parentDir, "Wall", CreateMode.OverwriteExisting);
}
}

37
VOffline/Services/Logic.cs
View file

@ -69,26 +69,37 @@ namespace VOffline.Services
.Distinct()
.ToImmutableList();
log.Debug($"Processing {JsonConvert.SerializeObject(modes)} for {string.Join(", ", identifiers.Select(x => $"[{x.name} {x.id}]"))}");
filesystemTools.LoadCache(log);
var downloaderTask = downloader.Process(token, log);
var rootDir = filesystemTools.MkDir(settings.OutputPath);
foreach (var identifier in identifiers)
try
{
var name = await vkApiUtils.GetName(identifier.id);
var workDir = filesystemTools.CreateSubdir(rootDir, name, CreateMode.MergeWithExisting);
log.Info($"id [{identifier.id}], name [{name}], path [{workDir.FullName}]");
foreach (var mode in modes)
foreach (var identifier in identifiers)
{
await ProcessTarget(identifier.id, workDir, mode, token, log);
var name = await vkApiUtils.GetName(identifier.id);
var workDir = filesystemTools.CreateSubdir(filesystemTools.RootDir, name, CreateMode.OverwriteExisting);
log.Info($"id [{identifier.id}], name [{name}], path [{workDir.FullName}]");
foreach (var mode in modes)
{
await ProcessTarget(identifier.id, workDir, mode, token, log);
}
}
queueProvider.Pending.CompleteAdding();
var downloadErrors = await downloaderTask;
foreach (var downloadError in downloadErrors)
{
log.Warn($"Failed {downloadError.DesiredName}", downloadError.Errors.LastOrDefault());
}
}
queueProvider.Pending.CompleteAdding();
var downloadErrors = await downloaderTask;
foreach (var downloadError in downloadErrors)
catch (TaskCanceledException)
{
log.Warn($"Failed {downloadError.DesiredName}", downloadError.Errors.LastOrDefault());
queueProvider.Pending.CompleteAdding();
log.Warn($"Abandoned {queueProvider.Pending.GetConsumingEnumerable().Count()} pending downloads");
}
finally
{
filesystemTools.SaveCache(log);
}
}

18
VOffline/Services/Storage/AttachmentProcessor.cs
View file

@ -1,24 +1,30 @@
using System.IO;
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using log4net;
using Microsoft.Extensions.DependencyInjection;
using Newtonsoft.Json;
using VkNet;
using VkNet.Model;
using VkNet.Model.Attachments;
using VOffline.Models.Storage;
using VOffline.Services.Handlers;
using VOffline.Services.Vk;
namespace VOffline.Services.Storage
{
public class AttachmentProcessor
{
public IServiceProvider ServiceProvider { get; }
private readonly VkApi vkApi;
private readonly VkApiUtils vkApiUtils;
private readonly FilesystemTools filesystemTools;
private readonly DownloadQueueProvider downloadQueueProvider;
public AttachmentProcessor(VkApi vkApi, VkApiUtils vkApiUtils, FilesystemTools filesystemTools, DownloadQueueProvider downloadQueueProvider)
public AttachmentProcessor(VkApi vkApi, VkApiUtils vkApiUtils, FilesystemTools filesystemTools, DownloadQueueProvider downloadQueueProvider, IServiceProvider serviceProvider)
{
ServiceProvider = serviceProvider;
this.vkApi = vkApi;
this.vkApiUtils = vkApiUtils;
this.filesystemTools = filesystemTools;
@ -55,13 +61,13 @@ namespace VOffline.Services.Storage
break;
case VkNet.Model.Attachments.AudioPlaylist audioPlaylist:
var playlistWithAudio = await vkApiUtils.ExpandPlaylist(audioPlaylist, token, log);
//await playlistHandler.Process(p, workDir, token, log)
log.Warn($"TODO: playlist attachment");
var playlistHandler = ServiceProvider.GetRequiredService<IHandler<PlaylistWithAudio>>();
await playlistHandler.Process(playlistWithAudio, workDir, token, log);
break;
case VkNet.Model.Attachments.Album album:
var albumWithPhoto = await vkApiUtils.ExpandAlbum(album, token, log);
//await albumHandler.Process(p, workDir, token, log)
log.Warn($"TODO: photoalbum attachment");
var albumHandler = ServiceProvider.GetRequiredService<IHandler<AlbumWithPhoto>>();
await albumHandler.Process(albumWithPhoto, workDir, token, log);
break;
case VkNet.Model.Attachments.Video video:
await downloadQueueProvider.EnqueueAll(video.ToDownloads(number, filesystemTools, workDir, log), token);

2
VOffline/Services/Storage/CreateMode.cs
View file

@ -4,6 +4,6 @@
{
AutoRenameCollisions,
ThrowIfExists,
MergeWithExisting
OverwriteExisting
}
}

5
VOffline/Services/Storage/DownloadQueueProvider.cs
View file

@ -13,7 +13,10 @@ namespace VOffline.Services.Storage
{
public DownloadQueueProvider(IOptionsSnapshot<Settings> settings)
{
Pending = new AsyncProducerConsumerQueue<IDownload>(settings.Value.DownloadQueueLimit);
var queueSizeLimit = settings.Value.DownloadQueueLimit;
Pending = queueSizeLimit > 0
? new AsyncProducerConsumerQueue<IDownload>(settings.Value.DownloadQueueLimit)
: new AsyncProducerConsumerQueue<IDownload>();
}
public async Task EnqueueAll(IEnumerable<IDownload> items, CancellationToken token)

136
VOffline/Services/Storage/FilesystemTools.cs
View file

@ -1,20 +1,63 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using log4net;
using Microsoft.Extensions.Options;
using Newtonsoft.Json;
using VOffline.Models;
namespace VOffline.Services.Storage
{
public class FilesystemTools
{
public DirectoryInfo MkDir(string path)
private readonly FileInfo cacheFile;
private readonly ConcurrentDictionary<string, DateTime> cache;
public DirectoryInfo RootDir { get; }
public FilesystemTools(IOptionsSnapshot<Settings> settings)
{
var dir = new DirectoryInfo(path);
dir.Create();
dir.Refresh();
return dir;
RootDir = MkDir(settings.Value.OutputPath);
cacheFile = new FileInfo(CombineCutPath(RootDir, CacheFilename));
cache = new ConcurrentDictionary<string, DateTime>();
}
public void LoadCache(ILog log)
{
lock (LockObject)
{
var lines = File.ReadAllLines(cacheFile.FullName);
var separator = new[] {' '};
foreach (var line in lines)
{
var items = line.Split(separator, 2);
var datetime = DateTime.ParseExact(items[0], "O", CultureInfo.InvariantCulture);
var path = items[1];
cache[path] = datetime;
}
}
log.Info($"Cache loaded: {cache.Count} items, {cacheFile.FullName}");
}
public void SaveCache(ILog log)
{
lock (LockObject)
{
var data = cache
.OrderBy(kv => kv.Value)
.Select(kv => $"{kv.Value:O} {kv.Key}");
File.WriteAllLines(cacheFile.FullName, data);
}
log.Info($"Cache saved: {cache.Count} items, {cacheFile.FullName}");
}
public DirectoryInfo CreateSubdir(DirectoryInfo parent, string desiredName, CreateMode mode)
@ -39,13 +82,15 @@ namespace VOffline.Services.Storage
{
throw new InvalidOperationException($"Dir already exists: {directory.FullName}");
}
break;
case CreateMode.MergeWithExisting:
case CreateMode.OverwriteExisting:
directory = new DirectoryInfo(CombineCutPath(parent, validName));
break;
default:
throw new ArgumentOutOfRangeException(nameof(mode), mode, null);
}
directory.Create();
directory.Refresh();
return directory;
@ -74,13 +119,15 @@ namespace VOffline.Services.Storage
{
throw new InvalidOperationException($"File already exists: {file.FullName}");
}
break;
case CreateMode.MergeWithExisting:
case CreateMode.OverwriteExisting:
file = new FileInfo(CombineCutPath(parent, validName));
break;
default:
throw new ArgumentOutOfRangeException(nameof(mode), mode, null);
}
file.Create().Close();
file.Refresh();
return file;
@ -90,10 +137,8 @@ namespace VOffline.Services.Storage
public async Task WriteFileWithCompletionMark(DirectoryInfo parent, string desiredName, Func<Task<string>> contentTaskFunc, CancellationToken token, ILog log)
{
var validName = MakeValidName(desiredName);
var completedName = $".{validName}.done.voffline";
var file = new FileInfo(CombineCutPath(parent, validName));
var completedFile = new FileInfo(CombineCutPath(parent, completedName));
if (completedFile.Exists)
if (cache.ContainsKey(file.FullName))
{
log.Debug($"Skipping [{file.FullName}] because marked as competed");
return;
@ -109,17 +154,14 @@ namespace VOffline.Services.Storage
await File.WriteAllTextAsync(file.FullName, content, token);
log.Info($"Saved [{desiredName}] as [{file.FullName}] with [{content.Length}] chars");
}
await File.WriteAllTextAsync(completedFile.FullName, $"{DateTime.Now:O}", token);
completedFile.Attributes |= FileAttributes.Hidden;
cache[file.FullName] = DateTime.Now;
}
public async Task WriteFileWithCompletionMark(DirectoryInfo parent, string desiredName, Func<Task<byte[]>> contentTaskFunc, CancellationToken token, ILog log)
{
var validName = MakeValidName(desiredName);
var completedName = $".{validName}.done.voffline";
var file = new FileInfo(CombineCutPath(parent, validName));
var completedFile = new FileInfo(CombineCutPath(parent, completedName));
if (completedFile.Exists)
if (cache.ContainsKey(file.FullName))
{
log.Debug($"Skipping [{file.FullName}] because marked as competed");
return;
@ -135,44 +177,16 @@ namespace VOffline.Services.Storage
await File.WriteAllBytesAsync(file.FullName, content, token);
log.Info($"Saved [{desiredName}] as [{file.FullName}] with [{content.Length}] bytes");
}
await File.WriteAllTextAsync(completedFile.FullName, $"{DateTime.Now:O}", token);
completedFile.Attributes |= FileAttributes.Hidden;
cache[file.FullName] = DateTime.Now;
}
public int Wipe(DirectoryInfo dir)
private DirectoryInfo MkDir(string path)
{
lock (LockObject)
{
var count = 0;
foreach (var f in dir.GetFiles())
{
f.Delete();
count++;
}
foreach (var d in dir.GetDirectories())
{
d.Delete(true);
count++;
}
return count;
}
}
public void MarkAsCompleted(DirectoryInfo dir)
{
lock (LockObject)
{
var file = CreateFile(dir, CompletedFilename, CreateMode.ThrowIfExists);
File.WriteAllText(file.FullName, $"{DateTime.Now:O}");
file.Attributes |= FileAttributes.Hidden;
}
}
public bool IsCompleted(DirectoryInfo dir)
{
var file = new FileInfo(CombineCutPath(dir, CompletedFilename));
return file.Exists;
var dir = new DirectoryInfo(path);
dir.Create();
dir.Refresh();
return dir;
}
/// <summary>
@ -223,15 +237,17 @@ namespace VOffline.Services.Storage
var extensionPart = Path.GetExtension(name);
while (namePart.Length > 1)
{
var testName = $"{namePart} (NNN){extensionPart}"; // extra filler for possible " (NNN)"
var testPath = Path.Combine(parentDir.FullName, testName);
if (testPath.Length < 248)
{
return Path.Combine(parentDir.FullName, $"{namePart}{extensionPart}");
}
//Path.GetFullPath(testPath); // should throw on long paths but does not work
namePart = namePart.Substring(0, Math.Max(1, namePart.Length - 1));
var testName = $"{namePart} (NNN){extensionPart}"; // extra filler for possible " (NNN)"
var testPath = Path.Combine(parentDir.FullName, testName);
if (testPath.Length < 248)
{
return Path.Combine(parentDir.FullName, $"{namePart}{extensionPart}");
}
//Path.GetFullPath(testPath); // should throw on long paths but does not work on net core?
namePart = namePart.Substring(0, Math.Max(1, namePart.Length - 1));
}
throw new PathTooLongException($"Tried to shorten [{name}] to [{namePart}{extensionPart}] but path [{parentDir.FullName}] is still too long");
}
@ -242,12 +258,12 @@ namespace VOffline.Services.Storage
private static readonly char[] AllBadChars =
Path.GetInvalidFileNameChars()
.Concat(Path.GetInvalidPathChars())
.Concat(new[] { '\\', '/', ':' })
.Concat(new[] {'\\', '/', ':'})
.Distinct()
.ToArray();
private static readonly object LockObject = new object();
private static readonly string CompletedFilename = MakeValidName(".done.voffline");
private static readonly string CacheFilename = MakeValidName(".cache.voffline");
}
}

12
VOffline/Services/Vk/AttachmentExtensions.cs
View file

@ -27,7 +27,7 @@ namespace VOffline.Services.Vk
}
else
{
filesystemTools.CreateFile(dir, $"{i} {trackName}.mp3.deleted", CreateMode.MergeWithExisting);
filesystemTools.CreateFile(dir, $"{i} {trackName}.mp3.deleted", CreateMode.OverwriteExisting);
}
}
@ -120,7 +120,7 @@ namespace VOffline.Services.Vk
{
yield break;
}
var pollDir = filesystemTools.CreateSubdir(dir, $"{i} {poll.GetName()}", CreateMode.MergeWithExisting);
var pollDir = filesystemTools.CreateSubdir(dir, $"{i} {poll.GetName()}", CreateMode.OverwriteExisting);
var imageDownloads = photos.SelectMany((p, j) => p.ToDownloads(j, filesystemTools, pollDir, log));
foreach (var imageDownload in imageDownloads)
{
@ -212,7 +212,7 @@ namespace VOffline.Services.Vk
{
if (!string.IsNullOrWhiteSpace(photo.Text))
{
var textFile = filesystemTools.CreateFile(dir, $"{i} {photo.Id}.txt", CreateMode.MergeWithExisting);
var textFile = filesystemTools.CreateFile(dir, $"{i} {photo.Id}.txt", CreateMode.OverwriteExisting);
await File.WriteAllTextAsync(textFile.FullName, photo.Text, token);
}
}
@ -232,20 +232,20 @@ namespace VOffline.Services.Vk
public static async Task SaveHumanReadableText(this Poll poll, int i, FilesystemTools filesystemTools, DirectoryInfo dir, CancellationToken token, ILog log)
{
var textFile = filesystemTools.CreateFile(dir, $"{i} {poll.GetName()}.txt", CreateMode.MergeWithExisting);
var textFile = filesystemTools.CreateFile(dir, $"{i} {poll.GetName()}.txt", CreateMode.OverwriteExisting);
await File.WriteAllTextAsync(textFile.FullName, poll.Serialize(), token);
}
public static async Task SaveHumanReadableText(this Link link, int i, FilesystemTools filesystemTools, DirectoryInfo dir, CancellationToken token, ILog log)
{
var textFile = filesystemTools.CreateFile(dir, $"{i} {link.Title}.txt", CreateMode.MergeWithExisting);
var textFile = filesystemTools.CreateFile(dir, $"{i} {link.Title}.txt", CreateMode.OverwriteExisting);
await File.WriteAllTextAsync(textFile.FullName, link.Serialize(), token);
}
public static async Task SaveHumanReadableText(this IReadOnlyList<Comment> comments, FilesystemTools filesystemTools, DirectoryInfo dir, CancellationToken token, ILog log)
{
var data = string.Join("\n\n", comments.Select(c => c.Serialize()));
var textFile = filesystemTools.CreateFile(dir, $"comments.txt", CreateMode.MergeWithExisting);
var textFile = filesystemTools.CreateFile(dir, $"comments.txt", CreateMode.OverwriteExisting);
await File.WriteAllTextAsync(textFile.FullName, data, token);
}
}