From 705bd15c65a9f0e024ab213a05fa34e22fb9d1e3 Mon Sep 17 00:00:00 2001 From: YaR Date: Thu, 2 Nov 2017 05:47:54 +0300 Subject: [PATCH 01/31] double folder listing requests fixed; refacted --- MailRuCloud/MailRuCloudApi/Base/Entry.cs | 69 ------- MailRuCloud/MailRuCloudApi/Base/File.cs | 23 +-- MailRuCloud/MailRuCloudApi/Base/Folder.cs | 30 ++- MailRuCloud/MailRuCloudApi/Base/IEntry.cs | 12 ++ .../Base/Requests/Types/FolderInfoResult.cs | 1 + .../MailRuCloudApi/Base/SplittedCloud.cs | 31 --- MailRuCloud/MailRuCloudApi/Base/WebDavPath.cs | 42 ++++- .../MailRuCloudApi/Extensions/DtoImport.cs | 143 +++++++++++--- MailRuCloud/MailRuCloudApi/MailRuCloud.cs | 177 ++++++++++-------- .../PathResolve/PathResolver.cs | 16 +- .../SharedFolderLinkCommand.cs | 13 +- .../SpecialCommands/SpecialCommandFabric.cs | 4 +- NWebDav/NWebDav.Server/WebDavUri.cs | 35 ++++ WebDavMailRuCloudStore/Cloud.cs | 4 +- WebDavMailRuCloudStore/Mailru/MkcolHandler.cs | 50 +++++ .../Mailru/RequestHandlerFactory.cs | 2 +- .../Mailru/StoreBase/MailruStore.cs | 46 +++-- .../Mailru/StoreBase/MailruStoreCollection.cs | 32 +--- 18 files changed, 440 insertions(+), 290 deletions(-) delete mode 100644 MailRuCloud/MailRuCloudApi/Base/Entry.cs create mode 100644 MailRuCloud/MailRuCloudApi/Base/IEntry.cs delete mode 100644 MailRuCloud/MailRuCloudApi/Base/SplittedCloud.cs create mode 100644 WebDavMailRuCloudStore/Mailru/MkcolHandler.cs diff --git a/MailRuCloud/MailRuCloudApi/Base/Entry.cs b/MailRuCloud/MailRuCloudApi/Base/Entry.cs deleted file mode 100644 index bff545d2..00000000 --- a/MailRuCloud/MailRuCloudApi/Base/Entry.cs +++ /dev/null @@ -1,69 +0,0 @@ -//----------------------------------------------------------------------- -// -// Mail.ru cloud client created in 2016. -// -// Korolev Erast. -//----------------------------------------------------------------------- - -using System; -using System.Collections.Generic; -using System.Diagnostics; - -namespace YaR.MailRuCloud.Api.Base -{ - /// - /// List of items in cloud. - /// - [DebuggerDisplay("{" + nameof(FullPath) + "}")] - public class Entry - { - /// - /// Initializes a new instance of the class. - /// - /// List of the folders. - /// List of the files. - /// The entry path on the server. - public Entry(IList folders, IList files, string path) - { - Folders = folders ?? new List(); - Files = files ?? new List(); - - NumberOfFolders = folders?.Count ?? 0; - NumberOfFiles = files?.Count ?? 0; - FullPath = path; - } - - /// - /// Gets number of the folders. - /// - public int NumberOfFolders { get; } - - /// - /// Gets number of the files. - /// - public int NumberOfFiles { get; } - - public int NumberOfItems => NumberOfFolders + NumberOfFiles; - - /// - /// Gets list of the folders with their specification. - /// - public IList Folders { get; } - - /// - /// Gets list of the files with their specification. - /// - public IList Files { get; } - - /// - /// Gets full entry path on the server. - /// - public string FullPath { get; } - - public long Size { get; set; } - public string WebLink { get; set; } - public bool IsFile { get; set; } - public string Name { get; set; } - public DateTime? CreationDate { get; set; } - } -} diff --git a/MailRuCloud/MailRuCloudApi/Base/File.cs b/MailRuCloud/MailRuCloudApi/Base/File.cs index d70c702a..bbd4deb5 100644 --- a/MailRuCloud/MailRuCloudApi/Base/File.cs +++ b/MailRuCloud/MailRuCloudApi/Base/File.cs @@ -16,13 +16,13 @@ namespace YaR.MailRuCloud.Api.Base /// Server file info. /// [DebuggerDisplay("{" + nameof(FullPath) + "}")] - public class File + public class File : IEntry { protected File() { } - public File(string fullPath, long size, string hash) + public File(string fullPath, long size, string hash = "") { FullPath = fullPath; _size = size; @@ -39,7 +39,7 @@ public File(string fullPath, long size, string hash) /// /// File name. //TODO: refact - public virtual string Name => FullPath.Substring(FullPath.LastIndexOf("/", StringComparison.Ordinal) + 1); + public virtual string Name => WebDavPath.Name(FullPath); //FullPath.Substring(FullPath.LastIndexOf("/", StringComparison.Ordinal) + 1); public string Extension => System.IO.Path.GetExtension(Name); @@ -70,11 +70,7 @@ public virtual FileSize Size public string FullPath { get => _fullPath; - set - { - _fullPath = value.Replace("\\", "/"); - if (!string.IsNullOrEmpty(Name) && !_fullPath.EndsWith("/" + Name)) _fullPath = _fullPath.TrimEnd('/') + "/" + Name; - } + protected set => _fullPath = WebDavPath.Clean(value); } public string Path => WebDavPath.Parent(FullPath); @@ -90,17 +86,13 @@ public string FullPath /// public virtual List Parts => new List {this}; - /// - /// Gets or sets base file size. - /// - /// File size. - internal FileSize PrimarySize => Size; - public virtual DateTime CreationTimeUtc { get; set; } public virtual DateTime LastWriteTimeUtc { get; set; } public virtual DateTime LastAccessTimeUtc { get; set; } public bool IsSplitted => Parts.Any(f => f.FullPath != FullPath); + public bool IsFile => true; + public void SetName(string destinationName) { string path = WebDavPath.Parent(FullPath); @@ -120,7 +112,6 @@ public void SetPath(string fullPath) { fiFile.FullPath = WebDavPath.Combine(fullPath, fiFile.Name); //TODO: refact } - } - } + }} } diff --git a/MailRuCloud/MailRuCloudApi/Base/Folder.cs b/MailRuCloud/MailRuCloudApi/Base/Folder.cs index f1bb190b..583925f8 100644 --- a/MailRuCloud/MailRuCloudApi/Base/Folder.cs +++ b/MailRuCloud/MailRuCloudApi/Base/Folder.cs @@ -6,8 +6,10 @@ //----------------------------------------------------------------------- using System; +using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Linq; namespace YaR.MailRuCloud.Api.Base { @@ -15,7 +17,7 @@ namespace YaR.MailRuCloud.Api.Base /// Server file info. /// [DebuggerDisplay("{" + nameof(FullPath) + "}")] - public class Folder + public class Folder : IEntry { /// /// Initializes a new instance of the class. @@ -33,25 +35,39 @@ public Folder(string fullPath) /// Folder size. /// Full folder path. /// Public folder link. - public Folder(int foldersCount, int filesCount, FileSize size, string fullPath, string publicLink = null):this(fullPath) + public Folder(FileSize size, string fullPath, string publicLink = null):this(fullPath) { - NumberOfFolders = foldersCount; - NumberOfFiles = filesCount; Size = size; PublicLink = publicLink; } + public IEnumerable Entries + { + get + { + foreach (var file in Files) + yield return file; + foreach (var folder in Folders) + yield return folder; + } + } + + public List Files { get; set; } = new List(); + + public List Folders { get; set; } = new List(); + + /// /// Gets number of folders in folder. /// /// Number of folders. - public int NumberOfFolders { get; } + public int NumberOfFolders => Entries.OfType().Count(); /// /// Gets number of files in folder. /// /// Number of files. - public int NumberOfFiles { get; } + public int NumberOfFiles => Entries.OfType().Count(); /// /// Gets folder name. @@ -96,5 +112,7 @@ public string FullPath public FileAttributes Attributes { get; set; } = FileAttributes.Directory; + + public bool IsFile => false; } } diff --git a/MailRuCloud/MailRuCloudApi/Base/IEntry.cs b/MailRuCloud/MailRuCloudApi/Base/IEntry.cs new file mode 100644 index 00000000..753723a7 --- /dev/null +++ b/MailRuCloud/MailRuCloudApi/Base/IEntry.cs @@ -0,0 +1,12 @@ +using System; + +namespace YaR.MailRuCloud.Api.Base +{ + public interface IEntry + { + bool IsFile { get; } + FileSize Size { get; } + string Name { get; } + DateTime CreationTimeUtc { get; } + } +} \ No newline at end of file diff --git a/MailRuCloud/MailRuCloudApi/Base/Requests/Types/FolderInfoResult.cs b/MailRuCloud/MailRuCloudApi/Base/Requests/Types/FolderInfoResult.cs index a38c852d..bcf4a088 100644 --- a/MailRuCloud/MailRuCloudApi/Base/Requests/Types/FolderInfoResult.cs +++ b/MailRuCloud/MailRuCloudApi/Base/Requests/Types/FolderInfoResult.cs @@ -11,6 +11,7 @@ public class FolderInfoResult public FolderInfoBody body { get; set; } public long time { get; set; } public int status { get; set; } + public bool IsFile { get; set; } } diff --git a/MailRuCloud/MailRuCloudApi/Base/SplittedCloud.cs b/MailRuCloud/MailRuCloudApi/Base/SplittedCloud.cs deleted file mode 100644 index 43ffedf1..00000000 --- a/MailRuCloud/MailRuCloudApi/Base/SplittedCloud.cs +++ /dev/null @@ -1,31 +0,0 @@ -using System.Linq; -using System.Text.RegularExpressions; -using System.Threading.Tasks; - -namespace YaR.MailRuCloud.Api.Base -{ - public class SplittedCloud : MailRuCloud - { - public SplittedCloud(string login, string password, ITwoFaHandler twoFaHandler) : base(login, password, twoFaHandler) - { - } - - public override async Task GetItems(string path) - { - Entry entry = await base.GetItems(path); - - if (null == entry) return null; - - var groupedFiles = entry.Files - .GroupBy(f => Regex.Match(f.Name, @"(?.*?)(\.wdmrc\.(crc|\d\d\d))?\Z").Groups["name"].Value, file => file) - .Select(group => group.Count() == 1 - ? group.First() - : new SplittedFile(group.ToList())) - .ToList(); - - var newEntry = new Entry(entry.Folders, groupedFiles, entry.FullPath) {Size = entry.Size}; - - return newEntry; - } - } -} diff --git a/MailRuCloud/MailRuCloudApi/Base/WebDavPath.cs b/MailRuCloud/MailRuCloudApi/Base/WebDavPath.cs index f20917d8..a32deeac 100644 --- a/MailRuCloud/MailRuCloudApi/Base/WebDavPath.cs +++ b/MailRuCloud/MailRuCloudApi/Base/WebDavPath.cs @@ -1,4 +1,5 @@ using System; +using System.ComponentModel; namespace YaR.MailRuCloud.Api.Base { @@ -10,7 +11,8 @@ public static string Combine(string a, string b) b = Clean(b); a = a.Trim('/'); b = b.TrimStart('/'); - string res = "/" + a + (string.IsNullOrEmpty(b) ? "" : "/" + b); + string res = a + (string.IsNullOrEmpty(b) ? "" : "/" + b); + if (!res.StartsWith("/")) res = "/" + res; return res; } @@ -25,7 +27,16 @@ public static string Clean(string path) public static string Parent(string path) { - int pos = path.LastIndexOf("/", StringComparison.Ordinal); + //TODO: refact + path = path.TrimEnd('/'); + + // cause we use >> as a sign of special command + int cmdPos = path.IndexOf(">>", StringComparison.Ordinal); + + int pos = cmdPos > 0 + ? path.LastIndexOf("/", 0, cmdPos + 1, StringComparison.Ordinal) + : path.LastIndexOf("/", StringComparison.Ordinal); + return pos > 0 ? path.Substring(0, pos) : "/"; @@ -33,13 +44,38 @@ public static string Parent(string path) public static string Name(string path) { + //TODO: refact path = path.TrimEnd('/'); - int pos = path.LastIndexOf("/", StringComparison.Ordinal); + + // cause we use >> as a sign of special command + int cmdPos = path.IndexOf(">>", StringComparison.Ordinal); + + int pos = cmdPos > 0 + ? path.LastIndexOf("/", 0, cmdPos + 1, StringComparison.Ordinal) + : path.LastIndexOf("/", StringComparison.Ordinal); string res = path.Substring(pos+1); return res; } public static string Root => "/"; + + public static WebDavPathParts Parts(string path) + { + //TODO: refact + var res = new WebDavPathParts + { + Parent = Parent(path), + Name = Name(path) + }; + + return res; + } + } + + public struct WebDavPathParts + { + public string Parent { get; set; } + public string Name { get; set; } } } diff --git a/MailRuCloud/MailRuCloudApi/Extensions/DtoImport.cs b/MailRuCloud/MailRuCloudApi/Extensions/DtoImport.cs index 01bad0eb..d1e85cfe 100644 --- a/MailRuCloud/MailRuCloudApi/Extensions/DtoImport.cs +++ b/MailRuCloud/MailRuCloudApi/Extensions/DtoImport.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text.RegularExpressions; using YaR.MailRuCloud.Api.Base; using YaR.MailRuCloud.Api.Base.Requests.Types; @@ -81,36 +82,126 @@ public static ShardInfo ToShardInfo(this ShardInfoResult webdata, ShardType shar private static readonly string[] FolderKinds = { "folder", "camera-upload", "mounted", "shared" }; - public static Entry ToEntry(this FolderInfoResult data) + //public static Entry ToEntry(this FolderInfoResult data) + //{ + // var entry = new Entry( + // data.body.list? + // .Where(it => FolderKinds.Contains(it.kind)) + // .Select(it => new Folder(it.count.folders, it.count.files, it.size, it.home, string.IsNullOrEmpty(it.weblink) ? "" : ConstSettings.PublishFileLink + it.weblink)) + // .ToList(), + // data.body.list? + // .Where(it => it.kind == "file") + // .Select(it => new File(it.home, it.size, it.hash) + // { + // PublicLink = + // string.IsNullOrEmpty(it.weblink) ? "" : ConstSettings.PublishFileLink + it.weblink, + // CreationTimeUtc = UnixTimeStampToDateTime(it.mtime), + // LastAccessTimeUtc = UnixTimeStampToDateTime(it.mtime), + // LastWriteTimeUtc = UnixTimeStampToDateTime(it.mtime), + // }).ToList(), + // data.body.home) + // { + // CreationDate = DateTime.Now, + // Name = data.body.name, + // IsFile = data.body.kind == "file", + // Size = data.body.size, + // //WebLink = data.body. + // //WebLink = (data.body?.list != null && data.body.list.Count > 0) + // // ? data.body.list[0].weblink + // // : string.Empty + // }; + + // return entry; + //} + + public static IEntry ToEntry(this FolderInfoResult data) + { + if (data.body.kind == "file") + { + var file = data.ToFile(); + return file; + } + + var folder = new Folder(data.body.size, WebDavPath.Combine(data.body.home ?? WebDavPath.Root, data.body.name)) + { + Folders = data.body.list? + .Where(it => FolderKinds.Contains(it.kind)) + .Select(item => item.ToFolder()) + .ToList(), + Files = data.body.list? + .Where(it => it.kind == "file") + .Select(item => item.ToFile()) + .ToGroupedFiles() + .ToList() + }; + + + return folder; + } + + + public static Folder ToFolder(this FolderInfoResult data) { - var entry = new Entry( - data.body.list? - .Where(it => FolderKinds.Contains(it.kind)) - .Select(it => new Folder(it.count.folders, it.count.files, it.size, it.home, string.IsNullOrEmpty(it.weblink) ? "" : ConstSettings.PublishFileLink + it.weblink)) - .ToList(), - data.body.list? - .Where(it => it.kind == "file") - .Select(it => new File(it.home, it.size, it.hash) - { - PublicLink = - string.IsNullOrEmpty(it.weblink) ? "" : ConstSettings.PublishFileLink + it.weblink, - CreationTimeUtc = UnixTimeStampToDateTime(it.mtime), - LastAccessTimeUtc = UnixTimeStampToDateTime(it.mtime), - LastWriteTimeUtc = UnixTimeStampToDateTime(it.mtime), - }).ToList(), - data.body.home) + var folder = new Folder(data.body.size, data.body.home) { - CreationDate = DateTime.Now, - Name = data.body.name, - IsFile = data.body.kind == "file", - Size = data.body.size, - //WebLink = data.body. - //WebLink = (data.body?.list != null && data.body.list.Count > 0) - // ? data.body.list[0].weblink - // : string.Empty + Folders = data.body.list? + .Where(it => FolderKinds.Contains(it.kind)) + .Select(item => item.ToFolder()) + .ToList(), + Files = data.body.list? + .Where(it => it.kind == "file") + .Select(item => item.ToFile()) + .ToGroupedFiles() + .ToList() }; - return entry; + return folder; + } + + public static File ToFile(this FolderInfoResult data, string filename = null) + { + if (string.IsNullOrEmpty(filename)) + { + return new File(WebDavPath.Combine(data.body.home ?? "", data.body.name), data.body.size); + } + + var groupedFile = data.body.list? + .Where(it => it.kind == "file") + .Select(it => it.ToFile()) + .ToGroupedFiles() + .First(it => it.Name == filename); + + return groupedFile; + } + + private static Folder ToFolder(this FolderInfoProps item) + { + var folder = new Folder(item.size, item.home, string.IsNullOrEmpty(item.weblink) ? "" : ConstSettings.PublishFileLink + item.weblink); + return folder; + } + + private static File ToFile(this FolderInfoProps item) + { + var file = new File(item.home, item.size, item.hash) + { + PublicLink = + string.IsNullOrEmpty(item.weblink) ? "" : ConstSettings.PublishFileLink + item.weblink, + CreationTimeUtc = UnixTimeStampToDateTime(item.mtime), + LastAccessTimeUtc = UnixTimeStampToDateTime(item.mtime), + LastWriteTimeUtc = UnixTimeStampToDateTime(item.mtime), + }; + return file; + } + + private static IEnumerable ToGroupedFiles(this IEnumerable list) + { + var groupedFiles = list + .GroupBy(f => Regex.Match(f.Name, @"(?.*?)(\.wdmrc\.(crc|\d\d\d))?\Z").Groups["name"].Value, + file => file) + .Select(group => group.Count() == 1 + ? group.First() + : new SplittedFile(group.ToList())); + return groupedFiles; } private static DateTime UnixTimeStampToDateTime(double unixTimeStamp) diff --git a/MailRuCloud/MailRuCloudApi/MailRuCloud.cs b/MailRuCloud/MailRuCloudApi/MailRuCloud.cs index 5137732e..1776be0a 100644 --- a/MailRuCloud/MailRuCloudApi/MailRuCloud.cs +++ b/MailRuCloud/MailRuCloudApi/MailRuCloud.cs @@ -41,57 +41,131 @@ public class MailRuCloud : IDisposable public MailRuCloud(string login, string password, ITwoFaHandler twoFaHandler) { CloudApi = new CloudApi(login, password, twoFaHandler); - _pathResolver = new PathResolver(CloudApi); + _pathResolver = new PathResolver(this); } - + public enum ItemType + { + File, + Folder, + Unknown + } /// /// Get list of files and folders from account. /// /// Path in the cloud to return the list of the items. + /// + /// /// List of the items. - public virtual async Task GetItems(string path) + public virtual async Task GetItem(string path, ItemType itemType = ItemType.Unknown, bool resolveLinks = true) { - string ulink = _pathResolver.AsRelationalWebLink(path); + string ulink = resolveLinks ? _pathResolver.AsRelationalWebLink(path) : string.Empty; - var data = await new FolderInfoRequest(CloudApi, string.IsNullOrEmpty(ulink) ? path : ulink, !string.IsNullOrEmpty(ulink)).MakeRequestAsync(); + var data = new FolderInfoRequest(CloudApi, string.IsNullOrEmpty(ulink) ? path : ulink, !string.IsNullOrEmpty(ulink)) + .MakeRequestAsync().ConfigureAwait(false); - if (!string.IsNullOrEmpty(ulink)) + if (itemType == ItemType.Unknown && !string.IsNullOrEmpty(ulink)) { - bool isFile = data.body.list.Any(it => it.weblink.TrimStart('/') == ulink.TrimStart('/')); + var infores = await new ItemInfoRequest(CloudApi, string.IsNullOrEmpty(ulink) ? path : ulink, !string.IsNullOrEmpty(ulink)) + .MakeRequestAsync().ConfigureAwait(false); + itemType = infores.body.kind == "file" + ? ItemType.File + : ItemType.Folder; + } - string trimpath = path; - if (isFile) trimpath = WebDavPath.Parent(path); + var datares = await data; - foreach (var propse in data.body.list) + if (itemType == ItemType.Unknown && string.IsNullOrEmpty(ulink)) + { + itemType = (await data).body.home == path + ? ItemType.Folder + : ItemType.File; + } + + // patch paths if linked item + if (!string.IsNullOrEmpty(ulink)) + { + string home = path; + if (itemType == ItemType.File) home = WebDavPath.Parent(path); + + //if (itemType == ItemType.Folder) + foreach (var propse in datares.body.list) { - propse.home = WebDavPath.Combine(trimpath, propse.name); + propse.home = WebDavPath.Combine(home, propse.name); } - data.body.home = trimpath; + datares.body.home = home; } - var entry = data.ToEntry(); + var entry = itemType == ItemType.File + ? (IEntry)datares.ToFile(WebDavPath.Name(path)) + : datares.ToFolder(); - - var flinks = _pathResolver.GetItems(entry.FullPath); - if (flinks.Any()) + if (itemType == ItemType.Folder && entry is Folder folder) { - foreach (var flink in flinks) + var flinks = _pathResolver.GetItems(folder.FullPath); + if (flinks.Any()) { - string linkpath = WebDavPath.Combine(entry.FullPath, flink.Name); - - if (!flink.IsFile) - entry.Folders.Add(new Folder(0, 0, 0, linkpath) { CreationTimeUtc = flink.CreationDate ?? DateTime.MinValue }); - else + foreach (var flink in flinks) { - if (entry.Files.All(inf => inf.FullPath != linkpath)) - entry.Files.Add(new File(linkpath, flink.Size, string.Empty)); + string linkpath = WebDavPath.Combine(folder.FullPath, flink.Name); + + if (!flink.IsFile) + folder.Folders.Add(new Folder(0, linkpath) { CreationTimeUtc = flink.CreationDate ?? DateTime.MinValue }); + else + { + if (folder.Files.All(inf => inf.FullPath != linkpath)) + folder.Files.Add(new File(linkpath, flink.Size)); + } } } } + + return entry; + + + + //======================================================================================================= + + //var data = await new FolderInfoRequest(CloudApi, string.IsNullOrEmpty(ulink) ? path : ulink, !string.IsNullOrEmpty(ulink)).MakeRequestAsync(); + + //if (!string.IsNullOrEmpty(ulink)) + //{ + // bool isFile = data.body.list.Any(it => it.weblink.TrimStart('/') == ulink.TrimStart('/')); + + // string trimpath = path; + // if (isFile) trimpath = WebDavPath.Parent(path); + + // foreach (var propse in data.body.list) + // { + // propse.home = WebDavPath.Combine(trimpath, propse.name); + // } + // data.body.home = trimpath; + //} + + //var entry = data.ToEntry(); + + + //var flinks = _pathResolver.GetItems(entry.FullPath); + //if (flinks.Any()) + //{ + // foreach (var flink in flinks) + // { + // string linkpath = WebDavPath.Combine(entry.FullPath, flink.Name); + + // if (!flink.IsFile) + // entry.Folders.Add(new Folder(0, 0, 0, linkpath) { CreationTimeUtc = flink.CreationDate ?? DateTime.MinValue }); + // else + // { + // if (entry.Files.All(inf => inf.FullPath != linkpath)) + // entry.Files.Add(new File(linkpath, flink.Size, string.Empty)); + // } + // } + //} + + //return entry; } @@ -105,17 +179,6 @@ public async Task GetDiskUsage() return data.ToDiskUsage(); } - /// - /// Get list of files and folders from account. - /// - /// Folder info. - /// List of the items. - public async Task GetItems(Folder folder) - { - return await GetItems(folder.FullPath); - } - - /// /// Abort all prolonged async operations. /// @@ -124,17 +187,6 @@ public void AbortAllAsyncThreads() CloudApi.CancelToken.Cancel(true); } - /// - /// Copying folder in another space on the server. - /// - /// Folder info to copying. - /// Destination entry on the server. - /// True or false operation result. - public async Task Copy(Folder folder, Entry destinationEntry) - { - return await Copy(folder, destinationEntry.FullPath); - } - /// /// Copying folder in another space on the server. /// @@ -157,17 +209,6 @@ public async Task Copy(Folder folder, string destinationPath) return !string.IsNullOrEmpty(await MoveOrCopy(folder.FullPath, destinationPath, false)); } - /// - /// Copying file in another space on the server. - /// - /// File info to copying. - /// Destination entry on the server. - /// True or false operation result. - public async Task Copy(File file, Entry destinationEntry) - { - return await Copy(file, destinationEntry.FullPath); - } - /// /// Copying file in another space on the server. /// @@ -235,17 +276,6 @@ public async Task Move(Folder folder, Folder destinationFolder) return await Move(folder, destinationFolder.FullPath); } - /// - /// Move folder in another space on the server. - /// - /// Folder info to moving. - /// Destination entry on the server. - /// True or false operation result. - public async Task Move(Folder folder, Entry destinationEntry) - { - return await Move(folder, destinationEntry.FullPath); - } - /// /// Move folder in another space on the server. /// @@ -257,17 +287,6 @@ public async Task Move(Folder folder, string destinationPath) return !string.IsNullOrEmpty(await MoveOrCopy(folder.FullPath, destinationPath, true)); } - /// - /// Move file in another space on the server. - /// - /// File info to move. - /// Destination entry on the server. - /// True or false operation result. - public async Task Move(File file, Entry destinationEntry) - { - return await Move(file, destinationEntry.FullPath); - } - /// /// Move file in another space on the server. /// diff --git a/MailRuCloud/MailRuCloudApi/PathResolve/PathResolver.cs b/MailRuCloud/MailRuCloudApi/PathResolve/PathResolver.cs index 826cd927..8888e669 100644 --- a/MailRuCloud/MailRuCloudApi/PathResolve/PathResolver.cs +++ b/MailRuCloud/MailRuCloudApi/PathResolve/PathResolver.cs @@ -7,6 +7,7 @@ using YaR.MailRuCloud.Api.Base; using YaR.MailRuCloud.Api.Base.Requests; using YaR.MailRuCloud.Api.Extensions; +using File = YaR.MailRuCloud.Api.Base.File; namespace YaR.MailRuCloud.Api.PathResolve { @@ -15,13 +16,13 @@ public class PathResolver private static readonly log4net.ILog Logger = log4net.LogManager.GetLogger(typeof(PathResolver)); public static string LinkContainerName = "item.links.wdmrc"; - private readonly CloudApi _api; + private readonly MailRuCloud _cloud; private ItemList _itemList; - public PathResolver(CloudApi api) + public PathResolver(MailRuCloud api) { - _api = api; + _cloud = api; Load(); } @@ -34,7 +35,8 @@ public void Save() string content = JsonConvert.SerializeObject(_itemList, Formatting.Indented); var data = Encoding.UTF8.GetBytes(content); - using (var stream = new UploadStream("/" + LinkContainerName, _api, data.Length)) + + using (var stream = _cloud.GetFileUploadStream(WebDavPath.Combine(WebDavPath.Root, LinkContainerName), data.Length)) { stream.Write(data, 0, data.Length); stream.Close(); @@ -45,11 +47,11 @@ public void Load() { Logger.Info($"Loading links from {LinkContainerName}"); - var flist = new FolderInfoRequest(_api, WebDavPath.Root).MakeRequestAsync().Result.ToEntry(); - var file = flist.Files.FirstOrDefault(f => f.Name == LinkContainerName); + var file = (File)_cloud.GetItem(WebDavPath.Combine(WebDavPath.Root, LinkContainerName), MailRuCloud.ItemType.File, false).Result; + if (file != null && file.Size > 3) //some clients put one/two/three-byte file before original file { - DownloadStream stream = new DownloadStream(file, _api); + DownloadStream stream = new DownloadStream(file, _cloud.CloudApi); using (StreamReader reader = new StreamReader(stream)) using (JsonTextReader jsonReader = new JsonTextReader(reader)) diff --git a/MailRuCloud/MailRuCloudApi/SpecialCommands/SharedFolderLinkCommand.cs b/MailRuCloud/MailRuCloudApi/SpecialCommands/SharedFolderLinkCommand.cs index f628895f..d909328a 100644 --- a/MailRuCloud/MailRuCloudApi/SpecialCommands/SharedFolderLinkCommand.cs +++ b/MailRuCloud/MailRuCloudApi/SpecialCommands/SharedFolderLinkCommand.cs @@ -22,18 +22,21 @@ public override Task Execute() { var m = Regex.Match(_param, @"(?snx-)link \s+ (https://?cloud.mail.ru/public)?(?/\w*/\w*)/? \s* (?.*) "); - var info = new ItemInfoRequest(_cloud.CloudApi, m.Groups["url"].Value, true).MakeRequestAsync().Result.ToEntry(); + if (!m.Success) return Task.FromResult(new SpecialCommandResult { Success = false }); - bool isFile = info.IsFile; - long size = info.Size; + var item = new ItemInfoRequest(_cloud.CloudApi, m.Groups["url"].Value, true).MakeRequestAsync().Result.ToEntry(); + + + bool isFile = item.IsFile; + long size = item.Size; string name = m.Groups["name"].Value; - if (string.IsNullOrWhiteSpace(name)) name = info.Name; + if (string.IsNullOrWhiteSpace(name)) name = item.Name; if (m.Success) { - _cloud.LinkItem(m.Groups["url"].Value, _path, name, isFile, size, info.CreationDate); + _cloud.LinkItem(m.Groups["url"].Value, _path, name, isFile, size, item.CreationTimeUtc); } return Task.FromResult(new SpecialCommandResult{Success = true}); diff --git a/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommandFabric.cs b/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommandFabric.cs index 5cec5111..313b8df2 100644 --- a/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommandFabric.cs +++ b/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommandFabric.cs @@ -3,9 +3,9 @@ namespace YaR.MailRuCloud.Api.SpecialCommands { - public static class SpecialCommandFabric + public class SpecialCommandFabric { - public static SpecialCommand Build(MailRuCloud cloud, string param) + public SpecialCommand Build(MailRuCloud cloud, string param) { if (null == param || !param.Contains("/>>")) return null; diff --git a/NWebDav/NWebDav.Server/WebDavUri.cs b/NWebDav/NWebDav.Server/WebDavUri.cs index ae226544..540ac329 100644 --- a/NWebDav/NWebDav.Server/WebDavUri.cs +++ b/NWebDav/NWebDav.Server/WebDavUri.cs @@ -71,9 +71,44 @@ public string BaseUrl } } + public UriAndName Parent + { + get + { + var trimmedUri = AbsoluteUri; + if (trimmedUri.EndsWith("/")) + trimmedUri = trimmedUri.TrimEnd('/'); + + // cause we use >> as a sign for special command + int cmdPos = trimmedUri.IndexOf("%3e%3e", StringComparison.Ordinal); + + int slashOffset = cmdPos > 0 + ? trimmedUri.LastIndexOf("/", cmdPos, StringComparison.InvariantCultureIgnoreCase) + : trimmedUri.LastIndexOf('/'); + + if (slashOffset == -1) + return null; + + // Separate name from path + return new UriAndName + { + Parent = new WebDavUri(trimmedUri.Substring(0, slashOffset)), + Name = Uri.UnescapeDataString(trimmedUri.Substring(slashOffset + 1)) + }; + + } + } + public override string ToString() { return _url; } } + + public class UriAndName +{ + public WebDavUri Parent { get; set; } + public string Name { get; set; } + } + } diff --git a/WebDavMailRuCloudStore/Cloud.cs b/WebDavMailRuCloudStore/Cloud.cs index f1b7518f..7872c78f 100644 --- a/WebDavMailRuCloudStore/Cloud.cs +++ b/WebDavMailRuCloudStore/Cloud.cs @@ -10,7 +10,7 @@ namespace YaR.WebDavMailRu.CloudStore { public static class Cloud { - private static readonly log4net.ILog Logger = log4net.LogManager.GetLogger(typeof(SplittedCloud)); + private static readonly log4net.ILog Logger = log4net.LogManager.GetLogger(typeof(Cloud)); public static void Init(string userAgent = "") { @@ -71,7 +71,7 @@ private static MailRuCloud.Api.MailRuCloud CreateCloud(HttpListenerBasicIdentity Logger.Error($"Cannot load two-factor auth handler {TwoFactorHandlerName}"); } - var cloud = new SplittedCloud(identity.Name, identity.Password, twoFaHandler); + var cloud = new MailRuCloud.Api.MailRuCloud(identity.Name, identity.Password, twoFaHandler); return cloud; } } diff --git a/WebDavMailRuCloudStore/Mailru/MkcolHandler.cs b/WebDavMailRuCloudStore/Mailru/MkcolHandler.cs new file mode 100644 index 00000000..f797a92e --- /dev/null +++ b/WebDavMailRuCloudStore/Mailru/MkcolHandler.cs @@ -0,0 +1,50 @@ +using System.Threading.Tasks; +using NWebDav.Server; +using NWebDav.Server.Helpers; +using NWebDav.Server.Http; +using NWebDav.Server.Stores; + +namespace YaR.WebDavMailRu.CloudStore.Mailru +{ + public class MkcolHandler : IRequestHandler + { + /// + /// Handle a MKCOL request. + /// + /// + /// The HTTP context of the request. + /// + /// + /// Store that is used to access the collections and items. + /// + /// + /// A task that represents the asynchronous MKCOL operation. The task + /// will always return upon completion. + /// + public async Task HandleRequestAsync(IHttpContext httpContext, IStore store) + { + // Obtain request and response + var request = httpContext.Request; + var response = httpContext.Response; + + // The collection must always be created inside another collection + var splitUri = request.Url.Parent; + + // Obtain the parent entry + var collection = await store.GetCollectionAsync(splitUri.Parent, httpContext).ConfigureAwait(false); + if (collection == null) + { + // Source not found + response.SetStatus(DavStatusCode.Conflict); + return true; + } + + // Create the collection + var result = await collection.CreateCollectionAsync(splitUri.Name, false, httpContext).ConfigureAwait(false); + + // Finished + response.SetStatus(result.Result); + return true; + } + } +} \ No newline at end of file diff --git a/WebDavMailRuCloudStore/Mailru/RequestHandlerFactory.cs b/WebDavMailRuCloudStore/Mailru/RequestHandlerFactory.cs index 4744be6a..e5d26cb7 100644 --- a/WebDavMailRuCloudStore/Mailru/RequestHandlerFactory.cs +++ b/WebDavMailRuCloudStore/Mailru/RequestHandlerFactory.cs @@ -13,7 +13,7 @@ public class RequestHandlerFactory : IRequestHandlerFactory { "GET", new NWebDav.Server.Handlers.GetAndHeadHandler() }, { "HEAD", new NWebDav.Server.Handlers.GetAndHeadHandler() }, { "LOCK", new NWebDav.Server.Handlers.LockHandler() }, - { "MKCOL", new NWebDav.Server.Handlers.MkcolHandler() }, + { "MKCOL", new MkcolHandler() }, { "MOVE", new MoveHandler() }, { "OPTIONS", new NWebDav.Server.Handlers.OptionsHandler() }, { "PROPFIND", new NWebDav.Server.Handlers.PropFindHandler() }, diff --git a/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStore.cs b/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStore.cs index 523a3c74..c2bdab18 100644 --- a/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStore.cs +++ b/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStore.cs @@ -32,27 +32,31 @@ public Task GetItemAsync(WebDavUri uri, IHttpContext httpContext) //TODO: clean this trash try { - var item = Cloud.Instance(identity).GetItems(path).Result; + var item = Cloud.Instance(identity).GetItem(path).Result; if (item != null) { - if (item.FullPath == path) - { - var dir = new Folder(item.NumberOfFolders, item.NumberOfFiles, item.Size, path); - return Task.FromResult(new MailruStoreCollection(httpContext, LockingManager, dir, IsWritable)); - } - var fa = item.Files.FirstOrDefault(k => k.FullPath == path); - if (fa != null) - return Task.FromResult(new MailruStoreItem(LockingManager, fa, IsWritable)); + return item.IsFile + ? Task.FromResult(new MailruStoreItem(LockingManager, (File)item, IsWritable)) + : Task.FromResult(new MailruStoreCollection(httpContext, LockingManager, (Folder)item, IsWritable)); - string parentPath = WebDavPath.Parent(path); - item = Cloud.Instance(identity).GetItems(parentPath).Result; - if (item != null) - { - var f = item.Files.FirstOrDefault(k => k.FullPath == path); - return null != f - ? Task.FromResult(new MailruStoreItem(LockingManager, f, IsWritable)) - : null; - } + //if (item.FullPath == path) + //{ + // var dir = new Folder(item.NumberOfFolders, item.NumberOfFiles, item.Size, path); + // return Task.FromResult(new MailruStoreCollection(httpContext, LockingManager, dir, IsWritable)); + //} + //var fa = item.Files.FirstOrDefault(k => k.FullPath == path); + //if (fa != null) + // return Task.FromResult(new MailruStoreItem(LockingManager, fa, IsWritable)); + + //string parentPath = WebDavPath.Parent(path); + //item = Cloud.Instance(identity).GetItems(parentPath).Result; + //if (item != null) + //{ + // var f = item.Files.FirstOrDefault(k => k.FullPath == path); + // return null != f + // ? Task.FromResult(new MailruStoreItem(LockingManager, f, IsWritable)) + // : null; + //} } } @@ -77,7 +81,11 @@ public Task GetItemAsync(WebDavUri uri, IHttpContext httpContext) public Task GetCollectionAsync(WebDavUri uri, IHttpContext httpContext) { var path = uri.Path; - return Task.FromResult(new MailruStoreCollection(httpContext, LockingManager, new Folder(path), IsWritable)); + + var folder = (Folder)Cloud.Instance(httpContext.Session.Principal.Identity) + .GetItem(path, MailRuCloud.Api.MailRuCloud.ItemType.Folder).Result; + + return Task.FromResult(new MailruStoreCollection(httpContext, LockingManager, folder, IsWritable)); } } } diff --git a/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStoreCollection.cs b/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStoreCollection.cs index f13b6404..a39bf887 100644 --- a/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStoreCollection.cs +++ b/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStoreCollection.cs @@ -292,30 +292,13 @@ public Task GetItemAsync(string name, IHttpContext httpContext) public Task> GetItemsAsync(IHttpContext httpContext) { - var item = Cloud.Instance(httpContext.Session.Principal.Identity).GetItems(_directoryInfo).Result; + var list = _directoryInfo.Entries + .Select(entry => entry.IsFile + ? (IStoreItem) new MailruStoreItem(LockingManager, (File) entry, IsWritable) + : new MailruStoreCollection(httpContext, LockingManager, (Folder) entry, IsWritable)) + .ToList(); - var items = item.Folders.Select(subDirectory => new MailruStoreCollection(httpContext, LockingManager, subDirectory, IsWritable)) - .Cast().ToList(); - - items.AddRange(item.Files.Select(file => new MailruStoreItem(LockingManager, file, IsWritable))); - - //var shares = item.Folders - // .Where(dir => !string.IsNullOrEmpty(dir.PublicLink)) - // .Select(dir => dir.FullPath + "\t" + dir.PublicLink) - // .ToList(); - //if (shares.Any()) - //{ - // string sharestr = shares - // .Aggregate((c, n) => c + "\r\n" + n); - - // items.Add(new MailruStoreItem( - // LockingManager, - // new MailRuCloudApi.File(_directoryInfo.FullPath + "/folder.info.wdmrc", sharestr.Length, - // string.Empty), - // false)); - //} - - return Task.FromResult>(items); + return Task.FromResult>(list); } public Task CreateItemAsync(string name, bool overwrite, IHttpContext httpContext) @@ -341,7 +324,8 @@ public Task CreateCollectionAsync(string name, bool overw var destinationPath = WebDavPath.Combine(FullPath, name); - var cmd = SpecialCommandFabric.Build(Cloud.Instance(httpContext.Session.Principal.Identity), destinationPath); + var cmdFabric = new SpecialCommandFabric(); + var cmd = cmdFabric.Build(Cloud.Instance(httpContext.Session.Principal.Identity), destinationPath); if (cmd != null) { var res = cmd.Execute().Result; From bc4f589e735b0ec3a813917513695499a2b42660 Mon Sep 17 00:00:00 2001 From: YaR Date: Thu, 2 Nov 2017 16:51:16 +0300 Subject: [PATCH 02/31] shards caching --- MailRuCloud/MailRuCloudApi/Base/Cached.cs | 46 +++++++++ MailRuCloud/MailRuCloudApi/Base/CloudApi.cs | 31 ++---- .../Base/Requests/ShardInfoRequest.cs | 12 +-- .../MailRuCloudApi/Extensions/DtoImport.cs | 97 +++---------------- 4 files changed, 76 insertions(+), 110 deletions(-) create mode 100644 MailRuCloud/MailRuCloudApi/Base/Cached.cs diff --git a/MailRuCloud/MailRuCloudApi/Base/Cached.cs b/MailRuCloud/MailRuCloudApi/Base/Cached.cs new file mode 100644 index 00000000..9a0c38af --- /dev/null +++ b/MailRuCloud/MailRuCloudApi/Base/Cached.cs @@ -0,0 +1,46 @@ +using System; + +namespace YaR.MailRuCloud.Api.Base +{ + public class Cached + { + private readonly TimeSpan _duration; + private DateTime _expiration; + private Lazy _value; + private readonly Func _valueFactory; + + public T Value + { + get + { + RefreshValueIfNeeded(); + return _value.Value; + } + } + + public Cached(Func valueFactory, TimeSpan duration) + { + _duration = duration; + _valueFactory = valueFactory; + + RefreshValueIfNeeded(); + } + + private readonly object _refreshLock = new object(); + + private void RefreshValueIfNeeded() + { + if (DateTime.Now >= _expiration) + { + lock (_refreshLock) + { + if (DateTime.Now >= _expiration) + { + _value = new Lazy(_valueFactory); + _expiration = DateTime.Now.Add(_duration); + } + } + } + } + } +} \ No newline at end of file diff --git a/MailRuCloud/MailRuCloudApi/Base/CloudApi.cs b/MailRuCloud/MailRuCloudApi/Base/CloudApi.cs index 064cfdad..d170a421 100644 --- a/MailRuCloud/MailRuCloudApi/Base/CloudApi.cs +++ b/MailRuCloud/MailRuCloudApi/Base/CloudApi.cs @@ -1,8 +1,10 @@ using System; +using System.Collections.Generic; using System.Security.Authentication; using System.Threading; using System.Threading.Tasks; using YaR.MailRuCloud.Api.Base.Requests; +using YaR.MailRuCloud.Api.Base.Requests.Types; using YaR.MailRuCloud.Api.Extensions; namespace YaR.MailRuCloud.Api.Base @@ -21,13 +23,7 @@ public class CloudApi : IDisposable /// Gets or sets account to connect with cloud. /// /// Account info. - public Account Account { get; set; } - - //public string DownloadToken { get; set; } - - - - + public Account Account { get; } public CloudApi(string login, string password, ITwoFaHandler twoFaHandler) { @@ -37,24 +33,25 @@ public CloudApi(string login, string password, ITwoFaHandler twoFaHandler) throw new AuthenticationException("Auth token has't been retrieved."); } - // !!!!!!!!!!!!!!!! Account.Info = GetAccountInfo().Result; + _cachedShards = new Cached>(() => new ShardInfoRequest(this).MakeRequestAsync().Result.ToShardInfo(), + TimeSpan.FromMinutes(2)); } - /// /// Get shard info that to do post get request. Can be use for anonymous user. /// /// Shard type as numeric type. - /// To get anonymous user. /// Shard info. - public async Task GetShardInfo(ShardType shardType, bool useAnonymousUser = false) + public async Task GetShardInfo(ShardType shardType) { - var data = await new ShardInfoRequest(this, useAnonymousUser).MakeRequestAsync(); - var shard = data.ToShardInfo(shardType); + var shards = await Task.Run(() => _cachedShards.Value); + var shard = shards[shardType]; Logger.Info($"Shard: ({shardType}){shard.Url}"); return shard; } + private readonly Cached> _cachedShards; + #region IDisposable Support private bool _disposedValue; @@ -75,13 +72,5 @@ public void Dispose() Dispose(true); } #endregion - - } - - - - - - } diff --git a/MailRuCloud/MailRuCloudApi/Base/Requests/ShardInfoRequest.cs b/MailRuCloud/MailRuCloudApi/Base/Requests/ShardInfoRequest.cs index 4827bf9f..3dfdd1af 100644 --- a/MailRuCloud/MailRuCloudApi/Base/Requests/ShardInfoRequest.cs +++ b/MailRuCloud/MailRuCloudApi/Base/Requests/ShardInfoRequest.cs @@ -4,19 +4,19 @@ namespace YaR.MailRuCloud.Api.Base.Requests { class ShardInfoRequest : BaseRequest { - private readonly bool _isAnonymous; - - public ShardInfoRequest(CloudApi cloudApi, bool isAnonymous) : base(cloudApi) + public ShardInfoRequest(CloudApi cloudApi) : base(cloudApi) { - _isAnonymous = isAnonymous; } - protected override string RelationalUri { get { - var uri = string.Format("{0}/api/v2/dispatcher?{2}={1}", ConstSettings.CloudDomain, !_isAnonymous ? CloudApi.Account.AuthToken : 2.ToString(), !_isAnonymous ? "token" : "api"); + //var uri = string.Format("{0}/api/v2/dispatcher?{2}={1}", ConstSettings.CloudDomain, !_isAnonymous ? CloudApi.Account.AuthToken : 2.ToString(), !_isAnonymous ? "token" : "api"); + var uri = string.Format("{0}/api/v2/dispatcher?api=2", ConstSettings.CloudDomain); + var token = CloudApi.Account.AuthToken; + if (!string.IsNullOrEmpty(token)) + uri += $"&token={token}"; return uri; } } diff --git a/MailRuCloud/MailRuCloudApi/Extensions/DtoImport.cs b/MailRuCloud/MailRuCloudApi/Extensions/DtoImport.cs index d1e85cfe..1e082724 100644 --- a/MailRuCloud/MailRuCloudApi/Extensions/DtoImport.cs +++ b/MailRuCloud/MailRuCloudApi/Extensions/DtoImport.cs @@ -23,97 +23,28 @@ public static DiskUsage ToDiskUsage(this AccountInfoResult data) - public static ShardInfo ToShardInfo(this ShardInfoResult webdata, ShardType shardType) + public static Dictionary ToShardInfo(this ShardInfoResult webdata) { - List shard; - - switch (shardType) - { - case ShardType.Video: - shard = webdata.body.video; - break; - case ShardType.ViewDirect: - shard = webdata.body.view_direct; - break; - case ShardType.WeblinkView: - shard = webdata.body.weblink_view; - break; - case ShardType.WeblinkVideo: - shard = webdata.body.weblink_video; - break; - case ShardType.WeblinkGet: - shard = webdata.body.weblink_get; - break; - case ShardType.WeblinkThumbnails: - shard = webdata.body.weblink_thumbnails; - break; - case ShardType.Auth: - shard = webdata.body.auth; - break; - case ShardType.View: - shard = webdata.body.view; - break; - case ShardType.Get: - shard = webdata.body.get; - break; - case ShardType.Upload: - shard = webdata.body.upload; - break; - case ShardType.Thumbnails: - shard = webdata.body.thumbnails; - break; - default: - throw new ArgumentOutOfRangeException(nameof(shardType), shardType, null); - } - - if (null == shard || shard.Count == 0) - throw new Exception("Cannot get shard info"); - - var res = new ShardInfo + var dict = new Dictionary { - Type = shardType, - Count = int.Parse(shard[0].count), - Url = shard[0].url - + {ShardType.Video, new ShardInfo{Type = ShardType.Video, Url = webdata.body.video[0].url} }, + {ShardType.ViewDirect, new ShardInfo{Type = ShardType.ViewDirect, Url = webdata.body.view_direct[0].url} }, + {ShardType.WeblinkView, new ShardInfo{Type = ShardType.WeblinkView, Url = webdata.body.weblink_view[0].url} }, + {ShardType.WeblinkVideo, new ShardInfo{Type = ShardType.WeblinkVideo, Url = webdata.body.weblink_video[0].url} }, + {ShardType.WeblinkGet, new ShardInfo{Type = ShardType.WeblinkGet, Url = webdata.body.weblink_get[0].url} }, + {ShardType.WeblinkThumbnails, new ShardInfo{Type = ShardType.WeblinkThumbnails, Url = webdata.body.weblink_thumbnails[0].url} }, + {ShardType.Auth, new ShardInfo{Type = ShardType.Auth, Url = webdata.body.auth[0].url} }, + {ShardType.View, new ShardInfo{Type = ShardType.View, Url = webdata.body.view[0].url} }, + {ShardType.Get, new ShardInfo{Type = ShardType.Get, Url = webdata.body.get[0].url} }, + {ShardType.Upload, new ShardInfo{Type = ShardType.Upload, Url = webdata.body.upload[0].url} }, + {ShardType.Thumbnails, new ShardInfo{Type = ShardType.Thumbnails, Url = webdata.body.thumbnails[0].url} } }; - return res; + return dict; } private static readonly string[] FolderKinds = { "folder", "camera-upload", "mounted", "shared" }; - //public static Entry ToEntry(this FolderInfoResult data) - //{ - // var entry = new Entry( - // data.body.list? - // .Where(it => FolderKinds.Contains(it.kind)) - // .Select(it => new Folder(it.count.folders, it.count.files, it.size, it.home, string.IsNullOrEmpty(it.weblink) ? "" : ConstSettings.PublishFileLink + it.weblink)) - // .ToList(), - // data.body.list? - // .Where(it => it.kind == "file") - // .Select(it => new File(it.home, it.size, it.hash) - // { - // PublicLink = - // string.IsNullOrEmpty(it.weblink) ? "" : ConstSettings.PublishFileLink + it.weblink, - // CreationTimeUtc = UnixTimeStampToDateTime(it.mtime), - // LastAccessTimeUtc = UnixTimeStampToDateTime(it.mtime), - // LastWriteTimeUtc = UnixTimeStampToDateTime(it.mtime), - // }).ToList(), - // data.body.home) - // { - // CreationDate = DateTime.Now, - // Name = data.body.name, - // IsFile = data.body.kind == "file", - // Size = data.body.size, - // //WebLink = data.body. - // //WebLink = (data.body?.list != null && data.body.list.Count > 0) - // // ? data.body.list[0].weblink - // // : string.Empty - // }; - - // return entry; - //} - public static IEntry ToEntry(this FolderInfoResult data) { if (data.body.kind == "file") From 9a4b59a31755293403ce09b9b4e0d88800f4ee0f Mon Sep 17 00:00:00 2001 From: YaR Date: Thu, 2 Nov 2017 17:12:07 +0300 Subject: [PATCH 03/31] download token cached --- MailRuCloud/MailRuCloudApi/Base/Account.cs | 42 +++++++++++-------- .../MailRuCloudApi/Base/DownloadStream.cs | 2 +- .../MailRuCloudApi/Extensions/DtoImport.cs | 6 ++- 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/MailRuCloud/MailRuCloudApi/Base/Account.cs b/MailRuCloud/MailRuCloudApi/Base/Account.cs index cf71e8e4..c1d69abf 100644 --- a/MailRuCloud/MailRuCloudApi/Base/Account.cs +++ b/MailRuCloud/MailRuCloudApi/Base/Account.cs @@ -3,6 +3,7 @@ using System.Security.Authentication; using System.Threading.Tasks; using YaR.MailRuCloud.Api.Base.Requests; +using YaR.MailRuCloud.Api.Extensions; namespace YaR.MailRuCloud.Api.Base { @@ -11,6 +12,8 @@ namespace YaR.MailRuCloud.Api.Base /// public class Account { + private static readonly log4net.ILog Logger = log4net.LogManager.GetLogger(typeof(Account)); + private readonly CloudApi _cloudApi; /// @@ -39,6 +42,10 @@ public Account(CloudApi cloudApi, string login, string password, ITwoFaHandler t var twoFaHandler1 = twoFaHandler; if (twoFaHandler1 != null) AuthCodeRequiredEvent += twoFaHandler1.Get; + + + DownloadToken = new Cached(() => new DownloadTokenRequest(_cloudApi).MakeRequestAsync().Result.ToToken(), + TimeSpan.FromSeconds(DownloadTokenExpiresSec)); } /// @@ -127,26 +134,27 @@ public async Task LoginAsync() return true; } + public readonly Cached DownloadToken; + private const int DownloadTokenExpiresSec = 2 * 60 * 60; + public DateTime TokenExpiresAt { get; private set; } private const int TokenExpiresInSec = 23 * 60 * 60; - - public string DownloadToken - { - get - { - if (string.IsNullOrEmpty(_downloadToken) || (DateTime.Now - _downloadTokenDate).TotalSeconds > DownloadTokenExpiresSec) - { - _downloadTokenDate = DateTime.Now; - var dtres = new DownloadTokenRequest(_cloudApi).MakeRequestAsync().Result; - _downloadToken = dtres.body.token; - } - return _downloadToken; - } - } - private string _downloadToken; - private DateTime _downloadTokenDate = DateTime.MinValue; - private const int DownloadTokenExpiresSec = 2 * 60 * 60; + //public string DownloadToken + //{ + // get + // { + // if (string.IsNullOrEmpty(_downloadToken) || (DateTime.Now - _downloadTokenDate).TotalSeconds > DownloadTokenExpiresSec) + // { + // _downloadTokenDate = DateTime.Now; + // _downloadToken = new DownloadTokenRequest(_cloudApi).MakeRequestAsync().Result.ToToken(); + // } + // return _downloadToken; + // } + //} + //private string _downloadToken; + //private DateTime _downloadTokenDate = DateTime.MinValue; + /// /// Need to add this function for all calls. diff --git a/MailRuCloud/MailRuCloudApi/Base/DownloadStream.cs b/MailRuCloud/MailRuCloudApi/Base/DownloadStream.cs index f6688383..f123e8a1 100644 --- a/MailRuCloud/MailRuCloudApi/Base/DownloadStream.cs +++ b/MailRuCloud/MailRuCloudApi/Base/DownloadStream.cs @@ -96,7 +96,7 @@ private async Task GetFileStream() { //var dtres = new DownloadTokenHtmlRequest(_cloud, file.PublicLink).MakeRequestAsync().Result; //downloadkey = dtres.body.token; - downloadkey = _cloud.Account.DownloadToken; + downloadkey = _cloud.Account.DownloadToken.Value; } var request = _shard.Type == ShardType.Get diff --git a/MailRuCloud/MailRuCloudApi/Extensions/DtoImport.cs b/MailRuCloud/MailRuCloudApi/Extensions/DtoImport.cs index 1e082724..819f07f3 100644 --- a/MailRuCloud/MailRuCloudApi/Extensions/DtoImport.cs +++ b/MailRuCloud/MailRuCloudApi/Extensions/DtoImport.cs @@ -20,7 +20,11 @@ public static DiskUsage ToDiskUsage(this AccountInfoResult data) return res; } - + public static string ToToken(this DownloadTokenResult data) + { + var res = data.body.token; + return res; + } public static Dictionary ToShardInfo(this ShardInfoResult webdata) From 9a795b241cd943df40110587e9f5d1aa535d114f Mon Sep 17 00:00:00 2001 From: YaR Date: Thu, 2 Nov 2017 17:39:45 +0300 Subject: [PATCH 04/31] auth token cached --- MailRuCloud/MailRuCloudApi/Base/Account.cs | 81 +++++-------------- MailRuCloud/MailRuCloudApi/Base/Cached.cs | 5 ++ .../Base/Requests/ShardInfoRequest.cs | 3 +- .../MailRuCloudApi/Extensions/DtoImport.cs | 7 ++ .../MailRuCloudApi/Extensions/Extensions.cs | 9 +++ WebDavMailRuCloudStore/Cloud.cs | 15 +--- 6 files changed, 43 insertions(+), 77 deletions(-) diff --git a/MailRuCloud/MailRuCloudApi/Base/Account.cs b/MailRuCloud/MailRuCloudApi/Base/Account.cs index c1d69abf..e4e9c009 100644 --- a/MailRuCloud/MailRuCloudApi/Base/Account.cs +++ b/MailRuCloud/MailRuCloudApi/Base/Account.cs @@ -1,6 +1,5 @@ using System; using System.Net; -using System.Security.Authentication; using System.Threading.Tasks; using YaR.MailRuCloud.Api.Base.Requests; using YaR.MailRuCloud.Api.Extensions; @@ -12,8 +11,6 @@ namespace YaR.MailRuCloud.Api.Base /// public class Account { - private static readonly log4net.ILog Logger = log4net.LogManager.GetLogger(typeof(Account)); - private readonly CloudApi _cloudApi; /// @@ -21,8 +18,6 @@ public class Account /// private CookieContainer _cookies; - //private readonly AuthCodeWindow _authCodeHandler = new AuthCodeWindow(); - /// /// Initializes a new instance of the class. /// @@ -46,19 +41,16 @@ public Account(CloudApi cloudApi, string login, string password, ITwoFaHandler t DownloadToken = new Cached(() => new DownloadTokenRequest(_cloudApi).MakeRequestAsync().Result.ToToken(), TimeSpan.FromSeconds(DownloadTokenExpiresSec)); + + AuthToken = new Cached(() => new AuthTokenRequest(_cloudApi).MakeRequestAsync().Result.ToToken(), + TimeSpan.FromSeconds(AuthTokenExpiresInSec)); } /// - /// Gets or sets connection proxy. + /// Gets connection proxy. /// /// Proxy settings. - public IWebProxy Proxy { get; set; } - - /// - /// Gets authorization token. - /// - /// Access token. - public string AuthToken { get; private set; } + public IWebProxy Proxy { get; } /// /// Gets account cookies. @@ -70,15 +62,15 @@ public Account(CloudApi cloudApi, string login, string password, ITwoFaHandler t /// Gets or sets login name. /// /// Account email. - public string LoginName { get; set; } + public string LoginName { get; } /// /// Gets or sets email password. /// /// Password related with login. - public string Password { get; set; } + private string Password { get; } - public AccountInfo Info { get; set; } + public AccountInfo Info { get; private set; } /// /// Authorize on MAIL.RU server. @@ -119,56 +111,29 @@ public async Task LoginAsync() await new EnsureSdcCookieRequest(_cloudApi) .MakeRequestAsync(); - AuthToken = new AuthTokenRequest(_cloudApi) - .MakeRequestAsync() - .ThrowIf(data => string.IsNullOrEmpty(data.body?.token), new AuthenticationException("Empty auth token")) - .body.token; - Info = new AccountInfo { FileSizeLimit = new AccountInfoRequest(_cloudApi).MakeRequestAsync().Result.body.cloud.file_size_limit }; - TokenExpiresAt = DateTime.Now.AddHours(TokenExpiresInSec); - return true; } - public readonly Cached DownloadToken; - private const int DownloadTokenExpiresSec = 2 * 60 * 60; - - public DateTime TokenExpiresAt { get; private set; } - private const int TokenExpiresInSec = 23 * 60 * 60; - - //public string DownloadToken - //{ - // get - // { - // if (string.IsNullOrEmpty(_downloadToken) || (DateTime.Now - _downloadTokenDate).TotalSeconds > DownloadTokenExpiresSec) - // { - // _downloadTokenDate = DateTime.Now; - // _downloadToken = new DownloadTokenRequest(_cloudApi).MakeRequestAsync().Result.ToToken(); - // } - // return _downloadToken; - // } - //} - //private string _downloadToken; - //private DateTime _downloadTokenDate = DateTime.MinValue; - + /// + /// Token for authorization + /// + public readonly Cached AuthToken; + private const int AuthTokenExpiresInSec = 23 * 60 * 60; /// - /// Need to add this function for all calls. + /// Token for downloading files /// - internal void CheckAuth() - { - if (LoginName == null || Password == null) - throw new AuthenticationException("Login or password is empty."); + public readonly Cached DownloadToken; + private const int DownloadTokenExpiresSec = 2 * 60 * 60; - if (string.IsNullOrEmpty(AuthToken)) - if (!Login()) - throw new AuthenticationException("Auth token has't been retrieved."); - } + + public delegate string AuthCodeRequiredDelegate(string login, bool isAutoRelogin); @@ -178,14 +143,4 @@ protected virtual string OnAuthCodeRequired(string login, bool isAutoRelogin) return AuthCodeRequiredEvent?.Invoke(login, isAutoRelogin); } } - - public static class Extensions - { - public static T ThrowIf(this Task data, Func func, Exception ex) - { - var res = data.Result; - if (func(res)) throw ex; - return res; - } - } } diff --git a/MailRuCloud/MailRuCloudApi/Base/Cached.cs b/MailRuCloud/MailRuCloudApi/Base/Cached.cs index 9a0c38af..290c8b43 100644 --- a/MailRuCloud/MailRuCloudApi/Base/Cached.cs +++ b/MailRuCloud/MailRuCloudApi/Base/Cached.cs @@ -42,5 +42,10 @@ private void RefreshValueIfNeeded() } } } + + public override string ToString() + { + return Value.ToString(); + } } } \ No newline at end of file diff --git a/MailRuCloud/MailRuCloudApi/Base/Requests/ShardInfoRequest.cs b/MailRuCloud/MailRuCloudApi/Base/Requests/ShardInfoRequest.cs index 3dfdd1af..ccfb08c5 100644 --- a/MailRuCloud/MailRuCloudApi/Base/Requests/ShardInfoRequest.cs +++ b/MailRuCloud/MailRuCloudApi/Base/Requests/ShardInfoRequest.cs @@ -12,9 +12,8 @@ protected override string RelationalUri { get { - //var uri = string.Format("{0}/api/v2/dispatcher?{2}={1}", ConstSettings.CloudDomain, !_isAnonymous ? CloudApi.Account.AuthToken : 2.ToString(), !_isAnonymous ? "token" : "api"); var uri = string.Format("{0}/api/v2/dispatcher?api=2", ConstSettings.CloudDomain); - var token = CloudApi.Account.AuthToken; + var token = CloudApi.Account.AuthToken.Value; if (!string.IsNullOrEmpty(token)) uri += $"&token={token}"; return uri; diff --git a/MailRuCloud/MailRuCloudApi/Extensions/DtoImport.cs b/MailRuCloud/MailRuCloudApi/Extensions/DtoImport.cs index 819f07f3..1f479dbe 100644 --- a/MailRuCloud/MailRuCloudApi/Extensions/DtoImport.cs +++ b/MailRuCloud/MailRuCloudApi/Extensions/DtoImport.cs @@ -20,6 +20,13 @@ public static DiskUsage ToDiskUsage(this AccountInfoResult data) return res; } + public static string ToToken(this AuthTokenResult data) + { + var res = data.body.token; + return res; + } + + public static string ToToken(this DownloadTokenResult data) { var res = data.body.token; diff --git a/MailRuCloud/MailRuCloudApi/Extensions/Extensions.cs b/MailRuCloud/MailRuCloudApi/Extensions/Extensions.cs index 6d1e38b0..ddcb6daf 100644 --- a/MailRuCloud/MailRuCloudApi/Extensions/Extensions.cs +++ b/MailRuCloud/MailRuCloudApi/Extensions/Extensions.cs @@ -1,9 +1,18 @@ using System; +using System.Threading.Tasks; namespace YaR.MailRuCloud.Api.Extensions { public static class Extensions { + public static T ThrowIf(this Task data, Func func, Exception ex) + { + var res = data.Result; + if (func(res)) throw ex; + return res; + } + + /// /// Finds the first exception of the requested type. /// diff --git a/WebDavMailRuCloudStore/Cloud.cs b/WebDavMailRuCloudStore/Cloud.cs index 7872c78f..48651eed 100644 --- a/WebDavMailRuCloudStore/Cloud.cs +++ b/WebDavMailRuCloudStore/Cloud.cs @@ -1,5 +1,4 @@ -using System; -using System.Collections.Concurrent; +using System.Collections.Concurrent; using System.Linq; using System.Net; using System.Security.Principal; @@ -25,16 +24,10 @@ public static void Init(string userAgent = "") public static MailRuCloud.Api.MailRuCloud Instance(IIdentity identityi) { var identity = (HttpListenerBasicIdentity) identityi; - //HttpListenerBasicIdentity identity = (HttpListenerBasicIdentity)context.Session.Principal.Identity; string key = identity.Name + identity.Password; if (CloudCache.TryGetValue(key, out var cloud)) - { - if (cloud.CloudApi.Account.TokenExpiresAt <= DateTime.Now) - CloudCache.TryRemove(key, out cloud); - else - return cloud; - } + return cloud; lock (Locker) { @@ -60,10 +53,8 @@ private static MailRuCloud.Api.MailRuCloud CreateCloud(HttpListenerBasicIdentity Logger.Warn($"Missing domain part ({domains}) in login, file and folder deleting will be denied"); } - - //2FA + //2FA authorization ITwoFaHandler twoFaHandler = null; - if (!string.IsNullOrEmpty(TwoFactorHandlerName)) { twoFaHandler = TwoFaHandlers.Get(TwoFactorHandlerName); From 866a1d7528fce1b6049166569f8f0c32ba8f1e27 Mon Sep 17 00:00:00 2001 From: YaR Date: Thu, 2 Nov 2017 17:44:45 +0300 Subject: [PATCH 05/31] account info refactored a bit --- MailRuCloud/MailRuCloudApi/Base/Account.cs | 7 +++---- MailRuCloud/MailRuCloudApi/Extensions/DtoImport.cs | 10 ++++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/MailRuCloud/MailRuCloudApi/Base/Account.cs b/MailRuCloud/MailRuCloudApi/Base/Account.cs index e4e9c009..d8c9018f 100644 --- a/MailRuCloud/MailRuCloudApi/Base/Account.cs +++ b/MailRuCloud/MailRuCloudApi/Base/Account.cs @@ -111,10 +111,9 @@ public async Task LoginAsync() await new EnsureSdcCookieRequest(_cloudApi) .MakeRequestAsync(); - Info = new AccountInfo - { - FileSizeLimit = new AccountInfoRequest(_cloudApi).MakeRequestAsync().Result.body.cloud.file_size_limit - }; + Info = (await new AccountInfoRequest(_cloudApi) + .MakeRequestAsync()) + .ToAccountInfo(); return true; } diff --git a/MailRuCloud/MailRuCloudApi/Extensions/DtoImport.cs b/MailRuCloud/MailRuCloudApi/Extensions/DtoImport.cs index 1f479dbe..b30e2710 100644 --- a/MailRuCloud/MailRuCloudApi/Extensions/DtoImport.cs +++ b/MailRuCloud/MailRuCloudApi/Extensions/DtoImport.cs @@ -9,6 +9,16 @@ namespace YaR.MailRuCloud.Api.Extensions { public static class DtoImport { + public static AccountInfo ToAccountInfo(this AccountInfoResult data) + { + var res = new AccountInfo + { + FileSizeLimit = data.body.cloud.file_size_limit + }; + return res; + } + + public static DiskUsage ToDiskUsage(this AccountInfoResult data) { var res = new DiskUsage From 30f1907754eebfe4084932dba3afb6519b793df9 Mon Sep 17 00:00:00 2001 From: YaR Date: Thu, 2 Nov 2017 18:06:21 +0300 Subject: [PATCH 06/31] environment params logging --- WDMRC.Console/Program.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/WDMRC.Console/Program.cs b/WDMRC.Console/Program.cs index 9f0c29b0..942a7db1 100644 --- a/WDMRC.Console/Program.cs +++ b/WDMRC.Console/Program.cs @@ -25,14 +25,13 @@ static void Main(string[] args) LoggerFactory.Factory = new Log4NetAdapter(); - ShowInfo(); - var result = Parser.Default.ParseArguments(args); var exitCode = result .MapResult( options => { + ShowInfo(options); Cloud.Init(options.UserAgent); Cloud.TwoFactorHandlerName = Config.TwoFactorAuthHandlerName; @@ -137,7 +136,7 @@ private static async Task DispatchHttpRequestsAsync(HttpListener httpListener, C } - private static void ShowInfo() + private static void ShowInfo(CommandLineOptions options) { string title = GetAssemblyAttribute(a => a.Product); string description = GetAssemblyAttribute(a => a.Description); @@ -147,6 +146,12 @@ private static void ShowInfo() System.Console.WriteLine($" {title}: {description}"); System.Console.WriteLine($" v.{version}"); System.Console.WriteLine($" {copyright}"); + + Logger.Info($"OS Version: {Environment.OSVersion}"); + Logger.Info($"CLR Version: {Environment.Version}"); + Logger.Info($"User interactive: {Environment.UserInteractive}"); + Logger.Info($"Version: {version}"); + Logger.Info($"Max threads count: {options.MaxThreadCount}"); } private static string GetAssemblyAttribute(Func value) where T : Attribute From 3106757abd865e425ef41422ceda2ea2b1d33b5a Mon Sep 17 00:00:00 2001 From: YaR Date: Thu, 2 Nov 2017 19:45:20 +0300 Subject: [PATCH 07/31] caching file info for speeding up bunches of partial reads --- .../Mailru/GetAndHeadHandler.cs | 194 ++++++++++++++++++ .../Mailru/RequestHandlerFactory.cs | 2 +- .../Mailru/StoreBase/MailruStore.cs | 7 + .../Mailru/StoreBase/StoreItemCache.cs | 96 +++++++++ 4 files changed, 298 insertions(+), 1 deletion(-) create mode 100644 WebDavMailRuCloudStore/Mailru/GetAndHeadHandler.cs create mode 100644 WebDavMailRuCloudStore/Mailru/StoreBase/StoreItemCache.cs diff --git a/WebDavMailRuCloudStore/Mailru/GetAndHeadHandler.cs b/WebDavMailRuCloudStore/Mailru/GetAndHeadHandler.cs new file mode 100644 index 00000000..e208e9b2 --- /dev/null +++ b/WebDavMailRuCloudStore/Mailru/GetAndHeadHandler.cs @@ -0,0 +1,194 @@ +using System; +using System.Globalization; +using System.IO; +using System.Threading.Tasks; +using NWebDav.Server; +using NWebDav.Server.Helpers; +using NWebDav.Server.Http; +using NWebDav.Server.Props; +using NWebDav.Server.Stores; +using YaR.MailRuCloud.Api.Base; +using YaR.WebDavMailRu.CloudStore.Mailru.StoreBase; + +namespace YaR.WebDavMailRu.CloudStore.Mailru +{ + + /// + /// Implementation of the GET and HEAD method. + /// + /// + /// The specification of the WebDAV GET and HEAD methods for collections + /// can be found in the + /// + /// WebDAV specification + /// . + /// + public class GetAndHeadHandler : IRequestHandler + { + /// + /// Handle a GET or HEAD request. + /// + /// + /// The HTTP context of the request. + /// + /// + /// Store that is used to access the collections and items. + /// + /// + /// A task that represents the asynchronous GET or HEAD operation. The + /// task will always return upon completion. + /// + public async Task HandleRequestAsync(IHttpContext httpContext, IStore store) + { + // Obtain request and response + var request = httpContext.Request; + var response = httpContext.Response; + + // Determine if we are invoked as HEAD + var head = request.HttpMethod == "HEAD"; + + // Determine the requested range + var range = request.GetRange(); + + // Obtain the WebDAV collection + var mrstore = (MailruStore) store; + //TODO: refact + var entry = mrstore.ItemCache.Get(request.Url, httpContext); // GetItemAsync(request.Url, httpContext).ConfigureAwait(false); + //var entry = await store.GetItemAsync(request.Url, httpContext).ConfigureAwait(false); + if (entry == null) + { + // Set status to not found + response.SetStatus(DavStatusCode.NotFound); + return true; + } + + // Add non-expensive headers based on properties + var propertyManager = entry.PropertyManager; + if (propertyManager != null) + { + // Add Last-Modified header + var lastModifiedUtc = (string)(await propertyManager.GetPropertyAsync(httpContext, entry, DavGetLastModified.PropertyName, true).ConfigureAwait(false)); + if (lastModifiedUtc != null) + response.SetHeaderValue("Last-Modified", lastModifiedUtc); + + // Add ETag + var etag = (string)(await propertyManager.GetPropertyAsync(httpContext, entry, DavGetEtag.PropertyName, true).ConfigureAwait(false)); + if (etag != null) + response.SetHeaderValue("Etag", etag); + + // Add type + var contentType = (string)(await propertyManager.GetPropertyAsync(httpContext, entry, DavGetContentType.PropertyName, true).ConfigureAwait(false)); + if (contentType != null) + response.SetHeaderValue("Content-Type", contentType); + + // Add language + var contentLanguage = (string)(await propertyManager.GetPropertyAsync(httpContext, entry, DavGetContentLanguage.PropertyName, true).ConfigureAwait(false)); + if (contentLanguage != null) + response.SetHeaderValue("Content-Language", contentLanguage); + } + + // Stream the actual entry + using (var stream = await entry.GetReadableStreamAsync(httpContext).ConfigureAwait(false)) + { + if (stream != null && stream != Stream.Null) + { + // Set the response + response.SetStatus(DavStatusCode.Ok); + + // Set the expected content length + try + { + // We can only specify the Content-Length header if the + // length is known (this is typically true for seekable streams) + if (stream.CanSeek) + { + // Add a header that we accept ranges (bytes only) + response.SetHeaderValue("Accept-Ranges", "bytes"); + + // Determine the total length + var length = stream.Length; + + // Check if an 'If-Range' was specified + if (range?.If != null) + { + var lastModifiedText = (string)await propertyManager.GetPropertyAsync(httpContext, entry, DavGetLastModified.PropertyName, true).ConfigureAwait(false); + var lastModified = DateTime.Parse(lastModifiedText, CultureInfo.InvariantCulture); + if (lastModified != range.If) + range = null; + } + + // Check if a range was specified + if (range != null) + { + var start = range.Start ?? 0; + var end = Math.Min(range.End ?? long.MaxValue, length - 1); + length = end - start + 1; + + // Write the range + response.SetHeaderValue("Content-Range", $"bytes {start}-{end} / {stream.Length}"); + + // Set status to partial result if not all data can be sent + if (length < stream.Length) + response.SetStatus(DavStatusCode.PartialContent); + } + + // Set the header, so the client knows how much data is required + response.SetHeaderValue("Content-Length", $"{length}"); + } + } + catch (NotSupportedException) + { + // If the content length is not supported, then we just skip it + } + + // HEAD method doesn't require the actual item data + if (!head) + await CopyToAsync(stream, response.Stream, range?.Start ?? 0, range?.End).ConfigureAwait(false); + } + else + { + // Set the response + response.SetStatus(DavStatusCode.NoContent); + } + } + return true; + } + + private async Task CopyToAsync(Stream src, Stream dest, long start, long? end) + { + // Skip to the first offset + if (start > 0) + { + // We prefer seeking instead of draining data + if (!src.CanSeek) + throw new IOException("Cannot use range, because the source stream isn't seekable"); + + src.Seek(start, SeekOrigin.Begin); + } + + // Determine the number of bytes to read + var bytesToRead = end.HasValue ? end.Value - start + 1 : long.MaxValue; + + // Read in 64KB blocks + var buffer = new byte[64 * 1024]; + + // Copy, until we don't get any data anymore + while (bytesToRead > 0) + { + // Read the requested bytes into memory + var requestedBytes = (int)Math.Min(bytesToRead, buffer.Length); + var bytesRead = await src.ReadAsync(buffer, 0, requestedBytes).ConfigureAwait(false); + + // We're done, if we cannot read any data anymore + if (bytesRead == 0) + return; + + // Write the data to the destination stream + await dest.WriteAsync(buffer, 0, bytesRead).ConfigureAwait(false); + + // Decrement the number of bytes left to read + bytesToRead -= bytesRead; + } + } + } +} \ No newline at end of file diff --git a/WebDavMailRuCloudStore/Mailru/RequestHandlerFactory.cs b/WebDavMailRuCloudStore/Mailru/RequestHandlerFactory.cs index e5d26cb7..012bd608 100644 --- a/WebDavMailRuCloudStore/Mailru/RequestHandlerFactory.cs +++ b/WebDavMailRuCloudStore/Mailru/RequestHandlerFactory.cs @@ -10,7 +10,7 @@ public class RequestHandlerFactory : IRequestHandlerFactory { { "COPY", new NWebDav.Server.Handlers.CopyHandler() }, { "DELETE", new DeleteHandler() }, - { "GET", new NWebDav.Server.Handlers.GetAndHeadHandler() }, + { "GET", new GetAndHeadHandler() }, { "HEAD", new NWebDav.Server.Handlers.GetAndHeadHandler() }, { "LOCK", new NWebDav.Server.Handlers.LockHandler() }, { "MKCOL", new MkcolHandler() }, diff --git a/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStore.cs b/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStore.cs index c2bdab18..f337012a 100644 --- a/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStore.cs +++ b/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStore.cs @@ -17,11 +17,18 @@ public MailruStore(bool isWritable = true, ILockingManager lockingManager = null { LockingManager = lockingManager ?? new InMemoryLockingManager(); IsWritable = isWritable; + + ItemCache = new StoreItemCache(this, TimeSpan.FromSeconds(20)) {CleanUpPeriod = TimeSpan.FromMinutes(1)}; } private bool IsWritable { get; } private ILockingManager LockingManager { get; } + /// + /// Caching files for multiple small reads + /// + public StoreItemCache ItemCache { get; } + public Task GetItemAsync(WebDavUri uri, IHttpContext httpContext) { //TODO: Refact diff --git a/WebDavMailRuCloudStore/Mailru/StoreBase/StoreItemCache.cs b/WebDavMailRuCloudStore/Mailru/StoreBase/StoreItemCache.cs new file mode 100644 index 00000000..35a55c22 --- /dev/null +++ b/WebDavMailRuCloudStore/Mailru/StoreBase/StoreItemCache.cs @@ -0,0 +1,96 @@ +using System; +using System.Collections.Concurrent; +using System.Linq; +using System.Threading; +using NWebDav.Server; +using NWebDav.Server.Http; +using NWebDav.Server.Stores; + +namespace YaR.WebDavMailRu.CloudStore.Mailru.StoreBase +{ + //TODO: not thread-safe, refact + public class StoreItemCache + { + public StoreItemCache(IStore store, TimeSpan expirePeriod) + { + _store = store; + _expirePeriod = expirePeriod; + + long cleanPreiod = (long) CleanUpPeriod.TotalMilliseconds; + _cleanTimer = new Timer(state => + { + if (!_items.Any()) return; + foreach (var item in _items) + { + if (DateTime.Now - item.Value.Created > TimeSpan.FromMinutes(5)) + { + _items.TryRemove(item.Key, out _); + } + } + }, null, cleanPreiod, cleanPreiod); + } + + private readonly IStore _store; + private readonly Timer _cleanTimer; + private readonly ConcurrentDictionary> _items = new ConcurrentDictionary>(); + private readonly object _locker = new object(); + + public TimeSpan CleanUpPeriod + { + get => _cleanUpPeriod; + set + { + _cleanUpPeriod = value; + long cleanPreiod = (long)value.TotalMilliseconds; + _cleanTimer.Change(cleanPreiod, cleanPreiod); + } + } + private TimeSpan _cleanUpPeriod = TimeSpan.FromMinutes(5); + + public IStoreItem Get(WebDavUri uri, IHttpContext context) + { + if (_items.TryGetValue(uri.AbsoluteUri, out var item)) + { + if (IsExpired(item)) + _items.TryRemove(uri.AbsoluteUri, out item); + else + return item.Item; + } + + lock (_locker) + { + if (!_items.TryGetValue(uri.AbsoluteUri, out item)) + { + item = new TimedItem + { + Created = DateTime.Now, + Item = _store.GetItemAsync(uri, context).Result + }; + + if (!_items.TryAdd(uri.AbsoluteUri, item)) + _items.TryGetValue(uri.AbsoluteUri, out item); + } + } + + return item.Item; + } + + private bool IsExpired(TimedItem item) + { + return DateTime.Now - item.Created > _expirePeriod; + } + + private readonly TimeSpan _expirePeriod; + private readonly TimeSpan _cleanupPeriod; + + + private class TimedItem + { + public DateTime Created { get; set; } + public T Item { get; set; } + } + } + + + +} \ No newline at end of file From a2bee3ce119ef67ad2a32e55f429b2be23a245bd Mon Sep 17 00:00:00 2001 From: YaR Date: Thu, 2 Nov 2017 19:48:30 +0300 Subject: [PATCH 08/31] code clean a bit --- WebDavMailRuCloudStore/Mailru/StoreBase/StoreItemCache.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/WebDavMailRuCloudStore/Mailru/StoreBase/StoreItemCache.cs b/WebDavMailRuCloudStore/Mailru/StoreBase/StoreItemCache.cs index 35a55c22..b898e92c 100644 --- a/WebDavMailRuCloudStore/Mailru/StoreBase/StoreItemCache.cs +++ b/WebDavMailRuCloudStore/Mailru/StoreBase/StoreItemCache.cs @@ -45,7 +45,6 @@ public TimeSpan CleanUpPeriod _cleanTimer.Change(cleanPreiod, cleanPreiod); } } - private TimeSpan _cleanUpPeriod = TimeSpan.FromMinutes(5); public IStoreItem Get(WebDavUri uri, IHttpContext context) { @@ -81,8 +80,7 @@ private bool IsExpired(TimedItem item) } private readonly TimeSpan _expirePeriod; - private readonly TimeSpan _cleanupPeriod; - + private TimeSpan _cleanUpPeriod = TimeSpan.FromMinutes(5); private class TimedItem { From 99cd9b197d9b43688cc7af0c90dcfbaa8e891780 Mon Sep 17 00:00:00 2001 From: YaR Date: Thu, 2 Nov 2017 19:54:35 +0300 Subject: [PATCH 09/31] item cache cleanup logging --- WebDavMailRuCloudStore/Mailru/StoreBase/MailruStore.cs | 2 +- WebDavMailRuCloudStore/Mailru/StoreBase/StoreItemCache.cs | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStore.cs b/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStore.cs index f337012a..9ec7b2d9 100644 --- a/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStore.cs +++ b/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStore.cs @@ -18,7 +18,7 @@ public MailruStore(bool isWritable = true, ILockingManager lockingManager = null LockingManager = lockingManager ?? new InMemoryLockingManager(); IsWritable = isWritable; - ItemCache = new StoreItemCache(this, TimeSpan.FromSeconds(20)) {CleanUpPeriod = TimeSpan.FromMinutes(1)}; + ItemCache = new StoreItemCache(this, TimeSpan.FromSeconds(20)) {CleanUpPeriod = TimeSpan.FromMinutes(5)}; } private bool IsWritable { get; } diff --git a/WebDavMailRuCloudStore/Mailru/StoreBase/StoreItemCache.cs b/WebDavMailRuCloudStore/Mailru/StoreBase/StoreItemCache.cs index b898e92c..c48f2093 100644 --- a/WebDavMailRuCloudStore/Mailru/StoreBase/StoreItemCache.cs +++ b/WebDavMailRuCloudStore/Mailru/StoreBase/StoreItemCache.cs @@ -11,6 +11,8 @@ namespace YaR.WebDavMailRu.CloudStore.Mailru.StoreBase //TODO: not thread-safe, refact public class StoreItemCache { + private static readonly log4net.ILog Logger = log4net.LogManager.GetLogger(typeof(StoreItemCache)); + public StoreItemCache(IStore store, TimeSpan expirePeriod) { _store = store; @@ -20,12 +22,15 @@ public StoreItemCache(IStore store, TimeSpan expirePeriod) _cleanTimer = new Timer(state => { if (!_items.Any()) return; + int removedCount = 0; foreach (var item in _items) { if (DateTime.Now - item.Value.Created > TimeSpan.FromMinutes(5)) { - _items.TryRemove(item.Key, out _); + bool removed = _items.TryRemove(item.Key, out _); + if (removed) removedCount++; } + Logger.Debug($"Items cache clean: removed {removedCount} expired items"); } }, null, cleanPreiod, cleanPreiod); } From 868e9df11c9ef3c8624adbfc8da2d9487379bc03 Mon Sep 17 00:00:00 2001 From: YaR Date: Thu, 2 Nov 2017 20:17:43 +0300 Subject: [PATCH 10/31] some code cleanup and logging --- WDMRC.Console/Program.cs | 4 +- .../{Cloud.cs => CloudManager.cs} | 6 ++- .../Mailru/StoreBase/MailruStore.cs | 4 +- .../Mailru/StoreBase/MailruStoreCollection.cs | 10 ++--- .../Mailru/StoreBase/MailruStoreItem.cs | 8 ++-- .../Mailru/StoreBase/StoreItemCache.cs | 37 +++++++++++-------- 6 files changed, 39 insertions(+), 30 deletions(-) rename WebDavMailRuCloudStore/{Cloud.cs => CloudManager.cs} (94%) diff --git a/WDMRC.Console/Program.cs b/WDMRC.Console/Program.cs index 942a7db1..cdca3202 100644 --- a/WDMRC.Console/Program.cs +++ b/WDMRC.Console/Program.cs @@ -32,8 +32,8 @@ static void Main(string[] args) options => { ShowInfo(options); - Cloud.Init(options.UserAgent); - Cloud.TwoFactorHandlerName = Config.TwoFactorAuthHandlerName; + CloudManager.Init(options.UserAgent); + CloudManager.TwoFactorHandlerName = Config.TwoFactorAuthHandlerName; var webdavProtocol = "http"; var webdavIp = "127.0.0.1"; diff --git a/WebDavMailRuCloudStore/Cloud.cs b/WebDavMailRuCloudStore/CloudManager.cs similarity index 94% rename from WebDavMailRuCloudStore/Cloud.cs rename to WebDavMailRuCloudStore/CloudManager.cs index 48651eed..710eeb76 100644 --- a/WebDavMailRuCloudStore/Cloud.cs +++ b/WebDavMailRuCloudStore/CloudManager.cs @@ -7,9 +7,9 @@ namespace YaR.WebDavMailRu.CloudStore { - public static class Cloud + public static class CloudManager { - private static readonly log4net.ILog Logger = log4net.LogManager.GetLogger(typeof(Cloud)); + private static readonly log4net.ILog Logger = log4net.LogManager.GetLogger(typeof(CloudManager)); public static void Init(string userAgent = "") { @@ -47,6 +47,8 @@ public static MailRuCloud.Api.MailRuCloud Instance(IIdentity identityi) private static MailRuCloud.Api.MailRuCloud CreateCloud(HttpListenerBasicIdentity identity) { + Logger.Info($"Cloud instance created for {identity.Name}"); + if (!ConstSettings.AvailDomains.Any(d => identity.Name.Contains($"@{d}."))) { string domains = ConstSettings.AvailDomains.Aggregate((c, n) => c + ", @" + n); diff --git a/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStore.cs b/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStore.cs index 9ec7b2d9..d94ec8a4 100644 --- a/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStore.cs +++ b/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStore.cs @@ -39,7 +39,7 @@ public Task GetItemAsync(WebDavUri uri, IHttpContext httpContext) //TODO: clean this trash try { - var item = Cloud.Instance(identity).GetItem(path).Result; + var item = CloudManager.Instance(identity).GetItem(path).Result; if (item != null) { return item.IsFile @@ -89,7 +89,7 @@ public Task GetCollectionAsync(WebDavUri uri, IHttpContext htt { var path = uri.Path; - var folder = (Folder)Cloud.Instance(httpContext.Session.Principal.Identity) + var folder = (Folder)CloudManager.Instance(httpContext.Session.Principal.Identity) .GetItem(path, MailRuCloud.Api.MailRuCloud.ItemType.Folder).Result; return Task.FromResult(new MailruStoreCollection(httpContext, LockingManager, folder, IsWritable)); diff --git a/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStoreCollection.cs b/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStoreCollection.cs index a39bf887..c817daf1 100644 --- a/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStoreCollection.cs +++ b/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStoreCollection.cs @@ -97,7 +97,7 @@ static byte[] GetBytes(string str) new DavQuotaAvailableBytes { - Getter = (context, collection) => collection.FullPath == "/" ? Cloud.Instance(context.Session.Principal.Identity).GetDiskUsage().Result.Free.DefaultValue : long.MaxValue, + Getter = (context, collection) => collection.FullPath == "/" ? CloudManager.Instance(context.Session.Principal.Identity).GetDiskUsage().Result.Free.DefaultValue : long.MaxValue, IsExpensive = true //folder listing performance }, @@ -325,7 +325,7 @@ public Task CreateCollectionAsync(string name, bool overw var destinationPath = WebDavPath.Combine(FullPath, name); var cmdFabric = new SpecialCommandFabric(); - var cmd = cmdFabric.Build(Cloud.Instance(httpContext.Session.Principal.Identity), destinationPath); + var cmd = cmdFabric.Build(CloudManager.Instance(httpContext.Session.Principal.Identity), destinationPath); if (cmd != null) { var res = cmd.Execute().Result; @@ -346,7 +346,7 @@ public Task CreateCollectionAsync(string name, bool overw try { - Cloud.Instance(httpContext.Session.Principal.Identity).CreateFolder(name, FullPath).Wait(); + CloudManager.Instance(httpContext.Session.Principal.Identity).CreateFolder(name, FullPath).Wait(); } catch (Exception exc) { @@ -382,7 +382,7 @@ public async Task MoveItemAsync(string sourceName, IStoreCollec if (item == null) return new StoreItemResult(DavStatusCode.NotFound); - var instance = Cloud.Instance(httpContext.Session.Principal.Identity); + var instance = CloudManager.Instance(httpContext.Session.Principal.Identity); if (destinationCollection is MailruStoreCollection destinationStoreCollection) { @@ -462,7 +462,7 @@ public Task DeleteItemAsync(string name, IHttpContext httpContext if (null == item) return Task.FromResult(DavStatusCode.NotFound); - Cloud.Instance(httpContext.Session.Principal.Identity).Remove(item).Wait(); + CloudManager.Instance(httpContext.Session.Principal.Identity).Remove(item).Wait(); return Task.FromResult(DavStatusCode.Ok); } catch (Exception exc) diff --git a/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStoreItem.cs b/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStoreItem.cs index 0f5f3e5a..102c92c9 100644 --- a/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStoreItem.cs +++ b/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStoreItem.cs @@ -166,7 +166,7 @@ private Stream OpenReadStream(MailRuCloud.Api.MailRuCloud cloud, long? start, lo public Task GetReadableStreamAsync(IHttpContext httpContext) //=> { - var cloud = Cloud.Instance((HttpListenerBasicIdentity)httpContext.Session.Principal.Identity); + var cloud = CloudManager.Instance((HttpListenerBasicIdentity)httpContext.Session.Principal.Identity); var range = httpContext.Request.GetRange(); return Task.FromResult(OpenReadStream(cloud, range?.Start, range?.End)); } @@ -190,7 +190,7 @@ public async Task UploadFromStreamAsync(IHttpContext httpContext, _fileInfo.Size = new FileSize(memStream.Length); using (var outputStream = IsWritable - ? Cloud.Instance(httpContext.Session.Principal.Identity).GetFileUploadStream(_fileInfo.FullPath, _fileInfo.Size) + ? CloudManager.Instance(httpContext.Session.Principal.Identity).GetFileUploadStream(_fileInfo.FullPath, _fileInfo.Size) : null) { memStream.Seek(0, SeekOrigin.Begin); @@ -207,7 +207,7 @@ public async Task UploadFromStreamAsync(IHttpContext httpContext, { // Copy the information to the destination stream using (var outputStream = IsWritable - ? Cloud.Instance(httpContext.Session.Principal.Identity).GetFileUploadStream(_fileInfo.FullPath, _fileInfo.Size) + ? CloudManager.Instance(httpContext.Session.Principal.Identity).GetFileUploadStream(_fileInfo.FullPath, _fileInfo.Size) : null) { await inputStream.CopyToAsync(outputStream).ConfigureAwait(false); @@ -234,7 +234,7 @@ public async Task CopyAsync(IStoreCollection destination, strin // check if the file already exists?? - await Cloud.Instance(httpContext.Session.Principal.Identity).Copy(_fileInfo, destinationPath); + await CloudManager.Instance(httpContext.Session.Principal.Identity).Copy(_fileInfo, destinationPath); return new StoreItemResult(DavStatusCode.Created); } diff --git a/WebDavMailRuCloudStore/Mailru/StoreBase/StoreItemCache.cs b/WebDavMailRuCloudStore/Mailru/StoreBase/StoreItemCache.cs index c48f2093..d9fbfda4 100644 --- a/WebDavMailRuCloudStore/Mailru/StoreBase/StoreItemCache.cs +++ b/WebDavMailRuCloudStore/Mailru/StoreBase/StoreItemCache.cs @@ -18,21 +18,9 @@ public StoreItemCache(IStore store, TimeSpan expirePeriod) _store = store; _expirePeriod = expirePeriod; - long cleanPreiod = (long) CleanUpPeriod.TotalMilliseconds; - _cleanTimer = new Timer(state => - { - if (!_items.Any()) return; - int removedCount = 0; - foreach (var item in _items) - { - if (DateTime.Now - item.Value.Created > TimeSpan.FromMinutes(5)) - { - bool removed = _items.TryRemove(item.Key, out _); - if (removed) removedCount++; - } - Logger.Debug($"Items cache clean: removed {removedCount} expired items"); - } - }, null, cleanPreiod, cleanPreiod); + long cleanPeriod = (long) CleanUpPeriod.TotalMilliseconds; + + _cleanTimer = new Timer(state => RemoveExpired() , null, cleanPeriod, cleanPeriod); } private readonly IStore _store; @@ -51,6 +39,25 @@ public TimeSpan CleanUpPeriod } } + public int RemoveExpired() + { + if (!_items.Any()) return 0; + + int removedCount = 0; + foreach (var item in _items) + { + if (DateTime.Now - item.Value.Created > TimeSpan.FromMinutes(5)) + { + bool removed = _items.TryRemove(item.Key, out _); + if (removed) removedCount++; + } + } + if (removedCount > 0) + Logger.Debug($"Items cache clean: removed {removedCount} expired items"); + + return removedCount; + } + public IStoreItem Get(WebDavUri uri, IHttpContext context) { if (_items.TryGetValue(uri.AbsoluteUri, out var item)) From 1d2de54de4f76b612817740f0694d2123ad5d1a0 Mon Sep 17 00:00:00 2001 From: YaR Date: Thu, 2 Nov 2017 22:18:05 +0300 Subject: [PATCH 11/31] code cleanup --- MailRuCloud/MailRuCloudApi/MailRuCloud.cs | 44 +------------------ .../Mailru/GetAndHeadHandler.cs | 3 +- .../Mailru/StoreBase/MailruStore.cs | 1 - 3 files changed, 2 insertions(+), 46 deletions(-) diff --git a/MailRuCloud/MailRuCloudApi/MailRuCloud.cs b/MailRuCloud/MailRuCloudApi/MailRuCloud.cs index 1776be0a..e0969c1c 100644 --- a/MailRuCloud/MailRuCloudApi/MailRuCloud.cs +++ b/MailRuCloud/MailRuCloudApi/MailRuCloud.cs @@ -56,7 +56,7 @@ public enum ItemType /// /// Path in the cloud to return the list of the items. /// - /// + /// /// List of the items. public virtual async Task GetItem(string path, ItemType itemType = ItemType.Unknown, bool resolveLinks = true) { @@ -124,48 +124,6 @@ public virtual async Task GetItem(string path, ItemType itemType = ItemT return entry; - - - - //======================================================================================================= - - //var data = await new FolderInfoRequest(CloudApi, string.IsNullOrEmpty(ulink) ? path : ulink, !string.IsNullOrEmpty(ulink)).MakeRequestAsync(); - - //if (!string.IsNullOrEmpty(ulink)) - //{ - // bool isFile = data.body.list.Any(it => it.weblink.TrimStart('/') == ulink.TrimStart('/')); - - // string trimpath = path; - // if (isFile) trimpath = WebDavPath.Parent(path); - - // foreach (var propse in data.body.list) - // { - // propse.home = WebDavPath.Combine(trimpath, propse.name); - // } - // data.body.home = trimpath; - //} - - //var entry = data.ToEntry(); - - - //var flinks = _pathResolver.GetItems(entry.FullPath); - //if (flinks.Any()) - //{ - // foreach (var flink in flinks) - // { - // string linkpath = WebDavPath.Combine(entry.FullPath, flink.Name); - - // if (!flink.IsFile) - // entry.Folders.Add(new Folder(0, 0, 0, linkpath) { CreationTimeUtc = flink.CreationDate ?? DateTime.MinValue }); - // else - // { - // if (entry.Files.All(inf => inf.FullPath != linkpath)) - // entry.Files.Add(new File(linkpath, flink.Size, string.Empty)); - // } - // } - //} - - //return entry; } diff --git a/WebDavMailRuCloudStore/Mailru/GetAndHeadHandler.cs b/WebDavMailRuCloudStore/Mailru/GetAndHeadHandler.cs index e208e9b2..5fbab3e5 100644 --- a/WebDavMailRuCloudStore/Mailru/GetAndHeadHandler.cs +++ b/WebDavMailRuCloudStore/Mailru/GetAndHeadHandler.cs @@ -7,7 +7,6 @@ using NWebDav.Server.Http; using NWebDav.Server.Props; using NWebDav.Server.Stores; -using YaR.MailRuCloud.Api.Base; using YaR.WebDavMailRu.CloudStore.Mailru.StoreBase; namespace YaR.WebDavMailRu.CloudStore.Mailru @@ -167,7 +166,7 @@ private async Task CopyToAsync(Stream src, Stream dest, long start, long? end) } // Determine the number of bytes to read - var bytesToRead = end.HasValue ? end.Value - start + 1 : long.MaxValue; + var bytesToRead = end - start + 1 ?? long.MaxValue; // Read in 64KB blocks var buffer = new byte[64 * 1024]; diff --git a/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStore.cs b/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStore.cs index d94ec8a4..f4c91af0 100644 --- a/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStore.cs +++ b/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStore.cs @@ -1,7 +1,6 @@ using System; using System.Linq; using System.Net; -using System.Text.RegularExpressions; using System.Threading.Tasks; using NWebDav.Server; using NWebDav.Server.Http; From 77a06b8d87f3472aa486f5344136eb5c9dd86f9b Mon Sep 17 00:00:00 2001 From: YaR Date: Thu, 2 Nov 2017 23:12:46 +0300 Subject: [PATCH 12/31] item deletion improved a bit --- .../Mailru/StoreBase/MailruStoreCollection.cs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStoreCollection.cs b/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStoreCollection.cs index c817daf1..98201873 100644 --- a/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStoreCollection.cs +++ b/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStoreCollection.cs @@ -448,11 +448,10 @@ private IStoreItem FindSubItem(string name) return string.IsNullOrEmpty(name) ? this : Items.FirstOrDefault(it => it.Name == name); } - public Task DeleteItemAsync(string name, IHttpContext httpContext) + public async Task DeleteItemAsync(string name, IHttpContext httpContext) { if (!IsWritable) - return Task.FromResult(DavStatusCode.PreconditionFailed); - + return DavStatusCode.PreconditionFailed; // Determine the full path var fullPath = WebDavPath.Combine(_directoryInfo.FullPath, name); @@ -460,15 +459,17 @@ public Task DeleteItemAsync(string name, IHttpContext httpContext { var item = FindSubItem(name); - if (null == item) return Task.FromResult(DavStatusCode.NotFound); + if (null == item) return DavStatusCode.NotFound; + + var cloud = CloudManager.Instance(httpContext.Session.Principal.Identity); + bool res = await cloud.Remove(item); - CloudManager.Instance(httpContext.Session.Principal.Identity).Remove(item).Wait(); - return Task.FromResult(DavStatusCode.Ok); + return res ? DavStatusCode.Ok : DavStatusCode.InternalServerError; } catch (Exception exc) { Logger.Log(LogLevel.Error, () => $"Unable to delete '{fullPath}' directory.", exc); - return Task.FromResult(DavStatusCode.InternalServerError); + return DavStatusCode.InternalServerError; } } From fdf418f9e9d52fc8685ff8928eb691346342045f Mon Sep 17 00:00:00 2001 From: YaR Date: Thu, 2 Nov 2017 23:13:08 +0300 Subject: [PATCH 13/31] code cleanup a bit --- .../Mailru/StoreBase/MailruStore.cs | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStore.cs b/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStore.cs index f4c91af0..6d11c8ea 100644 --- a/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStore.cs +++ b/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStore.cs @@ -30,12 +30,9 @@ public MailruStore(bool isWritable = true, ILockingManager lockingManager = null public Task GetItemAsync(WebDavUri uri, IHttpContext httpContext) { - //TODO: Refact - var identity = (HttpListenerBasicIdentity)httpContext.Session.Principal.Identity; var path = uri.Path; - //TODO: clean this trash try { var item = CloudManager.Instance(identity).GetItem(path).Result; @@ -44,26 +41,6 @@ public Task GetItemAsync(WebDavUri uri, IHttpContext httpContext) return item.IsFile ? Task.FromResult(new MailruStoreItem(LockingManager, (File)item, IsWritable)) : Task.FromResult(new MailruStoreCollection(httpContext, LockingManager, (Folder)item, IsWritable)); - - //if (item.FullPath == path) - //{ - // var dir = new Folder(item.NumberOfFolders, item.NumberOfFiles, item.Size, path); - // return Task.FromResult(new MailruStoreCollection(httpContext, LockingManager, dir, IsWritable)); - //} - //var fa = item.Files.FirstOrDefault(k => k.FullPath == path); - //if (fa != null) - // return Task.FromResult(new MailruStoreItem(LockingManager, fa, IsWritable)); - - //string parentPath = WebDavPath.Parent(path); - //item = Cloud.Instance(identity).GetItems(parentPath).Result; - //if (item != null) - //{ - // var f = item.Files.FirstOrDefault(k => k.FullPath == path); - // return null != f - // ? Task.FromResult(new MailruStoreItem(LockingManager, f, IsWritable)) - // : null; - //} - } } catch (AggregateException e) From e5fe991f86b57d9b0698aba2b0a2c1586dd9bf6f Mon Sep 17 00:00:00 2001 From: YaR Date: Thu, 2 Nov 2017 23:25:03 +0300 Subject: [PATCH 14/31] cleanup a bit --- .../MailRuCloudApi/Base/Requests/DownloadTokenRequest.cs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/MailRuCloud/MailRuCloudApi/Base/Requests/DownloadTokenRequest.cs b/MailRuCloud/MailRuCloudApi/Base/Requests/DownloadTokenRequest.cs index 0fba70b8..8365b921 100644 --- a/MailRuCloud/MailRuCloudApi/Base/Requests/DownloadTokenRequest.cs +++ b/MailRuCloud/MailRuCloudApi/Base/Requests/DownloadTokenRequest.cs @@ -40,13 +40,6 @@ protected override HttpWebRequest CreateRequest(string baseDomain = null) protected override string RelationalUri => _url; - //protected override byte[] CreateHttpContent() - //{ - // string data = $"Login={Uri.EscapeUriString(_login)}&Domain={ConstSettings.Domain}&Password={Uri.EscapeUriString(_password)}"; - - // return Encoding.UTF8.GetBytes(data); - //} - protected override RequestResponse DeserializeMessage(string responseText) { var m = Regex.Match(responseText, From 0f9b0e3e1190f72ec438456a3518b23a1d1c2a15 Mon Sep 17 00:00:00 2001 From: YaR Date: Thu, 2 Nov 2017 23:25:20 +0300 Subject: [PATCH 15/31] more http requests logging --- MailRuCloud/MailRuCloudApi/Base/DownloadStream.cs | 8 +++----- MailRuCloud/MailRuCloudApi/Base/UploadStream.cs | 4 ++++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/MailRuCloud/MailRuCloudApi/Base/DownloadStream.cs b/MailRuCloud/MailRuCloudApi/Base/DownloadStream.cs index f123e8a1..f9d5d31b 100644 --- a/MailRuCloud/MailRuCloudApi/Base/DownloadStream.cs +++ b/MailRuCloud/MailRuCloudApi/Base/DownloadStream.cs @@ -10,6 +10,8 @@ namespace YaR.MailRuCloud.Api.Base { public class DownloadStream : Stream { + private static readonly log4net.ILog Logger = log4net.LogManager.GetLogger(typeof(DownloadStream)); + private const int InnerBufferSize = 65536; private readonly IList _files; @@ -93,16 +95,12 @@ private async Task GetFileStream() //TODO: refact string downloadkey = string.Empty; if (_shard.Type == ShardType.WeblinkGet) - { - //var dtres = new DownloadTokenHtmlRequest(_cloud, file.PublicLink).MakeRequestAsync().Result; - //downloadkey = dtres.body.token; downloadkey = _cloud.Account.DownloadToken.Value; - } var request = _shard.Type == ShardType.Get ? (HttpWebRequest) WebRequest.Create($"{_shard.Url}{Uri.EscapeDataString(file.FullPath)}") : (HttpWebRequest)WebRequest.Create($"{_shard.Url}{new Uri(file.PublicLink).PathAndQuery.Remove(0, "/public".Length)}?key={downloadkey}"); - + Logger.Debug($"HTTP:{request.Method}:{request.RequestUri.AbsoluteUri}"); request.Headers.Add("Accept-Ranges", "bytes"); request.AddRange(instart, inend); diff --git a/MailRuCloud/MailRuCloudApi/Base/UploadStream.cs b/MailRuCloud/MailRuCloudApi/Base/UploadStream.cs index 46071c60..7028de7e 100644 --- a/MailRuCloud/MailRuCloudApi/Base/UploadStream.cs +++ b/MailRuCloud/MailRuCloudApi/Base/UploadStream.cs @@ -10,6 +10,8 @@ namespace YaR.MailRuCloud.Api.Base { internal class UploadStream : Stream { + private static readonly log4net.ILog Logger = log4net.LogManager.GetLogger(typeof(UploadStream)); + private readonly CloudApi _cloud; private readonly File _file; private readonly ShardInfo _shard; @@ -60,6 +62,8 @@ private void Initialize() _request.UserAgent = ConstSettings.UserAgent; _request.AllowWriteStreamBuffering = false; + Logger.Debug($"HTTP:{_request.Method}:{_request.RequestUri.AbsoluteUri}"); + _task = Task.Factory.FromAsync(_request.BeginGetRequestStream, asyncResult => _request.EndGetRequestStream(asyncResult), null); _task = _task.ContinueWith From 3e9dbf678f63e36cc0e96459144a7eba2dcb0954 Mon Sep 17 00:00:00 2001 From: YaR Date: Thu, 2 Nov 2017 23:26:24 +0300 Subject: [PATCH 16/31] removed shard selection logging because http requests logged --- MailRuCloud/MailRuCloudApi/Base/CloudApi.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/MailRuCloud/MailRuCloudApi/Base/CloudApi.cs b/MailRuCloud/MailRuCloudApi/Base/CloudApi.cs index d170a421..8acab73d 100644 --- a/MailRuCloud/MailRuCloudApi/Base/CloudApi.cs +++ b/MailRuCloud/MailRuCloudApi/Base/CloudApi.cs @@ -46,7 +46,6 @@ public async Task GetShardInfo(ShardType shardType) { var shards = await Task.Run(() => _cachedShards.Value); var shard = shards[shardType]; - Logger.Info($"Shard: ({shardType}){shard.Url}"); return shard; } From 010971cf8a889e2fec80cfd2e2e13ad73b34f0be Mon Sep 17 00:00:00 2001 From: YaR Date: Fri, 3 Nov 2017 00:45:40 +0300 Subject: [PATCH 17/31] code cleanup --- MailRuCloud/MailRuCloudApi/Base/File.cs | 15 +++++-- .../Base/SplittedUploadStream.cs | 43 +------------------ .../MailRuCloudApi/Base/UploadStream.cs | 2 + .../Mailru/StoreBase/MailruStoreCollection.cs | 4 +- 4 files changed, 17 insertions(+), 47 deletions(-) diff --git a/MailRuCloud/MailRuCloudApi/Base/File.cs b/MailRuCloud/MailRuCloudApi/Base/File.cs index bbd4deb5..1a0bdc94 100644 --- a/MailRuCloud/MailRuCloudApi/Base/File.cs +++ b/MailRuCloud/MailRuCloudApi/Base/File.cs @@ -38,9 +38,11 @@ public File(string fullPath, long size, string hash = "") /// Gets file name. /// /// File name. - //TODO: refact - public virtual string Name => WebDavPath.Name(FullPath); //FullPath.Substring(FullPath.LastIndexOf("/", StringComparison.Ordinal) + 1); + public virtual string Name => WebDavPath.Name(FullPath); + /// + /// Gets file extension + /// public string Extension => System.IO.Path.GetExtension(Name); /// @@ -64,7 +66,7 @@ public virtual FileSize Size } /// - /// Gets full file path with name in server. + /// Gets full file path with name on server. /// /// Full file path. public string FullPath @@ -73,6 +75,9 @@ public string FullPath protected set => _fullPath = WebDavPath.Clean(value); } + /// + /// Path to file (without filename) + /// public string Path => WebDavPath.Parent(FullPath); /// @@ -89,6 +94,10 @@ public string FullPath public virtual DateTime CreationTimeUtc { get; set; } public virtual DateTime LastWriteTimeUtc { get; set; } public virtual DateTime LastAccessTimeUtc { get; set; } + + /// + /// If file splitted to several phisical files + /// public bool IsSplitted => Parts.Any(f => f.FullPath != FullPath); public bool IsFile => true; diff --git a/MailRuCloud/MailRuCloudApi/Base/SplittedUploadStream.cs b/MailRuCloud/MailRuCloudApi/Base/SplittedUploadStream.cs index 65ec1233..ff52e486 100644 --- a/MailRuCloud/MailRuCloudApi/Base/SplittedUploadStream.cs +++ b/MailRuCloud/MailRuCloudApi/Base/SplittedUploadStream.cs @@ -129,59 +129,18 @@ public override void Close() { if (_files.Count > 1) { - string content = $"filename={_origfile.Name}\r\nsize = {_origfile.Size.DefaultValue}"; //TODO: calculate CRC32 \r\ncrc32 = C9EB1402\r\n"; + string content = $"filename={_origfile.Name}\r\nsize = {_origfile.Size.DefaultValue}"; var data = Encoding.UTF8.GetBytes(content); var stream = new UploadStream(_origfile.FullPath, _cloud, data.Length); stream.Write(data, 0, data.Length); stream.Close(); } - ////remove test file created with webdav - //var dele = new RemoveRequest(_cloud, _origfile.FullPath) - // .MakeRequestAsync().Result; - _uploadStream?.Close(); OnFileUploaded(_files); } - - //uint CalculateCrc(uint crc, byte[] buffer, int offset, int count) - //{ - // unchecked - // { - // for (int i = offset, end = offset + count; i < end; i++) - // crc = (crc >> 8) ^ CrcTable[(crc ^ buffer[i]) & 0xFF]; - // } - // return crc; - //} - - //private static readonly uint[] CrcTable = GenerateTable(); - - //private static uint[] GenerateTable() - //{ - // unchecked - // { - // uint[] table = new uint[256]; - - // const uint poly = 0xEDB88320; - // for (uint i = 0; i < table.Length; i++) - // { - // var crc = i; - // for (int j = 8; j > 0; j--) - // { - // if ((crc & 1) == 1) crc = (crc >> 1) ^ poly; - // else crc >>= 1; - // } - // table[i] = crc; - // } - - // return table; - // } - - //} - - public override bool CanRead => true; public override bool CanSeek => true; public override bool CanWrite => true; diff --git a/MailRuCloud/MailRuCloudApi/Base/UploadStream.cs b/MailRuCloud/MailRuCloudApi/Base/UploadStream.cs index 7028de7e..d883bdc5 100644 --- a/MailRuCloud/MailRuCloudApi/Base/UploadStream.cs +++ b/MailRuCloud/MailRuCloudApi/Base/UploadStream.cs @@ -134,10 +134,12 @@ public override void Write(byte[] buffer, int offset, int count) WriteBytesInStream(zbuffer, s, token, zcount); } // ReSharper disable once UnusedVariable + #pragma warning disable 168 catch (Exception ex) { return null; } + #pragma warning restore 168 return t.Result; }, diff --git a/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStoreCollection.cs b/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStoreCollection.cs index 98201873..3d11d1f7 100644 --- a/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStoreCollection.cs +++ b/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStoreCollection.cs @@ -177,7 +177,7 @@ static byte[] GetBytes(string str) new DavExtCollectionNoSubs { - Getter = (context, collection) => false //TODO: WTF? + Getter = (context, collection) => false }, new DavExtCollectionObjectCount @@ -361,7 +361,7 @@ public Task CreateCollectionAsync(string name, bool overw public Task UploadFromStreamAsync(IHttpContext httpContext, Stream source) { - throw new NotImplementedException(); + throw new NotImplementedException("Cannot upload a collection by stream"); } public async Task CopyAsync(IStoreCollection destinationCollection, string name, bool overwrite, IHttpContext httpContext) From 0270a26f7c1818f2143e170f9f5461d20d80b09b Mon Sep 17 00:00:00 2001 From: YaR Date: Fri, 3 Nov 2017 01:37:07 +0300 Subject: [PATCH 18/31] download token expire in 20min --- MailRuCloud/MailRuCloudApi/Base/Account.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MailRuCloud/MailRuCloudApi/Base/Account.cs b/MailRuCloud/MailRuCloudApi/Base/Account.cs index d8c9018f..1d6d6515 100644 --- a/MailRuCloud/MailRuCloudApi/Base/Account.cs +++ b/MailRuCloud/MailRuCloudApi/Base/Account.cs @@ -128,7 +128,7 @@ public async Task LoginAsync() /// Token for downloading files /// public readonly Cached DownloadToken; - private const int DownloadTokenExpiresSec = 2 * 60 * 60; + private const int DownloadTokenExpiresSec = 20 * 60; From afc3c3c0e1b74be95ae66d67c4a3a1e875373ee9 Mon Sep 17 00:00:00 2001 From: YaR Date: Fri, 3 Nov 2017 01:37:49 +0300 Subject: [PATCH 19/31] #94 delete command --- MailRuCloud/MailRuCloudApi/MailRuCloud.cs | 16 ++++++++ .../SpecialCommands/DeleteCommand.cs | 40 +++++++++++++++++++ .../SharedFolderJoinCommand.cs | 2 +- .../SharedFolderLinkCommand.cs | 6 +-- .../SpecialCommands/SpecialCommandFabric.cs | 5 ++- .../SpecialCommands/SpecialCommandResult.cs | 6 ++- .../Mailru/StoreBase/MailruStoreCollection.cs | 2 +- 7 files changed, 70 insertions(+), 7 deletions(-) create mode 100644 MailRuCloud/MailRuCloudApi/SpecialCommands/DeleteCommand.cs diff --git a/MailRuCloud/MailRuCloudApi/MailRuCloud.cs b/MailRuCloud/MailRuCloudApi/MailRuCloud.cs index e0969c1c..19914fd8 100644 --- a/MailRuCloud/MailRuCloudApi/MailRuCloud.cs +++ b/MailRuCloud/MailRuCloudApi/MailRuCloud.cs @@ -287,6 +287,22 @@ public async Task CreateFolder(string name, string createIn) return true; } + + /// + /// Remove item on server by path + /// + /// File or folder + /// True or false operation result. + public virtual async Task Remove(IEntry entry) + { + if (entry is File file) + return await Remove(file); + if (entry is Folder folder) + return await Remove(folder); + + return false; + } + /// /// Remove the file on server. /// diff --git a/MailRuCloud/MailRuCloudApi/SpecialCommands/DeleteCommand.cs b/MailRuCloud/MailRuCloudApi/SpecialCommands/DeleteCommand.cs new file mode 100644 index 00000000..b3ff61a6 --- /dev/null +++ b/MailRuCloud/MailRuCloudApi/SpecialCommands/DeleteCommand.cs @@ -0,0 +1,40 @@ +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using YaR.MailRuCloud.Api.Base; +using YaR.MailRuCloud.Api.Base.Requests; +using YaR.MailRuCloud.Api.Extensions; + +namespace YaR.MailRuCloud.Api.SpecialCommands +{ + public class DeleteCommand : SpecialCommand + { + private readonly MailRuCloud _cloud; + private readonly string _path; + private readonly string _param; + + public DeleteCommand(MailRuCloud cloud, string path, string param) + { + _cloud = cloud; + _path = WebDavPath.Clean(path); + _param = param.Replace('\\', '/'); + } + + public override async Task Execute() + { + string path; + if (string.IsNullOrWhiteSpace(_param)) + path = _path; + else if (_param.StartsWith("/")) + path = _param; + else + path = WebDavPath.Combine(_path, _param); + + var entry = await _cloud.GetItem(path); + if (null == entry) + return SpecialCommandResult.Fail; + + var res = await _cloud.Remove(entry); + return new SpecialCommandResult { IsSuccess = res }; + } + } +} \ No newline at end of file diff --git a/MailRuCloud/MailRuCloudApi/SpecialCommands/SharedFolderJoinCommand.cs b/MailRuCloud/MailRuCloudApi/SpecialCommands/SharedFolderJoinCommand.cs index d93c7fc0..73a02e32 100644 --- a/MailRuCloud/MailRuCloudApi/SpecialCommands/SharedFolderJoinCommand.cs +++ b/MailRuCloud/MailRuCloudApi/SpecialCommands/SharedFolderJoinCommand.cs @@ -31,7 +31,7 @@ private string Value public override Task Execute() { bool k = _cloud.CloneItem(_path, Value).Result; - return Task.FromResult(new SpecialCommandResult{Success = k}); + return Task.FromResult(new SpecialCommandResult{IsSuccess = k}); } } } \ No newline at end of file diff --git a/MailRuCloud/MailRuCloudApi/SpecialCommands/SharedFolderLinkCommand.cs b/MailRuCloud/MailRuCloudApi/SpecialCommands/SharedFolderLinkCommand.cs index d909328a..99bd0a43 100644 --- a/MailRuCloud/MailRuCloudApi/SpecialCommands/SharedFolderLinkCommand.cs +++ b/MailRuCloud/MailRuCloudApi/SpecialCommands/SharedFolderLinkCommand.cs @@ -20,9 +20,9 @@ public SharedFolderLinkCommand(MailRuCloud cloud, string path, string param) public override Task Execute() { - var m = Regex.Match(_param, @"(?snx-)link \s+ (https://?cloud.mail.ru/public)?(?/\w*/\w*)/? \s* (?.*) "); + var m = Regex.Match(_param, @"(?snx-)\s* (https://?cloud.mail.ru/public)?(?/\w*/\w*)/? \s* (?.*) "); - if (!m.Success) return Task.FromResult(new SpecialCommandResult { Success = false }); + if (!m.Success) return Task.FromResult(SpecialCommandResult.Fail); var item = new ItemInfoRequest(_cloud.CloudApi, m.Groups["url"].Value, true).MakeRequestAsync().Result.ToEntry(); @@ -39,7 +39,7 @@ public override Task Execute() _cloud.LinkItem(m.Groups["url"].Value, _path, name, isFile, size, item.CreationTimeUtc); } - return Task.FromResult(new SpecialCommandResult{Success = true}); + return Task.FromResult(SpecialCommandResult.Success); } } } \ No newline at end of file diff --git a/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommandFabric.cs b/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommandFabric.cs index 313b8df2..e9c86f16 100644 --- a/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommandFabric.cs +++ b/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommandFabric.cs @@ -10,11 +10,14 @@ public SpecialCommand Build(MailRuCloud cloud, string param) if (null == param || !param.Contains("/>>")) return null; + //TODO: refact this sheet int pos = param.LastIndexOf("/>>", StringComparison.Ordinal); string path = WebDavPath.Clean(param.Substring(0, pos + 1)); string data = param.Substring(pos + 3); - if (data.StartsWith("link ")) return new SharedFolderLinkCommand(cloud, path, data); + if (data.StartsWith("link ")) return new SharedFolderLinkCommand(cloud, path, data.Remove(0, 5)); + if (data.StartsWith("del ")) return new DeleteCommand(cloud, path, data.Remove(0, 4)); + if (data == "del") return new DeleteCommand(cloud, path, data.Remove(0, 3)); return new SharedFolderJoinCommand(cloud, path, data); } diff --git a/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommandResult.cs b/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommandResult.cs index f5171d08..47ed586e 100644 --- a/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommandResult.cs +++ b/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommandResult.cs @@ -2,6 +2,10 @@ { public class SpecialCommandResult { - public bool Success { get; set; } + public bool IsSuccess { get; set; } + + public static SpecialCommandResult Success => new SpecialCommandResult {IsSuccess = true}; + public static SpecialCommandResult Fail => new SpecialCommandResult { IsSuccess = false }; + } } \ No newline at end of file diff --git a/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStoreCollection.cs b/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStoreCollection.cs index 3d11d1f7..b4ada373 100644 --- a/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStoreCollection.cs +++ b/WebDavMailRuCloudStore/Mailru/StoreBase/MailruStoreCollection.cs @@ -329,7 +329,7 @@ public Task CreateCollectionAsync(string name, bool overw if (cmd != null) { var res = cmd.Execute().Result; - return Task.FromResult(new StoreCollectionResult(res.Success ? DavStatusCode.Created : DavStatusCode.PreconditionFailed)); + return Task.FromResult(new StoreCollectionResult(res.IsSuccess ? DavStatusCode.Created : DavStatusCode.PreconditionFailed)); } DavStatusCode result; From edc7cbdccd7f3f09faea36be7d719f38d377a677 Mon Sep 17 00:00:00 2001 From: YaR Date: Fri, 3 Nov 2017 02:18:46 +0300 Subject: [PATCH 20/31] #94 readme update --- readme.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/readme.md b/readme.md index 3d0f0cd1..91315c24 100644 --- a/readme.md +++ b/readme.md @@ -25,7 +25,17 @@ --help Display this help screen. --version Display version information. ``` -Settings in `wdmrc.exe.config` + +***Commands***
+Commands executed by making directory with special name +* Clone shared cloud.mail.ru file/folder to your account: `>>SHARED_FOLDER_LINK` +* Link shared folder without wasting your space: `>>link SHARED_FOLDER_LINK [linkname]` or manually edit file /item.links.wdmrc +* Fast delete: + * `>>del` current folder + * `>>del abc/cde` current folder/abc/cde + * `>>del /abc/cde` root/abc/cde + +***Settings*** in `wdmrc.exe.config` * Logging
``
It's standart [Apache log4net](https://logging.apache.org/log4net/) configurations, take a look for [examples](https://logging.apache.org/log4net/release/config-examples.html) @@ -45,9 +55,6 @@ Automatically split/join when uploading/downloading files larger than cloud allo [Russian FAQ](https://gist.github.com/yar229/4b702af114503546be1fe221bb098f27)
[Discussion on geektimes.ru](https://geektimes.ru/post/285520/) -***Commands*** -* Clone shared cloud.mail.ru file/folder to your account: make folder with name `>>SHARED_FOLDER_LINK` -* Link shared folder without wasting your space: make folder with name `>>link SHARED_FOLDER_LINK [linkname]` or manually edit file /item.links.wdmrc #### Windows From a85f21c68ccb07fd25fe42d2236d70f215739d2c Mon Sep 17 00:00:00 2001 From: YaR Date: Fri, 3 Nov 2017 14:30:26 +0300 Subject: [PATCH 21/31] version bump --- WDMRC.Console/WDMRC.Console.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WDMRC.Console/WDMRC.Console.csproj b/WDMRC.Console/WDMRC.Console.csproj index 7acdabfe..b700fdd8 100644 --- a/WDMRC.Console/WDMRC.Console.csproj +++ b/WDMRC.Console/WDMRC.Console.csproj @@ -11,8 +11,8 @@ yar229@yandex.ru WebDAV emulator for Cloud.mail.ru WebDAVCloudMailRu - 1.7.1.27 - 1.7.1.27 + 1.8.0.0 + 1.8.0.0 wdmrc YaR.CloudMailRu.Console From db55535d5b6956c7066490ff937c4720bce7c73c Mon Sep 17 00:00:00 2001 From: YaR Date: Fri, 3 Nov 2017 18:41:25 +0300 Subject: [PATCH 22/31] #97 `link check` command (no history yet) --- MailRuCloud/MailRuCloudApi/MailRuCloud.cs | 5 +++++ .../MailRuCloudApi/PathResolve/ItemList.cs | 2 +- .../PathResolve/PathResolver.cs | 22 +++++++++++++++++++ .../SpecialCommands/RemoveBadLinksCommand.cs | 21 ++++++++++++++++++ .../SpecialCommands/SpecialCommandFabric.cs | 1 + 5 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 MailRuCloud/MailRuCloudApi/SpecialCommands/RemoveBadLinksCommand.cs diff --git a/MailRuCloud/MailRuCloudApi/MailRuCloud.cs b/MailRuCloud/MailRuCloudApi/MailRuCloud.cs index 19914fd8..f04caab9 100644 --- a/MailRuCloud/MailRuCloudApi/MailRuCloud.cs +++ b/MailRuCloud/MailRuCloudApi/MailRuCloud.cs @@ -454,6 +454,11 @@ public void LinkItem(string url, string path, string name, bool isFile, long siz { _pathResolver.Add(url, path, name, isFile, size, creationDate); } + + public void RemoveDeadLinks() + { + _pathResolver.RemoveDeadLinks(true); + } } public delegate void FileUploadedDelegate(IEnumerable file); diff --git a/MailRuCloud/MailRuCloudApi/PathResolve/ItemList.cs b/MailRuCloud/MailRuCloudApi/PathResolve/ItemList.cs index 877364d0..d90bbdd9 100644 --- a/MailRuCloud/MailRuCloudApi/PathResolve/ItemList.cs +++ b/MailRuCloud/MailRuCloudApi/PathResolve/ItemList.cs @@ -5,7 +5,7 @@ namespace YaR.MailRuCloud.Api.PathResolve { public class ItemList { - public IList Items { get; } = new List(); + public List Items { get; } = new List(); } public class ItemLink diff --git a/MailRuCloud/MailRuCloudApi/PathResolve/PathResolver.cs b/MailRuCloud/MailRuCloudApi/PathResolve/PathResolver.cs index 8888e669..f7363d3f 100644 --- a/MailRuCloud/MailRuCloudApi/PathResolve/PathResolver.cs +++ b/MailRuCloud/MailRuCloudApi/PathResolve/PathResolver.cs @@ -102,8 +102,30 @@ public void RemoveItem(string path) _itemList.Items.Remove(z); Save(); } + } + + public void RemoveDeadLinks(bool doWriteHistory) + { + var removes = _itemList.Items.Where(it => !IsLinkAlive(it)).ToList(); + if (removes.Count == 0) return; + + _itemList.Items.RemoveAll(it => removes.Contains(it)); + + if (doWriteHistory) + { + //TODO:load item.links.history.wdmrc + //TODO:append removed + //TODO:save item.links.history.wdmrc + } + } + + private bool IsLinkAlive(ItemLink link) + { + string path = WebDavPath.Combine(link.MapTo, link.Name); + var entry = _cloud.GetItem(path); + return entry != null; } public string AsWebLink(string path) diff --git a/MailRuCloud/MailRuCloudApi/SpecialCommands/RemoveBadLinksCommand.cs b/MailRuCloud/MailRuCloudApi/SpecialCommands/RemoveBadLinksCommand.cs new file mode 100644 index 00000000..dbe36c47 --- /dev/null +++ b/MailRuCloud/MailRuCloudApi/SpecialCommands/RemoveBadLinksCommand.cs @@ -0,0 +1,21 @@ +using System.Threading.Tasks; +using YaR.MailRuCloud.Api.Base; + +namespace YaR.MailRuCloud.Api.SpecialCommands +{ + public class RemoveBadLinksCommand : SpecialCommand + { + private readonly MailRuCloud _cloud; + + public RemoveBadLinksCommand(MailRuCloud cloud) + { + _cloud = cloud; + } + + public override Task Execute() + { + _cloud.RemoveDeadLinks(); + return Task.FromResult(SpecialCommandResult.Success); + } + } +} \ No newline at end of file diff --git a/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommandFabric.cs b/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommandFabric.cs index e9c86f16..86f37dfc 100644 --- a/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommandFabric.cs +++ b/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommandFabric.cs @@ -15,6 +15,7 @@ public SpecialCommand Build(MailRuCloud cloud, string param) string path = WebDavPath.Clean(param.Substring(0, pos + 1)); string data = param.Substring(pos + 3); + if (data == "link check") return new RemoveBadLinksCommand(cloud); if (data.StartsWith("link ")) return new SharedFolderLinkCommand(cloud, path, data.Remove(0, 5)); if (data.StartsWith("del ")) return new DeleteCommand(cloud, path, data.Remove(0, 4)); if (data == "del") return new DeleteCommand(cloud, path, data.Remove(0, 3)); From e5d7faa79e2dc4aa952d38b00dc9fd3d9f57504e Mon Sep 17 00:00:00 2001 From: YaR Date: Fri, 3 Nov 2017 22:57:58 +0300 Subject: [PATCH 23/31] readme update - enable UTF-8 in Total Commander --- readme.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 91315c24..860d12d5 100644 --- a/readme.md +++ b/readme.md @@ -151,7 +151,9 @@ Use any client supports webdav. #### Remarks -* [**Total Commander**](http://www.ghisler.com/): requires to update `WebDAV plugin` to [v.2.9](http://ghisler.fileburst.com/fsplugins/webdav.zip) +* [**Total Commander**](http://www.ghisler.com/): + - requires to update `WebDAV plugin` to [v.2.9](http://ghisler.fileburst.com/fsplugins/webdav.zip) + - turn on `(connection properties) -> Send\Receive accents in URLs as UTF-8 Unicode` * [**WebDrive**](https://southrivertech.com/products/webdrive/): - check on option `(disk properties) -> HTTP Settings -> Use /allprop on PROPFIND's (slower performance)` - disable `(disk properties) -> HTTP Settings -> Do chunked upload for large files.` From a0903464f1e2bcc376825bd7907af9c662fc097d Mon Sep 17 00:00:00 2001 From: YaR Date: Fri, 3 Nov 2017 23:08:39 +0300 Subject: [PATCH 24/31] separator for webdav path --- MailRuCloud/MailRuCloudApi/Base/WebDavPath.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/MailRuCloud/MailRuCloudApi/Base/WebDavPath.cs b/MailRuCloud/MailRuCloudApi/Base/WebDavPath.cs index a32deeac..d8a39369 100644 --- a/MailRuCloud/MailRuCloudApi/Base/WebDavPath.cs +++ b/MailRuCloud/MailRuCloudApi/Base/WebDavPath.cs @@ -59,6 +59,7 @@ public static string Name(string path) } public static string Root => "/"; + public static string Separator => "/"; public static WebDavPathParts Parts(string path) { From d76eb218573a4f7b9a8b54b2b2b01219387e8c46 Mon Sep 17 00:00:00 2001 From: YaR Date: Fri, 3 Nov 2017 23:18:36 +0300 Subject: [PATCH 25/31] #97 improve del command --- .../PathResolve/PathResolver.cs | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/MailRuCloud/MailRuCloudApi/PathResolve/PathResolver.cs b/MailRuCloud/MailRuCloudApi/PathResolve/PathResolver.cs index f7363d3f..b5cdc206 100644 --- a/MailRuCloud/MailRuCloudApi/PathResolve/PathResolver.cs +++ b/MailRuCloud/MailRuCloudApi/PathResolve/PathResolver.cs @@ -2,11 +2,10 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Net; using System.Text; using Newtonsoft.Json; using YaR.MailRuCloud.Api.Base; -using YaR.MailRuCloud.Api.Base.Requests; -using YaR.MailRuCloud.Api.Extensions; using File = YaR.MailRuCloud.Api.Base.File; namespace YaR.MailRuCloud.Api.PathResolve @@ -106,7 +105,10 @@ public void RemoveItem(string path) public void RemoveDeadLinks(bool doWriteHistory) { - var removes = _itemList.Items.Where(it => !IsLinkAlive(it)).ToList(); + var removes = _itemList.Items + .AsParallel() + .WithDegreeOfParallelism(5) + .Where(it => !IsLinkAlive(it)).ToList(); if (removes.Count == 0) return; _itemList.Items.RemoveAll(it => removes.Contains(it)); @@ -118,14 +120,25 @@ public void RemoveDeadLinks(bool doWriteHistory) //TODO:save item.links.history.wdmrc } + Save(); } private bool IsLinkAlive(ItemLink link) { string path = WebDavPath.Combine(link.MapTo, link.Name); - var entry = _cloud.GetItem(path); - - return entry != null; + try + { + var entry = _cloud.GetItem(path).Result; + return entry != null; + } + catch (AggregateException e) + when ( // let's check if there really no file or just other network error + e.InnerException is WebException we && + (we.Response as HttpWebResponse)?.StatusCode == HttpStatusCode.NotFound + ) + { + return false; + } } public string AsWebLink(string path) From 29918377ef64ab923479f17cd8e00465fbce060c Mon Sep 17 00:00:00 2001 From: YaR Date: Fri, 3 Nov 2017 23:19:21 +0300 Subject: [PATCH 26/31] special commands refact --- .../SpecialCommands/DeleteCommand.cs | 33 ++++---- .../SpecialCommands/RemoveBadLinksCommand.cs | 11 ++- .../SharedFolderJoinCommand.cs | 16 ++-- .../SharedFolderLinkCommand.cs | 41 ++++------ .../SpecialCommands/SpecialCommand.cs | 51 +++++++++++- .../SpecialCommands/SpecialCommandFabric.cs | 79 +++++++++++++++++-- 6 files changed, 163 insertions(+), 68 deletions(-) diff --git a/MailRuCloud/MailRuCloudApi/SpecialCommands/DeleteCommand.cs b/MailRuCloud/MailRuCloudApi/SpecialCommands/DeleteCommand.cs index b3ff61a6..203374d0 100644 --- a/MailRuCloud/MailRuCloudApi/SpecialCommands/DeleteCommand.cs +++ b/MailRuCloud/MailRuCloudApi/SpecialCommands/DeleteCommand.cs @@ -1,40 +1,37 @@ -using System.Text.RegularExpressions; +using System.Collections.Generic; using System.Threading.Tasks; using YaR.MailRuCloud.Api.Base; -using YaR.MailRuCloud.Api.Base.Requests; -using YaR.MailRuCloud.Api.Extensions; namespace YaR.MailRuCloud.Api.SpecialCommands { public class DeleteCommand : SpecialCommand { - private readonly MailRuCloud _cloud; - private readonly string _path; - private readonly string _param; - - public DeleteCommand(MailRuCloud cloud, string path, string param) + public DeleteCommand(MailRuCloud cloud, string path, IList parames): base(cloud, path, parames) { - _cloud = cloud; - _path = WebDavPath.Clean(path); - _param = param.Replace('\\', '/'); } + protected override MinMax MinMaxParamsCount { get; } = new MinMax(0, 1); + public override async Task Execute() { string path; - if (string.IsNullOrWhiteSpace(_param)) - path = _path; - else if (_param.StartsWith("/")) - path = _param; + string param = Parames.Count == 0 ? string.Empty : Parames[0].Replace("\\", WebDavPath.Separator); + + if (Parames.Count == 0) + path = Path; + else if (param.StartsWith(WebDavPath.Separator)) + path = param; else - path = WebDavPath.Combine(_path, _param); + path = WebDavPath.Combine(Path, param); - var entry = await _cloud.GetItem(path); + var entry = await Cloud.GetItem(path); if (null == entry) return SpecialCommandResult.Fail; - var res = await _cloud.Remove(entry); + var res = await Cloud.Remove(entry); return new SpecialCommandResult { IsSuccess = res }; } } + + } \ No newline at end of file diff --git a/MailRuCloud/MailRuCloudApi/SpecialCommands/RemoveBadLinksCommand.cs b/MailRuCloud/MailRuCloudApi/SpecialCommands/RemoveBadLinksCommand.cs index dbe36c47..9b0cb38a 100644 --- a/MailRuCloud/MailRuCloudApi/SpecialCommands/RemoveBadLinksCommand.cs +++ b/MailRuCloud/MailRuCloudApi/SpecialCommands/RemoveBadLinksCommand.cs @@ -1,20 +1,19 @@ +using System.Collections.Generic; using System.Threading.Tasks; -using YaR.MailRuCloud.Api.Base; namespace YaR.MailRuCloud.Api.SpecialCommands { public class RemoveBadLinksCommand : SpecialCommand { - private readonly MailRuCloud _cloud; - - public RemoveBadLinksCommand(MailRuCloud cloud) + public RemoveBadLinksCommand(MailRuCloud cloud, string path, IList parames): base(cloud, path, parames) { - _cloud = cloud; } + protected override MinMax MinMaxParamsCount { get; } = new MinMax(0); + public override Task Execute() { - _cloud.RemoveDeadLinks(); + Cloud.RemoveDeadLinks(); return Task.FromResult(SpecialCommandResult.Success); } } diff --git a/MailRuCloud/MailRuCloudApi/SpecialCommands/SharedFolderJoinCommand.cs b/MailRuCloud/MailRuCloudApi/SpecialCommands/SharedFolderJoinCommand.cs index 73a02e32..be6f64dc 100644 --- a/MailRuCloud/MailRuCloudApi/SpecialCommands/SharedFolderJoinCommand.cs +++ b/MailRuCloud/MailRuCloudApi/SpecialCommands/SharedFolderJoinCommand.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Text.RegularExpressions; using System.Threading.Tasks; @@ -5,22 +6,17 @@ namespace YaR.MailRuCloud.Api.SpecialCommands { public class SharedFolderJoinCommand: SpecialCommand { - private readonly MailRuCloud _cloud; - private readonly string _path; - private readonly string _param; - - public SharedFolderJoinCommand(MailRuCloud cloud, string path, string param) + public SharedFolderJoinCommand(MailRuCloud cloud, string path, IList parames): base(cloud, path, parames) { - _cloud = cloud; - _path = path; - _param = param; } + protected override MinMax MinMaxParamsCount { get; } = new MinMax(1); + private string Value { get { - var m = Regex.Match(_param, @"(?snx-) (https://?cloud.mail.ru/public)?(?/\w*/?\w*)/?\s*"); + var m = Regex.Match(Parames[0], @"(?snx-) (https://?cloud.mail.ru/public)?(?/\w*/?\w*)/?\s*"); return m.Success ? m.Groups["data"].Value @@ -30,7 +26,7 @@ private string Value public override Task Execute() { - bool k = _cloud.CloneItem(_path, Value).Result; + bool k = Cloud.CloneItem(Path, Value).Result; return Task.FromResult(new SpecialCommandResult{IsSuccess = k}); } } diff --git a/MailRuCloud/MailRuCloudApi/SpecialCommands/SharedFolderLinkCommand.cs b/MailRuCloud/MailRuCloudApi/SpecialCommands/SharedFolderLinkCommand.cs index 99bd0a43..7c728278 100644 --- a/MailRuCloud/MailRuCloudApi/SpecialCommands/SharedFolderLinkCommand.cs +++ b/MailRuCloud/MailRuCloudApi/SpecialCommands/SharedFolderLinkCommand.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Text.RegularExpressions; using System.Threading.Tasks; using YaR.MailRuCloud.Api.Base.Requests; @@ -7,39 +8,31 @@ namespace YaR.MailRuCloud.Api.SpecialCommands { public class SharedFolderLinkCommand : SpecialCommand { - private readonly MailRuCloud _cloud; - private readonly string _path; - private readonly string _param; - - public SharedFolderLinkCommand(MailRuCloud cloud, string path, string param) + public SharedFolderLinkCommand(MailRuCloud cloud, string path, IList parames): base(cloud, path, parames) { - _cloud = cloud; - _path = path; - _param = param; } - public override Task Execute() - { - var m = Regex.Match(_param, @"(?snx-)\s* (https://?cloud.mail.ru/public)?(?/\w*/\w*)/? \s* (?.*) "); - - if (!m.Success) return Task.FromResult(SpecialCommandResult.Fail); + protected override MinMax MinMaxParamsCount { get; } = new MinMax(1, 2); - var item = new ItemInfoRequest(_cloud.CloudApi, m.Groups["url"].Value, true).MakeRequestAsync().Result.ToEntry(); - + public override async Task Execute() + { + var m = Regex.Match(Parames[0], @"(?snx-)\s* (https://?cloud.mail.ru/public)?(?/\w*/\w*)/? \s*"); - bool isFile = item.IsFile; - long size = item.Size; + if (!m.Success) return SpecialCommandResult.Fail; + //TODO: make method in MailRuCloud to get entry by url + var item = await new ItemInfoRequest(Cloud.CloudApi, m.Groups["url"].Value, true).MakeRequestAsync(); + var entry = item.ToEntry(); + if (null == entry) + return SpecialCommandResult.Fail; - string name = m.Groups["name"].Value; - if (string.IsNullOrWhiteSpace(name)) name = item.Name; + string name = Parames.Count > 1 && !string.IsNullOrWhiteSpace(Parames[1]) + ? Parames[1] + : entry.Name; - if (m.Success) - { - _cloud.LinkItem(m.Groups["url"].Value, _path, name, isFile, size, item.CreationTimeUtc); - } + Cloud.LinkItem(m.Groups["url"].Value, Path, name, item.IsFile, entry.Size, entry.CreationTimeUtc); - return Task.FromResult(SpecialCommandResult.Success); + return SpecialCommandResult.Success; } } } \ No newline at end of file diff --git a/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommand.cs b/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommand.cs index 30d396ea..2106475f 100644 --- a/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommand.cs +++ b/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommand.cs @@ -1,10 +1,57 @@ -using System.Threading.Tasks; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; namespace YaR.MailRuCloud.Api.SpecialCommands { public abstract class SpecialCommand { - public abstract Task Execute(); + protected readonly MailRuCloud Cloud; + protected readonly string Path; + protected readonly IList Parames; + + protected abstract MinMax MinMaxParamsCount { get; } + + protected SpecialCommand(MailRuCloud cloud, string path, IList parames) + { + Cloud = cloud; + Path = path; + Parames = parames; + + CheckParams(); + } + + public virtual Task Execute() + { + if (Parames.Count < MinMaxParamsCount.Min || Parames.Count > MinMaxParamsCount.Max) + return Task.FromResult(SpecialCommandResult.Fail); + + return Task.FromResult(SpecialCommandResult.Success); + } + + private void CheckParams() + { + if (Parames.Count < MinMaxParamsCount.Min || Parames.Count > MinMaxParamsCount.Max) + throw new ArgumentException("Invalid pameters count"); + } + + } + + public struct MinMax where T : IComparable + { + public MinMax(T min, T max) + { + if (min.CompareTo(max) > 0) throw new ArgumentException("min > max"); + Min = min; + Max = max; + } + + public MinMax(T one) : this(one, one) + { + } + + public T Min { get; } + public T Max { get; } } } diff --git a/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommandFabric.cs b/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommandFabric.cs index 86f37dfc..3fee981a 100644 --- a/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommandFabric.cs +++ b/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommandFabric.cs @@ -1,26 +1,89 @@ using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; using YaR.MailRuCloud.Api.Base; namespace YaR.MailRuCloud.Api.SpecialCommands { + /// + /// + /// public class SpecialCommandFabric { + private static readonly List CommandContainers = new List + { + new SpecialCommandContainer + { + Commands = new [] {"del"}, + CreateFunc = (cloud, path, param) => new DeleteCommand(cloud, path, param) + }, + new SpecialCommandContainer + { + Commands = new [] {"link"}, + CreateFunc = (cloud, path, param) => new SharedFolderLinkCommand(cloud, path, param) + }, + new SpecialCommandContainer + { + Commands = new [] {"link", "check"}, + CreateFunc = (cloud, path, param) => new RemoveBadLinksCommand(cloud, path, param) + }, + new SpecialCommandContainer + { + Commands = new [] {"join"}, + CreateFunc = (cloud, path, param) => new SharedFolderJoinCommand(cloud, path, param) + } + }; + + public SpecialCommand Build(MailRuCloud cloud, string param) { - if (null == param || !param.Contains("/>>")) - return null; + if (null == param || !param.Contains("/>>")) return null; - //TODO: refact this sheet int pos = param.LastIndexOf("/>>", StringComparison.Ordinal); string path = WebDavPath.Clean(param.Substring(0, pos + 1)); string data = param.Substring(pos + 3); - if (data == "link check") return new RemoveBadLinksCommand(cloud); - if (data.StartsWith("link ")) return new SharedFolderLinkCommand(cloud, path, data.Remove(0, 5)); - if (data.StartsWith("del ")) return new DeleteCommand(cloud, path, data.Remove(0, 4)); - if (data == "del") return new DeleteCommand(cloud, path, data.Remove(0, 3)); + var parames = ParseParameters(data); + var commandContainer = FindCommandContainer(parames); + if (commandContainer == null) return null; + + parames = parames.Skip(commandContainer.Commands.Length).ToList(); + var cmd = commandContainer.CreateFunc(cloud, path, parames); + + return cmd; + } + + private SpecialCommandContainer FindCommandContainer(IList parames) + { + var commandContainer = CommandContainers + .Where(cm => + cm.Commands.Length <= parames.Count && + cm.Commands.SequenceEqual(parames.Take(cm.Commands.Length))) + .Aggregate((agg, next) => next.Commands.Length > agg.Commands.Length ? next : agg); - return new SharedFolderJoinCommand(cloud, path, data); + return commandContainer; } + + private List ParseParameters(string paramString) + { + var list = Regex + .Matches(paramString, @"((""((?.*?)(?[\S]+))(\s)*)") + .Cast() + .Select(m => m.Groups["token"].Value) + .ToList(); + + return list; + } + + + + private class SpecialCommandContainer + { + public string[] Commands; + public Func, SpecialCommand> CreateFunc; + } + } + } \ No newline at end of file From cfa0ad86f5ce946e660f20a943a735a4273e4925 Mon Sep 17 00:00:00 2001 From: YaR Date: Fri, 3 Nov 2017 23:28:59 +0300 Subject: [PATCH 27/31] #97 readme update --- readme.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/readme.md b/readme.md index 860d12d5..a98d7599 100644 --- a/readme.md +++ b/readme.md @@ -28,9 +28,10 @@ ***Commands***
Commands executed by making directory with special name -* Clone shared cloud.mail.ru file/folder to your account: `>>SHARED_FOLDER_LINK` -* Link shared folder without wasting your space: `>>link SHARED_FOLDER_LINK [linkname]` or manually edit file /item.links.wdmrc -* Fast delete: +* `>>join SHARED_FOLDER_LINK` Clone shared cloud.mail.ru file/folder to your account +* `>>link SHARED_FOLDER_LINK [linkname]` Link shared folder without wasting your space (or manually edit file /item.links.wdmrc) +* `>>link check` Remove all dead links (may take time if there's a lot of links) +* `>>del` Fast delete (if your client makes recursive deletions of inner items) * `>>del` current folder * `>>del abc/cde` current folder/abc/cde * `>>del /abc/cde` root/abc/cde From edca2212ed49c54eef1504ddbbb763102b989e5e Mon Sep 17 00:00:00 2001 From: YaR Date: Fri, 3 Nov 2017 23:43:24 +0300 Subject: [PATCH 28/31] renamed PathResolver to LinkManager; comments added --- .../{PathResolve => Links}/ItemList.cs | 2 +- .../PathResolver.cs => Links/LinkManager.cs} | 47 ++++++++++++++++--- MailRuCloud/MailRuCloudApi/MailRuCloud.cs | 24 +++++----- 3 files changed, 54 insertions(+), 19 deletions(-) rename MailRuCloud/MailRuCloudApi/{PathResolve => Links}/ItemList.cs (91%) rename MailRuCloud/MailRuCloudApi/{PathResolve/PathResolver.cs => Links/LinkManager.cs} (76%) diff --git a/MailRuCloud/MailRuCloudApi/PathResolve/ItemList.cs b/MailRuCloud/MailRuCloudApi/Links/ItemList.cs similarity index 91% rename from MailRuCloud/MailRuCloudApi/PathResolve/ItemList.cs rename to MailRuCloud/MailRuCloudApi/Links/ItemList.cs index d90bbdd9..f2f11b2d 100644 --- a/MailRuCloud/MailRuCloudApi/PathResolve/ItemList.cs +++ b/MailRuCloud/MailRuCloudApi/Links/ItemList.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; -namespace YaR.MailRuCloud.Api.PathResolve +namespace YaR.MailRuCloud.Api.Links { public class ItemList { diff --git a/MailRuCloud/MailRuCloudApi/PathResolve/PathResolver.cs b/MailRuCloud/MailRuCloudApi/Links/LinkManager.cs similarity index 76% rename from MailRuCloud/MailRuCloudApi/PathResolve/PathResolver.cs rename to MailRuCloud/MailRuCloudApi/Links/LinkManager.cs index b5cdc206..34fa2da2 100644 --- a/MailRuCloud/MailRuCloudApi/PathResolve/PathResolver.cs +++ b/MailRuCloud/MailRuCloudApi/Links/LinkManager.cs @@ -8,26 +8,31 @@ using YaR.MailRuCloud.Api.Base; using File = YaR.MailRuCloud.Api.Base.File; -namespace YaR.MailRuCloud.Api.PathResolve +namespace YaR.MailRuCloud.Api.Links { - public class PathResolver + /// + /// Управление ссылками, привязанными к облаку + /// + public class LinkManager { - private static readonly log4net.ILog Logger = log4net.LogManager.GetLogger(typeof(PathResolver)); + private static readonly log4net.ILog Logger = log4net.LogManager.GetLogger(typeof(LinkManager)); public static string LinkContainerName = "item.links.wdmrc"; private readonly MailRuCloud _cloud; private ItemList _itemList; - public PathResolver(MailRuCloud api) + public LinkManager(MailRuCloud api) { _cloud = api; Load(); } - + /// + /// Сохранить в файл в облаке список ссылок + /// public void Save() { Logger.Info($"Saving links to {LinkContainerName}"); @@ -42,6 +47,9 @@ public void Save() } } + /// + /// Загрузить из файла в облаке список ссылок + /// public void Load() { Logger.Info($"Loading links from {LinkContainerName}"); @@ -68,6 +76,11 @@ public void Load() } } + /// + /// Получить список ссылок, привязанных к указанному пути в облаке + /// + /// Путь к каталогу в облаке + /// public List GetItems(string path) { var z = _itemList.Items @@ -88,6 +101,10 @@ public ItemLink GetItem(string path) return item; } + /// + /// Убрать ссылку + /// + /// public void RemoveItem(string path) { var name = WebDavPath.Name(path); @@ -103,6 +120,10 @@ public void RemoveItem(string path) } } + /// + /// Убрать все привязки на мёртвые ссылки + /// + /// public void RemoveDeadLinks(bool doWriteHistory) { var removes = _itemList.Items @@ -123,6 +144,11 @@ public void RemoveDeadLinks(bool doWriteHistory) Save(); } + /// + /// Проверка доступности ссылки + /// + /// + /// private bool IsLinkAlive(ItemLink link) { string path = WebDavPath.Combine(link.MapTo, link.Name); @@ -171,6 +197,15 @@ public string AsRelationalWebLink(string path) return link; } + /// + /// Привязать ссылку к облаку + /// + /// Ссылка + /// Путь в облаке, в который поместить ссылку + /// Имя для ссылки + /// Признак, что ссылка ведёт на файл, иначе - на папку + /// Размер данных по ссылке + /// Дата создания public void Add(string url, string path, string name, bool isFile, long size, DateTime? creationDate) { Load(); @@ -197,7 +232,7 @@ public void Add(string url, string path, string name, bool isFile, long size, Da private const string PublicBaseLink = "https://cloud.mail.ru/public"; - private const string PublicBaseLink1 = "https:/cloud.mail.ru/public"; + private const string PublicBaseLink1 = "https:/cloud.mail.ru/public"; //TODO: may be obsolete? private string GetRelaLink(string url) { diff --git a/MailRuCloud/MailRuCloudApi/MailRuCloud.cs b/MailRuCloud/MailRuCloudApi/MailRuCloud.cs index f04caab9..a95e4810 100644 --- a/MailRuCloud/MailRuCloudApi/MailRuCloud.cs +++ b/MailRuCloud/MailRuCloudApi/MailRuCloud.cs @@ -13,7 +13,7 @@ using YaR.MailRuCloud.Api.Base; using YaR.MailRuCloud.Api.Base.Requests; using YaR.MailRuCloud.Api.Extensions; -using YaR.MailRuCloud.Api.PathResolve; +using YaR.MailRuCloud.Api.Links; using File = YaR.MailRuCloud.Api.Base.File; namespace YaR.MailRuCloud.Api @@ -25,7 +25,7 @@ namespace YaR.MailRuCloud.Api /// public class MailRuCloud : IDisposable { - private readonly PathResolver _pathResolver; + private readonly LinkManager _linkManager; @@ -41,7 +41,7 @@ public class MailRuCloud : IDisposable public MailRuCloud(string login, string password, ITwoFaHandler twoFaHandler) { CloudApi = new CloudApi(login, password, twoFaHandler); - _pathResolver = new PathResolver(this); + _linkManager = new LinkManager(this); } public enum ItemType @@ -60,7 +60,7 @@ public enum ItemType /// List of the items. public virtual async Task GetItem(string path, ItemType itemType = ItemType.Unknown, bool resolveLinks = true) { - string ulink = resolveLinks ? _pathResolver.AsRelationalWebLink(path) : string.Empty; + string ulink = resolveLinks ? _linkManager.AsRelationalWebLink(path) : string.Empty; var data = new FolderInfoRequest(CloudApi, string.IsNullOrEmpty(ulink) ? path : ulink, !string.IsNullOrEmpty(ulink)) .MakeRequestAsync().ConfigureAwait(false); @@ -103,7 +103,7 @@ public virtual async Task GetItem(string path, ItemType itemType = ItemT if (itemType == ItemType.Folder && entry is Folder folder) { - var flinks = _pathResolver.GetItems(folder.FullPath); + var flinks = _linkManager.GetItems(folder.FullPath); if (flinks.Any()) { foreach (var flink in flinks) @@ -361,8 +361,8 @@ public Stream GetFileUploadStream(string destinationPath, long size) var file = files?.FirstOrDefault(); if (null == file) return; - if (file.Path == "/" && file.Name == PathResolver.LinkContainerName) - _pathResolver.Load(); + if (file.Path == "/" && file.Name == LinkManager.LinkContainerName) + _linkManager.Load(); }; return stream; @@ -381,7 +381,7 @@ private async Task Rename(string fullPath, string newName) if (res.status == 200) { - _pathResolver.ProcessRename(fullPath, newName); + _linkManager.ProcessRename(fullPath, newName); } return res.status == 200; } @@ -411,13 +411,13 @@ public async Task MoveOrCopy(string sourceFullPath, string destinationPa private async Task Remove(string fullPath) { //TODO: refact - string link = _pathResolver.AsRelationalWebLink(fullPath); + string link = _linkManager.AsRelationalWebLink(fullPath); if (!string.IsNullOrEmpty(link)) { //if folder is linked - do not delete inner files/folders if client deleting recursively //just try to unlink folder - _pathResolver.RemoveItem(fullPath); + _linkManager.RemoveItem(fullPath); return true; } @@ -452,12 +452,12 @@ public void Dispose() public void LinkItem(string url, string path, string name, bool isFile, long size, DateTime? creationDate) { - _pathResolver.Add(url, path, name, isFile, size, creationDate); + _linkManager.Add(url, path, name, isFile, size, creationDate); } public void RemoveDeadLinks() { - _pathResolver.RemoveDeadLinks(true); + _linkManager.RemoveDeadLinks(true); } } From 5d775f18674b239be914b130d49fad06ac2e2144 Mon Sep 17 00:00:00 2001 From: YaR Date: Fri, 3 Nov 2017 23:47:13 +0300 Subject: [PATCH 29/31] typo fixed --- MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommand.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommand.cs b/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommand.cs index 2106475f..30597cea 100644 --- a/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommand.cs +++ b/MailRuCloud/MailRuCloudApi/SpecialCommands/SpecialCommand.cs @@ -33,7 +33,7 @@ public virtual Task Execute() private void CheckParams() { if (Parames.Count < MinMaxParamsCount.Min || Parames.Count > MinMaxParamsCount.Max) - throw new ArgumentException("Invalid pameters count"); + throw new ArgumentException("Invalid parameters count"); } } From e7c3ebdcc60a67061b280d5f1e452590ef8b62aa Mon Sep 17 00:00:00 2001 From: YaR Date: Sat, 4 Nov 2017 00:14:29 +0300 Subject: [PATCH 30/31] shards expiration --- MailRuCloud/MailRuCloudApi/Base/Account.cs | 36 +++++++++++++++++-- MailRuCloud/MailRuCloudApi/Base/Cached.cs | 8 +++++ MailRuCloud/MailRuCloudApi/Base/CloudApi.cs | 28 +++++++-------- .../MailRuCloudApi/Base/DownloadStream.cs | 4 +-- .../MailRuCloudApi/Base/UploadStream.cs | 2 +- WDMRC.Console/Properties/launchSettings.json | 2 +- 6 files changed, 59 insertions(+), 21 deletions(-) diff --git a/MailRuCloud/MailRuCloudApi/Base/Account.cs b/MailRuCloud/MailRuCloudApi/Base/Account.cs index 1d6d6515..c4d4c9b8 100644 --- a/MailRuCloud/MailRuCloudApi/Base/Account.cs +++ b/MailRuCloud/MailRuCloudApi/Base/Account.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Net; using System.Threading.Tasks; using YaR.MailRuCloud.Api.Base.Requests; @@ -38,11 +39,25 @@ public Account(CloudApi cloudApi, string login, string password, ITwoFaHandler t if (twoFaHandler1 != null) AuthCodeRequiredEvent += twoFaHandler1.Get; + _cachedShards = new Cached>(() => new ShardInfoRequest(_cloudApi).MakeRequestAsync().Result.ToShardInfo(), + TimeSpan.FromSeconds(ShardsExpiresInSec)); - DownloadToken = new Cached(() => new DownloadTokenRequest(_cloudApi).MakeRequestAsync().Result.ToToken(), + DownloadToken = new Cached(() => + { + //TODO: not thread safe + var token = new DownloadTokenRequest(_cloudApi).MakeRequestAsync().Result.ToToken(); + _cachedShards.Expire(); + return token; + }, TimeSpan.FromSeconds(DownloadTokenExpiresSec)); - AuthToken = new Cached(() => new AuthTokenRequest(_cloudApi).MakeRequestAsync().Result.ToToken(), + AuthToken = new Cached(() => + { + //TODO: not thread safe + var token = new AuthTokenRequest(_cloudApi).MakeRequestAsync().Result.ToToken(); + DownloadToken.Expire(); + return token; + }, TimeSpan.FromSeconds(AuthTokenExpiresInSec)); } @@ -130,9 +145,24 @@ public async Task LoginAsync() public readonly Cached DownloadToken; private const int DownloadTokenExpiresSec = 20 * 60; + private readonly Cached> _cachedShards; + private const int ShardsExpiresInSec = 2 * 60; + + /// + /// Get shard info that to do post get request. Can be use for anonymous user. + /// + /// Shard type as numeric type. + /// Shard info. + public async Task GetShardInfo(ShardType shardType) + { + var shards = await Task.Run(() => _cachedShards.Value); + var shard = shards[shardType]; + return shard; + } + - + public delegate string AuthCodeRequiredDelegate(string login, bool isAutoRelogin); diff --git a/MailRuCloud/MailRuCloudApi/Base/Cached.cs b/MailRuCloud/MailRuCloudApi/Base/Cached.cs index 290c8b43..81ca82e1 100644 --- a/MailRuCloud/MailRuCloudApi/Base/Cached.cs +++ b/MailRuCloud/MailRuCloudApi/Base/Cached.cs @@ -47,5 +47,13 @@ public override string ToString() { return Value.ToString(); } + + public void Expire() + { + lock (_refreshLock) + { + _expiration = DateTime.MinValue; + } + } } } \ No newline at end of file diff --git a/MailRuCloud/MailRuCloudApi/Base/CloudApi.cs b/MailRuCloud/MailRuCloudApi/Base/CloudApi.cs index 8acab73d..70848d14 100644 --- a/MailRuCloud/MailRuCloudApi/Base/CloudApi.cs +++ b/MailRuCloud/MailRuCloudApi/Base/CloudApi.cs @@ -33,23 +33,23 @@ public CloudApi(string login, string password, ITwoFaHandler twoFaHandler) throw new AuthenticationException("Auth token has't been retrieved."); } - _cachedShards = new Cached>(() => new ShardInfoRequest(this).MakeRequestAsync().Result.ToShardInfo(), - TimeSpan.FromMinutes(2)); + //_cachedShards = new Cached>(() => new ShardInfoRequest(this).MakeRequestAsync().Result.ToShardInfo(), + // TimeSpan.FromMinutes(2)); } - /// - /// Get shard info that to do post get request. Can be use for anonymous user. - /// - /// Shard type as numeric type. - /// Shard info. - public async Task GetShardInfo(ShardType shardType) - { - var shards = await Task.Run(() => _cachedShards.Value); - var shard = shards[shardType]; - return shard; - } + ///// + ///// Get shard info that to do post get request. Can be use for anonymous user. + ///// + ///// Shard type as numeric type. + ///// Shard info. + //public async Task GetShardInfo(ShardType shardType) + //{ + // var shards = await Task.Run(() => _cachedShards.Value); + // var shard = shards[shardType]; + // return shard; + //} - private readonly Cached> _cachedShards; + //private readonly Cached> _cachedShards; #region IDisposable Support private bool _disposedValue; diff --git a/MailRuCloud/MailRuCloudApi/Base/DownloadStream.cs b/MailRuCloud/MailRuCloudApi/Base/DownloadStream.cs index f9d5d31b..c0184202 100644 --- a/MailRuCloud/MailRuCloudApi/Base/DownloadStream.cs +++ b/MailRuCloud/MailRuCloudApi/Base/DownloadStream.cs @@ -33,8 +33,8 @@ public DownloadStream(IList files, CloudApi cloud, long? start = null, lon _cloud = cloud; _shard = files.All(f => string.IsNullOrEmpty(f.PublicLink)) - ? _cloud.GetShardInfo(ShardType.Get).Result - : _cloud.GetShardInfo(ShardType.WeblinkGet).Result; + ? _cloud.Account.GetShardInfo(ShardType.Get).Result + : _cloud.Account.GetShardInfo(ShardType.WeblinkGet).Result; _files = files; _start = start; diff --git a/MailRuCloud/MailRuCloudApi/Base/UploadStream.cs b/MailRuCloud/MailRuCloudApi/Base/UploadStream.cs index d883bdc5..f387bc11 100644 --- a/MailRuCloud/MailRuCloudApi/Base/UploadStream.cs +++ b/MailRuCloud/MailRuCloudApi/Base/UploadStream.cs @@ -21,7 +21,7 @@ public UploadStream(string destinationPath, CloudApi cloud, long size) _cloud = cloud; _file = new File(destinationPath, size, null); - _shard = _cloud.GetShardInfo(ShardType.Upload).Result; + _shard = _cloud.Account.GetShardInfo(ShardType.Upload).Result; Initialize(); } diff --git a/WDMRC.Console/Properties/launchSettings.json b/WDMRC.Console/Properties/launchSettings.json index f1df0060..0a350f61 100644 --- a/WDMRC.Console/Properties/launchSettings.json +++ b/WDMRC.Console/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "WDMRC.Console_NetCore": { "commandName": "Project", - "commandLineArgs": "-p 3332 --maxthreads 1" + "commandLineArgs": "-p 801 --maxthreads 1" } } } \ No newline at end of file From d28b739363533571d6035154b800e556ff4e46fd Mon Sep 17 00:00:00 2001 From: YaR Date: Sat, 4 Nov 2017 00:50:14 +0300 Subject: [PATCH 31/31] #96 fusedav compatibility --- NWebDav/NWebDav.Server/Handlers/PropFindHandler.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NWebDav/NWebDav.Server/Handlers/PropFindHandler.cs b/NWebDav/NWebDav.Server/Handlers/PropFindHandler.cs index 80bad1aa..20d21a5d 100644 --- a/NWebDav/NWebDav.Server/Handlers/PropFindHandler.cs +++ b/NWebDav/NWebDav.Server/Handlers/PropFindHandler.cs @@ -157,6 +157,10 @@ public async Task HandleRequestAsync(IHttpContext httpContext, IStore stor { foreach (var propertyName in propertyList) await AddPropertyAsync(httpContext, xResponse, xProp, propertyManager, entry.Entry, propertyName, addedProperties).ConfigureAwait(false); + + //TODO: dirty fix! + // some clients reqire collection property + await AddPropertyAsync(httpContext, xResponse, xProp, propertyManager, entry.Entry, "collection", addedProperties).ConfigureAwait(false); } // Add the values (if any)