Merge branch 'develop2'

MailRuCloud/MailRuCloudApi/Base/Account.cs

@@ -31,12 +31,15 @@ public Account(CloudSettings settings, Credentials credentials)
             if (twoFaHandler1 != null)
                 AuthCodeRequiredEvent += twoFaHandler1.Get;
 
-
+            //TODO: make repo fabric
             RequestRepo = Protocol.WebM1Bin == settings.Protocol
                 ? (IRequestRepo)new WebM1RequestRepo(Proxy, Credentials, OnAuthCodeRequired)
                 : Protocol.WebV2 == settings.Protocol
                     ? new WebV2RequestRepo(Proxy, Credentials, OnAuthCodeRequired)
                     : throw new Exception("Unknown protocol");
+
+            if (!string.IsNullOrWhiteSpace(settings.UserAgent))
+                RequestRepo.HttpSettings.UserAgent = settings.UserAgent;
         }
 
         internal IRequestRepo RequestRepo { get; }

MailRuCloud/MailRuCloudApi/Base/ConstSettings.cs

@@ -7,13 +7,9 @@ public static class ConstSettings
     {
         public static readonly string[] AvailDomains = {"mail", "inbox", "bk", "list"};
 
-        public const string Domain = "mail.ru";
         public const string CloudDomain = "https://cloud.mail.ru";
-        public const string AuthDomain = "https://auth.mail.ru";
         public const string PublishFileLink = CloudDomain + "/public/";
 
-        //public static string UserAgent = "Mozilla / 5.0(Windows; U; Windows NT 5.1; en - US; rv: 1.9.0.1) Gecko / 2008070208 Firefox / 3.0.1";
-        public const string DefaultAcceptType = "text / html,application / xhtml + xml,application / xml; q = 0.9,*/*;q=0.8";
         public const string DefaultRequestType = "application/x-www-form-urlencoded";
 
         #region Big freaking list of mime types

MailRuCloud/MailRuCloudApi/Base/Repos/MobileRequestRepo.cs

@@ -25,7 +25,7 @@ class MobileRequestRepo : IRequestRepo
             UserAgent = "CloudDiskOWindows 17.12.0009 beta WzBbt1Ygbm"
         };
 
-        public int PendingDownloads { get; set; }
+        //public int PendingDownloads { get; set; }
 
         public MobileRequestRepo(IWebProxy proxy, IAuth auth)
         {
@@ -41,10 +41,10 @@ public MobileRequestRepo(IWebProxy proxy, IAuth auth)
                 },
                 value => TimeSpan.FromSeconds(MetaServerExpiresSec));
 
-            _downloadServer = new Cached<MobDownloadServerRequest.Result>(old =>
+            _downloadServer = new Cached<ServerRequest.Result>(old =>
                 {
                     Logger.Debug("DownloadServer expired, refreshing.");
-                    var server = new MobDownloadServerRequest(HttpSettings).MakeRequestAsync().Result;
+                    var server = new GetServerRequest(HttpSettings).MakeRequestAsync().Result;
                     return server;
                 },
                 value => TimeSpan.FromSeconds(DownloadServerExpiresSec));
@@ -56,7 +56,7 @@ public MobileRequestRepo(IWebProxy proxy, IAuth auth)
         private readonly Cached<MobMetaServerRequest.Result> _metaServer;
         private const int MetaServerExpiresSec = 20 * 60;
 
-        private readonly Cached<MobDownloadServerRequest.Result> _downloadServer;
+        private readonly Cached<ServerRequest.Result> _downloadServer;
         private const int DownloadServerExpiresSec = 20 * 60;
 
 

MailRuCloud/MailRuCloudApi/Base/Repos/ShardManager.cs

@@ -31,19 +31,30 @@ public ShardManager(HttpCommonSettings httpsettings, IAuth auth)
             CachedShards = new Cached<Dictionary<ShardType, ShardInfo>>(old => new ShardInfoRequest(httpsettings, auth).MakeRequestAsync().Result.ToShardInfo(),
                 value => TimeSpan.FromSeconds(ShardsExpiresInSec));
 
-            DownloadServersPending = new Pending<Cached<Requests.WebBin.MobDownloadServerRequest.Result>>(8,
-                () => new Cached<Requests.WebBin.MobDownloadServerRequest.Result>(old =>
+            DownloadServersPending = new Pending<Cached<Requests.WebBin.ServerRequest.Result>>(8,
+                () => new Cached<Requests.WebBin.ServerRequest.Result>(old =>
                     {
-                        Logger.Debug("Requesting new download server");
-                        var server = new Requests.WebBin.MobDownloadServerRequest(httpsettings).MakeRequestAsync().Result;
+                        var server = new Requests.WebBin.GetServerRequest(httpsettings).MakeRequestAsync().Result;
+                        Logger.Debug($"Download server changed to {server.Url}");
+                        return server;
+                    },
+                    value => TimeSpan.FromSeconds(DownloadServerExpiresSec)
+                ));
+
+            WeblinkDownloadServersPending = new Pending<Cached<Requests.WebBin.ServerRequest.Result>>(8,
+                () => new Cached<Requests.WebBin.ServerRequest.Result>(old =>
+                    {
+                        var server = new Requests.WebBin.WeblinkGetServerRequest(httpsettings).MakeRequestAsync().Result;
+                        Logger.Debug($"weblink Download server changed to {server.Url}");
                         return server;
                     },
                     value => TimeSpan.FromSeconds(DownloadServerExpiresSec)
                 ));
         }
 
 
-        public Pending<Cached<Requests.WebBin.MobDownloadServerRequest.Result>> DownloadServersPending { get; }
+        public Pending<Cached<Requests.WebBin.ServerRequest.Result>> DownloadServersPending { get; }
+        public Pending<Cached<Requests.WebBin.ServerRequest.Result>> WeblinkDownloadServersPending { get; }
 
         public ShardInfo MetaServer => new ShardInfo {Url = _metaServer.Value.Url, Count = _metaServer.Value.Unknown};
         private readonly Cached<Requests.WebBin.MobMetaServerRequest.Result> _metaServer;

MailRuCloud/MailRuCloudApi/Base/Repos/WebM1RequestRepo.cs

@@ -48,23 +48,28 @@ public Stream GetDownloadStream(File afile, long? start = null, long? end = null
 
         private DownloadStream GetDownloadStreamInternal(File afile, long? start = null, long? end = null)
         {
-            Cached<Requests.WebBin.MobDownloadServerRequest.Result> downServer = null;
+            bool isLinked = !string.IsNullOrEmpty(afile.PublicLink);
+
+            Cached<Requests.WebBin.ServerRequest.Result> downServer = null;
+            var pendingServers = isLinked
+                ? _shardManager.WeblinkDownloadServersPending
+                : _shardManager.DownloadServersPending;
             Stopwatch watch = new Stopwatch();
 
+            HttpWebRequest request = null;
             CustomDisposable<HttpWebResponse> ResponseGenerator(long instart, long inend, File file)
             {
                 var resp = Retry.Do(() =>
                 {
-                    downServer = _shardManager.DownloadServersPending.Next(downServer);
-
-                    bool isLinked = !string.IsNullOrEmpty(file.PublicLink);
+                    downServer = pendingServers.Next(downServer);
 
-                    string url = isLinked
-                        ? $"{GetShardInfo(ShardType.WeblinkGet).Result.Url}/{file.PublicLink}?token={Authent.AccessToken}"
-                        : $"{downServer.Value.Url}{Uri.EscapeDataString(file.FullPath.TrimStart('/'))}?client_id={HttpSettings.ClientId}&token={Authent.AccessToken}";
+                    string url =(isLinked
+                            ? $"{downServer.Value.Url}{file.PublicLink}"
+                            : $"{downServer.Value.Url}{Uri.EscapeDataString(file.FullPath.TrimStart('/'))}") +
+                        $"?client_id={HttpSettings.ClientId}&token={Authent.AccessToken}";
                     var uri = new Uri(url);
 
-                    var request = (HttpWebRequest) WebRequest.Create(uri.OriginalString);
+                    request = (HttpWebRequest) WebRequest.Create(uri.OriginalString);
 
                     request.AddRange(instart, inend);
                     request.Proxy = HttpSettings.Proxy;
@@ -85,7 +90,6 @@ CustomDisposable<HttpWebResponse> ResponseGenerator(long instart, long inend, Fi
 
                     request.Timeout = 15 * 1000;
                     request.ReadWriteTimeout = 15 * 1000;
-                    //request.ServicePoint.ConnectionLimit = int.MaxValue;
 
                     watch.Start();
                     var response = (HttpWebResponse)request.GetResponse();
@@ -94,7 +98,7 @@ CustomDisposable<HttpWebResponse> ResponseGenerator(long instart, long inend, Fi
                         Value = response,
                         OnDispose = () =>
                         {
-                            _shardManager.DownloadServersPending.Free(downServer);
+                            pendingServers.Free(downServer);
                             watch.Stop();
                             Logger.Debug($"HTTP:{request.Method}:{request.RequestUri.AbsoluteUri} ({watch.Elapsed.Milliseconds} ms)");
                         }
@@ -104,17 +108,15 @@ CustomDisposable<HttpWebResponse> ResponseGenerator(long instart, long inend, Fi
                     ((exception as WebException)?.Response as HttpWebResponse)?.StatusCode == HttpStatusCode.NotFound,
                 exception =>
                 {
-                    _shardManager.DownloadServersPending.Free(downServer);
-                    Logger.Warn($"Retrying on exception {exception.Message}");
+                    pendingServers.Free(downServer);
+                    Logger.Warn($"Retrying HTTP:{request.Method}:{request.RequestUri.AbsoluteUri} on exception {exception.Message}");
                 },
                 TimeSpan.FromSeconds(1), 2);
 
                 return resp;
             }
 
             var stream = new DownloadStream(ResponseGenerator, afile, start, end);
-            //stream.Open();
-
             return stream;
         }
 
@@ -139,17 +141,6 @@ public HttpWebRequest UploadRequest(ShardInfo shard, File file, UploadMultipartB
             return request;
         }
 
-
-        //public void BanShardInfo(ShardInfo banShard)
-        //{
-        //    if (!_bannedShards.Value.Any(bsh => bsh.Type == banShard.Type && bsh.Url == banShard.Url))
-        //    {
-        //        Logger.Warn($"Shard {banShard.Url} temporarily banned");
-        //        _bannedShards.Value.Add(banShard);
-        //    }
-        //}
-
-
         /// <summary>
         /// Get shard info that to do post get request. Can be use for anonymous user.
         /// </summary>
@@ -316,3 +307,4 @@ public async Task<AddFileResult> AddFile(string fileFullPath, string fileHash, F
         }
     }
 }
+

MailRuCloud/MailRuCloudApi/Base/Requests/BaseRequest.cs

@@ -25,7 +25,7 @@ protected BaseRequest(HttpCommonSettings settings, IAuth auth)
         protected virtual HttpWebRequest CreateRequest(string baseDomain = null)
         {
             string domain = string.IsNullOrEmpty(baseDomain) ? ConstSettings.CloudDomain : baseDomain;
-            var uriz = new Uri(new Uri(domain), RelationalUri);
+            var uriz = new Uri(new Uri(domain ?? throw new InvalidOperationException("Empty domain")), RelationalUri);
 
             // supressing escaping is obsolete and breaks, for example, chinese names
             // url generated for %E2%80%8E and %E2%80%8F seems ok, but mail.ru replies error
@@ -87,7 +87,7 @@ public async Task<T> MakeRequestAsync()
                         throw new RequestException(exceptionMessage)
                         {
                             StatusCode = response.StatusCode,
-                            ResponseBody = string.Empty, //responseText,
+                            ResponseBody = string.Empty,
                             Description = result.Description,
                             ErrorCode = result.ErrorCode
                         };
@@ -97,7 +97,10 @@ public async Task<T> MakeRequestAsync()
                     return retVal;
                 }
             }
+            // ReSharper disable once RedundantCatchClause
+            #pragma warning disable 168
             catch (Exception ex)
+            #pragma warning restore 168
             {
                 throw;
             }

MailRuCloud/MailRuCloudApi/Base/Requests/WebBin/GetServerRequest.cs

@@ -0,0 +1,13 @@
+using System.Net;
+
+namespace YaR.MailRuCloud.Api.Base.Requests.WebBin
+{
+    internal class GetServerRequest : ServerRequest
+    {
+        public GetServerRequest(HttpCommonSettings settings) : base(settings)
+        {
+        }
+
+        protected override string RelationalUri => "https://dispatcher.cloud.mail.ru/d";
+    }
+}

MailRuCloud/MailRuCloudApi/Base/Requests/WebBin/MobMetaServerRequest.cs

@@ -1,37 +1,13 @@
 using System;
-using System.Net;
 
 namespace YaR.MailRuCloud.Api.Base.Requests.WebBin
 {
-    internal class MobMetaServerRequest : BaseRequestString<MobMetaServerRequest.Result>
+    internal class MobMetaServerRequest : ServerRequest
     {
-        public MobMetaServerRequest(HttpCommonSettings settings) : base(settings, null)
+        public MobMetaServerRequest(HttpCommonSettings settings) : base(settings)
         {
         }
 
         protected override string RelationalUri => "https://dispatcher.cloud.mail.ru/m";
-
-        protected override RequestResponse<Result> DeserializeMessage(string data)
-        {
-            var datas = data.Split(new []{' '}, StringSplitOptions.RemoveEmptyEntries);
-            var msg = new RequestResponse<Result>
-            {
-                Ok = true,
-                Result = new Result
-                {
-                    Url = datas[0],
-                    Ip = datas[1],
-                    Unknown = int.Parse(datas[2])
-                }
-            };
-            return msg;
-        }
-
-        public class Result
-        {
-            public string Url { get; set; }
-            public string Ip { get; set; }
-            public int Unknown { get; set; }
-        }
     }
 }

MailRuCloud/MailRuCloudApi/Base/Requests/WebBin/MobDownloadServerRequest.cs → MailRuCloud/MailRuCloudApi/Base/Requests/WebBin/ServerRequest.cs

@@ -1,15 +1,14 @@
 using System;
-using System.Net;
 
 namespace YaR.MailRuCloud.Api.Base.Requests.WebBin
 {
-    internal class MobDownloadServerRequest : BaseRequestString<MobDownloadServerRequest.Result>
+    internal abstract class ServerRequest : BaseRequestString<ServerRequest.Result>
     {
-        public MobDownloadServerRequest(HttpCommonSettings settings) : base(settings, null)
+        public ServerRequest(HttpCommonSettings settings) : base(settings, null)
         {
         }
 
-        protected override string RelationalUri => "https://dispatcher.cloud.mail.ru/d";
+        //protected override string RelationalUri => "https://dispatcher.cloud.mail.ru/d";
 
         protected override RequestResponse<Result> DeserializeMessage(string data)
         {

MailRuCloud/MailRuCloudApi/Base/Requests/WebBin/WeblinkGetServerRequest.cs

@@ -3,35 +3,12 @@
 
 namespace YaR.MailRuCloud.Api.Base.Requests.WebBin
 {
-    internal class WeblinkGetServerRequest : BaseRequestString<WeblinkGetServerRequest.Result>
+    internal class WeblinkGetServerRequest : ServerRequest
     {
-        public WeblinkGetServerRequest(HttpCommonSettings settings) : base(settings, null)
+        public WeblinkGetServerRequest(HttpCommonSettings settings) : base(settings)
         {
         }
 
-        protected override string RelationalUri => "https://dispatcher.cloud.mail.ru/y";
-
-        protected override RequestResponse<Result> DeserializeMessage(string data)
-        {
-            var datas = data.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
-            var msg = new RequestResponse<Result>
-            {
-                Ok = true,
-                Result = new Result
-                {
-                    Url = datas[0],
-                    Ip = datas[1],
-                    Unknown = int.Parse(datas[2])
-                }
-            };
-            return msg;
-        }
-
-        public class Result
-        {
-            public string Url { get; set; }
-            public string Ip { get; set; }
-            public int Unknown { get; set; }
-        }
+        protected override string RelationalUri => "https://dispatcher.cloud.mail.ru/G";
     }
 }

MailRuCloud/MailRuCloudApi/Base/Requests/WebV2/CommonSettings.cs

@@ -0,0 +1,10 @@
+namespace YaR.MailRuCloud.Api.Base.Requests.WebV2
+{
+    internal static class CommonSettings
+    {
+        public const string Domain = "mail.ru";
+        public const string AuthDomain = "https://auth.mail.ru";
+
+        public const string DefaultAcceptType = "text / html,application / xhtml + xml,application / xml; q = 0.9,*/*;q=0.8";
+    }
+}

MailRuCloud/MailRuCloudApi/Base/Requests/WebV2/DownloadTokenRequest.cs

@@ -25,21 +25,19 @@ protected override string RelationalUri
 
     class DownloadTokenHtmlRequest : BaseRequestString<DownloadTokenResult>
     {
-        private readonly string _url;
-
         public DownloadTokenHtmlRequest(HttpCommonSettings settings, IAuth auth, string url) : base(settings, auth)
         {
-            _url = url;
+            RelationalUri = url;
         }
 
         protected override HttpWebRequest CreateRequest(string baseDomain = null)
         {
-            var request = base.CreateRequest(ConstSettings.AuthDomain);
-            request.Accept = ConstSettings.DefaultAcceptType;
+            var request = base.CreateRequest(CommonSettings.AuthDomain);
+            request.Accept = CommonSettings.DefaultAcceptType;
             return request;
         }
 
-        protected override string RelationalUri => _url;
+        protected override string RelationalUri { get; }
 
         protected override RequestResponse<DownloadTokenResult> DeserializeMessage(string responseText)
         {

MailRuCloud/MailRuCloudApi/Base/Requests/WebV2/EnsureSdcCookieRequest.cs

@@ -12,8 +12,8 @@ public EnsureSdcCookieRequest(HttpCommonSettings settings, IAuth auth)
 
         protected override HttpWebRequest CreateRequest(string baseDomain = null)
         {
-            var request = base.CreateRequest(ConstSettings.AuthDomain);
-            request.Accept = ConstSettings.DefaultAcceptType;
+            var request = base.CreateRequest(CommonSettings.AuthDomain);
+            request.Accept = CommonSettings.DefaultAcceptType;
             return request;
         }