From 8ce95770c06bd04af9b8b81248d6c60f17e00497 Mon Sep 17 00:00:00 2001 From: anurse Date: Fri, 8 Nov 2013 16:01:07 -0800 Subject: [PATCH] Got a backup job working! Now on to Async Completion management --- .../BackendConfiguration.cs | 35 ++++ .../Helpers/DapperExtensions.cs | 21 +++ .../Helpers/DateTimeOffsetExtensions.cs | 21 +++ .../SqlConnectionStringBuilderExtensions.cs | 45 +++++ .../Infrastructure/Job.cs | 53 +++++- .../Infrastructure/JobDispatcher.cs | 3 +- .../Infrastructure/JobInvocationContext.cs | 23 +++ .../Jobs/CreateOnlineDatabaseBackupJob.cs | 169 ++++++++++++++++++ .../Jobs/HelloWorldJob.cs | 44 ----- src/NuGetGallery.Backend/KnownServers.cs | 21 +++ src/NuGetGallery.Backend/Models/Database.cs | 72 ++++++++ .../Monitoring/WorkerEventSource.cs | 12 ++ .../NuGetGallery.Backend.csproj | 11 +- src/NuGetGallery.Backend/Strings.Designer.cs | 18 ++ src/NuGetGallery.Backend/Strings.resx | 6 + src/NuGetGallery.Backend/WorkerRole.cs | 2 + src/NuGetGallery.Backend/app.config | 6 + src/NuGetGallery.Backend/packages.config | 1 + .../Monitoring/Tables/InvocationsEntry.cs | 4 +- .../NuGetGallery.Core.csproj | 1 + src/NuGetGallery.Core/Strings.Designer.cs | 13 +- src/NuGetGallery.Core/Strings.resx | 3 + src/NuGetGallery.Core/Unit.cs | 36 ++++ .../Data/Model/DatabaseBackupMetadataFacts.cs | 38 ++++ .../JobDispatcherFacts.cs | 4 +- tests/NuGetGallery.Backend.Facts/JobFacts.cs | 17 +- .../NuGetGallery.Backend.Facts.csproj | 19 ++ .../packages.config | 5 + 28 files changed, 636 insertions(+), 67 deletions(-) create mode 100644 src/NuGetGallery.Backend/Helpers/DapperExtensions.cs create mode 100644 src/NuGetGallery.Backend/Helpers/DateTimeOffsetExtensions.cs create mode 100644 src/NuGetGallery.Backend/Helpers/SqlConnectionStringBuilderExtensions.cs create mode 100644 src/NuGetGallery.Backend/Infrastructure/JobInvocationContext.cs create mode 100644 src/NuGetGallery.Backend/Jobs/CreateOnlineDatabaseBackupJob.cs delete mode 100644 src/NuGetGallery.Backend/Jobs/HelloWorldJob.cs create mode 100644 src/NuGetGallery.Backend/KnownServers.cs create mode 100644 src/NuGetGallery.Backend/Models/Database.cs create mode 100644 src/NuGetGallery.Core/Unit.cs create mode 100644 tests/NuGetGallery.Backend.Facts/Data/Model/DatabaseBackupMetadataFacts.cs diff --git a/src/NuGetGallery.Backend/BackendConfiguration.cs b/src/NuGetGallery.Backend/BackendConfiguration.cs index e001066ad9..01dc30b996 100644 --- a/src/NuGetGallery.Backend/BackendConfiguration.cs +++ b/src/NuGetGallery.Backend/BackendConfiguration.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Data.SqlClient; +using System.Globalization; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -102,6 +103,40 @@ public static BackendConfiguration CreateAzure() key => RoleEnvironment.GetConfigurationSettingValue(key)); } + public SqlConnectionStringBuilder GetSqlServer(KnownSqlServer server) + { + switch (server) + { + case KnownSqlServer.Primary: + return PrimaryDatabase; + case KnownSqlServer.Warehouse: + return WarehouseDatabase; + default: + throw new InvalidOperationException(String.Format( + CultureInfo.CurrentCulture, + Strings.BackendConfiguration_UnknownSqlServer, + server.ToString())); + } + } + + public CloudStorageAccount GetStorageAccount(KnownStorageAccount account) + { + switch (account) + { + case KnownStorageAccount.Primary: + return PrimaryStorage; + case KnownStorageAccount.Backup: + return BackupStorage; + case KnownStorageAccount.Diagnostics: + return DiagnosticsStorage; + default: + throw new InvalidOperationException(String.Format( + CultureInfo.CurrentCulture, + Strings.BackendConfiguration_UnknownStorageAccount, + account.ToString())); + } + } + private static CloudStorageAccount TryGetStorageAccount(string name) { string val = RoleEnvironment.GetConfigurationSettingValue(name); diff --git a/src/NuGetGallery.Backend/Helpers/DapperExtensions.cs b/src/NuGetGallery.Backend/Helpers/DapperExtensions.cs new file mode 100644 index 0000000000..f90c52a6db --- /dev/null +++ b/src/NuGetGallery.Backend/Helpers/DapperExtensions.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Data.SqlClient; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Dapper +{ + public static class DapperExtensions + { + public static Task ExecuteAsync(this SqlConnection connection, string sql) + { + SqlCommand cmd = connection.CreateCommand(); + cmd.CommandText = sql; + cmd.CommandType = CommandType.Text; + return cmd.ExecuteNonQueryAsync(); + } + } +} diff --git a/src/NuGetGallery.Backend/Helpers/DateTimeOffsetExtensions.cs b/src/NuGetGallery.Backend/Helpers/DateTimeOffsetExtensions.cs new file mode 100644 index 0000000000..5a4735059a --- /dev/null +++ b/src/NuGetGallery.Backend/Helpers/DateTimeOffsetExtensions.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace System +{ + public static class DateTimeOffsetExtensions + { + public static bool IsOlderThan(this DateTimeOffset self, TimeSpan age) + { + return (DateTimeOffset.UtcNow - self) > age; + } + + public static bool IsYoungerThan(this DateTimeOffset self, TimeSpan age) + { + return (DateTimeOffset.UtcNow - self) < age; + } + } +} diff --git a/src/NuGetGallery.Backend/Helpers/SqlConnectionStringBuilderExtensions.cs b/src/NuGetGallery.Backend/Helpers/SqlConnectionStringBuilderExtensions.cs new file mode 100644 index 0000000000..e17f66f635 --- /dev/null +++ b/src/NuGetGallery.Backend/Helpers/SqlConnectionStringBuilderExtensions.cs @@ -0,0 +1,45 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace System.Data.SqlClient +{ + public static class SqlConnectionStringBuilderExtensions + { + public static SqlConnectionStringBuilder ChangeDatabase(this SqlConnectionStringBuilder self, string newDatabaseName) + { + return new SqlConnectionStringBuilder(self.ConnectionString) + { + InitialCatalog = newDatabaseName + }; + } + + public static Task ConnectTo(this SqlConnectionStringBuilder self) + { + return ConnectTo(self.ConnectionString); + } + + public static Task ConnectToMaster(this SqlConnectionStringBuilder self) + { + return ConnectTo(self, "master"); + } + + public static Task ConnectTo(this SqlConnectionStringBuilder self, string databaseName) + { + var newConnectionString = new SqlConnectionStringBuilder(self.ConnectionString) + { + InitialCatalog = databaseName + }; + return ConnectTo(newConnectionString.ConnectionString); + } + + private static async Task ConnectTo(string connection) + { + var c = new SqlConnection(connection); + await c.OpenAsync().ConfigureAwait(continueOnCapturedContext: false); + return c; + } + } +} diff --git a/src/NuGetGallery.Backend/Infrastructure/Job.cs b/src/NuGetGallery.Backend/Infrastructure/Job.cs index 51b2740181..44f3300de2 100644 --- a/src/NuGetGallery.Backend/Infrastructure/Job.cs +++ b/src/NuGetGallery.Backend/Infrastructure/Job.cs @@ -9,6 +9,8 @@ using System.Threading.Tasks; using NuGetGallery.Backend.Monitoring; using NuGetGallery.Jobs; +using System.Diagnostics; +using System.Data.SqlClient; namespace NuGetGallery.Backend { @@ -17,9 +19,11 @@ public abstract class Job private static readonly Regex NameExtractor = new Regex(@"^(?.*)Job$"); public virtual string Name { get; private set; } - public JobInvocation Invocation { get; protected set; } - public BackendConfiguration Config { get; protected set; } + public JobInvocationContext Context { get; protected set; } + public JobInvocation Invocation { get { return Context.Invocation; } } + public BackendConfiguration Config { get { return Context.Config; } } + protected Job() { Name = InferName(); @@ -31,15 +35,14 @@ protected Job(string name) Name = name; } - public virtual async Task Invoke(JobInvocation invocation, BackendConfiguration config) + public virtual async Task Invoke(JobInvocationContext context) { // Bind invocation information - Invocation = invocation; - Config = config; - BindProperties(invocation.Request.Parameters); + Context = context; + BindProperties(Invocation.Request.Parameters); // Invoke the job - WorkerEventSource.Log.JobStarted(invocation.Request.Name, invocation.Id); + WorkerEventSource.Log.JobStarted(Invocation.Request.Name, Invocation.Id); JobResult result; try { @@ -53,11 +56,11 @@ public virtual async Task Invoke(JobInvocation invocation, BackendCon if (result.Status != JobStatus.Faulted) { - WorkerEventSource.Log.JobCompleted(invocation.Request.Name, invocation.Id); + WorkerEventSource.Log.JobCompleted(Invocation.Request.Name, Invocation.Id); } else { - WorkerEventSource.Log.JobFaulted(invocation.Request.Name, result.Exception, invocation.Id); + WorkerEventSource.Log.JobFaulted(Invocation.Request.Name, result.Exception, Invocation.Id); } // Return the result @@ -93,8 +96,17 @@ protected virtual void BindProperty(PropertyDescriptor prop, string value) prop.SetValue(this, convertedValue); } + private IList _converters = new List() { + new SqlConnectionStringBuilderConverter() + }; + protected virtual object ConvertPropertyValue(PropertyDescriptor prop, string value) { + var converter = _converters.FirstOrDefault(c => c.CanConvertFrom(typeof(string)) && c.CanConvertTo(prop.PropertyType)); + if (converter != null) + { + return converter.ConvertFromString(value); + } return prop.Converter.ConvertFromString(value); } @@ -108,5 +120,28 @@ private string InferName() } return name; } + + private class SqlConnectionStringBuilderConverter : TypeConverter + { + public override bool CanConvertFrom(ITypeDescriptorContext context, Type sourceType) + { + return sourceType == typeof(string); + } + + public override bool CanConvertTo(ITypeDescriptorContext context, Type destinationType) + { + return destinationType == typeof(SqlConnectionStringBuilder); + } + + public override object ConvertFrom(ITypeDescriptorContext context, System.Globalization.CultureInfo culture, object value) + { + string strVal = value as string; + if (strVal == null) + { + return null; + } + return new SqlConnectionStringBuilder(strVal); + } + } } } diff --git a/src/NuGetGallery.Backend/Infrastructure/JobDispatcher.cs b/src/NuGetGallery.Backend/Infrastructure/JobDispatcher.cs index 225b34cde0..8460594bc9 100644 --- a/src/NuGetGallery.Backend/Infrastructure/JobDispatcher.cs +++ b/src/NuGetGallery.Backend/Infrastructure/JobDispatcher.cs @@ -48,9 +48,10 @@ public virtual async Task Dispatch(JobInvocation invocation) WorkerEventSource.Log.DispatchingRequest(invocation); JobResult result = null; + var context = new JobInvocationContext(invocation, Config, _monitor); try { - result = await job.Invoke(invocation, Config); + result = await job.Invoke(context); } catch (Exception ex) { diff --git a/src/NuGetGallery.Backend/Infrastructure/JobInvocationContext.cs b/src/NuGetGallery.Backend/Infrastructure/JobInvocationContext.cs new file mode 100644 index 0000000000..51e9cdb4fc --- /dev/null +++ b/src/NuGetGallery.Backend/Infrastructure/JobInvocationContext.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using NuGetGallery.Backend.Monitoring; +using NuGetGallery.Jobs; + +namespace NuGetGallery.Backend +{ + public class JobInvocationContext + { + public JobInvocation Invocation { get; private set; } + public BackendConfiguration Config { get; private set; } + public BackendMonitoringHub Monitor { get; private set; } + + public JobInvocationContext(JobInvocation invocation, BackendConfiguration config, BackendMonitoringHub monitor) + { + Invocation = invocation; + Config = config; + Monitor = monitor; + } + } +} diff --git a/src/NuGetGallery.Backend/Jobs/CreateOnlineDatabaseBackupJob.cs b/src/NuGetGallery.Backend/Jobs/CreateOnlineDatabaseBackupJob.cs new file mode 100644 index 0000000000..e92b96a1dd --- /dev/null +++ b/src/NuGetGallery.Backend/Jobs/CreateOnlineDatabaseBackupJob.cs @@ -0,0 +1,169 @@ +using System; +using System.Data.SqlClient; +using System.Collections.Generic; +using System.Diagnostics.Tracing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Dapper; +using NuGetGallery.Backend.Models; +using System.Globalization; + +namespace NuGetGallery.Backend.Jobs +{ + public class CreateOnlineDatabaseBackupJob : Job + { + /// + /// The target server, in the form of a known SQL Server (primary, warehouse, etc.) + /// + public KnownSqlServer TargetServer { get; set; } + + /// + /// The name of the database to back up + /// + public string TargetDatabaseName { get; set; } + + /// + /// A connection string to the database to be backed up. The user credentials must + /// also be valid for connecting to the master database on that server. + /// + public SqlConnectionStringBuilder TargetDatabaseConnection { get; set; } + + /// + /// The prefix to apply to the backup + /// + public string BackupPrefix { get; set; } + + /// + /// The maximum age of the latest backup. If there isn't one younger than this value, a backup + /// will be created. If there is one younger and Force is not specified, a backup + /// will not be created. + /// + public TimeSpan? MaxAge { get; set; } + + /// + /// Forces a new backup to be created + /// + public bool Force { get; set; } + + public override EventSource GetEventSource() + { + return JobEventSource.Log; + } + + protected internal override async Task Execute() + { + // Resolve the connection if not specified explicitly + if (TargetDatabaseConnection == null) + { + TargetDatabaseConnection = Config + .GetSqlServer(TargetServer) + .ChangeDatabase(TargetDatabaseName); + } + JobEventSource.Log.PreparingToBackup( + TargetDatabaseConnection.InitialCatalog, + TargetDatabaseConnection.DataSource); + // Connect to the master database + using (var connection = await TargetDatabaseConnection.ConnectToMaster()) + { + if (!Force && MaxAge != null) + { + // Get databases + JobEventSource.Log.GettingDatabaseList(TargetDatabaseConnection.DataSource); + var databases = await GetDatabases(connection); + + // Gather backups with matching prefix and order descending + var ordered = from db in databases + let backupMeta = db.GetBackupMetadata() + where backupMeta != null && + String.Equals( + BackupPrefix, + backupMeta.Prefix, + StringComparison.OrdinalIgnoreCase) + orderby backupMeta.Timestamp descending + select backupMeta; + + // Take the most recent one and check it's time + var mostRecent = ordered.FirstOrDefault(); + if (mostRecent != null && mostRecent.Timestamp.IsYoungerThan(MaxAge.Value)) + { + // Skip the backup + JobEventSource.Log.BackupWithinMaxAge(mostRecent, MaxAge.Value); + return; + } + } + + // Generate a backup name + string backupName = DatabaseBackup.GetName(BackupPrefix, DateTimeOffset.UtcNow); + + // Start a copy + // (have to build the SQL string manually because you can't parameterize CREATE DATABASE) + JobEventSource.Log.StartingCopy(TargetDatabaseConnection.InitialCatalog, backupName); + await connection.ExecuteAsync(String.Format( + CultureInfo.InvariantCulture, + "CREATE DATABASE {0} AS COPY OF {1}", + backupName, + TargetDatabaseConnection.InitialCatalog)); + JobEventSource.Log.StartedCopy(TargetDatabaseConnection.InitialCatalog, backupName); + + // Return a result to queue an async completion check in 5 minutes. + return JobResult.AsyncCompletion(new + { + DatabaseName = backupName + }, TimeSpan.FromMinutes(5)); + } + } + + protected internal virtual Task> GetDatabases(SqlConnection connection) + { + return connection.QueryAsync(@" + SELECT name, database_id, create_date, state + FROM sys.databases + "); + } + + [EventSource(Name="NuGet-Jobs-CreateOnlineDatabaseBackup")] + public class JobEventSource : EventSource + { + public static readonly JobEventSource Log = new JobEventSource(); + + private JobEventSource() { } + +#pragma warning disable 0618 + [Event( + eventId: 1, + Level = EventLevel.Informational, + Message = "Skipping backup. {0} is within maximum age of {1}.")] + [Obsolete("This method supports ETL infrastructure. Use other overloads instead")] + public void BackupWithinMaxAge(string name, string age) { WriteEvent(1, name, age); } + + [NonEvent] + public void BackupWithinMaxAge(DatabaseBackup mostRecent, TimeSpan timeSpan) { BackupWithinMaxAge(mostRecent.Db.name, timeSpan.ToString()); } + + [Event( + eventId: 2, + Level = EventLevel.Informational, + Message = "Getting list of databases on {0}")] + public void GettingDatabaseList(string server) { WriteEvent(2, server); } + + [Event( + eventId: 3, + Level = EventLevel.Informational, + Message = "Preparing to backup {1} on server {0}")] + public void PreparingToBackup(string server, string database) { WriteEvent(3, server, database); } + + [Event( + eventId: 4, + Level = EventLevel.Informational, + Message = "Starting copy of {0} to {1}")] + public void StartingCopy(string source, string destination) { WriteEvent(4, source, destination); } + + [Event( + eventId: 5, + Level = EventLevel.Informational, + Message = "Started copy of {0} to {1}")] + public void StartedCopy(string source, string destination) { WriteEvent(5, source, destination); } +#pragma warning restore 0618 + } + } +} diff --git a/src/NuGetGallery.Backend/Jobs/HelloWorldJob.cs b/src/NuGetGallery.Backend/Jobs/HelloWorldJob.cs deleted file mode 100644 index 12564639f1..0000000000 --- a/src/NuGetGallery.Backend/Jobs/HelloWorldJob.cs +++ /dev/null @@ -1,44 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.Tracing; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace NuGetGallery.Backend.Jobs -{ - public class HelloWorldJob : Job - { - public string Message { get; set; } - - public override EventSource GetEventSource() - { - return HelloWorldEventSource.Log; - } - - protected internal override Task Execute() - { - if (Message.Contains("you suck")) - { - HelloWorldEventSource.Log.TellingUserTheySuck(); - throw new Exception("NO YOU SUCK!"); - } - HelloWorldEventSource.Log.Hello(Message); - return Task.FromResult(null); - } - - [EventSource(Name = "NuGet-Jobs-HelloWorld")] - public class HelloWorldEventSource : EventSource - { - public static readonly HelloWorldEventSource Log = new HelloWorldEventSource(); - - private HelloWorldEventSource() { } - - [Event(eventId: 1, Message = "Hello world! Your message: {0}")] - public void Hello(string message) { WriteEvent(1, message); } - - [Event(eventId: 2, Message = "The user said we suck! NO THEY SUCK!")] - public void TellingUserTheySuck() { WriteEvent(2); } - } - } -} diff --git a/src/NuGetGallery.Backend/KnownServers.cs b/src/NuGetGallery.Backend/KnownServers.cs new file mode 100644 index 0000000000..b11a0f36a3 --- /dev/null +++ b/src/NuGetGallery.Backend/KnownServers.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace NuGetGallery.Backend +{ + public enum KnownSqlServer + { + Primary, + Warehouse + } + + public enum KnownStorageAccount + { + Primary, + Backup, + Diagnostics + } +} diff --git a/src/NuGetGallery.Backend/Models/Database.cs b/src/NuGetGallery.Backend/Models/Database.cs new file mode 100644 index 0000000000..cf2fd5311b --- /dev/null +++ b/src/NuGetGallery.Backend/Models/Database.cs @@ -0,0 +1,72 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace NuGetGallery.Backend.Models +{ + public class Database + { + public string name { get; set; } + public int database_id { get; set; } + public DateTime create_date { get; set; } + public DatabaseState state { get; set; } + + public DatabaseBackup GetBackupMetadata() + { + return DatabaseBackup.Create(this); + } + } + + public class DatabaseBackup { + private const string BackupTimestampFormat = "yyyyMMMdd_HHmm"; + private static readonly Regex BackupNameParser = new Regex(@"(?[^_]*)_(?\d{4}[A-Z]{3}\d{2}_\d{4})Z", RegexOptions.IgnoreCase); + private const string BackupNameFormat = "{0}_{1:" + BackupTimestampFormat + "}Z"; + + public Database Db { get; private set; } + public string Prefix { get; private set; } + public DateTimeOffset Timestamp { get; private set; } + + public DatabaseBackup(Database db, string prefix, DateTimeOffset timestamp) + { + Db = db; + Prefix = prefix; + Timestamp = timestamp; + } + + internal static DatabaseBackup Create(Database db) + { + var match = BackupNameParser.Match(db.name); + if (match.Success) + { + DateTimeOffset timestamp = DateTimeOffset.ParseExact( + match.Groups["timestamp"].Value, + BackupTimestampFormat, + CultureInfo.InvariantCulture, + DateTimeStyles.AssumeUniversal); + return new DatabaseBackup(db, match.Groups["prefix"].Value, timestamp); + } + return null; + } + + public static string GetName(string prefix, DateTimeOffset timestamp) + { + return String.Format(BackupNameFormat, prefix, timestamp); + } + } + + public enum DatabaseState : byte + { + ONLINE = 0, + RESTORING = 1, + RECOVERING = 2, + RECOVERY_PENDING = 3, + SUSPECT = 4, + EMERGENCY = 5, + OFFLINE = 6, + COPYING = 7 + } +} diff --git a/src/NuGetGallery.Backend/Monitoring/WorkerEventSource.cs b/src/NuGetGallery.Backend/Monitoring/WorkerEventSource.cs index 2fb4df7190..18b8a5ad93 100644 --- a/src/NuGetGallery.Backend/Monitoring/WorkerEventSource.cs +++ b/src/NuGetGallery.Backend/Monitoring/WorkerEventSource.cs @@ -242,6 +242,18 @@ public void JobExecuted(JobResponse response) { [NonEvent] public void JobFaulted(string jobName, Exception ex, Guid invocationId) { JobFaulted(jobName, ex.ToString(), ex.StackTrace, invocationId.ToString("N")); } + [Event( + eventId: 26, + Level = EventLevel.Verbose, + Message = "Invoking Query: {0}")] + public void InvokingQuery(string name) { WriteEvent(26, name); } + + [Event( + eventId: 27, + Level = EventLevel.Verbose, + Message = "Invoked Query: {0}")] + public void InvokedQuery(string name) { WriteEvent(27, name); } + #pragma warning restore 0618 } } diff --git a/src/NuGetGallery.Backend/NuGetGallery.Backend.csproj b/src/NuGetGallery.Backend/NuGetGallery.Backend.csproj index e705ea690a..c31291ce03 100644 --- a/src/NuGetGallery.Backend/NuGetGallery.Backend.csproj +++ b/src/NuGetGallery.Backend/NuGetGallery.Backend.csproj @@ -34,6 +34,9 @@ 4 + + ..\..\packages\Dapper.1.13\lib\net45\Dapper.dll + ..\..\packages\Microsoft.Data.Edm.5.6.0\lib\net40\Microsoft.Data.Edm.dll @@ -85,11 +88,16 @@ + + + + + - + True @@ -98,6 +106,7 @@ + diff --git a/src/NuGetGallery.Backend/Strings.Designer.cs b/src/NuGetGallery.Backend/Strings.Designer.cs index e90b569336..fc94cbe3a4 100644 --- a/src/NuGetGallery.Backend/Strings.Designer.cs +++ b/src/NuGetGallery.Backend/Strings.Designer.cs @@ -60,6 +60,24 @@ internal Strings() { } } + /// + /// Looks up a localized string similar to Unknown Sql Server: {0}. + /// + internal static string BackendConfiguration_UnknownSqlServer { + get { + return ResourceManager.GetString("BackendConfiguration_UnknownSqlServer", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Unknown Storage Account: {0}. + /// + internal static string BackendConfiguration_UnknownStorageAccount { + get { + return ResourceManager.GetString("BackendConfiguration_UnknownStorageAccount", resourceCulture); + } + } + /// /// Looks up a localized string similar to Unknown job: {0}. /// diff --git a/src/NuGetGallery.Backend/Strings.resx b/src/NuGetGallery.Backend/Strings.resx index 7225dcc358..3e92230800 100644 --- a/src/NuGetGallery.Backend/Strings.resx +++ b/src/NuGetGallery.Backend/Strings.resx @@ -117,6 +117,12 @@ System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + Unknown Sql Server: {0} + + + Unknown Storage Account: {0} + Unknown job: {0} diff --git a/src/NuGetGallery.Backend/WorkerRole.cs b/src/NuGetGallery.Backend/WorkerRole.cs index e56420fda9..2bd9a35a08 100644 --- a/src/NuGetGallery.Backend/WorkerRole.cs +++ b/src/NuGetGallery.Backend/WorkerRole.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Net; using System.Threading; +using System.Threading.Tasks; using Microsoft.WindowsAzure; using Microsoft.WindowsAzure.Diagnostics; using Microsoft.WindowsAzure.ServiceRuntime; @@ -20,6 +21,7 @@ public class WorkerRole : RoleEntryPoint public override void Run() { + // Start the runner on it's own "thread" and just sleep until cancelled _runner.Run(_cancelSource.Token).Wait(); } diff --git a/src/NuGetGallery.Backend/app.config b/src/NuGetGallery.Backend/app.config index eeb8f6cfd3..0de22d656d 100644 --- a/src/NuGetGallery.Backend/app.config +++ b/src/NuGetGallery.Backend/app.config @@ -7,5 +7,11 @@ + + + + + + \ No newline at end of file diff --git a/src/NuGetGallery.Backend/packages.config b/src/NuGetGallery.Backend/packages.config index ce8af92a50..696d4b6b51 100644 --- a/src/NuGetGallery.Backend/packages.config +++ b/src/NuGetGallery.Backend/packages.config @@ -1,5 +1,6 @@  + diff --git a/src/NuGetGallery.Core/Monitoring/Tables/InvocationsEntry.cs b/src/NuGetGallery.Core/Monitoring/Tables/InvocationsEntry.cs index 90adb03b5e..c5c158b3d1 100644 --- a/src/NuGetGallery.Core/Monitoring/Tables/InvocationsEntry.cs +++ b/src/NuGetGallery.Core/Monitoring/Tables/InvocationsEntry.cs @@ -17,8 +17,8 @@ public InvocationsEntry() { } public InvocationsEntry(Guid invocationId) { - PartitionKey = invocationId.ToString().Substring(0, 8); - RowKey = invocationId.ToString().Substring(8); + PartitionKey = invocationId.ToString("N").Substring(0, 8); + RowKey = invocationId.ToString("N").Substring(8); InvocationId = invocationId; } diff --git a/src/NuGetGallery.Core/NuGetGallery.Core.csproj b/src/NuGetGallery.Core/NuGetGallery.Core.csproj index 5d7e2ae30f..c037c7aafd 100644 --- a/src/NuGetGallery.Core/NuGetGallery.Core.csproj +++ b/src/NuGetGallery.Core/NuGetGallery.Core.csproj @@ -142,6 +142,7 @@ True Strings.resx + diff --git a/src/NuGetGallery.Core/Strings.Designer.cs b/src/NuGetGallery.Core/Strings.Designer.cs index 533c7023a1..ea8a9a64d3 100644 --- a/src/NuGetGallery.Core/Strings.Designer.cs +++ b/src/NuGetGallery.Core/Strings.Designer.cs @@ -68,8 +68,17 @@ internal static string CloudAuditingService_DuplicateAuditRecord { return ResourceManager.GetString("CloudAuditingService_DuplicateAuditRecord", resourceCulture); } } - } -} + + /// + /// Looks up a localized string similar to No handler for the {0} command is registered.. + /// + internal static string CommandExecutor_UnhandledCommand { + get { + return ResourceManager.GetString("CommandExecutor_UnhandledCommand", resourceCulture); + } + } + + /// /// Looks up a localized string similar to Table type {0} is not known. Only entities implementing IMonitoringTable are supported. /// internal static string MonitoringHub_UnknownTableType { diff --git a/src/NuGetGallery.Core/Strings.resx b/src/NuGetGallery.Core/Strings.resx index d54bd29a52..85cc1079ed 100644 --- a/src/NuGetGallery.Core/Strings.resx +++ b/src/NuGetGallery.Core/Strings.resx @@ -120,6 +120,9 @@ Unable to write audit record: '{0}'. Record already exists. + + No handler for the {0} command is registered. + Table type {0} is not known. Only entities implementing IMonitoringTable are supported diff --git a/src/NuGetGallery.Core/Unit.cs b/src/NuGetGallery.Core/Unit.cs new file mode 100644 index 0000000000..c5b15cdb16 --- /dev/null +++ b/src/NuGetGallery.Core/Unit.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace System +{ + // .NET Framework needs a unit type! + public class Unit : IEquatable + { + public static readonly Unit Instance = new Unit(); + + private Unit() { } + + public bool Equals(Unit other) + { + return ReferenceEquals(this, other); + } + + public override bool Equals(object obj) + { + return ReferenceEquals(this, obj); + } + + public override string ToString() + { + return ""; + } + + public override int GetHashCode() + { + return 0; // There can only be one! + } + } +} diff --git a/tests/NuGetGallery.Backend.Facts/Data/Model/DatabaseBackupMetadataFacts.cs b/tests/NuGetGallery.Backend.Facts/Data/Model/DatabaseBackupMetadataFacts.cs new file mode 100644 index 0000000000..12ccb0ed52 --- /dev/null +++ b/tests/NuGetGallery.Backend.Facts/Data/Model/DatabaseBackupMetadataFacts.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NuGetGallery.Backend.Models; +using Xunit; +using Xunit.Extensions; + +namespace NuGetGallery.Backend.Data.Model +{ + public class DatabaseBackupMetadataFacts + { + [Theory] + [InlineData("burkurp")] // No timestamp + [InlineData("backup_nov122013@1505")] // Wrong timestamp format + [InlineData("backup_November12_2013_4_43_pm_PST")] // Wrong timestamp format + [InlineData("Backup_12Nov2013_2043")] // Wrong timestamp format (missing trailing "Z") + public void ReturnsNullForNonMatchingDatabaseName(string name) + { + Assert.Null(DatabaseBackup.Create(new Database() { name = name })); + } + + [Theory] + [InlineData("Backup_2013Nov12_2043Z", "Backup", "2013-11-12T2043")] + [InlineData("Backup_2013nOv12_2043Z", "Backup", "2013-11-12T2043")] + [InlineData("Backup_1924Dec12_0042z", "Backup", "1924-12-12T0042")] + [InlineData("Burkurp_1924deC12_0042Z", "Burkurp", "1924-12-12T0042")] + [InlineData("WarehouseBackup_1924Dec12_0042Z", "WarehouseBackup_", "1924-12-12T0042")] + public void ParsesMatchingNameCorrectly(string name, string prefix, string expectedTimestamp) + { + var parsed = DatabaseBackup.Create(new Database() { name = name }); + Assert.NotNull(parsed); + Assert.Equal(prefix, parsed.Prefix); + Assert.Equal(expectedTimestamp, parsed.Timestamp.ToString("s")); + } + } +} diff --git a/tests/NuGetGallery.Backend.Facts/JobDispatcherFacts.cs b/tests/NuGetGallery.Backend.Facts/JobDispatcherFacts.cs index 27a4fa3ca2..7ed12e2f32 100644 --- a/tests/NuGetGallery.Backend.Facts/JobDispatcherFacts.cs +++ b/tests/NuGetGallery.Backend.Facts/JobDispatcherFacts.cs @@ -37,7 +37,7 @@ public async Task GivenJobWithName_ItCreatesAnInvocationAndInvokesJob() var request = new JobRequest("Test", "test", new Dictionary()); var invocation = new JobInvocation(Guid.NewGuid(), request, DateTimeOffset.UtcNow); - job.Setup(j => j.Invoke(invocation, dispatcher.Config)) + job.Setup(j => j.Invoke(It.IsAny())) .Returns(Task.FromResult(JobResult.Completed())); @@ -61,7 +61,7 @@ public async Task GivenJobWithName_ItReturnsResponseContainingInvocationAndResul var request = new JobRequest("Test", "test", new Dictionary()); var invocation = new JobInvocation(Guid.NewGuid(), request, DateTimeOffset.UtcNow); - job.Setup(j => j.Invoke(invocation, dispatcher.Config)) + job.Setup(j => j.Invoke(It.IsAny())) .Returns(Task.FromResult(JobResult.Faulted(ex))); // Act diff --git a/tests/NuGetGallery.Backend.Facts/JobFacts.cs b/tests/NuGetGallery.Backend.Facts/JobFacts.cs index e5976304e2..0ff955424e 100644 --- a/tests/NuGetGallery.Backend.Facts/JobFacts.cs +++ b/tests/NuGetGallery.Backend.Facts/JobFacts.cs @@ -55,9 +55,10 @@ public async Task GivenAnInvocation_ItSetsTheInvocationProperty() "Test", new Dictionary()), DateTimeOffset.UtcNow); + var context = new JobInvocationContext(invocation, BackendConfiguration.Create(), monitor: null); // Act - await job.Invoke(invocation, BackendConfiguration.Create()); + await job.Invoke(context); // Assert Assert.Same(invocation, job.Invocation); @@ -79,9 +80,10 @@ public async Task GivenParametersThatMatchPropertyNames_ItSetsPropertiesToThoseV {"NotMapped", "bar"} }), DateTimeOffset.UtcNow); + var context = new JobInvocationContext(invocation, BackendConfiguration.Create(), monitor: null); // Act - await job.Invoke(invocation, BackendConfiguration.Create()); + await job.Invoke(context); // Assert Assert.Equal("frob", job.TestParameter); @@ -102,9 +104,10 @@ public async Task GivenPropertiesWithConverters_ItUsesTheConverterToChangeTheVal {"ConvertValue", "frob"}, }), DateTimeOffset.UtcNow); + var context = new JobInvocationContext(invocation, BackendConfiguration.Create(), monitor: null); // Act - await job.Invoke(invocation, BackendConfiguration.Create()); + await job.Invoke(context); // Assert Assert.Equal("http://it.was.a.string/frob", job.ConvertValue.AbsoluteUri); @@ -122,9 +125,10 @@ public async Task GivenAJobExecutesWithoutException_ItReturnsCompletedJobResult( "Test", new Dictionary()), DateTimeOffset.UtcNow); + var context = new JobInvocationContext(invocation, BackendConfiguration.Create(), monitor: null); // Act - var result = await job.Object.Invoke(invocation, BackendConfiguration.Create()); + var result = await job.Object.Invoke(context); // Assert Assert.Equal(JobResult.Completed(), result); @@ -144,9 +148,10 @@ public async Task GivenAJobThrows_ItReturnsFaultedJobResult() DateTimeOffset.UtcNow); var ex = new NotImplementedException("Broked!"); job.Setup(j => j.Execute()).Throws(ex); - + var context = new JobInvocationContext(invocation, BackendConfiguration.Create(), monitor: null); + // Act - var result = await job.Object.Invoke(invocation, BackendConfiguration.Create()); + var result = await job.Object.Invoke(context); // Assert Assert.Equal(JobResult.Faulted(ex), result); diff --git a/tests/NuGetGallery.Backend.Facts/NuGetGallery.Backend.Facts.csproj b/tests/NuGetGallery.Backend.Facts/NuGetGallery.Backend.Facts.csproj index 09837d0c1d..ed94d1c4fb 100644 --- a/tests/NuGetGallery.Backend.Facts/NuGetGallery.Backend.Facts.csproj +++ b/tests/NuGetGallery.Backend.Facts/NuGetGallery.Backend.Facts.csproj @@ -33,11 +33,29 @@ 4 + + ..\..\packages\Microsoft.Data.Edm.5.2.0\lib\net40\Microsoft.Data.Edm.dll + + + ..\..\packages\Microsoft.Data.OData.5.2.0\lib\net40\Microsoft.Data.OData.dll + + + False + ..\..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll + + + False + ..\..\packages\WindowsAzure.Storage.2.1.0.3\lib\net40\Microsoft.WindowsAzure.Storage.dll + ..\..\packages\Moq.4.1.1309.1617\lib\net40\Moq.dll + + + ..\..\packages\System.Spatial.5.2.0\lib\net40\System.Spatial.dll + @@ -52,6 +70,7 @@ + diff --git a/tests/NuGetGallery.Backend.Facts/packages.config b/tests/NuGetGallery.Backend.Facts/packages.config index 7428834d83..08083a457c 100644 --- a/tests/NuGetGallery.Backend.Facts/packages.config +++ b/tests/NuGetGallery.Backend.Facts/packages.config @@ -1,6 +1,11 @@  + + + + + \ No newline at end of file