From ead35acce847195745a38910b07944ea3fafa582 Mon Sep 17 00:00:00 2001 From: Sakura Akeno Isayeki Date: Fri, 9 Sep 2022 12:23:47 +0200 Subject: [PATCH 01/20] Init `Nodsoft.WowsReplaysUnpack.FileStore` library project This project will hold the main implementation for proposal #22. --- .../Nodsoft.WowsReplaysUnpack.FileStore.csproj | 14 ++++++++++++++ WoWS-ReplaysUnpack.sln | 7 +++++++ 2 files changed, 21 insertions(+) create mode 100644 Nodsoft.WowsReplaysUnpack.FileStore/Nodsoft.WowsReplaysUnpack.FileStore.csproj diff --git a/Nodsoft.WowsReplaysUnpack.FileStore/Nodsoft.WowsReplaysUnpack.FileStore.csproj b/Nodsoft.WowsReplaysUnpack.FileStore/Nodsoft.WowsReplaysUnpack.FileStore.csproj new file mode 100644 index 0000000..962dd40 --- /dev/null +++ b/Nodsoft.WowsReplaysUnpack.FileStore/Nodsoft.WowsReplaysUnpack.FileStore.csproj @@ -0,0 +1,14 @@ + + + + Library + net6.0 + enable + enable + + + + + + + diff --git a/WoWS-ReplaysUnpack.sln b/WoWS-ReplaysUnpack.sln index b49c026..684624e 100644 --- a/WoWS-ReplaysUnpack.sln +++ b/WoWS-ReplaysUnpack.sln @@ -34,6 +34,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Library", "Library", "{367B EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nodsoft.WowsReplaysUnpack.ExtendedData", "Nodsoft.WowsReplaysUnpack.ExtendedData\Nodsoft.WowsReplaysUnpack.ExtendedData.csproj", "{A9D51B91-8D9E-4E8E-A1A7-26BD5223EC53}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nodsoft.WowsReplaysUnpack.FileStore", "Nodsoft.WowsReplaysUnpack.FileStore\Nodsoft.WowsReplaysUnpack.FileStore.csproj", "{5278C904-639B-4368-A98C-6F7F79B1683E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -64,6 +66,10 @@ Global {A9D51B91-8D9E-4E8E-A1A7-26BD5223EC53}.Debug|Any CPU.Build.0 = Debug|Any CPU {A9D51B91-8D9E-4E8E-A1A7-26BD5223EC53}.Release|Any CPU.ActiveCfg = Release|Any CPU {A9D51B91-8D9E-4E8E-A1A7-26BD5223EC53}.Release|Any CPU.Build.0 = Release|Any CPU + {5278C904-639B-4368-A98C-6F7F79B1683E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5278C904-639B-4368-A98C-6F7F79B1683E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5278C904-639B-4368-A98C-6F7F79B1683E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5278C904-639B-4368-A98C-6F7F79B1683E}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -72,6 +78,7 @@ Global {B7A3AA8B-E20E-4DE9-B63A-A1A96741EFCD} = {367BF192-3EC3-4A87-B8E4-3E444E7019D0} {18EF6FA2-5AF1-4277-81F4-7104BB35B3BD} = {367BF192-3EC3-4A87-B8E4-3E444E7019D0} {A9D51B91-8D9E-4E8E-A1A7-26BD5223EC53} = {367BF192-3EC3-4A87-B8E4-3E444E7019D0} + {5278C904-639B-4368-A98C-6F7F79B1683E} = {367BF192-3EC3-4A87-B8E4-3E444E7019D0} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {26B5CF6B-2C64-417B-8149-3DFD8E93D497} From fa9c60465cf183281bcfbaa9cef9a37713a43227 Mon Sep 17 00:00:00 2001 From: Sakura Akeno Isayeki Date: Fri, 9 Sep 2022 14:56:30 +0200 Subject: [PATCH 02/20] Breakout `IDefinitionLoader` from `IDefinitionStore` + Implementations --- .../Definitions/BaseDefinition.cs | 4 +- .../Definitions/DefaultDefinitionStore.cs | 140 ++---------------- .../Definitions/DefinitionDirectory.cs | 4 +- .../Definitions/EmbeddedDefinitionLoader.cs | 138 +++++++++++++++++ .../Definitions/EntityDefinition.cs | 6 +- .../Definitions/IDefinitionLoader.cs | 40 +++++ .../Definitions/IDefinitionStore.cs | 14 +- 7 files changed, 202 insertions(+), 144 deletions(-) create mode 100644 Nodsoft.WowsReplaysUnpack.Core/Definitions/EmbeddedDefinitionLoader.cs create mode 100644 Nodsoft.WowsReplaysUnpack.Core/Definitions/IDefinitionLoader.cs diff --git a/Nodsoft.WowsReplaysUnpack.Core/Definitions/BaseDefinition.cs b/Nodsoft.WowsReplaysUnpack.Core/Definitions/BaseDefinition.cs index 43885a1..b51cd17 100644 --- a/Nodsoft.WowsReplaysUnpack.Core/Definitions/BaseDefinition.cs +++ b/Nodsoft.WowsReplaysUnpack.Core/Definitions/BaseDefinition.cs @@ -39,7 +39,7 @@ protected BaseDefinition(Version clientVersion, IDefinitionStore definitionStore Name = name; _folder = folder; - ParseDefinitionFile(DefinitionStore.GetFileAsXml(ClientVersion, Name + ".def", _folder).DocumentElement!); + ParseDefinitionFile(DefinitionStore.Loader.GetFileAsXml(ClientVersion, Name + ".def", _folder).DocumentElement!); } @@ -63,7 +63,7 @@ private void ParseImplements(IEnumerable interfaces) { foreach (string @interface in interfaces) { - ParseDefinitionFile(DefinitionStore.GetFileAsXml(ClientVersion, @interface + ".def", _folder, "interfaces").DocumentElement!); + ParseDefinitionFile(DefinitionStore.Loader.GetFileAsXml(ClientVersion, @interface + ".def", _folder, "interfaces").DocumentElement!); } } diff --git a/Nodsoft.WowsReplaysUnpack.Core/Definitions/DefaultDefinitionStore.cs b/Nodsoft.WowsReplaysUnpack.Core/Definitions/DefaultDefinitionStore.cs index b086ded..4ff3274 100644 --- a/Nodsoft.WowsReplaysUnpack.Core/Definitions/DefaultDefinitionStore.cs +++ b/Nodsoft.WowsReplaysUnpack.Core/Definitions/DefaultDefinitionStore.cs @@ -3,6 +3,7 @@ using Nodsoft.WowsReplaysUnpack.Core.Exceptions; using Nodsoft.WowsReplaysUnpack.Core.Extensions; using System.Reflection; +using System.Runtime.CompilerServices; using System.Xml; namespace Nodsoft.WowsReplaysUnpack.Core.Definitions; @@ -17,23 +18,6 @@ public class DefaultDefinitionStore : IDefinitionStore /// protected const string EntitiesXml = "entities.xml"; - private readonly Version[] _supportedVersions; - - /// - /// Logger instance used by this definition store. - /// - protected readonly ILogger Logger; - - /// - /// Assembly of the Definition store (defaults to the implementation assembly). - /// - protected readonly Assembly Assembly; - - - /// - /// Version -> Definitions Directory - /// - protected readonly Dictionary DirectoryCache = new(); /// /// version_name -> Definition @@ -55,29 +39,21 @@ public class DefaultDefinitionStore : IDefinitionStore /// protected readonly Dictionary> TypeMappings = new(); - public DefaultDefinitionStore(ILogger logger) + public IDefinitionLoader Loader { get; } + + public DefaultDefinitionStore(IDefinitionLoader embeddedDefinitionLoader) { - Assembly = typeof(DefaultDefinitionStore).Assembly; - Logger = logger; - - string versionsDirectory = JoinPath(Assembly.FullName!.GetStringBeforeIndex(Consts.Comma), "Definitions", "Versions"); - - _supportedVersions = Assembly.GetManifestResourceNames() - .Where(name => name.StartsWith(versionsDirectory)) - .Select(name => name.GetStringAfterLength(versionsDirectory + Consts.Dot).GetStringBeforeIndex(Consts.Dot)[1..]) - .Distinct() - .Select(version => version.Split(Consts.Underscore).Select(int.Parse).ToArray()) - .Select(arr => new Version(arr[0], arr[1], arr[2])) - .OrderByDescending(version => version) - .ToArray(); + Loader = embeddedDefinitionLoader; } + + #region EntityDefinitions /// public virtual EntityDefinition GetEntityDefinition(Version clientVersion, int index) { - clientVersion = GetActualVersion(clientVersion); + clientVersion = Loader.GetExactVersion(clientVersion); string name = GetEntityDefinitionName(clientVersion, index); return GetEntityDefinition(clientVersion, name); @@ -86,8 +62,8 @@ public virtual EntityDefinition GetEntityDefinition(Version clientVersion, int i /// public virtual EntityDefinition GetEntityDefinition(Version clientVersion, string name) { - clientVersion = GetActualVersion(clientVersion); - string cacheKey = CacheKey(clientVersion.ToString(), name); + clientVersion = Loader.GetExactVersion(clientVersion); + string cacheKey = string.Join('_', clientVersion.ToString(), name); if (EntityDefinitionCache.TryGetValue(cacheKey, out EntityDefinition? definition)) { @@ -107,7 +83,7 @@ public virtual EntityDefinition GetEntityDefinition(Version clientVersion, strin /// The name of the entity definition. protected virtual string GetEntityDefinitionName(Version clientVersion, int index) { - string cacheKey = CacheKey(clientVersion.ToString(), (index - 1).ToString()); + string cacheKey = string.Join('_', clientVersion.ToString(), (index - 1).ToString()); if (EntityDefinitionIndexNameCache.TryGetValue(cacheKey, out string? name)) { @@ -137,26 +113,13 @@ protected virtual Dictionary GetEntityIndexes(Version clientVersion /// public virtual XmlDocument GetFileAsXml(Version clientVersion, string name, params string[] directoryNames) { - clientVersion = GetActualVersion(clientVersion); - DefinitionDirectory directory = FindDirectory(clientVersion, directoryNames); - - if (directory.Files.SingleOrDefault(f => f.Name.Equals(name, StringComparison.InvariantCultureIgnoreCase)) is not { } file) - { - throw new InvalidOperationException("File could not be found"); - } - - XmlReaderSettings settings = new() { IgnoreComments = true }; - XmlReader reader = XmlReader.Create(Assembly.GetManifestResourceStream(file.Path) ?? throw new InvalidOperationException("File not found"), settings); - XmlDocument xmlDocument = new(); - xmlDocument.Load(reader); - - return xmlDocument; + return Loader.GetFileAsXml(clientVersion, name, directoryNames); } /// public virtual DataTypeBase GetDataType(Version clientVersion, XmlNode typeOrArgXmlNode) { - clientVersion = GetActualVersion(clientVersion); + clientVersion = Loader.GetExactVersion(clientVersion); string versionString = clientVersion.ToString(); if (TypeMappings.TryGetValue(versionString, out Dictionary? typeMapping)) @@ -177,57 +140,6 @@ public virtual DataTypeBase GetDataType(Version clientVersion, XmlNode typeOrArg return GetDataTypeInternal(clientVersion, typeMapping, typeOrArgXmlNode); } - /// - /// Finds a definition directory by given names. - /// - /// The game client version. - /// The names of the directories. - /// The definition directory. - protected virtual DefinitionDirectory FindDirectory(Version clientVersion, IEnumerable directoryNames) - { - DefinitionDirectory folder = GetRootDirectory(clientVersion); - - foreach (string? folderName in directoryNames) - { - DefinitionDirectory? foundFolder = folder.Directories.SingleOrDefault(f => f.Name.Equals(folderName, StringComparison.InvariantCultureIgnoreCase)); - - if (foundFolder is null) - { - break; - } - - folder = foundFolder; - } - - return folder; - } - - /// - /// Gets the root definition directory for a given game client version. - /// - /// The game client version. - /// The root definition directory. - protected virtual DefinitionDirectory GetRootDirectory(Version clientVersion) - { - if (DirectoryCache.TryGetValue(clientVersion.ToString(), out DefinitionDirectory? rootDirectory)) - { - return rootDirectory; - } - - string scriptsDirectory = JoinPath(Assembly.FullName!.GetStringBeforeIndex(Consts.Comma), - "Definitions", "Versions", $"{Consts.Underscore}{clientVersion.ToString().Replace(Consts.Dot, Consts.Underscore)}", "scripts" - ); - - string[] fileNames = Assembly.GetManifestResourceNames() - .Where(name => name.StartsWith(scriptsDirectory)) - .ToArray(); - - rootDirectory = new("scripts", scriptsDirectory, fileNames); - DirectoryCache.Add(clientVersion.ToString(), rootDirectory); - - return rootDirectory; - } - /// /// Internal method for getting a data type. /// @@ -256,30 +168,4 @@ protected virtual DataTypeBase GetDataTypeInternal(Version clientVersion, Dictio } } } - - private Version GetActualVersion(Version version) - { - Version actualVersion = _supportedVersions.FirstOrDefault(v => version >= v) ?? throw new VersionNotSupportedException(_supportedVersions.Last(), version); - - if (actualVersion != version) - { - Logger.LogWarning("The requested version does not match the latest supported version. Requested: {requested}, Latest: {latest}", version, actualVersion); - } - - return actualVersion; - } - - /// - /// Joins parts of an XML path, separated by a dot. - /// - /// Parts of the path. - /// The joined path. - protected static string JoinPath(params string[] parts) => string.Join(Consts.Dot, parts); - - /// - /// Joins parts of a cache key, separated by an underscore. - /// - /// Parts of the cache key. - /// The joined cache key. - protected static string CacheKey(params string[] values) => string.Join(Consts.Underscore, values); } \ No newline at end of file diff --git a/Nodsoft.WowsReplaysUnpack.Core/Definitions/DefinitionDirectory.cs b/Nodsoft.WowsReplaysUnpack.Core/Definitions/DefinitionDirectory.cs index a4c2dfc..ff0ebbf 100644 --- a/Nodsoft.WowsReplaysUnpack.Core/Definitions/DefinitionDirectory.cs +++ b/Nodsoft.WowsReplaysUnpack.Core/Definitions/DefinitionDirectory.cs @@ -40,9 +40,9 @@ private void ParseChildrenRecursive(string[] fileNames) foreach (string fileName in fileNames) { - string actualFileName = fileName.GetStringAfterLength(Path + "."); + string actualFileName = fileName.GetStringAfterLength(Path + '.'); - if (actualFileName.Count(c => c == '.') is 1) + if (actualFileName.Count(static c => c is '.') is 1) { Files.Add(new(actualFileName, fileName)); } diff --git a/Nodsoft.WowsReplaysUnpack.Core/Definitions/EmbeddedDefinitionLoader.cs b/Nodsoft.WowsReplaysUnpack.Core/Definitions/EmbeddedDefinitionLoader.cs new file mode 100644 index 0000000..dfbf0a7 --- /dev/null +++ b/Nodsoft.WowsReplaysUnpack.Core/Definitions/EmbeddedDefinitionLoader.cs @@ -0,0 +1,138 @@ +using Microsoft.Extensions.Logging; +using Nodsoft.WowsReplaysUnpack.Core.Exceptions; +using Nodsoft.WowsReplaysUnpack.Core.Extensions; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Xml; + +namespace Nodsoft.WowsReplaysUnpack.Core.Definitions; + +/// +/// Provides a definition loader to load entity definitions from XML files embedded in the library's assembly. +/// +public class EmbeddedDefinitionLoader : IDefinitionLoader +{ + /// + /// Version -> Definitions Directory + /// + protected readonly Dictionary DirectoryCache = new(); + + /// + /// Logger instance used by this definition store. + /// + protected readonly ILogger Logger; + + protected internal Assembly Assembly { get; } + + /// + /// Lists all the game client versions that are supported by this definition loader. + /// + public IEnumerable SupportedVersions => _supportedVersions; + + private readonly Version[] _supportedVersions; + + public EmbeddedDefinitionLoader(ILogger logger) + { + Logger = logger; + Assembly = typeof(EmbeddedDefinitionLoader).Assembly; + + string versionsDirectory = string.Join('.', Assembly.FullName!.GetStringBeforeIndex(','), nameof(Definitions), "Versions"); + + _supportedVersions = Assembly.GetManifestResourceNames() + .Where(name => name.StartsWith(versionsDirectory)) + .Select(name => name.GetStringAfterLength(versionsDirectory + '.').GetStringBeforeIndex('.')[1..]) + .Distinct() + .Select(static version => version.Split('_').Select(int.Parse).ToArray()) + .Select(static arr => new Version(arr[0], arr[1], arr[2])) + .OrderByDescending(static version => version) + .ToArray(); + } + + /// + public virtual XmlDocument GetFileAsXml(Version clientVersion, string name, params string[] directoryNames) + { + clientVersion = GetExactVersion(clientVersion); + DefinitionDirectory directory = FindDirectory(clientVersion, directoryNames); + + if (directory.Files.SingleOrDefault(f => f.Name.Equals(name, StringComparison.InvariantCultureIgnoreCase)) is not { } file) + { + throw new InvalidOperationException("File could not be found"); + } + + XmlReaderSettings settings = new() { IgnoreComments = true }; + XmlReader reader = XmlReader.Create(Assembly.GetManifestResourceStream(file.Path) ?? throw new InvalidOperationException("File not found"), settings); + XmlDocument xmlDocument = new(); + xmlDocument.Load(reader); + + return xmlDocument; + } + + /// + /// Finds a definition directory by given names. + /// + /// The game client version. + /// The names of the directories. + /// The definition directory. + public virtual DefinitionDirectory FindDirectory(Version clientVersion, IEnumerable directoryNames) + { + DefinitionDirectory folder = GetRootDirectory(clientVersion); + + foreach (string? folderName in directoryNames) + { + DefinitionDirectory? foundFolder = folder.Directories.SingleOrDefault(f => f.Name.Equals(folderName, StringComparison.InvariantCultureIgnoreCase)); + + if (foundFolder is null) + { + break; + } + + folder = foundFolder; + } + + return folder; + } + + /// + /// Gets the root definition directory for a given game client version. + /// + /// The game client version. + /// The root definition directory. + public virtual DefinitionDirectory GetRootDirectory(Version clientVersion) + { + if (DirectoryCache.TryGetValue(clientVersion.ToString(), out DefinitionDirectory? rootDirectory)) + { + return rootDirectory; + } + + string scriptsDirectory = string.Join('.', Assembly.FullName!.GetStringBeforeIndex(','), + "Definitions", "Versions", $"_{clientVersion.ToString().Replace('.', '_')}", "scripts" + ); + + string[] fileNames = Assembly.GetManifestResourceNames() + .Where(name => name.StartsWith(scriptsDirectory)) + .ToArray(); + + rootDirectory = new("scripts", scriptsDirectory, fileNames); + DirectoryCache.Add(clientVersion.ToString(), rootDirectory); + + return rootDirectory; + } + + /// + /// Gets an exact version supported by this definition loader, or the closest version below it. + /// + /// The version to find. + /// The closest version supported by this definition loader. + /// Thrown when no version is supported by this definition loader. + public Version GetExactVersion(Version version) + { + Version actualVersion = SupportedVersions.FirstOrDefault(v => version >= v) ?? throw new VersionNotSupportedException(_supportedVersions.Last(), version); + + if (actualVersion != version) + { + Logger.LogWarning("The requested version does not match the latest supported version. Requested: {requested}, Latest: {latest}", version, actualVersion); + } + + return actualVersion; + } +} \ No newline at end of file diff --git a/Nodsoft.WowsReplaysUnpack.Core/Definitions/EntityDefinition.cs b/Nodsoft.WowsReplaysUnpack.Core/Definitions/EntityDefinition.cs index 55e949f..f286c97 100644 --- a/Nodsoft.WowsReplaysUnpack.Core/Definitions/EntityDefinition.cs +++ b/Nodsoft.WowsReplaysUnpack.Core/Definitions/EntityDefinition.cs @@ -18,10 +18,8 @@ public record EntityDefinition : BaseDefinition /// public List ClientMethods { get; private set; } = new(); - public EntityDefinition(Version clientVersion, IDefinitionStore definitionStore, - string name) : base(clientVersion, definitionStore, name, ENTITY_DEFS) - { - } + public EntityDefinition(Version clientVersion, IDefinitionStore definitionStore, string name) + : base(clientVersion, definitionStore, name, ENTITY_DEFS) { } /// /// Parses a .def file for the entity definition. diff --git a/Nodsoft.WowsReplaysUnpack.Core/Definitions/IDefinitionLoader.cs b/Nodsoft.WowsReplaysUnpack.Core/Definitions/IDefinitionLoader.cs new file mode 100644 index 0000000..6664b9f --- /dev/null +++ b/Nodsoft.WowsReplaysUnpack.Core/Definitions/IDefinitionLoader.cs @@ -0,0 +1,40 @@ +using Nodsoft.WowsReplaysUnpack.Core.Exceptions; +using System.Xml; + +namespace Nodsoft.WowsReplaysUnpack.Core.Definitions; + +public interface IDefinitionLoader +{ + /// + /// Reads XML data from a .def file (seeking through directories as needed). + /// + /// Game client version + /// Name of the .def file to read + /// Directories where to search for the .def file + /// XML data + XmlDocument GetFileAsXml(Version clientVersion, string name, params string[] directoryNames); + + /// + /// Gets the root definition directory for a given game client version. + /// + /// The game client version. + /// The root definition directory. + DefinitionDirectory GetRootDirectory(Version clientVersion); + + + /// + /// Finds a definition directory by given names. + /// + /// The game client version. + /// The names of the directories. + /// The definition directory. + DefinitionDirectory FindDirectory(Version clientVersion, IEnumerable directoryNames); + + /// + /// Gets an exact version supported by this definition loader, or the closest version below it. + /// + /// The version to find. + /// The closest version supported by this definition loader. + /// Thrown when no version is supported by this definition loader. + public Version GetExactVersion(Version version); +} \ No newline at end of file diff --git a/Nodsoft.WowsReplaysUnpack.Core/Definitions/IDefinitionStore.cs b/Nodsoft.WowsReplaysUnpack.Core/Definitions/IDefinitionStore.cs index f098e9c..870dace 100644 --- a/Nodsoft.WowsReplaysUnpack.Core/Definitions/IDefinitionStore.cs +++ b/Nodsoft.WowsReplaysUnpack.Core/Definitions/IDefinitionStore.cs @@ -9,6 +9,11 @@ namespace Nodsoft.WowsReplaysUnpack.Core.Definitions; /// public interface IDefinitionStore { + /// + /// The Definition loader used to load this store's definitions. + /// + IDefinitionLoader Loader { get; } + /// /// Gets the data type of an XML node. /// @@ -32,13 +37,4 @@ public interface IDefinitionStore /// Name of the property definition /// Property definition EntityDefinition GetEntityDefinition(Version clientVersion, string name); - - /// - /// Reads XML data from a .def file (seeking through directories as needed). - /// - /// Game client version - /// Name of the .def file to read - /// Directories where to search for the .def file - /// XML data - XmlDocument GetFileAsXml(Version clientVersion, string name, params string[] directoryNames); } From 00cc5e22e9ea039b05ce9d9d18beffc86496f554 Mon Sep 17 00:00:00 2001 From: Sakura Akeno Isayeki Date: Fri, 9 Sep 2022 14:57:01 +0200 Subject: [PATCH 03/20] Update DI extensions & builder to include `IDefinitionLoader` --- .../ReplayUnpackerBuilder.cs | 114 ++++++++++++++++++ .../ServiceCollectionExtensions.cs | 88 ++------------ 2 files changed, 122 insertions(+), 80 deletions(-) create mode 100644 Nodsoft.WowsReplaysUnpack/ReplayUnpackerBuilder.cs diff --git a/Nodsoft.WowsReplaysUnpack/ReplayUnpackerBuilder.cs b/Nodsoft.WowsReplaysUnpack/ReplayUnpackerBuilder.cs new file mode 100644 index 0000000..5c7a226 --- /dev/null +++ b/Nodsoft.WowsReplaysUnpack/ReplayUnpackerBuilder.cs @@ -0,0 +1,114 @@ +using Microsoft.Extensions.DependencyInjection; +using Nodsoft.WowsReplaysUnpack.Controllers; +using Nodsoft.WowsReplaysUnpack.Core.Definitions; +using Nodsoft.WowsReplaysUnpack.Services; + +namespace Nodsoft.WowsReplaysUnpack; + +/// +/// Provides a fluent API to build a WOWS replay data unpacker. +/// +public class ReplayUnpackerBuilder +{ + private bool replayDataParserAdded; + private bool definitionStoreAdded; + private bool definitionLoaderAdded; + + public IServiceCollection Services { get; } + + /// + /// Intializes a new instance of the class, + /// by registering a as baseline. + /// + /// + public ReplayUnpackerBuilder(IServiceCollection services) + { + Services = services; + AddReplayController(); + } + + /// + /// Registers a for use in the WOWS replay data unpacker. + /// + /// The type of the replay data parser. + /// The builder. + public ReplayUnpackerBuilder WithReplayDataParser() where TParser : class, IReplayDataParser + { + Services.AddScoped(); + replayDataParserAdded = true; + return this; + } + + /// + /// Registers a for use in the WOWS replay data unpacker. + /// + /// The type of the replay controller. + /// The builder. + public ReplayUnpackerBuilder AddReplayController() where TController : class, IReplayController + { + Services.AddScoped>(); + Services.AddScoped(); + return this; + } + + /// + /// Registers a for use in the WOWS replay data unpacker. + /// + /// The type of the definition loader. + /// The builder. + public ReplayUnpackerBuilder WithDefinitionLoader() where TLoader : class, IDefinitionLoader + { + Services.AddScoped(); + definitionLoaderAdded = true; + return this; + } + + /// + /// Registers a for use in the WOWS replay data unpacker. + /// + /// The type of the definition store. + /// The builder. + public ReplayUnpackerBuilder WithDefinitionStore() where TStore : class, IDefinitionStore + { + Services.AddSingleton(); + definitionStoreAdded = true; + return this; + } + + /// + /// Builds the WOWS replay data unpacker, registering any missing services. + /// + public void Build() + { + if (!replayDataParserAdded) + { + WithReplayDataParser(); + } + + if (!definitionStoreAdded) + { + WithDefinitionStore(); + } + + if (!definitionLoaderAdded) + { + WithDefinitionLoader(); + } + } +} + +public static class ReplayUnpackerBuilderExtensions +{ + /// + /// Registers the Embedded definition loader and the default definition store for the WOWS replay data unpacker. + /// These are considered the default definition services for the unpacker. + /// + /// The replay unpacker builder. + /// The service collection. + public static ReplayUnpackerBuilder WithDefaultDefinitions(this ReplayUnpackerBuilder builder) + { + builder.WithDefinitionLoader(); + builder.WithDefinitionStore(); + return builder; + } +} \ No newline at end of file diff --git a/Nodsoft.WowsReplaysUnpack/ServiceCollectionExtensions.cs b/Nodsoft.WowsReplaysUnpack/ServiceCollectionExtensions.cs index 53d4023..5cdee02 100644 --- a/Nodsoft.WowsReplaysUnpack/ServiceCollectionExtensions.cs +++ b/Nodsoft.WowsReplaysUnpack/ServiceCollectionExtensions.cs @@ -1,5 +1,4 @@ using Microsoft.Extensions.DependencyInjection; -using Nodsoft.WowsReplaysUnpack.Controllers; using Nodsoft.WowsReplaysUnpack.Core.Definitions; using Nodsoft.WowsReplaysUnpack.Services; @@ -18,25 +17,28 @@ public static class ServiceCollectionExtensions /// The service collection. /// The service collection. public static IServiceCollection AddWowsReplayUnpacker(this IServiceCollection services) - => services.AddWowsReplayUnpacker(); - + => services.AddWowsReplayUnpacker(); + /// /// Registers the WOWS replay data unpacker, - /// using the specified and + /// using the specified , and /// for parsing the replay data and definitions. /// /// The service collection to add the services to. /// The type of the replay data parser. /// The type of the definition store. + /// The type of the definition loader. /// The service collection. - public static IServiceCollection AddWowsReplayUnpacker(this IServiceCollection services) + public static IServiceCollection AddWowsReplayUnpacker(this IServiceCollection services) where TReplayDataParser : class, IReplayDataParser where TDefinitionStore : class, IDefinitionStore + where TDefinitionLoader : class, IDefinitionLoader => services.AddWowsReplayUnpacker(unpacker => { unpacker .WithReplayDataParser() - .WithDefinitionStore(); + .WithDefinitionStore() + .WithDefinitionLoader(); } ); @@ -56,78 +58,4 @@ public static IServiceCollection AddWowsReplayUnpacker(this IServiceCollection s services.AddScoped(); return services; } - - /// - /// Provides a fluent API to build a WOWS replay data unpacker. - /// - public class ReplayUnpackerBuilder - { - private bool replayDataParserAdded; - private bool definitionStoreAdded; - - public IServiceCollection Services { get; } - - /// - /// Intializes a new instance of the class, - /// by registering a as baseline. - /// - /// - public ReplayUnpackerBuilder(IServiceCollection services) - { - Services = services; - AddReplayController(); - } - - /// - /// Registers a for use in the WOWS replay data unpacker. - /// - /// The type of the replay data parser. - /// The builder. - public ReplayUnpackerBuilder WithReplayDataParser() where TParser : class, IReplayDataParser - { - Services.AddScoped(); - replayDataParserAdded = true; - return this; - } - - /// - /// Registers a for use in the WOWS replay data unpacker. - /// - /// The type of the replay controller. - /// The builder. - public ReplayUnpackerBuilder AddReplayController() where TController : class, IReplayController - { - Services.AddScoped>(); - Services.AddScoped(); - return this; - } - - /// - /// Registers a for use in the WOWS replay data unpacker. - /// - /// The type of the definition store. - /// The builder. - public ReplayUnpackerBuilder WithDefinitionStore() where TStore : class, IDefinitionStore - { - Services.AddSingleton(); - definitionStoreAdded = true; - return this; - } - - /// - /// Builds the WOWS replay data unpacker, registering any missing services. - /// - public void Build() - { - if (!replayDataParserAdded) - { - WithReplayDataParser(); - } - - if (!definitionStoreAdded) - { - WithDefinitionStore(); - } - } - } } From 19c4d88cb9b2c4cdb33f86abf54d09152f743ad1 Mon Sep 17 00:00:00 2001 From: stewieoO Date: Fri, 9 Sep 2022 15:38:47 +0200 Subject: [PATCH 04/20] Fix the DefinitionLoader Extraction --- ...nLoader.cs => AssemblyDefinitionLoader.cs} | 76 ++++++------------- .../Definitions/BaseDefinition.cs | 4 +- .../Definitions/DefaultDefinitionStore.cs | 63 ++++++++++----- .../Definitions/IDefinitionLoader.cs | 24 +----- .../Definitions/IDefinitionStore.cs | 18 +++-- .../ReplayUnpackerBuilder.cs | 41 +++++----- .../ServiceCollectionExtensions.cs | 2 +- 7 files changed, 103 insertions(+), 125 deletions(-) rename Nodsoft.WowsReplaysUnpack.Core/Definitions/{EmbeddedDefinitionLoader.cs => AssemblyDefinitionLoader.cs} (53%) diff --git a/Nodsoft.WowsReplaysUnpack.Core/Definitions/EmbeddedDefinitionLoader.cs b/Nodsoft.WowsReplaysUnpack.Core/Definitions/AssemblyDefinitionLoader.cs similarity index 53% rename from Nodsoft.WowsReplaysUnpack.Core/Definitions/EmbeddedDefinitionLoader.cs rename to Nodsoft.WowsReplaysUnpack.Core/Definitions/AssemblyDefinitionLoader.cs index dfbf0a7..1e8e6c7 100644 --- a/Nodsoft.WowsReplaysUnpack.Core/Definitions/EmbeddedDefinitionLoader.cs +++ b/Nodsoft.WowsReplaysUnpack.Core/Definitions/AssemblyDefinitionLoader.cs @@ -1,57 +1,42 @@ -using Microsoft.Extensions.Logging; -using Nodsoft.WowsReplaysUnpack.Core.Exceptions; -using Nodsoft.WowsReplaysUnpack.Core.Extensions; +using Nodsoft.WowsReplaysUnpack.Core.Extensions; using System.Reflection; -using System.Runtime.CompilerServices; using System.Xml; namespace Nodsoft.WowsReplaysUnpack.Core.Definitions; - -/// -/// Provides a definition loader to load entity definitions from XML files embedded in the library's assembly. -/// -public class EmbeddedDefinitionLoader : IDefinitionLoader +public class AssemblyDefinitionLoader : IDefinitionLoader { /// - /// Version -> Definitions Directory + /// Assembly of the Definition store (defaults to the implementation assembly). /// - protected readonly Dictionary DirectoryCache = new(); + protected readonly Assembly Assembly; /// - /// Logger instance used by this definition store. + /// Version -> Definitions Directory /// - protected readonly ILogger Logger; - - protected internal Assembly Assembly { get; } + protected readonly Dictionary DirectoryCache = new(); - /// - /// Lists all the game client versions that are supported by this definition loader. - /// - public IEnumerable SupportedVersions => _supportedVersions; + public AssemblyDefinitionLoader() + { - private readonly Version[] _supportedVersions; + Assembly = typeof(DefaultDefinitionStore).Assembly; + } - public EmbeddedDefinitionLoader(ILogger logger) + public Version[] GetSupportedVersions() { - Logger = logger; - Assembly = typeof(EmbeddedDefinitionLoader).Assembly; - - string versionsDirectory = string.Join('.', Assembly.FullName!.GetStringBeforeIndex(','), nameof(Definitions), "Versions"); - - _supportedVersions = Assembly.GetManifestResourceNames() + string versionsDirectory = JoinPath(Assembly.FullName!.GetStringBeforeIndex(Consts.Comma), "Definitions", "Versions"); + return Assembly.GetManifestResourceNames() .Where(name => name.StartsWith(versionsDirectory)) - .Select(name => name.GetStringAfterLength(versionsDirectory + '.').GetStringBeforeIndex('.')[1..]) + .Select(name => name.GetStringAfterLength(versionsDirectory + Consts.Dot).GetStringBeforeIndex(Consts.Dot)[1..]) .Distinct() - .Select(static version => version.Split('_').Select(int.Parse).ToArray()) - .Select(static arr => new Version(arr[0], arr[1], arr[2])) - .OrderByDescending(static version => version) + .Select(version => version.Split(Consts.Underscore).Select(int.Parse).ToArray()) + .Select(arr => new Version(arr[0], arr[1], arr[2])) + .OrderByDescending(version => version) .ToArray(); } /// public virtual XmlDocument GetFileAsXml(Version clientVersion, string name, params string[] directoryNames) { - clientVersion = GetExactVersion(clientVersion); DefinitionDirectory directory = FindDirectory(clientVersion, directoryNames); if (directory.Files.SingleOrDefault(f => f.Name.Equals(name, StringComparison.InvariantCultureIgnoreCase)) is not { } file) @@ -104,8 +89,8 @@ public virtual DefinitionDirectory GetRootDirectory(Version clientVersion) return rootDirectory; } - string scriptsDirectory = string.Join('.', Assembly.FullName!.GetStringBeforeIndex(','), - "Definitions", "Versions", $"_{clientVersion.ToString().Replace('.', '_')}", "scripts" + string scriptsDirectory = JoinPath(Assembly.FullName!.GetStringBeforeIndex(Consts.Comma), + "Definitions", "Versions", $"{Consts.Underscore}{clientVersion.ToString().Replace(Consts.Dot, Consts.Underscore)}", "scripts" ); string[] fileNames = Assembly.GetManifestResourceNames() @@ -117,22 +102,11 @@ public virtual DefinitionDirectory GetRootDirectory(Version clientVersion) return rootDirectory; } - + /// - /// Gets an exact version supported by this definition loader, or the closest version below it. + /// Joins parts of an XML path, separated by a dot. /// - /// The version to find. - /// The closest version supported by this definition loader. - /// Thrown when no version is supported by this definition loader. - public Version GetExactVersion(Version version) - { - Version actualVersion = SupportedVersions.FirstOrDefault(v => version >= v) ?? throw new VersionNotSupportedException(_supportedVersions.Last(), version); - - if (actualVersion != version) - { - Logger.LogWarning("The requested version does not match the latest supported version. Requested: {requested}, Latest: {latest}", version, actualVersion); - } - - return actualVersion; - } -} \ No newline at end of file + /// Parts of the path. + /// The joined path. + protected static string JoinPath(params string[] parts) => string.Join(Consts.Dot, parts); +} diff --git a/Nodsoft.WowsReplaysUnpack.Core/Definitions/BaseDefinition.cs b/Nodsoft.WowsReplaysUnpack.Core/Definitions/BaseDefinition.cs index b51cd17..43885a1 100644 --- a/Nodsoft.WowsReplaysUnpack.Core/Definitions/BaseDefinition.cs +++ b/Nodsoft.WowsReplaysUnpack.Core/Definitions/BaseDefinition.cs @@ -39,7 +39,7 @@ protected BaseDefinition(Version clientVersion, IDefinitionStore definitionStore Name = name; _folder = folder; - ParseDefinitionFile(DefinitionStore.Loader.GetFileAsXml(ClientVersion, Name + ".def", _folder).DocumentElement!); + ParseDefinitionFile(DefinitionStore.GetFileAsXml(ClientVersion, Name + ".def", _folder).DocumentElement!); } @@ -63,7 +63,7 @@ private void ParseImplements(IEnumerable interfaces) { foreach (string @interface in interfaces) { - ParseDefinitionFile(DefinitionStore.Loader.GetFileAsXml(ClientVersion, @interface + ".def", _folder, "interfaces").DocumentElement!); + ParseDefinitionFile(DefinitionStore.GetFileAsXml(ClientVersion, @interface + ".def", _folder, "interfaces").DocumentElement!); } } diff --git a/Nodsoft.WowsReplaysUnpack.Core/Definitions/DefaultDefinitionStore.cs b/Nodsoft.WowsReplaysUnpack.Core/Definitions/DefaultDefinitionStore.cs index 4ff3274..587d485 100644 --- a/Nodsoft.WowsReplaysUnpack.Core/Definitions/DefaultDefinitionStore.cs +++ b/Nodsoft.WowsReplaysUnpack.Core/Definitions/DefaultDefinitionStore.cs @@ -2,8 +2,6 @@ using Nodsoft.WowsReplaysUnpack.Core.DataTypes; using Nodsoft.WowsReplaysUnpack.Core.Exceptions; using Nodsoft.WowsReplaysUnpack.Core.Extensions; -using System.Reflection; -using System.Runtime.CompilerServices; using System.Xml; namespace Nodsoft.WowsReplaysUnpack.Core.Definitions; @@ -18,6 +16,14 @@ public class DefaultDefinitionStore : IDefinitionStore /// protected const string EntitiesXml = "entities.xml"; + private readonly Version[] _supportedVersions; + + /// + /// Logger instance used by this definition store. + /// + protected ILogger Logger { get; } + + public IDefinitionLoader DefinitionLoader { get; } /// /// version_name -> Definition @@ -39,21 +45,21 @@ public class DefaultDefinitionStore : IDefinitionStore /// protected readonly Dictionary> TypeMappings = new(); - public IDefinitionLoader Loader { get; } - - public DefaultDefinitionStore(IDefinitionLoader embeddedDefinitionLoader) + public DefaultDefinitionStore(ILogger logger, IDefinitionLoader definitionLoader) { - Loader = embeddedDefinitionLoader; - } - + Logger = logger; + DefinitionLoader = definitionLoader; + + _supportedVersions = DefinitionLoader.GetSupportedVersions(); + } #region EntityDefinitions /// public virtual EntityDefinition GetEntityDefinition(Version clientVersion, int index) { - clientVersion = Loader.GetExactVersion(clientVersion); + clientVersion = GetActualVersion(clientVersion); string name = GetEntityDefinitionName(clientVersion, index); return GetEntityDefinition(clientVersion, name); @@ -62,8 +68,8 @@ public virtual EntityDefinition GetEntityDefinition(Version clientVersion, int i /// public virtual EntityDefinition GetEntityDefinition(Version clientVersion, string name) { - clientVersion = Loader.GetExactVersion(clientVersion); - string cacheKey = string.Join('_', clientVersion.ToString(), name); + clientVersion = GetActualVersion(clientVersion); + string cacheKey = CacheKey(clientVersion.ToString(), name); if (EntityDefinitionCache.TryGetValue(cacheKey, out EntityDefinition? definition)) { @@ -83,7 +89,7 @@ public virtual EntityDefinition GetEntityDefinition(Version clientVersion, strin /// The name of the entity definition. protected virtual string GetEntityDefinitionName(Version clientVersion, int index) { - string cacheKey = string.Join('_', clientVersion.ToString(), (index - 1).ToString()); + string cacheKey = CacheKey(clientVersion.ToString(), (index - 1).ToString()); if (EntityDefinitionIndexNameCache.TryGetValue(cacheKey, out string? name)) { @@ -103,23 +109,20 @@ protected virtual string GetEntityDefinitionName(Version clientVersion, int inde /// A dictionary of entity indexes and their names. protected virtual Dictionary GetEntityIndexes(Version clientVersion) { - return GetFileAsXml(clientVersion, EntitiesXml).DocumentElement!.SelectSingleNode("ClientServerEntities")!.ChildNodes() + return DefinitionLoader.GetFileAsXml(GetActualVersion(clientVersion), EntitiesXml).DocumentElement!.SelectSingleNode("ClientServerEntities")!.ChildNodes() .Select((node, index) => new { node, index }) .ToDictionary(i => i.index, i => i.node.Name); } #endregion - /// - public virtual XmlDocument GetFileAsXml(Version clientVersion, string name, params string[] directoryNames) - { - return Loader.GetFileAsXml(clientVersion, name, directoryNames); - } + public virtual XmlDocument GetFileAsXml(Version clientVersion, string name, params string[] directoryNames) + => DefinitionLoader.GetFileAsXml(GetActualVersion(clientVersion), name, directoryNames); /// public virtual DataTypeBase GetDataType(Version clientVersion, XmlNode typeOrArgXmlNode) { - clientVersion = Loader.GetExactVersion(clientVersion); + clientVersion = GetActualVersion(clientVersion); string versionString = clientVersion.ToString(); if (TypeMappings.TryGetValue(versionString, out Dictionary? typeMapping)) @@ -127,7 +130,7 @@ public virtual DataTypeBase GetDataType(Version clientVersion, XmlNode typeOrArg return GetDataTypeInternal(clientVersion, typeMapping, typeOrArgXmlNode); } - XmlDocument aliasXml = GetFileAsXml(clientVersion, "alias.xml", "entity_defs"); + XmlDocument aliasXml = DefinitionLoader.GetFileAsXml(clientVersion, "alias.xml", "entity_defs"); typeMapping = new(); foreach (XmlNode node in aliasXml.DocumentElement!.ChildNodes) @@ -140,6 +143,7 @@ public virtual DataTypeBase GetDataType(Version clientVersion, XmlNode typeOrArg return GetDataTypeInternal(clientVersion, typeMapping, typeOrArgXmlNode); } + /// /// Internal method for getting a data type. /// @@ -168,4 +172,23 @@ protected virtual DataTypeBase GetDataTypeInternal(Version clientVersion, Dictio } } } + + private Version GetActualVersion(Version version) + { + Version actualVersion = _supportedVersions.FirstOrDefault(v => version >= v) ?? throw new VersionNotSupportedException(_supportedVersions.Last(), version); + + if (actualVersion != version) + { + Logger.LogWarning("The requested version does not match the latest supported version. Requested: {requested}, Latest: {latest}", version, actualVersion); + } + + return actualVersion; + } + + /// + /// Joins parts of a cache key, separated by an underscore. + /// + /// Parts of the cache key. + /// The joined cache key. + protected static string CacheKey(params string[] values) => string.Join(Consts.Underscore, values); } \ No newline at end of file diff --git a/Nodsoft.WowsReplaysUnpack.Core/Definitions/IDefinitionLoader.cs b/Nodsoft.WowsReplaysUnpack.Core/Definitions/IDefinitionLoader.cs index 6664b9f..ff6a484 100644 --- a/Nodsoft.WowsReplaysUnpack.Core/Definitions/IDefinitionLoader.cs +++ b/Nodsoft.WowsReplaysUnpack.Core/Definitions/IDefinitionLoader.cs @@ -14,27 +14,5 @@ public interface IDefinitionLoader /// XML data XmlDocument GetFileAsXml(Version clientVersion, string name, params string[] directoryNames); - /// - /// Gets the root definition directory for a given game client version. - /// - /// The game client version. - /// The root definition directory. - DefinitionDirectory GetRootDirectory(Version clientVersion); - - - /// - /// Finds a definition directory by given names. - /// - /// The game client version. - /// The names of the directories. - /// The definition directory. - DefinitionDirectory FindDirectory(Version clientVersion, IEnumerable directoryNames); - - /// - /// Gets an exact version supported by this definition loader, or the closest version below it. - /// - /// The version to find. - /// The closest version supported by this definition loader. - /// Thrown when no version is supported by this definition loader. - public Version GetExactVersion(Version version); + Version[] GetSupportedVersions(); } \ No newline at end of file diff --git a/Nodsoft.WowsReplaysUnpack.Core/Definitions/IDefinitionStore.cs b/Nodsoft.WowsReplaysUnpack.Core/Definitions/IDefinitionStore.cs index 870dace..5f06955 100644 --- a/Nodsoft.WowsReplaysUnpack.Core/Definitions/IDefinitionStore.cs +++ b/Nodsoft.WowsReplaysUnpack.Core/Definitions/IDefinitionStore.cs @@ -9,11 +9,6 @@ namespace Nodsoft.WowsReplaysUnpack.Core.Definitions; /// public interface IDefinitionStore { - /// - /// The Definition loader used to load this store's definitions. - /// - IDefinitionLoader Loader { get; } - /// /// Gets the data type of an XML node. /// @@ -21,7 +16,7 @@ public interface IDefinitionStore /// XML node to get the data type of /// Data type of the XML node DataTypeBase GetDataType(Version clientVersion, XmlNode typeOrArgXmlNode); - + /// /// Gets an entity definition by its index. /// @@ -29,7 +24,7 @@ public interface IDefinitionStore /// Index of the entity definition /// Entity definition EntityDefinition GetEntityDefinition(Version clientVersion, int index); - + /// /// Gets a property definition by its name. /// @@ -37,4 +32,13 @@ public interface IDefinitionStore /// Name of the property definition /// Property definition EntityDefinition GetEntityDefinition(Version clientVersion, string name); + + /// + /// Gets the definition file from the definition loader + /// + /// Game client version + /// Name of the .def file to read + /// Directories where to search for the .def file + /// XML data + XmlDocument GetFileAsXml(Version clientVersion, string name, params string[] directoryNames); } diff --git a/Nodsoft.WowsReplaysUnpack/ReplayUnpackerBuilder.cs b/Nodsoft.WowsReplaysUnpack/ReplayUnpackerBuilder.cs index 5c7a226..5d761f7 100644 --- a/Nodsoft.WowsReplaysUnpack/ReplayUnpackerBuilder.cs +++ b/Nodsoft.WowsReplaysUnpack/ReplayUnpackerBuilder.cs @@ -26,7 +26,7 @@ public ReplayUnpackerBuilder(IServiceCollection services) Services = services; AddReplayController(); } - + /// /// Registers a for use in the WOWS replay data unpacker. /// @@ -62,7 +62,7 @@ public ReplayUnpackerBuilder WithDefinitionLoader() where TLoader : cla definitionLoaderAdded = true; return this; } - + /// /// Registers a for use in the WOWS replay data unpacker. /// @@ -74,7 +74,22 @@ public ReplayUnpackerBuilder WithDefinitionStore() where TStore : class, definitionStoreAdded = true; return this; } - + + + // stewie says: No need for that since they will be added either way if you don't add other ones + ///// + ///// Registers the Assembly definition loader and the default definition store for the WOWS replay data unpacker. + ///// These are considered the default definition services for the unpacker. + ///// + ///// The replay unpacker builder. + ///// The service collection. + //public static ReplayUnpackerBuilder WithDefaultDefinitions(this ReplayUnpackerBuilder builder) + //{ + // builder.WithDefinitionLoader(); + // builder.WithDefinitionStore(); + // return builder; + //} + /// /// Builds the WOWS replay data unpacker, registering any missing services. /// @@ -89,26 +104,10 @@ public void Build() { WithDefinitionStore(); } - + if (!definitionLoaderAdded) { - WithDefinitionLoader(); + WithDefinitionLoader(); } } -} - -public static class ReplayUnpackerBuilderExtensions -{ - /// - /// Registers the Embedded definition loader and the default definition store for the WOWS replay data unpacker. - /// These are considered the default definition services for the unpacker. - /// - /// The replay unpacker builder. - /// The service collection. - public static ReplayUnpackerBuilder WithDefaultDefinitions(this ReplayUnpackerBuilder builder) - { - builder.WithDefinitionLoader(); - builder.WithDefinitionStore(); - return builder; - } } \ No newline at end of file diff --git a/Nodsoft.WowsReplaysUnpack/ServiceCollectionExtensions.cs b/Nodsoft.WowsReplaysUnpack/ServiceCollectionExtensions.cs index 5cdee02..e8ee16e 100644 --- a/Nodsoft.WowsReplaysUnpack/ServiceCollectionExtensions.cs +++ b/Nodsoft.WowsReplaysUnpack/ServiceCollectionExtensions.cs @@ -17,7 +17,7 @@ public static class ServiceCollectionExtensions /// The service collection. /// The service collection. public static IServiceCollection AddWowsReplayUnpacker(this IServiceCollection services) - => services.AddWowsReplayUnpacker(); + => services.AddWowsReplayUnpacker(); /// /// Registers the WOWS replay data unpacker, From 68a97d894d2104e17c71a994dbcbfb4eeb5c568e Mon Sep 17 00:00:00 2001 From: stewieoO Date: Fri, 9 Sep 2022 15:39:25 +0200 Subject: [PATCH 05/20] fix visibility of loader --- .../Definitions/DefaultDefinitionStore.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Nodsoft.WowsReplaysUnpack.Core/Definitions/DefaultDefinitionStore.cs b/Nodsoft.WowsReplaysUnpack.Core/Definitions/DefaultDefinitionStore.cs index 587d485..60115b7 100644 --- a/Nodsoft.WowsReplaysUnpack.Core/Definitions/DefaultDefinitionStore.cs +++ b/Nodsoft.WowsReplaysUnpack.Core/Definitions/DefaultDefinitionStore.cs @@ -23,7 +23,7 @@ public class DefaultDefinitionStore : IDefinitionStore /// protected ILogger Logger { get; } - public IDefinitionLoader DefinitionLoader { get; } + protected IDefinitionLoader DefinitionLoader { get; } /// /// version_name -> Definition From 728faf572a1bf9b56b635a7dc9a62492b7cfae96 Mon Sep 17 00:00:00 2001 From: Sakura Akeno Isayeki Date: Sun, 11 Sep 2022 07:16:57 +0200 Subject: [PATCH 06/20] Draft Filesystem defs loader implementation --- Directory.Build.props | 7 -- .../Definitions/AssemblyDefinitionLoader.cs | 20 +++--- .../Definitions/IDefinitionLoader.cs | 7 +- .../Nodsoft.WowsReplaysUnpack.Core.csproj | 2 +- .../Definitions/FileSystemDefinitionLoader.cs | 68 +++++++++++++++++++ .../FileSystemDefinitionLoaderOptions.cs | 17 +++++ ...Nodsoft.WowsReplaysUnpack.FileStore.csproj | 4 ++ 7 files changed, 106 insertions(+), 19 deletions(-) create mode 100644 Nodsoft.WowsReplaysUnpack.FileStore/Definitions/FileSystemDefinitionLoader.cs create mode 100644 Nodsoft.WowsReplaysUnpack.FileStore/Definitions/FileSystemDefinitionLoaderOptions.cs diff --git a/Directory.Build.props b/Directory.Build.props index 4a95f88..2c2c7db 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -4,9 +4,6 @@ True latest - - - Nodsoft Systems imkindaprogrammermyself, Sakura Akeno Isayeki, floribe2000, StewieoO WoWS Replays Unpack @@ -33,10 +30,6 @@ - - - - diff --git a/Nodsoft.WowsReplaysUnpack.Core/Definitions/AssemblyDefinitionLoader.cs b/Nodsoft.WowsReplaysUnpack.Core/Definitions/AssemblyDefinitionLoader.cs index 1e8e6c7..4862f10 100644 --- a/Nodsoft.WowsReplaysUnpack.Core/Definitions/AssemblyDefinitionLoader.cs +++ b/Nodsoft.WowsReplaysUnpack.Core/Definitions/AssemblyDefinitionLoader.cs @@ -5,6 +5,8 @@ namespace Nodsoft.WowsReplaysUnpack.Core.Definitions; public class AssemblyDefinitionLoader : IDefinitionLoader { + private static readonly XmlReaderSettings _xmlReaderSettings = new() { IgnoreComments = true }; + /// /// Assembly of the Definition store (defaults to the implementation assembly). /// @@ -21,16 +23,17 @@ public AssemblyDefinitionLoader() Assembly = typeof(DefaultDefinitionStore).Assembly; } + /// public Version[] GetSupportedVersions() { - string versionsDirectory = JoinPath(Assembly.FullName!.GetStringBeforeIndex(Consts.Comma), "Definitions", "Versions"); + string versionsDirectory = JoinPath(Assembly.FullName!.GetStringBeforeIndex(','), "Definitions", "Versions"); return Assembly.GetManifestResourceNames() .Where(name => name.StartsWith(versionsDirectory)) - .Select(name => name.GetStringAfterLength(versionsDirectory + Consts.Dot).GetStringBeforeIndex(Consts.Dot)[1..]) + .Select(name => name.GetStringAfterLength(versionsDirectory + '.').GetStringBeforeIndex('.')[1..]) .Distinct() - .Select(version => version.Split(Consts.Underscore).Select(int.Parse).ToArray()) - .Select(arr => new Version(arr[0], arr[1], arr[2])) - .OrderByDescending(version => version) + .Select(static version => version.Split('_').Select(int.Parse).ToArray()) + .Select(static arr => new Version(arr[0], arr[1], arr[2])) + .OrderByDescending(static version => version) .ToArray(); } @@ -43,9 +46,8 @@ public virtual XmlDocument GetFileAsXml(Version clientVersion, string name, para { throw new InvalidOperationException("File could not be found"); } - - XmlReaderSettings settings = new() { IgnoreComments = true }; - XmlReader reader = XmlReader.Create(Assembly.GetManifestResourceStream(file.Path) ?? throw new InvalidOperationException("File not found"), settings); + + XmlReader reader = XmlReader.Create(Assembly.GetManifestResourceStream(file.Path) ?? throw new InvalidOperationException("File not found"), _xmlReaderSettings); XmlDocument xmlDocument = new(); xmlDocument.Load(reader); @@ -108,5 +110,5 @@ public virtual DefinitionDirectory GetRootDirectory(Version clientVersion) /// /// Parts of the path. /// The joined path. - protected static string JoinPath(params string[] parts) => string.Join(Consts.Dot, parts); + protected static string JoinPath(params string[] parts) => string.Join('.', parts); } diff --git a/Nodsoft.WowsReplaysUnpack.Core/Definitions/IDefinitionLoader.cs b/Nodsoft.WowsReplaysUnpack.Core/Definitions/IDefinitionLoader.cs index ff6a484..4a158ec 100644 --- a/Nodsoft.WowsReplaysUnpack.Core/Definitions/IDefinitionLoader.cs +++ b/Nodsoft.WowsReplaysUnpack.Core/Definitions/IDefinitionLoader.cs @@ -1,5 +1,4 @@ -using Nodsoft.WowsReplaysUnpack.Core.Exceptions; -using System.Xml; +using System.Xml; namespace Nodsoft.WowsReplaysUnpack.Core.Definitions; @@ -14,5 +13,9 @@ public interface IDefinitionLoader /// XML data XmlDocument GetFileAsXml(Version clientVersion, string name, params string[] directoryNames); + /// + /// Gets all game client versions supported by this definition loader. + /// + /// An array of all supported game client versions. Version[] GetSupportedVersions(); } \ No newline at end of file diff --git a/Nodsoft.WowsReplaysUnpack.Core/Nodsoft.WowsReplaysUnpack.Core.csproj b/Nodsoft.WowsReplaysUnpack.Core/Nodsoft.WowsReplaysUnpack.Core.csproj index a38cb30..e61cd0a 100644 --- a/Nodsoft.WowsReplaysUnpack.Core/Nodsoft.WowsReplaysUnpack.Core.csproj +++ b/Nodsoft.WowsReplaysUnpack.Core/Nodsoft.WowsReplaysUnpack.Core.csproj @@ -1,4 +1,4 @@ - + Library diff --git a/Nodsoft.WowsReplaysUnpack.FileStore/Definitions/FileSystemDefinitionLoader.cs b/Nodsoft.WowsReplaysUnpack.FileStore/Definitions/FileSystemDefinitionLoader.cs new file mode 100644 index 0000000..397090c --- /dev/null +++ b/Nodsoft.WowsReplaysUnpack.FileStore/Definitions/FileSystemDefinitionLoader.cs @@ -0,0 +1,68 @@ +using Microsoft.Extensions.Options; +using Nodsoft.WowsReplaysUnpack.Core.Definitions; +using System.Runtime.CompilerServices; +using System.Xml; + +namespace Nodsoft.WowsReplaysUnpack.FileStore.Definitions; + +/// +/// Provides a definition loader using the filesystem as a backing store. +/// +public class FileSystemDefinitionLoader : IDefinitionLoader +{ + private static readonly XmlReaderSettings _xmlReaderSettings = new() + { + IgnoreComments = true + }; + + private readonly IOptionsMonitor _options; + + public FileSystemDefinitionLoader(IOptionsMonitor options) + { + _options = options; + } + + /// + public XmlDocument GetFileAsXml(Version clientVersion, string name, params string[] directoryNames) + { + string path = Path.Combine(_options.CurrentValue.RootDirectory, ToFilesystemString(clientVersion), "scripts", Path.Combine(directoryNames), name); + + if (!File.Exists(path)) + { + throw new FileNotFoundException($"Could not find definition file {name} in {path}"); + } + + using Stream stream = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read); + + // Use an XmlReader to load the file, as it is more efficient than loading the entire file into memory. + using XmlReader reader = XmlReader.Create(stream, _xmlReaderSettings); + XmlDocument document = new(); + document.Load(reader); + return document; + } + + /// + public Version[] GetSupportedVersions() + { + // For a set directory, we're expecting the child directories to be the versions. + // These will be structured by __ (e.g. 0_11_0), + // so we'll need to parse them accordingly into Version objects. + Version[] versions = new DirectoryInfo(_options.CurrentValue.RootDirectory).GetDirectories() + .Select(static dir => FromFilesystemString(dir.Name)) + .ToArray(); + + return versions; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static string ToFilesystemString(Version version) + { + return version.ToString().Replace('.', '_'); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Version FromFilesystemString(string version) + { + return new(version.Replace('_', '.')); + } +} \ No newline at end of file diff --git a/Nodsoft.WowsReplaysUnpack.FileStore/Definitions/FileSystemDefinitionLoaderOptions.cs b/Nodsoft.WowsReplaysUnpack.FileStore/Definitions/FileSystemDefinitionLoaderOptions.cs new file mode 100644 index 0000000..1319332 --- /dev/null +++ b/Nodsoft.WowsReplaysUnpack.FileStore/Definitions/FileSystemDefinitionLoaderOptions.cs @@ -0,0 +1,17 @@ +namespace Nodsoft.WowsReplaysUnpack.FileStore.Definitions; + +/// +/// Defines settings for the . +/// +public record FileSystemDefinitionLoaderOptions +{ + /// + /// Gets or sets the path to the directory containing the definitions. + /// + public string RootDirectory { get; set; } + + /// + /// Gets or sets whether the loader should poll the filesystem for changes. + /// + public bool EnableChangePolling { get; set; } +} \ No newline at end of file diff --git a/Nodsoft.WowsReplaysUnpack.FileStore/Nodsoft.WowsReplaysUnpack.FileStore.csproj b/Nodsoft.WowsReplaysUnpack.FileStore/Nodsoft.WowsReplaysUnpack.FileStore.csproj index 962dd40..692e682 100644 --- a/Nodsoft.WowsReplaysUnpack.FileStore/Nodsoft.WowsReplaysUnpack.FileStore.csproj +++ b/Nodsoft.WowsReplaysUnpack.FileStore/Nodsoft.WowsReplaysUnpack.FileStore.csproj @@ -11,4 +11,8 @@ + + + + From a2d0658c0e6c6f6d6ed3c4a21b3e7695c3c41017 Mon Sep 17 00:00:00 2001 From: Sakura Akeno Isayeki Date: Sun, 11 Sep 2022 07:17:30 +0200 Subject: [PATCH 07/20] Add integration tests for unpacker configurations (default + fs) --- .../Nodsoft.WowsReplaysUnpack.Tests.csproj | 1 + .../ReplayParsingExecutionTests.cs | 66 +++++++++++++++++++ .../ReplaySanitizerTests.cs | 19 ++---- Nodsoft.WowsReplaysUnpack.Tests/Utilities.cs | 23 +++++++ 4 files changed, 96 insertions(+), 13 deletions(-) create mode 100644 Nodsoft.WowsReplaysUnpack.Tests/ReplayParsingExecutionTests.cs create mode 100644 Nodsoft.WowsReplaysUnpack.Tests/Utilities.cs diff --git a/Nodsoft.WowsReplaysUnpack.Tests/Nodsoft.WowsReplaysUnpack.Tests.csproj b/Nodsoft.WowsReplaysUnpack.Tests/Nodsoft.WowsReplaysUnpack.Tests.csproj index b910a66..1faaac9 100644 --- a/Nodsoft.WowsReplaysUnpack.Tests/Nodsoft.WowsReplaysUnpack.Tests.csproj +++ b/Nodsoft.WowsReplaysUnpack.Tests/Nodsoft.WowsReplaysUnpack.Tests.csproj @@ -24,6 +24,7 @@ + diff --git a/Nodsoft.WowsReplaysUnpack.Tests/ReplayParsingExecutionTests.cs b/Nodsoft.WowsReplaysUnpack.Tests/ReplayParsingExecutionTests.cs new file mode 100644 index 0000000..07e499e --- /dev/null +++ b/Nodsoft.WowsReplaysUnpack.Tests/ReplayParsingExecutionTests.cs @@ -0,0 +1,66 @@ +using Microsoft.Extensions.DependencyInjection; +using Nodsoft.WowsReplaysUnpack.Core.Models; +using Nodsoft.WowsReplaysUnpack.FileStore.Definitions; +using Nodsoft.WowsReplaysUnpack.Services; +using Xunit; + +namespace Nodsoft.WowsReplaysUnpack.Tests; + +/// +/// Provides integration tests for the different replay parser configurations. +/// +public class ReplayParsingExecutionTests +{ + /// + /// Tests the parsing of a replay file, using the default configuration. + /// + [Fact] + public void TestDefaultParsing() + { + // Arrange + IServiceProvider services = new ServiceCollection() + .AddLogging() + .AddWowsReplayUnpacker() + .BuildServiceProvider(); + + ReplayUnpackerFactory unpackerFactory = services.GetRequiredService(); + IReplayUnpackerService service = unpackerFactory.GetUnpacker(); + + // Act + UnpackedReplay metadata = service.Unpack(Utilities.LoadReplay("good.wowsreplay")); + + // Assert + Assert.NotNull(metadata); + } + + /// + /// Tests the parsing of a replay file, using the filesystem defs loader. + /// + /// + /// This test requires the filesystem to be setup correctly, and definitions to be present. + /// + [Fact] + public void TestFilesystemParsing() + { + // Arrange + IServiceProvider services = new ServiceCollection() + .AddLogging() + .AddWowsReplayUnpacker(builder => builder + .WithDefinitionLoader()) + .Configure(options => + { + options.RootDirectory = options.RootDirectory = Path.Join(Directory.GetCurrentDirectory(), + "..", "..", "..", "..", "Nodsoft.WowsReplaysUnpack.Core", "Definitions", "Versions"); + }) + .BuildServiceProvider(); + + ReplayUnpackerFactory unpackerFactory = services.GetRequiredService(); + IReplayUnpackerService service = unpackerFactory.GetUnpacker(); + + // Act + UnpackedReplay metadata = service.Unpack(Utilities.LoadReplay("good.wowsreplay")); + + // Assert + Assert.NotNull(metadata); + } +} \ No newline at end of file diff --git a/Nodsoft.WowsReplaysUnpack.Tests/ReplaySanitizerTests.cs b/Nodsoft.WowsReplaysUnpack.Tests/ReplaySanitizerTests.cs index 4765e02..f77e69d 100644 --- a/Nodsoft.WowsReplaysUnpack.Tests/ReplaySanitizerTests.cs +++ b/Nodsoft.WowsReplaysUnpack.Tests/ReplaySanitizerTests.cs @@ -7,10 +7,13 @@ namespace Nodsoft.WowsReplaysUnpack.Tests; +/// +/// Provides tests for the replay RCE detection system. +/// public class ReplaySanitizerTests { private readonly ReplayUnpackerFactory _factory; - private readonly string _sampleFolder = Path.Join(Directory.GetCurrentDirectory(), "../../../..", "Replay-Samples"); + public ReplaySanitizerTests() { @@ -27,26 +30,16 @@ public ReplaySanitizerTests() [Fact] public void TestGoodReplay_Pass() { - UnpackedReplay replay = _factory.GetUnpacker().Unpack(LoadReplay("good.wowsreplay")); + UnpackedReplay replay = _factory.GetUnpacker().Unpack(Utilities.LoadReplay("good.wowsreplay")); Assert.NotNull(replay); } - private MemoryStream LoadReplay(string replayPath) - { - using FileStream fs = File.OpenRead(Path.Join(_sampleFolder, replayPath)); - MemoryStream ms = new(); - fs.CopyTo(ms); - ms.Position = 0; - - return ms; - } - /// /// Test malicious replay detection (Assets/payload.wowsreplay) /// [Fact] public void TestPayloadReplayDetection() { - Assert.Throws(() => _factory.GetUnpacker().Unpack(LoadReplay("payload.wowsreplay"))); + Assert.Throws(() => _factory.GetUnpacker().Unpack(Utilities.LoadReplay("payload.wowsreplay"))); } } \ No newline at end of file diff --git a/Nodsoft.WowsReplaysUnpack.Tests/Utilities.cs b/Nodsoft.WowsReplaysUnpack.Tests/Utilities.cs new file mode 100644 index 0000000..22695a1 --- /dev/null +++ b/Nodsoft.WowsReplaysUnpack.Tests/Utilities.cs @@ -0,0 +1,23 @@ +using System.Runtime.CompilerServices; + +namespace Nodsoft.WowsReplaysUnpack.Tests; + +/// +/// Helper methods used in tests. +/// +public static class Utilities +{ + private static readonly string _sampleFolder = Path.Join(Directory.GetCurrentDirectory(), "..", "..", "..", "..", "Replay-Samples"); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static MemoryStream LoadReplay(string replayPath) + { + using FileStream fs = File.Open(Path.Join(_sampleFolder, replayPath), FileMode.Open, FileAccess.Read, FileShare.Read); + MemoryStream ms = new(); + fs.CopyTo(ms); + fs.Close(); + ms.Position = 0; + + return ms; + } +} \ No newline at end of file From c4f426bdc7ea2b84fa63404fc9a68bd816803806 Mon Sep 17 00:00:00 2001 From: stewieoO Date: Sun, 11 Sep 2022 15:24:27 +0200 Subject: [PATCH 08/20] Fix Version Loading --- .../Nodsoft.WowsReplaysUnpack.Console.csproj | 1 + Nodsoft.WowsReplaysUnpack.Console/Program.cs | 20 +++++++++++++------ .../Definitions/FileSystemDefinitionLoader.cs | 1 + 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/Nodsoft.WowsReplaysUnpack.Console/Nodsoft.WowsReplaysUnpack.Console.csproj b/Nodsoft.WowsReplaysUnpack.Console/Nodsoft.WowsReplaysUnpack.Console.csproj index 1c38080..9a809b0 100644 --- a/Nodsoft.WowsReplaysUnpack.Console/Nodsoft.WowsReplaysUnpack.Console.csproj +++ b/Nodsoft.WowsReplaysUnpack.Console/Nodsoft.WowsReplaysUnpack.Console.csproj @@ -17,6 +17,7 @@ + diff --git a/Nodsoft.WowsReplaysUnpack.Console/Program.cs b/Nodsoft.WowsReplaysUnpack.Console/Program.cs index 3d0bd32..e6eeb1c 100644 --- a/Nodsoft.WowsReplaysUnpack.Console/Program.cs +++ b/Nodsoft.WowsReplaysUnpack.Console/Program.cs @@ -6,6 +6,7 @@ using Nodsoft.WowsReplaysUnpack.EntitySerializer; using Nodsoft.WowsReplaysUnpack.ExtendedData; using Nodsoft.WowsReplaysUnpack.ExtendedData.Models; +using Nodsoft.WowsReplaysUnpack.FileStore.Definitions; using Nodsoft.WowsReplaysUnpack.Services; using System; using System.Collections.Generic; @@ -17,11 +18,18 @@ FileStream _GetReplayFile(string name) => File.OpenRead(Path.Join(samplePath, name)); ServiceProvider? services = new ServiceCollection() - .AddWowsReplayUnpacker(builder => - { - //builder.AddReplayController(); - builder.AddExtendedData(); - }) + //.AddWowsReplayUnpacker(builder => + //{ + // //builder.AddReplayController(); + // //builder.AddExtendedData(); + //}) + .AddWowsReplayUnpacker(builder => builder + .WithDefinitionLoader()) + .Configure(options => + { + options.RootDirectory = options.RootDirectory = Path.Join(Directory.GetCurrentDirectory(), + "..", "..", "..", "..", "Nodsoft.WowsReplaysUnpack.Core", "Definitions", "Versions"); + }) .AddLogging(logging => { logging.ClearProviders(); @@ -42,7 +50,7 @@ //} - +var goodReplay = replayUnpacker.GetUnpacker().Unpack(_GetReplayFile("good.wowsreplay")); var alphaReplay = replayUnpacker.GetUnpacker().Unpack(_GetReplayFile("press_account_alpha.wowsreplay")); var bravoReplay = replayUnpacker.GetUnpacker().Unpack(_GetReplayFile("unfinished_replay.wowsreplay")); diff --git a/Nodsoft.WowsReplaysUnpack.FileStore/Definitions/FileSystemDefinitionLoader.cs b/Nodsoft.WowsReplaysUnpack.FileStore/Definitions/FileSystemDefinitionLoader.cs index 397090c..cb2e4a6 100644 --- a/Nodsoft.WowsReplaysUnpack.FileStore/Definitions/FileSystemDefinitionLoader.cs +++ b/Nodsoft.WowsReplaysUnpack.FileStore/Definitions/FileSystemDefinitionLoader.cs @@ -49,6 +49,7 @@ public Version[] GetSupportedVersions() // so we'll need to parse them accordingly into Version objects. Version[] versions = new DirectoryInfo(_options.CurrentValue.RootDirectory).GetDirectories() .Select(static dir => FromFilesystemString(dir.Name)) + .OrderByDescending(static version => version) .ToArray(); return versions; From a0c6573fb3cc56e4a7cac2f35b395a0192960180 Mon Sep 17 00:00:00 2001 From: stewieoO Date: Sun, 11 Sep 2022 15:25:21 +0200 Subject: [PATCH 09/20] Add Comment For GetSupportedVersions() --- Nodsoft.WowsReplaysUnpack.Core/Definitions/IDefinitionLoader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Nodsoft.WowsReplaysUnpack.Core/Definitions/IDefinitionLoader.cs b/Nodsoft.WowsReplaysUnpack.Core/Definitions/IDefinitionLoader.cs index 4a158ec..f9206f2 100644 --- a/Nodsoft.WowsReplaysUnpack.Core/Definitions/IDefinitionLoader.cs +++ b/Nodsoft.WowsReplaysUnpack.Core/Definitions/IDefinitionLoader.cs @@ -16,6 +16,6 @@ public interface IDefinitionLoader /// /// Gets all game client versions supported by this definition loader. /// - /// An array of all supported game client versions. + /// An array of all supported game client versions ordered newest -> oldest. Version[] GetSupportedVersions(); } \ No newline at end of file From 0862e5dee7d54974cfaf6491d9737af977b6df10 Mon Sep 17 00:00:00 2001 From: Sakura Akeno Isayeki Date: Mon, 12 Sep 2022 07:44:15 +0200 Subject: [PATCH 10/20] HACK: Disable tests parallelization This commit mitigates issue #23, pending a more potent fix. --- .../ReplayParsingExecutionTests.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Nodsoft.WowsReplaysUnpack.Tests/ReplayParsingExecutionTests.cs b/Nodsoft.WowsReplaysUnpack.Tests/ReplayParsingExecutionTests.cs index 07e499e..36c6625 100644 --- a/Nodsoft.WowsReplaysUnpack.Tests/ReplayParsingExecutionTests.cs +++ b/Nodsoft.WowsReplaysUnpack.Tests/ReplayParsingExecutionTests.cs @@ -4,6 +4,11 @@ using Nodsoft.WowsReplaysUnpack.Services; using Xunit; + +/* + * FIXME: Test parallelization is disabled due to a file loading issue. + */ +[assembly: CollectionBehavior(DisableTestParallelization = true)] namespace Nodsoft.WowsReplaysUnpack.Tests; /// From e66d12aecff0220aab378c5014f5576de843cace Mon Sep 17 00:00:00 2001 From: Yiheng Date: Sun, 1 Oct 2023 20:14:06 +1100 Subject: [PATCH 11/20] Support running build manually --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dd2c4cc..9e303a9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,6 +4,7 @@ on: - push - pull_request - workflow_call + - workflow_dispatch jobs: build: From 58b3436bd913dec4c427c9aff346f3ffe1e88bd1 Mon Sep 17 00:00:00 2001 From: Yiheng Date: Sun, 1 Oct 2023 20:18:22 +1100 Subject: [PATCH 12/20] Use nbgv 0.4.1 --- .github/workflows/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9e303a9..194c657 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,10 +1,10 @@ name: .NET Build & Test on: - - push - - pull_request - - workflow_call - - workflow_dispatch + push: + pull_request: + workflow_call: + workflow_dispatch: jobs: build: @@ -21,7 +21,7 @@ jobs: with: dotnet-version: 6.0.x - - uses: dotnet/nbgv@main + - uses: dotnet/nbgv@v0.4.1 id: nbgv - name: Restore dependencies From 61239059ccd376885df19e98f1e39405971784f4 Mon Sep 17 00:00:00 2001 From: Yiheng Date: Mon, 2 Oct 2023 10:50:29 +1100 Subject: [PATCH 13/20] Allow testing in parallel --- .../ReplayParsingExecutionTests.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Nodsoft.WowsReplaysUnpack.Tests/ReplayParsingExecutionTests.cs b/Nodsoft.WowsReplaysUnpack.Tests/ReplayParsingExecutionTests.cs index 36c6625..0cda896 100644 --- a/Nodsoft.WowsReplaysUnpack.Tests/ReplayParsingExecutionTests.cs +++ b/Nodsoft.WowsReplaysUnpack.Tests/ReplayParsingExecutionTests.cs @@ -5,10 +5,6 @@ using Xunit; -/* - * FIXME: Test parallelization is disabled due to a file loading issue. - */ -[assembly: CollectionBehavior(DisableTestParallelization = true)] namespace Nodsoft.WowsReplaysUnpack.Tests; /// From 689a09cdd2a6a454ba4c1920377b492c54ca71ba Mon Sep 17 00:00:00 2001 From: Yiheng Date: Mon, 2 Oct 2023 10:50:46 +1100 Subject: [PATCH 14/20] Debug multiple tasks in Program --- Nodsoft.WowsReplaysUnpack.Console/Program.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Nodsoft.WowsReplaysUnpack.Console/Program.cs b/Nodsoft.WowsReplaysUnpack.Console/Program.cs index e6eeb1c..ef00174 100644 --- a/Nodsoft.WowsReplaysUnpack.Console/Program.cs +++ b/Nodsoft.WowsReplaysUnpack.Console/Program.cs @@ -13,6 +13,7 @@ using System.IO; using System.Linq; using System.Runtime.Serialization; +using System.Threading.Tasks; string samplePath = Path.Join(Directory.GetCurrentDirectory(), "../../../..", "Replay-Samples"); FileStream _GetReplayFile(string name) => File.OpenRead(Path.Join(samplePath, name)); @@ -49,6 +50,12 @@ // Console.WriteLine($"[{GetGroupString(msg)}] {msg.EntityId} : {msg.MessageContent}"); //} +Task[] tasks = { }; +for (int i = 0; i < 10; i++) +{ + tasks.Append(Task.Run(() => replayUnpacker.GetUnpacker().Unpack(_GetReplayFile("good.wowsreplay")))); +} +await Task.WhenAll(tasks); var goodReplay = replayUnpacker.GetUnpacker().Unpack(_GetReplayFile("good.wowsreplay")); var alphaReplay = replayUnpacker.GetUnpacker().Unpack(_GetReplayFile("press_account_alpha.wowsreplay")); From 0d1f080ab26f16e76b41973769f4aae52329c573 Mon Sep 17 00:00:00 2001 From: Yiheng Date: Mon, 2 Oct 2023 10:51:08 +1100 Subject: [PATCH 15/20] Fixed issue with mutlithreading --- Nodsoft.WowsReplaysUnpack/Services/ReplayUnpackerService.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Nodsoft.WowsReplaysUnpack/Services/ReplayUnpackerService.cs b/Nodsoft.WowsReplaysUnpack/Services/ReplayUnpackerService.cs index 7e2f73a..d8f5ba6 100644 --- a/Nodsoft.WowsReplaysUnpack/Services/ReplayUnpackerService.cs +++ b/Nodsoft.WowsReplaysUnpack/Services/ReplayUnpackerService.cs @@ -8,6 +8,7 @@ using System.IO.Compression; using System.Text; using System.Text.Json; +using System.Threading; namespace Nodsoft.WowsReplaysUnpack.Services; @@ -21,6 +22,7 @@ public sealed class ReplayUnpackerService : ReplayUnpackerService, private readonly JsonSerializerOptions _jsonSerializerOptions = new() { PropertyNameCaseInsensitive = true }; private readonly IReplayDataParser _replayDataParser; private readonly IReplayController _replayController; + private static readonly SemaphoreSlim _semaphore = new SemaphoreSlim(1); public ReplayUnpackerService(IReplayDataParser replayDataParser, TController replayController) { @@ -69,7 +71,7 @@ Seek to offset 4 in the replay file (skipping the magic number) See http://wiki.vbaddict.net/pages/File_Replays for more details. */ options ??= new(); - + _semaphore.Wait(); BinaryReader binaryReader = new(stream); byte[] signature = binaryReader.ReadBytes(4); @@ -103,7 +105,7 @@ Seek to offset 4 in the replay file (skipping the magic number) { _replayController.HandleNetworkPacket(networkPacket, options); } - + _semaphore.Release(); return replay; } From 7c8bfef58111acb6cce7077ff1587b20cd0f8793 Mon Sep 17 00:00:00 2001 From: Yiheng Date: Mon, 2 Oct 2023 11:27:24 +1100 Subject: [PATCH 16/20] Added simple benchmark in Program --- Nodsoft.WowsReplaysUnpack.Console/Program.cs | 26 ++++++++++++++++--- .../Services/ReplayUnpackerService.cs | 4 +++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/Nodsoft.WowsReplaysUnpack.Console/Program.cs b/Nodsoft.WowsReplaysUnpack.Console/Program.cs index ef00174..d3dd6fd 100644 --- a/Nodsoft.WowsReplaysUnpack.Console/Program.cs +++ b/Nodsoft.WowsReplaysUnpack.Console/Program.cs @@ -50,12 +50,30 @@ // Console.WriteLine($"[{GetGroupString(msg)}] {msg.EntityId} : {msg.MessageContent}"); //} -Task[] tasks = { }; -for (int i = 0; i < 10; i++) +const int CYCLE = 20; +async Task syncTasks(bool sync) { - tasks.Append(Task.Run(() => replayUnpacker.GetUnpacker().Unpack(_GetReplayFile("good.wowsreplay")))); + List unpackedReplays = new List(); + if (sync) + { + for (int i = 0; i < CYCLE; i++) + { + replayUnpacker.GetUnpacker().Unpack(_GetReplayFile("good.wowsreplay")); + } + } + else + { + Parallel.ForEach(Enumerable.Range(0, CYCLE), (i) => + { + unpackedReplays.Add(replayUnpacker.GetUnpacker().Unpack(_GetReplayFile("good.wowsreplay"))); + }); + } + return unpackedReplays.ToArray(); } -await Task.WhenAll(tasks); + +DateTime start = DateTime.Now; +await syncTasks(false); +Console.WriteLine(DateTime.Now - start); var goodReplay = replayUnpacker.GetUnpacker().Unpack(_GetReplayFile("good.wowsreplay")); var alphaReplay = replayUnpacker.GetUnpacker().Unpack(_GetReplayFile("press_account_alpha.wowsreplay")); diff --git a/Nodsoft.WowsReplaysUnpack/Services/ReplayUnpackerService.cs b/Nodsoft.WowsReplaysUnpack/Services/ReplayUnpackerService.cs index d8f5ba6..da38623 100644 --- a/Nodsoft.WowsReplaysUnpack/Services/ReplayUnpackerService.cs +++ b/Nodsoft.WowsReplaysUnpack/Services/ReplayUnpackerService.cs @@ -77,6 +77,7 @@ Seek to offset 4 in the replay file (skipping the magic number) byte[] signature = binaryReader.ReadBytes(4); int jsonBlockCount = binaryReader.ReadInt32(); + _semaphore.Release(); // Verify replay signature if (!signature.SequenceEqual(ReplaySignature)) { @@ -86,11 +87,13 @@ Seek to offset 4 in the replay file (skipping the magic number) // The first block is the arena info // Read it and create the unpacked replay model UnpackedReplay replay = _replayController.CreateUnpackedReplay(ReadJsonBlock(binaryReader)); + _semaphore.Wait(); ReadExtraJsonBlocks(replay, binaryReader, jsonBlockCount); MemoryStream decryptedStream = new(); Decrypt(binaryReader, decryptedStream); + _semaphore.Release(); // Initial stream and reader not used anymore binaryReader.Dispose(); @@ -101,6 +104,7 @@ Seek to offset 4 in the replay file (skipping the magic number) decryptedStream.Dispose(); + _semaphore.Wait(); foreach (NetworkPacketBase networkPacket in _replayDataParser.ParseNetworkPackets(replayDataStream, options)) { _replayController.HandleNetworkPacket(networkPacket, options); From 8b6672bead730740c19ccb38eb4ec9cc15988220 Mon Sep 17 00:00:00 2001 From: Yiheng Date: Sat, 14 Oct 2023 00:16:13 +1100 Subject: [PATCH 17/20] Fix missing consts --- Nodsoft.WowsReplaysUnpack.Core/Consts.cs | 4 ++++ Nodsoft.WowsReplaysUnpack/Services/ReplayUnpackerService.cs | 5 +---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Nodsoft.WowsReplaysUnpack.Core/Consts.cs b/Nodsoft.WowsReplaysUnpack.Core/Consts.cs index 4ecbe1c..9a3c669 100644 --- a/Nodsoft.WowsReplaysUnpack.Core/Consts.cs +++ b/Nodsoft.WowsReplaysUnpack.Core/Consts.cs @@ -3,4 +3,8 @@ public static class Consts { public const int Infinity = 0xFFFF; + + public const char Underscore = '_'; + public const char Dot = '.'; + public const char Comma = ','; } \ No newline at end of file diff --git a/Nodsoft.WowsReplaysUnpack/Services/ReplayUnpackerService.cs b/Nodsoft.WowsReplaysUnpack/Services/ReplayUnpackerService.cs index 0ccadf7..18cdd3d 100644 --- a/Nodsoft.WowsReplaysUnpack/Services/ReplayUnpackerService.cs +++ b/Nodsoft.WowsReplaysUnpack/Services/ReplayUnpackerService.cs @@ -105,11 +105,8 @@ Seek to offset 4 in the replay file (skipping the magic number) // Decrypted stream not used anymore decryptedStream.Dispose(); - - _semaphore.Wait(); Version gameclientVersion = Version.Parse(arenaInfo.ClientVersionFromExe.Replace(',', '.')); - //foreach (NetworkPacketBase networkPacket in _replayDataParser.ParseNetworkPackets(replayDataStream, options)) - + _semaphore.Wait(); foreach (NetworkPacketBase networkPacket in _replayDataParser.ParseNetworkPackets(replayDataStream, options, gameclientVersion)) { _replayController.HandleNetworkPacket(networkPacket, options); From 542283f0d8e8b46fb0d09879068093a1d9b235be Mon Sep 17 00:00:00 2001 From: Yiheng Date: Sat, 14 Oct 2023 00:24:22 +1100 Subject: [PATCH 18/20] Update the samplePath --- Nodsoft.WowsReplaysUnpack.Console/Program.cs | 2 +- .../Extensions/StringExtensions.cs | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Nodsoft.WowsReplaysUnpack.Console/Program.cs b/Nodsoft.WowsReplaysUnpack.Console/Program.cs index d3dd6fd..b191b52 100644 --- a/Nodsoft.WowsReplaysUnpack.Console/Program.cs +++ b/Nodsoft.WowsReplaysUnpack.Console/Program.cs @@ -15,7 +15,7 @@ using System.Runtime.Serialization; using System.Threading.Tasks; -string samplePath = Path.Join(Directory.GetCurrentDirectory(), "../../../..", "Replay-Samples"); +string samplePath = Path.Join(Directory.GetCurrentDirectory(), "../../../../Nodsoft.WowsReplaysUnpack.Tests", "Replay-Samples"); FileStream _GetReplayFile(string name) => File.OpenRead(Path.Join(samplePath, name)); ServiceProvider? services = new ServiceCollection() diff --git a/Nodsoft.WowsReplaysUnpack.Core/Extensions/StringExtensions.cs b/Nodsoft.WowsReplaysUnpack.Core/Extensions/StringExtensions.cs index 79efe1c..79a1391 100644 --- a/Nodsoft.WowsReplaysUnpack.Core/Extensions/StringExtensions.cs +++ b/Nodsoft.WowsReplaysUnpack.Core/Extensions/StringExtensions.cs @@ -1,4 +1,11 @@ namespace Nodsoft.WowsReplaysUnpack.Core.Extensions; public static class StringExtensions -{ } \ No newline at end of file +{ + public static string GetStringBeforeIndex(this string str, char before) => str[..str.IndexOf(before)]; + + public static string GetStringBeforeIndex(this string str, string before) => str[..str.IndexOf(before, StringComparison.Ordinal)]; + + public static string GetStringAfterLength(this string str, string after) => str[after.Length..]; +} + \ No newline at end of file From 7d856bb3d18bd30148ecd675ec4c0d352e2bcebb Mon Sep 17 00:00:00 2001 From: Yiheng Date: Sat, 14 Oct 2023 00:41:32 +1100 Subject: [PATCH 19/20] Update sameFolder in tests --- Nodsoft.WowsReplaysUnpack.Tests/Utilities.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Nodsoft.WowsReplaysUnpack.Tests/Utilities.cs b/Nodsoft.WowsReplaysUnpack.Tests/Utilities.cs index 22695a1..6e58243 100644 --- a/Nodsoft.WowsReplaysUnpack.Tests/Utilities.cs +++ b/Nodsoft.WowsReplaysUnpack.Tests/Utilities.cs @@ -7,7 +7,7 @@ namespace Nodsoft.WowsReplaysUnpack.Tests; /// public static class Utilities { - private static readonly string _sampleFolder = Path.Join(Directory.GetCurrentDirectory(), "..", "..", "..", "..", "Replay-Samples"); + private static readonly string _sampleFolder = Path.Join(Directory.GetCurrentDirectory(), "Replay-Samples"); [MethodImpl(MethodImplOptions.AggressiveInlining)] public static MemoryStream LoadReplay(string replayPath) From 8aacd2c6778eed811666017b695eee253ac2077b Mon Sep 17 00:00:00 2001 From: Yiheng Date: Mon, 16 Oct 2023 22:02:09 +1100 Subject: [PATCH 20/20] Add a timeout to prevent dead locks --- .../Services/ReplayUnpackerService.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Nodsoft.WowsReplaysUnpack/Services/ReplayUnpackerService.cs b/Nodsoft.WowsReplaysUnpack/Services/ReplayUnpackerService.cs index 18cdd3d..f92b9b4 100644 --- a/Nodsoft.WowsReplaysUnpack/Services/ReplayUnpackerService.cs +++ b/Nodsoft.WowsReplaysUnpack/Services/ReplayUnpackerService.cs @@ -24,6 +24,7 @@ public sealed class ReplayUnpackerService : ReplayUnpackerService, private readonly IReplayDataParser _replayDataParser; private readonly IReplayController _replayController; private static readonly SemaphoreSlim _semaphore = new SemaphoreSlim(1); + private const int _semephoreTimeOut = 2000; public ReplayUnpackerService(IReplayDataParser replayDataParser, TController replayController) { @@ -72,7 +73,7 @@ Seek to offset 4 in the replay file (skipping the magic number) See http://wiki.vbaddict.net/pages/File_Replays for more details. */ options ??= new(); - _semaphore.Wait(); + _semaphore.Wait(_semephoreTimeOut); BinaryReader binaryReader = new(stream); byte[] signature = binaryReader.ReadBytes(4); @@ -89,7 +90,7 @@ Seek to offset 4 in the replay file (skipping the magic number) // Read it and create the unpacked replay model ArenaInfo arenaInfo = ReadJsonBlock(binaryReader); UnpackedReplay replay = _replayController.CreateUnpackedReplay(arenaInfo); - _semaphore.Wait(); + _semaphore.Wait(_semephoreTimeOut); ReadExtraJsonBlocks(replay, binaryReader, jsonBlockCount); MemoryStream decryptedStream = new(); @@ -106,7 +107,7 @@ Seek to offset 4 in the replay file (skipping the magic number) decryptedStream.Dispose(); Version gameclientVersion = Version.Parse(arenaInfo.ClientVersionFromExe.Replace(',', '.')); - _semaphore.Wait(); + _semaphore.Wait(_semephoreTimeOut); foreach (NetworkPacketBase networkPacket in _replayDataParser.ParseNetworkPackets(replayDataStream, options, gameclientVersion)) { _replayController.HandleNetworkPacket(networkPacket, options);