From a189836006b8696377bc758fca80c6021f6db7c2 Mon Sep 17 00:00:00 2001 From: khhs167 Date: Mon, 19 Jun 2023 14:53:06 +0200 Subject: [PATCH 1/2] Added CSV stuff --- ContentPipe/Content.cs | 71 +++++++++++++++++++++++++++++- ContentPipe/ContentDirectory.cs | 76 +++++++++++++-------------------- ContentPipe/IContentProvider.cs | 58 +++++++++++++++++++++++-- 3 files changed, 154 insertions(+), 51 deletions(-) diff --git a/ContentPipe/Content.cs b/ContentPipe/Content.cs index cbd1264..f84a530 100644 --- a/ContentPipe/Content.cs +++ b/ContentPipe/Content.cs @@ -1,4 +1,5 @@ using System.Text; +using System.Text.RegularExpressions; namespace ContentPipe; @@ -8,6 +9,19 @@ namespace ContentPipe; public static class Content { private static readonly Dictionary Providers = new Dictionary(); + + private static readonly Dictionary LoadedContent = new Dictionary(); + + /// + /// If Loads should be logged. This will slow down performance of loading by some amount! + /// + public static bool ShouldLogLoads = false; + + /// + /// A filter to run on all log load registrations, in case you want to ignore something. + /// This only affects the output log! + /// + public static Regex LogLoadIgnoreFilter = new Regex(""); /// /// Load a packed content directory(.cpkg file) @@ -20,6 +34,20 @@ public static void LoadDirectory(string path) Providers.Add(path, new PacketContentProvider(new ContentDirectory(path))); } + /// + /// Load a packed ContentDirectory where all content has a prefix. + /// + /// The path to the cpkg directory, without extension + /// The prefix to be used for said directory + /// The string to be used when unloading, as prefixing slightly modifies the string + public static string LoadPrefixed(string path, string prefix) + { + string pfxPath = prefix + path; + if(!Providers.ContainsKey(pfxPath)) + Providers.Add(pfxPath, new PrefixedContentProvider(prefix, new PacketContentProvider(new ContentDirectory(path)))); + return pfxPath; + } + /// /// Unload a packed content directory /// @@ -52,6 +80,14 @@ public static void LoadPhysicalDirectory(string path) Providers.Add(path, new PhysicalContentProvider(path)); } + private static void RegisterLoad(string resource) + { + if(!ShouldLogLoads) + return; + LoadedContent.TryAdd(resource, 0); + LoadedContent[resource]++; + } + /// /// Load/Fetch a content lump from loaded directories. Newer loaded directories are fetched from before other directories /// @@ -59,6 +95,7 @@ public static void LoadPhysicalDirectory(string path) /// The content lump loaded, or null if it is not available public static ContentLump? Load(string resource) { + RegisterLoad(resource); foreach (var provider in Providers.Values) { ContentLump? lump = provider.Load(resource); @@ -68,7 +105,7 @@ public static void LoadPhysicalDirectory(string path) } return null; } - + /// /// Load/Fetch a content lump from all loaded directories. Newer loaded directories are fetched from before other directories /// @@ -76,6 +113,7 @@ public static void LoadPhysicalDirectory(string path) /// The content lumps loaded public static ContentLump[] LoadAll(string resource) { + RegisterLoad(resource); List contentLumps = new List(); foreach (var provider in Providers.Values.Reverse()) { @@ -178,4 +216,35 @@ public static string[] LoadAllStrings(string resource) return strings.ToArray(); } + + /// + /// Get the load log if one is collected, formatted as CSV + /// + /// The load log + public static string WriteLoadLog(bool includeDeadResources = false) + { + StringWriter stringWriter = new StringWriter(); + stringWriter.WriteLine("File,Loads"); + + if (includeDeadResources) + { + foreach (var provider in Providers) + { + string[] resources = provider.Value.GetContent(); + foreach (var res in resources) + { + if(!String.IsNullOrWhiteSpace(res)) + LoadedContent.TryAdd(res, 0); + } + } + } + var lc = LoadedContent.OrderByDescending(l => l.Value); + foreach (var load in lc) + { + if(!LogLoadIgnoreFilter.IsMatch(load.Key)) + stringWriter.WriteLine(load.Key + "," + load.Value); + } + + return stringWriter.ToString(); + } } diff --git a/ContentPipe/ContentDirectory.cs b/ContentPipe/ContentDirectory.cs index 77e0537..5a01a67 100644 --- a/ContentPipe/ContentDirectory.cs +++ b/ContentPipe/ContentDirectory.cs @@ -8,8 +8,9 @@ namespace ContentPipe; /// public struct ContentDirectory { - private Dictionary contentLumps = new Dictionary(); + private ZipArchive archive; + /// /// Fetch a content lump, returns null if it is not available /// @@ -18,11 +19,24 @@ public ContentLump? this[string name] { get { - if (contentLumps.ContainsKey(name)) - return contentLumps[name]; - return null; + ZipArchiveEntry? entry = archive.GetEntry(name); + if (entry == null) + return null; + + string tempFile = Directory.GetCurrentDirectory() + "/temp-read"; + + entry.ExtractToFile(tempFile); + ContentLump contentLump = new() + { + Name = name, + Data = File.ReadAllBytes(tempFile) + }; + File.Delete(tempFile); + + return contentLump; } } + /// /// Compress a directory into a .cpkg file. It will be stored in the same directory as the source directory, @@ -32,32 +46,9 @@ public ContentLump? this[string name] /// The path to the directory to compress public static void CompressDirectory(string path) { - - if (!Directory.Exists(path)) - return; - - string[] files = Directory.GetFiles(path, "*", SearchOption.AllDirectories); - string filePath = Path.Combine(Path.GetDirectoryName(path)!, Path.GetFileName(path)) + ".cpkg"; - - List data = new List(); - - foreach (var file in files) - { - string localPath = Path.GetRelativePath(path, file); - ContentLump lump = new ContentLump(); - lump.Name = localPath; - lump.Data = File.ReadAllBytes(file); - data.Add(lump); - } - - byte[] compressedData = MessagePackSerializer.Serialize(data.ToArray()); - - FileStream fileStream = File.Open(filePath, FileMode.OpenOrCreate); - DeflateStream gZipStream = new DeflateStream(fileStream, CompressionLevel.SmallestSize, false); - gZipStream.Write(compressedData); - gZipStream.Close(); - fileStream.Close(); - + if(File.Exists(path + ".cpkg")) + File.Delete(path + ".cpkg"); + ZipFile.CreateFromDirectory(path, path + ".cpkg"); } /// @@ -66,23 +57,16 @@ public static void CompressDirectory(string path) /// The path to the file, excluding the extension public ContentDirectory(string path) { - - FileStream fileStream = File.Open(path + ".cpkg", FileMode.Open); - DeflateStream gZipStream = new DeflateStream(fileStream, CompressionMode.Decompress, false); - - MemoryStream memoryStream = new MemoryStream(); - gZipStream.CopyTo(memoryStream); - gZipStream.Close(); - fileStream.Close(); - - ContentLump[] data = MessagePackSerializer.Deserialize(memoryStream.ToArray()); - - foreach (var lump in data) + string fileName = path + ".cpkg"; + archive = ZipFile.OpenRead(fileName); + Content = new string[archive.Entries.Count]; + for (int i = 0; i < archive.Entries.Count; i++) { - contentLumps.Add(lump.Name, lump); + Content[i] = archive.Entries[i].FullName; } - } + + public readonly string[] Content; } /// @@ -94,12 +78,12 @@ public struct ContentLump /// /// The name of the content lump /// - [Key(0)] public string Name; + public string Name; /// /// The data of the ContentLump /// - [Key(1)] public byte[] Data = Array.Empty(); + public byte[] Data = Array.Empty(); /// /// Create a content lump with 0:ed fields. diff --git a/ContentPipe/IContentProvider.cs b/ContentPipe/IContentProvider.cs index 10326ca..018fc04 100644 --- a/ContentPipe/IContentProvider.cs +++ b/ContentPipe/IContentProvider.cs @@ -3,11 +3,44 @@ namespace ContentPipe; internal interface IContentProvider { public ContentLump? Load(string name); + public string[] GetContent(); } -internal struct PacketContentProvider : IContentProvider +internal readonly struct PrefixedContentProvider : IContentProvider { - private ContentDirectory directory; + private readonly IContentProvider provider; + private readonly string prefix; + + public PrefixedContentProvider(string prefix, IContentProvider provider) + { + this.prefix = prefix; + this.provider = provider; + } + + public ContentLump? Load(string name) + { + if (!name.StartsWith(prefix)) + return null; + + string deAliased = name[prefix.Length..]; + return provider.Load(deAliased); + } + + public string[] GetContent() + { + string[] content = provider.GetContent(); + for (int i = 0; i < content.Length; i++) + { + content[i] = prefix + content[i]; + } + return content; + } + +} + +internal readonly struct PacketContentProvider : IContentProvider +{ + private readonly ContentDirectory directory; public PacketContentProvider(ContentDirectory directory) { this.directory = directory; @@ -17,11 +50,16 @@ public PacketContentProvider(ContentDirectory directory) { return directory[name]; } + + public string[] GetContent() + { + return directory.Content; + } } -internal struct PhysicalContentProvider : IContentProvider +internal readonly struct PhysicalContentProvider : IContentProvider { - private string directory; + private readonly string directory; public PhysicalContentProvider(string directory) { this.directory = directory; @@ -39,4 +77,16 @@ public PhysicalContentProvider(string directory) } return null; } + + public string[] GetContent() + { + string[] files = Directory.GetFiles(directory, "*", SearchOption.AllDirectories); + + for (int i = 0; i < files.Length; i++) + { + files[i] = Path.GetRelativePath(directory, files[i]); + } + + return files; + } } \ No newline at end of file From ec25440e70a96da9b571c9a80c7968a8cf2c9eab Mon Sep 17 00:00:00 2001 From: khhs167 Date: Fri, 30 Jun 2023 16:29:48 +0200 Subject: [PATCH 2/2] Minor logging changes afaik --- ContentPipe/Extras.cs | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 ContentPipe/Extras.cs diff --git a/ContentPipe/Extras.cs b/ContentPipe/Extras.cs new file mode 100644 index 0000000..67e68f9 --- /dev/null +++ b/ContentPipe/Extras.cs @@ -0,0 +1,31 @@ +using System.Security.Cryptography; +using System.Text; + +namespace ContentPipe; + +internal static class Extras +{ + public static string ReadTerminatedString(this BinaryReader reader) + { + string s = ""; + char c; + while ((c = reader.ReadChar()) != '\0') + s += c; + return s; + } + + public static byte[] GetHash(this string inputString) + { + using (HashAlgorithm algorithm = SHA256.Create()) + return algorithm.ComputeHash(Encoding.UTF8.GetBytes(inputString)); + } + + public static string GetHashString(this string inputString) + { + StringBuilder sb = new StringBuilder(); + foreach (byte b in GetHash(inputString)) + sb.Append(b.ToString("X2")); + + return sb.ToString(); + } +}