chore: Speed up and cleanup level and zone loading; Add safer asset b…

…uffer reading (#1314)

* Remove std::couts littered throughout the base

* working

End of optimizations for now

going faster

* Remove extraneous compare function

std::less<LWOSCENEID> already does this in a map.

* gaming

* Update Zone.cpp

* dlu is moving to bitbucket again

* Update Level.cpp

---------

Co-authored-by: Jettford <[email protected]>

dCommon/BinaryIO.cpp

@@ -1,14 +1,6 @@
 #include "BinaryIO.h"
 #include <string>
 
-void BinaryIO::WriteString(const std::string& stringToWrite, std::ofstream& outstream) {
-	//BinaryWrite(outstream, uint32_t(stringToWrite.length()));
-
-	for (size_t i = 0; i < size_t(stringToWrite.length()); ++i) {
-		BinaryIO::BinaryWrite(outstream, stringToWrite[i]);
-	}
-}
-
 //For reading null-terminated strings
 std::string BinaryIO::ReadString(std::istream& instream) {
 	std::string toReturn;
@@ -23,36 +15,3 @@ std::string BinaryIO::ReadString(std::istream& instream) {
 
 	return toReturn;
 }
-
-//For reading strings of a specific size
-std::string BinaryIO::ReadString(std::istream& instream, size_t size) {
-	std::string toReturn;
-	char buffer;
-
-	for (size_t i = 0; i < size; ++i) {
-		BinaryIO::BinaryRead(instream, buffer);
-		toReturn += buffer;
-	}
-
-	return toReturn;
-}
-
-std::string BinaryIO::ReadWString(std::istream& instream) {
-	size_t size;
-	BinaryRead(instream, size);
-	//toReturn.resize(size);
-	std::string test;
-	unsigned char buf;
-
-	for (size_t i = 0; i < size; ++i) {
-		//instream.ignore(1);
-		BinaryRead(instream, buf);
-		test += buf;
-	}
-
-	//printf("%s\n", test.c_str());
-
-	//instream.read((char*)&toReturn[0], size * 2);
-	//std::string str(toReturn.begin(), toReturn.end());
-	return test;
-}

dCommon/BinaryIO.h

@@ -1,8 +1,17 @@
 #pragma once
+
+#ifndef __BINARYIO__H__
+#define __BINARYIO__H__
+
 #include <iostream>
 #include <fstream>
+#include <string>
+
+#include "Game.h"
+#include "Logger.h"
 
 namespace BinaryIO {
+
 	template<typename T>
 	std::ostream& BinaryWrite(std::ostream& stream, const T& value) {
 		return stream.write(reinterpret_cast<const char*>(&value), sizeof(T));
@@ -15,13 +24,51 @@ namespace BinaryIO {
 		return stream.read(reinterpret_cast<char*>(&value), sizeof(T));
 	}
 
-	void WriteString(const std::string& stringToWrite, std::ofstream& outstream);
+	enum class ReadType : int8_t {
+		WideString = 0,
+		String = 1,
+	};
+
+	template<typename SizeType>
+	inline void ReadString(std::istream& stream, std::u16string& value) {
+		static_assert(std::is_integral<SizeType>::value, "SizeType must be an integral type.");
+
+		SizeType size;
+		BinaryRead(stream, size);
+
+		if (!stream.good()) throw std::runtime_error("Failed to read from istream.");
+		value.resize(size);
+		stream.read(reinterpret_cast<char*>(value.data()), size * sizeof(uint16_t));
+	}
+
+	template<typename SizeType>
+	inline void ReadString(std::istream& stream, std::string& value, ReadType readType) {
+		static_assert(std::is_integral<SizeType>::value, "SizeType must be an integral type.");
+
+		SizeType size;
+		BinaryRead(stream, size);
+
+		if (!stream.good()) throw std::runtime_error("Failed to read from istream.");
+		value.resize(size);
+		if (readType == ReadType::WideString) {
+			uint16_t wideChar;
+
+			// Faster to do this than to read a u16string and convert it to a string since we only go through allocator once
+			for (SizeType i = 0; i < size; ++i) {
+				BinaryRead(stream, wideChar);
+				value[i] = static_cast<char>(wideChar);
+			}
+		} else {
+			stream.read(value.data(), size);
+		}
+	}
+
 	std::string ReadString(std::istream& instream);
-	std::string ReadString(std::istream& instream, size_t size);
-	std::string ReadWString(std::istream& instream);
 
 	inline bool DoesFileExist(const std::string& name) {
 		std::ifstream f(name.c_str());
 		return f.good();
 	}
 }
+
+#endif  //!__BINARYIO__H__

dCommon/FdbToSqlite.cpp

@@ -28,19 +28,17 @@ FdbToSqlite::Convert::Convert(std::string binaryOutPath) {
 	this->m_BinaryOutPath = binaryOutPath;
 }
 
-bool FdbToSqlite::Convert::ConvertDatabase(AssetMemoryBuffer& buffer) {
+bool FdbToSqlite::Convert::ConvertDatabase(AssetStream& buffer) {
 	if (m_ConversionStarted) return false;
 
-	std::istream cdClientBuffer(&buffer);
-
 	this->m_ConversionStarted = true;
 	try {
 		CDClientDatabase::Connect(m_BinaryOutPath + "/CDServer.sqlite");
 
 		CDClientDatabase::ExecuteQuery("BEGIN TRANSACTION;");
 
-		int32_t numberOfTables = ReadInt32(cdClientBuffer);
-		ReadTables(numberOfTables, cdClientBuffer);
+		int32_t numberOfTables = ReadInt32(buffer);
+		ReadTables(numberOfTables, buffer);
 
 		CDClientDatabase::ExecuteQuery("COMMIT;");
 	} catch (CppSQLite3Exception& e) {

dCommon/FdbToSqlite.h

@@ -7,7 +7,7 @@
 #include <iosfwd>
 #include <map>
 
-class AssetMemoryBuffer;
+#include "AssetManager.h"
 
 enum class eSqliteDataType : int32_t;
 
@@ -27,7 +27,7 @@ namespace FdbToSqlite {
 		 * 
 		 * @return true if the database was converted properly, false otherwise. 
 		 */
-		bool ConvertDatabase(AssetMemoryBuffer& buffer);
+		bool ConvertDatabase(AssetStream& buffer);
 
 		/**
 		 * @brief Reads a 32 bit int from the fdb file.

dCommon/dClient/AssetManager.cpp

@@ -152,13 +152,12 @@ bool AssetManager::GetFile(const char* name, char** data, uint32_t* len) {
 	return success;
 }
 
-AssetMemoryBuffer AssetManager::GetFileAsBuffer(const char* name) {
-	char* buf;
-	uint32_t len;
+AssetStream AssetManager::GetFile(const char* name) {
+	char* buf; uint32_t len;
 
 	bool success = this->GetFile(name, &buf, &len);
 
-	return AssetMemoryBuffer(buf, len, success);
+	return AssetStream(buf, len, success);
 }
 
 uint32_t AssetManager::crc32b(uint32_t base, uint8_t* message, size_t l) {

dCommon/dClient/AssetManager.h

@@ -25,6 +25,10 @@ struct AssetMemoryBuffer : std::streambuf {
 		this->setg(base, base, base + n);
 	}
 
+	~AssetMemoryBuffer() {
+		free(m_Base);
+	}
+
 	pos_type seekpos(pos_type sp, std::ios_base::openmode which) override {
 		return seekoff(sp - pos_type(off_type(0)), std::ios_base::beg, which);
 	}
@@ -40,9 +44,17 @@ struct AssetMemoryBuffer : std::streambuf {
 			setg(eback(), eback() + off, egptr());
 		return gptr() - eback();
 	}
+};
+
+struct AssetStream : std::istream {
+	AssetStream(char* base, std::ptrdiff_t n, bool success) : std::istream(new AssetMemoryBuffer(base, n, success)) {}
+
+	~AssetStream() {
+		delete rdbuf();
+	}
 
-	void close() {
-		free(m_Base);
+	operator bool() {
+		return reinterpret_cast<AssetMemoryBuffer*>(rdbuf())->m_Success;
 	}
 };
 
@@ -56,7 +68,7 @@ class AssetManager {
 
 	bool HasFile(const char* name);
 	bool GetFile(const char* name, char** data, uint32_t* len);
-	AssetMemoryBuffer GetFileAsBuffer(const char* name);
+	AssetStream GetFile(const char* name);
 
 private:
 	void LoadPackIndex();

dCommon/dClient/PackIndex.cpp

@@ -8,21 +8,10 @@ PackIndex::PackIndex(const std::filesystem::path& filePath) {
 
 	BinaryIO::BinaryRead<uint32_t>(m_FileStream, m_Version);
 	BinaryIO::BinaryRead<uint32_t>(m_FileStream, m_PackPathCount);
-
-	for (int i = 0; i < m_PackPathCount; i++) {
-		uint32_t stringLen = 0;
-		BinaryIO::BinaryRead<uint32_t>(m_FileStream, stringLen);
-
-		std::string path;
-
-		for (int j = 0; j < stringLen; j++) {
-			char inChar;
-			BinaryIO::BinaryRead<char>(m_FileStream, inChar);
-
-			path += inChar;
-		}
-
-		m_PackPaths.push_back(path);
+
+	m_PackPaths.resize(m_PackPathCount);
+	for (auto& item : m_PackPaths) {
+		BinaryIO::ReadString<uint32_t>(m_FileStream, item, BinaryIO::ReadType::String);
 	}
 
 	BinaryIO::BinaryRead<uint32_t>(m_FileStream, m_PackFileIndexCount);

dGame/UserManager.cpp

@@ -44,57 +44,53 @@ inline void StripCR(std::string& str) {
 void UserManager::Initialize() {
 	std::string line;
 
-	AssetMemoryBuffer fnBuff = Game::assetManager->GetFileAsBuffer("names/minifigname_first.txt");
-	if (!fnBuff.m_Success) {
+	auto fnStream = Game::assetManager->GetFile("names/minifigname_first.txt");
+	if (!fnStream) {
 		LOG("Failed to load %s", (Game::assetManager->GetResPath() / "names/minifigname_first.txt").string().c_str());
 		throw std::runtime_error("Aborting initialization due to missing minifigure name file.");
 	}
-	std::istream fnStream = std::istream(&fnBuff);
+
 	while (std::getline(fnStream, line, '\n')) {
 		std::string name = line;
 		StripCR(name);
 		m_FirstNames.push_back(name);
 	}
-	fnBuff.close();
 
-	AssetMemoryBuffer mnBuff = Game::assetManager->GetFileAsBuffer("names/minifigname_middle.txt");
-	if (!mnBuff.m_Success) {
+	auto mnStream = Game::assetManager->GetFile("names/minifigname_middle.txt");
+	if (!mnStream) {
 		LOG("Failed to load %s", (Game::assetManager->GetResPath() / "names/minifigname_middle.txt").string().c_str());
 		throw std::runtime_error("Aborting initialization due to missing minifigure name file.");
 	}
-	std::istream mnStream = std::istream(&mnBuff);
+
 	while (std::getline(mnStream, line, '\n')) {
 		std::string name = line;
 		StripCR(name);
 		m_MiddleNames.push_back(name);
 	}
-	mnBuff.close();
 
-	AssetMemoryBuffer lnBuff = Game::assetManager->GetFileAsBuffer("names/minifigname_last.txt");
-	if (!lnBuff.m_Success) {
+	auto lnStream = Game::assetManager->GetFile("names/minifigname_last.txt");
+	if (!lnStream) {
 		LOG("Failed to load %s", (Game::assetManager->GetResPath() / "names/minifigname_last.txt").string().c_str());
 		throw std::runtime_error("Aborting initialization due to missing minifigure name file.");
 	}
-	std::istream lnStream = std::istream(&lnBuff);
+
 	while (std::getline(lnStream, line, '\n')) {
 		std::string name = line;
 		StripCR(name);
 		m_LastNames.push_back(name);
 	}
-	lnBuff.close();
 
-	//Load our pre-approved names:
-	AssetMemoryBuffer chatListBuff = Game::assetManager->GetFileAsBuffer("chatplus_en_us.txt");
-	if (!chatListBuff.m_Success) {
+	// Load our pre-approved names:
+	auto chatListStream = Game::assetManager->GetFile("chatplus_en_us.txt");
+	if (!chatListStream) {
 		LOG("Failed to load %s", (Game::assetManager->GetResPath() / "chatplus_en_us.txt").string().c_str());
 		throw std::runtime_error("Aborting initialization due to missing chat whitelist file.");
 	}
-	std::istream chatListStream = std::istream(&chatListBuff);
+
 	while (std::getline(chatListStream, line, '\n')) {
 		StripCR(line);
 		m_PreapprovedNames.push_back(line);
 	}
-	chatListBuff.close();
 }
 
 UserManager::~UserManager() {

dGame/dInventory/Item.cpp

@@ -423,25 +423,16 @@ void Item::DisassembleModel(uint32_t numToDismantle) {
 	if (renderAssetSplit.empty()) return;
 
 	std::string lxfmlPath = "BrickModels" + lxfmlFolderName + "/" + GeneralUtils::SplitString(renderAssetSplit.back(), '.').at(0) + ".lxfml";
-	auto buffer = Game::assetManager->GetFileAsBuffer(lxfmlPath.c_str());
+	auto file = Game::assetManager->GetFile(lxfmlPath.c_str());
 
-	if (!buffer.m_Success) {
+	if (!file) {
 		LOG("Failed to load %s to disassemble model into bricks, check that this file exists", lxfmlPath.c_str());
 		return;
 	}
 
-	std::istream file(&buffer);
-
-	if (!file.good()) {
-		buffer.close();
-		return;
-	}
-
 	std::stringstream data;
 	data << file.rdbuf();
 
-	buffer.close();
-
 	uint32_t fileSize;
 	file.seekg(0, std::ios::end);
 	fileSize = static_cast<uint32_t>(file.tellg());

dGame/dPropertyBehaviors/ControlBehaviors.cpp

@@ -324,14 +324,9 @@ void ControlBehaviors::ProcessCommand(Entity* modelEntity, const SystemAddress&
 }
 
 ControlBehaviors::ControlBehaviors() {
-	auto blocksDefStreamBuffer = Game::assetManager->GetFileAsBuffer("ui\\ingame\\blocksdef.xml");
-	if (!blocksDefStreamBuffer.m_Success) {
-		LOG("failed to open blocksdef");
-		return;
-	}
-	std::istream blocksBuffer(&blocksDefStreamBuffer);
-	if (!blocksBuffer.good()) {
-		LOG("Blocks buffer is not good!");
+	auto blocksBuffer = Game::assetManager->GetFile("ui\\ingame\\blocksdef.xml");
+	if (!blocksBuffer) {
+		LOG("Failed to open blocksdef.xml");
 		return;
 	}
 
@@ -342,7 +337,7 @@ ControlBehaviors::ControlBehaviors() {
 	std::string buffer{};
 	bool commentBlockStart = false;
 	while (std::getline(blocksBuffer, read)) {
-		// tinyxml2 should handle comment blocks but the client has one that fails the processing.  
+		// tinyxml2 should handle comment blocks but the client has one that fails the processing.
 		// This preprocessing just removes all comments from the read file out of an abundance of caution.
 		if (read.find("<!--") != std::string::npos) {
 			commentBlockStart = true;