perf: Loot memory savings (#1165)

* Move away from constructor queries

Fix up other large tables to have proper backup lookups

Revert "idk im just dumb ig"

This reverts commit 5d5be5df53b8959b42b291613d7db749a65a3585.

idk im just dumb ig

* Fix slow components registry lookup

* add define for cdclient cache all

* Huge loot namespace rework

- Remove all excess memory usage
- do not cache components registry
- cache loot matrices on startup of the destroyable component
- convert loot singleton class to a namespace
- rework loot cdclient tables to operate closer to how someone would actually use them (basically doing the previous LootGenerator::LootGenerator caching but in those tables)
- Memory usage reduced by 10%+ across the board

* cache rebuild matrix

* Database: move reading to own function

Also change name of cache to PascalCase

* Database: Move common function rading

dDatabase/CDClientManager.cpp

@@ -55,7 +55,7 @@ CDClientManager::CDClientManager() {
 	CDBehaviorParameterTable::Instance().LoadValuesFromDatabase();
 	CDBehaviorTemplateTable::Instance().LoadValuesFromDatabase();
 	CDBrickIDTableTable::Instance().LoadValuesFromDatabase();
-	CDComponentsRegistryTable::Instance().LoadValuesFromDatabase();
+	CDCLIENT_DONT_CACHE_TABLE(CDComponentsRegistryTable::Instance().LoadValuesFromDatabase());
 	CDCurrencyTableTable::Instance().LoadValuesFromDatabase();
 	CDDestructibleComponentTable::Instance().LoadValuesFromDatabase();
 	CDEmoteTableTable::Instance().LoadValuesFromDatabase();
@@ -65,8 +65,8 @@ CDClientManager::CDClientManager() {
 	CDItemSetSkillsTable::Instance().LoadValuesFromDatabase();
 	CDItemSetsTable::Instance().LoadValuesFromDatabase();
 	CDLevelProgressionLookupTable::Instance().LoadValuesFromDatabase();
-	CDLootMatrixTable::Instance().LoadValuesFromDatabase();
-	CDLootTableTable::Instance().LoadValuesFromDatabase();
+	CDCLIENT_DONT_CACHE_TABLE(CDLootMatrixTable::Instance().LoadValuesFromDatabase());
+	CDCLIENT_DONT_CACHE_TABLE(CDLootTableTable::Instance().LoadValuesFromDatabase());
 	CDMissionEmailTable::Instance().LoadValuesFromDatabase();
 	CDMissionNPCComponentTable::Instance().LoadValuesFromDatabase();
 	CDMissionTasksTable::Instance().LoadValuesFromDatabase();

dDatabase/Tables/CDLootMatrixTable.cpp

@@ -1,5 +1,18 @@
 #include "CDLootMatrixTable.h"
 
+CDLootMatrix CDLootMatrixTable::ReadRow(CppSQLite3Query& tableData) const {
+	CDLootMatrix entry{};
+	if (tableData.eof()) return entry;
+	entry.LootTableIndex = tableData.getIntField("LootTableIndex", -1);
+	entry.RarityTableIndex = tableData.getIntField("RarityTableIndex", -1);
+	entry.percent = tableData.getFloatField("percent", -1.0f);
+	entry.minToDrop = tableData.getIntField("minToDrop", -1);
+	entry.maxToDrop = tableData.getIntField("maxToDrop", -1);
+	entry.flagID = tableData.getIntField("flagID", -1);
+	UNUSED(entry.gate_version = tableData.getStringField("gate_version", ""));
+	return entry;
+}
+
 void CDLootMatrixTable::LoadValuesFromDatabase() {
 
 	// First, get the size of the table
@@ -11,42 +24,35 @@ void CDLootMatrixTable::LoadValuesFromDatabase() {
 		tableSize.nextRow();
 	}
 
-	tableSize.finalize();
-
 	// Reserve the size
 	this->entries.reserve(size);
 
 	// Now get the data
 	auto tableData = CDClientDatabase::ExecuteQuery("SELECT * FROM LootMatrix");
 	while (!tableData.eof()) {
 		CDLootMatrix entry;
-		entry.LootMatrixIndex = tableData.getIntField("LootMatrixIndex", -1);
-		entry.LootTableIndex = tableData.getIntField("LootTableIndex", -1);
-		entry.RarityTableIndex = tableData.getIntField("RarityTableIndex", -1);
-		entry.percent = tableData.getFloatField("percent", -1.0f);
-		entry.minToDrop = tableData.getIntField("minToDrop", -1);
-		entry.maxToDrop = tableData.getIntField("maxToDrop", -1);
-		entry.id = tableData.getIntField("id", -1);
-		entry.flagID = tableData.getIntField("flagID", -1);
-		UNUSED(entry.gate_version = tableData.getStringField("gate_version", ""));
-
-		this->entries.push_back(entry);
+		uint32_t lootMatrixIndex = tableData.getIntField("LootMatrixIndex", -1);
+
+		this->entries[lootMatrixIndex].push_back(ReadRow(tableData));
 		tableData.nextRow();
 	}
-
-	tableData.finalize();
 }
 
-std::vector<CDLootMatrix> CDLootMatrixTable::Query(std::function<bool(CDLootMatrix)> predicate) {
+const LootMatrixEntries& CDLootMatrixTable::GetMatrix(uint32_t matrixId) {
+	auto itr = this->entries.find(matrixId);
+	if (itr != this->entries.end()) {
+		return itr->second;
+	}
 
-	std::vector<CDLootMatrix> data = cpplinq::from(this->entries)
-		>> cpplinq::where(predicate)
-		>> cpplinq::to_vector();
+	auto query = CDClientDatabase::CreatePreppedStmt("SELECT * FROM LootMatrix where LootMatrixIndex = ?;");
+	query.bind(1, static_cast<int32_t>(matrixId));
 
-	return data;
-}
+	auto tableData = query.execQuery();
+	while (!tableData.eof()) {
+		this->entries[matrixId].push_back(ReadRow(tableData));
+		tableData.nextRow();
+	}
 
-const std::vector<CDLootMatrix>& CDLootMatrixTable::GetEntries() const {
-	return this->entries;
+	return this->entries[matrixId];
 }
 

dDatabase/Tables/CDLootMatrixTable.h

@@ -4,26 +4,26 @@
 #include "CDTable.h"
 
 struct CDLootMatrix {
-	unsigned int LootMatrixIndex;           //!< The Loot Matrix Index
 	unsigned int LootTableIndex;            //!< The Loot Table Index
 	unsigned int RarityTableIndex;          //!< The Rarity Table Index
 	float percent;                   //!< The percent that this matrix is used?
 	unsigned int minToDrop;                 //!< The minimum amount of loot from this matrix to drop
 	unsigned int maxToDrop;                 //!< The maximum amount of loot from this matrix to drop
-	unsigned int id;                        //!< The ID of the Loot Matrix
 	unsigned int flagID;                    //!< ???
 	UNUSED(std::string gate_version);          //!< The Gate Version
 };
 
-class CDLootMatrixTable : public CDTable<CDLootMatrixTable> {
-private:
-	std::vector<CDLootMatrix> entries;
+typedef uint32_t LootMatrixIndex;
+typedef std::vector<CDLootMatrix> LootMatrixEntries;
 
+class CDLootMatrixTable : public CDTable<CDLootMatrixTable> {
 public:
 	void LoadValuesFromDatabase();
-	// Queries the table with a custom "where" clause
-	std::vector<CDLootMatrix> Query(std::function<bool(CDLootMatrix)> predicate);
 
-	const std::vector<CDLootMatrix>& GetEntries() const;
+	// Gets a matrix by ID or inserts a blank one if none existed.
+	const LootMatrixEntries& GetMatrix(uint32_t matrixId);
+private:
+	CDLootMatrix ReadRow(CppSQLite3Query& tableData) const;
+	std::unordered_map<LootMatrixIndex, LootMatrixEntries> entries;
 };
 

dDatabase/Tables/CDLootTableTable.cpp

@@ -1,5 +1,14 @@
 #include "CDLootTableTable.h"
 
+CDLootTable CDLootTableTable::ReadRow(CppSQLite3Query& tableData) const {
+	CDLootTable entry{};
+	if (tableData.eof()) return entry;
+	entry.itemid = tableData.getIntField("itemid", -1);
+	entry.MissionDrop = tableData.getIntField("MissionDrop", -1) == 1 ? true : false;
+	entry.sortPriority = tableData.getIntField("sortPriority", -1);
+	return entry;
+}
+
 void CDLootTableTable::LoadValuesFromDatabase() {
 
 	// First, get the size of the table
@@ -11,41 +20,35 @@ void CDLootTableTable::LoadValuesFromDatabase() {
 		tableSize.nextRow();
 	}
 
-	tableSize.finalize();
-
 	// Reserve the size
 	this->entries.reserve(size);
 
 	// Now get the data
 	auto tableData = CDClientDatabase::ExecuteQuery("SELECT * FROM LootTable");
 	while (!tableData.eof()) {
 		CDLootTable entry;
-		entry.id = tableData.getIntField("id", -1);
-		entry.itemid = tableData.getIntField("itemid", -1);
-		entry.LootTableIndex = tableData.getIntField("LootTableIndex", -1);
-		entry.id = tableData.getIntField("id", -1);
-		entry.MissionDrop = tableData.getIntField("MissionDrop", -1) == 1 ? true : false;
-		entry.sortPriority = tableData.getIntField("sortPriority", -1);
-
-		this->entries.push_back(entry);
+		uint32_t lootTableIndex = tableData.getIntField("LootTableIndex", -1);
+
+		this->entries[lootTableIndex].push_back(ReadRow(tableData));
 		tableData.nextRow();
 	}
-
-	tableData.finalize();
 }
 
-//! Queries the table with a custom "where" clause
-std::vector<CDLootTable> CDLootTableTable::Query(std::function<bool(CDLootTable)> predicate) {
+const LootTableEntries& CDLootTableTable::GetTable(uint32_t tableId) {
+	auto itr = this->entries.find(tableId);
+	if (itr != this->entries.end()) {
+		return itr->second;
+	}
 
-	std::vector<CDLootTable> data = cpplinq::from(this->entries)
-		>> cpplinq::where(predicate)
-		>> cpplinq::to_vector();
+	auto query = CDClientDatabase::CreatePreppedStmt("SELECT * FROM LootTable WHERE LootTableIndex = ?;");
+	query.bind(1, static_cast<int32_t>(tableId));
+	auto tableData = query.execQuery();
 
-	return data;
-}
+	while (!tableData.eof()) {
+		CDLootTable entry;
+		this->entries[tableId].push_back(ReadRow(tableData));
+		tableData.nextRow();
+	}
 
-//! Gets all the entries in the table
-const std::vector<CDLootTable>& CDLootTableTable::GetEntries() const {
-	return this->entries;
+	return this->entries[tableId];
 }
-

dDatabase/Tables/CDLootTableTable.h

@@ -6,20 +6,21 @@
 struct CDLootTable {
 	unsigned int itemid;                 //!< The LOT of the item
 	unsigned int LootTableIndex;         //!< The Loot Table Index
-	unsigned int id;                     //!< The ID
 	bool MissionDrop;               //!< Whether or not this loot table is a mission drop
 	unsigned int sortPriority;           //!< The sorting priority
 };
 
+typedef uint32_t LootTableIndex;
+typedef std::vector<CDLootTable> LootTableEntries;
+
 class CDLootTableTable : public CDTable<CDLootTableTable> {
 private:
-	std::vector<CDLootTable> entries;
+	CDLootTable ReadRow(CppSQLite3Query& tableData) const;
+	std::unordered_map<LootTableIndex, LootTableEntries> entries;
 
 public:
 	void LoadValuesFromDatabase();
 	// Queries the table with a custom "where" clause
-	std::vector<CDLootTable> Query(std::function<bool(CDLootTable)> predicate);
-
-	const std::vector<CDLootTable>& GetEntries() const;
+	const LootTableEntries& GetTable(uint32_t tableId);
 };
 

dDatabase/Tables/CDRarityTableTable.cpp

@@ -17,33 +17,18 @@ void CDRarityTableTable::LoadValuesFromDatabase() {
 	this->entries.reserve(size);
 
 	// Now get the data
-	auto tableData = CDClientDatabase::ExecuteQuery("SELECT * FROM RarityTable");
+	auto tableData = CDClientDatabase::ExecuteQuery("SELECT * FROM RarityTable order by randmax desc;");
 	while (!tableData.eof()) {
+		uint32_t rarityTableIndex = tableData.getIntField("RarityTableIndex", -1);
+
 		CDRarityTable entry;
-		entry.id = tableData.getIntField("id", -1);
 		entry.randmax = tableData.getFloatField("randmax", -1);
 		entry.rarity = tableData.getIntField("rarity", -1);
-		entry.RarityTableIndex = tableData.getIntField("RarityTableIndex", -1);
-
-		this->entries.push_back(entry);
+		entries[rarityTableIndex].push_back(entry);
 		tableData.nextRow();
 	}
-
-	tableData.finalize();
-}
-
-//! Queries the table with a custom "where" clause
-std::vector<CDRarityTable> CDRarityTableTable::Query(std::function<bool(CDRarityTable)> predicate) {
-
-	std::vector<CDRarityTable> data = cpplinq::from(this->entries)
-		>> cpplinq::where(predicate)
-		>> cpplinq::to_vector();
-
-	return data;
 }
 
-//! Gets all the entries in the table
-const std::vector<CDRarityTable>& CDRarityTableTable::GetEntries() const {
-	return this->entries;
+const std::vector<CDRarityTable>& CDRarityTableTable::GetRarityTable(uint32_t id) {
+	return entries[id];
 }
-

dDatabase/Tables/CDRarityTableTable.h

@@ -4,37 +4,20 @@
 #include "CDTable.h"
 
 struct CDRarityTable {
-	unsigned int id;
 	float randmax;
 	unsigned int rarity;
-	unsigned int RarityTableIndex;
-
-	friend bool operator> (const CDRarityTable& c1, const CDRarityTable& c2) {
-		return c1.rarity > c2.rarity;
-	}
-
-	friend bool operator>= (const CDRarityTable& c1, const CDRarityTable& c2) {
-		return c1.rarity >= c2.rarity;
-	}
-
-	friend bool operator< (const CDRarityTable& c1, const CDRarityTable& c2) {
-		return c1.rarity < c2.rarity;
-	}
-
-	friend bool operator<= (const CDRarityTable& c1, const CDRarityTable& c2) {
-		return c1.rarity <= c2.rarity;
-	}
 };
 
+typedef std::vector<CDRarityTable> RarityTable;
+
 class CDRarityTableTable : public CDTable<CDRarityTableTable> {
 private:
-	std::vector<CDRarityTable> entries;
+	typedef uint32_t RarityTableIndex;
+	std::unordered_map<RarityTableIndex, std::vector<CDRarityTable>> entries;
 
 public:
 	void LoadValuesFromDatabase();
-	// Queries the table with a custom "where" clause
-	std::vector<CDRarityTable> Query(std::function<bool(CDRarityTable)> predicate);
 
-	const std::vector<CDRarityTable>& GetEntries() const;
+	const std::vector<CDRarityTable>& GetRarityTable(uint32_t predicate);
 };
 

dGame/Entity.cpp

@@ -377,6 +377,7 @@ void Entity::Initialize() {
 					comp->SetIsSmashable(destCompData[0].isSmashable);
 
 					comp->SetLootMatrixID(destCompData[0].LootMatrixIndex);
+					Loot::CacheMatrix(destCompData[0].LootMatrixIndex);
 
 					// Now get currency information
 					uint32_t npcMinLevel = destCompData[0].level;
@@ -565,6 +566,7 @@ void Entity::Initialize() {
 
 			if (activityID > 0) {
 				comp->SetActivityId(activityID);
+				Loot::CacheMatrix(activityID);
 			}
 
 			const auto compTime = GetVar<float>(u"compTime");

dGame/dComponents/DestroyableComponent.cpp

@@ -735,18 +735,18 @@ void DestroyableComponent::Smash(const LWOOBJID source, const eKillType killType
 
 					auto* member = Game::entityManager->GetEntity(specificOwner);
 
-					if (member) LootGenerator::Instance().DropLoot(member, m_Parent, lootMatrixId, GetMinCoins(), GetMaxCoins());
+					if (member) Loot::DropLoot(member, m_Parent, lootMatrixId, GetMinCoins(), GetMaxCoins());
 				} else {
 					for (const auto memberId : team->members) { // Free for all
 						auto* member = Game::entityManager->GetEntity(memberId);
 
 						if (member == nullptr) continue;
 
-						LootGenerator::Instance().DropLoot(member, m_Parent, lootMatrixId, GetMinCoins(), GetMaxCoins());
+						Loot::DropLoot(member, m_Parent, lootMatrixId, GetMinCoins(), GetMaxCoins());
 					}
 				}
 			} else { // drop loot for non team user
-				LootGenerator::Instance().DropLoot(owner, m_Parent, GetLootMatrixID(), GetMinCoins(), GetMaxCoins());
+				Loot::DropLoot(owner, m_Parent, GetLootMatrixID(), GetMinCoins(), GetMaxCoins());
 			}
 		}
 	} else {
@@ -764,7 +764,7 @@ void DestroyableComponent::Smash(const LWOOBJID source, const eKillType killType
 
 				coinsTotal -= coinsToLose;
 
-				LootGenerator::Instance().DropLoot(m_Parent, m_Parent, -1, coinsToLose, coinsToLose);
+				Loot::DropLoot(m_Parent, m_Parent, -1, coinsToLose, coinsToLose);
 				character->SetCoins(coinsTotal, eLootSourceType::PICKUP);
 			}
 		}

dGame/dComponents/RacingControlComponent.cpp

@@ -388,7 +388,7 @@ void RacingControlComponent::HandleMessageBoxResponse(Entity* player, int32_t bu
 		}
 
         const auto score = playersRating * 10 + data->finished;
-		LootGenerator::Instance().GiveActivityLoot(player, m_Parent, m_ActivityID, score);
+		Loot::GiveActivityLoot(player, m_Parent, m_ActivityID, score);
 
 		// Giving rewards
 		GameMessages::SendNotifyRacingClient(

dGame/dComponents/RebuildComponent.cpp

@@ -486,7 +486,7 @@ void RebuildComponent::CompleteRebuild(Entity* user) {
 			auto* missionComponent = builder->GetComponent<MissionComponent>();
 			if (missionComponent) missionComponent->Progress(eMissionTaskType::ACTIVITY, m_ActivityId);
 		}
-		LootGenerator::Instance().DropActivityLoot(builder, m_Parent, m_ActivityId, 1);
+		Loot::DropActivityLoot(builder, m_Parent, m_ActivityId, 1);
 	}
 
 	// Notify scripts

dGame/dComponents/ScriptedActivityComponent.cpp

@@ -576,7 +576,7 @@ void ActivityInstance::RewardParticipant(Entity* participant) {
 			maxCoins = currencyTable[0].maxvalue;
 		}
 
-		LootGenerator::Instance().DropLoot(participant, m_Parent, activityRewards[0].LootMatrixIndex, minCoins, maxCoins);
+		Loot::DropLoot(participant, m_Parent, activityRewards[0].LootMatrixIndex, minCoins, maxCoins);
 	}
 }