From bd9fd0b82b7d32ea82a2f5430b4a951c7c9b22b5 Mon Sep 17 00:00:00 2001 From: Kylie Smith Date: Mon, 9 Oct 2023 16:53:07 +1000 Subject: [PATCH 1/4] Start of DB metrics --- src/database/database_server.cpp | 60 ++++++++++++++++++++++++++++++++ src/database/database_server.h | 17 +++++++++ 2 files changed, 77 insertions(+) diff --git a/src/database/database_server.cpp b/src/database/database_server.cpp index 6a12df4..20e095b 100644 --- a/src/database/database_server.cpp +++ b/src/database/database_server.cpp @@ -112,9 +112,11 @@ void DatabaseServer::HandleDatagram(const std::shared_ptr &dg) { break; case DBSERVER_OBJECT_DELETE_FIELD: case DBSERVER_OBJECT_DELETE_FIELDS: + // TODO: Implement this. Logger::Error("[DB] OBJECT_DELETE_FIELD(S) NOT YET IMPLEMENTED!"); break; case DBSERVER_OBJECT_SET_FIELD_IF_EMPTY: + // TODO: Implement this. Logger::Error("[DB] OBJECT_SET_FIELD_IF_EMPTY NOT YET IMPLEMENTED!"); break; case DBSERVER_OBJECT_SET_FIELD_IF_EQUALS: @@ -149,6 +151,10 @@ uint32_t DatabaseServer::AllocateDoId() { // We've been allocated a DoId! if (doIdObj) { + if (_freeChannelsGauge) { + _freeChannelsGauge->Decrement(); + } + return DatabaseUtils::BsonToNumber( doIdObj->view()["doId"]["next"].get_value()); } @@ -164,6 +170,10 @@ uint32_t DatabaseServer::AllocateDoId() { << close_document << finalize); if (freeObj) { + if (_freeChannelsGauge) { + _freeChannelsGauge->Decrement(); + } + return DatabaseUtils::BsonToNumber( freeObj->view()["doId"]["free"].get_array().value[0].get_value()); } @@ -190,6 +200,10 @@ void DatabaseServer::FreeDoId(const uint32_t &doId) { << "GLOBALS" << finalize, document{} << "$push" << open_document << "doId.free" << static_cast(doId) << close_document << finalize); + + if (_freeChannelsGauge) { + _freeChannelsGauge->Increment(); + } } catch (const mongocxx::operation_exception &e) { Logger::Error( std::format("[DB] Failed to free DoId: {}: {}", doId, e.what())); @@ -783,6 +797,52 @@ void DatabaseServer::InitMetrics() { } auto registry = Metrics::Instance()->GetRegistry(); + + auto &freeChannelsBuilder = prometheus::BuildGauge() + .Name("db_free_channels_size") + .Help("Number of free channels") + .Register(*registry); + + auto &opsCompletedBuilder = + prometheus::BuildCounter() + .Name("db_ops_completed") + .Help("Number of successful database operations") + .Register(*registry); + + auto &opsFailedBuilder = prometheus::BuildCounter() + .Name("db_ops_failed") + .Help("Number of failed database operations") + .Register(*registry); + + auto &opsTimeBuilder = + prometheus::BuildHistogram() + .Name("db_ops_time") + .Help("Time taken for a successful database operation to complete") + .Register(*registry); + + _freeChannelsGauge = &freeChannelsBuilder.Add({}); + + // Map operation types to a human-readable string. + // These will be displayed in Prometheus/Grafana. + const std::vector> OPERATIONS = { + {OperationType::CREATE_OBJECT, "create_object"}, + {OperationType::DELETE_OBJECT, "delete_object"}, + {OperationType::GET_OBJECT, "get_object"}, + {OperationType::GET_OBJECT_FIELDS, "get_fields"}, + {OperationType::SET_OBJECT_FIELDS, "set_fields"}, + {OperationType::UPDATE_OBJECT_FIELDS, "update_fields"}}; + + // Populate operation maps. + for (const auto &opType : OPERATIONS) { + _opsCompleted[opType.first] = + &opsCompletedBuilder.Add({{"op_type", opType.second}}); + _opsFailed[opType.first] = + &opsFailedBuilder.Add({{"op_type", opType.second}}); + _opsCompletionTime[opType.first] = &opsTimeBuilder.Add( + {{"op_type", opType.second}}, + prometheus::Histogram::BucketBoundaries{0, 500, 1000, 1500, 2000, 2500, + 3000, 3500, 4000, 4500, 5000}); + } } } // namespace Ardos diff --git a/src/database/database_server.h b/src/database/database_server.h index f749352..9772719 100644 --- a/src/database/database_server.h +++ b/src/database/database_server.h @@ -3,6 +3,8 @@ #include #include +#include +#include #include "../messagedirector/channel_subscriber.h" #include "../net/datagram_iterator.h" @@ -49,6 +51,21 @@ class DatabaseServer : public ChannelSubscriber { mongocxx::uri _uri; mongocxx::client _conn; mongocxx::database _db; + + prometheus::Gauge *_freeChannelsGauge = nullptr; + + enum OperationType { + CREATE_OBJECT, + DELETE_OBJECT, + GET_OBJECT, + GET_OBJECT_FIELDS, + SET_OBJECT_FIELDS, + UPDATE_OBJECT_FIELDS, + }; + + std::unordered_map _opsCompleted; + std::unordered_map _opsFailed; + std::unordered_map _opsCompletionTime; }; } // namespace Ardos From a4c104971a740ce9f8273d30496dd1f1634218f5 Mon Sep 17 00:00:00 2001 From: Kylie Smith Date: Tue, 10 Oct 2023 21:16:52 +1000 Subject: [PATCH 2/4] Calculate free channels metric --- src/database/database_server.cpp | 48 ++++++++++++++++++++++++++++---- src/database/database_server.h | 6 ++-- src/database/database_utils.cpp | 11 ++++++-- 3 files changed, 54 insertions(+), 11 deletions(-) diff --git a/src/database/database_server.cpp b/src/database/database_server.cpp index 20e095b..9fa2aa6 100644 --- a/src/database/database_server.cpp +++ b/src/database/database_server.cpp @@ -97,7 +97,7 @@ void DatabaseServer::HandleDatagram(const std::shared_ptr &dg) { HandleCreate(dgi, sender); break; case DBSERVER_OBJECT_DELETE: - HandleDelete(dgi, sender); + HandleDelete(dgi); break; case DBSERVER_OBJECT_GET_ALL: HandleGetAll(dgi, sender); @@ -108,7 +108,7 @@ void DatabaseServer::HandleDatagram(const std::shared_ptr &dg) { break; case DBSERVER_OBJECT_SET_FIELD: case DBSERVER_OBJECT_SET_FIELDS: - HandleSetField(dgi, sender, msgType == DBSERVER_OBJECT_SET_FIELDS); + HandleSetField(dgi, msgType == DBSERVER_OBJECT_SET_FIELDS); break; case DBSERVER_OBJECT_DELETE_FIELD: case DBSERVER_OBJECT_DELETE_FIELDS: @@ -317,8 +317,7 @@ void DatabaseServer::HandleCreateDone(const uint64_t &channel, PublishDatagram(dg); } -void DatabaseServer::HandleDelete(DatagramIterator &dgi, - const uint64_t &sender) { +void DatabaseServer::HandleDelete(DatagramIterator &dgi) { uint32_t doId = dgi.GetUint32(); try { @@ -510,7 +509,6 @@ void DatabaseServer::HandleGetField(DatagramIterator &dgi, } void DatabaseServer::HandleSetField(DatagramIterator &dgi, - const uint64_t &sender, const bool &multiple) { auto doId = dgi.GetUint32(); auto fieldCount = multiple ? dgi.GetUint16() : 1; @@ -843,6 +841,46 @@ void DatabaseServer::InitMetrics() { prometheus::Histogram::BucketBoundaries{0, 500, 1000, 1500, 2000, 2500, 3000, 3500, 4000, 4500, 5000}); } + + // Calculate the number of free channels we have left to allocate. + InitFreeChannelsMetric(); +} + +void DatabaseServer::InitFreeChannelsMetric() { + try { + // Get the next DoId we have ready to allocate. + auto doIdObj = _db["globals"].find_one( + document{} << "_id" + << "GLOBALS" + << "doId.next" << open_document << "$gte" + << static_cast(_minDoId) << close_document + << "doId.next" << open_document << "$lte" + << static_cast(_maxDoId) << close_document + << finalize); + + if (!doIdObj) { + _freeChannelsGauge->Set(0); + return; + } + + auto currDoId = DatabaseUtils::BsonToNumber( + doIdObj->view()["doId"]["next"].get_value()); + + auto freeDoIdArr = doIdObj->view()["doId"]["free"].get_array().value; + auto freeDoIds = std::distance(freeDoIdArr.begin(), freeDoIdArr.end()); + + _freeChannelsGauge->Set((double)(_maxDoId - currDoId + freeDoIds)); + } catch (const ConversionException &e) { + Logger::Error(std::format("[DB] Conversion error occurred while " + "calculating free channel metrics: {}", + e.what())); + _freeChannelsGauge->Set(0); + } catch (const mongocxx::operation_exception &e) { + Logger::Error(std::format("[DB] MongoDB error occurred while calculating " + "free channel metrics: {}", + e.what())); + _freeChannelsGauge->Set(0); + } } } // namespace Ardos diff --git a/src/database/database_server.h b/src/database/database_server.h index 9772719..1b95d49 100644 --- a/src/database/database_server.h +++ b/src/database/database_server.h @@ -26,14 +26,13 @@ class DatabaseServer : public ChannelSubscriber { void HandleCreateDone(const uint64_t &channel, const uint32_t &context, const uint32_t &doId); - void HandleDelete(DatagramIterator &dgi, const uint64_t &sender); + void HandleDelete(DatagramIterator &dgi); void HandleGetAll(DatagramIterator &dgi, const uint64_t &sender); void HandleGetField(DatagramIterator &dgi, const uint64_t &sender, const bool &multiple); - void HandleSetField(DatagramIterator &dgi, const uint64_t &sender, - const bool &multiple); + void HandleSetField(DatagramIterator &dgi, const bool &multiple); void HandleSetFieldEquals(DatagramIterator &dgi, const uint64_t &sender, const bool &multiple); @@ -41,6 +40,7 @@ class DatabaseServer : public ChannelSubscriber { const uint32_t &context); void InitMetrics(); + void InitFreeChannelsMetric(); uint32_t _minDoId; uint32_t _maxDoId; diff --git a/src/database/database_utils.cpp b/src/database/database_utils.cpp index ee6c2fc..92ee066 100644 --- a/src/database/database_utils.cpp +++ b/src/database/database_utils.cpp @@ -331,12 +331,16 @@ void DatabaseUtils::BsonToField(const DCSubatomicType &fieldType, dg.AddBlob(arrDg.GetData(), arrDg.Size()); break; } - case ST_uint32uint8array: + case ST_uint32uint8array: { if (value.type() != bsoncxx::type::k_array) { throw ConversionException("Expected array"); } - dg.AddUint16(value.get_array().value.length()); - for (size_t i = 0; i < value.get_array().value.length();) { + + auto arr = value.get_array().value; + auto arrLength = std::distance(arr.begin(), arr.end()); + + dg.AddUint16(arrLength); + for (size_t i = 0; i < arrLength;) { dg.AddUint32(BsonToNumber( value.get_array().value[i].get_value(), divisor)); dg.AddUint8(BsonToNumber( @@ -344,6 +348,7 @@ void DatabaseUtils::BsonToField(const DCSubatomicType &fieldType, i += 2; } break; + } case ST_char: if (value.type() != bsoncxx::type::k_string && value.get_string().value.length() != 1) { From 04ffe71ec010e7986d12bee22cdb3a45ea1bc289 Mon Sep 17 00:00:00 2001 From: Kylie Smith Date: Wed, 11 Oct 2023 15:29:00 +1000 Subject: [PATCH 3/4] Database metrics --- src/database/database_server.cpp | 130 ++++++++++++++++++++++++++++--- src/database/database_server.h | 23 +++--- 2 files changed, 135 insertions(+), 18 deletions(-) diff --git a/src/database/database_server.cpp b/src/database/database_server.cpp index 9fa2aa6..70f7b5e 100644 --- a/src/database/database_server.cpp +++ b/src/database/database_server.cpp @@ -212,6 +212,8 @@ void DatabaseServer::FreeDoId(const uint32_t &doId) { void DatabaseServer::HandleCreate(DatagramIterator &dgi, const uint64_t &sender) { + auto startTime = g_loop->now(); + uint32_t context = dgi.GetUint32(); uint16_t dcId = dgi.GetUint16(); @@ -222,7 +224,9 @@ void DatabaseServer::HandleCreate(DatagramIterator &dgi, if (!dcClass) { Logger::Error(std::format( "[DB] Received create for unknown distributed class: {}", dcId)); + HandleCreateDone(sender, context, INVALID_DO_ID); + ReportFailed(CREATE_OBJECT); return; } @@ -230,6 +234,7 @@ void DatabaseServer::HandleCreate(DatagramIterator &dgi, FieldMap objectFields; if (!DatabaseUtils::UnpackFields(dgi, fieldCount, objectFields)) { HandleCreateDone(sender, context, INVALID_DO_ID); + ReportFailed(CREATE_OBJECT); return; } @@ -238,7 +243,9 @@ void DatabaseServer::HandleCreate(DatagramIterator &dgi, Logger::Error(std::format( "[DB] Failed to create object: {} with non-belonging fields", dcClass->get_name())); + HandleCreateDone(sender, context, INVALID_DO_ID); + ReportFailed(CREATE_OBJECT); return; } @@ -273,6 +280,7 @@ void DatabaseServer::HandleCreate(DatagramIterator &dgi, "[DB] Failed to unpack object fields for create: {}", e.what())); HandleCreateDone(sender, context, INVALID_DO_ID); + ReportFailed(CREATE_OBJECT); return; } auto fields = builder << finalize; @@ -281,6 +289,7 @@ void DatabaseServer::HandleCreate(DatagramIterator &dgi, uint32_t doId = AllocateDoId(); if (doId == INVALID_DO_ID) { HandleCreateDone(sender, context, INVALID_DO_ID); + ReportFailed(CREATE_OBJECT); return; } @@ -300,11 +309,13 @@ void DatabaseServer::HandleCreate(DatagramIterator &dgi, FreeDoId(doId); HandleCreateDone(sender, context, INVALID_DO_ID); + ReportFailed(CREATE_OBJECT); return; } // The object has been created successfully. HandleCreateDone(sender, context, doId); + ReportCompleted(CREATE_OBJECT, startTime); } void DatabaseServer::HandleCreateDone(const uint64_t &channel, @@ -318,6 +329,8 @@ void DatabaseServer::HandleCreateDone(const uint64_t &channel, } void DatabaseServer::HandleDelete(DatagramIterator &dgi) { + auto startTime = g_loop->now(); + uint32_t doId = dgi.GetUint32(); try { @@ -328,6 +341,8 @@ void DatabaseServer::HandleDelete(DatagramIterator &dgi) { if (!result || result->deleted_count() != 1) { Logger::Error( std::format("[DB] Tried to delete non-existent object: {}", doId)); + + ReportFailed(DELETE_OBJECT); return; } @@ -335,14 +350,20 @@ void DatabaseServer::HandleDelete(DatagramIterator &dgi) { FreeDoId(doId); Logger::Verbose(std::format("[DB] Deleted object: {}", doId)); + + ReportCompleted(DELETE_OBJECT, startTime); } catch (const mongocxx::operation_exception &e) { Logger::Error(std::format( "[DB] Unexpected error while deleting object {}: {}", doId, e.what())); + + ReportFailed(DELETE_OBJECT); } } void DatabaseServer::HandleGetAll(DatagramIterator &dgi, const uint64_t &sender) { + auto startTime = g_loop->now(); + uint32_t context = dgi.GetUint32(); uint32_t doId = dgi.GetUint32(); @@ -353,14 +374,18 @@ void DatabaseServer::HandleGetAll(DatagramIterator &dgi, } catch (const mongocxx::operation_exception &e) { Logger::Error(std::format( "[DB] Unexpected error while fetching object {}: {}", doId, e.what())); + HandleContextFailure(DBSERVER_OBJECT_GET_ALL_RESP, sender, context); + ReportFailed(GET_OBJECT); return; } if (!obj) { Logger::Error( std::format("[DB] Failed to fetch non-existent object: {}", doId)); + HandleContextFailure(DBSERVER_OBJECT_GET_ALL_RESP, sender, context); + ReportFailed(GET_OBJECT); return; } @@ -372,7 +397,9 @@ void DatabaseServer::HandleGetAll(DatagramIterator &dgi, Logger::Error(std::format( "[DB] Encountered unknown dclass while fetching object {}: {}", doId, dclassName)); + HandleContextFailure(DBSERVER_OBJECT_GET_ALL_RESP, sender, context); + ReportFailed(GET_OBJECT); return; } @@ -405,7 +432,9 @@ void DatabaseServer::HandleGetAll(DatagramIterator &dgi, Logger::Error( std::format("[DB] Failed to unpack field fetching object {}: {} - {}", doId, dclassName, e.what())); + HandleContextFailure(DBSERVER_OBJECT_GET_ALL_RESP, sender, context); + ReportFailed(GET_OBJECT); return; } @@ -420,11 +449,15 @@ void DatabaseServer::HandleGetAll(DatagramIterator &dgi, dg->AddData(it.second); } PublishDatagram(dg); + + ReportCompleted(GET_OBJECT, startTime); } void DatabaseServer::HandleGetField(DatagramIterator &dgi, const uint64_t &sender, const bool &multiple) { + auto startTime = g_loop->now(); + auto ctx = dgi.GetUint32(); auto doId = dgi.GetUint32(); auto fieldCount = multiple ? dgi.GetUint16() : 1; @@ -438,14 +471,20 @@ void DatabaseServer::HandleGetField(DatagramIterator &dgi, document{} << "_id" << static_cast(doId) << finalize); } catch (const mongocxx::operation_exception &e) { Logger::Error(std::format( - "[DB] Unexpected error while setting field(s) on object {}: {}", doId, + "[DB] Unexpected error while getting field(s) on object {}: {}", doId, e.what())); + + HandleContextFailure(responseType, sender, ctx); + ReportFailed(GET_OBJECT_FIELDS); return; } if (!obj) { Logger::Error(std::format( - "[DB] Failed to set field(s) on non-existent object: {}", doId)); + "[DB] Failed to get field(s) on non-existent object: {}", doId)); + + HandleContextFailure(responseType, sender, ctx); + ReportFailed(GET_OBJECT_FIELDS); return; } @@ -455,8 +494,11 @@ void DatabaseServer::HandleGetField(DatagramIterator &dgi, DCClass *dcClass = g_dc_file->get_class_by_name(dclassName); if (!dcClass) { Logger::Error(std::format( - "[DB] Received set field(s) for unknown distributed class {}: {}", doId, + "[DB] Received get field(s) for unknown distributed class {}: {}", doId, dclassName)); + + HandleContextFailure(responseType, sender, ctx); + ReportFailed(GET_OBJECT_FIELDS); return; } @@ -474,7 +516,9 @@ void DatabaseServer::HandleGetField(DatagramIterator &dgi, Logger::Error(std::format("[DB] Encountered unexpected field while " "fetching object {}: {} - {}", doId, dclassName, fieldNum)); + HandleContextFailure(responseType, sender, ctx); + ReportFailed(GET_OBJECT_FIELDS); return; } @@ -491,7 +535,9 @@ void DatabaseServer::HandleGetField(DatagramIterator &dgi, Logger::Error( std::format("[DB] Failed to unpack field fetching object {}: {} - {}", doId, dclassName, e.what())); + HandleContextFailure(responseType, sender, ctx); + ReportFailed(GET_OBJECT_FIELDS); return; } @@ -506,10 +552,14 @@ void DatabaseServer::HandleGetField(DatagramIterator &dgi, dg->AddData(it.second); } PublishDatagram(dg); + + ReportCompleted(GET_OBJECT_FIELDS, startTime); } void DatabaseServer::HandleSetField(DatagramIterator &dgi, const bool &multiple) { + auto startTime = g_loop->now(); + auto doId = dgi.GetUint32(); auto fieldCount = multiple ? dgi.GetUint16() : 1; @@ -521,12 +571,16 @@ void DatabaseServer::HandleSetField(DatagramIterator &dgi, Logger::Error(std::format( "[DB] Unexpected error while setting field(s) on object {}: {}", doId, e.what())); + + ReportFailed(SET_OBJECT_FIELDS); return; } if (!obj) { Logger::Error(std::format( "[DB] Failed to set field(s) on non-existent object: {}", doId)); + + ReportFailed(SET_OBJECT_FIELDS); return; } @@ -538,6 +592,8 @@ void DatabaseServer::HandleSetField(DatagramIterator &dgi, Logger::Error(std::format( "[DB] Received set field(s) for unknown distributed class {}: {}", doId, dclassName)); + + ReportFailed(SET_OBJECT_FIELDS); return; } @@ -546,6 +602,8 @@ void DatabaseServer::HandleSetField(DatagramIterator &dgi, if (!DatabaseUtils::UnpackFields(dgi, fieldCount, objectFields)) { Logger::Error( std::format("[DB] Failed to unpack set field(s) for object: {}", doId)); + + ReportFailed(SET_OBJECT_FIELDS); return; } @@ -553,6 +611,8 @@ void DatabaseServer::HandleSetField(DatagramIterator &dgi, if (!DatabaseUtils::VerifyFields(dcClass, objectFields)) { Logger::Error(std::format("[DB] Failed to verify fields on object {}: {} ", doId, dclassName)); + + ReportFailed(SET_OBJECT_FIELDS); return; } @@ -578,6 +638,8 @@ void DatabaseServer::HandleSetField(DatagramIterator &dgi, Logger::Error(std::format( "[DB] Failed to unpack object fields for set field(s) {}: {}", doId, e.what())); + + ReportFailed(SET_OBJECT_FIELDS); return; } @@ -592,21 +654,29 @@ void DatabaseServer::HandleSetField(DatagramIterator &dgi, if (!updateOperation) { Logger::Error(std::format( "[DB] Set field(s) update operation failed for object {}", doId)); + + ReportFailed(SET_OBJECT_FIELDS); return; } Logger::Verbose(std::format("[DB] Set field(s) for object {}: {}", doId, bsoncxx::to_json(fieldBuilder.view()))); + + ReportCompleted(SET_OBJECT_FIELDS, startTime); } catch (const mongocxx::operation_exception &e) { Logger::Error(std::format( "[DB] Unexpected error while setting field(s) on object {}: {}", doId, e.what())); + + ReportFailed(SET_OBJECT_FIELDS); } } void DatabaseServer::HandleSetFieldEquals(DatagramIterator &dgi, const uint64_t &sender, const bool &multiple) { + auto startTime = g_loop->now(); + auto ctx = dgi.GetUint32(); auto doId = dgi.GetUint32(); auto fieldCount = multiple ? dgi.GetUint16() : 1; @@ -622,14 +692,18 @@ void DatabaseServer::HandleSetFieldEquals(DatagramIterator &dgi, Logger::Error(std::format( "[DB] Unexpected error while setting field(s) equals on object {}: {}", doId, e.what())); + HandleContextFailure(responseType, sender, ctx); + ReportFailed(UPDATE_OBJECT_FIELDS); return; } if (!obj) { Logger::Error(std::format( "[DB] Failed to set field(s) equals on non-existent object: {}", doId)); + HandleContextFailure(responseType, sender, ctx); + ReportFailed(UPDATE_OBJECT_FIELDS); return; } @@ -641,7 +715,9 @@ void DatabaseServer::HandleSetFieldEquals(DatagramIterator &dgi, Logger::Error(std::format("[DB] Received set field(s) equals for unknown " "distributed class {}: {}", doId, dclassName)); + HandleContextFailure(responseType, sender, ctx); + ReportFailed(UPDATE_OBJECT_FIELDS); return; } @@ -652,7 +728,9 @@ void DatabaseServer::HandleSetFieldEquals(DatagramIterator &dgi, expectedFields)) { Logger::Error(std::format( "[DB] Failed to unpack set field(s) equals for object: {}", doId)); + HandleContextFailure(responseType, sender, ctx); + ReportFailed(UPDATE_OBJECT_FIELDS); return; } @@ -661,14 +739,18 @@ void DatabaseServer::HandleSetFieldEquals(DatagramIterator &dgi, Logger::Error(std::format( "[DB] Failed to verify set field(s) equals for object {}: {} ", doId, dclassName)); + HandleContextFailure(responseType, sender, ctx); + ReportFailed(UPDATE_OBJECT_FIELDS); return; } if (!DatabaseUtils::VerifyFields(dcClass, expectedFields)) { Logger::Error(std::format( "[DB] Failed to verify expected field(s) equals for object {}: {} ", doId, dclassName)); + HandleContextFailure(responseType, sender, ctx); + ReportFailed(UPDATE_OBJECT_FIELDS); return; } @@ -719,6 +801,8 @@ void DatabaseServer::HandleSetFieldEquals(DatagramIterator &dgi, dg->AddData(it.second); } PublishDatagram(dg); + + ReportFailed(UPDATE_OBJECT_FIELDS); return; } @@ -744,7 +828,9 @@ void DatabaseServer::HandleSetFieldEquals(DatagramIterator &dgi, Logger::Error(std::format( "[DB] Failed to unpack object fields for set field(s) equals {}: {}", doId, e.what())); + HandleContextFailure(responseType, sender, ctx); + ReportFailed(UPDATE_OBJECT_FIELDS); return; } @@ -759,24 +845,30 @@ void DatabaseServer::HandleSetFieldEquals(DatagramIterator &dgi, if (!updateOperation) { Logger::Error(std::format( "[DB] Set field(s) equals operation failed for object {}", doId)); + HandleContextFailure(responseType, sender, ctx); + ReportFailed(UPDATE_OBJECT_FIELDS); return; } Logger::Verbose(std::format("[DB] Set field(s) equals for object {}: {}", doId, bsoncxx::to_json(fieldBuilder.view()))); + + // Success! Notify the sender. + auto dg = std::make_shared(sender, _channel, responseType); + dg->AddUint32(ctx); + dg->AddBool(true); + PublishDatagram(dg); + + ReportCompleted(UPDATE_OBJECT_FIELDS, startTime); } catch (const mongocxx::operation_exception &e) { Logger::Error(std::format( "[DB] Unexpected error while setting field(s) equals on object {}: {}", doId, e.what())); + HandleContextFailure(responseType, sender, ctx); + ReportFailed(UPDATE_OBJECT_FIELDS); } - - // Success! Notify the sender. - auto dg = std::make_shared(sender, _channel, responseType); - dg->AddUint32(ctx); - dg->AddBool(true); - PublishDatagram(dg); } void DatabaseServer::HandleContextFailure(const MessageTypes &type, @@ -883,4 +975,24 @@ void DatabaseServer::InitFreeChannelsMetric() { } } +void DatabaseServer::ReportCompleted(const DatabaseServer::OperationType &type, + const uvw::timer_handle::time &startTime) { + auto counter = _opsCompleted[type]; + if (counter) { + counter->Increment(); + } + + auto time = _opsCompletionTime[type]; + if (time) { + time->Observe((double)(g_loop->now() - startTime).count()); + } +} + +void DatabaseServer::ReportFailed(const DatabaseServer::OperationType &type) { + auto counter = _opsFailed[type]; + if (counter) { + counter->Increment(); + } +} + } // namespace Ardos diff --git a/src/database/database_server.h b/src/database/database_server.h index 1b95d49..1c49e15 100644 --- a/src/database/database_server.h +++ b/src/database/database_server.h @@ -5,6 +5,7 @@ #include #include #include +#include #include "../messagedirector/channel_subscriber.h" #include "../net/datagram_iterator.h" @@ -42,6 +43,19 @@ class DatabaseServer : public ChannelSubscriber { void InitMetrics(); void InitFreeChannelsMetric(); + enum OperationType { + CREATE_OBJECT, + DELETE_OBJECT, + GET_OBJECT, + GET_OBJECT_FIELDS, + SET_OBJECT_FIELDS, + UPDATE_OBJECT_FIELDS, + }; + + void ReportCompleted(const OperationType &type, + const uvw::timer_handle::time &startTime); + void ReportFailed(const OperationType &type); + uint32_t _minDoId; uint32_t _maxDoId; uint64_t _channel; @@ -54,15 +68,6 @@ class DatabaseServer : public ChannelSubscriber { prometheus::Gauge *_freeChannelsGauge = nullptr; - enum OperationType { - CREATE_OBJECT, - DELETE_OBJECT, - GET_OBJECT, - GET_OBJECT_FIELDS, - SET_OBJECT_FIELDS, - UPDATE_OBJECT_FIELDS, - }; - std::unordered_map _opsCompleted; std::unordered_map _opsFailed; std::unordered_map _opsCompletionTime; From 6850e0a75d00e7ea6a18fa075286419974519336 Mon Sep 17 00:00:00 2001 From: Kylie Smith Date: Wed, 11 Oct 2023 16:59:33 +1000 Subject: [PATCH 4/4] DBSS metrics --- src/stateserver/database_state_server.cpp | 68 +++++++++++++++++++++++ src/stateserver/database_state_server.h | 12 ++++ src/stateserver/loading_object.cpp | 16 +++++- src/stateserver/loading_object.h | 2 + src/stateserver/state_server.cpp | 11 ++-- 5 files changed, 101 insertions(+), 8 deletions(-) diff --git a/src/stateserver/database_state_server.cpp b/src/stateserver/database_state_server.cpp index dd73c69..8a718ad 100644 --- a/src/stateserver/database_state_server.cpp +++ b/src/stateserver/database_state_server.cpp @@ -4,6 +4,7 @@ #include "../util/config.h" #include "../util/logger.h" +#include "../util/metrics.h" #include "loading_object.h" namespace Ardos { @@ -30,18 +31,37 @@ DatabaseStateServer::DatabaseStateServer() : ChannelSubscriber() { // Start listening to DoId's in our listening range. SubscribeRange(min, max); + + // Initialize metrics. + InitMetrics(); } void DatabaseStateServer::ReceiveObject(DistributedObject *distObj) { _distObjs[distObj->GetDoId()] = distObj; + + if (_objectsGauge) { + _objectsGauge->Increment(); + } + + if (_objectsSize) { + _objectsSize->Observe((double)distObj->Size()); + } } void DatabaseStateServer::RemoveDistributedObject(const uint32_t &doId) { _distObjs.erase(doId); + + if (_objectsGauge) { + _objectsGauge->Decrement(); + } } void DatabaseStateServer::DiscardLoader(const uint32_t &doId) { _loadObjs.erase(doId); + + if (_loadingGauge) { + _loadingGauge->Decrement(); + } } void DatabaseStateServer::HandleDatagram(const std::shared_ptr &dg) { @@ -349,6 +369,54 @@ void DatabaseStateServer::HandleGetActivated(DatagramIterator &dgi, PublishDatagram(dg); } +void DatabaseStateServer::InitMetrics() { + // Make sure we want to collect metrics on this cluster. + if (!Metrics::Instance()->WantMetrics()) { + return; + } + + auto registry = Metrics::Instance()->GetRegistry(); + + auto &objectsBuilder = prometheus::BuildGauge() + .Name("dbss_objects_size") + .Help("Number of loaded distributed objects") + .Register(*registry); + + auto &loadingBuilder = prometheus::BuildGauge() + .Name("dbss_loading_size") + .Help("Number of objects currently loading") + .Register(*registry); + + auto &activateTimeBuilder = + prometheus::BuildHistogram() + .Name("dbss_activate_time") + .Help("Time taken for an object to load/activate") + .Register(*registry); + + auto &objectsSizeBuilder = + prometheus::BuildHistogram() + .Name("dbss_objects_bytes_size") + .Help("Byte-size of loaded distributed objects") + .Register(*registry); + + _objectsGauge = &objectsBuilder.Add({}); + _loadingGauge = &loadingBuilder.Add({}); + + _activateTime = &activateTimeBuilder.Add( + {}, prometheus::Histogram::BucketBoundaries{ + 0, 500, 1000, 1500, 2000, 2500, 3000, 3500, 4000, 4500, 5000}); + _objectsSize = &objectsSizeBuilder.Add( + {}, prometheus::Histogram::BucketBoundaries{0, 4, 16, 64, 256, 1024, 4096, + 16384, 65536}); +} + +void DatabaseStateServer::ReportActivateTime( + const uvw::timer_handle::time &startTime) { + if (_activateTime) { + _activateTime->Observe((double)(g_loop->now() - startTime).count()); + } +} + bool UnpackDBFields(DatagramIterator &dgi, DCClass *dclass, FieldMap &required, FieldMap &ram) { // Unload RAM and REQUIRED fields from database response. diff --git a/src/stateserver/database_state_server.h b/src/stateserver/database_state_server.h index 3aadcbb..14a203c 100644 --- a/src/stateserver/database_state_server.h +++ b/src/stateserver/database_state_server.h @@ -1,6 +1,8 @@ #ifndef ARDOS_DATABASE_STATE_SERVER_H #define ARDOS_DATABASE_STATE_SERVER_H +#include + #include "../messagedirector/channel_subscriber.h" #include "../net/datagram_iterator.h" #include "../util/globals.h" @@ -41,6 +43,10 @@ class DatabaseStateServer : public StateServerImplementation, void HandleGetActivated(DatagramIterator &dgi, const uint64_t &sender); + void InitMetrics(); + + void ReportActivateTime(const uvw::timer_handle::time &startTime); + uint64_t _dbChannel; std::unordered_map _distObjs; @@ -52,6 +58,12 @@ class DatabaseStateServer : public StateServerImplementation, std::unordered_map> _contextDatagrams; uint32_t _nextContext = 0; + + prometheus::Gauge *_objectsGauge = nullptr; + prometheus::Gauge *_loadingGauge = nullptr; + + prometheus::Histogram *_objectsSize = nullptr; + prometheus::Histogram *_activateTime = nullptr; }; } // namespace Ardos diff --git a/src/stateserver/loading_object.cpp b/src/stateserver/loading_object.cpp index 574d3b3..ee09292 100644 --- a/src/stateserver/loading_object.cpp +++ b/src/stateserver/loading_object.cpp @@ -12,8 +12,13 @@ LoadingObject::LoadingObject(DatabaseStateServer *stateServer, const std::unordered_set &contexts) : ChannelSubscriber(), _stateServer(stateServer), _doId(doId), _parentId(parentId), _zoneId(zoneId), - _context(stateServer->_nextContext++), _validContexts(contexts) { + _context(stateServer->_nextContext++), _validContexts(contexts), + _startTime(g_loop->now()) { SubscribeChannel(doId); + + if (_stateServer->_loadingGauge) { + _stateServer->_loadingGauge->Increment(); + } } LoadingObject::LoadingObject(DatabaseStateServer *stateServer, @@ -24,7 +29,7 @@ LoadingObject::LoadingObject(DatabaseStateServer *stateServer, : ChannelSubscriber(), _stateServer(stateServer), _doId(doId), _parentId(parentId), _zoneId(zoneId), _context(stateServer->_nextContext++), _validContexts(contexts), - _dclass(dclass) { + _dclass(dclass), _startTime(g_loop->now()) { SubscribeChannel(doId); // Unpack the RAM fields we received in the generate message. @@ -48,10 +53,14 @@ LoadingObject::LoadingObject(DatabaseStateServer *stateServer, _doId, field->get_name())); } } + + if (_stateServer->_loadingGauge) { + _stateServer->_loadingGauge->Increment(); + } } void LoadingObject::Start() { - if (!_validContexts.size()) { + if (_validContexts.empty()) { // Fetch our stored fields from the database. auto dg = std::make_shared(_stateServer->_dbChannel, _doId, DBSERVER_OBJECT_GET_ALL); @@ -173,6 +182,7 @@ void LoadingObject::HandleDatagram(const std::shared_ptr &dg) { } void LoadingObject::Finalize() { + _stateServer->ReportActivateTime(_startTime); _stateServer->DiscardLoader(_doId); ForwardDatagrams(); ChannelSubscriber::Shutdown(); diff --git a/src/stateserver/loading_object.h b/src/stateserver/loading_object.h index 8e9aaa3..22402ed 100644 --- a/src/stateserver/loading_object.h +++ b/src/stateserver/loading_object.h @@ -52,6 +52,8 @@ class LoadingObject : public ChannelSubscriber { FieldMap _ramFields; std::vector> _datagramQueue; + + uvw::timer_handle::time _startTime; }; } // namespace Ardos diff --git a/src/stateserver/state_server.cpp b/src/stateserver/state_server.cpp index 5d68768..e0fed9e 100644 --- a/src/stateserver/state_server.cpp +++ b/src/stateserver/state_server.cpp @@ -133,13 +133,14 @@ void StateServer::InitMetrics() { auto &objectsBuilder = prometheus::BuildGauge() .Name("ss_objects_size") - .Help("Number of distributed objects") + .Help("Number of loaded distributed objects") .Register(*registry); - auto &objectsSizeBuilder = prometheus::BuildHistogram() - .Name("ss_objects_bytes_size") - .Help("Bytes size of distributed objects") - .Register(*registry); + auto &objectsSizeBuilder = + prometheus::BuildHistogram() + .Name("ss_objects_bytes_size") + .Help("Byte-size of loaded distributed objects") + .Register(*registry); _objectsGauge = &objectsBuilder.Add({}); _objectsSizeHistogram = &objectsSizeBuilder.Add(