diff --git a/README.md b/README.md index 8617e48..592ffbd 100644 --- a/README.md +++ b/README.md @@ -360,6 +360,10 @@ Input Global variable(s) configuration (or set 'H2AGENT_GLOBAL_VARIABLE' to be non-interactive) [global-variable.json]: global-variable.json +Input File manager configuration to enable read cache (true|false) + (or set 'H2AGENT__FILE_MANAGER_ENABLE_READ_CACHE_CONFIGURATION' to be non-interactive) [true]: +true + Input Server configuration to ignore request body (true|false) (or set 'H2AGENT__SERVER_TRAFFIC_IGNORE_REQUEST_BODY_CONFIGURATION' to be non-interactive) [false]: false @@ -1741,7 +1745,7 @@ The **target** of information is classified after parsing the following possible You could, for example, simulate a database where a *DELETE* for an specific entry could infer through its provision an *out-state* for a foreign method like *GET*, so when getting that *URI* you could obtain a *404* (assumed this provision for the new *working-state* = *in-state* = *out-state* = "id-deleted"). By default, the same `uri` is used from the current event to the foreign method, but it could also be provided optionally giving more flexibility to generate virtual events with specific states. -- txtFile.`` *[string]*: dumps source (as string) over text file with the path provided. The path can be relative (to the execution directory) or absolute, and **admits variables substitution**. Note that paths to missing directories will fail to open (the process does not create tree hierarchy). It is considered long term file (file is closed 1 second after last write, by default) when a constant path is configured, because this is normally used for specific log files. On the other hand, when any substitution took place on the path provided it is considered as a dynamic name, so understood as short term file (file is opened, written and closed without delay, by default). Delays in microseconds are configurable on process startup. Check [command line](#command-line) for `--long-term-files-close-delay-usecs` and `--short-term-files-close-delay-usecs` options. +- txtFile.`` *[string]*: dumps source (as string) over text file with the path provided. The path can be relative (to the execution directory) or absolute, and **admits variables substitution**. Note that paths to missing directories will fail to open (the process does not create tree hierarchy). It is considered long term file (file is closed 1 second after last write, by default) when a constant path is configured, because this is normally used for specific log files. On the other hand, when any substitution may took place in the path provided (it has variables in the form `@{varname}`) it is considered as a dynamic name, so understood as short term file (file is opened, written and closed without delay, by default). **Note:** you can force short term type inserting a variable, for example with empty value: `txtFile./path/to/short-term-file.txt@{empty}`. Delays in microseconds are configurable on process startup. Check [command line](#command-line) for `--long-term-files-close-delay-usecs` and `--short-term-files-close-delay-usecs` options. - binFile.`` *[string]*: same as `txtFile` but writting binary data. @@ -2364,6 +2368,9 @@ Usage: schema [-h|--help] [--clean] [file]; Cleans/gets/updates current schema c Usage: global_variable [-h|--help] [--clean] [name|file]; Cleans/gets/updates current agent global variable configuration (http://localhost:8074/admin/v1/global-variable). Usage: files [-h|--help]; Gets the files processed. +Usage: files_configuration [-h|--help]; Manages files configuration (gets current status by default). + [--enable-read-cache] ; Enables cache for read operations. + [--disable-read-cache] ; Disables cache for read operations. Usage: configuration [-h|--help]; Gets agent general configuration. Usage: server_configuration [-h|--help]; Manages agent server configuration (gets current status by default). [--traffic-server-ignore-request-body] ; Ignores request body on server receptions. diff --git a/ct/src/conftest.py b/ct/src/conftest.py index 92af0cf..4ff6316 100644 --- a/ct/src/conftest.py +++ b/ct/src/conftest.py @@ -540,7 +540,7 @@ def send(content, responseBodyRef = VALID_GLOBAL_VARIABLES__RESPONSE_BODY, respo } ''' -FILE_GENERATION_PROVISION=''' +FILE_MANAGER_PROVISION=''' { "requestMethod": "GET", "requestUri":"/app/v1/foo/bar", @@ -550,9 +550,17 @@ def send(content, responseBodyRef = VALID_GLOBAL_VARIABLES__RESPONSE_BODY, respo "source": "eraser", "target": "txtFile./tmp/example.txt" }, + { + "source": "value./tmp/example.txt", + "target": "var.file" + }, { "source": "value.hello", - "target": "txtFile./tmp/example.txt" + "target": "txtFile.@{file}" + }, + { + "source": "txtFile./tmp/example.txt", + "target": "response.body.string" } ] } diff --git a/ct/src/files_operation/files_test.py b/ct/src/files_operation/files_test.py index aa90656..4871350 100644 --- a/ct/src/files_operation/files_test.py +++ b/ct/src/files_operation/files_test.py @@ -1,14 +1,14 @@ import pytest import json import time -from conftest import ADMIN_FILES_URI, string2dict, FILE_GENERATION_PROVISION +from conftest import ADMIN_FILES_URI, string2dict, FILE_MANAGER_PROVISION @pytest.mark.admin def test_001_i_want_to_get_process_files(h2ac_admin, admin_server_provision, h2ac_traffic): # Provision - admin_server_provision(string2dict(FILE_GENERATION_PROVISION)) + admin_server_provision(string2dict(FILE_MANAGER_PROVISION)) # Check file before traffic: skipped because the test could re-run #response = h2ac_admin.get(ADMIN_FILES_URI) @@ -16,17 +16,18 @@ def test_001_i_want_to_get_process_files(h2ac_admin, admin_server_provision, h2a # Send GET response = h2ac_traffic.get("/app/v1/foo/bar") - - # Check file - response = h2ac_admin.get(ADMIN_FILES_URI) - responseBodyRef = [{ "bytes":0, "path": "/tmp/example.txt", "state": "opened", "closeDelayUsecs": 1000000 }] - h2ac_admin.assert_response__status_body_headers(response, 200, responseBodyRef) - - # Wait 2 seconds (long-term file closes in 1 second by default) - time.sleep(2) - + h2ac_admin.assert_response__status_body_headers(response, 200, "hello") + +# # Check file +# response = h2ac_admin.get(ADMIN_FILES_URI) +# responseBodyRef = [{ "bytes":0, "path": "/tmp/example.txt", "state": "opened" }] +# h2ac_admin.assert_response__status_body_headers(response, 200, responseBodyRef) +# +# # Wait 2 seconds (long-term file closes in 1 second by default) +# time.sleep(2) +# # Check file response = h2ac_admin.get(ADMIN_FILES_URI) - responseBodyRef = [{ "bytes":5, "path": "/tmp/example.txt", "state": "closed", "closeDelayUsecs": 1000000 }] + responseBodyRef = [{ "bytes":5, "path": "/tmp/example.txt", "state": "closed" }] h2ac_admin.assert_response__status_body_headers(response, 200, responseBodyRef) diff --git a/src/http2/MyAdminHttp2Server.cpp b/src/http2/MyAdminHttp2Server.cpp index 8a68a55..76e12b7 100644 --- a/src/http2/MyAdminHttp2Server.cpp +++ b/src/http2/MyAdminHttp2Server.cpp @@ -417,6 +417,10 @@ void MyAdminHttp2Server::receiveGET(const std::string &uri, const std::string &p responseBody = getHttp2Server()->dataConfigurationAsJsonString(); statusCode = 200; } + else if (pathSuffix == "files/configuration") { + responseBody = getFileManager()->configurationAsJsonString(); + statusCode = 200; + } else if (pathSuffix == "files") { responseBody = getFileManager()->asJsonString(); statusCode = ((responseBody == "[]") ? 204:200); @@ -606,6 +610,23 @@ void MyAdminHttp2Server::receivePUT(const std::string &pathSuffix, const std::st ert::tracing::Logger::error("Cannot keep requests history if data storage is discarded", ERT_FILE_LOCATION); } } + else if (pathSuffix == "files/configuration") { + std::string readCache; + + if (!queryParams.empty()) { // https://stackoverflow.com/questions/978061/http-get-with-request-body#:~:text=Yes.,semantic%20meaning%20to%20the%20request. + std::map qmap = h2agent::model::extractQueryParameters(queryParams); + auto it = qmap.find("readCache"); + if (it != qmap.end()) readCache = it->second; + + success = (readCache == "true" || readCache == "false"); + } + + if (success) { + bool b_readCache = (readCache == "true"); + getFileManager()->enableReadCache(b_readCache); + LOGWARNING(ert::tracing::Logger::warning(ert::tracing::Logger::asString("File read cache: %s", b_readCache ? "true":"false"), ERT_FILE_LOCATION)); + } + } statusCode = success ? 200:400; } diff --git a/src/main.cpp b/src/main.cpp index 111c4ee..5a3dcb4 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -620,7 +620,7 @@ int main(int argc, char* argv[]) if (cmdOptionExists(argv, argv + argc, "--long-term-files-close-delay-usecs", value)) { int iValue = toNumber(value); - if (iValue < 1) + if (iValue < 0) { usage(EXIT_FAILURE, "Invalid '--long-term-files-close-delay-usecs' value. Must be greater or equal than 0."); } @@ -630,7 +630,7 @@ int main(int argc, char* argv[]) if (cmdOptionExists(argv, argv + argc, "--short-term-files-close-delay-usecs", value)) { int iValue = toNumber(value); - if (iValue < 1) + if (iValue < 0) { usage(EXIT_FAILURE, "Invalid '--short-term-files-close-delay-usecs' value. Must be greater or equal than 0."); } diff --git a/src/model/AdminServerProvision.cpp b/src/model/AdminServerProvision.cpp index 4241d1b..68c3324 100644 --- a/src/model/AdminServerProvision.cpp +++ b/src/model/AdminServerProvision.cpp @@ -718,8 +718,10 @@ bool AdminServerProvision::processTargets(std::shared_ptr transf } else { // assignments - bool shortTerm = (target != transformation->getTarget()); // something was replaced in target: path is considered arbitrary and dynamic: short term files - file_manager_->write(target/*path*/, targetS/*data*/, true/*text*/, (shortTerm ? configuration_->getShortTermFilesCloseDelayUsecs():configuration_->getLongTermFilesCloseDelayUsecs())); + bool longTerm =(transformation->getTargetPatterns().empty()); // path is considered fixed (long term files), instead of arbitrary and dynamic (short term files) + // even if @{varname} is missing (empty value) we consider the intention to allow force short term + // files type. + file_manager_->write(target/*path*/, targetS/*data*/, true/*text*/, (longTerm ? configuration_->getLongTermFilesCloseDelayUsecs():configuration_->getShortTermFilesCloseDelayUsecs())); } } else if (transformation->getTargetType() == Transformation::TargetType::TBinFile) { @@ -733,8 +735,10 @@ bool AdminServerProvision::processTargets(std::shared_ptr transf } else { // assignments - bool shortTerm = (target != transformation->getTarget()); // something was replaced in target: path is considered arbitrary and dynamic: short term files - file_manager_->write(target/*path*/, targetS/*data*/, false/*binary*/, (shortTerm ? configuration_->getShortTermFilesCloseDelayUsecs():configuration_->getLongTermFilesCloseDelayUsecs())); + bool longTerm =(transformation->getTargetPatterns().empty()); // path is considered fixed (long term files), instead of arbitrary and dynamic (short term files) + // even if @{varname} is missing (empty value) we consider the intention to allow force short term + // files type. + file_manager_->write(target/*path*/, targetS/*data*/, false/*binary*/, (longTerm ? configuration_->getLongTermFilesCloseDelayUsecs():configuration_->getShortTermFilesCloseDelayUsecs())); } } } diff --git a/src/model/FileManager.cpp b/src/model/FileManager.cpp index 72a1a15..190f2de 100644 --- a/src/model/FileManager.cpp +++ b/src/model/FileManager.cpp @@ -44,6 +44,51 @@ namespace h2agent namespace model { + +void FileManager::enableMetrics(ert::metrics::Metrics *metrics) { + + metrics_ = metrics; + + if (metrics_) { + ert::metrics::counter_family_ref_t cf = metrics->addCounterFamily("FileSystem_observed_operations_total", "H2agent file system operations"); + observed_open_operation_counter_ = &(cf.Add({{"operation", "open"}})); + observed_close_operation_counter_ = &(cf.Add({{"operation", "close"}})); + observed_write_operation_counter_ = &(cf.Add({{"operation", "write"}})); + observed_empty_operation_counter_ = &(cf.Add({{"operation", "empty"}})); + observed_delayed_close_operation_counter_ = &(cf.Add({{"operation", "delayedClose"}})); + observed_instant_close_operation_counter_ = &(cf.Add({{"operation", "instantClose"}})); + observed_error_open_operation_counter_ = &(cf.Add({{"success", "false"}, {"operation", "open"}})); + } +} + +void FileManager::incrementObservedOpenOperationCounter() { + if (metrics_) observed_open_operation_counter_->Increment(); +} + +void FileManager::incrementObservedCloseOperationCounter() { + if (metrics_) observed_close_operation_counter_->Increment(); +} + +void FileManager::incrementObservedWriteOperationCounter() { + if (metrics_) observed_write_operation_counter_->Increment(); +} + +void FileManager::incrementObservedEmptyOperationCounter() { + if (metrics_) observed_empty_operation_counter_->Increment(); +} + +void FileManager::incrementObservedDelayedCloseOperationCounter() { + if (metrics_) observed_delayed_close_operation_counter_->Increment(); +} + +void FileManager::incrementObservedInstantCloseOperationCounter() { + if (metrics_) observed_instant_close_operation_counter_->Increment(); +} + +void FileManager::incrementObservedErrorOpenOperationCounter() { + if (metrics_) observed_error_open_operation_counter_->Increment(); +} + void FileManager::write(const std::string &path, const std::string &data, bool textOrBinary, unsigned int closeDelayUs) { std::shared_ptr safeFile; @@ -56,11 +101,11 @@ void FileManager::write(const std::string &path, const std::string &data, bool t std::ios_base::openmode mode = std::ofstream::out | std::ios_base::app; // for text files if (!textOrBinary) mode |= std::ios::binary; - safeFile = std::make_shared(path, io_service_, metrics_, closeDelayUs, mode); + safeFile = std::make_shared(this, path, io_service_, mode); add(path, safeFile); } - safeFile->write(data); + safeFile->write(data, closeDelayUs); } bool FileManager::read(const std::string &path, std::string &data, bool textOrBinary) { @@ -77,11 +122,11 @@ bool FileManager::read(const std::string &path, std::string &data, bool textOrBi else { if (!textOrBinary) mode |= std::ios::binary; - safeFile = std::make_shared(path, io_service_, metrics_, 0, mode); + safeFile = std::make_shared(this, path, io_service_, mode); add(path, safeFile); } - data = safeFile->read(result, mode); + data = safeFile->read(result, mode, read_cache_); return result; } @@ -95,7 +140,7 @@ void FileManager::empty(const std::string &path) { safeFile = it->second; } else { - safeFile = std::make_shared(path, io_service_, metrics_); + safeFile = std::make_shared(this, path, io_service_); add(path, safeFile); } @@ -111,9 +156,17 @@ bool FileManager::clear() return result; } -std::string FileManager::asJsonString() const { +nlohmann::json FileManager::getConfigurationJson() const { - return ((size() != 0) ? getJson().dump() : "[]"); // server data is shown as an array + nlohmann::json result; + result["readCache"] = read_cache_ ? "enabled":"disabled"; + + return result; +} + +std::string FileManager::configurationAsJsonString() const { + + return (getConfigurationJson().dump()); } nlohmann::json FileManager::getJson() const { @@ -129,6 +182,12 @@ nlohmann::json FileManager::getJson() const { return result; } +std::string FileManager::asJsonString() const { + + return ((size() != 0) ? getJson().dump() : "[]"); // server data is shown as an array +} + + } } diff --git a/src/model/FileManager.hpp b/src/model/FileManager.hpp index cf7a87d..1e5aa45 100644 --- a/src/model/FileManager.hpp +++ b/src/model/FileManager.hpp @@ -37,6 +37,8 @@ SOFTWARE. #include +#include + #include #include @@ -65,6 +67,16 @@ class FileManager : public Map> // metrics (will be passed to SafeFile): ert::metrics::Metrics *metrics_{}; + ert::metrics::counter_t *observed_open_operation_counter_{}; + ert::metrics::counter_t *observed_close_operation_counter_{}; + ert::metrics::counter_t *observed_write_operation_counter_{}; + ert::metrics::counter_t *observed_empty_operation_counter_{}; + ert::metrics::counter_t *observed_delayed_close_operation_counter_{}; + ert::metrics::counter_t *observed_instant_close_operation_counter_{}; + ert::metrics::counter_t *observed_error_open_operation_counter_{}; + + bool read_cache_; + public: /** * File manager class @@ -76,7 +88,7 @@ class FileManager : public Map> * * @see SafeFile */ - FileManager(boost::asio::io_service *timersIoService = nullptr, ert::metrics::Metrics *metrics = nullptr) : io_service_(timersIoService), metrics_(metrics) {;} + FileManager(boost::asio::io_service *timersIoService = nullptr, ert::metrics::Metrics *metrics = nullptr) : io_service_(timersIoService), metrics_(metrics), read_cache_(false) {;} ~FileManager() = default; /** @@ -84,9 +96,28 @@ class FileManager : public Map> * * @param metrics Optional metrics object to compute counters */ - void enableMetrics(ert::metrics::Metrics *metrics) { - metrics_ = metrics; - } + void enableMetrics(ert::metrics::Metrics *metrics); + + /** incrementObservedOpenOperationCounter */ + void incrementObservedOpenOperationCounter(); + + /** incrementObservedCloseOperationCounter */ + void incrementObservedCloseOperationCounter(); + + /** incrementObservedEmptyOperationCounter */ + void incrementObservedEmptyOperationCounter(); + + /** incrementObservedWriteOperationCounter */ + void incrementObservedWriteOperationCounter(); + + /** incrementObservedDelayedCloseOperationCounter */ + void incrementObservedDelayedCloseOperationCounter(); + + /** incrementObservedInstantCloseOperationCounter */ + void incrementObservedInstantCloseOperationCounter(); + + /** incrementObservedErrorOpenOperationCounter */ + void incrementObservedErrorOpenOperationCounter(); /** * Write file @@ -102,6 +133,7 @@ class FileManager : public Map> /** * Read the file content. + * There is no close delay. Read transaction is atomic (open, read, close). * * @param path path file to read. Can be relative (to execution directory) or absolute. * @param data data read by reference. @@ -111,6 +143,15 @@ class FileManager : public Map> */ bool read(const std::string &path, std::string &data, bool textOrBinary); + /** + * Enables read cache to speed up content load + * + * @param enable Boolean to enable or disable cache. File manager disables cache by default. + */ + void enableReadCache(bool enable) { + read_cache_ = enable; + } + /** * Empty file * @@ -125,11 +166,18 @@ class FileManager : public Map> bool clear(); /** - * Json string representation for class information (json object) + * Builds json document for class configuration * - * @return Json string representation ('[]' for empty object). + * @return Json object */ - std::string asJsonString() const; + nlohmann::json getConfigurationJson() const; + + /** + * Json string representation for class configuration (json object) + * + * @return Json string representation. + */ + std::string configurationAsJsonString() const; /** * Builds json document for class information @@ -137,6 +185,13 @@ class FileManager : public Map> * @return Json object */ nlohmann::json getJson() const; + + /** + * Json string representation for class information (json object) + * + * @return Json string representation ('[]' for empty object). + */ + std::string asJsonString() const; }; } diff --git a/src/model/SafeFile.cpp b/src/model/SafeFile.cpp index 41e1fb5..bf2fe6d 100644 --- a/src/model/SafeFile.cpp +++ b/src/model/SafeFile.cpp @@ -38,6 +38,9 @@ SOFTWARE. #include #include +#include +#include + namespace h2agent { @@ -48,27 +51,15 @@ std::atomic SafeFile::CurrentOpenedFiles(0); std::mutex SafeFile::MutexOpenedFiles; std::condition_variable SafeFile::OpenedFilesCV; -SafeFile::SafeFile (const std::string& path, boost::asio::io_service *timersIoService, ert::metrics::Metrics *metrics, unsigned int closeDelayUs, std::ios_base::openmode mode): +SafeFile::SafeFile (FileManager *fileManager, const std::string& path, boost::asio::io_service *timersIoService, std::ios_base::openmode mode): path_(path), io_service_(timersIoService), - metrics_(metrics), - close_delay_us_(closeDelayUs), + file_manager_(fileManager), opened_(false), + read_cached_(false), timer_(nullptr) { max_open_files_ = sysconf(_SC_OPEN_MAX /* 1024 probably */) - 10 /* margin just in case the process open other files */; - - if (metrics_) { - ert::metrics::counter_family_ref_t cf = metrics->addCounterFamily("FileSystem_observed_operations_total", "H2agent file system operations"); - observed_open_operation_counter_ = &(cf.Add({{"operation", "open"}})); - observed_close_operation_counter_ = &(cf.Add({{"operation", "close"}})); - observed_write_operation_counter_ = &(cf.Add({{"operation", "write"}})); - observed_empty_operation_counter_ = &(cf.Add({{"operation", "empty"}})); - observed_delayed_close_operation_counter_ = &(cf.Add({{"operation", "delayedClose"}})); - observed_instant_close_operation_counter_ = &(cf.Add({{"operation", "instantClose"}})); - observed_error_open_operation_counter_ = &(cf.Add({{"success", "false"}, {"operation", "open"}})); - } - open(mode); } @@ -77,16 +68,14 @@ SafeFile::~SafeFile() { delete timer_; } -void SafeFile::delayedClose() { - LOGDEBUG(ert::tracing::Logger::debug(ert::tracing::Logger::asString("Close delay is: %lu", close_delay_us_), ERT_FILE_LOCATION)); - +void SafeFile::delayedClose(unsigned int closeDelayUs) { // metrics - if (metrics_) observed_delayed_close_operation_counter_->Increment(); + file_manager_->incrementObservedDelayedCloseOperationCounter(); //if (!io_service_) return; // protection - if (!timer_) timer_ = new boost::asio::deadline_timer(*io_service_, boost::posix_time::microseconds(close_delay_us_)); + if (!timer_) timer_ = new boost::asio::deadline_timer(*io_service_, boost::posix_time::microseconds(closeDelayUs)); timer_->cancel(); - timer_->expires_from_now(boost::posix_time::microseconds(close_delay_us_)); + timer_->expires_from_now(boost::posix_time::microseconds(closeDelayUs)); timer_->async_wait([this] (const boost::system::error_code& e) { if( e ) return; // probably, we were cancelled (boost::asio::error::operation_aborted) close(); @@ -108,14 +97,14 @@ bool SafeFile::open(std::ios_base::openmode mode) { if (file_.is_open()) { opened_ = true; CurrentOpenedFiles++; - LOGDEBUG(ert::tracing::Logger::debug(ert::tracing::Logger::asString("'%s' opened for writting (currently opened: %d)", path_.c_str(), CurrentOpenedFiles.load()), ERT_FILE_LOCATION)); + LOGDEBUG(ert::tracing::Logger::debug(ert::tracing::Logger::asString("'%s' opened (currently opened: %d)", path_.c_str(), CurrentOpenedFiles.load()), ERT_FILE_LOCATION)); // metrics - if (metrics_) observed_open_operation_counter_->Increment(); + file_manager_->incrementObservedOpenOperationCounter(); } else { - LOGWARNING(ert::tracing::Logger::warning(ert::tracing::Logger::asString("Failed open to write operation for '%s'", path_.c_str()), ERT_FILE_LOCATION)); + LOGWARNING(ert::tracing::Logger::warning(ert::tracing::Logger::asString("Failed to open '%s'", path_.c_str()), ERT_FILE_LOCATION)); // metrics - if (metrics_) observed_error_open_operation_counter_->Increment(); + file_manager_->incrementObservedErrorOpenOperationCounter(); return false; } //lock.unlock(); @@ -132,7 +121,7 @@ void SafeFile::close() { CurrentOpenedFiles--; LOGDEBUG(ert::tracing::Logger::debug(ert::tracing::Logger::asString("'%s' closed (currently opened: %d)", path_.c_str(), CurrentOpenedFiles.load()), ERT_FILE_LOCATION)); // metrics - if (metrics_) observed_close_operation_counter_->Increment(); + file_manager_->incrementObservedCloseOperationCounter(); lock.unlock(); OpenedFilesCV.notify_one(); @@ -143,10 +132,17 @@ void SafeFile::empty() { open(std::ofstream::out | std::ofstream::trunc); close(); // metrics - if (metrics_) observed_empty_operation_counter_->Increment(); + file_manager_->incrementObservedEmptyOperationCounter(); } -std::string SafeFile::read(bool &success, std::ios_base::openmode mode) { +std::string SafeFile::read(bool &success, std::ios_base::openmode mode, bool cached) { + + // Check cached data: + if (cached && read_cached_) { + success = true; + return data_; + } + std::string result; success = false; @@ -157,26 +153,29 @@ std::string SafeFile::read(bool &success, std::ios_base::openmode mode) { { result += chunk; } + LOGDEBUG( + std::string output; + h2agent::model::asAsciiString(result, output); + ert::tracing::Logger::debug(ert::tracing::Logger::asString("Read '%s': %s", path_.c_str(), output.c_str()), ERT_FILE_LOCATION); + ); } // close the file after reading it: close(); - // metrics (not needed by h2agent) - //if (metrics_) ?->Increment(); + if (cached) { + read_cached_ = success; + data_ = std::move(result); + return data_; + } return result; } -void SafeFile::setCloseDelayUs(unsigned int usecs) { - close_delay_us_ = usecs; -} - nlohmann::json SafeFile::getJson() const { nlohmann::json result; result["path"] = path_; - if (close_delay_us_ != 0) result["closeDelayUsecs"] = close_delay_us_; std::ifstream file( path_, std::ofstream::in | std::ios::ate | std::ios::binary); // valid also for text files std::string::size_type size = file.tellg(); @@ -185,35 +184,39 @@ nlohmann::json SafeFile::getJson() const { result["state"] = (opened_ ? "opened":"closed"); } else { - result["state"] = ("missing"); + result["state"] = "missing"; } file.close(); + if (read_cached_) result["readCache"] = "true"; return result; } -void SafeFile::write (const std::string& data) { +void SafeFile::write (const std::string& data, unsigned int closeDelayUs) { // Open file (lazy): if (!open()) return; + // trace delay + LOGDEBUG(ert::tracing::Logger::debug(ert::tracing::Logger::asString("Close delay for write operation is: %lu", closeDelayUs), ERT_FILE_LOCATION)); + // Write file: std::lock_guard lock(mutex_); file_.write(data.c_str(), data.size()); LOGDEBUG(ert::tracing::Logger::debug(ert::tracing::Logger::asString("Data written into '%s'", path_.c_str()), ERT_FILE_LOCATION)); // metrics - if (metrics_) observed_write_operation_counter_->Increment(); + file_manager_->incrementObservedWriteOperationCounter(); // Close file: - if (io_service_ && close_delay_us_ != 0) { - delayedClose(); + if (io_service_ && closeDelayUs != 0) { + delayedClose(closeDelayUs); } else { close(); // metrics - if (metrics_) observed_instant_close_operation_counter_->Increment(); + file_manager_->incrementObservedInstantCloseOperationCounter(); } } diff --git a/src/model/SafeFile.hpp b/src/model/SafeFile.hpp index ebb9aab..58ac7bc 100644 --- a/src/model/SafeFile.hpp +++ b/src/model/SafeFile.hpp @@ -46,13 +46,12 @@ SOFTWARE. #include #include -#include - namespace h2agent { namespace model { +class FileManager; /** * This class allows safe writting of text/binary files. @@ -69,49 +68,33 @@ class SafeFile { std::mutex mutex_; // write file mutex bool opened_; boost::asio::deadline_timer *timer_{}; - unsigned int close_delay_us_; boost::asio::io_service *io_service_{}; - ert::metrics::Metrics *metrics_{}; + std::string data_; // used for read cache, but never shown in json string representation (just in case it is huge) + bool read_cached_; - ert::metrics::counter_t *observed_open_operation_counter_{}; - ert::metrics::counter_t *observed_close_operation_counter_{}; - ert::metrics::counter_t *observed_write_operation_counter_{}; - ert::metrics::counter_t *observed_empty_operation_counter_{}; - ert::metrics::counter_t *observed_delayed_close_operation_counter_{}; - ert::metrics::counter_t *observed_instant_close_operation_counter_{}; - ert::metrics::counter_t *observed_error_open_operation_counter_{}; + FileManager *file_manager_{}; - void delayedClose(); + void delayedClose(unsigned int closeDelayUs); public: /** * Constructor * + * @param fileManager parent reference to file manager. * @param path file path to write. It could be relative (to execution path) or absolute. * @param timersIoService asio io service which will be used to delay close * operations with the intention to reduce overhead in some scenarios. By default * it is not used (if not provided in constructor), so delay is not performed * regardless the close delay configured. - * @param metrics underlaying reference for SafeFile in order to compute prometheus metrics * about I/O operations. It may be 'nullptr' if no metrics are enabled. - * @param closeDelayUs delay after last write operation, to close the file. By default - * it is configured to 1 second, something appropiate to log over long term files. - * Zero value means that no planned close is scheduled, so the file is opened, - * written and closed in the same moment. This could be interesting for write - * many different files when they are not rewritten. If they need to append - * data later, use @setCloseDelay to configure them as short term files - * taking into account the maximum number of files that your system could - * open simultaneously. This class will blocks new open operations when that - * limit is reached, to prevent file system errors. * @param mode open mode. By default, text files and append is selected. You * could anyway add other flags, for example for binary dumps: std::ios::binary */ - SafeFile (const std::string& path, + SafeFile (FileManager *fileManager, + const std::string& path, boost::asio::io_service *timersIoService = nullptr, - ert::metrics::Metrics *metrics = nullptr, - unsigned int closeDelayUs = 1000000 /* 1 second */, std::ios_base::openmode mode = std::ofstream::out | std::ios_base::app); ~SafeFile(); @@ -147,30 +130,11 @@ class SafeFile { * @param success success of the read operation. * @param mode open mode. By default, text files are assumed, but you could * pass binary flag: std::ios::binary + * @param cached enable cache mode to retrieve data * - * @return Content read. Empty if failed to read. - */ - std::string read(bool &success, std::ios_base::openmode mode = std::ifstream::in); - - /** - * Set the delay in microseconds to close an opened file after writting over it. - * - * A value of zero will disable the delayed procedure, and close will be done - * after write operation (instant close). - * - * The class constructor sets 1 second by default, appropiate for logging - * oriented files which represent the most common and reasonable usage. - * - * We could consider 'short term files' those which are going to be rewritten - * (like long term ones) but when there are many of them and maximum opened - * files limit is a factor to take into account. So if your application is - * writting many different files, to optimize for example a load traffic rate - * of 200k req/s with a limit of 1024 concurrent files, we need a maximum - * delay of 1024/200000 = 0,00512 = 5 msecs. - * - * @param usecs microseconds of delay. Zero disables planned close and will be done instantly. + * @return Content read. Empty if failed to read. with cache enabled, empty data could be returned with success (no way no know if the file was empty or failed to read). */ - void setCloseDelayUs(unsigned int usecs); + std::string read(bool &success, std::ios_base::openmode mode = std::ifstream::in, bool cached = false); /** * Json representation of the class instance @@ -182,9 +146,21 @@ class SafeFile { * Close could be delayed. * * @param data data to write - * @see setCloseDelay() + * @param closeDelayUs delay, after write operation, to close the file. By default + * it is configured to 1 second, something appropiate to log over long term files. + * Zero value means that no planned close is scheduled, so the file is opened, + * written and closed in the same moment. This could be interesting for write + * many different files when they are not rewritten. If they need to append + * data later, you could provide a smaller value to indicate that it is a + * short term file, but you may take into account the maximum number of files + * that your system could open simultaneously. This class will blocks new open + * operations when that limit is reached, to prevent file system errors, and this + * will be exposed as performance drop. So if your application is writting many + * different files, to optimize for example a load traffic rate of 200k req/s + * with a limit of 1024 concurrent files, we need a maximum delay of 1024 divided + * by 200000, that is to say, 0,00512 seconds = 5000 microseconds. */ - void write (const std::string& data); + void write(const std::string& data, unsigned int closeDelayUs = 1000000); }; } diff --git a/st/server-provision.json b/st/server-provision.json index f6146fb..a170923 100644 --- a/st/server-provision.json +++ b/st/server-provision.json @@ -77,6 +77,10 @@ { "source": "value.@{usecs}\n", "target": "txtFile.__PROVISION_LONG_TERM_LOGFILE__" + }, + { + "source": "txtFile.__SOURCE_FILE__", + "target": "response.body.json.string./sourceFileContent" } ] } diff --git a/st/start.sh b/st/start.sh index 26c7180..359f0e2 100755 --- a/st/start.sh +++ b/st/start.sh @@ -3,6 +3,7 @@ ############# # VARIABLES # ############# +SCR_DIR=$(readlink -f "$(dirname "$0")") DEFAULTS= [ "$1" = "-y" ] && DEFAULTS=true @@ -10,6 +11,10 @@ DEFAULTS= PROVISION_LONG_TERM_LOGFILE=/tmp/h2agent_benchmark_timestamp_usecs.log rm -f ${PROVISION_LONG_TERM_LOGFILE} +# Source file example +SOURCE_FILE=${SCR_DIR}/server-matching.json +H2AGENT__FILE_MANAGER_ENABLE_READ_CACHE_CONFIGURATION__dflt=true + # Default values H2AGENT_VALIDATE_SCHEMAS__dflt=n H2AGENT_SCHEMA__dflt=schema.json @@ -48,7 +53,7 @@ H2AGENT__ADMIN_PORT=8074 H2AGENT__TRAFFIC_PORT=8000 # Common variables -COMMON_VARS="H2AGENT_VALIDATE_SCHEMAS H2AGENT_SCHEMA H2AGENT_SERVER_MATCHING H2AGENT_SERVER_PROVISION H2AGENT_GLOBAL_VARIABLE H2AGENT__SERVER_TRAFFIC_IGNORE_REQUEST_BODY_CONFIGURATION H2AGENT__SERVER_TRAFFIC_DYNAMIC_REQUEST_BODY_ALLOCATION_CONFIGURATION H2AGENT__DATA_STORAGE_CONFIGURATION H2AGENT__DATA_PURGE_CONFIGURATION H2AGENT__BIND_ADDRESS H2AGENT__RESPONSE_DELAY_MS ST_REQUEST_METHOD ST_REQUEST_URL ST_LAUNCHER" +COMMON_VARS="H2AGENT_VALIDATE_SCHEMAS H2AGENT_SCHEMA H2AGENT_SERVER_MATCHING H2AGENT_SERVER_PROVISION H2AGENT_GLOBAL_VARIABLE H2AGENT__FILE_MANAGER_ENABLE_READ_CACHE_CONFIGURATION__dflt H2AGENT__SERVER_TRAFFIC_IGNORE_REQUEST_BODY_CONFIGURATION H2AGENT__SERVER_TRAFFIC_DYNAMIC_REQUEST_BODY_ALLOCATION_CONFIGURATION H2AGENT__DATA_STORAGE_CONFIGURATION H2AGENT__DATA_PURGE_CONFIGURATION H2AGENT__BIND_ADDRESS H2AGENT__RESPONSE_DELAY_MS ST_REQUEST_METHOD ST_REQUEST_URL ST_LAUNCHER" ############# # FUNCTIONS # @@ -176,7 +181,7 @@ init_report() { # EXECUTION # ############# echo -cd $(dirname $0) +cd ${SCR_DIR} [ "$1" = "-h" -o "$1" = "--help" ] && usage && exit 0 # Requirements @@ -194,6 +199,7 @@ read_value "Provision configuration" H2AGENT_SERVER_PROVISION [ ! -f "${H2AGENT_SERVER_PROVISION}" ] && echo "ERROR: missing file '${H2AGENT_SERVER_PROVISION}' !" && exit 1 read_value "Global variable(s) configuration" H2AGENT_GLOBAL_VARIABLE [ ! -f "${H2AGENT_GLOBAL_VARIABLE}" ] && echo "ERROR: missing file '${H2AGENT_GLOBAL_VARIABLE}' !" && exit 1 +read_value "File manager configuration to enable read cache" H2AGENT__FILE_MANAGER_ENABLE_READ_CACHE_CONFIGURATION "true|false" || exit 1 read_value "Server configuration to ignore request body" H2AGENT__SERVER_TRAFFIC_IGNORE_REQUEST_BODY_CONFIGURATION "true|false" || exit 1 read_value "Server configuration to perform dynamic request body allocation" H2AGENT__SERVER_TRAFFIC_DYNAMIC_REQUEST_BODY_ALLOCATION_CONFIGURATION "true|false" || exit 1 read_value "Server data storage configuration" H2AGENT__DATA_STORAGE_CONFIGURATION "discard-all|discard-history|keep-all" || exit 1 @@ -257,11 +263,15 @@ then mv ${TMP_DIR}/server-provision.json2 ${TMP_DIR}/server-provision.json fi sed -i 's|__PROVISION_LONG_TERM_LOGFILE__|'${PROVISION_LONG_TERM_LOGFILE}'|' ${TMP_DIR}/server-provision.json +sed -i 's|__SOURCE_FILE__|'${SOURCE_FILE}'|' ${TMP_DIR}/server-provision.json [ "${H2AGENT_VALIDATE_SCHEMAS}" = "y" ] && { h2a_admin_curl POST admin/v1/schema 201 ${H2AGENT_SCHEMA} || exit 1 ; } h2a_admin_curl POST admin/v1/server-matching 201 ${H2AGENT_SERVER_MATCHING} || exit 1 h2a_admin_curl POST admin/v1/server-provision 201 ${TMP_DIR}/server-provision.json || exit 1 +# File manager configuration +h2a_admin_curl PUT "admin/v1/files/configuration?readCache=${H2AGENT__FILE_MANAGER_ENABLE_READ_CACHE_CONFIGURATION}" 200 || exit 1 + # Server configuration case ${H2AGENT__SERVER_TRAFFIC_IGNORE_REQUEST_BODY_CONFIGURATION} in false) RECEIVE_REQUEST_BODY=true ;; diff --git a/tools/helpers.src b/tools/helpers.src index b10cb19..fef187f 100644 --- a/tools/helpers.src +++ b/tools/helpers.src @@ -72,6 +72,24 @@ files() { fi } +files_configuration() { + if [ "$1" = "-h" -o "$1" = "--help" ] + then + echo "Usage: files_configuration [-h|--help]; Manages files configuration (gets current status by default)." + echo " [--enable-read-cache] ; Enables cache for read operations." + echo " [--disable-read-cache] ; Disables cache for read operations." + return 0 + elif [ "$1" = "--enable-read-cache" ] + then + do_curl -XPUT "${ADMIN_URL}/files/configuration?readCache=true" + elif [ "$1" = "--disable-read-cache" ] + then + do_curl -XPUT "${ADMIN_URL}/files/configuration?readCache=false" + else + do_curl ${ADMIN_URL}/files/configuration + fi +} + configuration() { if [ "$1" = "-h" -o "$1" = "--help" ] then @@ -277,6 +295,7 @@ snapshot() { global_variable && json > ${dir}/global-variable.json global_variable_schema && json > ${dir}/global-variable_schema.json files && json > ${dir}/files.json + files_configuration && json > ${dir}/files-configuration.json configuration && json > ${dir}/configuration.json server_configuration && json > ${dir}/server-configuration.json server_data_configuration && json > ${dir}/server-data-configuration.json @@ -379,6 +398,7 @@ help() { schema -h global_variable -h files -h + files_configuration -h configuration -h server_configuration -h server_data_configuration -h diff --git a/ut/model/FileSystem/fileManager.cpp b/ut/model/FileSystem/fileManager.cpp index 03a887d..f507aea 100644 --- a/ut/model/FileSystem/fileManager.cpp +++ b/ut/model/FileSystem/fileManager.cpp @@ -11,7 +11,8 @@ nlohmann::json FileManagerJson = R"( { "bytes": 0, "path": "/tmp/h2agent.ut.Beethoven.txt", - "state": "closed" + "state": "closed", + "readCache": "true" }, { "bytes": 0, @@ -55,14 +56,28 @@ TEST_F(FileManager_test, FileManager) EXPECT_EQ(content, FileManagerSafeFileContent); EXPECT_TRUE(success); + // read with cache: + fm.enableReadCache(true); + success = fm.read(path1, content, true /* text */); + EXPECT_EQ(content, FileManagerSafeFileContent); + EXPECT_TRUE(success); + // empty: fm.empty(path1); fm.empty(path2); + + // read path1 (was cached) success = fm.read(path1, content, true /* text */); + EXPECT_EQ(content, FileManagerSafeFileContent); + EXPECT_TRUE(success); + + // read path2 (not cached: will be read again) + fm.enableReadCache(false); + success = fm.read(path2, content, true /* text */); EXPECT_EQ(content, ""); EXPECT_TRUE(success); - sleep(1); + //sleep(1); // Check json representation: EXPECT_EQ(fm.getJson(), FileManagerJson); @@ -72,5 +87,8 @@ TEST_F(FileManager_test, FileManager) // Check empty: EXPECT_EQ(fm.asJsonString(), "[]"); + + // Check configuration: + EXPECT_EQ(fm.configurationAsJsonString(), "{\"readCache\":\"disabled\"}"); } diff --git a/ut/model/FileSystem/safeFile.cpp b/ut/model/FileSystem/safeFile.cpp index 40440dd..87582bd 100644 --- a/ut/model/FileSystem/safeFile.cpp +++ b/ut/model/FileSystem/safeFile.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include #include @@ -10,8 +10,7 @@ nlohmann::json SafeFileJson = R"( { "bytes": 0, "path": "/tmp/h2agent.ut.safefile.txt", - "state": "closed", - "closeDelayUsecs": 500 + "state": "closed" } )"_json; @@ -21,10 +20,12 @@ std::string SafeFileContent = "hi"; class SafeFile_test : public ::testing::Test { public: + h2agent::model::FileManager *file_manager_{}; boost::asio::io_service *timers_io_service_{}; std::thread *timers_thread_{}; SafeFile_test() { + file_manager_ = new h2agent::model::FileManager(); // no metrics by default timers_io_service_ = new boost::asio::io_service(); timers_thread_ = new std::thread([&] { boost::asio::io_service::work work(*timers_io_service_); @@ -45,32 +46,32 @@ TEST_F(SafeFile_test, SafeFileWithCloseDelayed) EXPECT_EQ(h2agent::model::SafeFile::CurrentOpenedFiles.load(), 0); // 1 file opened: - h2agent::model::SafeFile fileCloseDelayed(SafeFileJson["path"], timers_io_service_, nullptr /* metrics */, 500); + h2agent::model::SafeFile file(file_manager_, SafeFileJson["path"], timers_io_service_); EXPECT_EQ(h2agent::model::SafeFile::CurrentOpenedFiles.load(), 1); - fileCloseDelayed.empty(); // empty to ensure - fileCloseDelayed.write(SafeFileContent); + file.empty(); // empty to ensure + file.write(SafeFileContent, 5000 /* close delay value */); - // Closed after 0.5 seconds: - boost::asio::deadline_timer exitTimer(*timers_io_service_, boost::posix_time::milliseconds(1000)); + // Closed after 5000 usecs (5 ms), we wait 50 ms (10x !) to ensure it is closed: + boost::asio::deadline_timer exitTimer(*timers_io_service_, boost::posix_time::milliseconds(50)); exitTimer.async_wait([&] (const boost::system::error_code& e) { timers_io_service_->stop(); }); timers_thread_->join(); EXPECT_EQ(h2agent::model::SafeFile::CurrentOpenedFiles.load(), 0); // Check file content: - //fileCloseDelayed.open(std::ofstream::in); + //file.open(std::ofstream::in); bool success; - EXPECT_EQ(fileCloseDelayed.read(success), SafeFileContent); + EXPECT_EQ(file.read(success), SafeFileContent); EXPECT_TRUE(success); // Check json representation: SafeFileJson["bytes"] = SafeFileContent.size(); - EXPECT_EQ(fileCloseDelayed.getJson(), SafeFileJson); + EXPECT_EQ(file.getJson(), SafeFileJson); // Check after emptied: - fileCloseDelayed.empty(); + file.empty(); SafeFileJson["bytes"] = 0; - EXPECT_EQ(fileCloseDelayed.getJson(), SafeFileJson); + EXPECT_EQ(file.getJson(), SafeFileJson); } TEST_F(SafeFile_test, SafeFileWithInstantClose) @@ -83,21 +84,18 @@ TEST_F(SafeFile_test, SafeFileWithInstantClose) EXPECT_EQ(h2agent::model::SafeFile::CurrentOpenedFiles.load(), 0); // 1 file opened: - h2agent::model::SafeFile fileInstantClose(SafeFileJson["path"], nullptr /* no timers io service will be used */, nullptr /* metrics */, 0 /* instant close */); + h2agent::model::SafeFile fileInstantClose(file_manager_, SafeFileJson["path"], nullptr /* no timers io service will be used */); SafeFileJson.erase("closeDelayUsecs"); - // No timers io service will be used regardless close delay value: - fileInstantClose.setCloseDelayUs(0); - EXPECT_EQ(h2agent::model::SafeFile::CurrentOpenedFiles.load(), 1); fileInstantClose.empty(); // empty to ensure - fileInstantClose.write(SafeFileContent); + fileInstantClose.write(SafeFileContent, 0 /* close delay value */); // Instant close: EXPECT_EQ(h2agent::model::SafeFile::CurrentOpenedFiles.load(), 0); // Check file content: - //fileCloseDelayed.open(std::ofstream::in); + //file.open(std::ofstream::in); bool success; EXPECT_EQ(fileInstantClose.read(success), SafeFileContent); EXPECT_TRUE(success);