From 5e0e0c8dd5534b337cbced8a4c6ebbb39d363e51 Mon Sep 17 00:00:00 2001 From: Derek D Date: Wed, 12 Jun 2024 18:40:14 +0000 Subject: [PATCH] [#5210] Rewrite make_copy so it stores rule files in memory --- .../irods/private/re/configuration.hpp | 1 + .../include/irods/private/re/rules.hpp | 7 + .../irods_rule_language/src/configuration.cpp | 219 +++++++++++++----- .../irods_rule_language/src/rules.cpp | 35 +++ 4 files changed, 201 insertions(+), 61 deletions(-) diff --git a/plugins/rule_engines/irods_rule_language/include/irods/private/re/configuration.hpp b/plugins/rule_engines/irods_rule_language/include/irods/private/re/configuration.hpp index cd185244b4..25f88b30ae 100644 --- a/plugins/rule_engines/irods_rule_language/include/irods/private/re/configuration.hpp +++ b/plugins/rule_engines/irods_rule_language/include/irods/private/re/configuration.hpp @@ -141,6 +141,7 @@ int clearResources( int resources ); int clearCoreRuleIndex( ); int clearAppRuleIndex( ); int readRuleStructAndRuleSetFromFile( const char *ruleBaseName, const char *rulesFileName ); +int readRuleStructAndRuleSetFromBuffer(const char* ruleBaseName, char* ruleBase); int loadRuleFromCacheOrFile( const char*, const char *irbSet ); int createCoreRuleIndex( ); int createAppRuleIndex( ); diff --git a/plugins/rule_engines/irods_rule_language/include/irods/private/re/rules.hpp b/plugins/rule_engines/irods_rule_language/include/irods/private/re/rules.hpp index 6198fc4a87..83b799219f 100644 --- a/plugins/rule_engines/irods_rule_language/include/irods/private/re/rules.hpp +++ b/plugins/rule_engines/irods_rule_language/include/irods/private/re/rules.hpp @@ -13,6 +13,13 @@ int readRuleSetFromFile( const char *ruleBaseName, RuleSet *ruleSet, Env *funcDesc, int* errloc, rError_t *errmsg, Region *r ); int readRuleSetFromLocalFile( const char *ruleBaseName, const char *fileName, RuleSet *ruleSet, Env *funcDesc, int *errloc, rError_t *errmsg, Region *r ); +int readRuleSetFromBuffer(const char* ruleBaseName, + char* ruleBody, + RuleSet* ruleSet, + Env* funcDesc, + int* errloc, + rError_t* errmsg, + Region* r); int parseAndComputeMsParamArrayToEnv( msParamArray_t *msParamArray, Env *global, ruleExecInfo_t *rei, int reiSaveFlag, rError_t *errmsg, Region *r ); int parseAndComputeRule( char *expr, Env *env, ruleExecInfo_t *rei, int reiSaveFlag, rError_t *errmsg, Region *r ); int parseAndComputeRuleNewEnv( char *expr, ruleExecInfo_t *rei, int reiSaveFlag, msParamArray_t *msParamArray, rError_t *errmsg, Region *r ); diff --git a/plugins/rule_engines/irods_rule_language/src/configuration.cpp b/plugins/rule_engines/irods_rule_language/src/configuration.cpp index e1fd210ed4..7f8d0ea0a3 100644 --- a/plugins/rule_engines/irods_rule_language/src/configuration.cpp +++ b/plugins/rule_engines/irods_rule_language/src/configuration.cpp @@ -21,10 +21,17 @@ #include "irods/private/re/reFuncDefs.hpp" #include "irods/Hasher.hpp" #include "irods/irods_hasher_factory.hpp" +#include "irods/irods_exception.hpp" #include "irods/irods_get_full_path_for_config_file.hpp" #include "irods/irods_log.hpp" #include +#include +#include +#include + +using log_re = irods::experimental::log::rule_engine; +constexpr auto err_buf_len = ERR_MSG_LEN * 1024; Cache::Cache() : address(NULL), /* unsigned char *address */ pointers(NULL), /* unsigned char *pointers */ @@ -418,23 +425,60 @@ int hash_rules(const std::vector &irbs, const int pid, std::string return 0; } -class make_copy { -public: - make_copy(const std::vector& _irbs, const int _pid) : irbs_(_irbs), pid_(_pid) { - for(auto const &irb : _irbs) { - boost::filesystem::copy_file(get_rule_base_path(irb), get_rule_base_path_copy(irb, _pid)); - } - } - ~make_copy() { - for(auto const &irb : irbs_) { - boost::filesystem::remove(get_rule_base_path_copy(irb, pid_)); +class in_memory_rulebases +{ + public: + explicit in_memory_rulebases(const std::vector& _irods_rule_bases) + { + for (auto const& irb : _irods_rule_bases) { + std::ifstream ifs(get_rule_base_path(irb)); + if (!ifs) { + log_re::error("in_memory_rulebases: input file stream [{}] failed", get_rule_base_path(irb)); + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-array-to-pointer-decay) + THROW(RULES_FILE_READ_ERROR, fmt::format("Failed to load rulebase [{}] into memory", irb)); } + rule_bases_.insert( + {irb, std::string({std::istreambuf_iterator(ifs), std::istreambuf_iterator()})}); } -private: - const std::vector irbs_; - const int pid_; + } + + const std::string& get_rulebase(const std::string& irb) + { + return rule_bases_.at(irb); + } + + private: + std::unordered_map rule_bases_; }; +int hash_rules_with_copy(const std::vector& irods_rule_bases, + std::string& digest, + in_memory_rulebases& copy) +{ + irods::Hasher hasher; + irods::error ret = irods::getHasher("md5", hasher); + if (!ret.ok()) { + int status = static_cast(ret.code()); + log_re::error("hash_rules_with_copy: cannot get hasher, status = {}", status); + return status; + } + + for (auto const& irb : irods_rule_bases) { + try { + hasher.update(copy.get_rulebase(irb)); + } + catch (std::out_of_range& ex) { + log_re::warn("hash_rules_with_copy: attempted to access nonexistent rulebase [{}]", irb); + } + } + + hasher.digest(digest); + + log_re::debug("hash_rules_with_copy: rulebases = [{}]; digest = [{}]", fmt::join(irods_rule_bases, ", "), digest); + + return 0; +} + int load_rules(const char* irbSet, const std::vector &irbs, const int pid, const time_type timestamp) { generateRegions(); generateRuleSets(); @@ -443,33 +487,51 @@ int load_rules(const char* irbSet, const std::vector &irbs, const i getSystemFunctions( ruleEngineConfig.sysFuncDescIndex->current, ruleEngineConfig.sysRegion ); } - make_copy copy_rule_base_files(irbs, pid); - for(auto const & irb: irbs) { - int i = readRuleStructAndRuleSetFromFile( irb.c_str(), get_rule_base_path_copy(irb, pid).c_str() ); - - if ( i != 0 ) { - ruleEngineConfig.ruleEngineStatus = INITIALIZED; - return i; + try { + in_memory_rulebases stored_rulebases(irbs); + for (auto const& irb : irbs) { + try { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) + auto* rule_ptr = const_cast(&stored_rulebases.get_rulebase(irb)[0]); + int i = readRuleStructAndRuleSetFromBuffer(irb.c_str(), rule_ptr); + + if (i != 0) { + ruleEngineConfig.ruleEngineStatus = INITIALIZED; + return i; + } + } + catch (std::out_of_range& ex) { + log_re::warn("{}: attempted to access nonexistent rulebase [{}]", __func__, irb); + } } - } - createCoreRuleIndex( ); + createCoreRuleIndex(); - /* set max timestamp */ - time_type_set( ruleEngineConfig.timestamp, timestamp ); - std::string hash; - int ret = hash_rules(irbs, pid, hash); - if (ret >= 0) { - rstrcpy(ruleEngineConfig.hash, hash.c_str(), CHKSUM_LEN); - } else { + /* set max timestamp */ + time_type_set(ruleEngineConfig.timestamp, timestamp); + std::string hash; + int ret = hash_rules_with_copy(irbs, hash, stored_rulebases); + if (ret >= 0) { + rstrcpy(static_cast(ruleEngineConfig.hash), hash.c_str(), CHKSUM_LEN); + } + else { + ruleEngineConfig.ruleEngineStatus = INITIALIZED; + return ret; + } + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg) + snprintf( + static_cast(ruleEngineConfig.ruleBase), sizeof(ruleEngineConfig.ruleBase), "%s", irbSet); ruleEngineConfig.ruleEngineStatus = INITIALIZED; - return ret; } - snprintf( ruleEngineConfig.ruleBase, sizeof( ruleEngineConfig.ruleBase ), "%s", irbSet ); - ruleEngineConfig.ruleEngineStatus = INITIALIZED; + catch (const irods::exception& e) { + log_re::error( + "{} encountered an exception in source file {}:{}", __PRETTY_FUNCTION__, __FILE__, e.what()); + return static_cast(e.code()); + } return 0; } +// NOLINTNEXTLINE(readability-function-cognitive-complexity) int loadRuleFromCacheOrFile( const char* inst_name, const char *irbSet ) { int res = 0; auto irbs = parse_irbSet(irbSet); @@ -522,35 +584,47 @@ int loadRuleFromCacheOrFile( const char* inst_name, const char *irbSet ) { rodsLog( LOG_ERROR, "Failed to restore cache." ); } else { int diffIrbSet = strcmp( cache->ruleBase, irbSet ) != 0; - - make_copy copy_rule_base_files(irbs, pid); - std::string hash; - int ret = hash_rules(irbs, pid, hash); - - int diffHash = ret < 0 || hash != cache->hash; - if ( diffIrbSet ) { - rodsLog( LOG_DEBUG, "Rule base set changed, old value is %s", cache->ruleBase ); - } - - if ( diffIrbSet || time_type_gt( timestamp, cache->timestamp ) || diffHash ) { - update = 1; - free( cache->address ); - rodsLog( LOG_DEBUG, "Rule base set or rule files modified, force refresh." ); - } else { - cache->cacheStatus = INITIALIZED; - ruleEngineConfig = *cache; - - /* generate extRuleSet */ - generateRegions(); - generateRuleSets(); - generateFunctionDescriptionTables(); - if ( ruleEngineConfig.ruleEngineStatus == UNINITIALIZED ) { - getSystemFunctions( ruleEngineConfig.sysFuncDescIndex->current, ruleEngineConfig.sysRegion ); + try { + in_memory_rulebases stored_rulebases(irbs); + std::string hash; + int ret = hash_rules_with_copy(irbs, hash, stored_rulebases); + + int diffHash = static_cast(ret < 0 || hash != static_cast(cache->hash)); + if (static_cast(diffIrbSet)) { + log_re::debug("Rule base set changed, old value is {}", cache->ruleBase); } - ruleEngineConfig.ruleEngineStatus = INITIALIZED; - - return res; + if (static_cast(diffIrbSet) || time_type_gt(timestamp, cache->timestamp) || + static_cast(diffHash)) { + update = 1; + // NOLINTNEXTLINE(cppcoreguidelines-no-malloc,cppcoreguidelines-owning-memory) + free(cache->address); + log_re::debug("Rule base set or rule files modified, force refresh."); + } + else { + cache->cacheStatus = INITIALIZED; + ruleEngineConfig = *cache; + + /* generate extRuleSet */ + generateRegions(); + generateRuleSets(); + generateFunctionDescriptionTables(); + if (ruleEngineConfig.ruleEngineStatus == UNINITIALIZED) { + getSystemFunctions( + ruleEngineConfig.sysFuncDescIndex->current, ruleEngineConfig.sysRegion); + } + + ruleEngineConfig.ruleEngineStatus = INITIALIZED; + + return res; + } + } + catch (const irods::exception& e) { + log_re::error("{} encountered an exception in file {}:{}", + __PRETTY_FUNCTION__, + __FILE__, + e.client_display_what()); + return static_cast(e.code()); } } } @@ -576,8 +650,7 @@ int loadRuleFromCacheOrFile( const char* inst_name, const char *irbSet ) { return res; } int readRuleStructAndRuleSetFromFile( const char *ruleBaseName, const char *rulesBaseFile ) { - - int errloc; + int errloc = 0; rError_t errmsgBuf; errmsgBuf.errMsg = NULL; errmsgBuf.len = 0; @@ -595,6 +668,30 @@ int readRuleStructAndRuleSetFromFile( const char *ruleBaseName, const char *rule return res; } +int readRuleStructAndRuleSetFromBuffer(const char* ruleBaseName, char* ruleBase) +{ + int errloc = 0; + rError_t errmsgBuf; + errmsgBuf.errMsg = nullptr; + errmsgBuf.len = 0; + + // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays) + auto buf = std::make_unique(err_buf_len * sizeof(char)); + int res = readRuleSetFromBuffer(ruleBaseName, + ruleBase, + ruleEngineConfig.coreRuleSet, + ruleEngineConfig.coreFuncDescIndex, + &errloc, + &errmsgBuf, + ruleEngineConfig.coreRegion); + if (res != 0) { + errMsgToString(&errmsgBuf, buf.get(), err_buf_len); + log_re::error(buf.get()); + } + freeRErrorContent(&errmsgBuf); + return res; +} + int availableRules() { return ( isComponentInitialized( ruleEngineConfig.appRuleSetStatus ) ? ruleEngineConfig.coreRuleSet->len : 0 ) + ( isComponentInitialized( ruleEngineConfig.appRuleSetStatus ) == INITIALIZED ? ruleEngineConfig.appRuleSet->len : 0 ); } diff --git a/plugins/rule_engines/irods_rule_language/src/rules.cpp b/plugins/rule_engines/irods_rule_language/src/rules.cpp index bcf77d9c0f..ce03eb7c2d 100644 --- a/plugins/rule_engines/irods_rule_language/src/rules.cpp +++ b/plugins/rule_engines/irods_rule_language/src/rules.cpp @@ -61,6 +61,41 @@ int readRuleSetFromLocalFile( const char *ruleBaseName, const char *rulesFileNam return 0; } +int readRuleSetFromBuffer(const char* ruleBaseName, + char* ruleBody, + RuleSet* ruleSet, + Env* funcDesc, + int* errloc, + rError_t* errmsg, + Region* r) +{ + FILE* file = fmemopen(ruleBody, strlen(ruleBody), "r"); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays) + char errbuf[ERR_MSG_LEN]; + if (file == nullptr) { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg) + snprintf(static_cast(errbuf), ERR_MSG_LEN, "readRuleSetFromBuffer() could not read rule from buffer"); + addRErrorMsg(errmsg, RULES_FILE_READ_ERROR, static_cast(errbuf)); + return RULES_FILE_READ_ERROR; + } + Pointer* e = newPointer(file, ruleBaseName); + int ret = parseRuleSet(e, ruleSet, funcDesc, errloc, errmsg, r); + deletePointer(e); + if (ret < 0) { + return ret; + } + + Node* errnode{}; + ExprType* restype = typeRuleSet(ruleSet, errmsg, &errnode, r); + if (getNodeType(restype) == T_ERROR) { + if (nullptr != errnode) { + *errloc = NODE_EXPR_POS(errnode); + } + return RE_TYPE_ERROR; + } + + return 0; +} int parseAndComputeMsParamArrayToEnv( msParamArray_t *var, Env *env, ruleExecInfo_t *rei, int reiSaveFlag, rError_t *errmsg, Region *r ) { int i;