From 3124ca3348f3ecb36f87dc7f39693338c34b9487 Mon Sep 17 00:00:00 2001 From: Alan King Date: Tue, 9 Apr 2024 16:55:14 -0400 Subject: [PATCH] [#7577] Add msi_replica_truncate microservice And added some tests. --- doxygen/microservices.cpp | 1 + scripts/core_tests_list.json | 1 + scripts/irods/test/test_all_rules.py | 118 +++++++++++++++++ server/re/include/irods/reAction.hpp | 2 + server/re/include/irods/reDataObjOpr.hpp | 2 + server/re/src/reDataObjOpr.cpp | 158 ++++++++++++++++++++--- 6 files changed, 261 insertions(+), 21 deletions(-) diff --git a/doxygen/microservices.cpp b/doxygen/microservices.cpp index e441cccb17..7b5536fd62 100644 --- a/doxygen/microservices.cpp +++ b/doxygen/microservices.cpp @@ -68,6 +68,7 @@ - #msiCheckPermission - Check if a data object permission is the same as the one given - #msiCheckOwner - Check if the user is the owner of the data object - #msiSetReplComment - Sets the data_comments attribute of a data object + - #msi_replica_truncate - Truncate a replica to the desired size \subsection msicollection Collection Microservices - #msiCollCreate - Create a collection diff --git a/scripts/core_tests_list.json b/scripts/core_tests_list.json index 1d1cf7104e..70d59ca4dc 100644 --- a/scripts/core_tests_list.json +++ b/scripts/core_tests_list.json @@ -2,6 +2,7 @@ "test_all_rules.Test_AllRules", "test_all_rules.Test_JSON_microservices", "test_all_rules.Test_msiDataObjRepl_checksum_keywords", + "test_all_rules.test_msi_replica_truncate", "test_auth.Test_Auth", "test_auth.test_iinit", "test_catalog", diff --git a/scripts/irods/test/test_all_rules.py b/scripts/irods/test/test_all_rules.py index f079b481cc..ae74b93a3d 100644 --- a/scripts/irods/test/test_all_rules.py +++ b/scripts/irods/test/test_all_rules.py @@ -2290,3 +2290,121 @@ def do_test(json_ptr, expected_value): do_test( '/other', 'null') do_test( '/bool1', 'true') do_test( '/bool2', 'false') + + +@unittest.skipUnless(IrodsConfig().default_rule_engine_plugin == 'irods_rule_engine_plugin-irods_rule_language', + 'Only implemented for NREP.') +class test_msi_replica_truncate(unittest.TestCase): + """These test msi_replica_truncate and are not meant to be thorough tests for the replica_truncate API.""" + + plugin_name = IrodsConfig().default_rule_engine_plugin + rep_instance = plugin_name + '-instance' + other_resource = 'msi_replica_truncate_resource' + original_contents = 'truncate this' + new_size = 100 + data_name = 'test_msi_replica_truncate.txt' + + @classmethod + def setUpClass(cls): + # Create a test user. + cls.user = session.mkuser_and_return_session('rodsuser', 'smeagol', 'spass', lib.get_hostname()) + + # Create a test resource. + with session.make_session_for_existing_admin() as admin_session: + lib.create_ufs_resource(admin_session, cls.other_resource, hostname=test.settings.HOSTNAME_2) + + @classmethod + def tearDownClass(cls): + with session.make_session_for_existing_admin() as admin_session: + # End the test user session, and remove the test user and test resource. + cls.user.__exit__() + admin_session.assert_icommand(['iadmin', 'rmuser', cls.user.username]) + lib.remove_resource(admin_session, cls.other_resource) + + def setUp(self): + # Create a data object to truncate. + self.logical_path = '/'.join([self.user.session_collection, self.data_name]) + self.user.assert_icommand(['istream', 'write', self.logical_path], input=self.original_contents) + self.assertTrue(lib.replica_exists_on_resource(self.user, self.logical_path, self.user.default_resource)) + self.user.assert_icommand(['irepl', '-R', self.other_resource, self.logical_path]) + self.assertTrue(lib.replica_exists_on_resource(self.user, self.logical_path, self.other_resource)) + + def tearDown(self): + # Remove the data object. + self.user.assert_icommand(['irm', '-f', self.logical_path]) + + def test_empty_string_input(self): + rule_text = "msi_replica_truncate('', *ignored)" + self.user.assert_icommand( + ['irule', '-r', self.rep_instance, rule_text, 'null', 'ruleExecOut'], + 'STDERR', ['-358000', 'OBJ_PATH_DOES_NOT_EXIST']) + self.assertEqual(len(self.original_contents), int(lib.get_replica_size(self.user, self.data_name, 0))) + self.assertEqual(len(self.original_contents), int(lib.get_replica_size(self.user, self.data_name, 1))) + + def test_string_input_that_just_says_null(self): + rule_text = "msi_replica_truncate('null', *ignored)" + self.user.assert_icommand( + ['irule', '-r', self.rep_instance, rule_text, 'null', 'ruleExecOut'], + 'STDERR', ['-358000', 'OBJ_PATH_DOES_NOT_EXIST']) + self.assertEqual(len(self.original_contents), int(lib.get_replica_size(self.user, self.data_name, 0))) + self.assertEqual(len(self.original_contents), int(lib.get_replica_size(self.user, self.data_name, 1))) + + def test_int_input(self): + rule_text = "msi_replica_truncate(1, *ignored)" + self.user.assert_icommand( + ['irule', '-r', self.rep_instance, rule_text, 'null', 'ruleExecOut'], + 'STDERR', ['-323000', 'USER_PARAM_TYPE_ERR']) + self.assertEqual(len(self.original_contents), int(lib.get_replica_size(self.user, self.data_name, 0))) + self.assertEqual(len(self.original_contents), int(lib.get_replica_size(self.user, self.data_name, 1))) + + def test_missing_data_size_keyword(self): + rule_text = "msi_replica_truncate('objPath={}', *ignored)".format(self.logical_path) + self.user.assert_icommand( + ['irule', '-r', self.rep_instance, rule_text, 'null', 'ruleExecOut'], + 'STDERR', ['-529022', 'UNIX_FILE_TRUNCATE_ERR', 'Invalid argument']) + self.assertEqual(len(self.original_contents), int(lib.get_replica_size(self.user, self.data_name, 0))) + self.assertEqual(len(self.original_contents), int(lib.get_replica_size(self.user, self.data_name, 1))) + + def test_invalid_keyword(self): + # "replica_number" is the invalid keyword. + rule_text = "msi_replica_truncate('objPath={}++++dataSize={}++++replica_number=1', *ignored)".format( + self.logical_path, self.new_size) + self.user.assert_icommand( + ['irule', '-r', self.rep_instance, rule_text, 'null', 'ruleExecOut'], + 'STDERR', ['-315000', 'USER_BAD_KEYWORD_ERR']) + self.assertEqual(len(self.original_contents), int(lib.get_replica_size(self.user, self.data_name, 0))) + self.assertEqual(len(self.original_contents), int(lib.get_replica_size(self.user, self.data_name, 1))) + + def test_minimum_valid_input(self): + rule_text = "msi_replica_truncate('{}++++dataSize={}', *ignored)".format(self.logical_path, self.new_size) + self.user.assert_icommand(['irule', '-r', self.rep_instance, rule_text, 'null', 'ruleExecOut']) + self.assertEqual(self.new_size, int(lib.get_replica_size(self.user, self.data_name, 0))) + self.assertEqual(len(self.original_contents), int(lib.get_replica_size(self.user, self.data_name, 1))) + + def test_object_path_keyword(self): + rule_text = "msi_replica_truncate('objPath={}++++dataSize={}', *ignored)".format(self.logical_path, self.new_size) + self.user.assert_icommand(['irule', '-r', self.rep_instance, rule_text, 'null', 'ruleExecOut']) + self.assertEqual(self.new_size, int(lib.get_replica_size(self.user, self.data_name, 0))) + self.assertEqual(len(self.original_contents), int(lib.get_replica_size(self.user, self.data_name, 1))) + + def test_replica_number_keyword(self): + rule_text = "msi_replica_truncate('objPath={}++++dataSize={}++++replNum=1', *ignored)".format( + self.logical_path, self.new_size) + self.user.assert_icommand(['irule', '-r', self.rep_instance, rule_text, 'null', 'ruleExecOut']) + self.assertEqual(len(self.original_contents), int(lib.get_replica_size(self.user, self.data_name, 0))) + self.assertEqual(self.new_size, int(lib.get_replica_size(self.user, self.data_name, 1))) + + def test_resource_name_keyword(self): + rule_text = "msi_replica_truncate('objPath={}++++dataSize={}++++rescName={}', *ignored)".format( + self.logical_path, self.new_size, self.other_resource) + self.user.assert_icommand(['irule', '-r', self.rep_instance, rule_text, 'null', 'ruleExecOut']) + self.assertEqual(len(self.original_contents), int(lib.get_replica_size(self.user, self.data_name, 0))) + self.assertEqual(self.new_size, int(lib.get_replica_size(self.user, self.data_name, 1))) + + def test_admin_keyword(self): + with session.make_session_for_existing_admin() as admin_session: + rule_text = "msi_replica_truncate('objPath={}++++dataSize={}++++irodsAdmin=', *ignored)".format( + self.logical_path, self.new_size) + admin_session.assert_icommand(['irule', '-r', self.rep_instance, rule_text, 'null', 'ruleExecOut']) + self.assertEqual(self.new_size, int(lib.get_replica_size(self.user, self.data_name, 0))) + self.assertEqual(len(self.original_contents), int(lib.get_replica_size(self.user, self.data_name, 1))) diff --git a/server/re/include/irods/reAction.hpp b/server/re/include/irods/reAction.hpp index 05e389efa6..ce6602623c 100644 --- a/server/re/include/irods/reAction.hpp +++ b/server/re/include/irods/reAction.hpp @@ -183,6 +183,8 @@ namespace irods table_[ "msiDataObjPhymv" ] = new irods::ms_table_entry( "msiDataObjPhymv", 6, std::function( msiDataObjPhymv ) ); table_[ "msiDataObjRename" ] = new irods::ms_table_entry( "msiDataObjRename", 4, std::function( msiDataObjRename ) ); table_[ "msiDataObjTrim" ] = new irods::ms_table_entry( "msiDataObjTrim", 6, std::function( msiDataObjTrim ) ); + // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) + table_[ "msi_replica_truncate" ] = new irods::ms_table_entry( "msi_replica_truncate", 2, std::function( msi_replica_truncate ) ); table_[ "msiCollCreate" ] = new irods::ms_table_entry( "msiCollCreate", 3, std::function( msiCollCreate ) ); table_[ "msiRmColl" ] = new irods::ms_table_entry( "msiRmColl", 3, std::function( msiRmColl ) ); table_[ "msiCollRepl" ] = new irods::ms_table_entry( "msiCollRepl", 3, std::function( msiCollRepl ) ); diff --git a/server/re/include/irods/reDataObjOpr.hpp b/server/re/include/irods/reDataObjOpr.hpp index be97e87843..7140998fc1 100644 --- a/server/re/include/irods/reDataObjOpr.hpp +++ b/server/re/include/irods/reDataObjOpr.hpp @@ -96,4 +96,6 @@ msiCollRsync( msParam_t *inpParam1, msParam_t *inpParam2, int _rsCollRsync( rsComm_t *rsComm, dataObjInp_t *dataObjInp, char *srcColl, char *destColl ); + +auto msi_replica_truncate(MsParam* _inp, MsParam* _out, RuleExecInfo* _rei) -> int; #endif /* RE_DATA_OBJ_OPR_H */ diff --git a/server/re/src/reDataObjOpr.cpp b/server/re/src/reDataObjOpr.cpp index e99ce38903..e777e6eea2 100644 --- a/server/re/src/reDataObjOpr.cpp +++ b/server/re/src/reDataObjOpr.cpp @@ -1,41 +1,43 @@ /// \file #include "irods/reDataObjOpr.hpp" + #include "irods/apiHeaderAll.h" -#include "irods/rsApiHandler.hpp" #include "irods/collection.hpp" -#include "irods/rsDataObjCreate.hpp" -#include "irods/rsDataObjOpen.hpp" +#include "irods/irods_at_scope_exit.hpp" +#include "irods/key_value_proxy.hpp" +#include "irods/rsApiHandler.hpp" +#include "irods/rsCloseCollection.hpp" +#include "irods/rsCollCreate.hpp" +#include "irods/rsCollRepl.hpp" +#include "irods/rsDataObjChksum.hpp" #include "irods/rsDataObjClose.hpp" -#include "irods/rsDataObjRead.hpp" -#include "irods/rsDataObjWrite.hpp" -#include "irods/rsDataObjUnlink.hpp" -#include "irods/rsDataObjRepl.hpp" #include "irods/rsDataObjCopy.hpp" -#include "irods/rsDataObjChksum.hpp" +#include "irods/rsDataObjCreate.hpp" #include "irods/rsDataObjLseek.hpp" +#include "irods/rsDataObjOpen.hpp" #include "irods/rsDataObjPhymv.hpp" +#include "irods/rsDataObjRead.hpp" #include "irods/rsDataObjRename.hpp" -#include "irods/rsDataObjTrim.hpp" -#include "irods/rsCollCreate.hpp" -#include "irods/rsRmColl.hpp" -#include "irods/rsPhyPathReg.hpp" -#include "irods/rsObjStat.hpp" +#include "irods/rsDataObjRepl.hpp" #include "irods/rsDataObjRsync.hpp" -#include "irods/rsOpenCollection.hpp" -#include "irods/rsReadCollection.hpp" -#include "irods/rsCloseCollection.hpp" +#include "irods/rsDataObjTrim.hpp" +#include "irods/rsDataObjUnlink.hpp" +#include "irods/rsDataObjWrite.hpp" #include "irods/rsExecCmd.hpp" -#include "irods/rsCollRepl.hpp" #include "irods/rsModDataObjMeta.hpp" -#include "irods/rsStructFileExtAndReg.hpp" #include "irods/rsModDataObjMeta.hpp" +#include "irods/rsObjStat.hpp" +#include "irods/rsOpenCollection.hpp" +#include "irods/rsPhyPathReg.hpp" +#include "irods/rsReadCollection.hpp" +#include "irods/rsRmColl.hpp" #include "irods/rsStructFileBundle.hpp" -#include "irods/irods_at_scope_exit.hpp" -#include "irods/key_value_proxy.hpp" +#include "irods/rsStructFileExtAndReg.hpp" +#include "irods/rs_replica_truncate.hpp" -#include #include +#include #include #include @@ -3266,3 +3268,117 @@ msiTarFileCreate( msParam_t *inpParam1, msParam_t *inpParam2, msParam_t *inpPara return rei->status; } + +/// Truncate a replica for the specified data object to the specified size. +/// +/// This microservice uses the #rs_replica_truncate API. +/// +/// \param[in] _inp \parblock +/// MsParam of type DataObjInp or a string taken as a msKeyValStr. msKeyValStr has the following general form: +/// keyword=[value][++++keyword=[value]]* +/// +/// The following keywords are required: +/// - "objPath": The full logical path to the target data object. Note: If a value is provided with no keyword, the +/// provided value will be assumed to be the object path. In other words, the input "/tempZone/home/alice/foo" is +/// equivalent to "objPath=/tempZone/home/alice/foo". +/// - "dataSize": The desired size of the replica after truncating. +/// +/// The following keywords are valid, but not required: +/// - "replNum": The replica number of the replica to truncate. +/// - "rescName": The name of the resource with the replica to truncate. Must be a root resource. +/// - "defRescName": The default resource to target in the absence of any other inputs or policy. +/// - "irodsAdmin": Flag indicating that the operation is to be performed with elevated privileges. No value +/// required. +/// \endparblock +/// \param[out] _out \parblock +/// MsParam of type string representing a JSON structure with the following form: +/// \code{.js} +/// { +/// // Resource hierarchy of the selected replica for truncate. If an error occurs before hierarchy resolution is +/// // completed, a null value will be here instead of a string. +/// "resource_hierarchy": , +/// // Replica number of the selected replica for truncate. If an error occurs before hierarchy resolution is +/// // completed, a null value will be here instead of an integer. +/// "replica_number": , +/// // A string containing any relevant message the server may wish to send to the user (including error messages). +/// // This value will always be a string, even if it is empty. +/// "message": +/// } +/// \endcode +/// The JSON microservices can be used to parse and examine the output. See #msi_json_parse. +/// \endparblock +/// \param[in,out] rei - The RuleExecInfo structure that is automatically handled by the rule engine. The user does not +/// include rei as a parameter in the rule invocation. +/// +/// \usage \parblock +/// \code{.py} +/// msi_replica_truncate("/tempZone/home/alice/science.txt++++dataSize=100++++replNum=0", *json_output_string); +/// \endcode +/// \endparblock +/// +/// \return An integer representing an iRODS error code, or 0. +/// \retval 0 on success. +/// \retval <0 on failure; an iRODS error code. +/// +/// \since 4.3.2 +auto msi_replica_truncate(MsParam* _inp, MsParam* _out, RuleExecInfo* _rei) -> int +{ + if (nullptr == _rei || nullptr == _rei->rsComm) { + msi_log::error("{}: Input rei or rei->rsComm is nullptr.", __func__); + return SYS_INTERNAL_NULL_INPUT_ERR; + } + + RsComm* comm = _rei->rsComm; + + DataObjInp data_obj_inp{}; + DataObjInp* data_obj_inp_ptr = &data_obj_inp; + const auto clear_kvp = irods::at_scope_exit{[&data_obj_inp] { clearKeyVal(&data_obj_inp.condInput); }}; + + // Initialize the dataSize to -1 to ensure that the caller sets the dataSize to something valid. + data_obj_inp.dataSize = -1; + + if (0 == std::strcmp(_inp->type, STR_MS_T)) { + char* bad_keyword_output_string = nullptr; + const auto free_bad_keyword_output_string = + // NOLINTNEXTLINE(cppcoreguidelines-no-malloc, cppcoreguidelines-owning-memory) + irods::at_scope_exit{[&bad_keyword_output_string] { std::free(bad_keyword_output_string); }}; + + // DATA_SIZE_FLAGS is not a typo. + const auto valid_keywords = + // NOLINTNEXTLINE(hicpp-signed-bitwise) + OBJ_PATH_FLAG | DATA_SIZE_FLAGS | REPL_NUM_FLAG | RESC_NAME_FLAG | DEF_RESC_NAME_FLAG | ADMIN_FLAG; + _rei->status = + parseMsKeyValStrForDataObjInp(_inp, &data_obj_inp, OBJ_PATH_KW, valid_keywords, &bad_keyword_output_string); + if (_rei->status < 0 && bad_keyword_output_string) { + const auto msg = fmt::format( + "{}: Input keyword [{}] error. error code: [{}]", __func__, bad_keyword_output_string, _rei->status); + msi_log::error(msg); + addRErrorMsg(&comm->rError, _rei->status, msg.c_str()); + return _rei->status; + } + } + else { + _rei->status = parseMspForDataObjInp(_inp, &data_obj_inp, &data_obj_inp_ptr, 0); + } + + if (_rei->status < 0) { + const auto msg = fmt::format( + "{}: Error occurred while parsing microservice parameters. error code: [{}]", __func__, _rei->status); + msi_log::error(msg); + addRErrorMsg(&comm->rError, _rei->status, msg.c_str()); + return _rei->status; + } + + char* output_string = nullptr; + // NOLINTNEXTLINE(cppcoreguidelines-no-malloc, cppcoreguidelines-owning-memory) + const auto free_output = irods::at_scope_exit{[&output_string] { std::free(output_string); }}; + _rei->status = rs_replica_truncate(comm, &data_obj_inp, &output_string); + if (_rei->status < 0) { + const auto msg = fmt::format( + "{}: rs_replica_truncate failed for [{}]. error code: [{}]", __func__, data_obj_inp.objPath, _rei->status); + msi_log::error(msg); + addRErrorMsg(&comm->rError, _rei->status, msg.c_str()); + } + fillStrInMsParam(_out, output_string); + return _rei->status; +} // msi_replica_truncate