diff --git a/.github/workflows/build-irods-centos.yml b/.github/workflows/build-irods-centos.yml index 0941a5a575..b9c5bea198 100644 --- a/.github/workflows/build-irods-centos.yml +++ b/.github/workflows/build-irods-centos.yml @@ -18,10 +18,11 @@ jobs: yum -y install gcc g++ libstdc++-static make rpm-build bzip2-devel curl-devel fakeroot openssl-devel pam-devel python-devel unixODBC unixODBC-devel zlib-devel python36-distro - name: Install Flex and Bison run: | - yum -y install centos-release-scl-rh + #yum -y install centos-release-scl-rh rpm --import https://www.cert.org/forensics/repository/forensics-expires-2022-04-03.asc - wget https://forensics.cert.org/cert-forensics-tools-release-el7.rpm - rpm -i ./cert-forensics-tools-release-el7.rpm + #wget https://forensics.cert.org/cert-forensics-tools-release-el7.rpm + #rpm -i ./cert-forensics-tools-release-el7.rpm + yum -y install https://forensics.cert.org/cert-forensics-tools-release-el7.rpm yum -y install flex bison - name: Install iRODS Externals run: | diff --git a/lib/api/include/irods/genquery2.h b/lib/api/include/irods/genquery2.h index 9ab142bc41..19f915f424 100644 --- a/lib/api/include/irods/genquery2.h +++ b/lib/api/include/irods/genquery2.h @@ -7,20 +7,30 @@ struct RcComm; /// The input data type used to invoke #rc_genquery2. /// +/// \note This data structure is part of an experimental API endpoint and may change in the future. +/// /// \since 4.3.2 typedef struct GenQuery2Input // NOLINT(modernize-use-using) { - /// TODO + /// The GenQuery2 query string to execute. + /// + /// This member variable MUST be non-empty. /// /// \since 4.3.2 char* query_string; - /// TODO + /// The zone to execute the query string against. + /// + /// This member variable is allowed to be set to NULL. /// /// \since 4.3.2 char* zone; - /// TODO + /// Controls whether the SQL derived from the query string is + /// executed or returned to the caller. + /// + /// When set to 0, the generated SQL will be executed. + /// When set to 1, the generated SQL will be returned to the caller. The SQL will not be executed. /// /// \since 4.3.2 int sql_only; @@ -33,7 +43,70 @@ typedef struct GenQuery2Input // NOLINT(modernize-use-using) extern "C" { #endif -/// TODO +/// Query the catalog using the GenQuery2 parser. +/// +/// \note This is an experimental API endpoint and may change in the future. +/// +/// The GenQuery2 parser supports the following: +/// - Enforces the iRODS permission model +/// - Logical AND, OR, and NOT +/// - Grouping via parentheses +/// - SQL CAST +/// - SQL GROUP BY +/// - SQL aggregate functions (e.g. count, sum, avg, etc) +/// - Per-column sorting via ORDER BY [ASC|DESC] +/// - SQL FETCH FIRST N ROWS ONLY (LIMIT offered as an alias) +/// - Metadata queries involving different iRODS entities (i.e. data objects, collections, users, and resources) +/// - Operators: =, !=, <, <=, >, >=, LIKE, BETWEEN, IS [NOT] NULL +/// - SQL keywords are case-insensitive +/// - Federation +/// - Escaping of single quotes +/// - Bytes encoded as hexadecimal (e.g. \x21) +/// +/// Limitations: +/// - Groups are not yet fully supported +/// - Cannot resolve tickets to data objects and collections using a single query +/// - Integer values must be treated as strings, except when used for OFFSET, LIMIT, FETCH FIRST N ROWS ONLY +/// +/// \param[in] _comm A pointer to a RcComm. +/// \param[in] _input A pointer to a GenQuery2Input. +/// \param[in,out] _output \parblock A pointer that will hold the results of the operation. +/// On success, the pointer will either hold a JSON string or a string representing the SQL derived from +/// the GenQuery2 query string. See GenQuery2Input::sql_only for details. +/// +/// The string is always heap-allocated and must be free'd by the caller using free(). +/// +/// On failure, the pointer will be NULL. +/// \endparblock +/// +/// \return An integer. +/// \retval 0 On success. +/// \retval <0 On failure. +/// +/// \b Example +/// \code{.cpp} +/// RcComm* comm = // Our iRODS connection. +/// +/// // Configure the input object for the API call. +/// struct GenQuery2Input input; +/// memset(&input, 0, sizeof(struct GenQuery2Input)); +/// +/// // This is the query that will be executed (i.e. input.sql_only is set to 0). +/// input.query_string = strdup("select COLL_NAME, DATA_NAME where RESC_ID = '10016'"); +/// +/// char* output = NULL; +/// const int ec = rc_genquery2(comm, &input, &output); +/// +/// // Handle error. +/// if (ec < 0) { +/// free(input.query_string); +/// return; +/// } +/// +/// // At this point, "output" should represent a JSON string. +/// // Parse the string as JSON and inspect the results. +/// // If "input.sql_only" was set to 1, "output" would hold the SQL derived from the GenQuery2 syntax. +/// \endcode /// /// \since 4.3.2 int rc_genquery2(struct RcComm* _comm, struct GenQuery2Input* _input, char** _output); diff --git a/lib/core/include/irods/irods_configuration_keywords.hpp b/lib/core/include/irods/irods_configuration_keywords.hpp index 7c1cb0631e..35eb6ad789 100644 --- a/lib/core/include/irods/irods_configuration_keywords.hpp +++ b/lib/core/include/irods/irods_configuration_keywords.hpp @@ -49,6 +49,7 @@ namespace irods extern const char* const KW_CFG_LOG_LEVEL_CATEGORY_AGENT_FACTORY; extern const char* const KW_CFG_LOG_LEVEL_CATEGORY_AGENT; extern const char* const KW_CFG_LOG_LEVEL_CATEGORY_DELAY_SERVER; + extern const char* const KW_CFG_LOG_LEVEL_CATEGORY_GENQUERY2; extern const char* const KW_CFG_LOG_LEVEL_CATEGORY_RESOURCE; extern const char* const KW_CFG_LOG_LEVEL_CATEGORY_DATABASE; extern const char* const KW_CFG_LOG_LEVEL_CATEGORY_AUTHENTICATION; diff --git a/lib/core/include/irods/irods_logger.hpp b/lib/core/include/irods/irods_logger.hpp index 5d3d1940ce..b935ee7365 100644 --- a/lib/core/include/irods/irods_logger.hpp +++ b/lib/core/include/irods/irods_logger.hpp @@ -87,6 +87,7 @@ namespace irods::experimental::log struct agent_factory {}; struct agent {}; struct delay_server {}; + struct genquery2 {}; struct resource {}; struct database {}; struct authentication {}; @@ -141,6 +142,7 @@ namespace irods::experimental::log using agent_factory = logger; using agent = logger; using delay_server = logger; + using genquery2 = logger; using resource = logger; using database = logger; using authentication = logger; @@ -834,6 +836,15 @@ namespace irods::experimental::log friend class logger; }; // class logger_config + template <> + class logger_config + { + static constexpr const char* const name = "genquery2"; + inline static level level = level::info; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + + friend class logger; + }; // class logger_config + template <> class logger_config { diff --git a/lib/core/src/irods_configuration_keywords.cpp b/lib/core/src/irods_configuration_keywords.cpp index bd23fd5163..05f04e0774 100644 --- a/lib/core/src/irods_configuration_keywords.cpp +++ b/lib/core/src/irods_configuration_keywords.cpp @@ -47,6 +47,7 @@ namespace irods const char* const KW_CFG_LOG_LEVEL_CATEGORY_AGENT_FACTORY{"agent_factory"}; const char* const KW_CFG_LOG_LEVEL_CATEGORY_AGENT{"agent"}; const char* const KW_CFG_LOG_LEVEL_CATEGORY_DELAY_SERVER{"delay_server"}; + const char* const KW_CFG_LOG_LEVEL_CATEGORY_GENQUERY2{"genquery2"}; const char* const KW_CFG_LOG_LEVEL_CATEGORY_RESOURCE{"resource"}; const char* const KW_CFG_LOG_LEVEL_CATEGORY_DATABASE{"database"}; const char* const KW_CFG_LOG_LEVEL_CATEGORY_AUTHENTICATION{"authentication"}; diff --git a/scripts/core_tests_list.json b/scripts/core_tests_list.json index a82fb76d74..d15cc51865 100644 --- a/scripts/core_tests_list.json +++ b/scripts/core_tests_list.json @@ -68,6 +68,7 @@ "test_ipwd", "test_iqmod", "test_iqstat", + "test_iquery", "test_iquest.Test_Iquest", "test_iquest.test_iquest_logical_or_operator_with_data_resc_hier", "test_iquest.test_iquest_with_data_resc_hier", diff --git a/scripts/irods/test/test_genquery2_microservices.py b/scripts/irods/test/test_genquery2_microservices.py index a122b1e1d5..5321884210 100644 --- a/scripts/irods/test/test_genquery2_microservices.py +++ b/scripts/irods/test/test_genquery2_microservices.py @@ -1,10 +1,7 @@ -import os -import sys import textwrap import unittest from . import session -from .. import lib from ..configuration import IrodsConfig rodsadmins = [('otherrods', 'rods')] diff --git a/scripts/irods/test/test_iquery.py b/scripts/irods/test/test_iquery.py new file mode 100644 index 0000000000..10d11f7ce5 --- /dev/null +++ b/scripts/irods/test/test_iquery.py @@ -0,0 +1,24 @@ +import json +import unittest + +from . import session +from .. import lib + +rodsadmins = [('otherrods', 'rods')] +rodsusers = [('alice', 'apass')] + +class Test_IQuery(session.make_sessions_mixin(rodsadmins, rodsusers), unittest.TestCase): + + def setUp(self): + super(Test_IQuery, self).setUp() + + self.admin = self.admin_sessions[0] + self.user = self.user_sessions[0] + + def tearDown(self): + super(Test_IQuery, self).tearDown() + + def test_iquery_can_run_a_query__issue_7570(self): + json_string = json.dumps([[self.user.session_collection]]) + self.user.assert_icommand( + ['iquery', f"select COLL_NAME where COLL_NAME = '{self.user.session_collection}'"], 'STDOUT', [json_string]) diff --git a/server/api/include/irods/rs_genquery2.hpp b/server/api/include/irods/rs_genquery2.hpp index b1fc4e53c4..92d88988d7 100644 --- a/server/api/include/irods/rs_genquery2.hpp +++ b/server/api/include/irods/rs_genquery2.hpp @@ -7,7 +7,70 @@ struct RsComm; -/// TODO +/// Query the catalog using the GenQuery2 parser. +/// +/// \note This is an experimental API endpoint and may change in the future. +/// +/// The GenQuery2 parser supports the following: +/// - Enforces the iRODS permission model +/// - Logical AND, OR, and NOT +/// - Grouping via parentheses +/// - SQL CAST +/// - SQL GROUP BY +/// - SQL aggregate functions (e.g. count, sum, avg, etc) +/// - Per-column sorting via ORDER BY [ASC|DESC] +/// - SQL FETCH FIRST N ROWS ONLY (LIMIT offered as an alias) +/// - Metadata queries involving different iRODS entities (i.e. data objects, collections, users, and resources) +/// - Operators: =, !=, <, <=, >, >=, LIKE, BETWEEN, IS [NOT] NULL +/// - SQL keywords are case-insensitive +/// - Federation +/// - Escaping of single quotes +/// - Bytes encoded as hexadecimal (e.g. \x21) +/// +/// Limitations: +/// - Groups are not yet fully supported +/// - Cannot resolve tickets to data objects and collections using a single query +/// - Integer values must be treated as strings, except when used for OFFSET, LIMIT, FETCH FIRST N ROWS ONLY +/// +/// \param[in] _comm A pointer to a RsComm. +/// \param[in] _input A pointer to a GenQuery2Input. +/// \param[in,out] _output \parblock A pointer that will hold the results of the operation. +/// On success, the pointer will either hold a JSON string or a string representing the SQL derived from +/// the GenQuery2 query string. See GenQuery2Input::sql_only for details. +/// +/// The string is always heap-allocated and must be free'd by the caller using free(). +/// +/// On failure, the pointer will be NULL. +/// \endparblock +/// +/// \return An integer. +/// \retval 0 On success. +/// \retval <0 On failure. +/// +/// \b Example +/// \code{.cpp} +/// RsComm* comm = // Our iRODS connection. +/// +/// // Configure the input object for the API call. +/// struct GenQuery2Input input; +/// memset(&input, 0, sizeof(struct GenQuery2Input)); +/// +/// // This is the query that will be executed (i.e. input.sql_only is set to 0). +/// input.query_string = strdup("select COLL_NAME, DATA_NAME where RESC_ID = '10016'"); +/// +/// char* output = NULL; +/// const int ec = rc_genquery2(comm, &input, &output); +/// +/// // Handle error. +/// if (ec < 0) { +/// free(input.query_string); +/// return; +/// } +/// +/// // At this point, "output" should represent a JSON string. +/// // Parse the string as JSON and inspect the results. +/// // If "input.sql_only" was set to 1, "output" would hold the SQL derived from the GenQuery2 syntax. +/// \endcode /// /// \since 4.3.2 int rs_genquery2(RsComm* _comm, GenQuery2Input* _input, char** _output); diff --git a/server/api/src/rs_genquery2.cpp b/server/api/src/rs_genquery2.cpp index bc39b9e9c2..d2a19b4b54 100644 --- a/server/api/src/rs_genquery2.cpp +++ b/server/api/src/rs_genquery2.cpp @@ -8,22 +8,17 @@ #include "irods/genquery2_sql.hpp" #include "irods/apiHandler.hpp" -//#include "irods/catalog.hpp" // Requires linking against libnanodbc.so #include "irods/irods_logger.hpp" #include "irods/irods_rs_comm_query.hpp" #include "irods/irods_server_properties.hpp" #include "irods/irods_version.h" -//#include "irods/procApiRequest.h" #include "irods/rodsConnect.h" #include "irods/rodsDef.h" #include "irods/rodsErrorTable.h" #include "irods/icatHighLevelRoutines.hpp" -//#include -//#include #include -//#include // For std::malloc. #include // For strdup. #include #include @@ -52,6 +47,8 @@ auto rs_genquery2(RsComm* _comm, GenQuery2Input* _input, char** _output) -> int log_api::trace("{}: Received: query_string=[{}], zone=[nullptr]", __func__, _input->query_string); } + *_output = nullptr; + rodsServerHost* host_info{}; if (const auto ec = getAndConnRcatHost(_comm, PRIMARY_RCAT, _input->zone, &host_info); ec < 0) { diff --git a/server/core/src/irods_api_calling_functions.cpp b/server/core/src/irods_api_calling_functions.cpp index 75ff9d848a..ed25ab1621 100644 --- a/server/core/src/irods_api_calling_functions.cpp +++ b/server/core/src/irods_api_calling_functions.cpp @@ -1,4 +1,3 @@ -#include "irods/genquery2.h" #include "irods/rcConnect.h" #include "irods/apiHeaderAll.h" #include "irods/apiHandler.hpp" diff --git a/server/core/src/rodsAgent.cpp b/server/core/src/rodsAgent.cpp index 34837161ac..bf24dceaed 100644 --- a/server/core/src/rodsAgent.cpp +++ b/server/core/src/rodsAgent.cpp @@ -314,6 +314,7 @@ void set_log_levels_for_all_log_categories() log_ns::agent::set_level(log_ns::get_level_from_config(irods::KW_CFG_LOG_LEVEL_CATEGORY_AGENT)); log_ns::legacy::set_level(log_ns::get_level_from_config(irods::KW_CFG_LOG_LEVEL_CATEGORY_LEGACY)); log_ns::resource::set_level(log_ns::get_level_from_config(irods::KW_CFG_LOG_LEVEL_CATEGORY_RESOURCE)); + log_ns::genquery2::set_level(log_ns::get_level_from_config(irods::KW_CFG_LOG_LEVEL_CATEGORY_GENQUERY2)); log_ns::database::set_level(log_ns::get_level_from_config(irods::KW_CFG_LOG_LEVEL_CATEGORY_DATABASE)); log_ns::authentication::set_level(log_ns::get_level_from_config(irods::KW_CFG_LOG_LEVEL_CATEGORY_AUTHENTICATION)); log_ns::api::set_level(log_ns::get_level_from_config(irods::KW_CFG_LOG_LEVEL_CATEGORY_API)); diff --git a/server/genquery2/src/genquery2_sql.cpp b/server/genquery2/src/genquery2_sql.cpp index 848b0f7824..e196d02672 100644 --- a/server/genquery2/src/genquery2_sql.cpp +++ b/server/genquery2/src/genquery2_sql.cpp @@ -7,7 +7,6 @@ #include "irods/irods_at_scope_exit.hpp" #include "irods/irods_logger.hpp" -#include "irods/irods_version.h" #include #include @@ -23,38 +22,6 @@ #include #include -#if IRODS_VERSION_INTEGER < 4003001 -namespace irods::experimental -{ - struct genquery2 - { - }; - - template <> - class log::logger_config - { - static constexpr const char* name = "genquery2"; - inline static log::level level = log::level::info; - friend class logger; - }; // class logger_config -} // namespace irods::experimental -#else -namespace irods::experimental::log -{ - struct genquery2 - { - }; - - template <> - class logger_config - { - static constexpr const char* name = "genquery2"; - inline static level level = level::info; - friend class logger; - }; // class logger_config -} // namespace irods::experimental::log -#endif - namespace { namespace gq = irods::experimental::api::genquery; @@ -69,11 +36,7 @@ namespace using vertices_size_type = boost::graph_traits::vertices_size_type; using edge_type = std::pair; -#if IRODS_VERSION_INTEGER < 4003001 - using log_gq = irods::experimental::log::logger; -#else - using log_gq = irods::experimental::log::logger; -#endif + using log_gq = irods::experimental::log::genquery2; // clang-format on struct gq_state @@ -139,7 +102,6 @@ namespace throw std::invalid_argument{fmt::format("table [{}] not supported", _table_name)}; } // to_index - // clang-format on // clang-format off constexpr auto table_edges = std::to_array({ @@ -1086,8 +1048,6 @@ namespace irods::experimental::api::genquery auto to_sql(const select& _select, const options& _opts) -> std::tuple> { try { - log_gq::set_level(irods::experimental::log::get_level_from_config("genquery2")); - gq_state state; log_gq::trace("### PHASE 1: Gather"); diff --git a/server/icat/include/irods/icatHighLevelRoutines.hpp b/server/icat/include/irods/icatHighLevelRoutines.hpp index 0cc3069980..13a69191b7 100644 --- a/server/icat/include/irods/icatHighLevelRoutines.hpp +++ b/server/icat/include/irods/icatHighLevelRoutines.hpp @@ -399,9 +399,9 @@ auto chl_check_auth_credentials(RsComm& _comm, /// Triggers policy associated with database operations. /// /// \param[in] _comm The communication object. -/// \param[in] _sql TODO -/// \param[in] _values TODO -/// \param[in,out] _output TODO +/// \param[in] _sql The SQL, generated by the GenQuery2 parser, to execute. +/// \param[in] _values The list of values to bind to the query. +/// \param[in,out] _output The pointer that will hold the results of the query. /// /// \return An integer. /// \retval 0 On success. diff --git a/server/re/src/msi_genquery2.cpp b/server/re/src/msi_genquery2.cpp index 88b803b42f..c223866ff3 100644 --- a/server/re/src/msi_genquery2.cpp +++ b/server/re/src/msi_genquery2.cpp @@ -5,8 +5,6 @@ #include "irods/msParam.h" #include "irods/irods_re_structs.hpp" #include "irods/irods_logger.hpp" -//#include "irods/irods_plugin_context.hpp" -//#include "irods/irods_re_plugin.hpp" #include "irods/irods_state_table.h" #include "irods/rodsError.h" #include "irods/rodsErrorTable.h" @@ -41,6 +39,19 @@ namespace std::vector gq2_context; } // anonymous namespace +/// Queries the catalog using the GenQuery2 parser. +/// +/// \note This microservice is experimental and may change in the future. +/// +/// \param[in,out] _handle The output parameter that will hold the handle to the resultset. +/// \param[in] _query_string The GenQuery2 string to execute. +/// \param[in] _rei This parameter is special and should be ignored. +/// +/// \return An integer. +/// \retval 0 On success. +/// \retval <0 On failure. +/// +/// \since 4.3.2 auto msi_genquery2_execute(MsParam* _handle, MsParam* _query_string, RuleExecInfo* _rei) -> int { log_msi::trace(__func__); @@ -75,6 +86,34 @@ auto msi_genquery2_execute(MsParam* _handle, MsParam* _query_string, RuleExecInf return 0; } // msi_genquery2_execute +/// Moves the cursor forward by one row. +/// +/// \note This microservice is experimental and may change in the future. +/// +/// \param[in] _handle The GenQuery2 handle. +/// \param[in] _rei This parameter is special and should be ignored. +/// +/// \return An integer. +/// \retval 0 If data can be read from the new row. +/// \retval <0 If the end of the resultset has been reached. +/// +/// \b Example +/// \code{.py} +/// # Execute a query. The results are stored in the Rule Engine Plugin. +/// msi_genquery2_execute(*handle, "select COLL_NAME, DATA_NAME order by DATA_NAME desc limit 1"); +/// +/// # Iterate over the results. +/// while (errorcode(genquery2_next_row(*handle)) == 0) { +/// genquery2_column(*handle, '0', *coll_name); # Copy the COLL_NAME into *coll_name. +/// genquery2_column(*handle, '1', *data_name); # Copy the DATA_NAME into *data_name. +/// writeLine("stdout", "logical path => [*coll_name/*data_name]"); +/// } +/// +/// # Free any resources used. This is handled for you when the agent is shut down as well. +/// genquery2_free(*handle); +/// \endcode +/// +/// \since 4.3.2 auto msi_genquery2_next_row(MsParam* _handle, RuleExecInfo* _rei) -> int { log_msi::trace(__func__); @@ -106,10 +145,6 @@ auto msi_genquery2_next_row(MsParam* _handle, RuleExecInfo* _rei) -> int log_msi::trace("{}: Skipping increment of row position [current_row=[{}]]. Returning 1.", __func__, ctx.current_row); - // TODO Update this. - // We must return ERROR(stop_code, "") to trigger correct usage of msi_genquery2_next_row(). - // Otherwise, the NREP can loop forever. Ultimately, this means we aren't allowed to return - // CODE(stop_code) to signal there is no new row available. return GENQUERY2_END_OF_RESULTSET; } catch (const irods::exception& e) { @@ -124,6 +159,20 @@ auto msi_genquery2_next_row(MsParam* _handle, RuleExecInfo* _rei) -> int return 0; } // msi_genquery2_next_row +/// Reads the value of a column from a row within a GenQuery2 resultset. +/// +/// \note This microservice is experimental and may change in the future. +/// +/// \param[in] _handle The GenQuery2 handle. +/// \param[in] _column_index The index of the column to read. The index must be passed as a string. +/// \param[in,out] _column_value The variable to write the value of the column to. +/// \param[in] _rei This parameter is special and should be ignored. +/// +/// \return An integer. +/// \retval 0 On success. +/// \retval <0 On failure. +/// +/// \since 4.3.2 auto msi_genquery2_column(MsParam* _handle, MsParam* _column_index, MsParam* _column_value, RuleExecInfo* _rei) -> int { log_msi::trace(__func__); @@ -172,6 +221,21 @@ auto msi_genquery2_column(MsParam* _handle, MsParam* _column_index, MsParam* _co return 0; } // msi_genquery2_column +/// Frees all resources associated with a GenQuery2 handle. +/// +/// \note This microservice is experimental and may change in the future. +/// +/// Users are expected to call this microservice when use of the GenQuery2 isn't needed any longer. +/// Failing to follow this rule can result in memory leaks. +/// +/// \param[in] _handle The GenQuery2 handle. +/// \param[in] _rei This parameter is special and should be ignored. +/// +/// \return An integer. +/// \retval 0 On success. +/// \retval <0 On failure. +/// +/// \since 4.3.2 auto msi_genquery2_free(MsParam* _handle, RuleExecInfo* _rei) -> int { log_msi::trace(__func__);