squash. Zone report change, Env file validation, schema updates, WIP …

…- removal of assumptions about file locations and directories.

lib/core/src/irods_environment_properties.cpp

@@ -30,6 +30,8 @@ namespace irods {
         // that
         std::string json_file;
         try {
+            // TODO What does the server need that lives under the home directory?
+            // What does the client need that lives under the home directory?
             json_file = get_irods_home_directory().string();
         } catch (const irods::exception& e) {
             irods::log(e);

lib/core/src/irods_logger.cpp

@@ -54,7 +54,7 @@ namespace irods::experimental::log
       public:
         test_mode_ipc_sink()
             : mutex_{ipc::open_or_create, shm_name}
-            , file_{(irods::get_irods_home_directory() / "log" / "test_mode_output.log").string(), std::ios_base::app}
+            , file_{(irods::get_irods_home_directory() / "log" / "test_mode_output.log").string(), std::ios_base::app} // TODO Defined by config file.
             , owner_pid_{getpid()}
         {
         }

schemas/configuration/v4/server_config.json

@@ -15,7 +15,7 @@
         "msiExecCmd_bin_directory": {"type": "string"},
         "plugin_directory": {"type": "string"},
         "home_directory": {"type": "string"},
-        "environment_file": {"type": "string"},
+        "version_file": {"type": "string"},
 
         "directories": {
             "type": "object",
@@ -26,7 +26,7 @@
                 "plugins": {"type": "string"},
                 "stacktraces": {"type": "string"}
             },
-            "required": [
+            "_required": [
                 "agent_info",
                 "home",
                 "msiExecCmd_bin",
@@ -291,6 +291,11 @@
         "ips_data_directory",
         "stacktrace_directory",
         "json_schema_file",
+        "environment_json_schema_file",
+        "msiExecCmd_bin_directory",
+        "plugin_directory",
+        "home_directory",
+        "version_file",
         "graceful_shutdown_timeout_in_seconds",
         "advanced_settings",
         "catalog_provider_hosts",

server/api/src/rsExecCmd.cpp

@@ -3,6 +3,7 @@
 /* This is script-generated code (for the most part).  */
 /* See dataObjRead.h for a description of this API call.*/
 
+#include "irods/irods_server_properties.hpp"
 #include <sys/types.h>
 #ifndef windows_platform
 #include <sys/wait.h>
@@ -26,6 +27,9 @@
 #include "irods/irods_default_paths.hpp"
 
 #include <boost/thread/mutex.hpp>
+
+#include <filesystem>
+
 boost::mutex ExecCmdMutex;
 int initExecCmdMutex() {
     return 0;
@@ -369,7 +373,8 @@ execCmd( execCmd_t *execCmdInp, int stdOutFd, int stdErrFd ) {
     char *av[LONG_NAME_LEN];
     int status;
 
-    const auto cmd_path = irods::get_irods_home_directory().append("msiExecCmd_bin").append(execCmdInp->cmd);
+    const std::filesystem::path msiExecCmd_bin_dir = irods::get_server_property<std::string>("msiExecCmd_bin_directory");
+    const auto cmd_path = msiExecCmd_bin_dir / execCmdInp->cmd;
     std::strncpy(cmdPath, cmd_path.c_str(), LONG_NAME_LEN);
     rodsLog( LOG_NOTICE, "execCmd:%s argv:%s", cmdPath, execCmdInp->cmdArgv );
     initCmdArg( av, execCmdInp->cmdArgv, cmdPath );

server/api/src/rsServerReport.cpp

@@ -251,6 +251,7 @@ irods::error get_host_system_information(json& _host_system_information)
         _host_system_information["uname"] = nullptr;
     }
 
+#if 0
     std::vector<std::string> args;
     args.push_back( "os_distribution_name" );
     std::string os_distribution_name;
@@ -272,6 +273,12 @@ irods::error get_host_system_information(json& _host_system_information)
         irods::log( PASS( ret ) );
         _host_system_information["os_distribution_version"] = nullptr;
     }
+#else
+    // Remove the dependency on Python. The uname property contains this information.
+    // It requires parsing to extract it, but the information is already available.
+    _host_system_information["os_distribution_name"] = nullptr;
+    _host_system_information["os_distribution_version"] = nullptr;
+#endif
 
     return SUCCESS();
 } // get_host_system_information
@@ -432,19 +439,11 @@ irods::error get_config_dir( json& _cfg_dir )
 
 irods::error load_version_file( json& _version )
 {
-    // =-=-=-=-=-=-=-
     // if json file exists, simply load that
-    fs::path version_file;
-    try {
-        version_file = irods::get_irods_home_directory();
-    } catch (const irods::exception& e) {
-        irods::log(e);
-        return ERROR(-1, "failed to get irods home directory");
-    }
-    version_file.append("version.json");
+    const auto version_file = irods::get_server_property<std::string>("version_file");
 
-    if ( fs::exists( version_file ) ) {
-        return load_json_file(version_file.generic_string(), _version);
+    if (fs::exists(version_file)) {
+        return load_json_file(version_file, _version);
     }
 
     return SUCCESS();

server/core/src/irods_resource_manager.cpp

@@ -41,7 +41,7 @@ namespace irods
     // =-=-=-=-=-=-=-
     /// @brief special resource for local file system operations only
     const std::string LOCAL_USE_ONLY_RESOURCE( "LOCAL_USE_ONLY_RESOURCE" );
-    const std::string LOCAL_USE_ONLY_RESOURCE_VAULT(
+    const std::string LOCAL_USE_ONLY_RESOURCE_VAULT( // TODO This does not appear to be used.
         (get_irods_home_directory() / "LOCAL_USE_ONLY_RESOURCE_VAULT").string());
     const std::string LOCAL_USE_ONLY_RESOURCE_TYPE( "unixfilesystem" );
 

server/core/src/miscServerFunct.cpp

@@ -3002,6 +3002,17 @@ irods::error get_script_output_single_line(
             return ERROR(RESOLUTION_ERROR, fmt::format("Could not resolve {} to an executable", script_language));
         }
 
+        // TODO Consider removing entirely. Currently, only the following things invoke this function:
+        // - server/api/src/rsServerReport.cpp:257:    ret = get_script_output_single_line( "python3", "system_identification.py", args, os_distribution_name );
+        // - server/api/src/rsServerReport.cpp:268:    ret = get_script_output_single_line( "python3", "system_identification.py", args, os_distribution_version );
+        // - server/main_server/src/main_server_control_plane.cpp:328:        irods::error ret = get_script_output_single_line("python3", "pid_age.py", args, pid_age);
+        //
+        // Have to support zone reports.
+        // The use of system_identification.py can be replaced with parsing of /etc/os-release
+        // and/or the POSIX uname() function. uname() will not return the same information, but
+        // maybe that's okay? Parsing /etc/os-release would give us more detailed OS information.
+        //
+        // The control plane is going away, so there's no need for pid_age.py to exist.
         auto script_path = fmt::format("{}/scripts/{}", irods::get_irods_home_directory().c_str(), script_name);
         if (!std::filesystem::exists(script_path)) {
             return ERROR(RESOLUTION_ERROR, fmt::format("Script file not found! {}", script_path));

server/core/src/procLog.cpp

@@ -27,6 +27,7 @@ initAndClearProcLog() {
 
 int
 initProcLog() {
+    // TODO No longer used. Remove this function
     const auto proc_log_dir = irods::get_irods_home_directory() / "log" / "proc";
     std::strncpy(ProcLogDir, proc_log_dir.c_str(), MAX_NAME_LEN);
     return 0;

server/main_server/src/agent_main.cpp

@@ -315,6 +315,21 @@ int main(int _argc, char* _argv[])
             return 1;
         }
 
+        // Load server side pluggable api entries.
+        irods::api_entry_table&  RsApiTable   = irods::get_server_api_table();
+        irods::pack_entry_table& ApiPackTable = irods::get_pack_table();
+        if (const auto ret = irods::init_api_table(RsApiTable, ApiPackTable, false); !ret.ok()) {
+            log_af::error("{}: init_api_table error for server: {}", __func__, ret.user_result());
+            return 1;
+        }
+
+        // Load client side pluggable api entries.
+        irods::api_entry_table& RcApiTable = irods::get_client_api_table();
+        if (const auto ret = irods::init_api_table(RcApiTable, ApiPackTable, false); !ret.ok()) {
+            log_af::error("{}: init_api_table error for server: {}", __func__, ret.user_result());
+            return 1;
+        }
+
         // Enter parent process main loop.
         // 
         // This process should never introduce threads. Everything it cares about must be handled
@@ -426,7 +441,7 @@ int main(int _argc, char* _argv[])
         if (0 == g_terminate_graceful) {
             // Instruct all agents to shutdown gracefully.
             // To avoid unnecessary complexity, we use SIGUSR1 as the termination signal for agents.
-            // Attempting to use SIGTERM would also notify the main server process, agent factory, and delay server.
+            // Attempting to use SIGTERM would also notify the main server process and delay server.
             kill(0, SIGUSR1);
         }
 
@@ -911,23 +926,6 @@ namespace
             cleanupAndExit(status);
         }
 
-        // load server side pluggable api entries
-        irods::api_entry_table&  RsApiTable   = irods::get_server_api_table();
-        irods::pack_entry_table& ApiPackTable = irods::get_pack_table();
-        ret = irods::init_api_table(RsApiTable, ApiPackTable, false);
-        if (!ret.ok()) {
-            log_agent::error(PASS(ret).result());
-            return 1;
-        }
-
-        // load client side pluggable api entries
-        irods::api_entry_table& RcApiTable = irods::get_client_api_table();
-        ret = irods::init_api_table(RcApiTable, ApiPackTable, false);
-        if (!ret.ok()) {
-            log_agent::error(PASS(ret).result());
-            return 1;
-        }
-
         std::string svc_role;
         ret = get_catalog_service_role(svc_role);
         if (!ret.ok()) {

server/main_server/src/main.cpp

@@ -310,7 +310,7 @@ namespace
         fmt::print(
 R"__(irodsServer - Launch an iRODS server
 
-Usage: irodsServer [OPTION]... CONFIG_DIRECTORY_PATH
+Usage: irodsServer [OPTION]... CONFIG_FILE_PATH
 
 TODO More words ...
 
@@ -337,31 +337,56 @@ Mandatory arguments to long options are mandatory for short options too.
             namespace jsonschema = jsoncons::jsonschema;
 
             std::ifstream config_file{_config_file_path};
+            if (!config_file) {
+                return false;
+            }
             const auto config = jsoncons::json::parse(config_file);
 
-            const auto schema_file = config.at("json_schema_file").as<std::string>();
-            fmt::print("{}: JSON schema file = [{}].\n", __func__, schema_file);
-            std::ifstream in{schema_file};
-            const auto schema = jsoncons::json::parse(in); // The stream object cannot be instantiated inline.
-            const auto compiled = jsonschema::make_json_schema(schema);
+            // NOLINTNEXTLINE(bugprone-lambda-function-name)
+            const auto do_validate = [fn = __func__](const auto& _config, const std::string& _schema_file) {
+                fmt::print("{}: JSON schema file = [{}].\n", fn, _schema_file);
+                std::ifstream in{_schema_file};
+                const auto schema = jsoncons::json::parse(in); // The stream object cannot be instantiated inline.
+                const auto compiled = jsonschema::make_json_schema(schema);
+
+                jsoncons::json_decoder<jsoncons::ojson> decoder;
+                compiled.validate(_config, decoder);
+                const auto json_result = decoder.get_result();
+
+                if (!json_result.empty()) {
+                    std::ostringstream out;
+                    out << pretty_print(json_result);
+                    fmt::print("{}: {}\n", fn, out.str());
+                    return false;
+                }
 
-            jsoncons::json_decoder<jsoncons::ojson> decoder;
-            compiled.validate(config, decoder);
-            const auto json_result = decoder.get_result();
+                return true;
+            };
 
-            if (!json_result.empty()) {
-                std::ostringstream out;
-                out << pretty_print(json_result);
-                fmt::print("{}: {}\n", __func__, out.str());
-                return false;
-            }
+            // Validate the server configuration. If that succeeds, move on to validating the
+            // irods_environment.json file.
+            if (do_validate(config, config.at("json_schema_file").as<std::string>())) {
+                std::string env_file;
+                std::string session_file;
+                if (const auto err = irods::get_json_environment_file(env_file, session_file); !err.ok()) {
+                    fmt::print("{}: {}\n", __func__, err.status());
+                    return false;
+                }
 
-            return true;
+                // Validate the irods_environment.json file referenced by the server configuration.
+                std::ifstream in{env_file};
+                if (!in) {
+                    return false;
+                }
+                const auto env_file_config = jsoncons::json::parse(in);
+                return do_validate(env_file_config, config.at("environment_json_schema_file").as<std::string>());
+            }
         }
         catch (const std::exception& e) {
             fmt::print("{}: {}\n", __func__, e.what());
-            return false;
         }
+
+        return false;
     } // validate_configuration
 
     auto daemonize() -> void
@@ -687,12 +712,12 @@ Mandatory arguments to long options are mandatory for short options too.
         }
 
         if (g_pid_ds > 0) {
-            log_server::info("{}: Sending SIGTERM to the delay server.", __func__);
+            log_server::info("{}: Sending SIGTERM to delay server.", __func__);
             kill(g_pid_ds, SIGTERM);
         }
 
-        log_server::info("{}: Sending SIGTERM to the agent factory.", __func__);
-        kill(g_pid_af, SIGTERM);
+        log_server::info("{}: Sending SIGQUIT to agent factory.", __func__);
+        kill(g_pid_af, SIGQUIT);
 
         // Reset this variable so that the delay server migration logic can handle
         // the relaunching of the delay server for us.

server/main_server/src/rodsServer.cpp

@@ -356,6 +356,8 @@ namespace
 
                 const std::regex pattern{pattern_string};
 
+                // TODO Will iRODS 5 need to do this at all once the plugins are loaded by the agent factory?
+                // Hopefully all of this code will disappear in the march to iRODS 5.
                 for (const auto& p : fs::directory_iterator{irods::get_irods_config_directory()}) {
                     if (std::regex_match(p.path().c_str(), pattern)) {
                         try {

server/re/src/reIn2p3SysRule.cpp

@@ -123,6 +123,7 @@ int rodsMonPerfLog( char *serverName, char *resc, char *output, ruleExecInfo_t *
         pthread_mutex_lock( &my_mutex );
 #endif
         /* append to the output log file */
+        // TODO Figure out where this should live. I'm thinking we replace serverMonPerf tools.
         const auto log_fname = irods::get_irods_home_directory() / "log" / fname;
         FILE *foutput = fopen(log_fname.c_str(), "a");
         if ( foutput != NULL ) {
@@ -576,6 +577,7 @@ int msiServerMonPerf( msParam_t *verb, msParam_t *ptime, ruleExecInfo_t *rei ) {
     /* read the config file or the iCAT to know the servers list to monitor */
     nresc = 0;
 
+    // TODO Update or replace
     const auto mon_cfg_file = irods::get_irods_home_directory() / MON_CFG_FILE;
     nservers = -1;  /* nservers = -1, no config file available, consider all resources for the monitoring */
     if ((filein = fopen(mon_cfg_file.c_str(), "r")) != NULL) {