Merge branch 'tickets/DM-33276' into main

lsst · Jan 18, 2022 · 68e2749 · 68e2749
2 parents 9f11c49 + 494d46b
commit 68e2749
Show file tree

Hide file tree

Showing 15 changed files with 25 additions and 545 deletions.
diff --git a/src/replica/ChunksApp.cc b/src/replica/ChunksApp.cc
@@ -222,11 +222,8 @@ int ChunksApp::runImpl() {
 
     // Limit execution timeout for requests if such limit was provided
     if (_timeoutSec != 0) {
-        bool const updatePersistentState = false;
-        serviceProvider()->config()->set<unsigned int>(
-                "controller", "request-timeout-sec", _timeoutSec, updatePersistentState);
-        serviceProvider()->config()->set<unsigned int>(
-                "xrootd", "request-timeout-sec", _timeoutSec, updatePersistentState);
+        serviceProvider()->config()->set<unsigned int>("controller", "request-timeout-sec", _timeoutSec);
+        serviceProvider()->config()->set<unsigned int>("xrootd", "request-timeout-sec", _timeoutSec);
     }
 
     ///////////////////////////////////////////////////////////////////

diff --git a/src/replica/ConfigApp.cc b/src/replica/ConfigApp.cc
@@ -81,7 +81,6 @@ ConfigApp::ConfigApp(int argc, char* argv[])
         "command",
         {"DUMP",
          "CONFIG_INIT_FILE",
-         "UPDATE_GENERAL",
          "UPDATE_WORKER", "ADD_WORKER", "DELETE_WORKER",
          "ADD_DATABASE_FAMILY", "DELETE_DATABASE_FAMILY",
          "ADD_DATABASE", "PUBLISH_DATABASE", "DELETE_DATABASE",
@@ -96,13 +95,13 @@ ConfigApp::ConfigApp(int argc, char* argv[])
         "Dump existing configuration parameters. The command allows specifying"
         " an optional scope to limit the output. If the parameter 'scope' is"
         " omitted then complete configuration will be printed. Allowed scopes:"
-        " 'GENERAL', 'WORKERS', 'FAMILIES', 'DATABASES'."
+        " 'WORKERS', 'FAMILIES', 'DATABASES'."
     ).optional(
         "scope",
         "This optional parameter narrows a scope of the operation down to a specific"
         " context. If no scope is specified then everything will be dumped.",
         _dumpScope,
-        vector<string>({"GENERAL", "WORKERS", "FAMILIES", "DATABASES"})
+        vector<string>({"WORKERS", "FAMILIES", "DATABASES"})
     );
 
     parser().command(
@@ -165,29 +164,6 @@ ConfigApp::ConfigApp(int argc, char* argv[])
         " This includes: replicas info and transaction contribution records."
     );
 
-    // Add options for the general parameters named as:
-    //   --<category>.<param>=<string>
-    // Note that since no database connection is available at this time (that would have
-    // required knowing a value of the parameter 'configUrl', and no parsing has been made
-    // yet) then the loop below will set the default value of each option to be the empty
-    // string. Any changes from that will be detected when processing the input. 
-    auto&& updateGeneralCmd = parser().command(
-        "UPDATE_GENERAL"
-    ).description(
-        "Update the general configuration parameters."
-    );
-    for (auto&& itr: ConfigurationSchema::parameters()) {
-        string const& category = itr.first;
-        for (auto&& param: itr.second) {
-            // The read-only parameters can't be updated programmatically.
-            if (ConfigurationSchema::readOnly(category, param)) continue;
-            _generalParams[category][param] = ConfigurationSchema::defaultValueAsString(category, param);
-            updateGeneralCmd.option(
-                category + "-" + param,
-                ConfigurationSchema::description(category, param),
-                _generalParams[category][param]);
-        }
-    }
 
     parser().command(
         "ADD_DATABASE_FAMILY"
@@ -335,7 +311,6 @@ int ConfigApp::runSubclassImpl() {
 
     if (_command == "DUMP") return _dump();
     if (_command == "CONFIG_INIT_FILE") return _configInitFile();
-    if (_command == "UPDATE_GENERAL") return _updateGeneral();
     if (_command == "UPDATE_WORKER") return _updateWorker();
     if (_command == "ADD_WORKER") return _addWorker();
     if (_command == "DELETE_WORKER") return _deleteWorker();
@@ -420,7 +395,6 @@ void ConfigApp::_configureWorkerOptions(detail::Command& command) {
 int ConfigApp::_dump() const {
     string const indent = "  ";
     cout << "\n" << indent << "CONFIG_URL: " << configUrl() << "\n" << "\n";
-    if (_dumpScope.empty() or _dumpScope == "GENERAL") dumpGeneralAsTable(indent);
     if (_dumpScope.empty() or _dumpScope == "WORKERS") dumpWorkersAsTable(indent);
     if (_dumpScope.empty() or _dumpScope == "FAMILIES") dumpFamiliesAsTable(indent);
     if (_dumpScope.empty() or _dumpScope == "DATABASES") dumpDatabasesAsTable(indent);
@@ -443,27 +417,6 @@ int ConfigApp::_configInitFile() const {
 }
 
 
-int ConfigApp::_updateGeneral() {
-    try {
-        // Note that options specified by a user will have non-empty values.
-        for (auto&& categoryItr: _generalParams) {
-            string const& category = categoryItr.first;
-            for (auto&& paramItr: categoryItr.second) {
-                string const& param = paramItr.first;
-                string const& value = paramItr.second;
-                if (!value.empty()) {
-                    config()->setFromString(category, param, value);
-                }
-            }
-        }
-    } catch (exception const& ex) {
-        LOGS(_log, LOG_LVL_ERROR, "ConfigApp::" << __func__ << ": " << ex.what());
-        throw;
-    }
-    return 0;
-}
-
-
 int ConfigApp::_updateWorker() const {
     // Configuration changes will be updated in the transient object obtained from
     // the database and then be saved to the the persistent configuration.

diff --git a/src/replica/ConfigApp.h b/src/replica/ConfigApp.h
@@ -92,12 +92,6 @@ class ConfigApp: public ConfigAppBase {
      */
     int _configInitFile() const;
 
-    /**
-     * Update the general configuration parameters
-     * @return A status code to be returned to the shell.
-     */
-    int _updateGeneral();
-
     /**
      * Update parameters of a worker
      * @return A status code to be returned to the shell.
@@ -181,9 +175,6 @@ class ConfigApp: public ConfigAppBase {
     /// of 1 (or any positive number) means - enable the read-write mode for the worker .
     int _workerReadOnly = -1;
 
-    /// General parameters
-    std::map<std::string, std::map<std::string, std::string>> _generalParams;
-
     /// For database families
     DatabaseFamilyInfo _familyInfo;
 

diff --git a/src/replica/ConfigAppBase.cc b/src/replica/ConfigAppBase.cc
@@ -75,33 +75,6 @@ int ConfigAppBase::runImpl() {
 }
 
 
-void ConfigAppBase::dumpGeneralAsTable(string const& indent) const {
-    // Extract general attributes and put them into the corresponding
-    // columns. Translate tables cell values into strings when required.
-    vector<string> categories;
-    vector<string> parameters;
-    vector<string> values;
-    vector<string> descriptions;
-    for (auto&& itr: ConfigurationSchema::parameters()) {
-        string const& category = itr.first;
-        for (auto&& param: itr.second) {
-            categories.push_back(category);
-            parameters.push_back(param);
-            values.push_back(ConfigurationSchema::securityContext(category, param) ?
-                    "xxxxxx" : _config->getAsString(category, param)
-            );
-            descriptions.push_back(ConfigurationSchema::description(category, param));
-        }
-    }
-    util::ColumnTablePrinter table("GENERAL PARAMETERS:", indent, verticalSeparator());
-    table.addColumn("category", categories, util::ColumnTablePrinter::LEFT);
-    table.addColumn("param", parameters, util::ColumnTablePrinter::LEFT);
-    table.addColumn("value", values);
-    table.addColumn("description", descriptions, util::ColumnTablePrinter::LEFT);
-    table.print(cout, false, false);
-}
-
-
 void ConfigAppBase::dumpWorkersAsTable(string const& indent, string const& caption) const {
 
     // Extract attributes of each worker and put them into the corresponding

diff --git a/src/replica/ConfigAppBase.h b/src/replica/ConfigAppBase.h
@@ -68,9 +68,8 @@ class ConfigAppBase: public Application {
      * Dump the Configuration into the standard output stream
      * @return A status code to be returned to the shell.
      */
-    void dumpGeneralAsTable(std::string const& indent) const;
     void dumpWorkersAsTable(std::string const& indent, std::string const& caption="WORKERS:") const;
-    void dumpFamiliesAsTable(std::string const& indent, std::string const& caption="DATABASES FAMILIES") const;
+    void dumpFamiliesAsTable(std::string const& indent, std::string const& caption="DATABASE FAMILIES:") const;
     void dumpDatabasesAsTable(std::string const& indent, std::string const& caption="DATABASES:") const;
 
 private:

diff --git a/src/replica/ConfigParserMySQL.cc b/src/replica/ConfigParserMySQL.cc
@@ -36,7 +36,7 @@ namespace lsst {
 namespace qserv {
 namespace replica {
 
-int const ConfigParserMySQL::expectedSchemaVersion = 6;
+int const ConfigParserMySQL::expectedSchemaVersion = 7;
 
 
 ConfigParserMySQL::ConfigParserMySQL(database::mysql::Connection::Ptr const& conn,
@@ -56,11 +56,6 @@ void ConfigParserMySQL::parse() {
 
     _parseVersion();
 
-    // Read and update the transient state the general parameters and defaults
-    // shared by all components of the Replication system. The table also provides
-    // default values for some critical parameters of the worker-side services.
-    _parseGeneral();
-
     // Parse groupped parameters
     _parseWorkers();
     _parseDatabaseFamilies();
@@ -94,40 +89,6 @@ void ConfigParserMySQL::_parseVersion() {
 }
 
 
-void ConfigParserMySQL::_parseGeneral() {
-    _conn->execute("SELECT * FROM " + _conn->sqlId("config"));
-    while (_conn->next(_row)) {
-        string category;
-        _row.get("category", category);
-
-        string param;
-        _row.get("param", param);
-
-        json obj;
-        try {
-            obj = _data.at(category).at(param);
-        } catch (exception const& ex) {
-            throw invalid_argument(
-                    _context + " no transient schema match for the parameter, category: '"
-                    + category + "' param: '" + param + "'.");
-        }
-        if (obj.is_string()) {
-            _storeGeneralParameter<string>(category, param);
-        } else if (obj.is_number_unsigned()) {
-            _storeGeneralParameter<uint64_t>(category, param);
-        } else if (obj.is_number_integer()) {
-            _storeGeneralParameter<int64_t>(category, param);
-        } else if (obj.is_number_float()) {
-            _storeGeneralParameter<double>(category, param);
-        } else {
-            throw invalid_argument(
-                    _context + " unsupported transient schema type for the parameter, category: '"
-                    + category + "' param: '" + param + "'.");
-        }
-    }
-}
-
-
 void ConfigParserMySQL::_parseWorkers() {
     json& defaults = _data.at("worker-defaults");
     _conn->execute("SELECT * FROM " + _conn->sqlId("config_worker"));

diff --git a/src/replica/ConfigParserMySQL.h b/src/replica/ConfigParserMySQL.h
@@ -33,7 +33,6 @@
 #include "replica/ConfigDatabase.h"
 #include "replica/ConfigDatabaseFamily.h"
 #include "replica/ConfigWorker.h"
-#include "replica/ConfigurationSchema.h"
 #include "replica/DatabaseMySQL.h"
 
 // This header declarations
@@ -86,9 +85,6 @@ class ConfigParserMySQL {
      */
     void _parseVersion();
 
-    /// Parse general (category/parameter) parameters.
-    void _parseGeneral();
-
     /**
      * Parse a collection of workers.
      *
@@ -105,26 +101,6 @@ class ConfigParserMySQL {
     /// Parse a collection of the databases.
     void _parseDatabases();
 
-    /**
-     * Extract a value of the general parameter into the requested type, sanitize the value
-     * if needed, and store it in the transent state.
-     * @throws std::runtime_error If a required field has NULL.
-     * @throws std::invalid_argument If the parameter's value didn't pass the validation.
-     */
-    template <typename T>
-    void _storeGeneralParameter(std::string const& category, std::string const& param) {
-        T value;
-        if (!_row.get("value", value)) {
-            throw std::runtime_error(
-                    _context + " NULL is not allowed, category:'" + category
-                    + "' param: '" + param + "'.");
-        }
-        // Sanitize the input to ensure it matches schema requirements before
-        // pushing the value into the configuration.
-        ConfigurationSchema::validate<T>(category, param, value);
-        _data[category][param] = value;
-    }
-
     template <typename T>
     T _parseParam(std::string const& name) {
         T value;