diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8ed0d534..1ec5836e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -285,6 +285,8 @@ jobs: fi lxc exec node-head -- sh -c "microceph.ceph -s" | fgrep "mon: 3 daemons" + - name: Test client configurations + run: ~/actionutils.sh check_client_configs upgrade-quincy-tests: name: Test quincy upgrades diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 10b019ce..abfa097f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,7 @@ other projects. The agreement can be found, and signed, here: https://ubuntu.com/legal/contributors -Refer to [Microceph Hacking Guide](/HACKING.md) for a basic outline of the codebase and build tools. +Refer to [MicroCeph Hacking Guide](/HACKING.md) for a basic outline of the codebase and build tools. ## Contributor guidelines diff --git a/README.md b/README.md index 7ebd7d62..0466ecd8 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,7 @@ $ sudo microceph.ceph status ## 👍 How Can I Contribute ? -1. Checkout [Microceph Hacking Guide](/HACKING.md) to start building and contributing to the codebase. +1. Checkout [MicroCeph Hacking Guide](/HACKING.md) to start building and contributing to the codebase. 2. Excited about [MicroCeph](https://snapcraft.io/microceph) ? Join our [Stargazers](https://github.com/canonical/microceph/stargazers) 3. Write reviews or tutorials to help spread the knowledge 📖 4. Participate in [Pull Requests](https://github.com/canonical/microceph/pulls) and help fix [Issues](https://github.com/canonical/microceph/issues) diff --git a/docs/.custom_wordlist.txt b/docs/.custom_wordlist.txt index f7bd7fd7..464f3b42 100644 --- a/docs/.custom_wordlist.txt +++ b/docs/.custom_wordlist.txt @@ -1,5 +1,6 @@ openstack ceph +sudo Howto configs osd @@ -9,6 +10,7 @@ cfg CFG MicroCeph +microceph OSDs MSD Ceph @@ -45,8 +47,15 @@ Pre mds mon rgw +rbd +RBD MgrReports scalable Mattermost integratable cfg +conf +qemu +writethrough +writeback + diff --git a/docs/how-to/rbd-client-cfg.rst b/docs/how-to/rbd-client-cfg.rst new file mode 100644 index 00000000..d02ce504 --- /dev/null +++ b/docs/how-to/rbd-client-cfg.rst @@ -0,0 +1,114 @@ +Configure RBD client cache in MicroCeph +======================================== + +MicroCeph supports setting, resetting, and listing client configurations which are exported to ceph.conf and are used by tools like qemu directly for configuring rbd cache. Below are the supported client configurations. + +.. list-table:: Supported Config Keys + :widths: 30 70 + :header-rows: 1 + + * - Key + - Description + * - rbd_cache + - Enable caching for RADOS Block Device (RBD). + * - rbd_cache_size + - The RBD cache size in bytes. + * - rbd_cache_writethrough_until_flush + - The number of seconds dirty data is in the cache before writeback starts. + * - rbd_cache_max_dirty + - The dirty limit in bytes at which the cache triggers write-back. If 0, uses write-through caching. + * - rbd_cache_target_dirty + - The dirty target before the cache begins writing data to the data storage. Does not block writes to the cache. + +1. Supported config keys can be configured using the 'set' command: + + .. code-block:: shell + + $ sudo microceph client config set rbd_cache true + $ sudo microceph client config set rbd_cache false --target alpha + $ sudo microceph client config set rbd_cache_size 2048MiB --target beta + + .. note:: + + Host level configuration changes can be made by passing the relevant hostname as the --target parameter. + +2. All the client configs can be queried using the 'list' command. + + .. code-block:: shell + + $ sudo microceph cluster config list + +---+----------------+---------+----------+ + | # | KEY | VALUE | HOST | + +---+----------------+---------+----------+ + | 0 | rbd_cache | true | beta | + +---+----------------+---------+----------+ + | 1 | rbd_cache | false | alpha | + +---+----------------+---------+----------+ + | 2 | rbd_cache_size | 2048MiB | beta | + +---+----------------+---------+----------+ + + Similarly, all the client configs of a particular host can be queried using the --target parameter. + + .. code-block:: shell + + $ sudo microceph cluster config list --target beta + +---+----------------+---------+----------+ + | # | KEY | VALUE | HOST | + +---+----------------+---------+----------+ + | 0 | rbd_cache | true | beta | + +---+----------------+---------+----------+ + | 1 | rbd_cache_size | 2048MiB | beta | + +---+----------------+---------+----------+ + + +3. A particular config key can be queried for using the 'get' command: + + .. code-block:: shell + + $ sudo microceph cluster config list + +---+----------------+---------+----------+ + | # | KEY | VALUE | HOST | + +---+----------------+---------+----------+ + | 0 | rbd_cache | true | beta | + +---+----------------+---------+----------+ + | 1 | rbd_cache | false | alpha | + +---+----------------+---------+----------+ + + Similarly, --target parameter can be used with get command to query for a particular config key/hostname pair. + + .. code-block:: shell + + $ sudo microceph cluster config rbd_cache --target alpha + +---+----------------+---------+----------+ + | # | KEY | VALUE | HOST | + +---+----------------+---------+----------+ + | 0 | rbd_cache | false | alpha | + +---+----------------+---------+----------+ + + +4. Resetting a config key (i.e. removing the configured key/value) can performed using the 'reset' command: + + .. code-block:: shell + + $ sudo microceph cluster config reset rbd_cache_size + $ sudo microceph cluster config list + +---+----------------+---------+----------+ + | # | KEY | VALUE | HOST | + +---+----------------+---------+----------+ + | 0 | rbd_cache | true | beta | + +---+----------------+---------+----------+ + | 1 | rbd_cache | false | alpha | + +---+----------------+---------+----------+ + + This operation can also be performed for a specific host as follows: + + .. code-block:: shell + + $ sudo microceph cluster config reset rbd_cache --target alpha + $ sudo microceph cluster config list + +---+----------------+---------+----------+ + | # | KEY | VALUE | HOST | + +---+----------------+---------+----------+ + | 0 | rbd_cache | true | beta | + +---+----------------+---------+----------+ + diff --git a/docs/reference/commands/client.rst b/docs/reference/commands/client.rst new file mode 100644 index 00000000..affd7b08 --- /dev/null +++ b/docs/reference/commands/client.rst @@ -0,0 +1,121 @@ +=========== +``client`` +=========== + +Manages MicroCeph clients + +Usage: + +.. code-block:: none + + microceph client [flags] + microceph client [command] + +Available commands: + +.. code-block:: none + + config Manage Ceph Client configs + +Global options: + +.. code-block:: none + + -d, --debug Show all debug messages + -h, --help Print help + --state-dir Path to store state information + -v, --verbose Show all information messages + --version Print version number + +``config`` +---------- + +Manages Ceph Cluster configs. + +Usage: + +.. code-block:: none + + microceph cluster config [flags] + microceph cluster config [command] + +Available Commands: + +.. code-block:: none + + get Fetches specified Ceph Client config + list Lists all configured Ceph Client configs + reset Removes specified Ceph Client configs + set Sets specified Ceph Client config + +``config set`` +-------------- + +Sets specified Ceph Client config + +Usage: + +.. code-block:: none + + microceph client config set [flags] + +Flags: + +.. code-block:: none + + --target string Specify a microceph node the provided config should be applied to. (default "*") + --wait Wait for configs to propagate across the cluster. (default true) + +``config get`` +-------------- + +Fetches specified Ceph Client config + +Usage: + +.. code-block:: none + + microceph client config get [flags] + +Flags: + +.. code-block:: none + + --target string Specify a microceph node the provided config should be applied to. (default "*") + +``config list`` +--------------- + +Lists all configured Ceph Client configs + +Usage: + +.. code-block:: none + + microceph client config list [flags] + +Flags: + +.. code-block:: none + + --target string Specify a microceph node the provided config should be applied to. (default "*") + +``config reset`` +---------------- + +Removes specified Ceph Client configs + +Usage: + +.. code-block:: none + + microceph client config reset [flags] + +Flags: + +.. code-block:: none + + --target string Specify a microceph node the provided config should be applied to. (default "*") + --wait Wait for required ceph services to restart post config reset. (default true) + --yes-i-really-mean-it Force microceph to reset all client config records for given key. + diff --git a/microceph/api/client_configs.go b/microceph/api/client_configs.go new file mode 100644 index 00000000..34cd7e57 --- /dev/null +++ b/microceph/api/client_configs.go @@ -0,0 +1,182 @@ +package api + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "net/http" + + "github.com/canonical/lxd/lxd/response" + "github.com/canonical/lxd/shared/logger" + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/ceph" + "github.com/canonical/microceph/microceph/client" + "github.com/canonical/microceph/microceph/common" + "github.com/canonical/microceph/microceph/database" + "github.com/canonical/microcluster/rest" + "github.com/canonical/microcluster/state" +) + +// Top level client API +var clientCmd = rest.Endpoint{ + Path: "client", +} + +// client configs API +var clientConfigsCmd = rest.Endpoint{ + Path: "client/configs", + Put: rest.EndpointAction{Handler: cmdClientConfigsPut, ProxyTarget: true}, + Get: rest.EndpointAction{Handler: cmdClientConfigsGet, ProxyTarget: true}, +} + +// cmdClientConfigsGet fetches multiple client config key entries from internal database. +func cmdClientConfigsGet(s *state.State, r *http.Request) response.Response { + var req types.ClientConfig + var configs database.ClientConfigItems + + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + return response.InternalError(err) + } + + if req.Host == common.ClientConfigGlobalHostConst { + configs, err = database.ClientConfigQuery.GetAll(s) + } else { + configs, err = database.ClientConfigQuery.GetAllForHost(s, req.Host) + } + if err != nil { + logger.Errorf("failed fetching client configs: %v for %v", err, req) + return response.SyncResponse(false, nil) + } + + logger.Infof("Database Response: %v", configs) + + return response.SyncResponse(true, configs.GetClientConfigSlice()) +} + +// cmdClientConfigsPut renders .conf file at that particular host. +func cmdClientConfigsPut(s *state.State, r *http.Request) response.Response { + // Check if microceph is bootstrapped. + err := s.Database.Transaction(s.Context, func(ctx context.Context, tx *sql.Tx) error { + isFsid, err := database.ConfigItemExists(ctx, tx, "fsid") + if err != nil || !isFsid { + return fmt.Errorf("client configuration cannot be performed before bootstrapping the cluster") + } + return nil + }) + if err != nil { + logger.Error(err.Error()) + return response.BadRequest(err) + } + + err = ceph.UpdateConfig(common.CephState{State: s}) + if err != nil { + logger.Error(err.Error()) + response.InternalError(err) + } + + return response.EmptySyncResponse +} + +// client configs key API +var clientConfigsKeyCmd = rest.Endpoint{ + Path: "client/configs/{key}", + Put: rest.EndpointAction{Handler: clientConfigsKeyPut, ProxyTarget: true}, + Get: rest.EndpointAction{Handler: clientConfigsKeyGet, ProxyTarget: true}, + Delete: rest.EndpointAction{Handler: clientConfigsKeyDelete, ProxyTarget: true}, +} + +// clientConfigsKeyGet fetches particular client config key entries from internal db. +func clientConfigsKeyGet(s *state.State, r *http.Request) response.Response { + var req types.ClientConfig + var configs database.ClientConfigItems + + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + return response.InternalError(err) + } + + if req.Host == common.ClientConfigGlobalHostConst { + configs, err = database.ClientConfigQuery.GetAllForKey(s, req.Key) + } else { + configs, err = database.ClientConfigQuery.GetAllForKeyAndHost(s, req.Key, req.Host) + } + if err != nil { + logger.Errorf("failed fetching client configs: %v for %v", err, req) + return response.InternalError(err) + } + + logger.Infof("Database Response: %v", configs) + + return response.SyncResponse(true, configs.GetClientConfigSlice()) +} + +// clientConfigsKeyPut sets particular client config key. +func clientConfigsKeyPut(s *state.State, r *http.Request) response.Response { + var req types.ClientConfig + + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + return response.InternalError(err) + } + + // If new config request is for global configuration. + err = database.ClientConfigQuery.AddNew(s, req.Key, req.Value, req.Host) + if err != nil { + return response.InternalError(err) + } + + // Trigger /conf file update across cluster. + clientConfigUpdate(s, req.Wait) + + return response.EmptySyncResponse +} + +// clientConfigsKeyDelete removes particular client config key entries from internal db. +func clientConfigsKeyDelete(s *state.State, r *http.Request) response.Response { + var req types.ClientConfig + + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + return response.InternalError(err) + } + + if req.Host == common.ClientConfigGlobalHostConst { + err = database.ClientConfigQuery.RemoveAllForKey(s, req.Key) + } else { + err = database.ClientConfigQuery.RemoveOneForKeyAndHost(s, req.Key, req.Host) + } + if err != nil { + return response.InternalError(err) + } + + // Trigger /conf file update across cluster. + clientConfigUpdate(s, req.Wait) + + return response.EmptySyncResponse +} + +// clientConfigUpdate performs ordered (one after other) updation of ceph.conf across the ceph cluster. +func clientConfigUpdate(s *state.State, wait bool) error { + if wait { + // Execute update conf synchronously + err := client.SendUpdateClientConfRequestToClusterMembers(common.CephState{State: s}) + if err != nil { + return err + } + + // Update on current host. + err = ceph.UpdateConfig(common.CephState{State: s}) + if err != nil { + return err + } + } else { // Execute update asynchronously + go func() { + client.SendUpdateClientConfRequestToClusterMembers(common.CephState{State: s}) + ceph.UpdateConfig(common.CephState{State: s}) // Restart on current host. + }() + } + + return nil +} diff --git a/microceph/api/endpoints.go b/microceph/api/endpoints.go index 7af08549..845fc417 100644 --- a/microceph/api/endpoints.go +++ b/microceph/api/endpoints.go @@ -18,4 +18,7 @@ var Endpoints = []rest.Endpoint{ mgrServiceCmd, monServiceCmd, rgwServiceCmd, + clientCmd, + clientConfigsCmd, + clientConfigsKeyCmd, } diff --git a/microceph/api/types/client_configs.go b/microceph/api/types/client_configs.go new file mode 100644 index 00000000..ecf1488a --- /dev/null +++ b/microceph/api/types/client_configs.go @@ -0,0 +1,12 @@ +package types + +// ClientConfig type holds parameters from the `client config` API request +type ClientConfig struct { + Key string `json:"key" yaml:"key"` + Value string `json:"value" yaml:"value"` + Host string `json:"host" yaml:"host"` + Wait bool `json:"wait" yaml:"wait"` +} + +// ClientConfigs is a slice of client configs +type ClientConfigs []ClientConfig diff --git a/microceph/ceph/bootstrap.go b/microceph/ceph/bootstrap.go index 22836ed8..36c7890b 100644 --- a/microceph/ceph/bootstrap.go +++ b/microceph/ceph/bootstrap.go @@ -103,7 +103,7 @@ func Bootstrap(s common.StateInterface) error { } // Re-generate the configuration from the database. - err = updateConfig(s) + err = UpdateConfig(s) if err != nil { return fmt.Errorf("Failed to re-generate the configuration: %w", err) } diff --git a/microceph/ceph/client_config.go b/microceph/ceph/client_config.go new file mode 100644 index 00000000..beed7a06 --- /dev/null +++ b/microceph/ceph/client_config.go @@ -0,0 +1,65 @@ +package ceph + +import ( + "fmt" + "reflect" + + "github.com/canonical/microceph/microceph/common" + "github.com/canonical/microceph/microceph/database" +) + +// ClientConfigT holds all the client configuration values *applicable* for +// the host machine. These values are consumed by configwriter for ceph.conf +// updation. This approach keeps the client config updation logic tied together +// and easily extendable for more keys. +type ClientConfigT struct { + IsCache string + CacheSize string + IsCacheWritethrough string + CacheMaxDirty string + CacheTargetDirty string +} + +// GetClientConfigForHost fetches all the applicable client configurations for the provided host. +func GetClientConfigForHost(s common.StateInterface, hostname string) (ClientConfigT, error) { + retval := ClientConfigT{} + + // Get all client configs for the current host. + configs, err := database.ClientConfigQuery.GetAllForHost(s.ClusterState(), hostname) + if err != nil { + return ClientConfigT{}, fmt.Errorf("could not query database for client configs: %v", err) + } + + setterTable := GetClientConfigSet() + for _, config := range configs { + // Populate client config table using the database values. + err = setFieldValue(&retval, fmt.Sprint(setterTable[config.Key]), config.Value) + if err != nil { + return ClientConfigT{}, fmt.Errorf("failed object population: %v", err) + } + } + + return retval, nil +} + +// setFieldValue populates the individual client configuration values into ClientConfigT object fields. +func setFieldValue(ogp *ClientConfigT, field string, value string) error { + r := reflect.ValueOf(ogp) + f := reflect.Indirect(r).FieldByName(field) + if f.Kind() != reflect.Invalid { + f.SetString(value) + return nil + } + return fmt.Errorf("cannot set field %s", field) +} + +// GetClientConfigSet provides the mapping between client config key and fieldname for population through reflection. +func GetClientConfigSet() common.Set { + return common.Set{ + "rbd_cache": "IsCache", + "rbd_cache_size": "CacheSize", + "rbd_cache_writethrough_until_flush": "IsCacheWritethrough", + "rbd_cache_max_dirty": "CacheMaxDirty", + "rbd_cache_target_dirty": "CacheTargetDirty", + } +} diff --git a/microceph/ceph/client_config_test.go b/microceph/ceph/client_config_test.go new file mode 100644 index 00000000..91bbfc8c --- /dev/null +++ b/microceph/ceph/client_config_test.go @@ -0,0 +1,65 @@ +package ceph + +import ( + "fmt" + "reflect" + "testing" + + "github.com/canonical/microceph/microceph/database" + "github.com/canonical/microceph/microceph/mocks" + "github.com/canonical/microcluster/state" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type ClientConfigSuite struct { + baseSuite + TestStateInterface *mocks.StateInterface +} + +func TestClientConfig(t *testing.T) { + suite.Run(t, new(ClientConfigSuite)) +} + +func (ccs *ClientConfigSuite) SetupTest() { + ccs.baseSuite.SetupTest() + + ccs.TestStateInterface = mocks.NewStateInterface(ccs.T()) + state := &state.State{} + + ccs.TestStateInterface.On("ClusterState").Return(state) +} + +func addGetHostConfigsExpectation(mci *mocks.ClientConfigQueryIntf, cs *state.State, hostname string) { + output := database.ClientConfigItems{} + count := 0 + for configKey, field := range GetClientConfigSet() { + count++ + output = append(output, database.ClientConfigItem{ + ID: count, + Host: hostname, + Key: configKey, + Value: fmt.Sprintf("%v", field), + }) + } + + mci.On("GetAllForHost", cs, hostname).Return(output, nil) +} + +func (ccs *ClientConfigSuite) TestFetchHostConfig() { + hostname := "testHostname" + + // Mock Client config query interface. + ccq := mocks.NewClientConfigQueryIntf(ccs.T()) + addGetHostConfigsExpectation(ccq, ccs.TestStateInterface.ClusterState(), hostname) + database.ClientConfigQuery = ccq + + configs, err := GetClientConfigForHost(ccs.TestStateInterface, hostname) + assert.NoError(ccs.T(), err) + + // check fields + metaConfigs := reflect.ValueOf(configs) + for i := 0; i < metaConfigs.NumField(); i++ { + assert.Equal(ccs.T(), metaConfigs.Field(i).Interface(), metaConfigs.Type().Field(i).Name) + } +} diff --git a/microceph/ceph/config.go b/microceph/ceph/config.go index d7b93680..e8b03860 100644 --- a/microceph/ceph/config.go +++ b/microceph/ceph/config.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strings" + "github.com/canonical/lxd/shared/logger" "github.com/canonical/microceph/microceph/api/types" "github.com/canonical/microceph/microceph/common" "github.com/canonical/microceph/microceph/database" @@ -48,8 +49,8 @@ func GetConstConfigTable() ConfigTable { } } -func GetConfigTableServiceSet() Set { - return Set{ +func GetConfigTableServiceSet() common.Set { + return common.Set{ "mon": struct{}{}, "mgr": struct{}{}, "osd": struct{}{}, @@ -162,7 +163,7 @@ func ListConfigs() (types.Configs, error) { } // updates the ceph config file. -func updateConfig(s common.StateInterface) error { +func UpdateConfig(s common.StateInterface) error { confPath := filepath.Join(os.Getenv("SNAP_DATA"), "conf") runPath := filepath.Join(os.Getenv("SNAP_DATA"), "run") @@ -207,19 +208,30 @@ func updateConfig(s common.StateInterface) error { conf := newCephConfig(confPath) address := s.ClusterState().Address().Hostname() + clientConfig, err := GetClientConfigForHost(s, s.ClusterState().Name()) + if err != nil { + logger.Errorf("Failed to pull Client Configurations: %v", err) + return err + } + err = conf.WriteConfig( map[string]any{ - "fsid": config["fsid"], - "runDir": runPath, - "monitors": strings.Join(monitorAddresses, ","), - "addr": address, - "ipv4": strings.Contains(address, "."), - "ipv6": strings.Contains(address, ":"), + "fsid": config["fsid"], + "runDir": runPath, + "monitors": strings.Join(monitorAddresses, ","), + "addr": address, + "ipv4": strings.Contains(address, "."), + "ipv6": strings.Contains(address, ":"), + "isCache": clientConfig.IsCache, + "cacheSize": clientConfig.CacheSize, + "isCacheWritethrough": clientConfig.IsCacheWritethrough, + "cacheMaxDirty": clientConfig.CacheMaxDirty, + "cacheTargetDirty": clientConfig.CacheTargetDirty, }, 0644, ) if err != nil { - return fmt.Errorf("Couldn't render ceph.conf: %w", err) + return fmt.Errorf("couldn't render ceph.conf: %w", err) } // Generate ceph.client.admin.keyring @@ -232,7 +244,7 @@ func updateConfig(s common.StateInterface) error { 0640, ) if err != nil { - return fmt.Errorf("Couldn't render ceph.client.admin.keyring: %w", err) + return fmt.Errorf("couldn't render ceph.client.admin.keyring: %w", err) } return nil diff --git a/microceph/ceph/configwriter.go b/microceph/ceph/configwriter.go index 98ee9f8f..381a296c 100644 --- a/microceph/ceph/configwriter.go +++ b/microceph/ceph/configwriter.go @@ -51,6 +51,13 @@ auth allow insecure global id reclaim = false public addr = {{.addr}} ms bind ipv4 = {{.ipv4}} ms bind ipv6 = {{.ipv6}} + +[client] +{{if .isCache}}rbd_cache = {{.isCache}}{{end}} +{{if .cacheSize}}rbd_cache_size = {{.cacheSize}}{{end}} +{{if .isCacheWritethrough}}rbd_cache_writethrough_until_flush = {{.isCacheWritethrough}}{{end}} +{{if .cacheMaxDirty}}rbd_cache_max_dirty = {{.cacheMaxDirty}}{{end}} +{{if .cacheTargetDirty}}rbd_cache_target_dirty = {{.cacheTargetDirty}}{{end}} `)), configFile: "ceph.conf", configDir: configDir, diff --git a/microceph/ceph/join.go b/microceph/ceph/join.go index 97c5d2c3..d068317e 100644 --- a/microceph/ceph/join.go +++ b/microceph/ceph/join.go @@ -25,7 +25,7 @@ func Join(s common.StateInterface) error { } // Generate the configuration from the database. - err := updateConfig(s) + err := UpdateConfig(s) if err != nil { return fmt.Errorf("Failed to generate the configuration: %w", err) } diff --git a/microceph/ceph/services.go b/microceph/ceph/services.go index 2e88fb61..59c09c8f 100644 --- a/microceph/ceph/services.go +++ b/microceph/ceph/services.go @@ -4,7 +4,6 @@ import ( "context" "database/sql" "fmt" - "github.com/canonical/microceph/microceph/common" "os" "path/filepath" "time" @@ -16,29 +15,13 @@ import ( "github.com/canonical/microcluster/state" "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/common" "github.com/canonical/microceph/microceph/database" "github.com/tidwall/gjson" ) -type Set map[string]struct{} - -func (sub Set) isIn(super Set) bool { - flag := true - - // mark flag false if any key from subset is not present in superset. - for key := range sub { - _, ok := super[key] - if !ok { - flag = false - break // Break the loop. - } - } - - return flag -} - // Table to map fetchFunc for workers (daemons) to a service. -var serviceWorkerTable = map[string](func() (Set, error)){ +var serviceWorkerTable = map[string](func() (common.Set, error)){ "osd": getUpOsds, "mon": getMons, } @@ -82,7 +65,7 @@ func RestartCephService(service string) error { } // All still not up - if !workers.isIn(iWorkers) { + if !workers.IsIn(iWorkers) { err := fmt.Errorf( "Attempt %d: Workers: %v not all present in %v", i, workers, iWorkers, ) @@ -98,8 +81,8 @@ func RestartCephService(service string) error { return nil } -func getMons() (Set, error) { - retval := Set{} +func getMons() (common.Set, error) { + retval := common.Set{} output, err := processExec.RunCommand("ceph", "mon", "dump", "-f", "json-pretty") if err != nil { logger.Errorf("Failed fetching Mon dump: %v", err) @@ -116,8 +99,8 @@ func getMons() (Set, error) { return retval, nil } -func getUpOsds() (Set, error) { - retval := Set{} +func getUpOsds() (common.Set, error) { + retval := common.Set{} output, err := processExec.RunCommand("ceph", "osd", "dump", "-f", "json-pretty") if err != nil { logger.Errorf("Failed fetching OSD dump: %v", err) diff --git a/microceph/ceph/start.go b/microceph/ceph/start.go index ae2d2f87..ced3ce84 100644 --- a/microceph/ceph/start.go +++ b/microceph/ceph/start.go @@ -49,7 +49,7 @@ func Start(s common.StateInterface) error { continue } - err = updateConfig(s) + err = UpdateConfig(s) if err != nil { time.Sleep(10 * time.Second) continue diff --git a/microceph/client/client_configs.go b/microceph/client/client_configs.go new file mode 100644 index 00000000..38a6467d --- /dev/null +++ b/microceph/client/client_configs.go @@ -0,0 +1,99 @@ +package client + +import ( + "context" + "fmt" + "time" + + "github.com/canonical/lxd/shared/api" + "github.com/canonical/lxd/shared/logger" + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/common" + "github.com/canonical/microcluster/client" +) + +func SetClientConfig(ctx context.Context, c *client.Client, data *types.ClientConfig) error { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*200) + defer cancel() + + err := c.Query(queryCtx, "PUT", api.NewURL().Path("client", "configs", data.Key), data, nil) + if err != nil { + return fmt.Errorf("failed setting client config: %w, Key: %s, Value: %s", err, data.Key, data.Value) + } + + return nil +} + +func ResetClientConfig(ctx context.Context, c *client.Client, data *types.ClientConfig) error { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*200) + defer cancel() + + err := c.Query(queryCtx, "DELETE", api.NewURL().Path("client", "configs", data.Key), data, nil) + if err != nil { + return fmt.Errorf("failed clearing client config: %w, Key: %s", err, data.Key) + } + + return nil +} + +func GetClientConfig(ctx context.Context, c *client.Client, data *types.ClientConfig) (types.ClientConfigs, error) { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*10) + defer cancel() + + configs := types.ClientConfigs{} + + err := c.Query(queryCtx, "GET", api.NewURL().Path("client", "configs", data.Key), data, &configs) + if err != nil { + return nil, fmt.Errorf("failed to fetch client config: %w, Key: %s", err, data.Key) + } + + return configs, nil +} + +func ListClientConfig(ctx context.Context, c *client.Client, data *types.ClientConfig) (types.ClientConfigs, error) { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*10) + defer cancel() + + configs := types.ClientConfigs{} + + err := c.Query(queryCtx, "GET", api.NewURL().Path("client", "configs"), data, &configs) + if err != nil { + return nil, fmt.Errorf("failed to fetch client config: %w, Key: %s", err, data.Key) + } + + return configs, nil +} + +// /client/configs/ +func UpdateClientConf(ctx context.Context, c *client.Client) error { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*20) + defer cancel() + + err := c.Query(queryCtx, "PUT", api.NewURL().Path("client", "configs"), nil, nil) + if err != nil { + return fmt.Errorf("failed to update the configuration file: %w", err) + } + + return nil +} + +// Sends the update conf request to every other member of the cluster. +func SendUpdateClientConfRequestToClusterMembers(s common.StateInterface) error { + // Get a collection of clients to every other cluster member, with the notification user-agent set. + cluster, err := s.ClusterState().Cluster(nil) + if err != nil { + logger.Errorf("failed to get a client for every cluster member: %v", err) + return err + } + + for _, remoteClient := range cluster { + // In order send restart to each cluster member and wait. + err = UpdateClientConf(s.ClusterState().Context, &remoteClient) + if err != nil { + logger.Errorf("update conf error: %v", err) + return err + } + } + + return nil +} diff --git a/microceph/client/configs.go b/microceph/client/configs.go new file mode 100644 index 00000000..f723a779 --- /dev/null +++ b/microceph/client/configs.go @@ -0,0 +1,49 @@ +package client + +import ( + "context" + "fmt" + "time" + + "github.com/canonical/lxd/shared/api" + "github.com/canonical/microceph/microceph/api/types" + microCli "github.com/canonical/microcluster/client" +) + +func SetConfig(ctx context.Context, c *microCli.Client, data *types.Config) error { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*200) + defer cancel() + + err := c.Query(queryCtx, "PUT", api.NewURL().Path("configs"), data, nil) + if err != nil { + return fmt.Errorf("failed setting cluster config: %w, Key: %s, Value: %s", err, data.Key, data.Value) + } + + return nil +} + +func ClearConfig(ctx context.Context, c *microCli.Client, data *types.Config) error { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*200) + defer cancel() + + err := c.Query(queryCtx, "DELETE", api.NewURL().Path("configs"), data, nil) + if err != nil { + return fmt.Errorf("failed clearing cluster config: %w, Key: %s", err, data.Key) + } + + return nil +} + +func GetConfig(ctx context.Context, c *microCli.Client, data *types.Config) (types.Configs, error) { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*5) + defer cancel() + + configs := types.Configs{} + + err := c.Query(queryCtx, "GET", api.NewURL().Path("configs"), data, &configs) + if err != nil { + return nil, fmt.Errorf("failed to fetch cluster config: %w, Key: %s", err, data.Key) + } + + return configs, nil +} diff --git a/microceph/client/client.go b/microceph/client/disks.go similarity index 58% rename from microceph/client/client.go rename to microceph/client/disks.go index 3c9c1e02..383d9e27 100644 --- a/microceph/client/client.go +++ b/microceph/client/disks.go @@ -14,44 +14,6 @@ import ( "github.com/canonical/microceph/microceph/api/types" ) -func SetConfig(ctx context.Context, c *microCli.Client, data *types.Config) error { - queryCtx, cancel := context.WithTimeout(ctx, time.Second*200) - defer cancel() - - err := c.Query(queryCtx, "PUT", api.NewURL().Path("configs"), data, nil) - if err != nil { - return fmt.Errorf("Failed setting cluster config: %w, Key: %s, Value: %s", err, data.Key, data.Value) - } - - return nil -} - -func ClearConfig(ctx context.Context, c *microCli.Client, data *types.Config) error { - queryCtx, cancel := context.WithTimeout(ctx, time.Second*200) - defer cancel() - - err := c.Query(queryCtx, "DELETE", api.NewURL().Path("configs"), data, nil) - if err != nil { - return fmt.Errorf("Failed clearing cluster config: %w, Key: %s", err, data.Key) - } - - return nil -} - -func GetConfig(ctx context.Context, c *microCli.Client, data *types.Config) (types.Configs, error) { - queryCtx, cancel := context.WithTimeout(ctx, time.Second*5) - defer cancel() - - configs := types.Configs{} - - err := c.Query(queryCtx, "GET", api.NewURL().Path("configs"), data, &configs) - if err != nil { - return nil, fmt.Errorf("Failed to fetch cluster config: %w, Key: %s", err, data.Key) - } - - return configs, nil -} - // AddDisk requests Ceph sets up a new OSD. func AddDisk(ctx context.Context, c *microCli.Client, data *types.DisksPost) error { queryCtx, cancel := context.WithTimeout(ctx, time.Second*120) @@ -59,7 +21,7 @@ func AddDisk(ctx context.Context, c *microCli.Client, data *types.DisksPost) err err := c.Query(queryCtx, "POST", api.NewURL().Path("disks"), data, nil) if err != nil { - return fmt.Errorf("Failed adding new disk: %w", err) + return fmt.Errorf("failed adding new disk: %w", err) } return nil @@ -74,7 +36,7 @@ func GetDisks(ctx context.Context, c *microCli.Client) (types.Disks, error) { err := c.Query(queryCtx, "GET", api.NewURL().Path("disks"), nil, &disks) if err != nil { - return nil, fmt.Errorf("Failed listing disks: %w", err) + return nil, fmt.Errorf("failed listing disks: %w", err) } return disks, nil @@ -89,7 +51,7 @@ func GetResources(ctx context.Context, c *microCli.Client) (*api.ResourcesStorag err := c.Query(queryCtx, "GET", api.NewURL().Path("resources"), nil, &storage) if err != nil { - return nil, fmt.Errorf("Failed listing storage devices: %w", err) + return nil, fmt.Errorf("failed listing storage devices: %w", err) } return &storage, nil @@ -104,7 +66,7 @@ func RemoveDisk(ctx context.Context, c *microCli.Client, data *types.DisksDelete // get disks and determine osd location disks, err := GetDisks(ctx, c) if err != nil { - return fmt.Errorf("Failed to get disks: %w", err) + return fmt.Errorf("failed to get disks: %w", err) } var location string for _, disk := range disks { @@ -114,7 +76,7 @@ func RemoveDisk(ctx context.Context, c *microCli.Client, data *types.DisksDelete } } if location == "" { - return fmt.Errorf("Failed to find location for osd.%d", data.OSD) + return fmt.Errorf("failed to find location for osd.%d", data.OSD) } c = c.UseTarget(location) @@ -122,9 +84,9 @@ func RemoveDisk(ctx context.Context, c *microCli.Client, data *types.DisksDelete if err != nil { // Checking if the error is a context deadline exceeded error if errors.Is(err, context.DeadlineExceeded) { - return fmt.Errorf("Failed to remove disk, timeout (%ds) reached - abort", data.Timeout) + return fmt.Errorf("failed to remove disk, timeout (%ds) reached - abort", data.Timeout) } - return fmt.Errorf("Failed to remove disk: %w", err) + return fmt.Errorf("failed to remove disk: %w", err) } return nil } diff --git a/microceph/cmd/microceph/client.go b/microceph/cmd/microceph/client.go new file mode 100644 index 00000000..a8940fd1 --- /dev/null +++ b/microceph/cmd/microceph/client.go @@ -0,0 +1,26 @@ +package main + +import ( + "github.com/spf13/cobra" +) + +type cmdClient struct { + common *CmdControl +} + +func (c *cmdClient) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "client", + Short: "Manage the MicroCeph clients", + } + + // Config Subcommand + clientConfigCmd := cmdClientConfig{common: c.common, client: c} + cmd.AddCommand(clientConfigCmd.Command()) + + // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 + cmd.Args = cobra.NoArgs + cmd.Run = func(cmd *cobra.Command, args []string) { _ = cmd.Usage() } + + return cmd +} diff --git a/microceph/cmd/microceph/client_config.go b/microceph/cmd/microceph/client_config.go new file mode 100644 index 00000000..b0d6a387 --- /dev/null +++ b/microceph/cmd/microceph/client_config.go @@ -0,0 +1,39 @@ +package main + +import ( + "github.com/spf13/cobra" +) + +type cmdClientConfig struct { + common *CmdControl + client *cmdClient +} + +func (c *cmdClientConfig) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "config", + Short: "Manage Ceph Client configs", + } + + // Get + clientConfigGetCmd := cmdClientConfigGet{common: c.common, client: c.client, clientConfig: c} + cmd.AddCommand(clientConfigGetCmd.Command()) + + // Set + clientConfigSetCmd := cmdClientConfigSet{common: c.common, client: c.client, clientConfig: c} + cmd.AddCommand(clientConfigSetCmd.Command()) + + // Reset + clientConfigResetCmd := cmdClientConfigReset{common: c.common, client: c.client, clientConfig: c} + cmd.AddCommand(clientConfigResetCmd.Command()) + + // List + clientConfigListCmd := cmdClientConfigList{common: c.common, client: c.client, clientConfig: c} + cmd.AddCommand(clientConfigListCmd.Command()) + + // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 + cmd.Args = cobra.NoArgs + cmd.Run = func(cmd *cobra.Command, args []string) { _ = cmd.Usage() } + + return cmd +} diff --git a/microceph/cmd/microceph/client_config_get.go b/microceph/cmd/microceph/client_config_get.go new file mode 100644 index 00000000..dc59444e --- /dev/null +++ b/microceph/cmd/microceph/client_config_get.go @@ -0,0 +1,80 @@ +package main + +import ( + "context" + "fmt" + + lxdCmd "github.com/canonical/lxd/shared/cmd" + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/ceph" + "github.com/canonical/microceph/microceph/client" + "github.com/canonical/microcluster/microcluster" + "github.com/spf13/cobra" +) + +type cmdClientConfigGet struct { + common *CmdControl + client *cmdClient + clientConfig *cmdClientConfig + + flagHost string +} + +func (c *cmdClientConfigGet) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "get ", + Short: "Fetches specified Ceph Client config", + RunE: c.Run, + } + + // * stands for global configs, hence all configs are global by default unless specified. + cmd.Flags().StringVar(&c.flagHost, "target", "*", "Specify a microceph node the provided config should be applied to.") + return cmd +} + +func (c *cmdClientConfigGet) Run(cmd *cobra.Command, args []string) error { + allowList := ceph.GetClientConfigSet() + + // Get can be called with a single key. + if len(args) != 1 { + return cmd.Help() + } + + _, ok := allowList[args[0]] + if !ok { + return fmt.Errorf("key %s is invalid. \nSupported Keys: %v", args[0], allowList.Keys()) + } + + m, err := microcluster.App(context.Background(), microcluster.Args{StateDir: c.common.FlagStateDir, Verbose: c.common.FlagLogVerbose, Debug: c.common.FlagLogDebug}) + if err != nil { + return fmt.Errorf("unable to configure MicroCeph: %w", err) + } + + cli, err := m.LocalClient() + if err != nil { + return err + } + + req := &types.ClientConfig{ + Key: args[0], + Host: c.flagHost, + } + + configs, err := client.GetClientConfig(context.Background(), cli, req) + if err != nil { + return err + } + + data := make([][]string, len(configs)) + for i, config := range configs { + data[i] = []string{fmt.Sprintf("%d", i), config.Key, config.Value, config.Host} + } + + header := []string{"#", "Key", "Value", "Host"} + err = lxdCmd.RenderTable(lxdCmd.TableFormatTable, header, data, configs) + if err != nil { + return err + } + + return nil +} diff --git a/microceph/cmd/microceph/client_config_list.go b/microceph/cmd/microceph/client_config_list.go new file mode 100644 index 00000000..408ee12f --- /dev/null +++ b/microceph/cmd/microceph/client_config_list.go @@ -0,0 +1,70 @@ +package main + +import ( + "context" + "fmt" + + lxdCmd "github.com/canonical/lxd/shared/cmd" + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/client" + "github.com/canonical/microcluster/microcluster" + "github.com/spf13/cobra" +) + +type cmdClientConfigList struct { + common *CmdControl + client *cmdClient + clientConfig *cmdClientConfig + + flagHost string +} + +func (c *cmdClientConfigList) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "list", + Short: "Lists all configured Ceph Client configs", + RunE: c.Run, + } + + // * stands for global configs, hence all configs are global by default unless specifies. + cmd.Flags().StringVar(&c.flagHost, "target", "*", "Specify a microceph node the provided config should be applied to.") + return cmd +} + +func (c *cmdClientConfigList) Run(cmd *cobra.Command, args []string) error { + if len(args) != 0 { + return cmd.Help() + } + + m, err := microcluster.App(context.Background(), microcluster.Args{StateDir: c.common.FlagStateDir, Verbose: c.common.FlagLogVerbose, Debug: c.common.FlagLogDebug}) + if err != nil { + return fmt.Errorf("unable to configure MicroCeph: %w", err) + } + + cli, err := m.LocalClient() + if err != nil { + return err + } + + req := &types.ClientConfig{ + Host: c.flagHost, + } + + configs, err := client.ListClientConfig(context.Background(), cli, req) + if err != nil { + return err + } + + data := make([][]string, len(configs)) + for i, config := range configs { + data[i] = []string{fmt.Sprintf("%d", i), config.Key, config.Value, config.Host} + } + + header := []string{"#", "Key", "Value", "Host"} + err = lxdCmd.RenderTable(lxdCmd.TableFormatTable, header, data, configs) + if err != nil { + return err + } + + return nil +} diff --git a/microceph/cmd/microceph/client_config_reset.go b/microceph/cmd/microceph/client_config_reset.go new file mode 100644 index 00000000..87281875 --- /dev/null +++ b/microceph/cmd/microceph/client_config_reset.go @@ -0,0 +1,77 @@ +package main + +import ( + "context" + "fmt" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/ceph" + "github.com/canonical/microceph/microceph/client" + "github.com/canonical/microceph/microceph/common" + "github.com/canonical/microcluster/microcluster" + "github.com/spf13/cobra" +) + +type cmdClientConfigReset struct { + common *CmdControl + client *cmdClient + clientConfig *cmdClientConfig + + flagWait bool + flagHost string + flagForce bool +} + +func (c *cmdClientConfigReset) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "reset ", + Short: "Removes specified Ceph Client configs", + RunE: c.Run, + } + + cmd.Flags().BoolVar(&c.flagWait, "wait", true, "Wait for required ceph services to restart post config reset.") + // * stands for global configs, hence all configs are global by default unless specifies. + cmd.Flags().StringVar(&c.flagHost, "target", "*", "Specify a microceph node the provided config should be applied to.") + cmd.Flags().BoolVar(&c.flagForce, "yes-i-really-mean-it", false, "Force microceph to reset all client configs records for given key.") + return cmd +} + +func (c *cmdClientConfigReset) Run(cmd *cobra.Command, args []string) error { + allowList := ceph.GetClientConfigSet() + if len(args) != 1 { + return cmd.Help() + } + + _, ok := allowList[args[0]] + if !ok { + return fmt.Errorf("resetting key %s is not supported.\nSupported Keys: %v", args[0], allowList.Keys()) + } + + if !c.flagForce { + return fmt.Errorf("WARNING: this will *PERMANENTLY REMOVE* all records of the %s key. %s", + args[0], common.CliForcePrompt) + } + + m, err := microcluster.App(context.Background(), microcluster.Args{StateDir: c.common.FlagStateDir, Verbose: c.common.FlagLogVerbose, Debug: c.common.FlagLogDebug}) + if err != nil { + return fmt.Errorf("unable to configure MicroCeph: %w", err) + } + + cli, err := m.LocalClient() + if err != nil { + return err + } + + req := &types.ClientConfig{ + Key: args[0], + Wait: c.flagWait, + Host: c.flagHost, + } + + err = client.ResetClientConfig(context.Background(), cli, req) + if err != nil { + return err + } + + return nil +} diff --git a/microceph/cmd/microceph/client_config_set.go b/microceph/cmd/microceph/client_config_set.go new file mode 100644 index 00000000..300b6699 --- /dev/null +++ b/microceph/cmd/microceph/client_config_set.go @@ -0,0 +1,70 @@ +package main + +import ( + "context" + "fmt" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/ceph" + "github.com/canonical/microceph/microceph/client" + "github.com/canonical/microcluster/microcluster" + "github.com/spf13/cobra" +) + +type cmdClientConfigSet struct { + common *CmdControl + client *cmdClient + clientConfig *cmdClientConfig + + flagWait bool + flagHost string +} + +func (c *cmdClientConfigSet) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "set ", + Short: "Sets specified Ceph Client config", + RunE: c.Run, + } + + cmd.Flags().BoolVar(&c.flagWait, "wait", true, "Wait for configs to propagate across the cluster.") + // * stands for global configs, hence all configs are global by default unless specifies. + cmd.Flags().StringVar(&c.flagHost, "target", "*", "Specify a microceph node the provided config should be applied to.") + return cmd +} + +func (c *cmdClientConfigSet) Run(cmd *cobra.Command, args []string) error { + allowList := ceph.GetClientConfigSet() + if len(args) != 2 { + return cmd.Help() + } + + _, ok := allowList[args[0]] + if !ok { + return fmt.Errorf("configuring key %s is not supported.\nSupported Keys: %v", args[0], allowList.Keys()) + } + + m, err := microcluster.App(context.Background(), microcluster.Args{StateDir: c.common.FlagStateDir, Verbose: c.common.FlagLogVerbose, Debug: c.common.FlagLogDebug}) + if err != nil { + return fmt.Errorf("unable to configure MicroCeph: %w", err) + } + + cli, err := m.LocalClient() + if err != nil { + return err + } + + req := &types.ClientConfig{ + Key: args[0], + Value: args[1], + Wait: c.flagWait, + Host: c.flagHost, + } + + err = client.SetClientConfig(context.Background(), cli, req) + if err != nil { + return err + } + + return nil +} diff --git a/microceph/cmd/microceph/cluster_migrate.go b/microceph/cmd/microceph/cluster_migrate.go index 0f350727..bf7e430b 100644 --- a/microceph/cmd/microceph/cluster_migrate.go +++ b/microceph/cmd/microceph/cluster_migrate.go @@ -2,6 +2,7 @@ package main import ( "context" + "github.com/canonical/lxd/shared/logger" "github.com/canonical/microceph/microceph/api/types" "github.com/canonical/microceph/microceph/client" diff --git a/microceph/cmd/microceph/main.go b/microceph/cmd/microceph/main.go index 36d99810..ba20f1af 100644 --- a/microceph/cmd/microceph/main.go +++ b/microceph/cmd/microceph/main.go @@ -61,6 +61,9 @@ func main() { var cmdDisk = cmdDisk{common: &commonCmd} app.AddCommand(cmdDisk.Command()) + var cmdClient = cmdClient{common: &commonCmd} + app.AddCommand(cmdClient.Command()) + app.InitDefaultHelpCmd() err := app.Execute() diff --git a/microceph/common/cluster.go b/microceph/common/cluster.go new file mode 100644 index 00000000..7ad5e21e --- /dev/null +++ b/microceph/common/cluster.go @@ -0,0 +1,26 @@ +package common + +import "github.com/canonical/lxd/shared/logger" + +func GetClusterMemberNames(s StateInterface) ([]string, error) { + leader, err := s.ClusterState().Leader() + if err != nil { + return nil, err + } + + members, err := leader.GetClusterMembers(s.ClusterState().Context) + if err != nil { + return nil, err + } + + logger.Infof("Cluster Members are: %v", members) + + memberNames := make([]string, len(members)) + for i, member := range members { + memberNames[i] = member.Name + } + + logger.Infof("Cluster Members Names are: %v", memberNames) + + return memberNames, nil +} diff --git a/microceph/common/constants.go b/microceph/common/constants.go index 54ac15aa..9a97e5b0 100644 --- a/microceph/common/constants.go +++ b/microceph/common/constants.go @@ -6,6 +6,10 @@ import ( "path/filepath" ) +const ClientConfigGlobalHostConst = "*" + +const CliForcePrompt = "If you are *ABSOLUTELY CERTAIN* that is what you want, pass --yes-i-really-mean-it." + type PathConst struct { ConfPath string RunPath string diff --git a/microceph/common/set.go b/microceph/common/set.go new file mode 100644 index 00000000..c0818541 --- /dev/null +++ b/microceph/common/set.go @@ -0,0 +1,30 @@ +package common + +type Set map[string]interface{} + +func (s Set) Keys() []string { + keys := make([]string, len(s)) + count := 0 + + for key := range s { + keys[count] = key + count++ + } + + return keys +} + +func (s Set) IsIn(super Set) bool { + flag := true + + // mark flag false if any key from subset is not present in superset. + for key := range s { + _, ok := super[key] + if !ok { + flag = false + break // Break the loop. + } + } + + return flag +} diff --git a/microceph/database/client_config.go b/microceph/database/client_config.go new file mode 100644 index 00000000..97725171 --- /dev/null +++ b/microceph/database/client_config.go @@ -0,0 +1,37 @@ +package database + +//go:generate -command mapper lxd-generate db mapper -t client_config.mapper.go +//go:generate mapper reset +// +//go:generate mapper stmt -d github.com/canonical/microcluster/cluster -e ClientConfigItem objects table=client_config +//go:generate mapper stmt -d github.com/canonical/microcluster/cluster -e ClientConfigItem objects-by-Key table=client_config +//go:generate mapper stmt -d github.com/canonical/microcluster/cluster -e ClientConfigItem objects-by-Host table=client_config +//go:generate mapper stmt -d github.com/canonical/microcluster/cluster -e ClientConfigItem objects-by-Key-and-Host table=client_config +//go:generate mapper stmt -d github.com/canonical/microcluster/cluster -e ClientConfigItem id table=client_config +//go:generate mapper stmt -d github.com/canonical/microcluster/cluster -e ClientConfigItem create table=client_config +//go:generate mapper stmt -d github.com/canonical/microcluster/cluster -e ClientConfigItem delete-by-Key table=client_config +//go:generate mapper stmt -d github.com/canonical/microcluster/cluster -e ClientConfigItem delete-by-Host table=client_config +//go:generate mapper stmt -d github.com/canonical/microcluster/cluster -e ClientConfigItem delete-by-Key-and-Host table=client_config +//go:generate mapper stmt -d github.com/canonical/microcluster/cluster -e ClientConfigItem update table=client_config + +// +//go:generate mapper method -i -d github.com/canonical/microcluster/cluster -e ClientConfigItem GetMany table=client_config +//go:generate mapper method -i -d github.com/canonical/microcluster/cluster -e ClientConfigItem GetOne table=client_config +//go:generate mapper method -i -d github.com/canonical/microcluster/cluster -e ClientConfigItem ID table=client_config +//go:generate mapper method -i -d github.com/canonical/microcluster/cluster -e ClientConfigItem Exists table=client_config +//go:generate mapper method -i -d github.com/canonical/microcluster/cluster -e ClientConfigItem Create table=client_config +//go:generate mapper method -i -d github.com/canonical/microcluster/cluster -e ClientConfigItem DeleteOne-by-Key-and-Host table=client_config +//go:generate mapper method -i -d github.com/canonical/microcluster/cluster -e ClientConfigItem DeleteMany-by-Key table=client_config +//go:generate mapper method -i -d github.com/canonical/microcluster/cluster -e ClientConfigItem Update table=client_config + +type ClientConfigItem struct { + ID int + Host string `db:"primary=yes&join=internal_cluster_members.name&joinon=client_config.member_id"` + Key string `db:"primary=yes"` + Value string +} + +type ClientConfigItemFilter struct { + Host *string + Key *string +} diff --git a/microceph/database/client_config.mapper.go b/microceph/database/client_config.mapper.go new file mode 100644 index 00000000..bc7de4c5 --- /dev/null +++ b/microceph/database/client_config.mapper.go @@ -0,0 +1,424 @@ +package database + +// The code below was generated by lxd-generate - DO NOT EDIT! + +import ( + "context" + "database/sql" + "errors" + "fmt" + "net/http" + "strings" + + "github.com/canonical/lxd/lxd/db/query" + "github.com/canonical/lxd/shared/api" + "github.com/canonical/microcluster/cluster" +) + +var _ = api.ServerEnvironment{} + +var clientConfigItemObjects = cluster.RegisterStmt(` +SELECT client_config.id, internal_cluster_members.name AS host, client_config.key, client_config.value + FROM client_config + JOIN internal_cluster_members ON client_config.member_id = internal_cluster_members.id + ORDER BY internal_cluster_members.id, client_config.key +`) + +var clientConfigItemObjectsByKey = cluster.RegisterStmt(` +SELECT client_config.id, internal_cluster_members.name AS host, client_config.key, client_config.value + FROM client_config + JOIN internal_cluster_members ON client_config.member_id = internal_cluster_members.id + WHERE ( client_config.key = ? ) + ORDER BY internal_cluster_members.id, client_config.key +`) + +var clientConfigItemObjectsByHost = cluster.RegisterStmt(` +SELECT client_config.id, internal_cluster_members.name AS host, client_config.key, client_config.value + FROM client_config + JOIN internal_cluster_members ON client_config.member_id = internal_cluster_members.id + WHERE ( host = ? ) + ORDER BY internal_cluster_members.id, client_config.key +`) + +var clientConfigItemObjectsByKeyAndHost = cluster.RegisterStmt(` +SELECT client_config.id, internal_cluster_members.name AS host, client_config.key, client_config.value + FROM client_config + JOIN internal_cluster_members ON client_config.member_id = internal_cluster_members.id + WHERE ( client_config.key = ? AND host = ? ) + ORDER BY internal_cluster_members.id, client_config.key +`) + +var clientConfigItemID = cluster.RegisterStmt(` +SELECT client_config.id FROM client_config + JOIN internal_cluster_members ON client_config.member_id = internal_cluster_members.id + WHERE internal_cluster_members.name = ? AND client_config.key = ? +`) + +var clientConfigItemCreate = cluster.RegisterStmt(` +INSERT INTO client_config (member_id, key, value) + VALUES ((SELECT internal_cluster_members.id FROM internal_cluster_members WHERE internal_cluster_members.name = ?), ?, ?) +`) + +var clientConfigItemDeleteByKey = cluster.RegisterStmt(` +DELETE FROM client_config WHERE key = ? +`) + +var clientConfigItemDeleteByHost = cluster.RegisterStmt(` +DELETE FROM client_config WHERE member_id = (SELECT internal_cluster_members.id FROM internal_cluster_members WHERE internal_cluster_members.name = ?) +`) + +var clientConfigItemDeleteByKeyAndHost = cluster.RegisterStmt(` +DELETE FROM client_config WHERE key = ? AND member_id = (SELECT internal_cluster_members.id FROM internal_cluster_members WHERE internal_cluster_members.name = ?) +`) + +var clientConfigItemUpdate = cluster.RegisterStmt(` +UPDATE client_config + SET member_id = (SELECT internal_cluster_members.id FROM internal_cluster_members WHERE internal_cluster_members.name = ?), key = ?, value = ? + WHERE id = ? +`) + +// clientConfigItemColumns returns a string of column names to be used with a SELECT statement for the entity. +// Use this function when building statements to retrieve database entries matching the ClientConfigItem entity. +func clientConfigItemColumns() string { + return "client_config.id, internal_cluster_members.name AS host, client_config.key, client_config.value" +} + +// getClientConfigItems can be used to run handwritten sql.Stmts to return a slice of objects. +func getClientConfigItems(ctx context.Context, stmt *sql.Stmt, args ...any) ([]ClientConfigItem, error) { + objects := make([]ClientConfigItem, 0) + + dest := func(scan func(dest ...any) error) error { + c := ClientConfigItem{} + err := scan(&c.ID, &c.Host, &c.Key, &c.Value) + if err != nil { + return err + } + + objects = append(objects, c) + + return nil + } + + err := query.SelectObjects(ctx, stmt, dest, args...) + if err != nil { + return nil, fmt.Errorf("Failed to fetch from \"client_config\" table: %w", err) + } + + return objects, nil +} + +// getClientConfigItemsRaw can be used to run handwritten query strings to return a slice of objects. +func getClientConfigItemsRaw(ctx context.Context, tx *sql.Tx, sql string, args ...any) ([]ClientConfigItem, error) { + objects := make([]ClientConfigItem, 0) + + dest := func(scan func(dest ...any) error) error { + c := ClientConfigItem{} + err := scan(&c.ID, &c.Host, &c.Key, &c.Value) + if err != nil { + return err + } + + objects = append(objects, c) + + return nil + } + + err := query.Scan(ctx, tx, sql, dest, args...) + if err != nil { + return nil, fmt.Errorf("Failed to fetch from \"client_config\" table: %w", err) + } + + return objects, nil +} + +// GetClientConfigItems returns all available ClientConfigItems. +// generator: ClientConfigItem GetMany +func GetClientConfigItems(ctx context.Context, tx *sql.Tx, filters ...ClientConfigItemFilter) ([]ClientConfigItem, error) { + var err error + + // Result slice. + objects := make([]ClientConfigItem, 0) + + // Pick the prepared statement and arguments to use based on active criteria. + var sqlStmt *sql.Stmt + args := []any{} + queryParts := [2]string{} + + if len(filters) == 0 { + sqlStmt, err = cluster.Stmt(tx, clientConfigItemObjects) + if err != nil { + return nil, fmt.Errorf("Failed to get \"clientConfigItemObjects\" prepared statement: %w", err) + } + } + + for i, filter := range filters { + if filter.Key != nil && filter.Host != nil { + args = append(args, []any{filter.Key, filter.Host}...) + if len(filters) == 1 { + sqlStmt, err = cluster.Stmt(tx, clientConfigItemObjectsByKeyAndHost) + if err != nil { + return nil, fmt.Errorf("Failed to get \"clientConfigItemObjectsByKeyAndHost\" prepared statement: %w", err) + } + + break + } + + query, err := cluster.StmtString(clientConfigItemObjectsByKeyAndHost) + if err != nil { + return nil, fmt.Errorf("Failed to get \"clientConfigItemObjects\" prepared statement: %w", err) + } + + parts := strings.SplitN(query, "ORDER BY", 2) + if i == 0 { + copy(queryParts[:], parts) + continue + } + + _, where, _ := strings.Cut(parts[0], "WHERE") + queryParts[0] += "OR" + where + } else if filter.Key != nil && filter.Host == nil { + args = append(args, []any{filter.Key}...) + if len(filters) == 1 { + sqlStmt, err = cluster.Stmt(tx, clientConfigItemObjectsByKey) + if err != nil { + return nil, fmt.Errorf("Failed to get \"clientConfigItemObjectsByKey\" prepared statement: %w", err) + } + + break + } + + query, err := cluster.StmtString(clientConfigItemObjectsByKey) + if err != nil { + return nil, fmt.Errorf("Failed to get \"clientConfigItemObjects\" prepared statement: %w", err) + } + + parts := strings.SplitN(query, "ORDER BY", 2) + if i == 0 { + copy(queryParts[:], parts) + continue + } + + _, where, _ := strings.Cut(parts[0], "WHERE") + queryParts[0] += "OR" + where + } else if filter.Host != nil && filter.Key == nil { + args = append(args, []any{filter.Host}...) + if len(filters) == 1 { + sqlStmt, err = cluster.Stmt(tx, clientConfigItemObjectsByHost) + if err != nil { + return nil, fmt.Errorf("Failed to get \"clientConfigItemObjectsByHost\" prepared statement: %w", err) + } + + break + } + + query, err := cluster.StmtString(clientConfigItemObjectsByHost) + if err != nil { + return nil, fmt.Errorf("Failed to get \"clientConfigItemObjects\" prepared statement: %w", err) + } + + parts := strings.SplitN(query, "ORDER BY", 2) + if i == 0 { + copy(queryParts[:], parts) + continue + } + + _, where, _ := strings.Cut(parts[0], "WHERE") + queryParts[0] += "OR" + where + } else if filter.Host == nil && filter.Key == nil { + return nil, fmt.Errorf("Cannot filter on empty ClientConfigItemFilter") + } else { + return nil, fmt.Errorf("No statement exists for the given Filter") + } + } + + // Select. + if sqlStmt != nil { + objects, err = getClientConfigItems(ctx, sqlStmt, args...) + } else { + queryStr := strings.Join(queryParts[:], "ORDER BY") + objects, err = getClientConfigItemsRaw(ctx, tx, queryStr, args...) + } + + if err != nil { + return nil, fmt.Errorf("Failed to fetch from \"client_config\" table: %w", err) + } + + return objects, nil +} + +// GetClientConfigItem returns the ClientConfigItem with the given key. +// generator: ClientConfigItem GetOne +func GetClientConfigItem(ctx context.Context, tx *sql.Tx, host string, key string) (*ClientConfigItem, error) { + filter := ClientConfigItemFilter{} + filter.Host = &host + filter.Key = &key + + objects, err := GetClientConfigItems(ctx, tx, filter) + if err != nil { + return nil, fmt.Errorf("Failed to fetch from \"client_config\" table: %w", err) + } + + switch len(objects) { + case 0: + return nil, api.StatusErrorf(http.StatusNotFound, "ClientConfigItem not found") + case 1: + return &objects[0], nil + default: + return nil, fmt.Errorf("More than one \"client_config\" entry matches") + } +} + +// GetClientConfigItemID return the ID of the ClientConfigItem with the given key. +// generator: ClientConfigItem ID +func GetClientConfigItemID(ctx context.Context, tx *sql.Tx, host string, key string) (int64, error) { + stmt, err := cluster.Stmt(tx, clientConfigItemID) + if err != nil { + return -1, fmt.Errorf("Failed to get \"clientConfigItemID\" prepared statement: %w", err) + } + + row := stmt.QueryRowContext(ctx, host, key) + var id int64 + err = row.Scan(&id) + if errors.Is(err, sql.ErrNoRows) { + return -1, api.StatusErrorf(http.StatusNotFound, "ClientConfigItem not found") + } + + if err != nil { + return -1, fmt.Errorf("Failed to get \"client_config\" ID: %w", err) + } + + return id, nil +} + +// ClientConfigItemExists checks if a ClientConfigItem with the given key exists. +// generator: ClientConfigItem Exists +func ClientConfigItemExists(ctx context.Context, tx *sql.Tx, host string, key string) (bool, error) { + _, err := GetClientConfigItemID(ctx, tx, host, key) + if err != nil { + if api.StatusErrorCheck(err, http.StatusNotFound) { + return false, nil + } + + return false, err + } + + return true, nil +} + +// CreateClientConfigItem adds a new ClientConfigItem to the database. +// generator: ClientConfigItem Create +func CreateClientConfigItem(ctx context.Context, tx *sql.Tx, object ClientConfigItem) (int64, error) { + // Check if a ClientConfigItem with the same key exists. + exists, err := ClientConfigItemExists(ctx, tx, object.Host, object.Key) + if err != nil { + return -1, fmt.Errorf("Failed to check for duplicates: %w", err) + } + + if exists { + return -1, api.StatusErrorf(http.StatusConflict, "This \"client_config\" entry already exists") + } + + args := make([]any, 3) + + // Populate the statement arguments. + args[0] = object.Host + args[1] = object.Key + args[2] = object.Value + + // Prepared statement to use. + stmt, err := cluster.Stmt(tx, clientConfigItemCreate) + if err != nil { + return -1, fmt.Errorf("Failed to get \"clientConfigItemCreate\" prepared statement: %w", err) + } + + // Execute the statement. + result, err := stmt.Exec(args...) + if err != nil { + return -1, fmt.Errorf("Failed to create \"client_config\" entry: %w", err) + } + + id, err := result.LastInsertId() + if err != nil { + return -1, fmt.Errorf("Failed to fetch \"client_config\" entry ID: %w", err) + } + + return id, nil +} + +// DeleteClientConfigItem deletes the ClientConfigItem matching the given key parameters. +// generator: ClientConfigItem DeleteOne-by-Key-and-Host +func DeleteClientConfigItem(ctx context.Context, tx *sql.Tx, key string, host string) error { + stmt, err := cluster.Stmt(tx, clientConfigItemDeleteByKeyAndHost) + if err != nil { + return fmt.Errorf("Failed to get \"clientConfigItemDeleteByKeyAndHost\" prepared statement: %w", err) + } + + result, err := stmt.Exec(key, host) + if err != nil { + return fmt.Errorf("Delete \"client_config\": %w", err) + } + + n, err := result.RowsAffected() + if err != nil { + return fmt.Errorf("Fetch affected rows: %w", err) + } + + if n == 0 { + return api.StatusErrorf(http.StatusNotFound, "ClientConfigItem not found") + } else if n > 1 { + return fmt.Errorf("Query deleted %d ClientConfigItem rows instead of 1", n) + } + + return nil +} + +// DeleteClientConfigItems deletes the ClientConfigItem matching the given key parameters. +// generator: ClientConfigItem DeleteMany-by-Key +func DeleteClientConfigItems(ctx context.Context, tx *sql.Tx, key string) error { + stmt, err := cluster.Stmt(tx, clientConfigItemDeleteByKey) + if err != nil { + return fmt.Errorf("Failed to get \"clientConfigItemDeleteByKey\" prepared statement: %w", err) + } + + result, err := stmt.Exec(key) + if err != nil { + return fmt.Errorf("Delete \"client_config\": %w", err) + } + + _, err = result.RowsAffected() + if err != nil { + return fmt.Errorf("Fetch affected rows: %w", err) + } + + return nil +} + +// UpdateClientConfigItem updates the ClientConfigItem matching the given key parameters. +// generator: ClientConfigItem Update +func UpdateClientConfigItem(ctx context.Context, tx *sql.Tx, host string, key string, object ClientConfigItem) error { + id, err := GetClientConfigItemID(ctx, tx, host, key) + if err != nil { + return err + } + + stmt, err := cluster.Stmt(tx, clientConfigItemUpdate) + if err != nil { + return fmt.Errorf("Failed to get \"clientConfigItemUpdate\" prepared statement: %w", err) + } + + result, err := stmt.Exec(object.Host, object.Key, object.Value, id) + if err != nil { + return fmt.Errorf("Update \"client_config\" entry failed: %w", err) + } + + n, err := result.RowsAffected() + if err != nil { + return fmt.Errorf("Fetch affected rows: %w", err) + } + + if n != 1 { + return fmt.Errorf("Query updated %d rows instead of 1", n) + } + + return nil +} diff --git a/microceph/database/client_config_extras.go b/microceph/database/client_config_extras.go new file mode 100644 index 00000000..c76958b2 --- /dev/null +++ b/microceph/database/client_config_extras.go @@ -0,0 +1,340 @@ +package database + +import ( + "context" + "database/sql" + "fmt" + + "github.com/canonical/lxd/lxd/db/query" + "github.com/canonical/lxd/shared/api" + "github.com/canonical/lxd/shared/logger" + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/common" + "github.com/canonical/microcluster/cluster" + "github.com/canonical/microcluster/state" +) + +var _ = api.ServerEnvironment{} + +var globalClientConfigItemObjects = cluster.RegisterStmt(` +SELECT client_config.id, client_config.key, client_config.value FROM client_config + WHERE client_config.member_id IS NULL + ORDER BY client_config.key +`) + +var globalClientConfigItemObjectByKey = cluster.RegisterStmt(` +SELECT client_config.id, client_config.key, client_config.value FROM client_config + WHERE ( client_config.key = ? AND client_config.member_id IS NULL ) +`) + +var globalClientConfigItemCreateOrUpdate = cluster.RegisterStmt(` +INSERT OR REPLACE INTO client_config (member_id, key, value) + VALUES (NULL, ?, ?) +`) + +var clientConfigItemCreateOrUpdate = cluster.RegisterStmt(` +INSERT OR REPLACE INTO client_config (member_id, key, value) + VALUES ((SELECT internal_cluster_members.id FROM internal_cluster_members WHERE internal_cluster_members.name = ?), ?, ?) +`) + +// Slice of ClientConfigItem(s) +type ClientConfigItems []ClientConfigItem + +// GetClientConfigSlice translates a slice of ClientConfigItems (used in DB ops) to types.ClientConfigs (used in API ops) +func (cci ClientConfigItems) GetClientConfigSlice() types.ClientConfigs { + var host string + ccs := make(types.ClientConfigs, len(cci)) + for i, configItem := range cci { + if len(configItem.Host) > 0 { + host = configItem.Host + } else { + host = common.ClientConfigGlobalHostConst + } + + ccs[i] = types.ClientConfig{ + Key: configItem.Key, + Value: configItem.Value, + Host: host, + } + } + + return ccs +} + +type ClientConfigQueryIntf interface { + + // Add Method + AddNew(s *state.State, key string, value string, host string) error + + // Fetch Methods + GetAll(s *state.State) (ClientConfigItems, error) + GetAllForKey(s *state.State, key string) (ClientConfigItems, error) + GetAllForHost(s *state.State, host string) (ClientConfigItems, error) + GetAllForKeyAndHost(s *state.State, key string, host string) (ClientConfigItems, error) + + // Delete Methods + RemoveAllForKey(s *state.State, key string) error + RemoveOneForKeyAndHost(s *state.State, key string, host string) error +} + +type ClientConfigQueryImpl struct{} + +// Add Method +func (ccq ClientConfigQueryImpl) AddNew(s *state.State, key string, value string, host string) error { + err := s.Database.Transaction(s.Context, func(ctx context.Context, tx *sql.Tx) error { + data := ClientConfigItem{ + Key: key, + Value: value, + Host: host, + } + // Add record to database. + err := createOrUpdateClientConfigItem(ctx, tx, data) + if err != nil { + return fmt.Errorf("failed to add client config: %v", err) + } + + return nil + }) + if err != nil { + return err + } + return nil +} + +// Fetch Methods +func (ccq ClientConfigQueryImpl) GetAll(s *state.State) (ClientConfigItems, error) { + globalConfigs, err := ccq.GetGlobalConfigs(s, "") + if err != nil { + return nil, fmt.Errorf("failed to fetch global client configs: %w", err) + } + + logger.Infof("Global Configs: %v", globalConfigs) + + hostConfigs, err := ccq.GetAllForFilter(s) + if err != nil { + return nil, fmt.Errorf("failed to fetch host configured client configs: %w", err) + } + + logger.Infof("Host Configs: %v", hostConfigs) + + return append(globalConfigs, hostConfigs...), nil +} + +func (ccq ClientConfigQueryImpl) GetAllForKey(s *state.State, key string) (ClientConfigItems, error) { + globalConfigs, err := ccq.GetGlobalConfigs(s, key) + if err != nil { + return nil, fmt.Errorf("failed to fetch global client configs, key %s: %w", key, err) + } + + logger.Infof("Global Configs: %v", globalConfigs) + + hostConfigs, err := ccq.GetAllForFilter(s, ClientConfigItemFilter{Host: nil, Key: &key}) + if err != nil { + return nil, fmt.Errorf("failed to fetch host configured client configs, key %s: %w", key, err) + } + + logger.Infof("Host Configs: %v", hostConfigs) + + return append(globalConfigs, hostConfigs...), nil +} + +func (ccq ClientConfigQueryImpl) GetAllForHost(s *state.State, host string) (ClientConfigItems, error) { + globalConfigs, err := ccq.GetGlobalConfigs(s, "") + if err != nil { + return nil, fmt.Errorf("failed to fetch global client configs, host %s: %w", host, err) + } + + logger.Infof("Global Configs: %v", globalConfigs) + + hostConfigs, err := ccq.GetAllForFilter(s, ClientConfigItemFilter{Host: &host, Key: nil}) + if err != nil { + return nil, fmt.Errorf("failed to fetch host client configs, host %s: %w", host, err) + } + + logger.Infof("Host Configs: %v", hostConfigs) + + return squashClientConfigs(globalConfigs, hostConfigs), nil +} + +func (ccq ClientConfigQueryImpl) GetAllForKeyAndHost(s *state.State, key string, host string) (ClientConfigItems, error) { + return ccq.GetAllForFilter(s, ClientConfigItemFilter{Host: &host, Key: &key}) +} + +func (ccq ClientConfigQueryImpl) GetAllForFilter(s *state.State, filters ...ClientConfigItemFilter) (ClientConfigItems, error) { + var err error + var retval []ClientConfigItem + + err = s.Database.Transaction(s.Context, func(ctx context.Context, tx *sql.Tx) error { + retval, err = GetClientConfigItems(ctx, tx, filters...) + if err != nil { + return err + } + return nil + }) + if err != nil { + return retval, err + } + return retval, nil +} + +// Fetch client configs using registered sql stmt and args +func (ccq ClientConfigQueryImpl) GetGlobalConfigs(s *state.State, key string) ([]ClientConfigItem, error) { + var err error + objects := make([]ClientConfigItem, 0) + + // scan handler for global configs. + dest := func(scan func(dest ...any) error) error { + c := ClientConfigItem{Host: common.ClientConfigGlobalHostConst} + err := scan(&c.ID, &c.Key, &c.Value) + if err != nil { + return err + } + + objects = append(objects, c) + + logger.Infof("Object: %v, Objects: %v", c, objects) + return nil + } + + err = s.Database.Transaction(s.Context, func(ctx context.Context, tx *sql.Tx) error { + if len(key) != 0 { + return getOneGlobalConfigByKey(ctx, tx, dest, key) + } + + return getAllGlobalConfigs(ctx, tx, dest) + }) + if err != nil { + return nil, err + } + + return objects, nil +} + +// Delete Methods +func (ccq ClientConfigQueryImpl) RemoveAllForKey(s *state.State, key string) error { + err := s.Database.Transaction(s.Context, func(ctx context.Context, tx *sql.Tx) error { + err := DeleteClientConfigItems(ctx, tx, key) + if err != nil { + return fmt.Errorf("failed to clean existing keys %s: %v", key, err) + } + + return nil + }) + if err != nil { + return err + } + return nil +} + +func (ccq ClientConfigQueryImpl) RemoveOneForKeyAndHost(s *state.State, key string, host string) error { + err := s.Database.Transaction(s.Context, func(ctx context.Context, tx *sql.Tx) error { + err := DeleteClientConfigItem(ctx, tx, key, host) + if err != nil { + return fmt.Errorf("failed to clean existing keys %s: %v", key, err) + } + + return nil + }) + if err != nil { + return err + } + return nil +} + +/******************** HELPER FUNCTIONS ********************/ +// createOrUpdateClientConfigItem adds a new ClientConfigItem to the database or updates the existing one if it exists. +func createOrUpdateClientConfigItem(ctx context.Context, tx *sql.Tx, object ClientConfigItem) error { + var stmtIndex int + var args []any + + // Populate the statement arguments. + if object.Host == common.ClientConfigGlobalHostConst { + args = append(args, object.Key) + args = append(args, object.Value) + stmtIndex = globalClientConfigItemCreateOrUpdate + } else { + args = append(args, object.Host) + args = append(args, object.Key) + args = append(args, object.Value) + stmtIndex = clientConfigItemCreateOrUpdate + } + + // Prepared statement to use. + stmt, err := cluster.Stmt(tx, stmtIndex) + if err != nil { + return fmt.Errorf("failed to prepare statement for %v: %w", object, err) + } + + // Execute the statement. + _, err = stmt.Exec(args...) + if err != nil { + return fmt.Errorf("failed to insert %v: %w", object, err) + } + + return nil +} + +// Squash host configs over global configs to generate a slice of applicable configs for a host. +func squashClientConfigs(globalConfigs ClientConfigItems, hostConfigs ClientConfigItems) ClientConfigItems { + configMap := map[string]ClientConfigItem{} + + // Populate global configs in the output configs object. + for _, config := range globalConfigs { + configMap[config.Key] = config + } + + logger.Infof("Map post global key updation: %v", configMap) + + // Overwrite global configs if host level config exists. + for _, config := range hostConfigs { + configMap[config.Key] = config + } + + logger.Infof("Map post host key updation: %v", configMap) + + configs := ClientConfigItems{} + for _, item := range configMap { + configs = append(configs, item) + } + + logger.Infof("Squashed slice: %v", configs) + + return configs +} + +// getAllGlobalConfigs performs sql query for all global configurations. +func getAllGlobalConfigs(ctx context.Context, tx *sql.Tx, rowFunc query.Dest) error { + queryStr, err := cluster.StmtString(globalClientConfigItemObjects) + if err != nil { + return fmt.Errorf("failed to parse sql stmt table: %w", err) + } + + logger.Infof("Query Str: %s", queryStr) + + err = query.Scan(ctx, tx, queryStr, rowFunc) + if err != nil { + return fmt.Errorf("failed to fetch from client_config table: %w", err) + } + + return nil +} + +// getOneGlobalConfigByKey performs sql query for a single global configuration using config key. +func getOneGlobalConfigByKey(ctx context.Context, tx *sql.Tx, rowFunc query.Dest, key string) error { + queryStr, err := cluster.StmtString(globalClientConfigItemObjectByKey) + if err != nil { + return fmt.Errorf("failed to parse sql stmt table: %w", err) + } + + logger.Infof("Query Str: %s", queryStr) + + err = query.Scan(ctx, tx, queryStr, rowFunc, key) + if err != nil { + return fmt.Errorf("failed to fetch from client_config table: %w", err) + } + + return nil +} + +// Singleton for mocker +var ClientConfigQuery ClientConfigQueryIntf = ClientConfigQueryImpl{} diff --git a/microceph/database/disk_extras.go b/microceph/database/disk_extras.go index 91fda831..dca9251e 100644 --- a/microceph/database/disk_extras.go +++ b/microceph/database/disk_extras.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "github.com/canonical/microceph/microceph/api/types" "github.com/canonical/lxd/lxd/db/query" diff --git a/microceph/database/schema.go b/microceph/database/schema.go index 525dad8c..9e90468b 100644 --- a/microceph/database/schema.go +++ b/microceph/database/schema.go @@ -12,6 +12,7 @@ import ( // Each entry will increase the database schema version by one, and will be applied after internal schema updates. var SchemaExtensions = map[int]schema.Update{ 1: schemaUpdate1, + 2: schemaUpdate2, } func schemaUpdate1(ctx context.Context, tx *sql.Tx) error { @@ -46,3 +47,22 @@ CREATE TABLE services ( return err } + +// Adds client config table in database schema. +func schemaUpdate2(ctx context.Context, tx *sql.Tx) error { + stmt := ` +CREATE TABLE client_config ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + member_id INTEGER, + key TEXT NOT NULL, + value TEXT NOT NULL, + FOREIGN KEY (member_id) REFERENCES "internal_cluster_members" (id) ON DELETE CASCADE +); + +CREATE UNIQUE INDEX cc_index ON client_config(coalesce(member_id, 0), key); + ` + + _, err := tx.Exec(stmt) + + return err +} diff --git a/microceph/mocks/ClientConfigQueryIntf.go b/microceph/mocks/ClientConfigQueryIntf.go new file mode 100644 index 00000000..8208492c --- /dev/null +++ b/microceph/mocks/ClientConfigQueryIntf.go @@ -0,0 +1,174 @@ +// Code generated by mockery v2.30.10. DO NOT EDIT. + +package mocks + +import ( + state "github.com/canonical/microcluster/state" + mock "github.com/stretchr/testify/mock" + database "github.com/canonical/microceph/microceph/database" +) + +// ClientConfigQueryIntf is an autogenerated mock type for the ClientConfigQueryIntf type +type ClientConfigQueryIntf struct { + mock.Mock +} + +// AddNew provides a mock function with given fields: s, key, value, host +func (_m *ClientConfigQueryIntf) AddNew(s *state.State, key string, value string, host string) error { + ret := _m.Called(s, key, value, host) + + var r0 error + if rf, ok := ret.Get(0).(func(*state.State, string, string, string) error); ok { + r0 = rf(s, key, value, host) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// GetAll provides a mock function with given fields: s +func (_m *ClientConfigQueryIntf) GetAll(s *state.State) (database.ClientConfigItems, error) { + ret := _m.Called(s) + + var r0 database.ClientConfigItems + var r1 error + if rf, ok := ret.Get(0).(func(*state.State) (database.ClientConfigItems, error)); ok { + return rf(s) + } + if rf, ok := ret.Get(0).(func(*state.State) database.ClientConfigItems); ok { + r0 = rf(s) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(database.ClientConfigItems) + } + } + + if rf, ok := ret.Get(1).(func(*state.State) error); ok { + r1 = rf(s) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetAllForHost provides a mock function with given fields: s, host +func (_m *ClientConfigQueryIntf) GetAllForHost(s *state.State, host string) (database.ClientConfigItems, error) { + ret := _m.Called(s, host) + + var r0 database.ClientConfigItems + var r1 error + if rf, ok := ret.Get(0).(func(*state.State, string) (database.ClientConfigItems, error)); ok { + return rf(s, host) + } + if rf, ok := ret.Get(0).(func(*state.State, string) database.ClientConfigItems); ok { + r0 = rf(s, host) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(database.ClientConfigItems) + } + } + + if rf, ok := ret.Get(1).(func(*state.State, string) error); ok { + r1 = rf(s, host) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetAllForKey provides a mock function with given fields: s, key +func (_m *ClientConfigQueryIntf) GetAllForKey(s *state.State, key string) (database.ClientConfigItems, error) { + ret := _m.Called(s, key) + + var r0 database.ClientConfigItems + var r1 error + if rf, ok := ret.Get(0).(func(*state.State, string) (database.ClientConfigItems, error)); ok { + return rf(s, key) + } + if rf, ok := ret.Get(0).(func(*state.State, string) database.ClientConfigItems); ok { + r0 = rf(s, key) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(database.ClientConfigItems) + } + } + + if rf, ok := ret.Get(1).(func(*state.State, string) error); ok { + r1 = rf(s, key) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetAllForKeyAndHost provides a mock function with given fields: s, key, host +func (_m *ClientConfigQueryIntf) GetAllForKeyAndHost(s *state.State, key string, host string) (database.ClientConfigItems, error) { + ret := _m.Called(s, key, host) + + var r0 database.ClientConfigItems + var r1 error + if rf, ok := ret.Get(0).(func(*state.State, string, string) (database.ClientConfigItems, error)); ok { + return rf(s, key, host) + } + if rf, ok := ret.Get(0).(func(*state.State, string, string) database.ClientConfigItems); ok { + r0 = rf(s, key, host) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(database.ClientConfigItems) + } + } + + if rf, ok := ret.Get(1).(func(*state.State, string, string) error); ok { + r1 = rf(s, key, host) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// RemoveAllForKey provides a mock function with given fields: s, key +func (_m *ClientConfigQueryIntf) RemoveAllForKey(s *state.State, key string) error { + ret := _m.Called(s, key) + + var r0 error + if rf, ok := ret.Get(0).(func(*state.State, string) error); ok { + r0 = rf(s, key) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// RemoveOneForKeyAndHost provides a mock function with given fields: s, key, host +func (_m *ClientConfigQueryIntf) RemoveOneForKeyAndHost(s *state.State, key string, host string) error { + ret := _m.Called(s, key, host) + + var r0 error + if rf, ok := ret.Get(0).(func(*state.State, string, string) error); ok { + r0 = rf(s, key, host) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// NewClientConfigQueryIntf creates a new instance of ClientConfigQueryIntf. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewClientConfigQueryIntf(t interface { + mock.TestingT + Cleanup(func()) +}) *ClientConfigQueryIntf { + mock := &ClientConfigQueryIntf{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/tests/scripts/actionutils.sh b/tests/scripts/actionutils.sh index fe9fcd65..f57c5840 100755 --- a/tests/scripts/actionutils.sh +++ b/tests/scripts/actionutils.sh @@ -127,6 +127,39 @@ function refresh_snap() { done } +function check_client_configs() { + # Issue cluster wide client config set. + lxc exec node-head -- sh -c "microceph client config set rbd_cache true" + # Issue host specific client config set for each worker node. + for id in 1 2 3 ; do + lxc exec node-wrk${id} -- sh -c "microceph client config set rbd_cache_size $((512*$id))" + done + + # Verify client configs post set on each node. + for id in 1 2 3 ; do + res1=$(lxc exec node-wrk${id} -- sh -c "cat /var/snap/microceph/current/conf/ceph.conf | grep -c 'rbd_cache = true'") + res2=$(lxc exec node-wrk${id} -- sh -c "cat /var/snap/microceph/current/conf/ceph.conf | grep -c 'rbd_cache_size = $((512*$id))'") + if (($res1 -ne "1")) || (($res2 -ne "1")) ; then + # required configs not present. + exit 1 + fi + done + + # Reset client configs + lxc exec node-head -- sh -c "microceph client config reset rbd_cache" + lxc exec node-head -- sh -c "microceph client config reset rbd_cache_size" + + # Verify client configs post reset on each node. + for id in 1 2 3 ; do + res1=$(lxc exec node-wrk${id} -- sh -c "cat /var/snap/microceph/current/conf/ceph.conf | grep -c 'rbd_cache '") + res2=$(lxc exec node-wrk${id} -- sh -c "cat /var/snap/microceph/current/conf/ceph.conf | grep -c 'rbd_cache_size'") + if (($res1 -ne "0")) || (($res2 -ne "0")) ; then + # Incorrect configs present. + exit 1 + fi + done +} + function bootstrap_head() { # Bootstrap microceph on the head node lxc exec node-head -- sh -c "microceph cluster bootstrap"