Merge pull request juju#17103 from jack-w-shaw/JUJU-5381_back_enable_…

…ha_with_microceph

juju#17103

Use a (currently edge) microceph charm, integrated with the ceph-radosgw charm to provide an s3 endpoint the controller can use for distrbuted block storage

## Checklist

- [x] Code style: imports ordered, good names, simple structure, etc
- [x] Comments saying why design decisions were made
- ~[ ] Go unit tests, with comments saying what you're testing~
- [x] [Integration tests](https://github.com/juju/juju/tree/main/tests), with comments saying what you're testing
- ~[ ] [doc.go](https://discourse.charmhub.io/t/readme-in-packages/451) added or updated in changed packages~

## QA steps

NOTE: These integration tests currently fail. This is expected

#### lxd
```
./main.sh -v controller test_enable_ha
```

#### aws
```
./main.sh -v -c aws -p ec2 controller test_enable_ha
```

Both of these runs should successfully go into HA. They will fail exiting HA, however

tests/includes/microceph.sh

@@ -0,0 +1,32 @@
+enable_microceph_backed_storage() {
+	juju switch controller
+
+	if [[ $BOOTSTRAP_PROVIDER == "lxd" || $BOOTSTRAP_PROVIDER == "localhost" ]]; then
+		# Deploy to a virtual machine so loopback devices work out of the box
+		juju deploy microceph --channel="latest/edge" --config "snap-channel=reef/stable" --constraints "root-disk=16G cores=4 virt-type=virtual-machine"
+	else
+		juju deploy microceph --channel="latest/edge" --config "snap-channel=reef/stable" --constraints "root-disk=16G cores=4"
+	fi
+
+	# microceph is currently not a stable charm, so remove with force. Drop this when we use a stable release
+	add_clean_func "juju remove-application -m controller microceph --force"
+
+	juju deploy ceph-radosgw --channel "reef/stable"
+
+	wait_for "microceph" "$(idle_condition "microceph" 2)"
+
+	juju add-storage microceph/0 osd-standalone="loop,2G,3"
+	juju integrate microceph ceph-radosgw
+	juju expose ceph-radosgw
+
+	# NOTE: This can sometimes take surprisigly long
+	wait_for "ceph-radosgw" "$(active_idle_condition "ceph-radosgw")"
+
+	gw_ip=$(juju status --format json | jq -r '.applications["ceph-radosgw"].units["ceph-radosgw/0"]["public-address"]')
+	key=$(juju ssh microceph/0 "sudo radosgw-admin user create --uid juju --display-name Juju" | jq -r ".keys[0]")
+	juju controller-config \
+		"object-store-type=s3" \
+		"object-store-s3-endpoint=http://${gw_ip}:80" \
+		"object-store-s3-static-key=$(echo "$key" | jq -r ".access_key")" \
+		"object-store-s3-static-secret=$(echo "$key" | jq -r ".secret_key")"
+}

tests/suites/controller/enable_ha.sh

@@ -32,7 +32,7 @@ wait_for_ha() {
 	# shellcheck disable=SC2143
 	until [[ "$(juju show-controller --format=json | jq -r '.[] | .["controller-machines"] | .[] | select(.["ha-status"] == "ha-enabled") | .["instance-id"]' | wc -l | grep "${amount}")" ]]; do
 		echo "[+] (attempt ${attempt}) polling ha"
-		juju show-controller 2>&1 | sed 's/^/    | /g'
+		juju show-controller 2>&1 | yq '.[]["controller-machines"]' | sed 's/^/    | /g'
 		sleep "${SHORT_TIMEOUT}"
 		attempt=$((attempt + 1))
 
@@ -85,6 +85,8 @@ run_enable_ha() {
 
 	juju deploy jameinel-ubuntu-lite
 
+	enable_microceph_backed_storage
+
 	juju enable-ha
 
 	wait_for_controller_machines 3
@@ -96,13 +98,15 @@ run_enable_ha() {
 	# remove-machine will fail. Wait for the config to be
 	# settled before trying to tear down.
 	juju switch controller
-	wait_for "controller" "$(idle_condition "controller" 0 0)"
-	wait_for "controller" "$(idle_condition "controller" 0 1)"
-	wait_for "controller" "$(idle_condition "controller" 0 2)"
+	wait_for "controller" "$(idle_condition "controller" 1 0)"
+	wait_for "controller" "$(idle_condition "controller" 1 1)"
+	wait_for "controller" "$(idle_condition "controller" 1 2)"
 
 	juju switch enable-ha
-	juju remove-machine -m controller 1
-	juju remove-machine -m controller 2
+	controller_1=$(juju status -m controller --format json | jq -r '.applications.controller.units["controller/1"].machine')
+	juju remove-machine -m controller "${controller_1}"
+	controller_2=$(juju status -m controller --format json | jq -r '.applications.controller.units["controller/2"].machine')
+	juju remove-machine -m controller "${controller_2}"
 
 	wait_for_controller_no_leader
 	wait_for_controller_leader