From 7b3f99f0b8b41f4c9aac42c6052a765b2d9ce03a Mon Sep 17 00:00:00 2001
From: Jack Shaw <jack.shaw@canonical.com>
Date: Mon, 25 Mar 2024 09:42:11 +0000
Subject: [PATCH 1/2] Back enable-ha integration test with microceph storage

Use a (currently edge) microceph charm, integrated with the ceph-radosgw
charm to provide an s3 endpoint the controller can use for distrbuted
block storage
---
 tests/includes/microceph.sh          | 32 ++++++++++++++++++++++++++++
 tests/suites/controller/enable_ha.sh | 14 +++++++-----
 2 files changed, 41 insertions(+), 5 deletions(-)
 create mode 100644 tests/includes/microceph.sh

diff --git a/tests/includes/microceph.sh b/tests/includes/microceph.sh
new file mode 100644
index 00000000000..8d89a5f78ed
--- /dev/null
+++ b/tests/includes/microceph.sh
@@ -0,0 +1,32 @@
+enable_microceph_backed_storage() {
+	juju switch controller
+
+	if [[ $BOOTSTRAP_PROVIDER == "lxd" || $BOOTSTRAP_PROVIDER == "localhost" ]]; then
+		# Deploy to a virtual machine so loopback devices work out of the box
+		juju deploy microceph --channel="latest/edge" --config "snap-channel=reef/stable" --constraints "root-disk=16G cores=4 virt-type=virtual-machine"
+	else
+		juju deploy microceph --channel="latest/edge" --config "snap-channel=reef/stable" --constraints "root-disk=16G cores=4"
+	fi
+
+	# microceph is currently not a stable charm, so remove with force. Drop this when we use a stable release
+	add_clean_func "juju remove-application -m controller microceph --force"
+
+	juju deploy ceph-radosgw --channel "reef/stable"
+
+	wait_for "microceph" "$(idle_condition "microceph" 2)"
+
+	juju add-storage microceph/0 osd-standalone="loop,2G,3"
+	juju integrate microceph ceph-radosgw
+	juju expose ceph-radosgw
+
+	# NOTE: This can sometimes take surprisigly long
+	wait_for "ceph-radosgw" "$(active_idle_condition "ceph-radosgw")"
+
+	gw_ip=$(juju status --format json | jq -r '.applications["ceph-radosgw"].units["ceph-radosgw/0"]["public-address"]')
+	key=$(juju ssh microceph/0 "sudo radosgw-admin user create --uid juju --display-name Juju" | jq -r ".keys[0]")
+	juju controller-config \
+		"object-store-type=s3" \
+		"object-store-s3-endpoint=http://${gw_ip}:80" \
+		"object-store-s3-static-key=$(echo "$key" | jq -r ".access_key")" \
+		"object-store-s3-static-secret=$(echo "$key" | jq -r ".secret_key")"
+}
diff --git a/tests/suites/controller/enable_ha.sh b/tests/suites/controller/enable_ha.sh
index d2972299ff0..61ef8fd1628 100644
--- a/tests/suites/controller/enable_ha.sh
+++ b/tests/suites/controller/enable_ha.sh
@@ -85,6 +85,8 @@ run_enable_ha() {
 
 	juju deploy jameinel-ubuntu-lite
 
+	enable_microceph_backed_storage
+
 	juju enable-ha
 
 	wait_for_controller_machines 3
@@ -96,13 +98,15 @@ run_enable_ha() {
 	# remove-machine will fail. Wait for the config to be
 	# settled before trying to tear down.
 	juju switch controller
-	wait_for "controller" "$(idle_condition "controller" 0 0)"
-	wait_for "controller" "$(idle_condition "controller" 0 1)"
-	wait_for "controller" "$(idle_condition "controller" 0 2)"
+	wait_for "controller" "$(idle_condition "controller" 1 0)"
+	wait_for "controller" "$(idle_condition "controller" 1 1)"
+	wait_for "controller" "$(idle_condition "controller" 1 2)"
 
 	juju switch enable-ha
-	juju remove-machine -m controller 1
-	juju remove-machine -m controller 2
+	controller_1=$(juju status -m controller --format json | jq -r '.applications.controller.units["controller/1"].machine')
+	juju remove-machine -m controller "${controller_1}"
+	controller_2=$(juju status -m controller --format json | jq -r '.applications.controller.units["controller/2"].machine')
+	juju remove-machine -m controller "${controller_2}"
 
 	wait_for_controller_no_leader
 	wait_for_controller_leader

From 965c4913984bf5338732eb1f089c716dee8317e4 Mon Sep 17 00:00:00 2001
From: Jack Shaw <jack.shaw@canonical.com>
Date: Tue, 26 Mar 2024 13:24:22 +0000
Subject: [PATCH 2/2] Filter output from show-controller to show important info

This is a quality of life improvement. Significantly reduce console spam
---
 tests/suites/controller/enable_ha.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/suites/controller/enable_ha.sh b/tests/suites/controller/enable_ha.sh
index 61ef8fd1628..c2da925ddf7 100644
--- a/tests/suites/controller/enable_ha.sh
+++ b/tests/suites/controller/enable_ha.sh
@@ -32,7 +32,7 @@ wait_for_ha() {
 	# shellcheck disable=SC2143
 	until [[ "$(juju show-controller --format=json | jq -r '.[] | .["controller-machines"] | .[] | select(.["ha-status"] == "ha-enabled") | .["instance-id"]' | wc -l | grep "${amount}")" ]]; do
 		echo "[+] (attempt ${attempt}) polling ha"
-		juju show-controller 2>&1 | sed 's/^/    | /g'
+		juju show-controller 2>&1 | yq '.[]["controller-machines"]' | sed 's/^/    | /g'
 		sleep "${SHORT_TIMEOUT}"
 		attempt=$((attempt + 1))