Configure rbd mirroring

This change configure rbd mirroring between both clusters, and runs a
self test ensuring that mirroring works in both ways.

Since rbd mirroring depends on rook in both clusters, we must run it at
the end after both managed clusters are ready. The start script
configure mirroring in both ways in parallel, by splitting code to
configure and waiting, and waiting for both clusters in parallel. The
test scripts uses the same way to test mirroring in parallel.

Another way to do parallel configuration and testing would be to an
executor (like drenv uses) in the script. Yet another way would be to
run the global scripts in parallel, so we can configure and test rbd
mirroring while deploying and testing klusterlet. I think we need to
look at this later, after we add the metro DR environment.

Example usage:

    $ drenv start regional-dr.yaml
    2022-09-29 19:54:21,317 INFO    [env] Using regional-dr.yaml
    2022-09-29 19:54:21,322 INFO    [dr1] Starting cluster
    2022-09-29 19:54:21,323 INFO    [dr2] Starting cluster
    2022-09-29 19:54:21,324 INFO    [hub] Starting cluster
    2022-09-29 19:55:03,834 INFO    [dr2] Cluster started in 42.51 seconds
    2022-09-29 19:55:03,834 INFO    [dr2] Starting olm/start
    2022-09-29 19:55:21,664 INFO    [hub] Cluster started in 60.34 seconds
    2022-09-29 19:55:21,664 INFO    [hub] Starting olm/start
    2022-09-29 19:55:35,304 INFO    [dr2] olm/start completed in 31.47 seconds
    2022-09-29 19:55:35,304 INFO    [dr2] Starting rook/start
    2022-09-29 19:55:46,641 INFO    [dr1] Cluster started in 85.32 seconds
    2022-09-29 19:55:46,641 INFO    [dr1] Starting olm/start
    2022-09-29 19:55:48,224 INFO    [hub] olm/start completed in 26.56 seconds
    2022-09-29 19:55:48,224 INFO    [hub] Starting cluster-manager/start
    2022-09-29 19:56:15,552 INFO    [dr1] olm/start completed in 28.91 seconds
    2022-09-29 19:56:15,552 INFO    [dr1] Starting rook/start
    2022-09-29 19:57:03,221 INFO    [hub] cluster-manager/start completed in 75.00 seconds
    2022-09-29 19:58:26,053 INFO    [dr2] rook/start completed in 170.75 seconds
    2022-09-29 19:58:26,053 INFO    [dr2] Starting minio/start
    2022-09-29 19:58:47,126 INFO    [dr2] minio/start completed in 21.07 seconds
    2022-09-29 19:59:06,685 INFO    [dr1] rook/start completed in 171.13 seconds
    2022-09-29 19:59:06,685 INFO    [dr1] Starting minio/start
    2022-09-29 19:59:20,789 INFO    [dr1] minio/start completed in 14.10 seconds
    2022-09-29 19:59:20,789 INFO    [env] Starting klusterlet/start
    2022-09-29 20:00:12,250 INFO    [env] klusterlet/start completed in 51.46 seconds
    2022-09-29 20:00:12,250 INFO    [env] Starting klusterlet/test
    2022-09-29 20:00:35,980 INFO    [env] klusterlet/test completed in 23.73 seconds
    2022-09-29 20:00:35,980 INFO    [env] Starting rbd-mirror/start
    2022-09-29 20:01:26,111 INFO    [env] rbd-mirror/start completed in 50.13 seconds
    2022-09-29 20:01:26,111 INFO    [env] Starting rbd-mirror/test
    2022-09-29 20:01:41,611 INFO    [env] rbd-mirror/test completed in 15.50 seconds
    2022-09-29 20:01:41,611 INFO    [env] Started in 440.29 seconds

Signed-off-by: Nir Soffer <[email protected]>

test/rbd-mirror/rbd-mirror-secret.yaml

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: The RamenDR authors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+---
+apiVersion: v1
+data:
+  pool: "$pool"
+  token: "$token"
+kind: Secret
+metadata:
+  name: "$name"
+  namespace: rook-ceph
+type: Opaque

test/rbd-mirror/rbd-mirror.yaml

@@ -0,0 +1,12 @@
+# SPDX-FileCopyrightText: The RamenDR authors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+---
+apiVersion: ceph.rook.io/v1
+kind: CephRBDMirror
+metadata:
+  name: my-rbd-mirror
+  namespace: rook-ceph
+spec:
+  count: 1

test/rbd-mirror/rbd-pvc.yaml

@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: The RamenDR authors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: rbd-pvc
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+  storageClassName: rook-ceph-block

test/rbd-mirror/start

@@ -0,0 +1,147 @@
+#!/usr/bin/env -S python3 -u
+
+# SPDX-FileCopyrightText: The RamenDR authors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import base64
+import json
+import sys
+
+import drenv
+
+POOL_NAME = "replicapool"
+
+
+def fetch_secret_info(cluster):
+    info = {}
+
+    drenv.log_progress(f"Getting mirroring info site name for cluster {cluster}")
+    info["name"] = drenv.kubectl(
+        "get", "cephblockpools.ceph.rook.io", POOL_NAME,
+        "--output", "jsonpath={.status.mirroringInfo.site_name}",
+        "--namespace", "rook-ceph",
+        profile=cluster,
+    )
+
+    drenv.log_progress(f"Getting rbd mirror boostrap peer secret name for cluster {cluster}")
+    secret_name = drenv.kubectl(
+        "get", "cephblockpools.ceph.rook.io", POOL_NAME,
+        "--output", "jsonpath={.status.info.rbdMirrorBootstrapPeerSecretName}",
+        "--namespace", "rook-ceph",
+        profile=cluster,
+    )
+
+    drenv.log_progress(f"Getting secret {secret_name} token for cluster {cluster}")
+    info["token"] = drenv.kubectl(
+        "get", "secret", secret_name,
+        "--output", "jsonpath={.data.token}",
+        "--namespace", "rook-ceph",
+        profile=cluster,
+    )
+
+    # Must be encoded as base64 in secret .data section.
+    info["pool"] = base64.b64encode(POOL_NAME.encode()).decode()
+
+    return info
+
+
+def configure_rbd_mirroring(cluster, peer_info):
+    drenv.log_progress(f"Applying rbd mirror secret in cluster {cluster}")
+
+    template = drenv.template("rbd-mirror/rbd-mirror-secret.yaml")
+    yaml = template.substitute(peer_info)
+    drenv.kubectl(
+        "apply",
+        "--filename", "-",
+        "--namespace", "rook-ceph",
+        input=yaml,
+        profile=cluster,
+    )
+
+    drenv.log_progress(f"Configure peers for cluster {cluster}")
+    patch = {
+        "spec": {
+            "mirroring": {
+                "peers": {
+                    "secretNames": [
+                        peer_info["name"]]}}}}
+    drenv.kubectl(
+        "patch", "cephblockpool",  POOL_NAME,
+        "--type", "merge",
+        "--patch", json.dumps(patch),
+        "--namespace", "rook-ceph",
+        profile=cluster,
+    )
+
+    drenv.log_progress(f"Apply rbd mirror to cluster {cluster}")
+    drenv.kubectl(
+        "apply",
+        "--filename", "rbd-mirror/rbd-mirror.yaml",
+        "--namespace", "rook-ceph",
+        profile=cluster,
+    )
+
+
+def wait_until_pool_mirroring_is_healthy(cluster):
+    drenv.log_progress(f"Waiting until ceph mirror daemon is healthy in cluster {cluster}")
+    drenv.kubectl(
+        "wait", "cephblockpools.ceph.rook.io", POOL_NAME,
+        "--for", "jsonpath={.status.mirroringStatus.summary.daemon_health}=OK",
+        "--namespace", "rook-ceph",
+        "--timeout", "300s",
+        profile=cluster,
+    )
+
+    drenv.log_progress(f"Waiting until ceph mirror is healthy in cluster {cluster}")
+    drenv.kubectl(
+        "wait", "cephblockpools.ceph.rook.io", POOL_NAME,
+        "--for", "jsonpath={.status.mirroringStatus.summary.health}=OK",
+        "--namespace", "rook-ceph",
+        "--timeout", "300s",
+        profile=cluster,
+    )
+
+    drenv.log_progress(f"Waiting until ceph mirror image is healthy in cluster {cluster}")
+    drenv.kubectl(
+        "wait", "cephblockpools.ceph.rook.io", POOL_NAME,
+        "--for", "jsonpath={.status.mirroringStatus.summary.image_health}=OK",
+        "--namespace", "rook-ceph",
+        "--timeout", "300s",
+        profile=cluster,
+    )
+
+
+def deploy_vrc_sample(cluster):
+    drenv.log_progress(f"Applying vrc sample in cluster {cluster}")
+    drenv.kubectl(
+        "apply",
+        "--filename", "rbd-mirror/vrc-sample.yaml",
+        "--namespace", "rook-ceph",
+        profile=cluster,
+    )
+
+
+if len(sys.argv) != 3:
+    print(f"Usage: {sys.argv[0]} cluster1 cluster2")
+    sys.exit(1)
+
+cluster1 = sys.argv[1]
+cluster2 = sys.argv[2]
+
+cluster1_info = fetch_secret_info(cluster1)
+cluster2_info = fetch_secret_info(cluster2)
+
+drenv.log_progress(f"Setting up mirroring from '{cluster2}' to '{cluster1}'")
+configure_rbd_mirroring(cluster1, cluster2_info)
+
+drenv.log_progress(f"Setting up mirroring from '{cluster1}' to '{cluster2}'")
+configure_rbd_mirroring(cluster2, cluster1_info)
+
+wait_until_pool_mirroring_is_healthy(cluster1)
+wait_until_pool_mirroring_is_healthy(cluster2)
+
+deploy_vrc_sample(cluster1)
+deploy_vrc_sample(cluster2)
+
+drenv.log_progress("Mirroring was setup successfully")

test/rbd-mirror/test

@@ -0,0 +1,134 @@
+#!/usr/bin/env -S python3 -u
+
+# SPDX-FileCopyrightText: The RamenDR authors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+import time
+
+import drenv
+
+POOL_NAME = "replicapool"
+PVC_NAME = "rbd-pvc"
+
+
+def test_volume_replication(primary, secondary):
+    drenv.log_progress(f"Deploying pvc {PVC_NAME} in cluster {primary}")
+    drenv.kubectl(
+        "apply",
+        "--filename", f"rbd-mirror/{PVC_NAME}.yaml",
+        "--namespace", "rook-ceph",
+        profile=primary,
+    )
+
+    drenv.log_progress(f"Waiting until pvc {PVC_NAME} is bound in cluster {primary}")
+    drenv.kubectl(
+        "wait", "pvc", PVC_NAME,
+        "--for", "jsonpath={.status.phase}=Bound",
+        "--namespace", "rook-ceph",
+        "--timeout", "300s",
+        profile=primary,
+    )
+
+    drenv.log_progress(f"Deploying vr vr-sample in cluster {primary}")
+    drenv.kubectl(
+        "apply",
+        "--filename", "rbd-mirror/vr-sample.yaml",
+        "--namespace", "rook-ceph",
+        profile=primary,
+    )
+
+    drenv.log_progress(f"Waiting until vr vr-sample is completed in cluster {primary}")
+    drenv.kubectl(
+        "wait", "volumereplication", "vr-sample",
+        "--for", "condition=Completed",
+        "--namespace", "rook-ceph",
+        "--timeout", "60s",
+        profile=primary,
+    )
+
+    drenv.log_progress(f"Waiting until vr vr-sample state is primary in cluster {primary}")
+    drenv.kubectl(
+        "wait", "volumereplication", "vr-sample",
+        "--for", "jsonpath={.status.state}=Primary",
+        "--namespace", "rook-ceph",
+        "--timeout", "60s",
+        profile=primary,
+    )
+
+    drenv.log_progress(f"Looking up pvc {PVC_NAME} pv name in cluster {primary}")
+    pv_name = drenv.kubectl(
+        "get", f"pvc/{PVC_NAME}",
+        "--output", "jsonpath={.spec.volumeName}",
+        "--namespace", "rook-ceph",
+        profile=primary,
+    )
+
+    drenv.log_progress(f"Looking up rbd image for pv {pv_name} in cluster {primary}")
+    rbd_image = drenv.kubectl(
+        "get", f"pv/{pv_name}",
+        "--output", "jsonpath={.spec.csi.volumeAttributes.imageName}",
+        "--namespace", "rook-ceph",
+        profile=primary,
+    )
+
+    drenv.log_progress(f"rbd image {rbd_image} info in cluster {primary}")
+    drenv.kubectl(
+        "exec", "deploy/rook-ceph-tools", "--namespace", "rook-ceph", "--",
+        "rbd", "info", rbd_image, "--pool", POOL_NAME,
+        profile=primary,
+    )
+
+    drenv.log_progress(f"Waiting until rbd image {rbd_image} is created in cluster {secondary}")
+    for i in range(60):
+        time.sleep(1)
+        out = drenv.kubectl(
+            "exec", "deploy/rook-ceph-tools", "--namespace", "rook-ceph", "--",
+            "rbd", "list", "--pool", POOL_NAME,
+            profile=secondary,
+        )
+        if rbd_image in out:
+            drenv.kubectl(
+                "exec", "deploy/rook-ceph-tools", "--namespace", "rook-ceph", "--",
+                "rbd", "info", rbd_image, "--pool", POOL_NAME,
+                profile=secondary,
+            )
+            break
+    else:
+        raise RuntimeError(f"Timeout waiting for image {rbd_image}")
+
+    drenv.log_progress(f"vr vr-sample info on primary cluster {primary}")
+    drenv.kubectl(
+        "get", "volumereplication", "vr-sample",
+        "--output", "yaml",
+        "--namespace", "rook-ceph",
+        profile=primary,
+    )
+
+    drenv.log_progress(f"Deleting vr vr-sample in primary cluster {primary}")
+    drenv.kubectl(
+        "delete", "volumereplication", "vr-sample",
+        "--namespace", "rook-ceph",
+        profile=primary,
+    )
+
+    drenv.log_progress(f"Deleting pvc {PVC_NAME} in primary cluster {primary}")
+    drenv.kubectl(
+        "delete", "pvc", PVC_NAME,
+        "--namespace", "rook-ceph",
+        profile=primary,
+    )
+
+    drenv.log_progress(f"Replication from cluster {primary} to cluster {secondary} successeded")
+
+
+if len(sys.argv) != 3:
+    print(f"Usage: {sys.argv[0]} cluster1 cluster2")
+    sys.exit(1)
+
+cluster1 = sys.argv[1]
+cluster2 = sys.argv[2]
+
+test_volume_replication(cluster1, cluster2)
+test_volume_replication(cluster2, cluster1)

test/rbd-mirror/vr-sample.yaml

@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: The RamenDR authors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+---
+apiVersion: replication.storage.openshift.io/v1alpha1
+kind: VolumeReplication
+metadata:
+  name: vr-sample
+spec:
+  volumeReplicationClass: vrc-sample
+  replicationState: primary
+  dataSource:
+    kind: PersistentVolumeClaim
+    name: rbd-pvc
+  autoResync: true

test/rbd-mirror/vrc-sample.yaml

@@ -0,0 +1,15 @@
+# SPDX-FileCopyrightText: The RamenDR authors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+---
+apiVersion: replication.storage.openshift.io/v1alpha1
+kind: VolumeReplicationClass
+metadata:
+  name: vrc-sample
+spec:
+  provisioner: rook-ceph.rbd.csi.ceph.com
+  parameters:
+    replication.storage.openshift.io/replication-secret-name: rook-csi-rbd-provisioner
+    replication.storage.openshift.io/replication-secret-namespace: rook-ceph
+    schedulingInterval: 1m

test/regional-dr.yaml

@@ -31,3 +31,7 @@ scripts:
     args: ["dr1", "dr2", "hub"]
   - file: klusterlet/test
     args: ["dr1", "dr2", "hub"]
+  - file: rbd-mirror/start
+    args: ["dr1", "dr2"]
+  - file: rbd-mirror/test
+    args: ["dr1", "dr2"]