[kubevirtci]: refactor cluster-sync target to work with kubevirtci

.gitleaks.toml

@@ -0,0 +1,5 @@
+[allowlist]
+  description = "kubevirtci allowlist"
+  paths = [
+    '''kubevirtci\/cluster-up\/hack\/common.key$''',
+  ]

Makefile

@@ -21,15 +21,12 @@
 		goveralls \
 		release-description \
 		bazel-build-images push-images \
-		fossa
+		fossa \
+		bump-kubevirtci
 all: build
 
 build:  wasp manifest-generator
 
-ifeq ($(origin KUBEVIRT_RELEASE), undefined)
-	KUBEVIRT_RELEASE="latest_nightly"
-endif
-
 all: manifests build-images
 
 manifests:
@@ -39,10 +36,10 @@ builder-push:
 	./hack/build/build-builder.sh
 
 cluster-up:
-	eval "KUBEVIRT_RELEASE=${KUBEVIRT_RELEASE} KUBEVIRT_SWAP_ON=true ./cluster-up/up.sh"
+	./hack/cluster-up.sh
 
 cluster-down:
-	./cluster-up/down.sh
+	./kubevirtci/cluster-up/down.sh
 
 push-images:
 	eval "DOCKER_PREFIX=${DOCKER_PREFIX} DOCKER_TAG=${DOCKER_TAG}  ./hack/build/build-docker.sh push"
@@ -89,3 +86,6 @@ fmt:
 
 run: build
 	sudo ./wasp
+
+bump-kubevirtci:
+	./hack/bump-kubevirtci.sh

cluster-sync/clean.sh

@@ -1,8 +1,9 @@
 #!/bin/bash -e
 
 source ./hack/build/config.sh
-source ./cluster-up/hack/common.sh
-source ./cluster-up/cluster/${KUBEVIRT_PROVIDER}/provider.sh
+source ./hack/config-kubevirtci.sh
+source ./kubevirtci/cluster-up/hack/common.sh
+source ./kubevirtci/cluster-up/cluster/${KUBEVIRT_PROVIDER}/provider.sh
 source cluster-sync/install.sh
 
 echo "Cleaning up ..."

cluster-sync/install.sh

@@ -22,4 +22,58 @@ function delete_wasp {
   else
     echo "File ./_out/manifests/release/wasp.yaml does not exist."
   fi
+}
+
+function install_kubevirt() {
+
+  #Check if kubevrit is already installed
+  if kubectl get crd kubevirts.kubevirt.io >> /dev/null 2>&1 ; then
+    echo "Bypassing Kubevirt deployment since it's already installed"
+    return
+  fi
+
+  if [ "$KUBEVIRT_RELEASE" = "latest_nightly" ]; then
+    LATEST=$(curl -L https://storage.googleapis.com/kubevirt-prow/devel/nightly/release/kubevirt/kubevirt/latest)
+    kubectl apply -f https://storage.googleapis.com/kubevirt-prow/devel/nightly/release/kubevirt/kubevirt/${LATEST}/kubevirt-operator.yaml
+    kubectl apply -f https://storage.googleapis.com/kubevirt-prow/devel/nightly/release/kubevirt/kubevirt/${LATEST}/kubevirt-cr.yaml
+  elif [ "$KUBEVIRT_RELEASE" = "latest_stable" ]; then
+    RELEASE=$(curl https://storage.googleapis.com/kubevirt-prow/release/kubevirt/kubevirt/stable.txt)
+    kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${RELEASE}/kubevirt-operator.yaml
+    kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${RELEASE}/kubevirt-cr.yaml
+  else
+    kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_RELEASE}/kubevirt-operator.yaml
+    kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_RELEASE}/kubevirt-cr.yaml
+  fi
+  # Ensure the KubeVirt CRD is created
+  count=0
+  until kubectl get crd kubevirts.kubevirt.io; do
+      ((count++)) && ((count == 30)) && echo "KubeVirt CRD not found" && exit 1
+      echo "waiting for KubeVirt CRD"
+      sleep 1
+  done
+
+  # Ensure the KubeVirt API is available
+  count=0
+  until kubectl api-resources --api-group=kubevirt.io | grep kubevirts; do
+      ((count++)) && ((count == 30)) && echo "KubeVirt API not found" && exit 1
+      echo "waiting for KubeVirt API"
+      sleep 1
+  done
+
+
+  # Ensure the KubeVirt CR is created
+  count=0
+  until kubectl -n kubevirt get kv kubevirt; do
+      ((count++)) && ((count == 30)) && echo "KubeVirt CR not found" && exit 1
+      echo "waiting for KubeVirt CR"
+      sleep 1
+  done
+
+  # Wait until KubeVirt is ready
+  count=0
+  until kubectl wait -n kubevirt kv kubevirt --for condition=Available --timeout 5m; do
+      ((count++)) && ((count == 5)) && echo "KubeVirt not ready in time" && exit 1
+      echo "Error waiting for KubeVirt to be Available, sleeping 1m and retrying"
+      sleep 1m
+  done
 }

cluster-sync/sync.sh

@@ -7,8 +7,8 @@ echo wasp
 
 source ./hack/build/config.sh
 source ./hack/build/common.sh
-source ./cluster-up/hack/common.sh
-source ./cluster-up/cluster/${KUBEVIRT_PROVIDER}/provider.sh
+source ./kubevirtci/cluster-up/hack/common.sh
+source ./kubevirtci/cluster-up/cluster/${KUBEVIRT_PROVIDER}/provider.sh
 
 if [ "${KUBEVIRT_PROVIDER}" = "external" ]; then
    WASP_SYNC_PROVIDER="external"
@@ -91,6 +91,8 @@ function wait_wasp_available {
 
 mkdir -p ./_out/tests
 
+install_kubevirt
+
 # Install WASP
 install_wasp
 

hack/build/common.sh

@@ -44,5 +44,6 @@ VENDOR_DIR=${WASP_DIR}/vendor
 ARCHITECTURE="${BUILD_ARCH:-$(uname -m)}"
 HOST_ARCHITECTURE="$(uname -m)"
 WASP_CRI="$(determine_wasp_bin)"
+KUBEVIRT_RELEASE=${KUBEVIRT_RELEASE:-"latest_nightly"}
 
 

hack/build/config.sh

@@ -13,7 +13,7 @@
 #limitations under the License.
 WASP_IMAGE_NAME=${WASP_IMAGE_NAME:-wasp}
 
-DOCKER_PREFIX=${DOCKER_PREFIX:-"quay.io/bmordeha"}
+DOCKER_PREFIX=${DOCKER_PREFIX:-"registry:5000/openshift-virtualization"}
 DOCKER_TAG=${DOCKER_TAG:-latest}
 VERBOSITY=${VERBOSITY:-1}
 PULL_POLICY=${PULL_POLICY:-Always}

hack/bump-kubevirtci.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -ex
+
+source $(dirname "$0")/config.sh
+
+val=$(curl -L https://storage.googleapis.com/kubevirt-prow/release/kubevirt/kubevirtci/latest)
+sed -i "/^[[:blank:]]*kubevirtci_git_hash[[:blank:]]*=/s/=.*/=\"${val}\"/" hack/config.sh
+
+hack/sync-kubevirtci.sh

hack/cluster-up.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+#
+# This file is part of the KubeVirt project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Copyright 2021 Red Hat, Inc.
+#
+
+set -e
+set -x
+
+source hack/config-kubevirtci.sh
+KUBEVIRT_DEPLOY_CDI=${KUBEVIRT_DEPLOY_CDI:-true}
+KUBEVIRT_SWAP_ON=${KUBEVIRT_SWAP_ON:-true}
+source "${KUBEVIRTCI_PATH}up.sh"

hack/config-kubevirtci.sh

@@ -0,0 +1,22 @@
+#
+# This file is part of the KubeVirt project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Copyright the KubeVirt Authors.
+#
+#
+
+BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+KUBEVIRTCI_PATH="${BASE_DIR}/kubevirtci/cluster-up/"
+KUBEVIRTCI_CONFIG_PATH="${BASE_DIR}/kubevirtci/_ci-configs"

hack/config.sh

@@ -0,0 +1 @@
+kubevirtci_git_hash="2412171619-fbd31717"

hack/sync-kubevirtci.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+set -ex
+
+# Required for kubevirtci_git_hash
+source $(dirname "$0")/config.sh
+
+WASP_DIR="$(
+    cd "$(dirname "$BASH_SOURCE[0]")/../"
+    pwd
+)"
+
+# update cluster-up if needed
+version_file="kubevirtci/cluster-up/version.txt"
+sha_file="kubevirtci/cluster-up-sha.txt"
+download_cluster_up=true
+function getClusterUpShasum() {
+    (
+        cd ${WASP_DIR}
+        # We use LC_ALL=C to make sort canonical between machines, this is
+        # from sort man page [1]:
+        # ```
+        # *** WARNING *** The locale specified by the environment affects sort
+        # order.  Set LC_ALL=C to get the traditional sort order that uses
+        # native byte values.
+        # ```
+        # [1] https://man7.org/linux/man-pages/man1/sort.1.html
+        find kubevirtci/cluster-up -type f | LC_ALL=C sort | xargs sha1sum | sha1sum | awk '{print $1}'
+    )
+}
+
+# check if we got a new cluster-up git commit hash
+if [[ -f "${version_file}" ]] && [[ $(cat ${version_file}) == ${kubevirtci_git_hash} ]]; then
+    # check if files are modified
+    current_sha=$(getClusterUpShasum)
+    if [[ -f "${sha_file}" ]] && [[ $(cat ${sha_file}) == ${current_sha} ]]; then
+        echo "cluster-up is up to date and not modified"
+        download_cluster_up=false
+    else
+        echo "cluster-up was modified"
+    fi
+else
+    echo "cluster-up git commit hash was updated"
+fi
+if [[ "$download_cluster_up" == true ]]; then
+    echo "downloading cluster-up"
+    rm -rf kubevirtci/cluster-up
+    (
+        cd kubevirtci
+        curl --fail -L https://github.com/kubevirt/kubevirtci/archive/refs/tags/${kubevirtci_git_hash}.tar.gz | tar xz kubevirtci-${kubevirtci_git_hash}/cluster-up --strip-component 1
+    )
+
+    echo ${kubevirtci_git_hash} >${version_file}
+    new_sha=$(getClusterUpShasum)
+    echo ${new_sha} >${sha_file}
+    echo "KUBEVIRTCI_TAG=${kubevirtci_git_hash}" >>kubevirtci/cluster-up/hack/common.sh
+fi

kubevirtci/cluster-up-sha.txt

@@ -0,0 +1 @@
+c8e41b0449bf6b53024d0f1b6eb4f07a7795e9ee

kubevirtci/cluster-up/.gitignore

@@ -0,0 +1 @@
+cluster/kind-k8s-sriov*/certcreator/*.cert

kubevirtci/cluster-up/README.md

@@ -0,0 +1,36 @@
+# cluster-up
+
+## Prerequisites: podman or docker
+
+cluster-up requires that either podman or docker be installed on the host.
+
+If podman is being used, it is also necessary to enable podman socket with:
+
+```
+sudo systemctl enable podman.socket
+sudo systemctl start podman.socket
+```
+
+for more information see:
+
+https://github.com/kubevirt/kubevirtci/blob/main/PODMAN.md
+
+
+## How to use cluster-up
+
+This directory provides a wrapper around gocli. It can be vendored into other
+git repos and integrated to provide in the kubevirt well-known cluster commands
+like `make cluster-up` and `make cluster-down`.
+
+In order to properly use it, one has to vendor this folder from a git tag,
+which can be found on the github release page.
+
+Then, before calling one of the make targets, the environment variable
+`KUBEVIRTCI_TAG` must be exported and set to the tag which was used to vendor
+kubevirtci. It allow the content to find the right `gocli` version.
+
+```
+export KUBEVIRTCI_TAG=`curl -L -Ss https://storage.googleapis.com/kubevirt-prow/release/kubevirt/kubevirtci/latest`
+```
+
+Find more kubevirtci tags at https://quay.io/repository/kubevirtci/gocli?tab=tags.

kubevirtci/cluster-up/check.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+#
+# This file is part of the KubeVirt project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Copyright 2019 Red Hat, Inc.
+#
+
+set -e
+if [ ! -c /dev/kvm ]; then
+	echo "[ERR ] missing /dev/kvm"
+else
+	echo "[ OK ] found /dev/kvm"
+fi
+
+KVM_ARCH=""
+KVM_NESTED="unknown"
+KVM_HPAGE="unknown"
+if [ -f "/sys/module/kvm_intel/parameters/nested" ]; then
+	KVM_NESTED=$( cat /sys/module/kvm_intel/parameters/nested )
+	KVM_ARCH="intel"
+elif [ -f "/sys/module/kvm_amd/parameters/nested" ]; then
+	KVM_NESTED=$( cat /sys/module/kvm_amd/parameters/nested )
+	KVM_ARCH="amd"
+elif [ -f "/sys/module/kvm/parameters/nested" ]; then
+	KVM_NESTED=$( cat /sys/module/kvm/parameters/nested )
+	KVM_ARCH="s390x"
+	KVM_HPAGE=$( cat /sys/module/kvm/parameters/hpage )
+fi
+
+function is_enabled() {
+	if [ "$1" == "1" ]; then
+		return 0
+	fi
+	if [ "$1" == "Y" ] || [ "$1" == "y" ]; then
+		return 0
+	fi
+	return 1
+}
+
+if is_enabled "$KVM_NESTED"; then
+	echo "[ OK ] $KVM_ARCH nested virtualization enabled"
+else
+	echo "[ERR ] $KVM_ARCH nested virtualization not enabled"
+fi
+
+if is_enabled "$KVM_HPAGE" && [ "$(uname -m)" = "s390x" ]; then
+	echo "[ERR ] $KVM_HPAGE KVM hugepage enabled. It needs to be disabled while nested virtualization is enabled for s390x"
+fi