introduce image-converter

Signed-off-by: Benny Zlotnik <[email protected]>

Makefile

@@ -57,6 +57,7 @@ POPULATOR_CONTROLLER_IMAGE ?= $(REGISTRY)/$(REGISTRY_ORG)/populator-controller:$
 OVIRT_POPULATOR_IMAGE ?= $(REGISTRY)/$(REGISTRY_ORG)/ovirt-populator:$(REGISTRY_TAG)
 OPENSTACK_POPULATOR_IMAGE ?= $(REGISTRY)/$(REGISTRY_ORG)/openstack-populator:$(REGISTRY_TAG)
 OVA_PROVIDER_SERVER_IMAGE ?= $(REGISTRY)/$(REGISTRY_ORG)/forklift-ova-provider-server:$(REGISTRY_TAG)
+IMAGE_CONVERTER_IMAGE ?= $(REGISTRY)/$(REGISTRY_ORG)/image-converter:$(REGISTRY_TAG)
 
 ### External images
 MUST_GATHER_IMAGE ?= quay.io/kubev2v/forklift-must-gather:latest
@@ -296,6 +297,17 @@ build-ova-provider-server-image: check_container_runtime
 		$(BAZEL_OPTS) \
 		--action_env CONTAINER_CMD=$(CONTAINER_CMD)
 
+build-image-converter-image: check_container_runtime
+	export CONTAINER_CMD=$(CONTAINER_CMD); \
+	bazel run cmd/image-converter:image-converter-image \
+		$(BAZEL_OPTS) \
+		--action_env CONTAINER_CMD=$(CONTAINER_CMD)
+
+push-image-converter-image: build-image-converter-image
+	$(CONTAINER_CMD) tag bazel/cmd/image-converter:image-converter-image $(IMAGE_CONVERTER_IMAGE)
+	$(CONTAINER_CMD) push $(IMAGE_CONVERTER_IMAGE)
+
+
 push-ova-provider-server-image: build-ova-provider-server-image
 	$(CONTAINER_CMD) tag bazel/cmd/ova-provider-server:ova-provider-server-image $(OVA_PROVIDER_SERVER_IMAGE)
 	$(CONTAINER_CMD) push $(OVA_PROVIDER_SERVER_IMAGE)

cmd/image-converter/BUILD.bazel

@@ -0,0 +1,91 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+load(
+    "@io_bazel_rules_docker//container:container.bzl",
+    "container_image",
+    "container_layer",
+)
+load("@bazeldnf//:deps.bzl", "rpmtree")
+
+go_library(
+    name = "image-converter_lib",
+    srcs = ["image-converter.go"],
+    importpath = "github.com/konveyor/forklift-controller/cmd/image-converter",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//vendor/k8s.io/api/core/v1:core",
+        "//vendor/k8s.io/klog/v2:klog",
+        "//vendor/sigs.k8s.io/controller-runtime/pkg/client",
+        "//vendor/sigs.k8s.io/controller-runtime/pkg/client/config",
+    ],
+)
+
+go_binary(
+    name = "image-converter",
+    embed = [":image-converter_lib"],
+    visibility = ["//visibility:public"],
+)
+
+container_layer(
+    name = "base-layer",
+    tars = [
+        ":deps",
+    ],
+)
+
+container_image(
+    name = "image-converter-image",
+    directory = "/usr/local/bin/",
+    entrypoint = ["/usr/local/bin/image-converter"],
+    files = [":image-converter"],
+    layers = [":base-layer"],
+    visibility = ["//visibility:public"],
+)
+
+rpmtree(
+    name = "deps",
+    rpms = [
+        "@alternatives-0__1.24-1.el9.x86_64//rpm",
+        "@basesystem-0__11-13.el9.x86_64//rpm",
+        "@bash-0__5.1.8-6.el9.x86_64//rpm",
+        "@centos-gpg-keys-0__9.0-23.el9.x86_64//rpm",
+        "@centos-stream-release-0__9.0-23.el9.x86_64//rpm",
+        "@centos-stream-repos-0__9.0-23.el9.x86_64//rpm",
+        "@coreutils-single-0__8.32-34.el9.x86_64//rpm",
+        "@crypto-policies-0__20231113-1.gite9247c2.el9.x86_64//rpm",
+        "@filesystem-0__3.16-2.el9.x86_64//rpm",
+        "@glib2-0__2.68.4-12.el9.x86_64//rpm",
+        "@glibc-0__2.34-88.el9.x86_64//rpm",
+        "@glibc-common-0__2.34-88.el9.x86_64//rpm",
+        "@glibc-langpack-unm-0__2.34-88.el9.x86_64//rpm",
+        "@gnutls-0__3.8.2-1.el9.x86_64//rpm",
+        "@libacl-0__2.3.1-4.el9.x86_64//rpm",
+        "@libaio-0__0.3.111-13.el9.x86_64//rpm",
+        "@libattr-0__2.5.1-3.el9.x86_64//rpm",
+        "@libblkid-0__2.37.4-15.el9.x86_64//rpm",
+        "@libcap-0__2.48-9.el9.x86_64//rpm",
+        "@libffi-0__3.4.2-8.el9.x86_64//rpm",
+        "@libgcc-0__11.4.1-2.3.el9.x86_64//rpm",
+        "@libidn2-0__2.3.0-7.el9.x86_64//rpm",
+        "@libmount-0__2.37.4-15.el9.x86_64//rpm",
+        "@libselinux-0__3.6-0.rc1.1.el9.x86_64//rpm",
+        "@libsepol-0__3.6-0.rc1.1.el9.x86_64//rpm",
+        "@libtasn1-0__4.16.0-8.el9.x86_64//rpm",
+        "@libunistring-0__0.9.10-15.el9.x86_64//rpm",
+        "@liburing-0__2.3-2.el9.x86_64//rpm",
+        "@libuuid-0__2.37.4-15.el9.x86_64//rpm",
+        "@ncurses-base-0__6.2-10.20210508.el9.x86_64//rpm",
+        "@ncurses-libs-0__6.2-10.20210508.el9.x86_64//rpm",
+        "@nettle-0__3.9.1-1.el9.x86_64//rpm",
+        "@numactl-libs-0__2.0.16-3.el9.x86_64//rpm",
+        "@p11-kit-0__0.25.3-2.el9.x86_64//rpm",
+        "@p11-kit-trust-0__0.25.3-2.el9.x86_64//rpm",
+        "@pcre-0__8.44-3.el9.3.x86_64//rpm",
+        "@pcre2-0__10.40-4.el9.x86_64//rpm",
+        "@pcre2-syntax-0__10.40-4.el9.x86_64//rpm",
+        "@qemu-img-17__8.1.0-4.el9.x86_64//rpm",
+        "@setup-0__2.13.7-9.el9.x86_64//rpm",
+        "@tzdata-0__2023c-1.el9.x86_64//rpm",
+        "@zlib-0__1.2.11-41.el9.x86_64//rpm",
+    ],
+    visibility = ["//visibility:public"],
+)

cmd/image-converter/README.md

@@ -0,0 +1,74 @@
+# Image Converter
+
+Convert the format of images. Since KubeVirt requires RAW images, we sometimes have to convert images before attaching them to a VM.
+
+## Measuring
+
+1. Upload cirros PVC
+
+```shell
+# Download cirros image
+$ curl -LO https://download.cirros-cloud.net/0.6.2/cirros-0.6.2-x86_64-disk.img
+
+# Create PVC with above image
+$ virtctl image-upload pvc cirros --size=1Gi --image-path=cirros-0.6.2-x86_64-disk.img -n default --storage-class nfs-csi --insecure
+```
+2. Create service account in target namespace
+
+```yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: forklift-populator-controller
+  namespace: default
+```
+
+TODO: create a separate account?
+
+3. Create converter pod job
+
+```yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: measure-pvc
+  namespace: default
+spec:
+  template:
+    spec:
+      containers:
+      - name: measure-pvc
+        image: quay.io/bzlotnik/image-converter:latest
+        args:
+          - "-command"
+          - "measure"
+          - "-src-path"
+          - "/mnt/disk.img"
+          - "-pvc-name"
+          - "cirros"
+          - "-namespace"
+          - "default"
+         - "-target-format"
+          - "raw"
+          - "-src-format"
+          - "qcow2"
+        volumeMounts:
+        - name: cirros
+          mountPath: /mnt
+      restartPolicy: Never
+      volumes:
+      - name: cirros
+        persistentVolumeClaim:
+          claimName: cirros
+      serviceAccountName: forklift-populator-controller
+```
+
+4. Check the result
+
+```shell
+$ kubectl get pvc mypvc -n default -o jsonpath='"forklift.konveyor.io/required-size: "{.metadata.annotations.forklift\.konveyor\.io/required-size}'
+```
+
+## Converting
+
+TODO

cmd/image-converter/image-converter.go

@@ -0,0 +1,155 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"os/exec"
+	"strconv"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	ctrl "sigs.k8s.io/controller-runtime/pkg/client/config"
+)
+
+const apiGroup = "forklift.konveyor.io"
+
+func main() {
+	var srcVolPath, dstVolPath, sourceFormat, targetFormat, pvcName, namespace, command string
+
+	flag.StringVar(&srcVolPath, "src-path", "", "Source volume path")
+	flag.StringVar(&dstVolPath, "dst-path", "", "Target volume path")
+	flag.StringVar(&sourceFormat, "src-format", "", "Format of the source volume")
+	flag.StringVar(&targetFormat, "target-format", "", "Format of the target volume")
+	flag.StringVar(&pvcName, "pvc-name", "", "Name of PVC to measure")
+	flag.StringVar(&namespace, "namespace", "", "Namespace of PVC to measure")
+	flag.StringVar(&command, "command", "", "Command to run (convert, measure)")
+
+	flag.Parse()
+
+	klog.Info("Running command: ", command)
+
+	switch command {
+	case "convert":
+		err := convert(srcVolPath, dstVolPath, sourceFormat, targetFormat)
+		if err != nil {
+			klog.Fatal(err)
+		}
+	case "measure":
+		requiredSize, err := measure(srcVolPath, sourceFormat, targetFormat)
+		if err != nil {
+			klog.Fatal(err)
+		}
+
+		err = annotatePvc(pvcName, namespace, requiredSize)
+		if err != nil {
+			klog.Fatal(err)
+		}
+	}
+}
+
+func convert(srcVolPath, dstVolPath, sourceFormat, targetFormat string) error {
+	cmd := exec.Command(
+		"qemu-img",
+		"convert",
+		"-p",
+		srcVolPath,
+		dstVolPath,
+		"-f", sourceFormat,
+		"-O", targetFormat,
+	)
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+
+	scanner := bufio.NewScanner(stdout)
+	for scanner.Scan() {
+		line := scanner.Text()
+		klog.Info(line)
+	}
+
+	err = cmd.Wait()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func measure(srcVolPath, targetFormat, sourceFormat string) (int64, error) {
+	cmd := exec.Command(
+		"qemu-img",
+		"measure",
+		"-O", targetFormat,
+		"-f", sourceFormat,
+		srcVolPath,
+		"--output", "json",
+	)
+
+	klog.Info("Executing command: ", cmd.String())
+
+	stdout, err := cmd.StdoutPipe()
+	if err := cmd.Start(); err != nil {
+		return -1, err
+	}
+
+	outputBytes, err := io.ReadAll(stdout)
+	if err != nil {
+		return -1, err
+	}
+
+	type MeasureOutput struct {
+		Required       int64 `json:"required"`
+		FullyAllocated int64 `json:"fully-allocated"`
+	}
+
+	var measure MeasureOutput
+	if err := json.Unmarshal(outputBytes, &measure); err != nil {
+		return -1, err
+	}
+
+	klog.Info("output: ", measure)
+
+	return measure.Required, nil
+}
+
+func annotatePvc(pvcName, namespace string, required int64) error {
+	cfg := ctrl.GetConfigOrDie()
+	c, err := client.New(cfg, client.Options{})
+	if err != nil {
+		return err
+	}
+
+	pvc := &corev1.PersistentVolumeClaim{}
+	err = c.Get(context.TODO(), client.ObjectKey{
+		Namespace: namespace,
+		Name:      pvcName,
+	}, pvc)
+	if err != nil {
+		return err
+	}
+
+	patch := client.MergeFrom(pvc.DeepCopy())
+
+	if pvc.Annotations == nil {
+		pvc.Annotations = map[string]string{}
+	}
+	pvc.Annotations[fmt.Sprintf("%s/%s", apiGroup, "required-size")] = strconv.FormatInt(required, 10)
+
+	err = c.Patch(context.TODO(), pvc, patch)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

hack/image-converter-rpm-deps.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -e
+
+bazeldnf_repos="--repofile rpm/stream9-repo.yaml"
+if [ "${CUSTOM_REPO}" ]; then
+    bazeldnf_repos="--repofile ${CUSTOM_REPO} ${bazeldnf_repos}"
+fi
+
+bazel run \
+    //:bazeldnf -- fetch \
+    ${bazeldnf_repos}
+
+converter_deps="
+qemu-img
+"
+
+bazel run \
+        //:bazeldnf -- rpmtree \
+        --public \
+        --nobest \
+        --buildfile cmd/image-converter/BUILD.bazel \
+        --name deps \
+        --basesystem centos-stream-release \
+        ${bazeldnf_repos} \
+        $converter_deps
+