From 8bd16be656804a71a6a3721d7c3952243d8fa81a Mon Sep 17 00:00:00 2001 From: Ben Oukhanov Date: Thu, 26 Sep 2024 10:34:07 +0300 Subject: [PATCH] fix(CNV-48974): download disk img from vmexport-api - Install missing nbdkit-curl-plugin pkg - Fix wrong volume names - Fix wrong logs - Create secret (and get export token) - Get and create certificate file - Cleanup deployed resources Secret is used by VMExport and is needed to set a new generated token. This token is used by the VMExport API to accept calls to the endpoints. Certificate is provided by VMExport and is needed to be used by the client in order to make HTTPs client requests. Signed-off-by: Ben Oukhanov --- Dockerfile | 2 +- examples/kubevirt-disk-uploader-tekton.yaml | 2 +- kubevirt-disk-uploader.yaml | 9 +- main.go | 269 ++++++++++++++---- .../kubevirt-disk-uploader-task-run.yaml | 2 +- 5 files changed, 228 insertions(+), 56 deletions(-) diff --git a/Dockerfile b/Dockerfile index d5fc22d..77777b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o kubevirt-disk-uploader . FROM quay.io/fedora/fedora-minimal:39 -RUN microdnf install -y nbdkit qemu-img && microdnf clean all -y +RUN microdnf install -y nbdkit nbdkit-curl-plugin qemu-img && microdnf clean all -y COPY --from=builder /app/kubevirt-disk-uploader /usr/local/bin/kubevirt-disk-uploader ENTRYPOINT ["/usr/local/bin/kubevirt-disk-uploader"] diff --git a/examples/kubevirt-disk-uploader-tekton.yaml b/examples/kubevirt-disk-uploader-tekton.yaml index 3483fe1..72d65f7 100644 --- a/examples/kubevirt-disk-uploader-tekton.yaml +++ b/examples/kubevirt-disk-uploader-tekton.yaml @@ -252,7 +252,7 @@ spec: - name: VM_NAME value: example-vm-tekton - name: VOLUME_NAME - value: datavolumedisk + value: example-dv-tekton - name: IMAGE_DESTINATION value: quay.io/boukhano/example-vm-tekton-exported:latest - name: PUSH_TIMEOUT diff --git a/kubevirt-disk-uploader.yaml b/kubevirt-disk-uploader.yaml index 45e3370..f9d9327 100644 --- a/kubevirt-disk-uploader.yaml +++ b/kubevirt-disk-uploader.yaml @@ -15,6 +15,9 @@ rules: - apiGroups: [""] resources: ["secrets"] verbs: ["get", "create"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["get"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -53,8 +56,12 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: VM_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name command: ["/usr/local/bin/kubevirt-disk-uploader"] - # args: ["--vmname", "example-vm", "--volumename", "datavolumedisk", "--imagedestination", "quay.io/boukhano/example-vm-exported:latest", "--pushtimeout", "120"] + # args: ["--vmname", "example-vm", "--volumename", "example-dv", "--imagedestination", "quay.io/boukhano/example-vm-exported:latest", "--pushtimeout", "120"] resources: requests: memory: 3Gi diff --git a/main.go b/main.go index 5b99988..80af01d 100644 --- a/main.go +++ b/main.go @@ -2,8 +2,10 @@ package main import ( "context" + "crypto/rand" "fmt" "log" + "math/big" "os" "os/exec" "time" @@ -18,6 +20,7 @@ import ( cobra "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" kvcorev1 "kubevirt.io/api/core/v1" v1beta1 "kubevirt.io/api/export/v1beta1" @@ -26,44 +29,71 @@ import ( ) const ( - pollInterval = 15 * time.Second - pollTimeout = 3600 * time.Second - diskPath string = "./tmp/disk.img.gz" - diskPathDecompressed string = "./tmp/disk.img" - diskPathConverted string = "./tmp/disk.qcow2" + pollInterval = 15 * time.Second + pollTimeout = 3600 * time.Second + diskPath string = "./tmp/disk.qcow2" + certificatePath string = "./tmp/tls.crt" + kvExportTokenKey string = "token" + kvExportTokenHeader string = "x-kubevirt-export-token" + kvExportTokenLength int = 20 ) -func applyVirtualMachineExport(client kubecli.KubevirtClient, vmNamespace, vmName string) error { - log.Println("Applying VirtualMachineExport object...") +func createSecret(client kubecli.KubevirtClient, vmNamespace, vmName string) error { + token, err := generateSecureRandomString(kvExportTokenLength) + if err != nil { + return err + } - vmExport := &v1beta1.VirtualMachineExport{ + v1Secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: vmName, + Namespace: vmNamespace, + }, + StringData: map[string]string{ + kvExportTokenKey: token, + }, + } + + if err := setPodOwnerReference(client, v1Secret); err != nil { + return err + } + + _, err = client.CoreV1().Secrets(vmNamespace).Create(context.Background(), v1Secret, metav1.CreateOptions{}) + return err +} + +func createVirtualMachineExport(client kubecli.KubevirtClient, vmNamespace, vmName string) error { + v1VmExport := &v1beta1.VirtualMachineExport{ ObjectMeta: metav1.ObjectMeta{ Name: vmName, Namespace: vmNamespace, }, Spec: v1beta1.VirtualMachineExportSpec{ + TokenSecretRef: &vmName, Source: corev1.TypedLocalObjectReference{ - APIGroup: &kvcorev1.GroupVersion.Version, + APIGroup: &kvcorev1.SchemeGroupVersion.Group, Kind: kvcorev1.VirtualMachineGroupVersionKind.Kind, Name: vmName, }, }, } - _, err := client.VirtualMachineExport(vmNamespace).Create(context.Background(), vmExport, metav1.CreateOptions{}) + if err := setPodOwnerReference(client, v1VmExport); err != nil { + return err + } + + _, err := client.VirtualMachineExport(vmNamespace).Create(context.Background(), v1VmExport, metav1.CreateOptions{}) return err } -func getRawDiskUrlFromVirtualMachineExport(client kubecli.KubevirtClient, vmNamespace, vmName, volumeName string) (string, error) { - log.Println("Waiting for VirtualMachineExport to be ready...") - - vmExport, err := getVirtualMachineExportOnceReady(client, vmNamespace, vmName) +func getRawDiskUrlFromVolumes(client kubecli.KubevirtClient, vmNamespace, vmName, volumeName string) (string, error) { + vmExport, err := client.VirtualMachineExport(vmNamespace).Get(context.Background(), vmName, metav1.GetOptions{}) if err != nil { return "", err } if vmExport.Status.Links == nil && vmExport.Status.Links.Internal == nil { - return "", fmt.Errorf("No links found in VirtualMachineExport status.") + return "", fmt.Errorf("no links found in VirtualMachineExport status") } for _, volume := range vmExport.Status.Links.Internal.Volumes { @@ -77,41 +107,65 @@ func getRawDiskUrlFromVirtualMachineExport(client kubecli.KubevirtClient, vmName } } } - return "", fmt.Errorf("Could not get raw disk URL from the VirtualMachineExport object.") + return "", fmt.Errorf("volume %s is not found in VirtualMachineExport internal volumes", volumeName) } -func getVirtualMachineExportOnceReady(client kubecli.KubevirtClient, vmNamespace, vmName string) (*v1beta1.VirtualMachineExport, error) { - var vmExport *v1beta1.VirtualMachineExport - +func waitUntilVirtualMachineExportReady(client kubecli.KubevirtClient, vmNamespace, vmName string) error { poller := func(ctx context.Context) (bool, error) { vmExport, err := client.VirtualMachineExport(vmNamespace).Get(ctx, vmName, metav1.GetOptions{}) if err != nil { return false, err } - if vmExport.Status.Phase == v1beta1.Ready { + if vmExport.Status != nil && vmExport.Status.Phase == v1beta1.Ready { return true, nil } return false, nil } - err := wait.PollUntilContextTimeout(context.Background(), pollInterval, pollTimeout, true, poller) + return wait.PollUntilContextTimeout(context.Background(), pollInterval, pollTimeout, true, poller) +} + +func getCertificateFromVirtualMachineExport(client kubecli.KubevirtClient, vmNamespace, vmName string) (string, error) { + vmExport, err := client.VirtualMachineExport(vmNamespace).Get(context.Background(), vmName, metav1.GetOptions{}) if err != nil { - return nil, fmt.Errorf("Failed to wait for VirtualMachineExport to be ready: %v", err) + return "", err + } + + if vmExport.Status.Links == nil && vmExport.Status.Links.Internal == nil { + return "", fmt.Errorf("no links found in VirtualMachineExport status") } - return vmExport, nil + + content := vmExport.Status.Links.Internal.Cert + if content == "" { + return "", fmt.Errorf("no certificate found in VirtualMachineExport status") + } + return content, nil } -func convertRawDiskImageToQcow2(rawDiskUrl string) error { - log.Println("Converting raw disk image to qcow2 format...") +func getExportToken(client kubecli.KubevirtClient, vmNamespace, vmName string) (string, error) { + secret, err := client.CoreV1().Secrets(vmNamespace).Get(context.Background(), vmName, metav1.GetOptions{}) + if err != nil { + return "", err + } + data := secret.Data[kvExportTokenKey] + if len(data) == 0 { + return "", fmt.Errorf("failed to get export token from '%s/%s'", vmNamespace, vmName) + } + return string(data), nil +} + +func downloadDiskImageFromURL(rawDiskUrl, exportToken string) error { cmd := exec.Command( "nbdkit", "-r", "curl", rawDiskUrl, + fmt.Sprintf("header=%s: %s", kvExportTokenHeader, exportToken), + fmt.Sprintf("cainfo=%s", certificatePath), "--run", - fmt.Sprintf("qemu-img convert \"$uri\" -O qcow2 %s", diskPathConverted), + fmt.Sprintf("qemu-img convert \"$uri\" -O qcow2 %s", diskPath), ) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr @@ -120,15 +174,13 @@ func convertRawDiskImageToQcow2(rawDiskUrl string) error { return err } - if fileInfo, err := os.Stat(diskPathConverted); err != nil || fileInfo.Size() == 0 { - return fmt.Errorf("Converted file does not exist or is empty.") + if fileInfo, err := os.Stat(diskPath); err != nil || fileInfo.Size() == 0 { + return fmt.Errorf("disk image file does not exist or is empty") } - - log.Println("Conversion to qcow2 format completed successfully.") return nil } -func buildContainerDisk(diskPath string) (v1.Image, error) { +func buildContainerDisk() (v1.Image, error) { layer, err := tarball.LayerFromOpener(tar.StreamLayerOpener(diskPath)) if err != nil { log.Fatalf("Error creating layer from file: %v", err) @@ -140,8 +192,6 @@ func buildContainerDisk(diskPath string) (v1.Image, error) { log.Fatalf("Error appending layer: %v", err) return nil, err } - - log.Println("Image built successfully", image) return image, nil } @@ -158,45 +208,74 @@ func pushContainerDisk(image v1.Image, imageDestination string, pushTimeout int) log.Fatalf("Error pushing image: %v", err) return err } - - log.Println("Image pushed successfully") return nil } -func run(vmNamespace, vmName, volumeName, imageDestination string, pushTimeout int) error { - client, err := kubecli.GetKubevirtClient() - if err != nil { +func run(client kubecli.KubevirtClient, vmNamespace, vmName, volumeName, imageDestination string, pushTimeout int) error { + log.Printf("Creating a new Secret '%s/%s' object...", vmNamespace, vmName) + + if err := createSecret(client, vmNamespace, vmName); err != nil { return err } - env := os.Getenv("VM_NAMESPACE") - if env != "" { - vmNamespace = env + log.Printf("Creating a new VirtualMachineExport '%s/%s' object...", vmNamespace, vmName) + + if err := createVirtualMachineExport(client, vmNamespace, vmName); err != nil { + return err } - if vmNamespace == "" { - return fmt.Errorf("VM namespace is not defined. Set VM_NAMESPACE or parameter.") + log.Println("Waiting for VirtualMachineExport status to be ready...") + + if err := waitUntilVirtualMachineExportReady(client, vmNamespace, vmName); err != nil { + return err } - if err := applyVirtualMachineExport(client, vmNamespace, vmName); err != nil { + log.Println("Getting raw disk URL from the VirtualMachineExport object status...") + + rawDiskUrl, err := getRawDiskUrlFromVolumes(client, vmNamespace, vmName, volumeName) + if err != nil { return err } - rawDiskUrl, err := getRawDiskUrlFromVirtualMachineExport(client, vmNamespace, vmName, volumeName) + log.Println("Creating TLS certificate file from the VirtualMachineExport object status...") + + certificate, err := getCertificateFromVirtualMachineExport(client, vmNamespace, vmName) if err != nil { return err } - if err := convertRawDiskImageToQcow2(rawDiskUrl); err != nil { + if err := createFile(certificate); err != nil { return err } - image, err := buildContainerDisk(diskPathConverted) + log.Println("Getting export token from the Secret object...") + + exportToken, err := getExportToken(client, vmNamespace, vmName) if err != nil { return err } - return pushContainerDisk(image, imageDestination, pushTimeout) + log.Println("Downloading disk image from the VirtualMachineExport server...") + + if err := downloadDiskImageFromURL(rawDiskUrl, exportToken); err != nil { + return err + } + + log.Println("Building a new container image...") + + image, err := buildContainerDisk() + if err != nil { + return err + } + + log.Println("Pushing new container image to the container registry...") + + if err := pushContainerDisk(image, imageDestination, pushTimeout); err != nil { + return err + } + + log.Println("Successfully uploaded to the container registry.") + return nil } func main() { @@ -210,13 +289,16 @@ func main() { Use: "kubevirt-disk-uploader", Short: "Extracts disk and uploads it to a container registry", Run: func(cmd *cobra.Command, args []string) { - log.Println("Extracts disk and uploads it to a container registry...") - - if err := run(vmNamespace, vmName, volumeName, imageDestination, pushTimeout); err != nil { + client, err := kubecli.GetKubevirtClient() + if err != nil { log.Panicln(err) } - log.Println("Succesfully extracted disk image and uploaded it in a new container image to container registry.") + namespace := getNamespace(vmNamespace) + + if err := run(client, namespace, vmName, volumeName, imageDestination, pushTimeout); err != nil { + log.Panicln(err) + } }, } @@ -234,3 +316,86 @@ func main() { os.Exit(1) } } + +func getNamespace(vmNamespace string) string { + namespace := os.Getenv("VM_NAMESPACE") + if namespace != "" { + return namespace + } + + return vmNamespace +} + +func createFile(data string) error { + file, err := os.OpenFile(certificatePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return fmt.Errorf("failed to create file: %w", err) + } + defer file.Close() + + _, err = file.WriteString(data) + if err != nil { + return fmt.Errorf("failed to write content to file: %w", err) + } + return nil +} + +func generateSecureRandomString(n int) (string, error) { + // Alphanums is the list of alphanumeric characters used to create a securely generated random string + Alphanums := "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + + ret := make([]byte, n) + for i := range ret { + num, err := rand.Int(rand.Reader, big.NewInt(int64(len(Alphanums)))) + if err != nil { + return "", err + } + ret[i] = Alphanums[num.Int64()] + } + + return string(ret), nil +} + +func getTaskRunPod(client kubecli.KubevirtClient) (*corev1.Pod, error) { + podName, isSet := os.LookupEnv("VM_NAME") + if !isSet { + return nil, fmt.Errorf("pod name env variable is not set") + } + + podNamespace, isSet := os.LookupEnv("VM_NAMESPACE") + if !isSet { + return nil, fmt.Errorf("pod namespace env variable is not set") + } + + pod := &corev1.Pod{} + pod, err := client.CoreV1().Pods(podNamespace).Get(context.Background(), podName, metav1.GetOptions{}) + return pod, err +} + +func setPodOwnerReference(client kubecli.KubevirtClient, object metav1.Object) error { + pod, err := getTaskRunPod(client) + if err != nil { + return err + } + + if object.GetNamespace() != pod.GetNamespace() { + return fmt.Errorf("can't create owner reference for objects in different namespaces") + } + + scheme := runtime.NewScheme() + corev1.AddToScheme(scheme) + + gvks, _, err := scheme.ObjectKinds(pod) + if err != nil { + return fmt.Errorf("could not get GroupVersionKind for object: %w", err) + } + ref := metav1.OwnerReference{ + APIVersion: gvks[0].GroupVersion().String(), + Kind: gvks[0].Kind, + UID: pod.GetUID(), + Name: pod.GetName(), + } + + object.SetOwnerReferences([]metav1.OwnerReference{ref}) + return nil +} diff --git a/tasks/kubevirt-disk-uploader/0.5.0/tests/kubevirt-disk-uploader-task-run.yaml b/tasks/kubevirt-disk-uploader/0.5.0/tests/kubevirt-disk-uploader-task-run.yaml index 3c5b057..9028559 100644 --- a/tasks/kubevirt-disk-uploader/0.5.0/tests/kubevirt-disk-uploader-task-run.yaml +++ b/tasks/kubevirt-disk-uploader/0.5.0/tests/kubevirt-disk-uploader-task-run.yaml @@ -9,7 +9,7 @@ spec: - name: VM_NAME value: example-vm-tekton - name: VOLUME_NAME - value: datavolumedisk + value: example-dv-tekton - name: IMAGE_DESTINATION value: quay.io/boukhano/example-vm-tekton-exported:latest - name: PUSH_TIMEOUT