Add a testiso scenario installing coreOS to an iscsi target then boot it

I reused the same approach as other liveISO tests, wrapping
all the setup in a butane config.
Systemd units do all the steps, taking care of the ordering.
It is more verbose compared to imperatively executing
commands to the VM through SSH but the avaiable tooling to do SSH
commands in testiso is not as complete as in regular kola tests,
so doing SSH would end up in a lot of error handling code.
A future work is to merge the testiso scenarios with regular kola tests.

Extra storage mounted into /var to avoir running out of space in the
liveiso.

When installing to the iscsi target with coreos-install
we pass an ignition config to write to a serial device
when the boot is complete.
This serial device is passed through to the kola
completion serial device, so it completes the test.

mantle/cmd/kola/resources/iscsi_butane_setup.yaml

@@ -0,0 +1,142 @@
+variant: fcos
+version: 1.5.0
+storage:
+  filesystems:
+    - path: /var
+      device: /dev/disk/by-id/virtio-var
+      format: ext4
+      wipe_filesystem: true
+      label: var
+      with_mount_unit: true
+  files:
+    - path: /etc/containers/systemd/target.container
+      contents:
+        inline: |
+            [Unit]
+            Description=Targetd container
+            Documentation=https://github.com/jbtrystram/targetcli-containers
+            After=local-fs.target network-online.target After=nss-lookup.target dev-disk-by\x2did-virtio\x2dtarget.device
+            Wants=network-online.target
+            OnFailure=emergency.target
+            [Container]
+            Image=quay.io/jbtrystram/targetcli:latest
+            ContainerName=target
+            Network=host
+            Volume=/dev/disk/by-id/virtio-target:/dev/disk/by-id/virtio-target
+            Volume=/lib/modules:/lib/modules
+            Volume=/sys/kernel/config:/sys/kernel/config
+            PodmanArgs=--privileged
+            [Install]
+            # Start by default on boot
+            WantedBy=multi-user.target
+    - path: /usr/local/bin/targetcli_script
+      mode: 0755
+      contents:
+          inline: |
+            #!/bin/bash
+            set -xeuo pipefail
+            podman exec target bash -exc "
+            targetcli /backstores/block create name=coreos dev=/dev/disk/by-id/virtio-target
+            targetcli iscsi/ create iqn.2023-10.coreos.target.vm:coreos
+            targetcli iscsi/iqn.2023-10.coreos.target.vm:coreos/tpg1/luns create /backstores/block/coreos
+            targetcli iscsi/iqn.2023-10.coreos.target.vm:coreos/tpg1/ set attribute authentication=0 demo_mode_write_protect=0 generate_node_acls=1 cache_dynamic_acls=1
+            "
+            # Will return 0 if the discovery yield a valid portal
+            iscsiadm -m discovery -p 127.0.0.1 -t st | grep iqn.2023-10.coreos.target.vm:coreos
+    - path: /mnt/workdir-tmp/boot.ipxe
+      mode: 0644
+      contents:
+        inline: |
+            #!ipxe
+            set initiator-iqn iqn.2023-11.coreos.diskless:testsetup
+            sanboot iscsi:10.0.2.15::::iqn.2023-10.coreos.target.vm:coreos
+    - path: /usr/local/bin/install-coreos-iscsi
+      mode: 0755
+      contents:
+        inline: |
+          #!/bin/bash
+          set -euxo
+          # Mount the iscsi target
+          iscsiadm -m discovery -t st -p 127.0.0.1
+          iscsiadm -m node -T iqn.2023-10.coreos.target.vm:coreos -l
+          # Give a bit of time to udev to create the persistent names paths
+          sleep 2 
+          # Install coreos
+          coreos-installer install \
+            /dev/disk/by-path/ip-127.0.0.1\:3260-iscsi-iqn.2023-10.coreos.target.vm\:coreos-lun-0 \
+            --append-karg rd.iscsi.firmware=1 --append-karg ip=ibft \
+            --console ttyS0 \
+            -i /mnt/workdir-tmp/nested-ign.json
+          # Unmount the disk
+          iscsiadm --mode node --logoutall=all
+    - path: /etc/containers/systemd/coreos-iscsi-vm.container
+      contents:
+        inline: |
+          [Unit]
+          Description=Boot VM over iSCSI
+          After=network-online.target After=nss-lookup.target install-coreos-to-iscsi-target.service 
+          Wants=network-online.target install-coreos-to-iscsi-target.service 
+          Requires=install-coreos-to-iscsi-target.service
+          OnFailure=emergency.target
+          [Container]
+          Image=quay.io/coreos-assembler/coreos-assembler
+          ContainerName=iscsiboot
+          Volume=/mnt/workdir-tmp/:/mnt/workdir-tmp/
+          Volume=/dev/virtio-ports/testiscsicompletion:/mnt/serial
+          PodmanArgs=--privileged
+          Network=host
+          Exec=shell -- kola qemuexec --netboot /mnt/workdir-tmp/boot.ipxe --usernet-addr 10.0.3.0/24 -- -device virtio-serial -chardev file,id=iscsi-completion-virtio,path=/mnt/serial,append=on -device virtserialport,chardev=iscsi-completion-virtio,name=testiscsicompletion
+          [Install]
+          # Start by default on boot
+          WantedBy=multi-user.target
+          [Service]
+          # fix permissions on the serial device before passing it as a volume
+          ExecStartPre=chmod 777 /dev/virtio-ports/testiscsicompletion
+    - path: /mnt/workdir-tmp/nested-ign.json
+      contents:
+        inline: | 
+          {
+            "ignition": {
+              "version": "3.1.0"
+            },
+            "systemd": {
+              "units": [
+                {
+                  "contents": "[Unit]\nDescription=iSCSI Boot Signal Completion\nAfter=multi-user.target\nOnFailureJobMode=isolate\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/bin/sh -c '/usr/bin/echo \"iscsi-boot-ok\" \u003e/dev/virtio-ports/testiscsicompletion \u0026\u0026 systemctl poweroff'\n[Install]\nRequiredBy=multi-user.target\n",
+                  "enabled": true,
+                  "name": "successful-boot-signal.service"
+                }
+              ]
+            }
+          }
+systemd:
+    units:
+    - name: setup-targetcli.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Setup targetcli
+        Requires=target.service
+        After=target.service
+        ConditionFirstBoot=true
+        OnFailure=emergency.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStart=/usr/local/bin/targetcli_script
+        [Install]
+        WantedBy=multi-user.target
+    - name: install-coreos-to-iscsi-target.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Mount an iscsi target and install coreOS into it
+        Requires=setup-targetcli.service
+        After=setup-targetcli.service
+        OnFailure=emergency.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStart=/usr/local/bin/install-coreos-iscsi
+        [Install]
+        WantedBy=multi-user.target

mantle/cmd/kola/testiso.go

@@ -22,6 +22,7 @@ package main
 import (
 	"bufio"
 	"context"
+	_ "embed"
 	"fmt"
 	"io"
 	"os"
@@ -88,6 +89,7 @@ var (
 		"iso-offline-install.bios",
 		"iso-offline-install.mpath.bios",
 		"iso-offline-install-fromram.4k.uefi",
+		"iso-offline-install-iscsi.bios",
 		"miniso-install.bios",
 		"miniso-install.nm.bios",
 		"miniso-install.4k.uefi",
@@ -109,6 +111,7 @@ var (
 		"miniso-install.s390fw",
 		"miniso-install.nm.s390fw",
 		"miniso-install.4k.nm.s390fw",
+		"iso-offline-install-iscsi.bios",
 	}
 	tests_ppc64le = []string{
 		"iso-live-login.ppcfw",
@@ -121,6 +124,7 @@ var (
 		"miniso-install.4k.nm.ppcfw",
 		"pxe-online-install.ppcfw",
 		"pxe-offline-install.4k.ppcfw",
+		"iso-offline-install-iscsi.bios",
 	}
 	tests_aarch64 = []string{
 		"iso-live-login.uefi",
@@ -136,6 +140,7 @@ var (
 		"pxe-offline-install.4k.uefi",
 		"pxe-online-install.uefi",
 		"pxe-online-install.4k.uefi",
+		"iso-offline-install-iscsi.bios",
 	}
 )
 
@@ -319,6 +324,9 @@ RequiredBy=coreos-installer.target
 # for target system
 RequiredBy=multi-user.target`, nmConnectionId, nmConnectionFile)
 
+//go:embed resources/iscsi_butane_setup.yaml
+var iscsi_butane_config string
+
 func init() {
 	cmdTestIso.Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image")
 	cmdTestIso.Flags().BoolVar(&console, "console", false, "Connect qemu console to terminal, turn off automatic initramfs failure checking")
@@ -587,6 +595,8 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) {
 			duration, err = testLiveIso(ctx, inst, filepath.Join(outputDir, test), false)
 		case "miniso-install":
 			duration, err = testLiveIso(ctx, inst, filepath.Join(outputDir, test), true)
+		case "iso-offline-install-iscsi":
+			duration, err = testLiveInstalliscsi(ctx, inst, filepath.Join(outputDir, test))
 		default:
 			plog.Fatalf("Unknown test name:%s", test)
 		}
@@ -955,3 +965,77 @@ func testAsDisk(ctx context.Context, outdir string) (time.Duration, error) {
 
 	return awaitCompletion(ctx, mach, outdir, completionChannel, nil, []string{liveOKSignal})
 }
+
+// iscsi_butane_setup.yaml contain the full butane config but here is an overview of the setup
+// 1 - Boot a live ISO with two extra 10G disks with labels "target" and "var"
+//   - Format and mount `virtio-var` to var
+//
+// 2 - target.container -> start an iscsi target, using quay.io/jbtrystram/targetcli
+// 3 - setup-targetcli.service calls /usr/local/bin/targetcli_script:
+//   - instructs targetcli to serve /dev/disk/by-id/virtio-target as an iscsi target
+//   - disables authentication
+//   - verifies the iscsi service is active and reachable
+//
+// 4 - install-coreos-to-iscsi-target.service calls /usr/local/bin/install-coreos-iscsi:
+//   - mount iscsi target
+//   - run coreos-installer on the mounted block device
+//   - unmount iscsi
+//
+// 5 - coreos-iscsi-vm.container start a coreos-assemble:
+//   - launch cosa qemuexec instructing it to boot from an iPXE script
+//     wich in turns mount the iscsi target and load kernel
+//   - note the virtserial port device: we pass through the serial port that was created by kola for test completion
+//
+// 6 - /mnt/workdir-tmp/nested-ign.json contains an ignition config:
+//   - when the system is booted, write a success string to /dev/virtio-ports/testiscsicompletion
+//   - As this serial device is mapped to the host serial device, the test concludes
+func testLiveInstalliscsi(ctx context.Context, inst platform.Install, outdir string) (time.Duration, error) {
+
+	builddir := kola.CosaBuild.Dir
+	isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path)
+	builder, err := newBaseQemuBuilder(outdir)
+	if err != nil {
+		return 0, err
+	}
+	defer builder.Close()
+	if err := builder.AddIso(isopath, "", false); err != nil {
+		return 0, err
+	}
+
+	completionChannel, err := builder.VirtioChannelRead("testiscsicompletion")
+	if err != nil {
+		return 0, err
+	}
+
+	// empty disk to use as an iscsi target to install coreOS on and subseqently boot
+	// Also add a 10G disk that we will mount on /var, to increase space available when pulling containers
+	err = builder.AddDisksFromSpecs([]string{"10G:serial=target", "10G:serial=var"})
+	if err != nil {
+		return 0, err
+	}
+
+	// We need more memory to start another VM within !
+	builder.MemoryMiB = 4096
+
+	var iscsiTargetConfig = conf.Butane(iscsi_butane_config)
+
+	config, err := iscsiTargetConfig.Render(conf.FailWarnings)
+	if err != nil {
+		return 0, err
+	}
+	// Add a failure target to stop the test if something go wrong rather than waiting for the 10min timeout
+	config.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable)
+
+	// enable network
+	builder.EnableUsermodeNetworking([]platform.HostForwardPort{}, "")
+
+	builder.SetConfig(config)
+
+	mach, err := builder.Exec()
+	if err != nil {
+		return 0, errors.Wrapf(err, "running iso")
+	}
+	defer mach.Destroy()
+
+	return awaitCompletion(ctx, mach, outdir, completionChannel, nil, []string{"iscsi-boot-ok"})
+}