Test VM boot using an SPDK bdev_aio disk

Reproducing with QEMU (Incomplete, WIP)

0. Setup

sudo apt install qemu-system-x86 mtools

export SPDK_BIN=/opt/spdk/bin

echo '{"subsystems":[{"subsystem":"bdev","config":[{"method":"bdev_aio_create","params":{"name":"aio0","block_size":512,"filename":"disk.raw","readonly":false}}}]}' \
> spdk_conf.json

1. Download & create image

wget https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img

wget https://raw.githubusercontent.com/cloud-hypervisor/cloud-hypervisor/main/test_data/cloud-init/ubuntu/local/user-data
wget https://raw.githubusercontent.com/cloud-hypervisor/cloud-hypervisor/main/test_data/cloud-init/ubuntu/local/network-config
wget https://raw.githubusercontent.com/cloud-hypervisor/cloud-hypervisor/main/test_data/cloud-init/ubuntu/local/meta-data

rm -f /tmp/ubuntu-cloudinit.img
mkdosfs -n CIDATA -C /tmp/ubuntu-cloudinit.img 8192
mcopy -oi /tmp/ubuntu-cloudinit.img -s ./user-data ::
mcopy -oi /tmp/ubuntu-cloudinit.img -s ./network-config ::
mcopy -oi /tmp/ubuntu-cloudinit.img -s ./meta-data ::

# convert image to raw format
qemu-img convert -p -f qcow2 -O raw jammy-server-cloudimg-amd64.img jammy.raw
truncate -s 5G jammy.raw

# for unencrypted, just a simple copy
rm -rf disk.raw
cp jammy.raw disk.raw

2. Start vhost

export SPDK_SCRIPTS=/opt/spdk/scripts
HUGEMEM=5120 $SPDK_SCRIPTS/setup.sh

$SPDK_BIN/vhost --config spdk_conf.json -S /var/tmp

# in another window
# ONLY one of following commands, depending on if you want encrypted or unencrypted
$SPDK_SCRIPTS/rpc.py vhost_create_blk_controller vhost.1 aio0

3a. Run qemu

sudo qemu-system-x86_64 \
   --enable-kvm -cpu host -smp 2 -m 4G \
   -object memory-backend-file,id=mem0,size=1G,mem-path=/dev/hugepages,share=on -numa node,memdev=mem0 \
   -chardev socket,id=spdk_vhost_blk0,path=/var/tmp/vhost.1,reconnect=1 \
   -device vhost-user-blk-pci,chardev=spdk_vhost_blk0,num-queues=4,bootindex=0 \
   -drive file=/tmp/ubuntu-cloudinit.img,format=raw \
   -nographic

Login with:

• User: cloud
• Password: cloud123

3b. Run cloud-hypervisor

wget https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/v33.0/cloud-hypervisor

wget https://github.com/cloud-hypervisor/rust-hypervisor-firmware/releases/download/0.4.2/hypervisor-fw

chmod a+x cloud-hypervisor

./cloud-hypervisor \
    --kernel ./hypervisor-fw \
    --disk path=/tmp/ubuntu-cloudinit.img \
    --disk vhost_user=true,socket=/var/tmp/vhost.1,num_queues=1,queue_size=256 \
    --cpus boot=1 \
    --memory size=4G,hugepages=on,shared=true \
    --net "tap=,mac=,ip=,mask=" \
    --serial tty \
    --console off

4. Commands inside the VM

dd if=/dev/random of=1.txt bs=512 count=1000000
sync 1.txt
dd if=/dev/random of=2.txt bs=512 count=1000000
sync 2.txt
lsblk