From 6ca5f360683db140274f0408a889ebfb2c83e8a9 Mon Sep 17 00:00:00 2001 From: Roman Penyaev Date: Fri, 20 Oct 2023 18:34:34 +0200 Subject: [PATCH 01/12] pkg/alpine: sort community packages No functional changes. Signed-off-by: Roman Penyaev --- pkg/alpine/mirrors/3.16/community | 50 +++++++++++++++---------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/pkg/alpine/mirrors/3.16/community b/pkg/alpine/mirrors/3.16/community index 98b87e6702..50a357de6e 100644 --- a/pkg/alpine/mirrors/3.16/community +++ b/pkg/alpine/mirrors/3.16/community @@ -1,7 +1,22 @@ +fio +fmt +hwinfo +i2c-tools-dev +iw +libgudev-dev +librados +librbd +libvirt +libvirt-client +libvirt-common-drivers +libvirt-daemon +libvirt-libs +libvirt-lxc +libvirt-qemu libvncserver libvncserver-dev -qemu-img perf +pkgconf py3-cachecontrol py3-colorama py3-contextlib2 @@ -13,37 +28,22 @@ py3-msgpack py3-pep517 py3-pip py3-progress +py3-pycryptodome py3-pytoml py3-retrying py3-tomli py3-webencodings -py3-pycryptodome +qemu +qemu-img +qemu-system-x86_64 sudo -tini -fio sysstat -hwinfo -iw -i2c-tools-dev -qemu-system-x86_64 -qemu -tpm2-tss +tini tpm2-abrmd -tpm2-tss-esys +tpm2-tss tpm2-tss-dev -tpm2-tss-sys -tpm2-tss-tctildr +tpm2-tss-esys tpm2-tss-fapi tpm2-tss-rc -pkgconf -libgudev-dev -librados -librbd -libvirt -libvirt-client -libvirt-common-drivers -libvirt-daemon -libvirt-libs -libvirt-lxc -libvirt-qemu -fmt +tpm2-tss-sys +tpm2-tss-tctildr From 7a46cf6752cc50793b0ac8b1650025ed999e0e0a Mon Sep 17 00:00:00 2001 From: Roman Penyaev Date: Fri, 20 Oct 2023 18:36:10 +0200 Subject: [PATCH 02/12] pkg/alpine: add tio package tio is a simple TTY terminal, which will be used by EVE for attaching to the application consoles in following patches. Stay tuned. Signed-off-by: Roman Penyaev --- pkg/alpine/mirrors/3.16/community | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/alpine/mirrors/3.16/community b/pkg/alpine/mirrors/3.16/community index 50a357de6e..715f029a51 100644 --- a/pkg/alpine/mirrors/3.16/community +++ b/pkg/alpine/mirrors/3.16/community @@ -39,6 +39,7 @@ qemu-system-x86_64 sudo sysstat tini +tio tpm2-abrmd tpm2-tss tpm2-tss-dev From d8d2f27248aea4d4d53d5a11e1305de793b0d569 Mon Sep 17 00:00:00 2001 From: Roman Penyaev Date: Sun, 22 Oct 2023 13:32:09 +0200 Subject: [PATCH 03/12] debug/Dockerfile: switch to new alpine hash with `tio` package Add `tio` and `socat` utilities for the debug service EVE container. These tools will be used for application console attach. Signed-off-by: Roman Penyaev --- pkg/debug/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/debug/Dockerfile b/pkg/debug/Dockerfile index a721c312e5..d0e703ad9f 100644 --- a/pkg/debug/Dockerfile +++ b/pkg/debug/Dockerfile @@ -4,7 +4,7 @@ # has a fast path for stack unwinding. This also happens # to be a perfect place to put any other kind of debug info # into the package: see abuild/etc/abuild.conf. -FROM lfedge/eve-alpine:fad44e3702708a8d044663a20fd98d933dddb41e as build +FROM lfedge/eve-alpine:cbf02c2c126f210933ec9bdb142eb080b400fd76 as build ENV BUILD_PKGS abuild curl tar make linux-headers patch g++ git gcc ncurses-dev autoconf # Feel free to add additional packages here, but be aware that # EVE's rootfs image can be no larger than 300Mb (and don't @@ -15,8 +15,8 @@ ENV PKGS openssl openssh-client openssh-server tini util-linux ca-certificates p # These packages are not available on the riscv arch, so I have no idea how # deliver those, but still install them on other archs. -ENV PKGS_amd64 procps tar dmidecode iptables dhcpcd -ENV PKGS_arm64 procps tar dmidecode iptables dhcpcd +ENV PKGS_amd64 procps tar dmidecode iptables dhcpcd tio socat +ENV PKGS_arm64 procps tar dmidecode iptables dhcpcd tio socat RUN eve-alpine-deploy.sh From 5c7900704066649868821eb10c8ca0e0e5930448 Mon Sep 17 00:00:00 2001 From: Roman Penyaev Date: Sun, 22 Oct 2023 13:33:46 +0200 Subject: [PATCH 04/12] dom0-tools/bin/eve: add `list-app-consoles` and `attach-app-console` commands The `list-app-consoles` command returns all running QEMU (KVM) consoles in the following format: # /persist/eve list-app-consoles PID APP-UUID CONS-TYPE CONS-ID --- -------- --------- --------- 3883 e4e2f56d-b833-4562-a86f-be654d6387ba VM e4e2f56d-b833-4562-a86f-be654d6387ba.1.1/cons 4072 f6d348cc-9c31-4f8b-8c4f-a4aae4590b97 CONTAINER f6d348cc-9c31-4f8b-8c4f-a4aae4590b97.1.2/cons 4072 f6d348cc-9c31-4f8b-8c4f-a4aae4590b97 VM f6d348cc-9c31-4f8b-8c4f-a4aae4590b97.1.2/prime-cons The `attach-app-console` command uses a console ID from the `list-app-consoles` as a parameter: # eve attach-app-console e4e2f56d-b833-4562-a86f-be654d6387ba.1.1/cons [20:26:15.116] tio v1.37 [20:26:15.116] Press ctrl-t q to quit [20:26:15.116] Connected Ubuntu 18.04.6 LTS user hvc0 user login: The `attach-app-console` command attaches to the virtual QEMU console and pumps bytes between socket and PTY, so that `tio` terminal can attach to the corresponding PTY. This gives a fully working terminal of the guest. Since `socat` and `tio` utilities exist only in the debug service EVE container, the only way to use `attach-app-console` is to do the `eve enter debug` prior the console attach. This is done by the `eve` script. Signed-off-by: Roman Penyaev --- pkg/dom0-ztools/rootfs/bin/eve | 54 ++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/pkg/dom0-ztools/rootfs/bin/eve b/pkg/dom0-ztools/rootfs/bin/eve index 58796ed110..f8e7c464a9 100755 --- a/pkg/dom0-ztools/rootfs/bin/eve +++ b/pkg/dom0-ztools/rootfs/bin/eve @@ -2,6 +2,7 @@ CTR_CMD="ctr --namespace services.linuxkit" CTR_UA_CMD="ctr --namespace eve-user-apps" +KVM_RUN_DIR="/run/hypervisor/kvm" help() { cat <<__EOT__ @@ -15,6 +16,8 @@ Welcome to EVE! pause resume destroy + list-app-consoles Outputs list of available consoles for attach + attach-app-console Attach to the application console, see 'eve list-app-consoles' for details persist list persist attach config mount @@ -114,6 +117,10 @@ dump_mem() { http_debug_request "/dump/memory" } +is_in_debug_service() { + grep -q '/eve/services/debug' < /proc/self/cgroup +} + case "$1" in exec) NO_FORK="-F" if [ "$2" = "--fork" ]; then @@ -128,6 +135,53 @@ case "$1" in # shellcheck disable=SC2086 exec nsenter ${NO_FORK} -a -t "${ID:-1}" "$CMD" "$@" ;; + list-app-consoles) + if ! is_in_debug_service; then + # List can be executed only in debug service container + # due to missing convenient GNU tools + eve enter debug "eve $1" + exit + fi + printf "PID\tAPP-UUID\t\t\t\tCONS-TYPE\tCONS-ID\n" + # shellcheck disable=SC3045 + printf "---\t--------\t\t\t\t---------\t---------\n" + for pid in $(pgrep qemu-system); do + # shellcheck disable=SC2009 + name=$(ps -p "$pid" --no-headers -o command | grep -o '\-name\s\+[^ ]\+' | awk '{print $2}') + # shellcheck disable=SC2009 + uuid=$(ps -p "$pid" --no-headers -o command | grep -o '\-uuid\s\+[^ ]\+' | awk '{print $2}') + dir="$KVM_RUN_DIR/$name" + if [ -e "$dir/prime-cons" ]; then + printf "%s\t%s\tCONTAINER\t%s/cons\n" "$pid" "$uuid" "$name" + printf "%s\t%s\tVM\t\t%s/prime-cons\n" "$pid" "$uuid" "$name" + else + printf "%s\t%s\tVM\t\t%s/cons\n" "$pid" "$uuid" "$name" + fi + done + ;; + attach-app-console) [ -z "$2" ] && help + if ! is_in_debug_service; then + # Attach can be executed only in debug service container + # due to missing convenient GNU tools + eve enter debug "eve $1 $2" + exit + fi + CONS="$KVM_RUN_DIR/$2" + if [ ! -e "$CONS" ]; then + echo "Error: console '$2' does not exist." + echo "Try to use 'eve list-app-consoles'." + exit 1 + fi + PTY=$(dirname "$CONS")/pty-$$ + # Start socat in background to pump bytes between console socket and PTY + trap 'kill $SOCAT_PID 2>/dev/null' EXIT HUP INT QUIT TERM + socat "unix-connect:$CONS" "pty,link=$PTY" & + SOCAT_PID=$! + # Give some time to socat to create PTY + sleep 1 + # Start tio + tio "$PTY" + ;; enter) # shellcheck disable=SC2086 ${CTR_CMD} t exec -t --exec-id $(basename $(mktemp)) ${2:-pillar} ${3:-sh -l} ;; From ce5628f29b51af041038d1b152f9f99f8514edd8 Mon Sep 17 00:00:00 2001 From: Roman Penyaev Date: Sun, 22 Oct 2023 13:38:22 +0200 Subject: [PATCH 05/12] pillar/kvm: create another virtual console for the container only This console is called a 'prime-cons' (could not think a better name) and will connect the hosting Vm of a container with EVE environment. So for a container there will be two virtual consoles: 'cons' and 'prime-cons'. The 'cons' is attached to the input and output of the entry process of a container (this functionality exists for ages), but a new 'prime-cons' is attached to the input and output of the getty process of the hosting Vm. The 'prime-cons' gives a way to execute commands on Vm of a container. Signed-off-by: Roman Penyaev --- pkg/pillar/hypervisor/kvm.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pkg/pillar/hypervisor/kvm.go b/pkg/pillar/hypervisor/kvm.go index cecb340d4d..1a55f044d6 100644 --- a/pkg/pillar/hypervisor/kvm.go +++ b/pkg/pillar/hypervisor/kvm.go @@ -158,6 +158,22 @@ const qemuConfTemplate = `# This file is automatically generated by domainmgr chardev = "charserial0" name = "org.lfedge.eve.console.0" +{{if .DomainConfig.IsOCIContainer}} +[chardev "charserial1"] + backend = "socket" + mux = "on" + path = "` + kvmStateDir + `{{.DomainConfig.DisplayName}}/prime-cons" + server = "on" + wait = "off" + logfile = "/dev/fd/1" + logappend = "on" + +[device] + driver = "virtconsole" + chardev = "charserial1" + name = "org.lfedge.eve.console.prime" +{{end}} + {{if .DomainConfig.EnableVnc}} [vnc "default"] vnc = "0.0.0.0:{{if .DomainConfig.VncDisplay}}{{.DomainConfig.VncDisplay}}{{else}}0{{end}}" From a1aa103ef43a5112b308ab0cc380a78343af5657 Mon Sep 17 00:00:00 2001 From: Roman Penyaev Date: Sun, 22 Oct 2023 13:42:30 +0200 Subject: [PATCH 06/12] xen-tools/init-initrd: start `agetty` in loop for the container Vm Start `agetty` in loop on the '/dev/hvc1' virtual console (the other end of this console is a 'prime-cons' visible on the EVE). Since by default init process does not have any SID set (is 0), agetty fails to set control terminal, so job control does not work. In order to overcome this problem 'agetty' is called by the 'setsid' command, which creates a SID. Signed-off-by: Roman Penyaev --- pkg/xen-tools/Dockerfile | 2 +- pkg/xen-tools/initrd/base.files | 1 + pkg/xen-tools/initrd/init-initrd | 10 ++++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/pkg/xen-tools/Dockerfile b/pkg/xen-tools/Dockerfile index c35c61c6e6..cdb6a4f8af 100644 --- a/pkg/xen-tools/Dockerfile +++ b/pkg/xen-tools/Dockerfile @@ -3,7 +3,7 @@ FROM lfedge/eve-uefi:d821658883d6748d8bbf0d6640c62288e3ce8c6f as uefi-build FROM lfedge/eve-alpine:0f4e313d0d84ac313ea35e966def9ef96f61aafb as runx-build -ENV BUILD_PKGS mkinitfs gcc musl-dev e2fsprogs chrony +ENV BUILD_PKGS mkinitfs gcc musl-dev e2fsprogs chrony agetty RUN eve-alpine-deploy.sh RUN rm -f /sbin/poweroff /etc/mkinitfs/features.d/base.files diff --git a/pkg/xen-tools/initrd/base.files b/pkg/xen-tools/initrd/base.files index 8f7e27e868..1d9c141e37 100644 --- a/pkg/xen-tools/initrd/base.files +++ b/pkg/xen-tools/initrd/base.files @@ -1,4 +1,5 @@ /bin/busybox +/sbin/agetty /sbin/mke2fs /lib/libext2fs.so.2* /lib/libcom_err.so.2* diff --git a/pkg/xen-tools/initrd/init-initrd b/pkg/xen-tools/initrd/init-initrd index 236901d83e..e86a875a0e 100755 --- a/pkg/xen-tools/initrd/init-initrd +++ b/pkg/xen-tools/initrd/init-initrd @@ -149,6 +149,16 @@ EOF fi fi +run_agetty_loop() +{ + while true; do + setsid agetty --autologin root --noclear hvc1 linux + done +} + +echo "Executing agetty" +run_agetty_loop & + cmd=`cat /mnt/cmdline` echo "Executing $cmd" From 3739b257585ff70aa7871fe5809fe9179101c01b Mon Sep 17 00:00:00 2001 From: Roman Penyaev Date: Sun, 22 Oct 2023 13:06:11 +0200 Subject: [PATCH 07/12] xen-tools/chroot2: move chroot() and chdir() to a child process There is no need to do chroot for the parent process before actual clone is called, all the environment preparation should be as close as possible to the actual execvp(). The other motivation for this change is the next patch, where PID file of the child process will be created on the host, but not in the chrooted container environment. So postpone chroot and move it to the child process. Signed-off-by: Roman Penyaev --- pkg/xen-tools/initrd/chroot2.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/pkg/xen-tools/initrd/chroot2.c b/pkg/xen-tools/initrd/chroot2.c index 7459edbfb8..72762d9894 100644 --- a/pkg/xen-tools/initrd/chroot2.c +++ b/pkg/xen-tools/initrd/chroot2.c @@ -9,6 +9,8 @@ typedef struct clone_args clone_args; struct clone_args { + const char *chroot; + const char *workdir; char **args; char *command; uid_t uid, gid; @@ -20,6 +22,10 @@ static char child_stack[STACK_SIZE]; /* Space for child's stack */ static int childFunc(void *args) { clone_args *parsed_args = (clone_args *)args; + + chroot(parsed_args->chroot); + chdir(parsed_args->workdir); + mount("proc", "/proc", "proc", 0, NULL); setgid(parsed_args->gid); @@ -37,16 +43,17 @@ int main(int argc, char **argv) { setsid(); ioctl(0, TIOCSCTTY, 1); - chroot(argv[1]); - chdir(argv[2]); - uid = strtol(argv[3], &endptr, 10); gid = strtol(argv[4], &endptr, 10); - args.uid = uid; - args.gid = gid; - args.command = argv[5]; - args.args = argv + 5; + args = (struct clone_args) { + .chroot = argv[1], + .workdir = argv[2], + .uid = uid, + .gid = gid, + .command = argv[5], + .args = argv + 5, + }; child_pid = clone(childFunc, child_stack + STACK_SIZE, CLONE_NEWPID | SIGCHLD, (void *)(&args)); From a6f7925f118b1226103398662e367c779cafbd62 Mon Sep 17 00:00:00 2001 From: Roman Penyaev Date: Sun, 22 Oct 2023 13:16:13 +0200 Subject: [PATCH 08/12] xen-tools/chroot2: do logging on clone() error check Also code tweaks. Signed-off-by: Roman Penyaev --- pkg/xen-tools/initrd/chroot2.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pkg/xen-tools/initrd/chroot2.c b/pkg/xen-tools/initrd/chroot2.c index 72762d9894..ea5009bb6d 100644 --- a/pkg/xen-tools/initrd/chroot2.c +++ b/pkg/xen-tools/initrd/chroot2.c @@ -1,10 +1,11 @@ #define _GNU_SOURCE #include -#include +#include +#include #include -#include #include -#include +#include +#include typedef struct clone_args clone_args; @@ -55,7 +56,13 @@ int main(int argc, char **argv) { .args = argv + 5, }; - child_pid = clone(childFunc, child_stack + STACK_SIZE, CLONE_NEWPID | SIGCHLD, (void *)(&args)); + child_pid = clone(childFunc, child_stack + STACK_SIZE, + CLONE_NEWPID | SIGCHLD, &args); + if (child_pid < 0) { + perror("clone() failed:"); + return -1; + } waitpid(child_pid, NULL, 0); + return 0; } From 2735602ed819452f154ed84b0d7f4768ea568f79 Mon Sep 17 00:00:00 2001 From: Roman Penyaev Date: Sun, 22 Oct 2023 13:19:00 +0200 Subject: [PATCH 09/12] xen-tools/chroot2,init-initrd: write pid of the container entry process Make chroot2 utility responsible for writing a PID of the cloned child entry point process to the file passed as 5th argument, e.g. chroot2 /mnt/rootfs / 0 0 /mnt/entrypoint-pid /bin/sh ^^^^^^^^^^^^^^^^^^^ file which will be created by the chroot2 and will contain a PID of a container entry point process. The PID will be used for a container namespace enter in the following patch. The `init-initrd` was changed correspondingly and now the script passes /mnt/entrypoint-pid to the chroot2 as 5th argument. Signed-off-by: Roman Penyaev --- pkg/xen-tools/initrd/chroot2.c | 27 ++++++++++++++++++++++++--- pkg/xen-tools/initrd/init-initrd | 8 ++++++-- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/pkg/xen-tools/initrd/chroot2.c b/pkg/xen-tools/initrd/chroot2.c index ea5009bb6d..8347771960 100644 --- a/pkg/xen-tools/initrd/chroot2.c +++ b/pkg/xen-tools/initrd/chroot2.c @@ -1,9 +1,12 @@ #define _GNU_SOURCE +#include #include #include #include #include #include +#include +#include #include #include @@ -36,10 +39,12 @@ static int childFunc(void *args) } int main(int argc, char **argv) { + const char *pid_file = argv[5]; uid_t uid, gid; char *endptr; pid_t child_pid; struct clone_args args; + int fd; setsid(); ioctl(0, TIOCSCTTY, 1); @@ -52,10 +57,9 @@ int main(int argc, char **argv) { .workdir = argv[2], .uid = uid, .gid = gid, - .command = argv[5], - .args = argv + 5, + .command = argv[6], + .args = argv + 6, }; - child_pid = clone(childFunc, child_stack + STACK_SIZE, CLONE_NEWPID | SIGCHLD, &args); if (child_pid < 0) { @@ -63,6 +67,23 @@ int main(int argc, char **argv) { return -1; } + /* + * Open a file and write a PID of the child process in order + * to do attach to its namespace. + */ + fd = open(pid_file, O_CREAT|O_WRONLY|O_TRUNC, S_IRUSR|S_IWUSR); + if (fd < 0) { + /* Don't consider as fatal */ + perror("open(pid_file) failed:"); + } else { + char buf[64]; + int len; + + len = snprintf(buf, sizeof(buf), "%u\n", child_pid); + write(fd, buf, len); + close(fd); + } + waitpid(child_pid, NULL, 0); return 0; } diff --git a/pkg/xen-tools/initrd/init-initrd b/pkg/xen-tools/initrd/init-initrd index e86a875a0e..93549849ce 100755 --- a/pkg/xen-tools/initrd/init-initrd +++ b/pkg/xen-tools/initrd/init-initrd @@ -168,13 +168,17 @@ if [ -f /mnt/ug ]; then fi echo "Executing with uid gid: $ug" +# File which will contain a PID of the started entrypoint container +# process. File will be used for the `eve-enter-container` script. +pid_file="/mnt/entrypoint-pid" + if grep -q "console=tty0" /proc/cmdline; then #shellcheck disable=SC2086 #we have tty0 console primary, so will add output to hvc0 for logging - eval /chroot2 /mnt/rootfs "${WORKDIR:-/}" $ug $cmd 2>&1 | tee -i /dev/hvc0 + eval /chroot2 /mnt/rootfs "${WORKDIR:-/}" $ug $pid_file $cmd 2>&1 | tee -i /dev/hvc0 else #shellcheck disable=SC2086 - eval /chroot2 /mnt/rootfs "${WORKDIR:-/}" $ug $cmd <> /dev/console 2>&1 + eval /chroot2 /mnt/rootfs "${WORKDIR:-/}" $ug $pid_file $cmd <> /dev/console 2>&1 fi # once the command exits -- the only thing left is shut everything down From 6830fa9e8a766347ed73128502c38a5c3e6ef67c Mon Sep 17 00:00:00 2001 From: Roman Penyaev Date: Sun, 22 Oct 2023 13:23:47 +0200 Subject: [PATCH 10/12] xen-tools: add `eve-enter-container` script Once terminal is attached to the Vm console which hosts a container user can execute `eve-enter-container` script and enter corresponding container. Script uses `nsenter -t $pid` and pid is extracted from the /mnt/entrypoint-pid file, created by the chroot2 utility. Signed-off-by: Roman Penyaev --- pkg/xen-tools/Dockerfile | 1 + pkg/xen-tools/initrd/base.files | 1 + pkg/xen-tools/initrd/eve-enter-container | 13 +++++++++++++ 3 files changed, 15 insertions(+) create mode 100755 pkg/xen-tools/initrd/eve-enter-container diff --git a/pkg/xen-tools/Dockerfile b/pkg/xen-tools/Dockerfile index cdb6a4f8af..cd25679cd6 100644 --- a/pkg/xen-tools/Dockerfile +++ b/pkg/xen-tools/Dockerfile @@ -12,6 +12,7 @@ COPY initrd/init-initrd initrd/mount_disk.sh initrd/udhcpc_script.sh / COPY initrd/poweroff /sbin/poweroff COPY initrd/chroot2.c initrd/hacf.c /tmp/ COPY initrd/00000080 /etc/acpi/PWRF/ +COPY initrd/eve-enter-container /bin/ RUN gcc -s -o /chroot2 /tmp/chroot2.c RUN gcc -s -o /hacf /tmp/hacf.c RUN mkinitfs -n -F base -i /init-initrd -o /runx-initrd diff --git a/pkg/xen-tools/initrd/base.files b/pkg/xen-tools/initrd/base.files index 1d9c141e37..e812f44d03 100644 --- a/pkg/xen-tools/initrd/base.files +++ b/pkg/xen-tools/initrd/base.files @@ -1,4 +1,5 @@ /bin/busybox +/bin/eve-enter-container /sbin/agetty /sbin/mke2fs /lib/libext2fs.so.2* diff --git a/pkg/xen-tools/initrd/eve-enter-container b/pkg/xen-tools/initrd/eve-enter-container new file mode 100755 index 0000000000..952f3b6954 --- /dev/null +++ b/pkg/xen-tools/initrd/eve-enter-container @@ -0,0 +1,13 @@ +#!/bin/sh + +# PID of the started entrypoint container process. +# PID file is created from the `init-initrd` script. +PID_FILE="/mnt/entrypoint-pid" + +if [ ! -f "$PID_FILE" ]; then + echo "Error: PID file '$PID_FILE' not found. Container did not start?" + exit 1 +fi + +PID=$(cat "$PID_FILE") +nsenter -t "$PID" -m -u -i -n -p -r/mnt/rootfs -w/mnt/rootfs From 026fd12d37cec0be9b202902d0d70187cc59d80d Mon Sep 17 00:00:00 2001 From: Roman Penyaev Date: Sat, 21 Oct 2023 22:40:47 +0200 Subject: [PATCH 11/12] docs/DEBUGGING.md: add 'application console' section The application console section describes usage of application consoles. Signed-off-by: Roman Penyaev --- docs/DEBUGGING.md | 69 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/docs/DEBUGGING.md b/docs/DEBUGGING.md index 0f5e7aa9f5..345e545a78 100644 --- a/docs/DEBUGGING.md +++ b/docs/DEBUGGING.md @@ -93,6 +93,75 @@ tail -F /run/diag.out In addition this information is provided to application instances on the device using [the diag API endpoint](./ECO-METADATA.md). +## Application console + +A running application on an EVE device has a console for input or output. You can attach to the application console from the EVE device as a control terminal if the application (VM or Container) listens to the TTY line and communicates with the virtual console /dev/hvc0 device. For example for popular linux distributions deployed as VM application this is usually the case. + +First list applications consoles of all running QEMU (KVM) processes: + +```bash +# eve list-app-consoles +PID APP-UUID CONS-TYPE CONS-ID +--- -------- --------- --------- +3883 e4e2f56d-b833-4562-a86f-be654d6387ba VM e4e2f56d-b833-4562-a86f-be654d6387ba.1.1/cons +4072 f6d348cc-9c31-4f8b-8c4f-a4aae4590b97 CONTAINER f6d348cc-9c31-4f8b-8c4f-a4aae4590b97.1.2/cons +4072 f6d348cc-9c31-4f8b-8c4f-a4aae4590b97 VM f6d348cc-9c31-4f8b-8c4f-a4aae4590b97.1.2/prime-cons + +``` + +Where fields are: + +* PID - the process ID of the QEMU process +* APP-UUID - UUID of the application +* CONS-TYPE - Type of the console +* CONS-ID - ID of the console, should be used for attaching to the console by passing the console ID to the `eve attach-app-console` command + +Different application types may have different consoles (as mentioned above). An application of type "Virtual Machine" can only have a console of type "VM", which leads to the console of the user application; An application of the "Container" type has two types of console: the console of the "VM" type leads to the Virtual Machine that hosts the container, the console of the "CONTAINER" type leads to the user container itself. + +Choose console ID you need to attach and pass it as an argument to the `eve attach-app-console` command: + +```bash +# eve attach-app-console e4e2f56d-b833-4562-a86f-be654d6387ba.1.1/cons +[20:26:15.116] tio v1.37 +[20:26:15.116] Press ctrl-t q to quit +[20:26:15.116] Connected + + +Ubuntu 18.04.6 LTS user hvc0 + +user login: +``` + +Note: `tio` utility is used as a simple TTY terminal, so in order to quit the session please press `ctrl-t q` or read the `tio` manual for additional commands. + +The same 'cons' console ID can be used for the Container application, but please be aware if container does not start a shell then terminal is very limited and can be used only for reading for the console output, but not for executing commands. + +In order to attach to the console of the hosting Vm of the Container application another console ID should be used which is named `prime-cons`: + +```bash +# eve attach-app-console f6d348cc-9c31-4f8b-8c4f-a4aae4590b97.1.2/prime-cons +[20:41:47.124] tio v1.37 +[20:41:47.124] Press ctrl-t q to quit +[20:41:47.124] Connected + +~ # +``` + +The `prime-cons` console exists only for the Container applications and is always reachable for executing commands on the Vm which hosts corresponding container. + +Once terminal responds on the `prime-cons` console it is possible to enter container by executing the `eve-enter-container` command: + +```bash +~ # eve-enter-container +(none):/# ps awux +PID USER TIME COMMAND + 1 root 0:00 /bin/sh + 6 root 0:00 -ash + 7 root 0:00 ps awux +(none):/# exit +~ # +``` + ## Reboots EVE is architected in such a way that if any service is unresponsive for a period of time, the entire device will reboot. When this happens a BootReason is constructed and sent in the device info message to the controller. If there is a golang panic there can also be useful information found in `/persist/agentdebug/`. From 23cc14a80a53a63ffcb945f1baf6dc5d91357ff6 Mon Sep 17 00:00:00 2001 From: Roman Penyaev Date: Fri, 3 Nov 2023 16:48:27 +0100 Subject: [PATCH 12/12] pillar/kvm_test: reflect QEMU config changes Every change in the QEMU config in the 'kvm.go' should be reflected in the corresponding 'kvm_test.go' file, otherwise unit-test fails. Signed-off-by: Roman Penyaev --- pkg/pillar/hypervisor/kvm_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkg/pillar/hypervisor/kvm_test.go b/pkg/pillar/hypervisor/kvm_test.go index 04fa2eef38..c19973a2d8 100644 --- a/pkg/pillar/hypervisor/kvm_test.go +++ b/pkg/pillar/hypervisor/kvm_test.go @@ -181,6 +181,8 @@ func TestCreateDomConfigOnlyCom1(t *testing.T) { name = "org.lfedge.eve.console.0" + + #[device "video0"] # driver = "qxl-vga" # ram_size = "67108864" @@ -443,6 +445,8 @@ func TestCreateDomConfigOnlyCom1(t *testing.T) { name = "org.lfedge.eve.console.0" + + #[device "video0"] # driver = "qxl-vga" # ram_size = "67108864" @@ -681,6 +685,8 @@ func TestCreateDomConfigOnlyCom1(t *testing.T) { name = "org.lfedge.eve.console.0" + + #[device "video0"] # driver = "qxl-vga" # ram_size = "67108864" @@ -1117,6 +1123,8 @@ func domConfigArm64() string { name = "org.lfedge.eve.console.0" + + #[device "video0"] # driver = "qxl-vga" # ram_size = "67108864" @@ -1396,6 +1404,8 @@ func domConfigAmd64FML() string { name = "org.lfedge.eve.console.0" + + #[device "video0"] # driver = "qxl-vga" # ram_size = "67108864" @@ -1686,6 +1696,8 @@ func domConfigAmd64Legacy() string { name = "org.lfedge.eve.console.0" + + #[device "video0"] # driver = "qxl-vga" # ram_size = "67108864" @@ -1967,6 +1979,8 @@ func domConfigAmd64() string { name = "org.lfedge.eve.console.0" + + #[device "video0"] # driver = "qxl-vga" # ram_size = "67108864"