[Runtime] CRaC: Restore stdout and stderr when run CRaC in unprivileg…

…ed docker.

Summary: The stdout and stderr are pipe files when run in docker, restore these pipe files is rather tricky. Write the fds to a file named pipefds, then after criu checkpoint successfully, the criu execute the criuengine as a postdump callback. In the callback, append the pipe info of java process to the file pipefds. It read the pipefds when restore, than pass the pipe info as --inherit-fd to criu. The problem is criuengine cannot get the pipefds file path if run with nonprivilged. To fix this, set the environment CRAC_IMAGE_DIR explictly when do checkpointing.

Testing: jdk/jdk/crac/stdoutInDocker/TestStdoutInDocker.sh

Reviewers: lei.yul,denghui.ddh

Issue: #867

src/java.base/unix/native/criuengine/criuengine.c

@@ -214,6 +214,7 @@ static int checkpoint(pid_t jvm, const char *basedir, const char *self,
     }
     *arg++ = NULL;
 
+    setenv("CRAC_IMAGE_DIR", imagedir, 1);
     execv(criu, (char **)args);
     perror("criu dump");
     exit(1);
@@ -564,17 +565,23 @@ static int read_pseudo_file(const char *imagedir, pseudo_file_func func, const c
 }
 
 static int post_dump(void) {
-  char *imagedir = getenv("CRTOOLS_IMAGE_DIR");
+  //"CRTOOLS_IMAGE_DIR" is a symbol link like "/proc/227/fd/100/",
+  //and 227 is the pid of criu.If run with unprivileged mode, current
+  //process has no permission to access the fd 100.
+  //So use CRAC_IMAGE_DIR environment which setting before checkpointing.
+  char *imagedir = getenv("CRAC_IMAGE_DIR");
   if (!imagedir) {
-    fprintf(stderr, MSGPREFIX "cannot find CRTOOLS_IMAGE_DIR env\n");
+    fprintf(stderr, MSGPREFIX "cannot find CRAC_IMAGE_DIR env\n");
     return 1;
   }
   char *jvm_pid= getenv("CRTOOLS_INIT_PID");
   if (!jvm_pid) {
     fprintf(stderr, MSGPREFIX "cannot find CRTOOLS_INIT_PID env in post_dump callback\n");
     return 1;
   }
-  append_pipeinfo(imagedir, jvm_pid);
+  if (append_pipeinfo(imagedir, jvm_pid)) {
+    return 1;
+  }
   return read_pseudo_file(imagedir, checkpoint_pseudo_persistent_file, "checkpoint pseudo persistent file ");
 }
 
@@ -844,10 +851,12 @@ static int append_pipeinfo(const char* imagedir,const char* jvm_pid) {
   char buff[1024];
 
   if (-1 == asprintf(&path, "%s/" PIPE_FDS, imagedir)) {
+    fprintf(stderr, "asprintf for pipefds failed. imagedir:%s", imagedir);
     return -1;
   }
 
   if (stat(path, &st) != 0) {
+    fprintf(stderr, "%s not exist.", path);
     free(path);
     return 0;
   }
@@ -859,9 +868,10 @@ static int append_pipeinfo(const char* imagedir,const char* jvm_pid) {
     return -1;
   }
 
+  free(path);
+
   if (fgets(buff, sizeof(buff), f) == NULL) {
     perror("read from pipefds error");
-    free(path);
     fclose(f);
     return -1;
   }
@@ -877,15 +887,13 @@ static int append_pipeinfo(const char* imagedir,const char* jvm_pid) {
     if (is_exist_pipefd(jvm_pid, fd)) {
       if (read_fd_link(jvm_pid, fd, fdpath, sizeof(fdpath)) == -1) {
         fclose(f);
-        free(path);
         return -1;
       }
       fprintf(f, "%d,%s\n", fd, fdpath);
     }
     token = strtok(NULL, ",");
   }
   fclose(f);
-  free(path);
   return 0;
 }
 

test/jdk/jdk/crac/stdoutInDocker/Dockerfile

@@ -0,0 +1,8 @@
+FROM alibaba-cloud-linux-3-registry.cn-hangzhou.cr.aliyuncs.com/alinux3/alinux3:latest
+ADD jdk.tar.gz /
+RUN setcap 'cap_checkpoint_restore+eip cap_setpcap+eip' /jdk/lib/criu
+RUN groupadd -g 1000 app && useradd -u 1000 -g app -s /bin/sh  -d /home/app -m app
+USER app:app
+ENV JAVA_HOME=/jdk
+ADD classes.tar.gz takepid.sh runapp.sh /home/app
+CMD /home/app/runapp.sh

test/jdk/jdk/crac/stdoutInDocker/TestStdoutInDocker.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2024, Alibaba Group Holding Limited. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+import jdk.crac.Context;
+import jdk.crac.Core;
+import jdk.crac.Resource;
+
+/**
+ * @summary Test run "docker logs" can get output from stdout/stderr correctly.
+ */
+
+public class TestStdoutInDocker {
+    public static void main(String[] args) throws Exception {
+        System.out.println("Message from stdout before doing checkpoint");
+        System.err.println("Message from stderr before doing checkpoint");
+        Core.getGlobalContext().register(new Resource() {
+            @Override
+            public void beforeCheckpoint(Context<? extends Resource> context) throws Exception {
+            }
+
+            @Override
+            public void afterRestore(Context<? extends Resource> context) throws Exception {
+                System.out.println("Message from stdout in afterRestore callback");
+                System.err.println("Message from stderr in afterRestore callback");
+            }
+        });
+        Core.checkpointRestore();
+        System.out.println("Message from stdout afterRestore");
+        System.err.println("Message from stderr afterRestore");
+        int sleepTime = Integer.parseInt(System.getenv("SLEEP_TIME"));
+        System.out.println("Sleep time is : " + sleepTime);
+        Thread.sleep(sleepTime);
+    }
+}

test/jdk/jdk/crac/stdoutInDocker/TestStdoutInDocker.sh

@@ -0,0 +1,131 @@
+#! /bin/sh -x
+#
+# Copyright (c) 2024, Alibaba Group Holding Limited. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+
+# @test
+# @summary  Test "docker logs" can get stdout/stderr after restoring.
+#           Test docker run with the combination of (privileged/unprivileged) and (interactive/detach) modes.
+# @requires docker.support
+# @requires os.family == "linux"
+# @requires os.arch=="amd64" | os.arch=="aarch64"
+# @build TestStdoutInDocker
+# @run shell TestStdoutInDocker.sh
+
+TEST_IMAGE="crac-stdout-err-testimage:latest"
+PRIV_PARAM="--privileged --user root"
+NON_PRIV_PARAM="--security-opt seccomp=unconfined --cap-add CHECKPOINT_RESTORE --cap-add CAP_SETPCAP --env CRAC_UNPRIV_OPT=-XX:+CRaCUnprivileged"
+CHECKPOINT_PARAM="--env CRAC_INHERIT_OPT=-XX:CRaCRestoreInheritPipeFds=1,2 --env DO_CHECKPOINT=true --env CRAC_IMAGE_DIR=/cr"
+RESTORE_PARAM="--env DO_RESTORE=true --env CRAC_IMAGE_DIR=/cr"
+CR_DIR="/tmp/cr"
+
+resetDir() {
+  rm -rf $1
+  mkdir -p $1
+}
+
+assertExist() {
+  grep "$1" $2
+  if [ $? -ne 0 ]; then
+    echo "$? not found $1 in file $2"
+    exit 1
+  fi
+}
+
+checkOutput() {
+  assertExist 'Message from stderr in afterRestore callback' $1
+  assertExist 'Message from stdout in afterRestore callback' $1
+  assertExist 'Message from stderr afterRestore' $1
+  assertExist 'Message from stdout afterRestore' $1
+}
+
+killDockerPs() {
+  docker ps | grep  ${TEST_IMAGE} | awk '{print $1}' | xargs docker kill &> /dev/null
+}
+
+executCmd() {
+  echo "Run command: [$1]"
+  eval $1
+}
+
+# cr dir is used to save image files, it need to mount as volume when run docker
+# So avoid the permission denied error, use a directory in /tmp and grant with 0777
+resetCrDir() {
+  rm -rf ${CR_DIR}
+  mkdir ${CR_DIR}
+  chmod 0777 ${CR_DIR}
+}
+
+runInteractive() {
+  resetCrDir
+  executCmd "docker run $1 ${CHECKPOINT_PARAM} -v ${CR_DIR}:/cr ${TEST_IMAGE}"
+  executCmd "docker run $1 ${RESTORE_PARAM} --env SLEEP_TIME=3000 -v ${CR_DIR}:/cr ${TEST_IMAGE} &> ${CR_DIR}/docker.log"
+  checkOutput ${CR_DIR}/docker.log
+}
+
+runDetach() {
+  resetCrDir
+  executCmd "docker run $1 -d ${CHECKPOINT_PARAM} -v ${CR_DIR}:/cr ${TEST_IMAGE}"
+  sleep 3
+  killDockerPs
+
+  executCmd "docker run $1 -d ${RESTORE_PARAM} --env SLEEP_TIME=30000 -v ${CR_DIR}:/cr ${TEST_IMAGE}"
+  sleep 3
+  cid=$(docker ps | grep ${TEST_IMAGE} | awk '{print $1}') 
+  docker logs ${cid} &> ${CR_DIR}/docker.log
+  killDockerPs
+  checkOutput ${CR_DIR}/docker.log
+}
+
+preCheck() {
+  match=$(uname -r | awk -F'.' '{print $1>=5 && $2>=9; }')
+  if [ $match -ne 1 ]; then
+    echo "OS version should >= 5.9"
+    exit 0
+  fi
+}
+
+buildTestImage() {
+  docker rmi -f ${TEST_IMAGE} 
+  resetDir ${WORK_DIR}/baseimage-tmp
+  pushd baseimage-tmp
+  mkdir jdk classes
+  cp -R ${TESTJAVA}/* jdk
+  cp -R ${TESTCLASSES}/* classes
+  cp ${TESTSRCPATH}/*.sh ${TESTSRCPATH}/Dockerfile .
+  chmod +x *.sh
+  tar zcvf jdk.tar.gz jdk
+  tar zcvf classes.tar.gz classes
+  rm -rf jdk classes
+  docker build -t ${TEST_IMAGE} .
+  popd
+  rm -rf baseimage-tmp 
+}
+
+WORK_DIR=$(pwd)
+preCheck
+buildTestImage
+runDetach "$PRIV_PARAM"
+runDetach "$NON_PRIV_PARAM"
+runInteractive "$PRIV_PARAM"
+runInteractive "$NON_PRIV_PARAM"

test/jdk/jdk/crac/stdoutInDocker/runapp.sh

@@ -0,0 +1,17 @@
+#! /bin/sh
+APP_DIR=$(dirname "$0")
+JAVA=${JAVA_HOME}/bin/java
+if [[ "${DO_CHECKPOINT}" = "true" ]];
+then
+    for i in {1..100}
+    do
+        sh ${APP_DIR}/takepid.sh
+    done
+    ${JAVA} -XX:CRaCCheckpointTo=${CRAC_IMAGE_DIR} ${CRAC_UNPRIV_OPT} ${CRAC_INHERIT_OPT} -cp ${APP_DIR}/classes TestStdoutInDocker
+elif [[ "${DO_RESTORE}" = "true" ]];
+then
+    export CRAC_CRIU_OPTS="-o ${CRAC_IMAGE_DIR}/restore.log -vvvv"
+    ${JAVA} -XX:CRaCRestoreFrom=${CRAC_IMAGE_DIR} ${CRAC_UNPRIV_OPT}  -cp ${APP_DIR}/classes TestStdoutInDocker
+else
+    ${JAVA} -cp ${APP_DIR}/classes TestStdoutInDocker
+fi

test/jdk/jdk/crac/stdoutInDocker/takepid.sh

@@ -0,0 +1,2 @@
+#! /bin/sh
+echo "nothing" > /dev/null