Merge 3.0-dev to 3.0 from latest 3.0-dev build (#8828)

.github/workflows/validate-cg-manifest.sh

@@ -39,11 +39,11 @@ ignore_no_source_tarball=" \
   hyphen-mn \
   initramfs \
   javapackages-tools-meta \
+  kata-packages-uvm \
   kde-filesystem \
   kf \
   livepatching \
   lua-rpm-macros \
-  multilib-rpm-config \
   opencl-filesystem \
   patterns-ceph-containers \
   pyproject-rpm-macros \
@@ -225,7 +225,7 @@ do
       # Parsing output instead of using error codes because 'wget' returns code 8 for FTP, even if the file exists.
       # Sample HTTP(S) output:  Remote file exists.
       # Sample FTP output:      File ‘time-1.9.tar.gz’ exists.
-      if ! wget --secure-protocol=TLSv1_2 --spider --timeout=2 --tries=10 "${manifesturl}" 2>&1 | grep -qP "^(Remote file|File ‘.*’) exists.*"
+      if ! wget --secure-protocol=TLSv1_2 --spider --timeout=30 --tries=10 "${manifesturl}" 2>&1 | grep -qP "^(Remote file|File ‘.*’) exists.*"
       then
         echo "Registration for $name:$version has invalid URL '$manifesturl' (could not download)"  >> bad_registrations.txt
       fi

.pipelines/containerSourceData/base/Dockerfile-Base-Template

@@ -1,12 +1,32 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
+ARG BUILDER_IMAGE
 ARG BASE_IMAGE
 
-FROM $BASE_IMAGE
+FROM $BUILDER_IMAGE as builder
 
+ARG USERNAME=nonroot
+ARG USER_UID=65532
+ARG USER_GID=$USER_UID
+
+RUN mkdir -p /staging/etc \
+    && groupadd --gid $USER_GID $USERNAME \
+    && useradd --gid $USER_GID $USERNAME --uid $USER_UID --create-home --comment "Nonroot User" \
+    # Copy user/group info to staging
+    && cp /etc/group /staging/etc/group \
+    && cp /etc/passwd /staging/etc/passwd \
+    && cp -r /home /staging/home
+
+FROM $BASE_IMAGE AS final
+
+ARG USER_UID=65532
+ARG USER_GID=$USER_UID
 ARG EULA=@EULA_FILE@
 
 COPY $EULA .
 
+COPY --from=builder /staging/etc/ /etc/
+COPY --from=builder --chown=${USER_UID}:${USER_GID} /staging/home/ /home/
+
 CMD [ "bash" ]

.pipelines/containerSourceData/base/Dockerfile-Distroless-Template

@@ -1,10 +1,30 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
+ARG BUILDER_IMAGE
 ARG BASE_IMAGE
 
-FROM $BASE_IMAGE
+FROM $BUILDER_IMAGE as builder
 
+ARG USERNAME=nonroot
+ARG USER_UID=65532
+ARG USER_GID=$USER_UID
+
+RUN mkdir -p /staging/etc \
+    && groupadd --gid $USER_GID $USERNAME \
+    && useradd --gid $USER_GID $USERNAME --uid $USER_UID --create-home --comment "Nonroot User" \
+    # Copy user/group info to staging
+    && cp /etc/group /staging/etc/group \
+    && cp /etc/passwd /staging/etc/passwd \
+    && cp -r /home /staging/home
+
+FROM $BASE_IMAGE AS final
+
+ARG USER_UID=65532
+ARG USER_GID=$USER_UID
 ARG EULA=@EULA_FILE@
 
 COPY $EULA .
+
+COPY --from=builder /staging/etc/ /etc/
+COPY --from=builder --chown=${USER_UID}:${USER_GID} /staging/home/ /home/

.pipelines/containerSourceData/nodejs/distroless/holdback-nodejs18.pkg

@@ -4,5 +4,3 @@ coreutils
 gmp
 grep
 libselinux
-pcre
-pcre-libs

.pipelines/containerSourceData/prometheus/distroless/holdback-prometheus.pkg

@@ -4,5 +4,3 @@ coreutils
 gmp
 grep
 libselinux
-pcre
-pcre-libs

.pipelines/containerSourceData/prometheusadapter/distroless/holdback-prometheusadapter.pkg

@@ -4,5 +4,3 @@ coreutils
 gmp
 grep
 libselinux
-pcre
-pcre-libs

.pipelines/containerSourceData/python/distroless/holdback-python.pkg

@@ -4,5 +4,3 @@ coreutils
 gmp
 grep
 libselinux
-pcre
-pcre-libs

.pipelines/containerSourceData/scripts/BuildBaseContainers.sh

@@ -22,12 +22,12 @@ set -e
 #   │   containerSourceData
 #   │   ├── base
 #   │   │   ├── Dockerfile-Base-Template
-#   │   │   ├── Dockerfile-Base-Nonroot-Template
 #   │   │   ├── Dockerfile-Distroless-Template
-#   │   │   ├── Dockerfile-Distroless-Nonroot-Template
 #   │   container_tarballs
 #   │   ├── container_base
 #   │   │   ├── core-3.0.20240101.tar.gz
+#   │   ├── core_container_builder
+#   │   │   ├── core-container-builder-3.0.20240101.tar.gz
 #   │   ├── distroless_base
 #   │   │   ├── distroless-base-3.0.20240101.tar.gz
 #   │   ├── distroless_debug
@@ -100,6 +100,7 @@ function validate_inputs {
     fi
 
     BASE_TARBALL=$(find "$CONTAINER_TARBALLS_DIR" -name "core-[0-9.]*.tar.gz")
+    BASE_BUILDER_TARBALL=$(find "$CONTAINER_TARBALLS_DIR" -name "core-container-builder-[0-9.]*.tar.gz")
     DISTROLESS_BASE_TARBALL=$(find "$CONTAINER_TARBALLS_DIR" -name "distroless-base-[0-9.]*.tar.gz")
     DISTROLESS_DEBUG_TARBALL=$(find "$CONTAINER_TARBALLS_DIR" -name "distroless-debug-[0-9.]*.tar.gz")
     DISTROLESS_MINIMAL_TARBALL=$(find "$CONTAINER_TARBALLS_DIR" -name "distroless-minimal-[0-9.]*.tar.gz")
@@ -162,6 +163,7 @@ function initialization {
     EULA_FILE_NAME="EULA-Container.txt"
 
     # Image types
+    BASE_BUILDER="base-builder"
     BASE="base"
     DISTROLESS="distroless"
     MARINARA="marinara"
@@ -176,32 +178,27 @@ function initialization {
     echo "BUILD_ID                      -> $BUILD_ID"
 
     IMAGE_TAG=$BASE_IMAGE_TAG-$ARCHITECTURE
-    NONROOT_IMAGE_TAG=$AZL_VERSION-nonroot.$BUILD_ID-$ARCHITECTURE
 
     # Set various image names.
     BASE_IMAGE_NAME="$ACR_NAME_FULL/base/core:$IMAGE_TAG"
-    BASE_NONROOT_IMAGE_NAME="$ACR_NAME_FULL/base/core:$NONROOT_IMAGE_TAG"
-
     DISTROLESS_BASE_IMAGE_NAME="$ACR_NAME_FULL/distroless/base:$IMAGE_TAG"
-    DISTROLESS_BASE_NONROOT_IMAGE_NAME="$ACR_NAME_FULL/distroless/base:$NONROOT_IMAGE_TAG"
-
     DISTROLESS_MINIMAL_IMAGE_NAME="$ACR_NAME_FULL/distroless/minimal:$IMAGE_TAG"
-    DISTROLESS_MINIMAL_NONROOT_IMAGE_NAME="$ACR_NAME_FULL/distroless/minimal:$NONROOT_IMAGE_TAG"
-
-    DISTROLESS_DEBUG_NONROOT_IMAGE_NAME="$ACR_NAME_FULL/distroless/debug:$NONROOT_IMAGE_TAG"
     DISTROLESS_DEBUG_IMAGE_NAME="$ACR_NAME_FULL/distroless/debug:$IMAGE_TAG"
-
     MARINARA_IMAGE_NAME="$ACR_NAME_FULL/marinara:$IMAGE_TAG"
 
     echo "BASE_IMAGE_NAME                       -> $BASE_IMAGE_NAME"
-    echo "BASE_NONROOT_IMAGE_NAME               -> $BASE_NONROOT_IMAGE_NAME"
     echo "DISTROLESS_BASE_IMAGE_NAME            -> $DISTROLESS_BASE_IMAGE_NAME"
-    echo "DISTROLESS_BASE_NONROOT_IMAGE_NAME    -> $DISTROLESS_BASE_NONROOT_IMAGE_NAME"
     echo "DISTROLESS_MINIMAL_IMAGE_NAME         -> $DISTROLESS_MINIMAL_IMAGE_NAME"
-    echo "DISTROLESS_MINIMAL_NONROOT_IMAGE_NAME -> $DISTROLESS_MINIMAL_NONROOT_IMAGE_NAME"
     echo "DISTROLESS_DEBUG_IMAGE_NAME           -> $DISTROLESS_DEBUG_IMAGE_NAME"
-    echo "DISTROLESS_DEBUG_NONROOT_IMAGE_NAME   -> $DISTROLESS_DEBUG_NONROOT_IMAGE_NAME"
     echo "MARINARA_IMAGE_NAME                   -> $MARINARA_IMAGE_NAME"
+
+    ROOT_FOLDER="$(git rev-parse --show-toplevel)"
+    EULA_FILE_PATH="$ROOT_FOLDER/.pipelines/container_artifacts/data"
+}
+
+function build_builder_image {
+    echo "+++ Build builder image"
+    docker import - "$BASE_BUILDER" < "$BASE_BUILDER_TARBALL"
 }
 
 function docker_build {
@@ -217,8 +214,6 @@ function docker_build {
     local build_dir="$WORK_DIR/container_build_dir"
     mkdir -p "$build_dir"
 
-    ROOT_FOLDER="$(git rev-parse --show-toplevel)"
-    EULA_FILE_PATH="$ROOT_FOLDER/.pipelines/container_artifacts/data"
     if [ -d "$EULA_FILE_PATH" ]; then
         cp "$EULA_FILE_PATH/$EULA_FILE_NAME" "$build_dir"/
     fi
@@ -229,6 +224,7 @@ function docker_build {
 
     echo "+++ Build image: $image_full_name"
     docker build . \
+        --build-arg BUILDER_IMAGE="$BASE_BUILDER" \
         --build-arg EULA="$EULA_FILE_NAME" \
         --build-arg BASE_IMAGE="$temp_image" \
         -t "$image_full_name" \
@@ -243,47 +239,27 @@ function docker_build {
     save_container_image "$image_type" "$image_full_name"
 }
 
-function docker_build_custom {
-    local image_type=$1
-    local image_full_name=$2
-    local final_image_to_use=$3
-    local dockerfile=$4
-
-    # $WORK_DIR has $RPMS_DIR directory and $LOCAL_REPO_FILE file.
-    pushd "$WORK_DIR" > /dev/null
-
-    echo "+++ Build image: $image_full_name"
-    docker build . \
-        --build-arg BASE_IMAGE="$BASE_IMAGE_NAME" \
-        --build-arg FINAL_IMAGE="$final_image_to_use" \
-        --build-arg AZL_VERSION="$AZL_VERSION" \
-        --build-arg RPMS="$RPMS_DIR" \
-        --build-arg LOCAL_REPO_FILE="$LOCAL_REPO_FILE" \
-        -t "$image_full_name" \
-        -f "$CONTAINER_SRC_DIR/base/$dockerfile" \
-        --no-cache \
-        --progress=plain
-
-    popd > /dev/null
-
-    publish_to_acr "$image_full_name"
-    save_container_image "$image_type" "$image_full_name"
-}
-
 function docker_build_marinara {
     echo "+++ Build Marinara image: $MARINARA_IMAGE_NAME"
 
     local build_dir="$WORK_DIR/marinara_build_dir"
     mkdir -p "$build_dir"
     git clone "https://github.com/microsoft/$MARINARA.git" "$build_dir"
-    pushd "$build_dir"
 
-    sed -E "s|^FROM mcr\..*installer$|FROM $BASE_IMAGE_NAME as installer|g" -i "dockerfile-$MARINARA"
+    if [ -d "$EULA_FILE_PATH" ]; then
+        cp "$EULA_FILE_PATH/$EULA_FILE_NAME" "$build_dir"/
+    fi
+
+    pushd "$build_dir" > /dev/null
+
+    sed -E "s|^FROM mcr\..*installer$|FROM $BASE_BUILDER as installer|g" -i "dockerfile-$MARINARA"
 
     docker build . \
         -t "$MARINARA_IMAGE_NAME" \
         -f dockerfile-$MARINARA \
         --build-arg AZL_VERSION="$AZL_VERSION" \
+        --build-arg INSTALL_DEPENDENCIES=false \
+        --build-arg EULA=$EULA_FILE_NAME \
         --no-cache \
         --progress=plain
 
@@ -315,21 +291,15 @@ function save_container_image {
 
 function build_images {
     echo "+++ Build images"
-
     docker_build $BASE "$BASE_IMAGE_NAME" "$BASE_TARBALL" "Dockerfile-Base-Template"
     docker_build $DISTROLESS "$DISTROLESS_BASE_IMAGE_NAME" "$DISTROLESS_BASE_TARBALL" "Dockerfile-Distroless-Template"
     docker_build $DISTROLESS "$DISTROLESS_MINIMAL_IMAGE_NAME" "$DISTROLESS_MINIMAL_TARBALL" "Dockerfile-Distroless-Template"
     docker_build $DISTROLESS "$DISTROLESS_DEBUG_IMAGE_NAME" "$DISTROLESS_DEBUG_TARBALL" "Dockerfile-Distroless-Template"
-
-    docker_build_custom $BASE "$BASE_NONROOT_IMAGE_NAME" "" "Dockerfile-Base-Nonroot-Template"
-    docker_build_custom $DISTROLESS "$DISTROLESS_BASE_NONROOT_IMAGE_NAME" "$DISTROLESS_BASE_IMAGE_NAME" "Dockerfile-Distroless-Nonroot-Template"
-    docker_build_custom $DISTROLESS "$DISTROLESS_MINIMAL_NONROOT_IMAGE_NAME" "$DISTROLESS_MINIMAL_IMAGE_NAME" "Dockerfile-Distroless-Nonroot-Template"
-    docker_build_custom $DISTROLESS "$DISTROLESS_DEBUG_NONROOT_IMAGE_NAME" "$DISTROLESS_DEBUG_IMAGE_NAME" "Dockerfile-Distroless-Nonroot-Template"
-
     docker_build_marinara
 }
 
 print_inputs
 validate_inputs
 initialization
+build_builder_image
 build_images