Cleanup base sources

Signed-off-by: Christian Berendt <[email protected]>

.github/workflows/build-container-images.yml

@@ -26,6 +26,13 @@ jobs:
           submodules: true
       - uses: docker/setup-qemu-action@v2
       - uses: docker/setup-buildx-action@v2
+      # NOTE: https://rolandsdev.blog/posts/how-to-enable-experimental-docker-in-github-workflows/
+      - name: Enable experimental features for the Docker daemon and CLI
+        run: |
+          echo $'{\n  "experimental": true\n}' | sudo tee /etc/docker/daemon.json
+          mkdir -p ~/.docker
+          echo $'{\n  "experimental": "enabled"\n}' | sudo tee ~/.docker/config.json
+          sudo service docker restart
       - uses: docker/login-action@v2
         with:
           registry: ${{ secrets.DOCKER_REGISTRY }}

files/cleanup-base-sources.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+for e in $(find /*-base-source -name '*config-generator*'); do
+    if [[ -d $e ]]; then
+        mkdir -p /x/$e
+        cp -r $e/* /x/$e
+    else
+        f=$(dirname $e)
+        mkdir -p /x/$f
+        cp -r $f/* /x/$f
+    fi
+done
+
+rm -rf /*-base-source
+mv /x/* /
+rm -rf /x

scripts/004-build.sh

@@ -42,6 +42,9 @@ done
 
 # Build & squash base images
 
+# The line can be commented out for tests to build keystone images only.
+KOLLA_IMAGES_BASE=^keystone-base
+
 kolla-build \
   --template-override templates/$OPENSTACK_VERSION/template-overrides.j2 \
   --config-file $KOLLA_CONF \
@@ -52,6 +55,9 @@ kolla-build \
 
 # Build images
 
+# The line can be commented out for tests to build keystone images only.
+KOLLA_IMAGES=^keystone
+
 kolla-build \
   --template-override templates/$OPENSTACK_VERSION/template-overrides.j2 \
   --config-file $KOLLA_CONF \
@@ -65,4 +71,10 @@ if grep -q "Failed with status: error" kolla-build-$BUILD_ID.log; then
     exit 1
 fi
 
+# Cleanup images
+
+python3 src/cleanup-images.py
+
+# List images
+
 docker images

src/cleanup-images.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+
+import os
+import shutil
+import subprocess
+import tempfile
+
+from docker import APIClient, DockerClient
+from loguru import logger
+
+VERSION = os.environ.get("OPENSTACK_VERSION", "zed")
+FILTERS = {"label": f"de.osism.release.openstack={VERSION}"}
+
+SUPPORTED_IMAGES = ["keystone"]
+
+client = DockerClient()
+client_api = APIClient()
+
+for image in client.images.list(filters=FILTERS):
+    # skip images without a tag
+    if not image.tags:
+        continue
+
+    tag = image.tags[0]
+
+    # Check if the image is supported
+    if not [x for x in SUPPORTED_IMAGES if x in image.labels["name"]]:
+        logger.info(f"Image {tag} not supported")
+        continue
+
+    logger.info(f"Cleaning up image {tag}")
+
+    # Image layers
+
+    # 0: "sha256:59c56aee1fb4dbaeb334aef06088b49902105d1ea0c15a9e5a2a9ce560fa4c5d" <-- ubuntu:22.04
+    # 1: "sha256:295ebbacd98b98d173f5fba0371e4fee8bbdf9ff0ad2676670ddb2b521482004" <-- base
+    # 2: "sha256:bd9f5e1f11d685268d42d7bf3a7850fdd7b6767bfcfbdb1bcc0b31227dae8f91" <-- openstack-base
+    # 3: "sha256:6ae01f1fd7a8c29837ddc7ed50e78ba70666f5967bdaf8438f07da2971d3d024" <-- service-base
+    # 4: "sha256:591d1c8305854fabc586786673d87f55eb142f0efed7a002f696c6a1c4c26880" <-- service
+
+    # Image history
+
+    # NOTE: the missing image IDs available at build time
+
+    # 0: ec1a31f60d76   17 hours ago                                                   1.85MB <-- service
+    # 1: <missing>      17 hours ago                                                   52.1MB <-- service-base
+    # 2: <missing>      17 hours ago                                                   633MB  <-- openstack-base
+    # 3: <missing>      17 hours ago                                                   169MB  <-- base
+    # 4: <missing>      9 days ago     /bin/sh -c #(nop)  CMD ["/bin/bash"]            0B     <-- start of ubuntu 22.04
+    # 5: <missing>      9 days ago     /bin/sh -c #(nop) ADD file:140fb5108b4a2861b…   77.8MB
+    # 6: <missing>      9 days ago     /bin/sh -c #(nop)  LABEL org.opencontainers.…   0B
+    # 7: <missing>      9 days ago     /bin/sh -c #(nop)  LABEL org.opencontainers.…   0B
+    # 8: <missing>      9 days ago     /bin/sh -c #(nop)  ARG LAUNCHPAD_BUILD_ARCH     0B
+    # 9: <missing>      9 days ago     /bin/sh -c #(nop)  ARG RELEASE                  0B
+
+    openstack_base_layer = 3
+
+    with tempfile.TemporaryDirectory() as d:
+        shutil.copy("files/cleanup-base-sources.sh", d)
+
+        dockerfile = os.path.join(d, "Dockerfile")
+        logger.info(f"Generating Dockerfile in {d}")
+
+        # Get current username
+        result = client.containers.run(tag, "bash -c 'whoami'")
+        username = result.decode("utf-8")
+
+        with open(dockerfile, "w+") as fp:
+            fp.write(f"FROM {tag}\n")
+            fp.write("COPY cleanup-base-sources.sh /cleanup-base-sources.sh\n")
+            fp.write("USER root\n")
+            fp.write("RUN bash /cleanup-base-sources.sh\n")
+            fp.write("RUN rm /cleanup-base-sources.sh\n")
+            fp.write(f"USER {username}\n")
+
+        client.images.build(path=d, tag=f"{tag}-after-cleanup", squash=True)
+
+        # Squash the image
+        logger.info(f"Squashing with openstack-base layer {openstack_base_layer}")
+        cmds = [
+            "docker-squash",
+            f"--tag {tag}-after-squash",
+            f"--from-layer {openstack_base_layer + 5}",
+            f"{tag}-after-cleanup",
+        ]
+        subprocess.check_output(cmds, stderr=subprocess.STDOUT)
+
+        # Re-tag the image
+        client.images.remove(tag)
+        client.images.tag(f"{tag}-after-squash", tag)