Merge branch 'tickets/DM-33748' into main

.github/workflows/ci.yml

@@ -7,10 +7,15 @@ on:
     branches:
       - main
 jobs:
-  build:
-    name: Build image
+
+  update-base-images:
+    name: Update base images
     runs-on: ubuntu-20.04
+    outputs:
+      build-rebuilt: ${{ steps.rebuild.outputs.build-rebuilt }}
+      run-base-rebuilt: ${{ steps.rebuild.outputs.run-base-rebuilt }}
     steps:
+
       - name: Install python
         uses: actions/setup-python@v2
         with:
@@ -32,45 +37,107 @@ jobs:
           username: ${{ secrets.FM_DOCKERHUB_TOKEN_USER }}
           password: ${{ secrets.FM_DOCKERHUB_TOKEN }}
 
-      - name: Prepare lite-build image
+      - name: Rebuild and push build image if needed
+        id: rebuild
         run: |
-          ./admin/local/cli/qserv --log-level DEBUG build-build-image \
-          --pull-image \
-          --push-image
+          if [[ $(./admin/local/cli/qserv dh-image-exists build-base) == "True" ]]; then
+            echo "Build image already on docker hub; skipping..."
+            echo "::set-output name=build-rebuilt::False"
+          else
+            ./admin/local/cli/qserv --log-level DEBUG build-build-image --push-image
+            echo "::set-output name=build-rebuilt::True"
+          fi
         env:
           QSERV_DH_USER: ${{ secrets.FM_DOCKERHUB_TOKEN_USER }}
           QSERV_DH_TOKEN: ${{ secrets.FM_DOCKERHUB_TOKEN }}
 
-      - name: Prepare lite-run-base image
+      - name: Rebuild and push run base image if needed
         run: |
-          ./admin/local/cli/qserv --log-level DEBUG build-run-base-image \
-          --pull-image \
-          --push-image
+          if [[ $(./admin/local/cli/qserv dh-image-exists run-base) == "True" ]]; then
+            echo "Run base image already on docker hub; skipping..."
+            echo "::set-output name=run-base-rebuilt::False"
+          else
+            ./admin/local/cli/qserv --log-level DEBUG build-run-base-image --push-image
+            echo "::set-output name=run-base-rebuilt::True"
+          fi
         env:
           QSERV_DH_USER: ${{ secrets.FM_DOCKERHUB_TOKEN_USER }}
           QSERV_DH_TOKEN: ${{ secrets.FM_DOCKERHUB_TOKEN }}
 
-      - name: Prepare user build image
+  update-mariadb-image:
+    name: Update MariaDB image
+    runs-on: ubuntu-20.04
+    outputs:
+      rebuilt: ${{ steps.rebuild.outputs.rebuilt }}
+    steps:
+
+      - name: Install python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+
+      - name: Install click
         run: |
-          ./admin/local/cli/qserv --log-level DEBUG build-user-build-image \
-          --group docker_outer
+          python -m pip install --upgrade pip
+          pip install click pyyaml requests
 
-      - name: Prepare maraidb image
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0  # 0 is "all history and branch tags"
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.FM_DOCKERHUB_TOKEN_USER }}
+          password: ${{ secrets.FM_DOCKERHUB_TOKEN }}
+
+      - name: Rebuild and push if needed
+        id: rebuild
         run: |
-          ./admin/local/cli/qserv --log-level DEBUG build-mariadb-image \
-          --pull-image \
-          --push-image
+          if [[ $(./admin/local/cli/qserv dh-image-exists mariadb) == "True" ]]; then
+            echo "MariaDB image already on docker hub; skipping..."
+            echo "::set-output name=rebuilt::False"
+          else
+            ./admin/local/cli/qserv --log-level DEBUG build-mariadb-image --push-image
+            echo "::set-output name=rebuilt::True"
+          fi
         env:
           QSERV_DH_USER: ${{ secrets.FM_DOCKERHUB_TOKEN_USER }}
           QSERV_DH_TOKEN: ${{ secrets.FM_DOCKERHUB_TOKEN }}
 
-      # Pull-image on this stage should only ever have an effect when rerunnning a job from
-      # the github actions web page. This is useful for re-running integration tests on
-      # a successful build to sniff out intermittent integration test issues.
-      # To rerun the whole build, push a new sha to the branch.
-      #
-      # Specify -j2 because the github actions virutal machine has 2 cores, so restrict the
-      # number of jobs to the same.
+  update-run-image:
+    name: Update Qserv image
+    runs-on: ubuntu-20.04
+    needs: update-base-images
+    steps:
+
+      - name: Install python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+
+      - name: Install click
+        run: |
+          python -m pip install --upgrade pip
+          pip install click pyyaml requests
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0  # 0 is "all history and branch tags"
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.FM_DOCKERHUB_TOKEN_USER }}
+          password: ${{ secrets.FM_DOCKERHUB_TOKEN }}
+
+      - name: Prepare user build image
+        run: |
+          ./admin/local/cli/qserv --log-level DEBUG build-user-build-image \
+          --group docker_outer
+
       - name: Build lite-qserv image
         run: |
           ./admin/local/cli/qserv --log-level DEBUG build \
@@ -81,6 +148,33 @@ jobs:
           QSERV_DH_USER: ${{ secrets.FM_DOCKERHUB_TOKEN_USER }}
           QSERV_DH_TOKEN: ${{ secrets.FM_DOCKERHUB_TOKEN }}
 
+  compose-integration-tests:
+    name: Integration tests (compose)
+    runs-on: ubuntu-20.04
+    needs: [update-mariadb-image, update-run-image]
+    steps:
+
+      - name: Install python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+
+      - name: Install click
+        run: |
+          python -m pip install --upgrade pip
+          pip install click pyyaml requests
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0  # 0 is "all history and branch tags"
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.FM_DOCKERHUB_TOKEN_USER }}
+          password: ${{ secrets.FM_DOCKERHUB_TOKEN }}
+
       - name: Launch qserv
         run: |
           ./admin/local/cli/qserv --log-level DEBUG up
@@ -99,9 +193,14 @@ jobs:
           ./admin/local/cli/qserv --log-level DEBUG down \
           -v
 
-      - name: Notify Slack if CI failed on main
+  notify-on-fail:
+    name: Notify Slack if fail on main
+    runs-on: ubuntu-20.04
+    needs: [compose-integration-tests]
+    if: github.ref == 'refs/heads/main' && failure()
+    steps:
+      - name: Notify
         uses: voxmedia/github-action-slack-notify-build@v1
-        if: github.ref == 'refs/heads/main' && failure()
         with:
           channel_id: G2JPZ3GC8  # this is the channel id of the dm_db_team room
           status: FAILED

src/admin/python/lsst/qserv/admin/qservCli/opt.py

@@ -38,6 +38,10 @@
 _log = logging.getLogger(__name__)
 
 
+# The location of the root directory ("qserv root") relative to this file:
+relative_qserv_root = "../../../../../../../../"
+
+
 class EV:
     """Base class for classes that use environment variables to assign values
     and defaults to options.
@@ -93,7 +97,17 @@ def used_for(self) -> str:
         """Get the description of what the EnvVal is used for, for help output."""
         return self.description
 
-    def val(self, default: Optional[str] = None) -> Optional[str]:
+    def val(self) -> Optional[str]:
+        """Get the value.
+
+        Returns
+        -------
+        value : `str` or `None`
+            The value of the environment variable or None
+        """
+        return super().var_val
+
+    def val_with_default(self, default: str) -> str:
         """Get the value.
 
         Parameters
@@ -160,35 +174,109 @@ def help(self, preamble: str) -> str:
         return msg
 
 
-base_dockerfile = "admin/tools/docker/base/Dockerfile"
-user_dockerfile = "admin/tools/docker/build-user/Dockerfile"
-run_base_dockerfile = "admin/tools/docker/base/Dockerfile"
-mariadb_dockerfile = "admin/tools/docker/mariadb/Dockerfile"
-# The location of the root directory ("qserv root") relative to this file:
-relative_qserv_root = "../../../../../../../../"
+class ImageName:
 
+    """Generate an image name and/or tag based on image type, considering:
+    * image type (see `image_types`)
+    * the last time an image type's docker file has changed
+    * if the QSERV_IMAGE_TAG environment variable is set
+    """
 
-def tagged_image_name(image_name: str, dockerfiles: Optional[List[str]]) -> Optional[str]:
-    """Get the image name to use, tagged with a tag indicating the changelist
-    when the dockerfiles were most recently changed.
+    # paths to dockerfiles relative to qserv root:
+    base_dockerfile = "admin/tools/docker/base/Dockerfile"
+    user_dockerfile = "admin/tools/docker/build-user/Dockerfile"
+    run_base_dockerfile = "admin/tools/docker/base/Dockerfile"
+    mariadb_dockerfile = "admin/tools/docker/mariadb/Dockerfile"
 
-    Parameters
-    ----------
-    image_name : `str`
-        The image base name (without tag).
-    dockerfiles : `List` [ `str` ] or `None`
-        If provided, the list of dockerfiles to consider when generating the
-        tag. If `None`, get the current git tag.
-
-    Returns
-    -------
-    image_name : `str`
-        The image name + tag
-    """
-    qserv_root = qserv_root_ev.val()
-    if qserv_root is None:
-        return None
-    return f"{image_name}:{image_tag_ev.val(get_description(dockerfiles, qserv_root))}"
+    image_types = ["qserv", "run-base", "mariadb", "build-base", "build-user"]
+
+    def __init__(self, image: str):
+        if image not in self.image_types:
+            raise RuntimeError(f"Unexpected image type: {image}")
+        self.image = image
+
+    @property
+    def tagged_name(self) -> str:
+        """Get the image name+tag.
+
+        Returns
+        -------
+        name_and_tag : `str`
+            The image name + tag
+        """
+        return self.name_with_tag(self.tag)
+
+    def name_with_tag(self, tag: str) -> str:
+        """Get the image name+tag using the provided tag.
+
+        Parameters
+        ----------
+        tag : str
+            The tag to add to the image name.
+
+        Returns
+        -------
+        name_and_tag : `str`
+            The image name + tag
+        """
+        return f"{self.name}:{tag}"
+
+    @property
+    def name(self) -> str:
+        """Get the image name.
+
+        Returns
+        -------
+        name : `str`
+            The image name
+        """
+        if self.image == "qserv":
+            return "qserv/lite-qserv"
+        if self.image == "run-base":
+            return "qserv/lite-run-base"
+        if self.image == "mariadb":
+            return "qserv/lite-mariadb"
+        if self.image == "build-base":
+            return "qserv/lite-build"
+        if self.image == "build-user":
+            return f"qserv/lite-build-{getpass.getuser()}"
+        raise RuntimeError(f"Invalid image type: {self.image}")
+
+    @property
+    def tag(self) -> str:
+        """Get the image tag.
+
+        Returns
+        -------
+        tag : `str`
+            The image tag
+        """
+        qserv_root = qserv_root_ev.val()
+        if qserv_root is None:
+            raise RuntimeError("qserv root was unexpectedly None.")
+        return image_tag_ev.val_with_default(get_description(self.dockerfiles, qserv_root))
+
+    @property
+    def dockerfiles(self) -> Optional[List[str]]:
+        """Get the path(s) (relative to qserv root) of the dockerfile(s)
+        associated with the current image type.
+
+        Returns
+        -------
+        dockerfiles : list [str] or None
+            The relative paths to the dockerfiles.
+        """
+        if self.image == "qserv":
+            return None
+        if self.image == "run-base":
+            return [self.run_base_dockerfile]
+        if self.image == "mariadb":
+            return [self.mariadb_dockerfile]
+        if self.image == "build-base":
+            return [self.base_dockerfile]
+        if self.image == "build-user":
+            return [self.base_dockerfile, self.user_dockerfile]
+        raise RuntimeError(f"Invalid image type: {self.image}")
 
 
 qserv_root_ev = FlagEnvVal(
@@ -200,27 +288,27 @@ def tagged_image_name(image_name: str, dockerfiles: Optional[List[str]]) -> Opti
 qserv_image_ev = FlagEnvVal(
     "--qserv-image",
     "QSERV_IMAGE",
-    tagged_image_name("qserv/lite-qserv", None),
+    ImageName("qserv").tagged_name,
 )
 run_base_image_ev = FlagEnvVal(
     "--run-base-image",
     "QSERV_RUN_BASE_IMAGE",
-    tagged_image_name("qserv/lite-run-base", [run_base_dockerfile]),
+    ImageName("run-base").tagged_name,
 )
 mariadb_image_ev = FlagEnvVal(
     "--mariadb-image",
     "QSERV_MARIADB_IMAGE",
-    tagged_image_name("qserv/lite-mariadb", [mariadb_dockerfile]),
+    ImageName("mariadb").tagged_name,
 )
 build_image_ev = FlagEnvVal(
     "--build-image",
     "QSERV_BUILD_IMAGE",
-    tagged_image_name("qserv/lite-build", [base_dockerfile]),
+    ImageName("build-base").tagged_name,
 )
 user_build_image_ev = FlagEnvVal(
     "--user-build-image",
     "QSERV_USER_BUILD_IMAGE",
-    tagged_image_name(f"qserv/lite-build-{getpass.getuser()}", [base_dockerfile, user_dockerfile]),
+    ImageName("build-user").tagged_name,
 )
 # qserv root default is derived by the relative path to the qserv folder from
 # the locaiton of this file.