Merge branch 'master' into chore/update-vlmd-validation

.github/workflows/build_and_push_nf_base_images.yml

@@ -0,0 +1,85 @@
+name: Push new nextflow images to ECR
+on:
+  workflow_dispatch:
+  push:
+    branches: master
+    paths:
+      - 'nextflow-base-images/**' #Runs every time this folder gets updated
+      - '.github/workflows/build_and_push_nf_base_images.yml'
+  schedule:
+    - cron: '0 0 * * 6'  # Run at midnight UTC every Saturday
+
+jobs:
+  build_and_push_nf_images:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          path: containers
+
+
+      - name: Checkout other private repository
+        uses: actions/checkout@v3
+        with:
+          repository: uc-cdis/base-images
+          token: ${{ secrets.PLANXCYBORG_TOKEN }}
+          path: base-images
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Set up AWS CLI
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_ACCT_654654631253_ECR }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_ACCT_654654631253_ECR }}
+          aws-region: us-east-1
+
+      - name: Login to Amazon ECR
+        run: |
+          aws ecr-public get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin public.ecr.aws/u5x5h6w3
+
+      - name: Build and push Docker images
+        run: |
+              dir=base-images/amazonlinux-base/
+              echo "Building an image present in $dir"
+              image_name=nextflow-approved/public
+              tag_name=$(basename "$dir")
+              docker build -t public.ecr.aws/u5x5h6w3/$image_name:$tag_name $dir
+              docker push public.ecr.aws/u5x5h6w3/$image_name:$tag_name
+              echo "Built an image with name --> $image_name:$tag_name"
+
+              for dir in containers/nextflow-base-images/*/;do
+                  echo "Building an image present in $dir"
+                  image_name=nextflow-approved/public
+                  tag_name=$(basename "$dir")
+                  docker build -t public.ecr.aws/u5x5h6w3/$image_name:$tag_name $dir
+                  docker push public.ecr.aws/u5x5h6w3/$image_name:$tag_name
+                  echo "Built an image with name --> $image_name:$tag_name"
+              done
+
+      - name: Trigger `get_layer_info_for_nf_imgs.yml` workflow
+        env:
+          GH_TOKEN: ${{ secrets.PLANXCYBORG_TOKEN }}
+        run: |
+              gh api \
+              --method POST \
+              -H "Accept: application/vnd.github+json" \
+              -H "X-GitHub-Api-Version: 2022-11-28" \
+              /repos/uc-cdis/containers/actions/workflows/get_layer_info_for_nf_imgs.yml/dispatches \
+              -f "ref=master"
+
+      - name: Delete untagged docker images from public ECR
+        run: |
+            REPO_NAME=nextflow-approved/public
+            IMAGE_IDS=$(aws ecr-public describe-images --repository-name $REPO_NAME --query 'imageDetails[?imageTags==null].imageDigest' --output text)
+            if [ -n "$IMAGE_IDS" ]; then
+              for IMAGE_ID in $IMAGE_IDS; do
+                echo "Deleting image with SHA hash - $IMAGE_ID"
+                aws ecr-public batch-delete-image --repository-name $REPO_NAME --image-ids imageDigest=$IMAGE_ID
+              done
+            else
+              echo "No untagged images to delete."
+            fi

.github/workflows/get_layer_info_for_nf_imgs.yml

@@ -1,14 +1,12 @@
 name: Update Nextflow-approved image's layers
 on:
+  # Primarily this workflow is only expected to be triggered by `build_and_push_nf_base_images.yml`, these extra triggers are just added for convenience.
   workflow_dispatch:
   push:
     branches: master
     paths:
-      - 'nextflow-base-images/**' #Runs every time this folder gets updated
+      - 'nextflow-base-images/**'
       - '.github/workflows/get_layer_info_for_nf_imgs.yml'
-  schedule:
-    - cron: '0 0 * * 0'  # Run at midnight UTC every Sunday
-
 jobs:
   get_image_layers:
     runs-on: ubuntu-latest
@@ -24,6 +22,16 @@ jobs:
           TOKEN=$(curl -s https://public.ecr.aws/token/ | jq -r .token)
           layer_json="{}"
           while IFS= read -r image_url; do
+            # Ignore lines that start with #
+              if [[ "$image_url" == \#* ]]; then
+                echo "Skipping this line -- $image_url"
+                continue
+              fi
+            # Strip the first * if a line starts with *
+              if [[ "$image_url" == \** ]]; then
+                echo "Stripping * from this line -- $image_url"
+                image_url="${image_url:1}"
+              fi
               manifest_url=$(echo ${image_url} | sed 's|public\.ecr\.aws/\(.*\):\(.*\)|https://public.ecr.aws/v2/\1/manifests/\2|')
               tag_name=$(echo ${image_url} | sed 's|\(.*\):\(.*\)|\2|')
               echo $manifest_url, $tag_name

.secrets.baseline

@@ -58,6 +58,14 @@
     }
   ],
   "results": {
+    ".github/workflows/build_and_push_nf_base_images.yml": [
+      {
+        "hashed_secret": "8ca221cf185226c33b30e6b150183acb037eff56",
+        "is_verified": false,
+        "line_number": 76,
+        "type": "Base64 High Entropy String"
+      }
+    ],
     "BRH-notebooks/combined_demos/BDCat_Biolincc_Framingham_study_exploration.ipynb": [
       {
         "hashed_secret": "4604f7226a4eca86fab1f34f8204fa9223b2c27e",

nextflow-base-images/allowed_base_images.txt

@@ -1,3 +1,4 @@
+# Note that base images with an asterisk `*` are temporarily unavailable for use and are being updated.
 public.ecr.aws/u5x5h6w3/nextflow-approved/public:amazonlinux-base
 public.ecr.aws/u5x5h6w3/nextflow-approved/public:gen3-cuda-11.8-ubuntu22.04-openssl
 public.ecr.aws/u5x5h6w3/nextflow-approved/public:gen3-cuda-12.3-ubuntu22.04-openssl

nextflow-base-images/gen3-cuda-12.3-torch2.2-ubuntu22.04-openssl/Dockerfile

@@ -0,0 +1,25 @@
+FROM public.ecr.aws/u5x5h6w3/nextflow-approved/public:gen3-cuda-12.3-ubuntu22.04-openssl
+
+# Set environment variables
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install system dependencies
+RUN apt-get update && \
+    apt-get install -y \
+        git \
+        python3-pip \
+        python3-dev \
+        python3-opencv \
+        libglib2.0-0
+
+# Upgrade pip
+RUN python3 -m pip install --upgrade pip
+
+# Install PyTorch and torchvision
+RUN pip3 install torch -f https://download.pytorch.org/whl/cu123/torch_stable.html
+
+# Set the working directory
+WORKDIR /app
+
+# Set the entrypoint
+ENTRYPOINT [ "python3" ]

nextflow-base-images/gen3-cuda-12.3-ubuntu22.04-openssl/Dockerfile

@@ -0,0 +1,36 @@
+# Use the specified base image
+FROM nvcr.io/nvidia/cuda:12.3.1-base-ubuntu22.04
+
+RUN apt-get purge -y --auto-remove openssl && apt-get autoremove && apt-get autoclean
+
+RUN apt-get update && apt-get -y upgrade && apt install -y wget
+
+RUN apt install -y build-essential && \
+    apt-get install -y python3 && \
+    apt-get install -y python3-pip
+
+# install openssl 3.0.8 as it is required for FIPS compliance.
+WORKDIR /tmp
+RUN wget https://www.openssl.org/source/openssl-3.0.8.tar.gz && \
+    tar -xzvf openssl-3.0.8.tar.gz && \
+    rm openssl-3.0.8.tar.gz
+
+WORKDIR /tmp/openssl-3.0.8
+RUN ./Configure enable-fips && \
+    make && \
+    make install
+
+# Changing adding `/usr/local/lib` as a prefix to LD_LIBRARY_PATH will
+# give precedence to OpenSSL 3.0.8 library files over the 3.0.2
+ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH
+
+# Make config changes ti ensure FIPS compliance
+RUN sed -i 's$# .include fipsmodule.cnf$.include /usr/local/ssl/fipsmodule.cnf$g' /usr/local/ssl/openssl.cnf
+RUN sed -i 's$providers = provider_sect$providers = provider_sect\nalg_section = algorithm_sect$g' /usr/local/ssl/openssl.cnf
+RUN sed -i 's$# fips = fips_sect$fips = fips_sect$g' /usr/local/ssl/openssl.cnf
+RUN sed -i -e 's$# activate = 1$activate = 1 \n\n[algorithm_sect]\ndefault_properties = fips=yes$g' /usr/local/ssl/openssl.cnf
+
+
+# Clean up the temporary directory
+WORKDIR /
+RUN rm -rf /tmp/openssl-3.0.8