Add image sync workflow

.github/workflows/sync-images.yaml

@@ -0,0 +1,60 @@
+name: sync images
+on: push
+
+jobs:
+  sync_images:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        component:
+          - daskhub
+          - jupyterhub
+          - kubeflow
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v3
+        # TODO: Revert to main branch before merging
+        with:
+          ref: skopeo
+
+      # Uncommment for SSH-able terminal session within runner to aid debugging
+      # - uses: actions/checkout@v2
+      # - name: Setup upterm session
+      #   uses: lhotari/action-upterm@v1
+
+      - name: Check component manifest for changes
+        uses: dorny/paths-filter@v2
+        id: changes
+        with:
+          filters: |
+            manifest:
+              - skopeo-manifests/${{ matrix.component }}.yml
+
+      # NOTE: Need skopeo > v1.09 to avoid this issue:
+      # https://github.com/containers/skopeo/issues/1874
+      # so use containerized version
+
+      - name: Sync component images
+        id: image-sync
+        if: ${{ steps.changes.outputs.manifest == 'true' }}
+        run: |-
+          docker run --entrypoint /usr/bin/env \
+            -v $GITHUB_WORKSPACE:/home/skopeo/azimuth-charts \
+            quay.io/skopeo/stable:v1.11 \
+            skopeo sync \
+              --src yaml \
+              --dest docker \
+              --dest-creds ${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} \
+              --scoped \
+              --all \
+              /home/skopeo/azimuth-charts/skopeo-manifests/${{ matrix.component }}.yml \
+              ghcr.io/stackhpc/azimuth-charts \
+              |& tee sync-output.txt
+
+      - name: Check output for any sync errors
+        run: |
+          ERR_COUNT=$(grep "level=error" sync-output.txt | wc -l)
+          if [[ $ERR_COUNT -gt 0 ]]; then
+            echo "Found $ERR_COUNT logged error messages in output of image sync step"
+            exit 1
+          fi

.gitignore

@@ -1,6 +1,7 @@
 */charts/*
 */Chart.lock
+.vscode
 # Ignore `helm dependency build` output
-kubeflow-azimuth/kubeflow-azimuth-chart/**
-kubeflow-azimuth/kubeflow-crds/**
+kubeflow-azimuth/kubeflow-azimuth-chart
+kubeflow-azimuth/kubeflow-crds
 kubeflow-azimuth/kustomize-build-output.yml

daskhub-azimuth/values.yaml

@@ -19,6 +19,10 @@ daskhub:
               String("image", default="pangeo/base-notebook:2022.05.10", label="Image"),
               handler=option_handler,
           )
+    # Use stackhpc's ghcr image copy
+    traefik:
+      image: 
+        name: ghcr.io/stackhpc/azimuth-charts/docker.io/library/traefik
 
   jupyterhub:
     prePuller:
@@ -32,10 +36,16 @@ daskhub:
       service:
         type: ClusterIP
       chp:
+        # Use stackhpc's ghcr image copy
+        image:
+          name: ghcr.io/stackhpc/azimuth-charts/docker.io/jupyterhub/configurable-http-proxy
         networkPolicy:
           enabled: false
 
     hub:
+      # Use stackhpc's ghcr image copy
+      image:
+        name: ghcr.io/stackhpc/azimuth-charts/docker.io/jupyterhub/k8s-hub
       networkPolicy:
         enabled: false
       # Configure the authentication to respect the X-Remote-User header sent by Zenith from Azimuth
@@ -81,6 +91,9 @@ daskhub:
           c.JupyterHub.authenticator_class = RemoteUserAuthenticator
 
     singleuser:
+      # Use stackhpc's ghcr image copy
+      image:
+        name: ghcr.io/stackhpc/azimuth-charts/docker.io/pangeo/base-notebook
       networkPolicy:
         enabled: false
       defaultUrl: /lab

jupyterhub-azimuth/values.yaml

@@ -15,10 +15,16 @@ jupyterhub:
     service:
       type: ClusterIP
     chp:
+      # Use stackhpc's ghcr image copy
+      image:
+        name: ghcr.io/stackhpc/azimuth-charts/docker.io/jupyterhub/configurable-http-proxy
       networkPolicy:
         enabled: false
 
   hub:
+    # Use stackhpc's ghcr image copy
+    image:
+      name: ghcr.io/stackhpc/azimuth-charts/docker.io/jupyterhub/k8s-hub
     networkPolicy:
       enabled: false
     extraConfig:
@@ -63,6 +69,9 @@ jupyterhub:
         c.JupyterHub.authenticator_class = RemoteUserAuthenticator
 
   singleuser:
+    # Use stackhpc's ghcr image copy
+    image:
+      name: ghcr.io/stackhpc/azimuth-charts/docker.io/jupyterhub/k8s-singleuser-sample
     networkPolicy:
       enabled: false
     defaultUrl: /lab

kubeflow-azimuth/build-chart.sh

@@ -5,6 +5,28 @@ if [[ ! $(kustomize version) == *v5.* ]]; then
     echo "Please install a valid version then try again."
     exit 1
 fi
-kustomize build overlay/ --output kustomize-build-output.yml
+
+OUTPUT_FILE=kustomize-build-output.yml
+kustomize build overlay/ --output $OUTPUT_FILE
+
+# NOTE(scott): kustomize image source patches don't capture 
+# default notebook images used by kubeflow jupyterhub platform 
+# since these are defined within the data.'spawner_ui_config.yaml'
+# field of the ConfigMap 'jupyter-web-app-config-xxxxxxx'
+# Use sed here to replace these images with ghcr versions
+IMAGES=(
+    "jupyter-scipy"
+    "jupyter-pytorch-full"
+    "jupyter-pytorch-cuda-full"
+    "jupyter-tensorflow-full"
+    "jupyter-tensorflow-cuda-full"
+)
+for image in ${IMAGES[@]}; do
+    # Backup suffix is required on MacOS
+    sed -i.bak "s|kubeflownotebookswg/${image}|ghcr.io/stackhpc/azimuth-charts/docker.io/kubeflownotebookswg/${image}|g" $OUTPUT_FILE
+    # suffix to -i option is mandatory on MacOS sed, remove backup file here
+    rm $OUTPUT_FILE.bak
+done
+
+# Convert kustomize output to helm chart directory structure
 python3 to-helm-chart.py
-# git add ../kubeflow-azimuth-chart

kubeflow-azimuth/overlay/kustomization.yaml

@@ -37,4 +37,129 @@ patches:
         value: autoscaling/v2
     target:
       kind: HorizontalPodAutoscaler
-      version: v2beta2
+      version: v2beta2
+
+# Use StackHPC's ghcr for relevant container images
+images:
+- name: docker.io/istio/pilot
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/istio/pilot
+- name: docker.io/istio/proxyv2
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/istio/proxyv2
+- name: docker.io/kubeflowkatib/katib-controller
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/kubeflowkatib/katib-controller
+- name: docker.io/kubeflowkatib/katib-db-manager
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/kubeflowkatib/katib-db-manager
+- name: docker.io/kubeflowkatib/katib-ui
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/kubeflowkatib/katib-ui
+- name: docker.io/kubeflownotebookswg/centraldashboard
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/kubeflownotebookswg/centraldashboard
+- name: docker.io/kubeflownotebookswg/jupyter-web-app
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/kubeflownotebookswg/jupyter-web-app
+- name: docker.io/kubeflownotebookswg/kfam
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/kubeflownotebookswg/kfam
+- name: docker.io/kubeflownotebookswg/notebook-controller
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/kubeflownotebookswg/notebook-controller
+- name: docker.io/kubeflownotebookswg/poddefaults-webhook
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/kubeflownotebookswg/poddefaults-webhook
+- name: docker.io/kubeflownotebookswg/profile-controller
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/kubeflownotebookswg/profile-controller
+- name: docker.io/kubeflownotebookswg/tensorboard-controller
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/kubeflownotebookswg/tensorboard-controller
+- name: docker.io/kubeflownotebookswg/tensorboards-web-app
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/kubeflownotebookswg/tensorboards-web-app
+- name: docker.io/kubeflownotebookswg/volumes-web-app
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/kubeflownotebookswg/volumes-web-app
+- name: docker.io/metacontrollerio/metacontroller
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/metacontrollerio/metacontroller
+- name: gcr.io/arrikto/kubeflow/oidc-authservice
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/arrikto/kubeflow/oidc-authservice
+- name: gcr.io/knative-releases/knative.dev/eventing/cmd/controller
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/knative-releases/knative.dev/eventing/cmd/controller
+- name: gcr.io/knative-releases/knative.dev/eventing/cmd/webhook
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/knative-releases/knative.dev/eventing/cmd/webhook
+- name: gcr.io/knative-releases/knative.dev/net-istio/cmd/controller
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/knative-releases/knative.dev/net-istio/cmd/controller
+- name: gcr.io/knative-releases/knative.dev/net-istio/cmd/webhook
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/knative-releases/knative.dev/net-istio/cmd/webhook
+- name: gcr.io/knative-releases/knative.dev/serving/cmd/activator
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/knative-releases/knative.dev/serving/cmd/activator
+- name: gcr.io/knative-releases/knative.dev/serving/cmd/autoscaler
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/knative-releases/knative.dev/serving/cmd/autoscaler
+- name: gcr.io/knative-releases/knative.dev/serving/cmd/controller
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/knative-releases/knative.dev/serving/cmd/controller
+- name: gcr.io/knative-releases/knative.dev/serving/cmd/domain-mapping-webhook
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/knative-releases/knative.dev/serving/cmd/domain-mapping-webhook
+- name: gcr.io/knative-releases/knative.dev/serving/cmd/domain-mapping
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/knative-releases/knative.dev/serving/cmd/domain-mapping
+- name: gcr.io/knative-releases/knative.dev/serving/cmd/webhook
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/knative-releases/knative.dev/serving/cmd/webhook
+- name: gcr.io/kubebuilder/kube-rbac-proxy
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/kubebuilder/kube-rbac-proxy
+- name: gcr.io/kubebuilder/kube-rbac-proxy
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/kubebuilder/kube-rbac-proxy
+- name: gcr.io/ml-pipeline/api-server
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/ml-pipeline/api-server
+- name: gcr.io/ml-pipeline/cache-server
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/ml-pipeline/cache-server
+- name: gcr.io/ml-pipeline/frontend
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/ml-pipeline/frontend
+- name: gcr.io/ml-pipeline/metadata-envoy
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/ml-pipeline/metadata-envoy
+- name: gcr.io/ml-pipeline/metadata-writer
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/ml-pipeline/metadata-writer
+- name: gcr.io/ml-pipeline/minio
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/ml-pipeline/minio
+- name: gcr.io/ml-pipeline/mysql
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/ml-pipeline/mysql
+- name: gcr.io/ml-pipeline/persistenceagent
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/ml-pipeline/persistenceagent
+- name: gcr.io/ml-pipeline/scheduledworkflow
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/ml-pipeline/scheduledworkflow
+- name: gcr.io/ml-pipeline/viewer-crd-controller
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/ml-pipeline/viewer-crd-controller
+- name: gcr.io/ml-pipeline/visualization-server
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/ml-pipeline/visualization-server
+- name: gcr.io/ml-pipeline/workflow-controller
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/ml-pipeline/workflow-controller
+- name: gcr.io/tfx-oss-public/ml_metadata_store_server
+  newName: ghcr.io/stackhpc/azimuth-charts/gcr.io/tfx-oss-public/ml_metadata_store_server
+- name: docker.io/kserve/kserve-controller
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/kserve/kserve-controller
+- name: docker.io/kserve/models-web-app
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/kserve/models-web-app
+- name: docker.io/kubeflow/training-operator
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/kubeflow/training-operator
+- name: docker.io/kubeflownotebookswg/jupyter-pytorch-cuda-full
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/kubeflownotebookswg/jupyter-pytorch-cuda-full
+- name: docker.io/kubeflownotebookswg/jupyter-pytorch-full
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/kubeflownotebookswg/jupyter-pytorch-full
+- name: docker.io/kubeflownotebookswg/jupyter-scipy
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/kubeflownotebookswg/jupyter-scipy
+- name: docker.io/kubeflownotebookswg/jupyter-tensorflow-cuda-full
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/kubeflownotebookswg/jupyter-tensorflow-cuda-full
+- name: docker.io/kubeflownotebookswg/jupyter-tensorflow-full
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/kubeflownotebookswg/jupyter-tensorflow-full
+- name: mysql
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/library/mysql
+- name: python
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/library/python
+- name: quay.io/jetstack/cert-manager-cainjector
+  newName: ghcr.io/stackhpc/azimuth-charts/quay.io/jetstack/cert-manager-cainjector
+- name: quay.io/jetstack/cert-manager-controller
+  newName: ghcr.io/stackhpc/azimuth-charts/quay.io/jetstack/cert-manager-controller
+- name: quay.io/jetstack/cert-manager-webhook
+  newName: ghcr.io/stackhpc/azimuth-charts/quay.io/jetstack/cert-manager-webhook
+- name: registry.k8s.io/coredns/coredns
+  newName: ghcr.io/stackhpc/azimuth-charts/registry.k8s.io/coredns/coredns
+- name: registry.k8s.io/etcd
+  newName: ghcr.io/stackhpc/azimuth-charts/registry.k8s.io/etcd
+- name: registry.k8s.io/kube-apiserver
+  newName: ghcr.io/stackhpc/azimuth-charts/registry.k8s.io/kube-apiserver
+- name: registry.k8s.io/kube-controller-manager
+  newName: ghcr.io/stackhpc/azimuth-charts/registry.k8s.io/kube-controller-manager
+- name: registry.k8s.io/kube-proxy
+  newName: ghcr.io/stackhpc/azimuth-charts/registry.k8s.io/kube-proxy
+- name: registry.k8s.io/kube-scheduler
+  newName: ghcr.io/stackhpc/azimuth-charts/registry.k8s.io/kube-scheduler
+- name: docker.io/tensorflow/tensorflow
+  newName: ghcr.io/stackhpc/azimuth-charts/docker.io/tensorflow/tensorflow

kubeflow-azimuth/to-helm-chart.py

@@ -1,24 +1,25 @@
 import yaml, re, shutil
 from pathlib import Path
 
+
 def make_helm_chart_template(chart_path, chart_yml, values_yml):
     """Creates a template directory structure for a helm chart"""
-    print('Creating Helm chart at', chart_path.absolute())
-    # Remove any existing content at chart path 
+    print("Creating Helm chart at", chart_path.absolute())
+    # Remove any existing content at chart path
     # TODO: Add user confirmation and/or --force cmd line arg for deletion?
     try:
         shutil.rmtree(chart_path)
     except FileNotFoundError:
         pass
     # Create Helm chart directory structure
     chart_path.mkdir()
-    (chart_path / 'templates').mkdir()
-    (chart_path / 'crds').mkdir()
+    (chart_path / "templates").mkdir()
+    (chart_path / "crds").mkdir()
     # Write Chart.yaml
-    with open(chart_path / 'Chart.yaml', 'w') as file:
+    with open(chart_path / "Chart.yaml", "w") as file:
         file.write(chart_yml)
     # Write values.yaml
-    with open(chart_path / 'values.yaml', 'w') as file:
+    with open(chart_path / "values.yaml", "w") as file:
         file.write(values_yml)
 
 
@@ -57,40 +58,40 @@ def make_helm_chart_template(chart_path, chart_yml, values_yml):
     "required": []
 }
 """
-with open(main_chart_path / 'values.schema.json', 'w') as schema_file:
+with open(main_chart_path / "values.schema.json", "w") as schema_file:
     schema_file.write(json_schema)
 
 # Write manifest files
-with open('kustomize-build-output.yml', 'r') as input_file:
+with open("kustomize-build-output.yml", "r") as input_file:
     # NOTE: Read input file as str instead of yaml to preserve newlines
-    # all_manifests = yaml.load_all(input_file) 
+    # all_manifests = yaml.load_all(input_file)
     all_manifests = input_file.read().split("\n---\n")
     for i, manifest_str in enumerate(all_manifests):
-
         # Convert to yaml for field queries
         manifest = yaml.safe_load(manifest_str)
-        
+
         # NOTE: CRDs and namespaces are placed in separate sub-chart since trying to
         # bundle all manifests into a single helm chart creates a helm release secret
         # > 1MB which etcd then refuses to store so installation fails
-        manifest_name = manifest['metadata']['name'].replace('.', '-') + f'-{i+1}.yml'
-        if manifest['kind'] == 'CustomResourceDefinition':
-            manifest_path = crd_chart_path / 'crds' / manifest_name
-        elif manifest['kind'] == 'Namespace':
-            manifest_path = crd_chart_path / 'templates' / manifest_name
+        manifest_name = manifest["metadata"]["name"].replace(".", "-") + f"-{i+1}.yml"
+        if manifest["kind"] == "CustomResourceDefinition":
+            manifest_path = crd_chart_path / "crds" / manifest_name
+        elif manifest["kind"] == "Namespace":
+            manifest_path = crd_chart_path / "templates" / manifest_name
         else:
-            manifest_path = main_chart_path / 'templates' / manifest_name
-        print(f'{i+1}.\t Writing {manifest_path}')
-        
+            manifest_path = main_chart_path / "templates" / manifest_name
+        print(f"{i+1}.\t Writing {manifest_path}")
+
         # NOTE: Some manifest files have '{{' and '}}' instances in comments
         # These need to be escaped so that helm doesn't try to template them
         # Regex should match everying within a curly bracket that isn't a curly bracket itself
-        manifest_str = re.sub(r"{{([^\{\}]*)}}", r'{{ "{{" }}\1{{ "}}" }}', manifest_str)
+        manifest_str = re.sub(
+            r"{{([^\{\}]*)}}", r'{{ "{{" }}\1{{ "}}" }}', manifest_str
+        )
 
         # Write manifest to file
         # NOTE: Avoid using yaml.dumps here as it doesn't properly preserve multi-line
-        # yaml blocks (e.g. key: | \n ...) and instead replaces all newlines with '\n' 
+        # yaml blocks (e.g. key: | \n ...) and instead replaces all newlines with '\n'
         # inside blocks, making final manifests less readable.
-        with open(manifest_path, 'w') as output_file:
+        with open(manifest_path, "w") as output_file:
             output_file.write(manifest_str)
-