Merge pull request #23 from HumairAK/ubi_images

Use UBI images and make samples optional
opendatahub-io · Mar 11, 2024 · 2ef1607 · 2ef1607
2 parents 54e1724 + 03cb11f
commit 2ef1607
Show file tree

Hide file tree

Showing 7 changed files with 166 additions and 137 deletions.
diff --git a/backend/Dockerfile b/backend/Dockerfile
@@ -1,4 +1,4 @@
-# Copyright 2021-2022 The Kubeflow Authors
+# Copyright 2021-2024 The Kubeflow Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,74 +12,50 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# 1. Build api server application
-FROM golang:1.20.4-buster as builder
-RUN apt-get update && apt-get install -y cmake clang musl-dev openssl
-WORKDIR /go/src/github.com/kubeflow/pipelines
-COPY . .
-RUN GO111MODULE=on go build -o /bin/apiserver backend/src/apiserver/*.go
-# Check licenses and comply with license terms.
-RUN ./hack/install-go-licenses.sh
-# First, make sure there's no forbidden license.
-RUN go-licenses check ./backend/src/apiserver
-RUN go-licenses csv ./backend/src/apiserver > /tmp/licenses.csv && \
-  diff /tmp/licenses.csv backend/third_party_licenses/apiserver.csv && \
-  go-licenses save ./backend/src/apiserver --save_path /tmp/NOTICES
-
-# 2. Compile preloaded pipeline samples
-FROM python:3.7 as compiler
-RUN apt-get update -y && apt-get install --no-install-recommends -y -q default-jdk python3-setuptools python3-dev jq
-RUN wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py
-COPY backend/requirements.txt .
-RUN python3 -m pip install -r requirements.txt --no-cache-dir
-
-# Downloading Argo CLI so that the samples are validated
-ENV ARGO_VERSION v3.3.10
-RUN curl -sLO https://github.com/argoproj/argo-workflows/releases/download/${ARGO_VERSION}/argo-linux-amd64.gz && \
-    gunzip argo-linux-amd64.gz && \
-    chmod +x argo-linux-amd64 && \
-    mv ./argo-linux-amd64 /usr/local/bin/argo
-
-WORKDIR /
-COPY ./samples /samples
-COPY backend/src/apiserver/config/sample_config.json /samples/
-
-# Compiling the preloaded samples.
-# The default image is replaced with the GCR-hosted python image.
-RUN set -e; \
-    < /samples/sample_config.json jq .[].file --raw-output | while read pipeline_yaml; do \
-    pipeline_py="${pipeline_yaml%.yaml}"; \
-    python3 "$pipeline_py"; \
-    done
-
-# 3. Start api web server
-FROM debian:stable
-
-ARG COMMIT_SHA=unknown
-ENV COMMIT_SHA=${COMMIT_SHA}
-ARG TAG_NAME=unknown
-ENV TAG_NAME=${TAG_NAME}
+# Build arguments
+ARG SOURCE_CODE=.
+
+#@follow_tag(registry.access.redhat.com/ubi8/go-toolset:1.20)
+FROM registry.access.redhat.com/ubi8/go-toolset:1.20 as builder
+
+## Build args to be used at this step
+ARG SOURCE_CODE
+
+USER root
+
+RUN dnf install -y cmake clang openssl
+
+COPY ${SOURCE_CODE}/go.mod ./
+COPY ${SOURCE_CODE}/go.sum ./
+
+# Copy the source
+COPY ${SOURCE_CODE}/ ./
+
+RUN GO111MODULE=on go build -o /bin/apiserver ./backend/src/apiserver/ && \
+    dnf clean all
+
+#@follow_tag(registry.access.redhat.com/ubi8/ubi-minimal:8.9)
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.9
 
 WORKDIR /bin
 
-COPY backend/src/apiserver/config/ /config
+COPY --from=builder /opt/app-root/src/backend/src/apiserver/config/ /config
 COPY --from=builder /bin/apiserver /bin/apiserver
 
-# Copy licenses and notices.
-COPY --from=builder /tmp/licenses.csv /third_party/licenses.csv
-COPY --from=builder /tmp/NOTICES /third_party/NOTICES
-COPY --from=compiler /samples/ /samples/
 RUN chmod +x /bin/apiserver
 
+USER root
+
 # Adding CA certificate so API server can download pipeline through URL and wget is used for liveness/readiness probe command
-RUN apt-get update && apt-get install -y ca-certificates wget
+RUN microdnf install -y ca-certificates wget
 
-# Pin sample doc links to the commit that built the backend image
-RUN sed -E "s#/(blob|tree)/master/#/\1/${COMMIT_SHA}/#g" -i /config/sample_config.json && \
-    sed -E "s/%252Fmaster/%252F${COMMIT_SHA}/#g" -i /config/sample_config.json
+USER 1001
 
 # Expose apiserver port
 EXPOSE 8888
 
 # Start the apiserver
-CMD /bin/apiserver --config=/config --sampleconfig=/config/sample_config.json -logtostderr=true
+CMD /bin/apiserver --config=/config -logtostderr=true
+
+LABEL name="ds-pipelines-api-server" \
+      summary="DSP Server"
diff --git a/backend/Dockerfile.driver b/backend/Dockerfile.driver
@@ -1,4 +1,4 @@
-# Copyright 2023 The Kubeflow Authors
+# Copyright 2021-2024 The Kubeflow Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,31 +12,37 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.20.9-alpine3.17 as builder
+# Build arguments
+ARG SOURCE_CODE=.
 
-WORKDIR /go/src/github.com/kubeflow/pipelines
-COPY . .
+# Use ubi8/nodejs-14 as base image
+#@follow_tag(registry.access.redhat.com/ubi8/go-toolset:1.20)
+FROM registry.access.redhat.com/ubi8/go-toolset:1.20 as builder
 
-RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -tags netgo -ldflags '-extldflags "-static"' -o /bin/driver ./backend/src/v2/cmd/driver/*.go
 
-# Check licenses and comply with license terms.
-RUN ./hack/install-go-licenses.sh
-# First, make sure there's no forbidden license.
-RUN go-licenses check ./backend/src/v2/cmd/driver
-RUN go-licenses csv ./backend/src/v2/cmd/driver > /tmp/licenses.csv && \
-    diff /tmp/licenses.csv backend/third_party_licenses/driver.csv && \
-    go-licenses save ./backend/src/v2/cmd/driver --save_path /tmp/NOTICES
+## Build args to be used at this step
+ARG SOURCE_CODE
+
+## Switch to root as required for some operations
+USER root
+
+COPY ${SOURCE_CODE}/go.mod ./
+COPY ${SOURCE_CODE}/go.sum ./
 
-FROM alpine:3.17
+# Copy the source
+COPY ${SOURCE_CODE}/ ./
 
-RUN adduser -S appuser
-USER appuser
+RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -tags netgo -ldflags '-extldflags "-static"' -o /bin/driver ./backend/src/v2/cmd/driver/*.go
+
+#@follow_tag(registry.access.redhat.com/ubi8/ubi-minimal:8.9)
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.9
 
 WORKDIR /bin
 
 COPY --from=builder /bin/driver /bin/driver
-# Copy licenses and notices.
-COPY --from=builder /tmp/licenses.csv /third_party/licenses.csv
-COPY --from=builder /tmp/NOTICES /third_party/NOTICES
+RUN chmod +x /bin/driver
+
+ENTRYPOINT ["/bin/driver"]
 
-ENTRYPOINT [ "/bin/driver" ]
+LABEL name="ds-pipelines-driver" \
+      summary="DSP Driver"
diff --git a/backend/Dockerfile.launcher b/backend/Dockerfile.launcher
@@ -1,4 +1,4 @@
-# Copyright 2023 The Kubeflow Authors
+# Copyright 2021-2024 The Kubeflow Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,31 +12,39 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.20.9-alpine3.17 as builder
+# Build arguments
+ARG SOURCE_CODE=.
+ARG CI_CONTAINER_VERSION="unknown"
 
-WORKDIR /go/src/github.com/kubeflow/pipelines
-COPY . .
 
-RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -tags netgo -ldflags '-extldflags "-static"' -o /bin/launcher-v2 ./backend/src/v2/cmd/launcher-v2/*.go
+# Use ubi8/nodejs-14 as base image
+#@follow_tag(registry.access.redhat.com/ubi8/go-toolset:1.20)
+FROM registry.access.redhat.com/ubi8/go-toolset:1.20 as builder
+
 
-# Check licenses and comply with license terms.
-RUN ./hack/install-go-licenses.sh
-# First, make sure there's no forbidden license.
-RUN go-licenses check ./backend/src/v2/cmd/launcher-v2
-RUN go-licenses csv ./backend/src/v2/cmd/launcher-v2 > /tmp/licenses.csv && \
-    diff /tmp/licenses.csv backend/third_party_licenses/launcher.csv && \
-    go-licenses save ./backend/src/v2/cmd/launcher-v2 --save_path /tmp/NOTICES
+## Build args to be used at this step
+ARG SOURCE_CODE
 
-FROM alpine:3.17
+## Switch to root as required for some operations
+USER root
 
-RUN adduser -S appuser
-USER appuser
+COPY ${SOURCE_CODE}/go.mod ./
+COPY ${SOURCE_CODE}/go.sum ./
+
+# Copy the source
+COPY ${SOURCE_CODE}/ ./
+
+RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -tags netgo -ldflags '-extldflags "-static"' -o /bin/launcher-v2 ./backend/src/v2/cmd/launcher-v2/*.go
+
+#@follow_tag(registry.access.redhat.com/ubi8/ubi-minimal:8.9)
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.9
 
 WORKDIR /bin
 
 COPY --from=builder /bin/launcher-v2 /bin/launcher-v2
-# Copy licenses and notices.
-COPY --from=builder /tmp/licenses.csv /third_party/licenses.csv
-COPY --from=builder /tmp/NOTICES /third_party/NOTICES
+RUN chmod +x /bin/launcher-v2
+
+ENTRYPOINT ["/bin/launcher-v2"]
 
-ENTRYPOINT [ "/bin/launcher-v2" ]
+LABEL name="ds-pipelines-launcher" \
+      summary="DSP launcher"
diff --git a/backend/Dockerfile.persistenceagent b/backend/Dockerfile.persistenceagent
@@ -12,35 +12,35 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.20.4-alpine3.17 as builder
+# Build arguments
+ARG SOURCE_CODE=.
+ARG CI_CONTAINER_VERSION="unknown"
 
-WORKDIR /go/src/github.com/kubeflow/pipelines
-COPY . .
 
-# Needed musl-dev for github.com/mattn/go-sqlite3
-RUN apk update && apk upgrade && \
-    apk add --no-cache bash git openssh gcc musl-dev
+# Use ubi8/go-toolset as base image
+#@follow_tag(registry.access.redhat.com/ubi8/go-toolset:1.20)
+FROM registry.access.redhat.com/ubi8/go-toolset:1.20 as builder
 
-RUN GO111MODULE=on go build -o /bin/persistence_agent backend/src/agent/persistence/*.go
-# Check licenses and comply with license terms.
-RUN ./hack/install-go-licenses.sh
-# First, make sure there's no forbidden license.
-RUN go-licenses check ./backend/src/agent/persistence
-RUN go-licenses csv ./backend/src/agent/persistence > /tmp/licenses.csv && \
-    diff /tmp/licenses.csv backend/third_party_licenses/persistence_agent.csv && \
-    go-licenses save ./backend/src/agent/persistence --save_path /tmp/NOTICES
+## Build args to be used at this step
+ARG SOURCE_CODE
+
+USER root
 
-FROM alpine:3.17
+RUN dnf install -y bash git openssh gcc
 
-RUN adduser -S appuser
-USER appuser
+COPY ${SOURCE_CODE}/go.mod ./
+COPY ${SOURCE_CODE}/go.sum ./
 
+# Copy the source
+COPY ${SOURCE_CODE}/ ./
+
+RUN GO111MODULE=on go build -o /bin/persistence_agent backend/src/agent/persistence/*.go
+
+#@follow_tag(registry.access.redhat.com/ubi8/ubi-minimal:8.9)
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.9
 WORKDIR /bin
 
 COPY --from=builder /bin/persistence_agent /bin/persistence_agent
-# Copy licenses and notices.
-COPY --from=builder /tmp/licenses.csv /third_party/licenses.csv
-COPY --from=builder /tmp/NOTICES /third_party/NOTICES
 
 ENV NAMESPACE ""
 
@@ -51,3 +51,7 @@ ENV TTL_SECONDS_AFTER_WORKFLOW_FINISH 86400
 ENV NUM_WORKERS 2
 
 CMD persistence_agent --logtostderr=true --namespace=${NAMESPACE} --ttlSecondsAfterWorkflowFinish=${TTL_SECONDS_AFTER_WORKFLOW_FINISH} --numWorker ${NUM_WORKERS}
+
+LABEL name="ds-pipelines-persistenceagent" \
+      summary="DSP persistenceagent"
+
diff --git a/backend/Dockerfile.scheduledworkflow b/backend/Dockerfile.scheduledworkflow
@@ -12,38 +12,48 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.20.4-alpine3.17 as builder
+# Build arguments
+ARG SOURCE_CODE=.
 
-WORKDIR /go/src/github.com/kubeflow/pipelines
-COPY . .
+# Use ubi8/nodejs-14 as base image
+#@follow_tag(registry.access.redhat.com/ubi8/go-toolset:1.19)
+FROM registry.access.redhat.com/ubi8/go-toolset:1.19 as builder
 
-# Needed musl-dev for github.com/mattn/go-sqlite3
-RUN apk update && apk upgrade && \
-    apk add --no-cache bash git openssh gcc musl-dev
+## Build args to be used at this step
+ARG SOURCE_CODE
+
+## Switch to root as required for some operations
+USER root
+
+RUN dnf upgrade -y && \
+    dnf install -y bash \
+                   git \
+                   openssh \
+                   gcc && \
+    dnf clean all && rm -rf /var/cache/yum
 
-RUN GO111MODULE=on go build -o /bin/controller backend/src/crd/controller/scheduledworkflow/*.go
-# Check licenses and comply with license terms.
-RUN ./hack/install-go-licenses.sh
-# First, make sure there's no forbidden license.
-RUN go-licenses check ./backend/src/crd/controller/scheduledworkflow
-RUN go-licenses csv ./backend/src/crd/controller/scheduledworkflow > /tmp/licenses.csv && \
-    diff /tmp/licenses.csv backend/third_party_licenses/swf.csv && \
-    go-licenses save ./backend/src/crd/controller/scheduledworkflow --save_path /tmp/NOTICES
 
-FROM alpine:3.17
+COPY ${SOURCE_CODE}/go.mod ./
+COPY ${SOURCE_CODE}/go.sum ./
 
-RUN apk --no-cache add tzdata
+# Copy the source
+COPY ${SOURCE_CODE}/ ./
 
-RUN adduser -S appuser
-USER appuser
+RUN GO111MODULE=on go build -o /bin/controller backend/src/crd/controller/scheduledworkflow/*.go
 
+#@follow_tag(registry.access.redhat.com/ubi8/ubi-minimal:8.8)
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8
 WORKDIR /bin
 
 COPY --from=builder /bin/controller /bin/controller
-# Copy licenses and notices.
-COPY --from=builder /tmp/licenses.csv /third_party/licenses.csv
-COPY --from=builder /tmp/NOTICES /third_party/NOTICES
+RUN chmod +x /bin/controller
+
+RUN microdnf makecache && \
+     microdnf install -y tzdata.noarch
 
 ENV NAMESPACE ""
 
 CMD /bin/controller --logtostderr=true --namespace=${NAMESPACE}
+
+LABEL name="ds-pipelines-scheduledworkflow" \
+      summary="DSP Scheduled Workflow Controller"
diff --git a/backend/src/apiserver/client/util.go b/backend/src/apiserver/client/util.go
@@ -19,6 +19,7 @@ import (
 	"github.com/pkg/errors"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
+	"os"
 )
 
 func getKubernetesClientset(clientParams util.ClientParameters) (*kubernetes.Clientset, error) {
@@ -35,3 +36,15 @@ func getKubernetesClientset(clientParams util.ClientParameters) (*kubernetes.Cli
 	}
 	return clientSet, nil
 }
+
+// PathExists exists returns whether the given file or directory exists
+func PathExists(path string) (bool, error) {
+	_, err := os.Stat(path)
+	if err == nil {
+		return true, nil
+	}
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	return false, err
+}