Skip to content

Commit

Permalink
chore: Move Data Science Pipelines Dockerfiles to a different path
Browse files Browse the repository at this point in the history
Signed-off-by: VaniHaripriya <[email protected]>
  • Loading branch information
VaniHaripriya committed Nov 25, 2024
1 parent 7c3b020 commit 3066dd2
Show file tree
Hide file tree
Showing 16 changed files with 418 additions and 156 deletions.
10 changes: 5 additions & 5 deletions .github/workflows/build-images.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -74,17 +74,17 @@ jobs:
matrix:
include:
- image: ds-pipelines-api-server
dockerfile: backend/Dockerfile
dockerfile: odh_images/Dockerfile
- image: ds-pipelines-frontend
dockerfile: frontend/Dockerfile
- image: ds-pipelines-persistenceagent
dockerfile: backend/Dockerfile.persistenceagent
dockerfile: odh_images/Dockerfile.persistenceagent
- image: ds-pipelines-scheduledworkflow
dockerfile: backend/Dockerfile.scheduledworkflow
dockerfile: odh_images/Dockerfile.scheduledworkflow
- image: ds-pipelines-driver
dockerfile: backend/Dockerfile.driver
dockerfile: odh_images/Dockerfile.driver
- image: ds-pipelines-launcher
dockerfile: backend/Dockerfile.launcher
dockerfile: odh_images/Dockerfile.launcher
steps:
- uses: actions/checkout@v3
- uses: ./.github/actions/build
Expand Down
10 changes: 5 additions & 5 deletions .github/workflows/build-master.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,17 @@ jobs:
matrix:
include:
- image: ds-pipelines-api-server
dockerfile: backend/Dockerfile
dockerfile: odh_images/Dockerfile
- image: ds-pipelines-frontend
dockerfile: frontend/Dockerfile
- image: ds-pipelines-persistenceagent
dockerfile: backend/Dockerfile.persistenceagent
dockerfile: odh_images/Dockerfile.persistenceagent
- image: ds-pipelines-scheduledworkflow
dockerfile: backend/Dockerfile.scheduledworkflow
dockerfile: odh_images/Dockerfile.scheduledworkflow
- image: ds-pipelines-driver
dockerfile: backend/Dockerfile.driver
dockerfile: odh_images/Dockerfile.driver
- image: ds-pipelines-launcher
dockerfile: backend/Dockerfile.launcher
dockerfile: odh_images/Dockerfile.launcher
steps:
- uses: actions/checkout@v3
- name: Generate Tag
Expand Down
10 changes: 5 additions & 5 deletions .github/workflows/build-prs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -69,17 +69,17 @@ jobs:
matrix:
include:
- image: ds-pipelines-api-server
dockerfile: backend/Dockerfile
dockerfile: odh_images/Dockerfile
- image: ds-pipelines-frontend
dockerfile: frontend/Dockerfile
- image: ds-pipelines-persistenceagent
dockerfile: backend/Dockerfile.persistenceagent
dockerfile: odh_images/Dockerfile.persistenceagent
- image: ds-pipelines-scheduledworkflow
dockerfile: backend/Dockerfile.scheduledworkflow
dockerfile: odh_images/Dockerfile.scheduledworkflow
- image: ds-pipelines-driver
dockerfile: backend/Dockerfile.driver
dockerfile: odh_images/Dockerfile.driver
- image: ds-pipelines-launcher
dockerfile: backend/Dockerfile.launcher
dockerfile: odh_images/Dockerfile.launcher
steps:
- uses: actions/checkout@v3
- name: Build Image
Expand Down
69 changes: 47 additions & 22 deletions backend/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2021-2024 The Kubeflow Authors
# Copyright 2021-2022 The Kubeflow Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand All @@ -12,47 +12,72 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# Build arguments
ARG SOURCE_CODE=.
# 1. Build api server application
FROM golang:1.21.7-bookworm as builder
RUN apt-get update && apt-get install -y cmake clang musl-dev openssl
WORKDIR /go/src/github.com/kubeflow/pipelines
COPY . .
RUN GO111MODULE=on go build -o /bin/apiserver backend/src/apiserver/*.go
# Check licenses and comply with license terms.
RUN ./hack/install-go-licenses.sh
# First, make sure there's no forbidden license.
RUN go-licenses check ./backend/src/apiserver
RUN go-licenses csv ./backend/src/apiserver > /tmp/licenses.csv && \
diff /tmp/licenses.csv backend/third_party_licenses/apiserver.csv && \
go-licenses save ./backend/src/apiserver --save_path /tmp/NOTICES

# 2. Compile preloaded pipeline samples
FROM python:3.8 as compiler
RUN apt-get update -y && apt-get install --no-install-recommends -y -q default-jdk python3-setuptools python3-dev jq
RUN wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py
COPY backend/requirements.txt .
RUN python3 -m pip install -r requirements.txt --no-cache-dir

FROM registry.access.redhat.com/ubi8/go-toolset:1.21 as builder

USER root
# Downloading Argo CLI so that the samples are validated
ENV ARGO_VERSION v3.4.17
RUN curl -sLO https://github.com/argoproj/argo-workflows/releases/download/${ARGO_VERSION}/argo-linux-amd64.gz && \
gunzip argo-linux-amd64.gz && \
chmod +x argo-linux-amd64 && \
mv ./argo-linux-amd64 /usr/local/bin/argo

RUN dnf install -y cmake clang openssl

COPY ${SOURCE_CODE}/go.mod ./
COPY ${SOURCE_CODE}/go.sum ./
WORKDIR /
COPY ./samples /samples
COPY backend/src/apiserver/config/sample_config.json /samples/

RUN GO111MODULE=on go mod download
# Compiling the preloaded samples.
# The default image is replaced with the GCR-hosted python image.
RUN set -e; \
< /samples/sample_config.json jq .[].file --raw-output | while read pipeline_yaml; do \
pipeline_py="${pipeline_yaml%.yaml}"; \
python3 "$pipeline_py"; \
done

# Copy the source
COPY ${SOURCE_CODE}/ ./
# 3. Start api web server
FROM debian:stable

RUN GO111MODULE=on go build -o /bin/apiserver ./backend/src/apiserver/ && \
dnf clean all

FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
ARG COMMIT_SHA=unknown
ENV COMMIT_SHA=${COMMIT_SHA}
ARG TAG_NAME=unknown
ENV TAG_NAME=${TAG_NAME}
ENV LOG_LEVEL info

WORKDIR /bin

COPY --from=builder /opt/app-root/src/backend/src/apiserver/config/ /config
COPY backend/src/apiserver/config/ /config
COPY --from=builder /bin/apiserver /bin/apiserver

# Copy licenses and notices.
COPY --from=builder /tmp/licenses.csv /third_party/licenses.csv
COPY --from=builder /tmp/NOTICES /third_party/NOTICES
COPY --from=compiler /samples/ /samples/
RUN chmod +x /bin/apiserver

USER root

# Adding CA certificate so API server can download pipeline through URL and wget is used for liveness/readiness probe command
RUN microdnf install -y ca-certificates wget
RUN apt-get update && apt-get install -y ca-certificates wget

USER 1001
# Pin sample doc links to the commit that built the backend image
RUN sed -E "s#/(blob|tree)/master/#/\1/${COMMIT_SHA}/#g" -i /config/sample_config.json && \
sed -E "s/%252Fmaster/%252F${COMMIT_SHA}/#g" -i /config/sample_config.json

# Expose apiserver port
EXPOSE 8888
Expand Down
9 changes: 1 addition & 8 deletions backend/Dockerfile.cacheserver
Original file line number Diff line number Diff line change
Expand Up @@ -19,19 +19,12 @@ RUN apk update && apk upgrade && \
apk add --no-cache bash git openssh gcc musl-dev

WORKDIR /go/src/github.com/kubeflow/pipelines

COPY ./go.mod ./
COPY ./go.sum ./
COPY ./hack/install-go-licenses.sh ./hack/

RUN GO111MODULE=on go mod download
RUN ./hack/install-go-licenses.sh

COPY . .

RUN GO111MODULE=on go build -o /bin/cache_server backend/src/cache/*.go

# Check licenses and comply with license terms.
RUN ./hack/install-go-licenses.sh
# First, make sure there's no forbidden license.
RUN go-licenses check ./backend/src/cache
RUN go-licenses csv ./backend/src/cache > /tmp/licenses.csv && \
Expand Down
44 changes: 19 additions & 25 deletions backend/Dockerfile.driver
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2021-2024 The Kubeflow Authors
# Copyright 2023 The Kubeflow Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand All @@ -12,37 +12,31 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# Build arguments
ARG SOURCE_CODE=.
FROM golang:1.21.7-alpine3.19 as builder

# Use ubi8/nodejs-14 as base image
FROM registry.access.redhat.com/ubi8/go-toolset:1.21 as builder
WORKDIR /go/src/github.com/kubeflow/pipelines
COPY . .

RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -tags netgo -ldflags '-extldflags "-static"' -o /bin/driver ./backend/src/v2/cmd/driver/*.go

## Build args to be used at this step
ARG SOURCE_CODE

## Switch to root as required for some operations
USER root

COPY ${SOURCE_CODE}/go.mod ./
COPY ${SOURCE_CODE}/go.sum ./

RUN GO111MODULE=on go mod download

# Copy the source
COPY ${SOURCE_CODE}/ ./
# Check licenses and comply with license terms.
RUN ./hack/install-go-licenses.sh
# First, make sure there's no forbidden license.
RUN go-licenses check ./backend/src/v2/cmd/driver
RUN go-licenses csv ./backend/src/v2/cmd/driver > /tmp/licenses.csv && \
diff /tmp/licenses.csv backend/third_party_licenses/driver.csv && \
go-licenses save ./backend/src/v2/cmd/driver --save_path /tmp/NOTICES

RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -tags netgo -ldflags '-extldflags "-static"' -o /bin/driver ./backend/src/v2/cmd/driver/*.go
FROM alpine:3.19

FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
RUN adduser -S appuser
USER appuser

WORKDIR /bin

COPY --from=builder /bin/driver /bin/driver
RUN chmod +x /bin/driver

ENTRYPOINT ["/bin/driver"]
# Copy licenses and notices.
COPY --from=builder /tmp/licenses.csv /third_party/licenses.csv
COPY --from=builder /tmp/NOTICES /third_party/NOTICES

LABEL name="ds-pipelines-driver" \
summary="DSP Driver"
ENTRYPOINT [ "/bin/driver" ]
46 changes: 19 additions & 27 deletions backend/Dockerfile.launcher
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2021-2024 The Kubeflow Authors
# Copyright 2023 The Kubeflow Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand All @@ -12,39 +12,31 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# Build arguments
ARG SOURCE_CODE=.
ARG CI_CONTAINER_VERSION="unknown"
FROM golang:1.21.7-alpine3.19 as builder

WORKDIR /go/src/github.com/kubeflow/pipelines
COPY . .

# Use ubi8/nodejs-14 as base image
FROM registry.access.redhat.com/ubi8/go-toolset:1.21 as builder


## Build args to be used at this step
ARG SOURCE_CODE

## Switch to root as required for some operations
USER root

COPY ${SOURCE_CODE}/go.mod ./
COPY ${SOURCE_CODE}/go.sum ./

RUN GO111MODULE=on go mod download
RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -tags netgo -ldflags '-extldflags "-static"' -o /bin/launcher-v2 ./backend/src/v2/cmd/launcher-v2/*.go

# Copy the source
COPY ${SOURCE_CODE}/ ./
# Check licenses and comply with license terms.
RUN ./hack/install-go-licenses.sh
# First, make sure there's no forbidden license.
RUN go-licenses check ./backend/src/v2/cmd/launcher-v2
RUN go-licenses csv ./backend/src/v2/cmd/launcher-v2 > /tmp/licenses.csv && \
diff /tmp/licenses.csv backend/third_party_licenses/launcher.csv && \
go-licenses save ./backend/src/v2/cmd/launcher-v2 --save_path /tmp/NOTICES

RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -tags netgo -ldflags '-extldflags "-static"' -o /bin/launcher-v2 ./backend/src/v2/cmd/launcher-v2/*.go
FROM alpine:3.19

FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
RUN adduser -S appuser
USER appuser

WORKDIR /bin

COPY --from=builder /bin/launcher-v2 /bin/launcher-v2
RUN chmod +x /bin/launcher-v2

ENTRYPOINT ["/bin/launcher-v2"]
# Copy licenses and notices.
COPY --from=builder /tmp/licenses.csv /third_party/licenses.csv
COPY --from=builder /tmp/NOTICES /third_party/NOTICES

LABEL name="ds-pipelines-launcher" \
summary="DSP launcher"
ENTRYPOINT [ "/bin/launcher-v2" ]
40 changes: 20 additions & 20 deletions backend/Dockerfile.persistenceagent
Original file line number Diff line number Diff line change
Expand Up @@ -12,35 +12,35 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# Build arguments
ARG SOURCE_CODE=.
ARG CI_CONTAINER_VERSION="unknown"
FROM golang:1.21.7-alpine3.19 as builder

WORKDIR /go/src/github.com/kubeflow/pipelines
COPY . .

# Use ubi8/go-toolset as base image
FROM registry.access.redhat.com/ubi8/go-toolset:1.21 as builder
# Needed musl-dev for github.com/mattn/go-sqlite3
RUN apk update && apk upgrade && \
apk add --no-cache bash git openssh gcc musl-dev

## Build args to be used at this step
ARG SOURCE_CODE

USER root

RUN dnf install -y bash git openssh gcc

COPY ${SOURCE_CODE}/go.mod ./
COPY ${SOURCE_CODE}/go.sum ./

RUN GO111MODULE=on go mod download
RUN GO111MODULE=on go build -o /bin/persistence_agent backend/src/agent/persistence/*.go
# Check licenses and comply with license terms.
RUN ./hack/install-go-licenses.sh
# First, make sure there's no forbidden license.
RUN go-licenses check ./backend/src/agent/persistence
RUN go-licenses csv ./backend/src/agent/persistence > /tmp/licenses.csv && \
diff /tmp/licenses.csv backend/third_party_licenses/persistence_agent.csv && \
go-licenses save ./backend/src/agent/persistence --save_path /tmp/NOTICES

# Copy the source
COPY ${SOURCE_CODE}/ ./
FROM alpine:3.19

RUN GO111MODULE=on go build -o /bin/persistence_agent backend/src/agent/persistence/*.go
RUN adduser -S appuser
USER appuser

FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
WORKDIR /bin

COPY --from=builder /bin/persistence_agent /bin/persistence_agent
# Copy licenses and notices.
COPY --from=builder /tmp/licenses.csv /third_party/licenses.csv
COPY --from=builder /tmp/NOTICES /third_party/NOTICES

ENV NAMESPACE ""

Expand Down
Loading

0 comments on commit 3066dd2

Please sign in to comment.