Merge pull request #5 from cloudogu/feature/4-package-and-release-crd…

…s-in-a-separate-helm-chart Feature/4 package and release crds in a separate helm chart
cloudogu · Oct 5, 2023 · 9538ec8 · 9538ec8
2 parents 08f6ce5 + 56fd365
commit 9538ec8
Show file tree

Hide file tree

Showing 18 changed files with 865 additions and 150 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- [#4] Add CRD-Release to Jenkinsfile
 
 ## [v0.1.0] - 2023-09-05
 

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -187,9 +187,8 @@ void stageStaticAnalysisSonarQube() {
 
 void stageAutomaticRelease(Makefile makefile) {
     if (gitflow.isReleaseBranch()) {
-        String releaseVersion = git.getSimpleBranchName()
-        String dockerReleaseVersion = releaseVersion.split("v")[1]
         String controllerVersion = makefile.getVersion()
+        String releaseVersion = "v${controllerVersion}".toString()
 
         stage('Build & Push Image') {
             withCredentials([usernamePassword(credentialsId: 'cesmarvin',
@@ -200,10 +199,10 @@ void stageAutomaticRelease(Makefile makefile) {
                         "login ${CES_MARVIN_USERNAME}\n" +
                         "password ${CES_MARVIN_PASSWORD}\" >> ~/.netrc"
             }
-            def dockerImage = docker.build("cloudogu/${repositoryName}:${dockerReleaseVersion}")
+            def dockerImage = docker.build("cloudogu/${repositoryName}:${controllerVersion}")
             sh "rm ~/.netrc"
             docker.withRegistry('https://registry.hub.docker.com/', 'dockerHubCredentials') {
-                dockerImage.push("${dockerReleaseVersion}")
+                dockerImage.push("${controllerVersion}")
             }
         }
 
@@ -238,11 +237,16 @@ void stageAutomaticRelease(Makefile makefile) {
                     .mountJenkinsUser()
                     .inside("--volume ${WORKSPACE}:/go/src/${project} -w /go/src/${project}")
                             {
-                                make 'k8s-helm-package-release'
+                                // Package operator-chart & crd-chart
+                                make 'helm-package-release'
+                                make 'crd-helm-package'
 
+                                // Push charts
                                 withCredentials([usernamePassword(credentialsId: 'harborhelmchartpush', usernameVariable: 'HARBOR_USERNAME', passwordVariable: 'HARBOR_PASSWORD')]) {
                                     sh ".bin/helm registry login ${registry} --username '${HARBOR_USERNAME}' --password '${HARBOR_PASSWORD}'"
+
                                     sh ".bin/helm push target/helm/${repositoryName}-${controllerVersion}.tgz oci://${registry}/${registry_namespace}/"
+                                    sh ".bin/helm push target/helm-crd/${repositoryName}-crd-${controllerVersion}.tgz oci://${registry}/${registry_namespace}/"
                                 }
                             }
         }

diff --git a/Makefile b/Makefile
@@ -5,7 +5,7 @@ VERSION=0.1.0
 IMAGE_DEV=${K3CES_REGISTRY_URL_PREFIX}/${ARTIFACT_ID}:${VERSION}
 IMAGE=cloudogu/${ARTIFACT_ID}:${VERSION}
 GOTAG?=1.21
-MAKEFILES_VERSION=7.13.1
+MAKEFILES_VERSION=8.5.0
 LINT_VERSION=v1.52.1
 STAGE?=production
 

diff --git a/build/make/bats/Dockerfile b/build/make/bats/Dockerfile
@@ -4,4 +4,4 @@ ARG BATS_TAG
 FROM ${BATS_BASE_IMAGE}:${BATS_TAG}
 
 # Make bash more findable by scripts and tests
-RUN apk add make git bash
+RUN apk add make git bash
diff --git a/build/make/coder-lib.sh b/build/make/coder-lib.sh
@@ -0,0 +1,167 @@
+#!/bin/bash
+# a collection of helpful functions to update coder workspaces for rapid development
+set -e -u -x -o pipefail
+
+function getCoderUser() {
+  coder users show me -o json | jq -r '.username'
+}
+
+function getAllWorkspaces() {
+  coder list -c workspace | tail -n+2
+}
+
+function doesWorkspaceExist() {
+  coderUser="$1"
+  workspaceName="$2"
+
+  workspace=$(coder list -a -o json | jq -r "select(.[].owner_name == \"${coderUser}\" and  .[].name == \"${workspaceName}\") | .[0].name")
+  if [ -z "$workspace" ]; then
+    return 1 #workspace does not exist
+  else
+    return 0
+  fi
+}
+
+function generateUniqueWorkspaceName() {
+  local wantedWorkspacePrefix="$1"
+  # use time to make name unique
+  local time
+  time=$(date +'%H-%M-%S')
+  local lengthOfTime=${#time}
+  local delimiter='-'
+  local lengthOfDelimiter=${#delimiter}
+  # trim prefix, as workspace names are limited to 32 chars
+  local trimmedPrefix="${wantedWorkspacePrefix:0:$((32 - lengthOfDelimiter - lengthOfTime))}"
+  local uniqueName="${trimmedPrefix}${delimiter}${time}"
+  # '--' is forbidden in coder, replace multiple '-' with a single one.
+  echo "${uniqueName}" | awk '{gsub(/[-]+/,"-")}1'
+  # returns sth like 'myPrefix-12-45-23'
+}
+
+function buildImage() {
+  local tag="$1"
+  local buildDir="${2:-./build}"
+  local secretDir="${3:-./secretArgs}"
+  local containerExec="${4:-podman}"
+  local secretArgs=()
+  # include build-secrets if there are any
+  # shellcheck disable=SC2231
+  for secretPath in $secretDir/*; do
+    # do not match .sh scripts
+    [[ $secretPath == *.sh ]] && continue
+    local secretName
+    secretName=$(basename "$secretPath")
+    secretArgs+=("--secret=id=$secretName,src=$secretDir/$secretName")
+  done
+  if [ "$containerExec" = "podman" ]; then
+    $containerExec build -t "$tag" --pull=newer "$buildDir" "${secretArgs[@]}"
+  else
+    $containerExec build -t "$tag" --pull "$buildDir" "${secretArgs[@]}"
+  fi
+}
+
+function doTrivyConvert() {
+  local trivyFlags=$1
+  local outputFile=$2
+  local containerExec=$3
+  local jsonScanToConvert=$4
+
+  local containerJsonScanFile="/tmp/scan.json"
+
+  # shellcheck disable=SC2086
+  # as globbing is what we want here
+  "$containerExec" run --rm --pull=always \
+    -v trivy-cache:/root/.cache \
+    -v "$jsonScanToConvert:$containerJsonScanFile" \
+    aquasec/trivy -q \
+    convert $trivyFlags "$containerJsonScanFile"  > "$outputFile"
+}
+
+function uploadTemplate() {
+  local templateDir="${1:?"Error. you need to add the template directory as the first parameter"}"
+  local templateName="${2:?"Error. you need to add the template name as the second parameter"}"
+  # for terraform variables (not editable by workspace users)
+  local variablesFile="${templateDir}/variables.yaml"
+  if [ -f "$variablesFile" ]; then
+    local doesVariablesFileExist=1
+  fi
+  if ! coder template push -y -d "$templateDir" ${doesVariablesFileExist:+--variables-file "$variablesFile"} "$templateName"; then
+    # if template does not exist yet, create it in coder
+    coder template create -y -d "$templateDir" ${doesVariablesFileExist:+--variables-file "$variablesFile"} "$templateName"
+  fi
+}
+
+function createNewWorkspace() {
+  local templateName="$1"
+  local workspaceName="$2"
+  # 3. param is optional, set it to autofill prompts for coder params
+  local templateDir="${3-unset}"
+  local richParametersFile="${templateDir}/rich-parameters.yaml"
+  if [ -n "${templateDir+x}" ] && [ -f "$richParametersFile" ]; then
+    local doesRichParametersFileExist=1
+  fi
+  coder create -t "$templateName" -y "$workspaceName" ${doesRichParametersFileExist:+--rich-parameter-file "$richParametersFile"}
+}
+
+function removeAllOtherWorkspaces() {
+  local CODER_USER="$1"
+  local WORKSPACE_PREFIX="$2"
+  local IGNORED_WORKSPACE="$3"
+  WORKSPACES="$(getAllWorkspaces)"
+  for ws in $WORKSPACES; do
+    if [ "$ws" != "$CODER_USER/$IGNORED_WORKSPACE" ] && [[ "$ws" =~ ^"$CODER_USER/$WORKSPACE_PREFIX" ]]; then
+      echo "delete $ws"
+      if ! coder delete "$ws" -y; then
+        #do it twice as podman always throws an error at the first time
+        coder delete "$ws" -y
+      fi
+    fi
+  done
+}
+
+function updateWorkspace() {
+  local coderUser="$1"
+  local workspaceName="$2"
+  local qualifiedWorkspaceName="$coderUser/$workspaceName"
+  if ! coder stop "$qualifiedWorkspaceName" -y; then
+    #do it twice as podman always throws an error at the first time
+    coder stop "$qualifiedWorkspaceName" -y
+  fi
+  coder update "$qualifiedWorkspaceName"
+}
+
+function startTestWorkspace() {
+  local coderUser="$1"
+  local templateDir="$2"
+  local workspacePrefix="$3"
+  local templateName="$4"
+  local reuseTestWorkspace="$5"
+
+  local newWorkspaceName
+  if [ "$reuseTestWorkspace" = false ]; then
+    newWorkspaceName="$(generateUniqueWorkspaceName "$workspacePrefix")"
+    # do that before deleting others, so that i don't need to wait
+    createNewWorkspace "$templateName" "$newWorkspaceName" "$templateDir"
+    # trim prefix as the name of the workspace can also get trimmed
+    removeAllOtherWorkspaces "$coderUser" "${workspacePrefix:0:22}" "$newWorkspaceName"
+  else
+    newWorkspaceName="$workspacePrefix"
+    if ! doesWorkspaceExist "$coderUser" "$newWorkspaceName"; then
+      createNewWorkspace "$templateName" "$newWorkspaceName" "$templateDir"
+    else
+      updateWorkspace "$coderUser" "$newWorkspaceName"
+    fi
+  fi
+}
+
+function uploadToNexus() {
+  local fileToUpload="$1"
+  local fileNameNexus="${fileToUpload##*/}"
+  local templateName="$2"
+  local releaseVersion="$3"
+  local nexusUrl="${4:-https://ecosystem.cloudogu.com/nexus/repository/itz-bund/coder}"
+  set +x #disable command printing because of the password
+  curl --progress-bar -u "$(cat secrets/nexus-user):$(cat secrets/nexus-pw)" --upload-file "$fileToUpload" \
+    "$nexusUrl/$templateName/$releaseVersion/$fileNameNexus"
+  set -x
+}
diff --git a/build/make/coder.mk b/build/make/coder.mk
@@ -0,0 +1,160 @@
+SHELL := /bin/bash
+
+IMAGE_TAG?=${IMAGE_REGISTRY}/coder/coder-${TEMPLATE_NAME}:${VERSION}
+REUSE_TEST_WORKSPACE?=false
+
+#BUILD_DIR given via variables.mk
+TEMPLATE_DIR=${WORKDIR}/template
+CONTAINER_BUILD_DIR=${WORKDIR}/container
+SECRETS_DIR=${WORKDIR}/secrets
+CODER_LIB_PATH=${BUILD_DIR}/make/coder-lib.sh
+
+RELEASE_DIR=${WORKDIR}/release
+MAKE_CHANGE_TOKEN_DIR=${RELEASE_DIR}/make
+CONTAINER_FILE?=${CONTAINER_BUILD_DIR}/Dockerfile
+CONTAINER_IMAGE_CHANGE_TOKEN?=${MAKE_CHANGE_TOKEN_DIR}/${TEMPLATE_NAME}_image_id.txt
+CONTAINER_IMAGE_TAR?=${RELEASE_DIR}/${TEMPLATE_NAME}.tar
+CONTAINER_IMAGE_TARGZ?=${RELEASE_DIR}/${TEMPLATE_NAME}.tar.gz
+CONTAINER_IMAGE_TRIVY_SCAN_JSON?=${RELEASE_DIR}/trivy.json
+CONTAINER_IMAGE_TRIVY_SCAN_TABLE?=${RELEASE_DIR}/trivy.txt
+CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_TABLE?=${RELEASE_DIR}/trivy_critical.txt
+CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_JSON?=${RELEASE_DIR}/trivy_critical.json
+
+IMAGE_REGISTRY?=registry.cloudogu.com
+IMAGE_REGISTRY_USER_FILE?=${SECRETS_DIR}/harbor-user
+IMAGE_REGISTRY_PW_FILE?=${SECRETS_DIR}/harbor-pw
+
+CHANGELOG_FILE=${WORKDIR}/CHANGELOG.md
+TEMPLATE_RELEASE_TAR_GZ=${RELEASE_DIR}/${TEMPLATE_NAME}-template.tar.gz
+
+TEST_WORKSPACE_PREFIX?=test-${TEMPLATE_NAME}
+# only set if coder is installed, so that there is no problem with build and release targets
+CODER_USER?=$(shell if [ -x "$(command -v coder)" ]; then coder users show me -o json | jq -r '.username'; else echo ""; fi )
+
+CONTAINER_BIN?=$(shell if [ -x "$(command -v podman)" ]; then echo "podman"; else echo "docker"; fi)
+GOPASS_BIN?=$(shell command -v gopass 2> /dev/null)
+
+EXCLUDED_TEMPLATE_FILES?=rich-parameters.yaml variables.yaml
+
+
+##@ Coder template development
+
+${SECRETS_DIR}:
+	mkdir -p ${SECRETS_DIR}
+
+${IMAGE_REGISTRY_USER_FILE}: ${SECRETS_DIR}
+ifeq ($(ENVIRONMENT), local)
+		@echo "Found developer environment. creating secret ${IMAGE_REGISTRY_USER_FILE}"
+		@${GOPASS_BIN} show ces/websites/registry.cloudogu.com/robot_coder_jenkins | tail -n 1 | sed -e "s/^username: //" > ${IMAGE_REGISTRY_USER_FILE};
+else
+		@echo "Found CI environment. Please create secrets yourself"
+endif
+
+${IMAGE_REGISTRY_PW_FILE}: ${SECRETS_DIR}
+ifeq ($(ENVIRONMENT), local)
+		@echo "Found developer environment. creating secret ${IMAGE_REGISTRY_PW_FILE}"
+		@${GOPASS_BIN} show ces/websites/registry.cloudogu.com/robot_coder_jenkins | head -n 1  > ${IMAGE_REGISTRY_PW_FILE};
+else
+		@echo "Found CI environment. Please create secrets yourself"
+endif
+
+.PHONY: loadGopassSecrets
+loadGopassSecrets: ${IMAGE_REGISTRY_USER_FILE} ${IMAGE_REGISTRY_PW_FILE} ${ADDITIONAL_SECRETS_TARGET} ## load secrets from gopass into secret files, so that the build process works locally
+
+.PHONY: imageRegistryLogin
+imageRegistryLogin: loadGopassSecrets ${IMAGE_REGISTRY_USER_FILE} ${IMAGE_REGISTRY_PW_FILE} ## log in to the registry
+	@${CONTAINER_BIN} login -u "$$(cat ${IMAGE_REGISTRY_USER_FILE})" --password-stdin '${IMAGE_REGISTRY}' < ${IMAGE_REGISTRY_PW_FILE}
+
+.PHONY: imageRegistryLogout
+imageRegistryLogout: ## log out of the registry
+	@${CONTAINER_BIN} logout '${IMAGE_REGISTRY}'
+
+.PHONY: buildImage
+buildImage: buildImage-$(ENVIRONMENT) ## build the container image
+
+.PHONY: buildImage-local
+buildImage-local: imageRegistryLogin ${CONTAINER_IMAGE_CHANGE_TOKEN} ## build the container image locally
+	@echo "if the build is not triggered without a change in the dockerfile, try to delete ${CONTAINER_IMAGE_CHANGE_TOKEN}"
+
+.PHONY: buildImage-ci
+buildImage-ci: ${CONTAINER_IMAGE_CHANGE_TOKEN} ## build the container image without automatic secret management
+
+${CONTAINER_IMAGE_CHANGE_TOKEN}: ${CONTAINER_FILE}
+	@. ${CODER_LIB_PATH} && buildImage ${IMAGE_TAG} ${CONTAINER_BUILD_DIR} ${SECRETS_DIR} ${CONTAINER_BIN}
+	@mkdir -p ${MAKE_CHANGE_TOKEN_DIR}
+	@${CONTAINER_BIN} image ls --format="{{.ID}}" ${IMAGE_TAG} > ${CONTAINER_IMAGE_CHANGE_TOKEN}
+
+.PHONY: uploadTemplate
+uploadTemplate: ## upload template to coder server
+	@. ${CODER_LIB_PATH} && uploadTemplate ${TEMPLATE_DIR} ${TEMPLATE_NAME}
+
+.PHONY: startTestWorkspace
+startTestWorkspace: ## start a test workspace with coder
+	 @. ${CODER_LIB_PATH} && startTestWorkspace ${CODER_USER} ${TEMPLATE_DIR} ${TEST_WORKSPACE_PREFIX}  ${TEMPLATE_NAME} ${REUSE_TEST_WORKSPACE}
+
+.PHONY: createImageRelease
+createImageRelease: ${CONTAINER_IMAGE_TARGZ} ## export the container image as a tar.gz
+
+${CONTAINER_IMAGE_TAR}: ${CONTAINER_IMAGE_CHANGE_TOKEN}
+	${CONTAINER_BIN} save "${IMAGE_TAG}" -o ${CONTAINER_IMAGE_TAR}
+
+${CONTAINER_IMAGE_TARGZ}: ${CONTAINER_IMAGE_TAR}
+	gzip -f --keep "${CONTAINER_IMAGE_TAR}"
+
+.PHONY: trivyscanImage
+trivyscanImage: ${CONTAINER_IMAGE_TRIVY_SCAN_JSON} ${CONTAINER_IMAGE_TRIVY_SCAN_TABLE} ${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_TABLE} ${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_JSON} ## do a trivy scan for the workspace image in various output formats
+
+${CONTAINER_IMAGE_TRIVY_SCAN_JSON}: ${CONTAINER_IMAGE_TAR}
+	${CONTAINER_BIN} run --rm --pull=always \
+		-v "trivy-cache:/root/.cache" \
+		-v "${CONTAINER_IMAGE_TAR}:/tmp/image.tar" \
+		aquasec/trivy -q \
+		image --scanners vuln --input /tmp/image.tar -f json --timeout 15m \
+		> ${CONTAINER_IMAGE_TRIVY_SCAN_JSON}
+
+${CONTAINER_IMAGE_TRIVY_SCAN_TABLE}: ${CONTAINER_IMAGE_TRIVY_SCAN_JSON}
+	@. ${CODER_LIB_PATH} && \
+    		doTrivyConvert "--format table" ${CONTAINER_IMAGE_TRIVY_SCAN_TABLE} ${CONTAINER_BIN} ${CONTAINER_IMAGE_TRIVY_SCAN_JSON}
+
+${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_TABLE}: ${CONTAINER_IMAGE_TRIVY_SCAN_JSON}
+	@. ${CODER_LIB_PATH} && \
+		doTrivyConvert "--format table --severity CRITICAL" ${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_TABLE} ${CONTAINER_BIN} ${CONTAINER_IMAGE_TRIVY_SCAN_JSON}
+
+${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_JSON}: ${CONTAINER_IMAGE_TRIVY_SCAN_JSON}
+	@. ${CODER_LIB_PATH} && \
+    		doTrivyConvert "--format json --severity CRITICAL" ${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_JSON} ${CONTAINER_BIN} ${CONTAINER_IMAGE_TRIVY_SCAN_JSON}
+
+.PHONY: createTemplateRelease
+createTemplateRelease: ## generate template.tar.gz with all files needed for customers
+	# remove release dir first as 'cp' cannot merge and will place the source dir inside the target dir if it already exists
+	rm -rf "${RELEASE_DIR}/${TEMPLATE_NAME}"
+	cp -r "${TEMPLATE_DIR}" "${RELEASE_DIR}/${TEMPLATE_NAME}/"
+	#copy changelog
+	cp "${CHANGELOG_FILE}" "${RELEASE_DIR}/${TEMPLATE_NAME}/"
+	# remove excludes
+	for file in "${EXCLUDED_TEMPLATE_FILES}"; do \
+		rm -f "${RELEASE_DIR}/${TEMPLATE_NAME}/$$file"; \
+	done
+	tar -czf "${RELEASE_DIR}/${TEMPLATE_NAME}-template.tar.gz" -C "${RELEASE_DIR}" "${TEMPLATE_NAME}"
+
+.PHONY: createRelease ## generate template- and container archives and the trivy scans
+createRelease: createTemplateRelease ${CONTAINER_IMAGE_TARGZ} trivyscanImage ## create the image.tar.gz, template.tar.gz and trivy scans
+
+.PHONY: cleanCoderRelease
+cleanCoderRelease: ## clean release directory
+	rm -rf "${RELEASE_DIR}"
+	mkdir -p "${RELEASE_DIR}"
+
+.PHONY: pushImage
+pushImage: ## push the container image into the registry
+	${CONTAINER_BIN} push ${IMAGE_TAG}
+
+.PHONY: uploadRelease
+uploadRelease: createTemplateRelease ${CONTAINER_IMAGE_TARGZ} ${CONTAINER_IMAGE_TRIVY_SCAN_JSON} ${CONTAINER_IMAGE_TRIVY_SCAN_TABLE} ${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_TABLE} ${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_JSON} ## upload release artifacts to nexus
+	@. ${CODER_LIB_PATH} && uploadToNexus ${TEMPLATE_RELEASE_TAR_GZ} ${TEMPLATE_NAME} ${VERSION}
+	@. ${CODER_LIB_PATH} && uploadToNexus ${CONTAINER_IMAGE_TRIVY_SCAN_JSON} ${TEMPLATE_NAME} ${VERSION}
+	@. ${CODER_LIB_PATH} && uploadToNexus ${CONTAINER_IMAGE_TRIVY_SCAN_TABLE} ${TEMPLATE_NAME} ${VERSION}
+	@. ${CODER_LIB_PATH} && uploadToNexus ${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_TABLE} ${TEMPLATE_NAME} ${VERSION}
+	@. ${CODER_LIB_PATH} && uploadToNexus ${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_JSON} ${TEMPLATE_NAME} ${VERSION}
+	@. ${CODER_LIB_PATH} && uploadToNexus ${CONTAINER_IMAGE_TARGZ} ${TEMPLATE_NAME} ${VERSION}
+