feat(workflows): integrate hadolint (dockle failed) #1970

Workflow file for this run

.github/workflows/build_push.yaml at e6b58b5

	# This workflow:
	# * Builds, tests, and scans all images
	# * (optionally) pushes the images to ACR
	#
	# This workflow triggers on:
	# * a push to master
	# * any create/synchronize to a PR (eg: any time you push an update to a PR).
	#
	# Image build/test/scan will run on any of the above events.
	# Image push will run only if:
	# * this is a push to master
	# * if the PR triggering this event has the label 'auto-deploy'
	#
	# To configure this workflow:
	#
	# 1. Set up the following secrets in your workspace:
	# a. REGISTRY_USERNAME with ACR username
	# b. REGISTRY_PASSWORD with ACR Password
	# c. AZURE_CREDENTIALS with the output of `az ad sp create-for-rbac --sdk-auth`
	# d. DEV_REGISTRY_USERNAME with the DEV ACR username
	# e. DEV_REGISTRY_PASSWORD with the DEV ACR Password
	#
	# 2. Change the values for the REGISTRY_NAME, CLUSTER_NAME, CLUSTER_RESOURCE_GROUP and NAMESPACE environment variables (below in build-push).
	name: build_and_push
	on:
	schedule:
	# Execute at 2am EST every day
	- cron: '0 21 * * *'
	push:
	branches:
	- 'master'
	pull_request:
	types:
	- 'opened'
	- 'synchronize'
	- 'reopened'
	env:
	SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

	jobs:
	# Any checks that run pre-build
	pre-build-checks:
	runs-on: ubuntu-20.04
	steps:
	- uses: actions/checkout@master

	- name: Assert committed ./output folder matches `make generate-dockerfiles` output
	run: \|
	sudo apt-get install --yes make
	make clean
	make generate-dockerfiles
	if ! git diff --quiet output/; then
	echo 'output folder and docker-bits/resources out of sync!'
	exit 1
	fi

	build-push:
	env:
	REGISTRY_NAME: k8scc01covidacr
	DEV_REGISTRY_NAME: k8scc01covidacrdev
	CLUSTER_NAME: k8s-cancentral-01-covid-aks
	CLUSTER_RESOURCE_GROUP: k8s-cancentral-01-covid-aks
	LOCAL_REPO: localhost:5000
	TRIVY_VERSION: "v0.31.3"
	strategy:
	fail-fast: false
	matrix:
	notebook:
	# TODO: Pull this from a settings file or Makefile, that way Make can have the same list
	# - docker-stacks-datascience-notebook # Debugging
	- rstudio
	- sas
	- jupyterlab-cpu
	- jupyterlab-pytorch
	- jupyterlab-tensorflow
	- remote-desktop
	needs: pre-build-checks
	runs-on: ubuntu-latest
	services:
	registry:
	image: registry:2
	ports:
	- 5000:5000
	steps:
	- name: Set ENV variables for a PR containing the auto-deploy tag
	if: github.event_name == 'pull_request' && contains( github.event.pull_request.labels.*.name, 'auto-deploy')
	run: \|
	echo "REGISTRY=k8scc01covidacrdev.azurecr.io" >> "$GITHUB_ENV"
	echo "IMAGE_VERSION=dev" >> "$GITHUB_ENV"

	- name: Set ENV variables for pushes to master
	if: github.event_name == 'push' && github.ref == 'refs/heads/master'
	run: \|
	echo "REGISTRY=k8scc01covidacr.azurecr.io" >> "$GITHUB_ENV"
	echo "IMAGE_VERSION=v1" >> "$GITHUB_ENV"
	echo "IS_LATEST=true" >> "$GITHUB_ENV"

	- uses: actions/checkout@master

	- name: Echo disk usage before clean up
	run: ./.github/scripts/echo_usage.sh

	- name: Free up all available disk space before building
	run: ./.github/scripts/cleanup_runner.sh

	- name: Echo disk usage before build start
	run: ./.github/scripts/echo_usage.sh

	- name: Get current notebook name
	id: notebook-name
	shell: bash
	# testing without the -> echo NOTEBOOK_NAME=${{ matrix.notebook }} >> $GITHUB_OUTPUT
	run: \|
	echo ::set-output name=NOTEBOOK_NAME::${{ matrix.notebook }}


	# Connect to Azure Container registry (ACR)
	- uses: azure/docker-login@v1
	with:
	login-server: ${{ env.REGISTRY_NAME }}.azurecr.io
	username: ${{ secrets.REGISTRY_USERNAME }}
	password: ${{ secrets.REGISTRY_PASSWORD }}

	# Connect to Azure DEV Container registry (ACR)
	- uses: azure/docker-login@v1
	with:
	login-server: ${{ env.DEV_REGISTRY_NAME }}.azurecr.io
	username: ${{ secrets.DEV_REGISTRY_USERNAME }}
	password: ${{ secrets.DEV_REGISTRY_PASSWORD }}

	# Image building/storing locally
	- name: Make Dockerfiles
	run: make generate-dockerfiles

	# make build emits full_image_name, image_tag, and image_repo outputs
	- name: Build image
	id: build-image
	run: make build/${{ matrix.notebook }} REPO=${{ env.LOCAL_REPO }}

	- name: Echo disk usage after build completion
	run: ./.github/scripts/echo_usage.sh

	- name: Add standard tag names (short sha, sha, and branch) and any other post-build activity
	run: make post-build/${{ matrix.notebook }} REPO=${{ env.LOCAL_REPO }}

	- name: Push image to local registry (default pushes all tags)
	run: make push/${{ matrix.notebook }} REPO=${{ env.LOCAL_REPO }}

	# Image testing

	- name: Set Up Python for Test Suite
	uses: actions/setup-python@v4
	with:
	python-version: '3.10'

	- name: Set up venv for Test Suite
	run: \|
	python -m pip install --upgrade pip
	make install-python-dev-venv

	- name: Test image
	run: make test/${{ matrix.notebook }} REPO=${{ env.LOCAL_REPO }}

	# Image scanning

	# Free up space from build process (containerscan action will run out of space if we don't)
	- run: ./.github/scripts/cleanup_runner.sh

	# Scan image for vulnerabilities
	- name: Aqua Security Trivy image scan
	# see https://github.com/StatCan/aaw-private/issues/11 -- should be re-enabled
	if: steps.notebook-name.outputs.NOTEBOOK_NAME != 'sas'
	run: \|
	curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh \| sh -s -- -b /usr/local/bin ${{ env.TRIVY_VERSION }}
	trivy image ${{ steps.build-image.outputs.full_image_name }} --exit-code 1 --timeout=20m --security-checks vuln --severity CRITICAL

	# Run Dockle
	- name: Run dockle
	uses: goodwithtech/dockle-action@main
	with:
	# This seems to not work as expected, it seems to take from Makefile 201 but idk why it fails here
	image: ${{ steps.build-image.outputs.full_image_name }}
	#image: localhost:5000/${{ matrix.notebook }}:${{ github.sha }}
	format: 'list'
	exit-code: '0'
	# Must determine what we want to do action-wise (fail? let it continue?)
	# exit-level: 'fatal'
	ignore: 'DKL-DI-0006'

	# Push image to ACR
	# Pushes if this is a push to master or an update to a PR that has auto-deploy label
	- name: Test if we should push to ACR
	id: should-i-push
	if: \|
	github.event_name == 'push' \|\|
	(
	github.event_name == 'pull_request' &&
	contains( github.event.pull_request.labels.*.name, 'auto-deploy')
	)
	run: echo "::set-output name=boolean::true"
	#testing without -> echo 'boolean=true' >> $GITHUB_OUTPUT

	# Pull the local image back, then "build" it (will just tag the pulled image)
	- name: Pull image back from local repo
	if: steps.should-i-push.outputs.boolean == 'true'
	run: docker pull ${{ steps.build-image.outputs.full_image_name }}

	# Rename the localhost:5000/imagename:tag built above to use the real repo
	# (get above's name from build-image's output)
	- name: Tag images with real repository
	if: steps.should-i-push.outputs.boolean == 'true'
	run: >
	make post-build/${{ matrix.notebook }} DEFAULT_REPO=$REGISTRY IS_LATEST=$IS_LATEST
	IMAGE_VERSION=$IMAGE_VERSION SOURCE_FULL_IMAGE_NAME=${{ steps.build-image.outputs.full_image_name }}

	- name: Push image to registry
	if: steps.should-i-push.outputs.boolean == 'true'
	run: \|
	make push/${{ matrix.notebook }} DEFAULT_REPO=$REGISTRY

	- name: Slack Notification
	if: failure() && github.event_name=='schedule'
	uses: act10ns/slack@v1
	with:
	status: failure
	message: Build failed. https://github.com/StatCan/aaw-kubeflow-containers/actions/runs/${{github.run_id}}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

feat(workflows): integrate hadolint (dockle failed) #1970

Workflow file

feat(workflows): integrate hadolint (dockle failed) #1970

Jobs

Run details

Workflow file for this run