Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adds workflow that helps the bots with intermediate files. #2145

Merged
merged 13 commits into from
Sep 18, 2023
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
name: "renovate-/ospo-bot: go mod tidy and make package-specs"
on:
push:
branches:
- "renovate/**"
- "dependabot/**"
paths:
- ".github/workflows/dependency-update-go-mod-tidy.yaml"
- "./src/**/go.mod"
- "./src/**/go.sum"
jobs:
go-mod-tidy:
runs-on: ubuntu-latest
permissions:
pull-requests: write
contents: write
steps:
# We potentially want to add at the end a commit by the author of the most recent
# commit in this branch. However github has some protection which prevents workflows
# to run in case a commit has been pushed with the default job-specific github-token.
# For this case we need to use another one here.
#
# For more information, see:
# <https://docs.github.com/en/actions/security-guides/automatic-token-authentication#using-the-github_token-in-a-workflow>
- uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: true
token: ${{ secrets.APP_AUTOSCALER_CI_TOKEN }} # With push token that can trigger new PR jobs
- name: Configure git
shell: bash
run: |
joergdw marked this conversation as resolved.
Show resolved Hide resolved
#! /usr/bin/env bash
set -eu -o pipefail

declare -r commit_author_name="${{github.event.head_commit.author.name}}"
declare -r commit_author_email="${{github.event.head_commit.author.email}}"

git config --global --add safe.directory "${GITHUB_WORKSPACE}"
git config user.name "${commit_author_name}"
git config user.email "${commit_author_email}"
Comment on lines +32 to +41

Check failure

Code scanning / CodeQL

Expression injection in Actions

Potential injection from the ${{{{ github.event.head_commit.author.name }}}}, which may be controlled by an external user. Potential injection from the ${{{{ github.event.head_commit.author.email }}}}, which may be controlled by an external user.
- name: go-mod-tidy and make package-specs
shell: bash
run: |
joergdw marked this conversation as resolved.
Show resolved Hide resolved
joergdw marked this conversation as resolved.
Show resolved Hide resolved
joergdw marked this conversation as resolved.
Show resolved Hide resolved
joergdw marked this conversation as resolved.
Show resolved Hide resolved
joergdw marked this conversation as resolved.
Show resolved Hide resolved
joergdw marked this conversation as resolved.
Show resolved Hide resolved
joergdw marked this conversation as resolved.
Show resolved Hide resolved
#! /usr/bin/env bash
set -eu -o pipefail

# We need the subsequent standard-message to determine if the last commit
# has already cleaned up everything. In this case this workflow should not
# change anything and we exit early.
# An alternative would be to use a tag for this. But this does affect the whole
# PR instead of just the latest commit.
declare -r tidy_message='🤖🦾🛠️ go mod tidy & make package-specs'

declare -r commit_author_name="${{github.event.head_commit.author.name}}"
declare -r commit_message="${{github.event.head_commit.message}}"

if [[ ! "${commit_author_name}" =~ ('dependabot'|'renovate')'[bot]' ]] \
|| [[ "${commit_message}" == "${tidy_message}" ]]
then
echo 'This commit was not by a known bot or already an automatic `go mod tidy`! Exiting …'
exit 0
fi

declare -r current_branch="$(git branch --show-current)"
git checkout 'HEAD~1'
pushd './src/acceptance/assets/app/go_app'
make generate
popd
git checkout "${current_branch}"

make package-specs

declare -i -r changed_files=$(git status --porcelain | wc --lines)
if ((changed_files >= 0))
then
git add .
git commit --message="${tidy_message}"
git push
fi

Check failure

Code scanning / CodeQL

Expression injection in Actions

Potential injection from the ${{{{ github.event.head_commit.author.name }}}}, which may be controlled by an external user. Potential injection from the ${{{{ github.event.head_commit.message }}}}, which may be controlled by an external user.
33 changes: 18 additions & 15 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ db_type := postgres
DB_HOST := localhost
DBURL := $(shell case "${db_type}" in\
(postgres) printf "postgres://postgres:postgres@${DB_HOST}/autoscaler?sslmode=disable"; ;; \
(mysql) printf "root@tcp(${DB_HOST})/autoscaler?tls=false"; ;; esac)
(mysql) printf "root@tcp(${DB_HOST})/autoscaler?tls=false"; ;; esac)
DEBUG := false
MYSQL_TAG := 8
POSTGRES_TAG := 12
Expand Down Expand Up @@ -162,7 +162,7 @@ target/start-db-postgres_CI_true:
waitfor_postgres_CI_false:
@echo -n " - waiting for ${db_type} ."
@COUNTER=0; until $$(docker exec postgres pg_isready &>/dev/null) || [ $$COUNTER -gt 10 ]; do echo -n "."; sleep 1; let COUNTER+=1; done;\
if [ $$COUNTER -gt 10 ]; then echo; echo "Error: timed out waiting for postgres. Try \"make clean\" first." >&2 ; exit 1; fi
if [ $$COUNTER -gt 10 ]; then echo; echo "Error: timed out waiting for postgres. Try \"make clean\" first." >&2 ; exit 1; fi
waitfor_postgres_CI_true:
@echo " - no ci postgres checks"

Expand Down Expand Up @@ -193,14 +193,14 @@ waitfor_mysql_CI_false:
waitfor_mysql_CI_true:
@echo -n " - Waiting for table creation"
@which mysql >/dev/null &&\
{\
T=0;\
until [[ ! -z "$(shell mysql -u "root" -h "${DB_HOST}" --port=3306 -qfsBe "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='autoscaler'" 2> /dev/null)" ]]\
|| [[ $${T} -gt 30 ]];\
do echo -n "."; sleep 1; T=$$((T+1)); done;\
}
{\
T=0;\
until [[ ! -z "$(shell mysql -u "root" -h "${DB_HOST}" --port=3306 -qfsBe "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='autoscaler'" 2> /dev/null)" ]]\
|| [[ $${T} -gt 30 ]];\
do echo -n "."; sleep 1; T=$$((T+1)); done;\
}
@[ ! -z "$(shell mysql -u "root" -h "${DB_HOST}" --port=3306 -qfsBe "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='autoscaler'" 2> /dev/null)" ]\
|| { echo "ERROR: Mysql timed out creating database"; exit 1; }
|| { echo "ERROR: Mysql timed out creating database"; exit 1; }


.PHONY: stop-db
Expand Down Expand Up @@ -256,23 +256,26 @@ acceptance-release: clean-acceptance mod-tidy vendor build-test-app
@tar --create --auto-compress --directory="src" --file="${ACCEPTANCE_TESTS_FILE}" 'acceptance'
.PHONY: mod-tidy
mod-tidy:
@for folder in $$(find . -maxdepth 3 -name "go.mod" -exec dirname {} \;);\
do\
cd $${folder}; echo " - go mod tidying '$${folder}'"; go mod tidy; cd - >/dev/null;\
@for folder in $$(find . -maxdepth 6 -name 'go.mod' -exec dirname {} \;) ;\
do \
pushd "$${folder}" ;\
echo " - go mod tidying '$${folder}'" ;\
go mod tidy ;\
popd ;\
done

.PHONY: mod-download
mod-download:
@for folder in $$(find . -maxdepth 3 -name "go.mod" -exec dirname {} \;);\
do\
cd $${folder}; echo " - go mod download '$${folder}'"; go mod download; cd - >/dev/null;\
cd $${folder}; echo " - go mod download '$${folder}'"; go mod download; cd - >/dev/null;\
done

.PHONY: vendor
vendor:
@for folder in $$(find . -maxdepth 3 -name "go.mod" -exec dirname {} \;);\
do\
cd $${folder}; echo " - go mod vendor '$${folder}'"; go mod vendor; cd - >/dev/null;\
cd $${folder}; echo " - go mod vendor '$${folder}'"; go mod vendor; cd - >/dev/null;\
done

.PHONY: fakes
Expand Down Expand Up @@ -396,4 +399,4 @@ docker-image: docker-login
validate-openapi-specs: $(wildcard ./api/*.openapi.yaml)
for file in $^ ; do \
swagger-cli validate "$${file}" ; \
done
done
10 changes: 7 additions & 3 deletions flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,11 @@
hardeningDisable = [ "fortify" ];

shellHook = ''
echo -ne '\033[1;33m' '\033[5m'
aes_terminal_font_yellow='\e[38;2;255;255;0m'
aes_terminal_font_blink='\e[5m'
aes_terminal_reset='\e[0m'

echo -ne "$aes_terminal_font_yellow" "$aes_terminal_font_blink"
cat << 'EOF'
⚠️ If `whoami` does not work properly on your computer, `bosh ssh` commands may fail.
The solution is, to provide your nix dev-shell the path to the `libnss_sss.so.2` of
Expand All @@ -72,9 +76,9 @@
Adapt the following line to contain the correct path:
export LD_PRELOAD='/lib/x86_64-linux-gnu/libnss_sss.so.2'
EOF
echo -ne '\033[0m'
echo -ne "$aes_terminal_reset"
'';
};
});
};
}
}
123 changes: 80 additions & 43 deletions src/acceptance/assets/app/go_app/Makefile
Original file line number Diff line number Diff line change
@@ -1,75 +1,112 @@
.ONESHELL:
SHELL := /bin/bash
.SHELLFLAGS := -eu -o pipefail -c ${SHELLFLAGS}
aes_terminal_font_yellow := \e[38;2;255;255;0m
aes_terminal_reset := \e[0m


# TODO: Do we need the next line?
MAKEFLAGS=
GO_VERSION := $(shell go version | sed -e 's/^[^0-9.]*\([0-9.]*\).*/\1/')
GO_DEPENDENCIES := $(shell find . -type f -name '*.go')
PACKAGE_DIRS := $(shell go list ./... | grep -v /vendor/ | grep -v e2e)
CGO_ENABLED = 0
export GOWORK=off

binaries=$(shell find . -name "main.go" -exec dirname {} \; | cut -d/ -f2 | sort | uniq)
test_dirs=$(shell find . -name "*_test.go" -exec dirname {} \; | cut -d/ -f2 | sort | uniq)
GO_VERSION = $(shell go version | sed --expression='s/^[^0-9.]*\([0-9.]*\).*/\1/')
GO_DEPENDENCIES = $(shell find . -type f -name '*.go')
PACKAGE_DIRS = $(shell go list ./... | grep --invert-match /vendor/ | grep --invert-match e2e)
CGO_ENABLED := 0
export GOWORK := off

binaries = $(shell find . -name "main.go" -exec dirname {} \; \
| cut --delimiter='/' --fields='2' | sort | uniq)
test_dirs = $(shell find . -name "*_test.go" -exec dirname {} \; \
| cut --delimiter='/' --fields='2' | sort | uniq)

GINKGO_OPTS :=-r --race --require-suite --randomize-all --cover ${OPTS}
export CONFIG ?= ../../../../acceptance_config.json

GINKGO_VERSION = v$(shell cat ../../../../../.tool-versions | grep --regexp='ginkgo' | cut --delimiter=' ' --fields='2')
GOLANGCI_LINT_VERSION = v$(shell cat ../../../../../.tool-versions | grep --regexp='golangci-lint' | cut --delimiter=' ' --fields='2')



openapi-spec-path := ../../../../../api
openapi-specs-list := $(wildcard ${openapi-spec-path}/*.openapi.yaml)
appfakes-path := ./internal/app/appfakes

.PHONY: generate-fakes
generate-fakes: ${appfakes-path} $(wildcard ${appfakes-path}/*.go)
${appfakes-path} $(wildcard ${appfakes-path}/*.go) &: ./internal/generate.go ${openapi-specs-list}
@echo -ne '${aes_terminal_font_yellow}'
@echo -e '⚠️ The client-fakes generated from the openapi-specification depend on\n' \
'the files ./go.mod and ./go.sum. This has not been reflected in this\n' \
'make-target to avoid cyclic dependencies because `go mod tidy`, which\n' \
'modifies both files, depends itself on the client-fakes.'
@echo -ne '${aes_terminal_reset}'
go generate ./...



.PHONY: go-mod-tidy
go-mod-tidy: ${appfakes-path} $(wildcard ${appfakes-path}/*.go)
go mod tidy

GINKGO_OPTS=-r --race --require-suite --randomize-all --cover ${OPTS}
export CONFIG?=../../../../acceptance_config.json

GINKGO_VERSION=v$(shell cat ../../../../../.tool-versions | grep ginkgo | cut -d " " -f 2 )
GOLANGCI_LINT_VERSION=v$(shell cat ../../../../../.tool-versions | grep golangci-lint | cut -d " " -f 2 )

.PHONY: build
build: generate
echo "# building test-app"
rm -rf build/* || true
mkdir -p build/
CGO_ENABLED='${CGO_ENABLED}' GOOS='linux' GOARCH='amd64' go build -o build/app
cp app_manifest.yml build/manifest.yml
build: ./build/app ./build/manifest.yml
./build/app ./build/manifest.yml: ./go.mod ${appfakes-path} $(wildcard ${appfakes-path}/*.go)
echo '# building test-app'
mkdir -p build
CGO_ENABLED='${CGO_ENABLED}' GOOS='linux' GOARCH='amd64' go build -o './build/app'
cp './app_manifest.yml' './build/manifest.yml'



.PHONY: build_tests
build_tests: $(addprefix build_test-,$(test_dirs))

build_test-%:
build_test-%: ${appfakes-path} $(wildcard ${appfakes-path}/*.go)
@echo " - building '$*' tests"
@export build_folder=${PWD}/build/tests/$* &&\
mkdir -p $${build_folder} &&\
cd $* &&\
for package in $$( go list ./... | sed 's|.*/autoscaler/$*|.|' | awk '{ print length, $$0 }' | sort -n -r | cut -d" " -f2- );\
do\
export test_file=$${build_folder}/$${package}.test;\
echo " - compiling $${package} to $${test_file}";\
go test -c -o $${test_file} $${package};\
done;

@export build_folder='${PWD}/build/tests/$*'
@mkdir -p '$${build_folder}'
cd $*
for package in $$(go list './...' | sed 's|.*/autoscaler/$*|.|' | awk '{ print length, $$0 }' | \
sort --numeric-sort --reverse | cut --delimiter=' ' --fields='2-')
do
export test_file="$${build_folder}/$${package}.test"
echo " - compiling $${package} to $${test_file}"
go test -c -o "$${test_file}" "$${package}"
done


.PHONY: check lint lint-fix test
check: lint build test
test: generate
@echo "Running tests"
go run github.com/onsi/ginkgo/v2/ginkgo@${GINKGO_VERSION} run ${GINKGO_OPTS} ./...

lint: generate
@go run github.com/golangci/golangci-lint/cmd/golangci-lint@${GOLANGCI_LINT_VERSION} run
test: generate-fakes
@echo 'Running tests'
go run 'github.com/onsi/ginkgo/v2/ginkgo@${GINKGO_VERSION}' run ${GINKGO_OPTS} './...'

lint: generate-fakes
@go run 'github.com/golangci/golangci-lint/cmd/golangci-lint@${GOLANGCI_LINT_VERSION}' run

lint-fix: generate-fakes
go run 'github.com/golangci/golangci-lint/cmd/golangci-lint@${GOLANGCI_LINT_VERSION}' run --fix


lint-fix: generate
go run github.com/golangci/golangci-lint/cmd/golangci-lint@${GOLANGCI_LINT_VERSION} run --fix

.PHONY: start
start: build
docker run -it --name go_app -v $$PWD/build/:/cf/bin -p 8080:8080 --rm cloudfoundry/cflinuxfs4 /cf/bin/app
docker run --interactive --tty --name go_app --volume="$${PWD}/build/:/cf/bin" \
--publish '8080:8080' --rm 'cloudfoundry/cflinuxfs4' '/cf/bin/app'



.PHONY: deploy
deploy: build
./deploy.sh

openapi-spec-path := ../../../../../api
openapi-specs-list := $(wildcard ${openapi-spec-path}/*.openapi.yaml)

generate: go.mod ${openapi-specs-list}
go generate ./...

.PHONY: clean
clean:
@echo "# cleaning autoscaler"
@go clean -cache -testcache
@rm -rf build
@rm -rf internal/app/appfakes
@rm --force --recursive './build'
@rm --force --recursive './internal/app/appfakes'