diff --git a/.github/workflows/bosh-release-checks.yaml b/.github/workflows/bosh-release-checks.yaml index bb6a139e58..1048639cc8 100644 --- a/.github/workflows/bosh-release-checks.yaml +++ b/.github/workflows/bosh-release-checks.yaml @@ -46,7 +46,7 @@ jobs: run: | set -e export PATH=$PATH:/usr/local/maven/bin - make mod-tidy vendor db scheduler + make go-mod-tidy vendor db scheduler - name: Build Dev Release id: build diff --git a/.github/workflows/dependency-update.go-mod-tidy_and_make-package-specs.yaml b/.github/workflows/dependency-update.go-mod-tidy_and_make-package-specs.yaml new file mode 100644 index 0000000000..62e46359a0 --- /dev/null +++ b/.github/workflows/dependency-update.go-mod-tidy_and_make-package-specs.yaml @@ -0,0 +1,85 @@ +name: "renovate-/ospo-bot: go mod tidy and make package-specs" +on: + push: + branches: + - "renovate/**" + - "dependabot/**" + paths: + - ".github/workflows/dependency-update-go-mod-tidy.yaml" + - "./src/**/go.mod" + - "./src/**/go.sum" +jobs: + go-mod-tidy: + runs-on: ubuntu-latest + permissions: + pull-requests: write + contents: write + steps: + # We potentially want to add at the end a commit by the author of the most recent + # commit in this branch. However github has some protection which prevents workflows + # to run in case a commit has been pushed with the default job-specific github-token. + # For this case we need to use another one here. + # + # For more information, see: + # + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: true + token: ${{ secrets.APP_AUTOSCALER_CI_TOKEN }} # With push token that can trigger new PR jobs + - name: Configure git + shell: bash + run: | + #! /usr/bin/env bash + set -eu -o pipefail + + declare -r commit_author_name="${{github.event.head_commit.author.name}}" + declare -r commit_author_email="${{github.event.head_commit.author.email}}" + + git config --global --add safe.directory "${GITHUB_WORKSPACE}" + git config user.name "${commit_author_name}" + git config user.email "${commit_author_email}" + - name: go-mod-tidy and make package-specs + shell: bash + run: | + #! /usr/bin/env bash + set -eu -o pipefail + + # We need the subsequent standard-message to determine if the last commit + # has already cleaned up everything. In this case this workflow should not + # change anything and we exit early. + # An alternative would be to use a tag for this. But this does affect the whole + # PR instead of just the latest commit. + declare -r tidy_message='🤖🦾🛠️ go mod tidy & make package-specs' + + declare -r commit_author_name="${{github.event.head_commit.author.name}}" + declare -r commit_message="${{github.event.head_commit.message}}" + + if [[ ! "${commit_author_name}" =~ ('dependabot'|'renovate')'[bot]' ]] \ + || [[ "${commit_message}" == "${tidy_message}" ]] + then + echo 'This commit was not by a known bot or already an automatic `go mod tidy`! Exiting …' + exit 0 + fi + + # Generated files are needed for `go mod tidy` which is a dependency of the + # target `package-specs`. However the generation of them itself already + # requires go-modules to be tidied up. So we need to generate the files + # before changing `go.mod` and `go.sum`. + declare -r current_branch="$(git branch --show-current)" + git checkout 'HEAD~1' + make --directory='./src/acceptance/assets/app/go_app' generate-fakes + make --directory='./src/autoscaler' generate-fakes + git checkout "${current_branch}" + + # ⚠️ For this workflow to be successful, the subsequent line must not + # trigger again the creation of the generated files. + make package-specs + + declare -i -r changed_files=$(git status --porcelain | wc --lines) + if ((changed_files >= 0)) + then + git add . + git commit --message="${tidy_message}" + git push + fi diff --git a/.github/workflows/tidy-go-mod.yaml b/.github/workflows/tidy-go-mod.yaml index 6bf8f7b984..ed1c27c028 100644 --- a/.github/workflows/tidy-go-mod.yaml +++ b/.github/workflows/tidy-go-mod.yaml @@ -19,7 +19,7 @@ jobs: run: git config --global --add safe.directory "${GITHUB_WORKSPACE}" - name: run go mod tidy on all .mod's - run: make mod-tidy + run: make go-mod-tidy - name: Check if there is any change id: get_changes diff --git a/Makefile b/Makefile index e9a41daf09..7b480c7ad9 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,17 @@ SHELL := /bin/bash .SHELLFLAGS := -eu -o pipefail -c ${SHELLFLAGS} MAKEFLAGS = -s + go_modules := $(shell find . -maxdepth 3 -name "*.mod" -exec dirname {} \; | sed 's|\./src/||' | sort) all_modules := $(go_modules) db scheduler + MVN_OPTS = "-Dmaven.test.skip=true" OS := $(shell . /etc/lsb-release &>/dev/null && echo $${DISTRIB_ID} || uname) db_type := postgres DB_HOST := localhost DBURL := $(shell case "${db_type}" in\ (postgres) printf "postgres://postgres:postgres@${DB_HOST}/autoscaler?sslmode=disable"; ;; \ - (mysql) printf "root@tcp(${DB_HOST})/autoscaler?tls=false"; ;; esac) + (mysql) printf "root@tcp(${DB_HOST})/autoscaler?tls=false"; ;; esac) DEBUG := false MYSQL_TAG := 8 POSTGRES_TAG := 12 @@ -162,7 +164,7 @@ target/start-db-postgres_CI_true: waitfor_postgres_CI_false: @echo -n " - waiting for ${db_type} ." @COUNTER=0; until $$(docker exec postgres pg_isready &>/dev/null) || [ $$COUNTER -gt 10 ]; do echo -n "."; sleep 1; let COUNTER+=1; done;\ - if [ $$COUNTER -gt 10 ]; then echo; echo "Error: timed out waiting for postgres. Try \"make clean\" first." >&2 ; exit 1; fi + if [ $$COUNTER -gt 10 ]; then echo; echo "Error: timed out waiting for postgres. Try \"make clean\" first." >&2 ; exit 1; fi waitfor_postgres_CI_true: @echo " - no ci postgres checks" @@ -193,14 +195,14 @@ waitfor_mysql_CI_false: waitfor_mysql_CI_true: @echo -n " - Waiting for table creation" @which mysql >/dev/null &&\ - {\ - T=0;\ - until [[ ! -z "$(shell mysql -u "root" -h "${DB_HOST}" --port=3306 -qfsBe "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='autoscaler'" 2> /dev/null)" ]]\ - || [[ $${T} -gt 30 ]];\ - do echo -n "."; sleep 1; T=$$((T+1)); done;\ - } + {\ + T=0;\ + until [[ ! -z "$(shell mysql -u "root" -h "${DB_HOST}" --port=3306 -qfsBe "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='autoscaler'" 2> /dev/null)" ]]\ + || [[ $${T} -gt 30 ]];\ + do echo -n "."; sleep 1; T=$$((T+1)); done;\ + } @[ ! -z "$(shell mysql -u "root" -h "${DB_HOST}" --port=3306 -qfsBe "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='autoscaler'" 2> /dev/null)" ]\ - || { echo "ERROR: Mysql timed out creating database"; exit 1; } + || { echo "ERROR: Mysql timed out creating database"; exit 1; } .PHONY: stop-db @@ -243,36 +245,57 @@ spec-test: bundle exec rspec .PHONY: release -bosh-release: mod-tidy vendor scheduler db build/autoscaler-test.tgz +bosh-release: go-mod-tidy vendor scheduler db build/autoscaler-test.tgz build/autoscaler-test.tgz: @echo " - building bosh release into build/autoscaler-test.tgz" @mkdir -p build @bosh create-release --force --timestamp-version --tarball=build/autoscaler-test.tgz .PHONY: acceptance-release -acceptance-release: clean-acceptance mod-tidy vendor build-test-app +acceptance-release: clean-acceptance go-mod-tidy vendor build-test-app @echo " - building acceptance test release '${VERSION}' to dir: '${DEST}' " @mkdir -p ${DEST} @tar --create --auto-compress --directory="src" --file="${ACCEPTANCE_TESTS_FILE}" 'acceptance' -.PHONY: mod-tidy -mod-tidy: - @for folder in $$(find . -maxdepth 3 -name "go.mod" -exec dirname {} \;);\ - do\ - cd $${folder}; echo " - go mod tidying '$${folder}'"; go mod tidy; cd - >/dev/null;\ - done + + + +.PHONY: go-mod-tidy +go-mod-tidy: acceptance.go-mod-tidy autoscaler.go-mod-tidy changelog.go-mod-tidy \ + changeloglockcleander.go-mod-tidy test-app.go-mod-tidy + +go-acceptance-dir := ./src/acceptance +go-autoscaler-dir := ./src/autoscaler +go-changelog-dir := ./src/changelog +go-changeloglockcleander-dir := ./src/changeloglockcleaner +go-test-app-dir := ./src/acceptance/assets/app/go_app + +.PHONY: acceptance.go-mod-tidy autoscaler.go-mod-tidy changelog.go-mod-tidy \ + changeloglockcleander.go-mod-tidy test-app.go-mod-tidy +acceptance.go-mod-tidy: + make --directory='${go-acceptance-dir}' go-mod-tidy +autoscaler.go-mod-tidy: + make --directory='${go-autoscaler-dir}' go-mod-tidy +changelog.go-mod-tidy: + make --directory='${go-changelog-dir}' go-mod-tidy +changeloglockcleander.go-mod-tidy: + make --directory='${go-changeloglockcleander-dir}' go-mod-tidy +test-app.go-mod-tidy: + make --directory='${go-test-app-dir}' go-mod-tidy + + .PHONY: mod-download mod-download: @for folder in $$(find . -maxdepth 3 -name "go.mod" -exec dirname {} \;);\ do\ - cd $${folder}; echo " - go mod download '$${folder}'"; go mod download; cd - >/dev/null;\ + cd $${folder}; echo " - go mod download '$${folder}'"; go mod download; cd - >/dev/null;\ done .PHONY: vendor vendor: @for folder in $$(find . -maxdepth 3 -name "go.mod" -exec dirname {} \;);\ do\ - cd $${folder}; echo " - go mod vendor '$${folder}'"; go mod vendor; cd - >/dev/null;\ + cd $${folder}; echo " - go mod vendor '$${folder}'"; go mod vendor; cd - >/dev/null;\ done .PHONY: fakes @@ -295,7 +318,7 @@ markdownlint-cli: .PHONY: deploy deploy-autoscaler deploy-register-cf deploy-autoscaler-bosh deploy-cleanup deploy-autoscaler: deploy -deploy: mod-tidy vendor uaac db scheduler deploy-autoscaler-bosh deploy-register-cf +deploy: go-mod-tidy vendor uaac db scheduler deploy-autoscaler-bosh deploy-register-cf deploy-register-cf: echo " - registering broker with cf" [ "$${BUILDIN_MODE}" == "false" ] && { ${CI_DIR}/autoscaler/scripts/register-broker.sh; } || echo " - Not registering broker due to buildin mode enabled" @@ -367,7 +390,7 @@ run-performance: run-act: ${AUTOSCALER_DIR}/scripts/run_act.sh;\ -package-specs: mod-tidy vendor +package-specs: go-mod-tidy vendor @echo " - Updating the package specs" @./scripts/sync-package-specs @@ -396,4 +419,4 @@ docker-image: docker-login validate-openapi-specs: $(wildcard ./api/*.openapi.yaml) for file in $^ ; do \ swagger-cli validate "$${file}" ; \ - done \ No newline at end of file + done diff --git a/flake.nix b/flake.nix index 71225e69a8..01f0cdf312 100644 --- a/flake.nix +++ b/flake.nix @@ -63,7 +63,11 @@ hardeningDisable = [ "fortify" ]; shellHook = '' - echo -ne '\033[1;33m' '\033[5m' + aes_terminal_font_yellow='\e[38;2;255;255;0m' + aes_terminal_font_blink='\e[5m' + aes_terminal_reset='\e[0m' + + echo -ne "$aes_terminal_font_yellow" "$aes_terminal_font_blink" cat << 'EOF' ⚠️ If `whoami` does not work properly on your computer, `bosh ssh` commands may fail. The solution is, to provide your nix dev-shell the path to the `libnss_sss.so.2` of @@ -72,9 +76,9 @@ Adapt the following line to contain the correct path: export LD_PRELOAD='/lib/x86_64-linux-gnu/libnss_sss.so.2' EOF - echo -ne '\033[0m' + echo -ne "$aes_terminal_reset" ''; }; }); }; -} \ No newline at end of file +} diff --git a/src/acceptance/Makefile b/src/acceptance/Makefile index 71357933d0..cf287d533f 100644 --- a/src/acceptance/Makefile +++ b/src/acceptance/Makefile @@ -1,20 +1,32 @@ SHELL := /bin/bash -.SHELLFLAGS = -euo pipefail -c -MAKEFLAGS = -s +.SHELLFLAGS := -eu -o pipefail -c +MAKEFLAGS := -s + GO := GO111MODULE=on GO15VENDOREXPERIMENT=1 go GO_NOMOD := GO111MODULE=off go -GO_VERSION := $(shell $(GO) version | sed -e 's/^[^0-9.]*\([0-9.]*\).*/\1/') -GO_DEPENDENCIES := $(shell find . -type f -name '*.go') -PACKAGE_DIRS := $(shell $(GO) list ./... | grep -v /vendor/ | grep -v e2e) -CGO_ENABLED = 0 -export GOWORK=off -BUILDTAGS := +GO_VERSION = $(shell go version | sed --expression='s/^[^0-9.]*\([0-9.]*\).*/\1/') +GO_DEPENDENCIES = $(shell find . -type f -name '*.go') +PACKAGE_DIRS = $(shell go list './...' | grep --invert-match --regexp='/vendor/' \ + | grep --invert-match --regexp='e2e') + +CGO_ENABLED := 0 +export GOWORK := off + +GINKGO_VERSION = v$(shell cat '../../.tool-versions' | grep --regexp='ginkgo' \ + | cut --delimiter=' ' --fields='2') +GOLANGCI_LINT_VERSION = v$(shell cat '../../.tool-versions' | grep --regexp='golangci-lint' \ + | cut --delimiter=' ' --fields='2' ) + + +.PHONY: go-mod-tidy +go-mod-tidy: + go mod tidy + -GINKGO_VERSION=v$(shell cat ../../.tool-versions | grep ginkgo | cut -d " " -f 2 ) -GOLANGCI_LINT_VERSION=v$(shell cat ../../.tool-versions | grep golangci-lint | cut -d " " -f 2 ) -test_dirs=$(shell find . -name "*_test.go" -exec dirname {} \; | cut -d/ -f2 | sort | uniq) +test_dirs = $(shell find . -name "*_test.go" -exec dirname {} \; | cut --delimiter='/' --fields='2' \ + | sort | uniq) build_tests: $(addprefix build_test-,$(test_dirs)) build_test-%: @echo " - building '$*' tests" @@ -23,9 +35,9 @@ build_test-%: cd $* &&\ for package in $$( go list ./... | sed 's|.*/autoscaler/$*|.|' | awk '{ print length, $$0 }' | sort -n -r | cut -d" " -f2- );\ do\ - export test_file=$${build_folder}/$${package}.test;\ - echo " - compiling $${package} to $${test_file}";\ - go test -c -o $${test_file} $${package};\ + export test_file=$${build_folder}/$${package}.test;\ + echo " - compiling $${package} to $${test_file}";\ + go test -c -o $${test_file} $${package};\ done; check: lint build_tests diff --git a/src/acceptance/assets/app/go_app/Makefile b/src/acceptance/assets/app/go_app/Makefile index 78ba8a95f4..74ccec8c02 100644 --- a/src/acceptance/assets/app/go_app/Makefile +++ b/src/acceptance/assets/app/go_app/Makefile @@ -1,75 +1,128 @@ .ONESHELL: SHELL := /bin/bash .SHELLFLAGS := -eu -o pipefail -c ${SHELLFLAGS} +aes_terminal_font_yellow := \e[38;2;255;255;0m +aes_terminal_reset := \e[0m +# TODO: Do we need the next line? +MAKEFLAGS = -MAKEFLAGS= -GO_VERSION := $(shell go version | sed -e 's/^[^0-9.]*\([0-9.]*\).*/\1/') -GO_DEPENDENCIES := $(shell find . -type f -name '*.go') -PACKAGE_DIRS := $(shell go list ./... | grep -v /vendor/ | grep -v e2e) -CGO_ENABLED = 0 -export GOWORK=off +GO_VERSION = $(shell go version | sed --expression='s/^[^0-9.]*\([0-9.]*\).*/\1/') +GO_DEPENDENCIES = $(shell find . -type f -name '*.go') +PACKAGE_DIRS = $(shell go list './...' | grep --invert-match --regexp='/vendor/' \ + | grep --invert-match --regexp='e2e') +CGO_ENABLED := 0 +export GOWORK := off -binaries=$(shell find . -name "main.go" -exec dirname {} \; | cut -d/ -f2 | sort | uniq) -test_dirs=$(shell find . -name "*_test.go" -exec dirname {} \; | cut -d/ -f2 | sort | uniq) +binaries = $(shell find . -name "main.go" -exec dirname {} \; \ + | cut --delimiter='/' --fields='2' | sort | uniq) +test_dirs = $(shell find . -name "*_test.go" -exec dirname {} \; \ + | cut --delimiter='/' --fields='2' | sort | uniq) + +GINKGO_OPTS :=-r --race --require-suite --randomize-all --cover ${OPTS} +export CONFIG ?= ../../../../acceptance_config.json + +GINKGO_VERSION = v$(shell cat ../../../../../.tool-versions | grep --regexp='ginkgo' | cut --delimiter=' ' --fields='2') +GOLANGCI_LINT_VERSION = v$(shell cat ../../../../../.tool-versions | grep --regexp='golangci-lint' | cut --delimiter=' ' --fields='2') + + + +openapi-spec-path := ../../../../../api +openapi-specs-list := $(wildcard ${openapi-spec-path}/*.openapi.yaml) + +# The presence of the subsequent directory indicates wheather the fakes still need to be generated +# or not. +app-fakes-dir := ./internal/app/appfakes +app-fakes-files := $(wildcard ${app-fakes-dir}/*.go) + +.PHONY: generate-fakes +generate-fakes: ${app-fakes-dir} ${app-fakes-files} +${app-fakes-dir} ${app-fakes-files} &: ./go.mod ./go.sum ./internal/generate.go ${openapi-specs-list} + go generate ./... + + + +# The generated fakes are order-only dependencies for `go-mod-tidy`. The reason is, that for +# `go-mod-tidy` the generated fakes need to be present but fortunately not necessarily +# up-to-date. This is fortunate because the generation of the fake requires the files `go.mod` and +# `go.sum` to be already tidied up, introducing a cyclic dependency otherwise. But that would make +# any modification to `go.mod` or `go.sum` impossible. This definition now makes it possible to +# update `go.mod` and `go.sum` as follows: +# 1. `make generate-fakes` +# 2. Update `go.mod` and/or `go.sum` +# 3. `make go-mod-tidy` +# 4. Optionally: `make generate-fakes` to update the fakes as well. +# +# For more information on order-only dependencies, see: +# +.PHONY: go-mod-tidy +go-mod-tidy: ./go.mod ./go.sum ${GO_DEPENDENCIES} | ${app-fakes-dir} ${app-fakes-files} + @echo -ne '${aes_terminal_font_yellow}' + @echo -e '⚠️ Warning: The client-fakes generated from the openapi-specification may be\n' \ + 'outdated. Please consider re-generating them, if this is relevant.' + @echo -ne '${aes_terminal_reset}' + go mod tidy -GINKGO_OPTS=-r --race --require-suite --randomize-all --cover ${OPTS} -export CONFIG?=../../../../acceptance_config.json -GINKGO_VERSION=v$(shell cat ../../../../../.tool-versions | grep ginkgo | cut -d " " -f 2 ) -GOLANGCI_LINT_VERSION=v$(shell cat ../../../../../.tool-versions | grep golangci-lint | cut -d " " -f 2 ) .PHONY: build -build: generate - echo "# building test-app" - rm -rf build/* || true - mkdir -p build/ - CGO_ENABLED='${CGO_ENABLED}' GOOS='linux' GOARCH='amd64' go build -o build/app - cp app_manifest.yml build/manifest.yml +build: ./build/app ./build/manifest.yml +./build/app ./build/manifest.yml: ./go.mod ${app-fakes-dir} ${app-fakes-files} + echo '# building test-app' + mkdir -p build + CGO_ENABLED='${CGO_ENABLED}' GOOS='linux' GOARCH='amd64' go build -o './build/app' + cp './app_manifest.yml' './build/manifest.yml' + +.PHONY: build_tests build_tests: $(addprefix build_test-,$(test_dirs)) -build_test-%: +build_test-%: ${app-fakes-dir} ${app-fakes-files} @echo " - building '$*' tests" - @export build_folder=${PWD}/build/tests/$* &&\ - mkdir -p $${build_folder} &&\ - cd $* &&\ - for package in $$( go list ./... | sed 's|.*/autoscaler/$*|.|' | awk '{ print length, $$0 }' | sort -n -r | cut -d" " -f2- );\ - do\ - export test_file=$${build_folder}/$${package}.test;\ - echo " - compiling $${package} to $${test_file}";\ - go test -c -o $${test_file} $${package};\ - done; - + @export build_folder='${PWD}/build/tests/$*' + @mkdir -p "$${build_folder}" + cd $* + for package in $$(go list './...' | sed 's|.*/autoscaler/$*|.|' | awk '{ print length, $$0 }' | \ + sort --numeric-sort --reverse | cut --delimiter=' ' --fields='2-') + do + export test_file="$${build_folder}/$${package}.test" + echo " - compiling $${package} to $${test_file}" + go test -c -o "$${test_file}" "$${package}" + done + + +.PHONY: check lint lint-fix test check: lint build test -test: generate - @echo "Running tests" - go run github.com/onsi/ginkgo/v2/ginkgo@${GINKGO_VERSION} run ${GINKGO_OPTS} ./... -lint: generate - @go run github.com/golangci/golangci-lint/cmd/golangci-lint@${GOLANGCI_LINT_VERSION} run +test: generate-fakes + @echo 'Running tests' + go run 'github.com/onsi/ginkgo/v2/ginkgo@${GINKGO_VERSION}' run ${GINKGO_OPTS} './...' + +lint: generate-fakes + @go run 'github.com/golangci/golangci-lint/cmd/golangci-lint@${GOLANGCI_LINT_VERSION}' run + +lint-fix: generate-fakes + go run 'github.com/golangci/golangci-lint/cmd/golangci-lint@${GOLANGCI_LINT_VERSION}' run --fix + -lint-fix: generate - go run github.com/golangci/golangci-lint/cmd/golangci-lint@${GOLANGCI_LINT_VERSION} run --fix .PHONY: start start: build - docker run -it --name go_app -v $$PWD/build/:/cf/bin -p 8080:8080 --rm cloudfoundry/cflinuxfs4 /cf/bin/app + docker run --interactive --tty --name go_app --volume="$${PWD}/build/:/cf/bin" \ + --publish '8080:8080' --rm 'cloudfoundry/cflinuxfs4' '/cf/bin/app' + + .PHONY: deploy deploy: build ./deploy.sh -openapi-spec-path := ../../../../../api -openapi-specs-list := $(wildcard ${openapi-spec-path}/*.openapi.yaml) -generate: go.mod ${openapi-specs-list} - go generate ./... .PHONY: clean clean: @echo "# cleaning autoscaler" @go clean -cache -testcache - @rm -rf build - @rm -rf internal/app/appfakes \ No newline at end of file + @rm --force --recursive './build' + @rm --force --recursive './internal/app/appfakes' diff --git a/src/autoscaler/.gitignore b/src/autoscaler/.gitignore index 378eac25d3..27eecdf61d 100644 --- a/src/autoscaler/.gitignore +++ b/src/autoscaler/.gitignore @@ -1 +1,2 @@ -build +build/ +fakes/ diff --git a/src/autoscaler/Makefile b/src/autoscaler/Makefile index dcebe611d1..c12357694a 100644 --- a/src/autoscaler/Makefile +++ b/src/autoscaler/Makefile @@ -1,11 +1,16 @@ SHELL := /bin/bash -.SHELLFLAGS = -euo pipefail -c -MAKEFLAGS = -s -GO_VERSION := $(shell go version | sed -e 's/^[^0-9.]*\([0-9.]*\).*/\1/') -GO_DEPENDENCIES := $(shell find . -type f -name '*.go') -PACKAGE_DIRS := $(shell go list ./... | grep -v /vendor/ | grep -v e2e) - -CGO_ENABLED = 1 # This is set to enforce dynamic linking which is a requirement of dynatrace. +.SHELLFLAGS := -eu -o pipefail -c +MAKEFLAGS := -s +aes_terminal_font_yellow := \e[38;2;255;255;0m +aes_terminal_reset := \e[0m + +GO_VERSION = $(shell go version | sed --expression='s/^[^0-9.]*\([0-9.]*\).*/\1/') +GO_DEPENDENCIES = $(shell find . -type f -name '*.go') +PACKAGE_DIRS = $(shell go list './...' | grep --invert-match --regexp='/vendor/' \ + | grep --invert-match --regexp='e2e') + +# `CGO_ENABLED := 1` is required to enforce dynamic linking which is a requirement of dynatrace. +CGO_ENABLED := 1 BUILDTAGS := export GOWORK=off BUILDFLAGS := -ldflags '-linkmode=external' @@ -20,6 +25,43 @@ export GO111MODULE=on GINKGO_OPTS=-r --race --require-suite --randomize-all --cover ${OPTS} GINKGO_VERSION=v$(shell cat ../../.tool-versions | grep ginkgo | cut -d " " -f 2 ) +# The presence of the subsequent directory indicates wheather the fakes still need to be generated +# or not. +app-fakes-dir := ./fakes +app-fakes-files := $(wildcard ${app-fakes-dir}/*.go) + +.PHONY: generate-fakes +generate-fakes: ${app-fakes-dir} ${app-fakes-files} +${app-fakes-dir} ${app-fakes-files} &: ./go.mod ./go.sum ./generate-fakes.go + @echo "# Generating counterfeits" + mkdir -p '${app-fakes-dir}' + COUNTERFEITER_NO_GENERATE_WARNING='true' go generate ./... + + + +# The generated fakes are order-only dependencies for `go-mod-tidy`. The reason is, that for +# `go-mod-tidy` the generated fakes need to be present but fortunately not necessarily +# up-to-date. This is fortunate because the generation of the fake requires the files `go.mod` and +# `go.sum` to be already tidied up, introducing a cyclic dependency otherwise. But that would make +# any modification to `go.mod` or `go.sum` impossible. This definition now makes it possible to +# update `go.mod` and `go.sum` as follows: +# 1. `make generate-fakes` +# 2. Update `go.mod` and/or `go.sum` +# 3. `make go-mod-tidy` +# 4. Optionally: `make generate-fakes` to update the fakes as well. +# +# For more information on order-only dependencies, see: +# +.PHONY: go-mod-tidy +go-mod-tidy: ./go.mod ./go.sum ${GO_DEPENDENCIES} | ${app-fakes-dir} ${app-fakes-files} + @echo -ne '${aes_terminal_font_yellow}' + @echo -e '⚠️ Warning: The client-fakes generated from the openapi-specification may be\n' \ + 'outdated. Please consider re-generating them, if this is relevant.' + @echo -ne '${aes_terminal_reset}' + go mod tidy + + + build-%: @echo "# building $*" @CGO_ENABLED=$(CGO_ENABLED) go build $(BUILDTAGS) $(BUILDFLAGS) -o build/$* $*/cmd/$*/main.go @@ -36,9 +78,9 @@ build_test-%: generate cd $* &&\ for package in $$( go list ./... | sed 's|.*/autoscaler/$*|.|' | awk '{ print length, $$0 }' | sort -n -r | cut -d" " -f2- );\ do\ - export test_file=$${build_folder}/$${package}.test;\ - echo " - compiling $${package} to $${test_file}";\ - go test -c -o $${test_file} $${package};\ + export test_file=$${build_folder}/$${package}.test;\ + echo " - compiling $${package} to $${test_file}";\ + go test -c -o $${test_file} $${package};\ done; check: fmt lint build test @@ -75,5 +117,5 @@ lint: clean: @echo "# cleaning autoscaler" @go clean -cache -testcache - @rm -rf build - @rm -rf fakes/fake_*.go + @rm --force --recursive 'build' + @rm --force --recursive 'fakes' diff --git a/src/autoscaler/fakes/.gitignore b/src/autoscaler/fakes/.gitignore deleted file mode 100644 index e3834a00fe..0000000000 --- a/src/autoscaler/fakes/.gitignore +++ /dev/null @@ -1 +0,0 @@ -fake_*.go \ No newline at end of file diff --git a/src/autoscaler/fakes/fakes.go b/src/autoscaler/fakes/fakes.go deleted file mode 100644 index 0b0a1f8f28..0000000000 --- a/src/autoscaler/fakes/fakes.go +++ /dev/null @@ -1,30 +0,0 @@ -package fakes - -// Multiple go:generate directives instead of counterfeiter:generate due to https://github.com/maxbrunsfeld/counterfeiter/issues/254 -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_wshelper.go ../metricsgateway/helpers WSHelper -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_cf_client.go ../cf CFClient -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_policy_db.go ../db PolicyDB -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_scalingengine_db.go ../db ScalingEngineDB -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_scheduler_db.go ../db SchedulerDB -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_scalingengine.go ../scalingengine ScalingEngine -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_binding_db.go ../db BindingDB -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_app_metric_db.go ../db AppMetricDB -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_instancemetrics_db.go ../db InstanceMetricsDB -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_credentials.go ../cred_helper Credentials -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_storedprocedure_db.go ../db StoredProcedureDB -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_metric_forwarder.go ../metricsforwarder/forwarder MetricForwarder -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_plan_checker.go ../api/plancheck PlanChecker -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_log_cache_client.go ../eventgenerator/client LogCacheClientReader -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_envelope_processor.go ../envelopeprocessor EnvelopeProcessor -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_log_cache_creator.go ../eventgenerator/client LogCacheClientCreator -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_metric_server_creator_creator.go ../eventgenerator/client MetricServerClientCreator -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_go_log_cache_client.go ../eventgenerator/client GoLogCacheClient -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_grpc.go ../eventgenerator/client GRPCOptions -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_envelope_processor_creator.go ../envelopeprocessor EnvelopeProcessorCreator -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_plan_checker.go ../api/plancheck PlanChecker -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_ratelimiter.go ../ratelimiter Limiter -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_httpstatus_collector.go ../healthendpoint HTTPStatusCollector -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_database_status.go ../healthendpoint DatabaseStatus -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_emitter.go ../metricsgateway Emitter -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_operator.go ../operator Operator -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fake_sychronizer.go ../scalingengine/schedule ActiveScheduleSychronizer diff --git a/src/autoscaler/generate-fakes.go b/src/autoscaler/generate-fakes.go new file mode 100644 index 0000000000..536f6d59d9 --- /dev/null +++ b/src/autoscaler/generate-fakes.go @@ -0,0 +1,30 @@ +package fakes + +// Multiple go:generate directives instead of counterfeiter:generate due to https://github.com/maxbrunsfeld/counterfeiter/issues/254 +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_wshelper.go ./metricsgateway/helpers WSHelper +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_cf_client.go ./cf CFClient +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_policy_db.go ./db PolicyDB +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_scalingengine_db.go ./db ScalingEngineDB +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_scheduler_db.go ./db SchedulerDB +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_scalingengine.go ./scalingengine ScalingEngine +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_binding_db.go ./db BindingDB +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_app_metric_db.go ./db AppMetricDB +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_instancemetrics_db.go ./db InstanceMetricsDB +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_credentials.go ./cred_helper Credentials +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_storedprocedure_db.go ./db StoredProcedureDB +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_metric_forwarder.go ./metricsforwarder/forwarder MetricForwarder +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_plan_checker.go ./api/plancheck PlanChecker +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_log_cache_client.go ./eventgenerator/client LogCacheClientReader +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_envelope_processor.go ./envelopeprocessor EnvelopeProcessor +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_log_cache_creator.go ./eventgenerator/client LogCacheClientCreator +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_metric_server_creator_creator.go ./eventgenerator/client MetricServerClientCreator +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_go_log_cache_client.go ./eventgenerator/client GoLogCacheClient +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_grpc.go ./eventgenerator/client GRPCOptions +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_envelope_processor_creator.go ./envelopeprocessor EnvelopeProcessorCreator +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_plan_checker.go ./api/plancheck PlanChecker +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_ratelimiter.go ./ratelimiter Limiter +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_httpstatus_collector.go ./healthendpoint HTTPStatusCollector +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_database_status.go ./healthendpoint DatabaseStatus +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_emitter.go ./metricsgateway Emitter +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_operator.go ./operator Operator +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_sychronizer.go ./scalingengine/schedule ActiveScheduleSychronizer diff --git a/src/changelog/Makefile b/src/changelog/Makefile index 9456abdcfd..3f6cc342aa 100644 --- a/src/changelog/Makefile +++ b/src/changelog/Makefile @@ -1,18 +1,30 @@ SHELL := /bin/bash -.SHELLFLAGS = -euo pipefail -c -MAKEFLAGS = -s +.SHELLFLAGS := -eu -o pipefail -c +MAKEFLAGS := -s + GO := GO111MODULE=on GO15VENDOREXPERIMENT=1 go GO_NOMOD := GO111MODULE=off go -GO_VERSION := $(shell $(GO) version | sed -e 's/^[^0-9.]*\([0-9.]*\).*/\1/') -GO_DEPENDENCIES := $(shell find . -type f -name '*.go') -PACKAGE_DIRS := $(shell $(GO) list ./... | grep -v /vendor/ | grep -v e2e) -CGO_ENABLED = 0 -export GOWORK=off +GO_VERSION := $(shell ${GO} version | sed --expression='s/^[^0-9.]*\([0-9.]*\).*/\1/') +GO_DEPENDENCIES = $(shell find . -type f -name '*.go') +PACKAGE_DIRS = $(shell ${GO} list './...' | grep --invert-match --regexp='/vendor/' \ + | grep --invert-match --regexp='e2e') + +CGO_ENABLED := 0 +export GOWORK := off BUILDTAGS := -GINKGO_VERSION=v$(shell cat ../../.tool-versions | grep ginkgo | cut -d " " -f 2 ) -GOLANGCI_LINT_VERSION=v$(shell cat ../../.tool-versions | grep golangci-lint | cut -d " " -f 2 ) +GINKGO_VERSION = v$(shell cat '../../.tool-versions' | grep --regexp='ginkgo' \ + | cut --delimiter=' ' --fields='2') +GOLANGCI_LINT_VERSION = v$(shell cat '../../.tool-versions' | grep --regexp='golangci-lint' \ + | cut --delimiter=' ' --fields='2' ) + + +.PHONY: go-mod-tidy +go-mod-tidy: + go mod tidy + + build: @echo "# building changelog" @@ -27,9 +39,9 @@ build_test-%: cd $* &&\ for package in $$( go list ./... | sed 's|.*/autoscaler/$*|.|' | awk '{ print length, $$0 }' | sort -n -r | cut -d" " -f2- );\ do\ - export test_file=$${build_folder}/$${package}.test;\ - echo " - compiling $${package} to $${test_file}";\ - go test -c -o $${test_file} $${package};\ + export test_file=$${build_folder}/$${package}.test;\ + echo " - compiling $${package} to $${test_file}";\ + go test -c -o $${test_file} $${package};\ done; check: fmt lint build test diff --git a/src/changeloglockcleaner/Makefile b/src/changeloglockcleaner/Makefile index 127a46c7ea..1a4a060222 100644 --- a/src/changeloglockcleaner/Makefile +++ b/src/changeloglockcleaner/Makefile @@ -1,18 +1,30 @@ SHELL := /bin/bash -.SHELLFLAGS = -euo pipefail -c -MAKEFLAGS = -s +.SHELLFLAGS := -eu -o pipefail -c +MAKEFLAGS := -s + GO := GO111MODULE=on GO15VENDOREXPERIMENT=1 go GO_NOMOD := GO111MODULE=off go -GO_VERSION := $(shell $(GO) version | sed -e 's/^[^0-9.]*\([0-9.]*\).*/\1/') -GO_DEPENDENCIES := $(shell find . -type f -name '*.go') -PACKAGE_DIRS := $(shell $(GO) list ./... | grep -v /vendor/ | grep -v e2e) -CGO_ENABLED = 0 -export GOWORK=off +GO_VERSION := $(shell ${GO} version | sed --expression='s/^[^0-9.]*\([0-9.]*\).*/\1/') +GO_DEPENDENCIES = $(shell find . -type f -name '*.go') +PACKAGE_DIRS = $(shell ${GO} list './...' | grep --invert-match --regexp='/vendor/' \ + | grep --invert-match --regexp='e2e') + +CGO_ENABLED := 0 +export GOWORK := off BUILDTAGS := -GINKGO_VERSION=v$(shell cat ../../.tool-versions | grep ginkgo | cut -d " " -f 2 ) -GOLANGCI_LINT_VERSION=v$(shell cat ../../.tool-versions | grep golangci-lint | cut -d " " -f 2 ) +GINKGO_VERSION = v$(shell cat '../../.tool-versions' | grep --regexp='ginkgo' \ + | cut --delimiter=' ' --fields='2') +GOLANGCI_LINT_VERSION = v$(shell cat '../../.tool-versions' | grep --regexp='golangci-lint' \ + | cut --delimiter=' ' --fields='2' ) + + +.PHONY: go-mod-tidy +go-mod-tidy: + go mod tidy + + build: @echo "# building changeloglockcleaner" @@ -27,9 +39,9 @@ build_test-%: cd $* &&\ for package in $$( go list ./... | sed 's|.*/autoscaler/$*|.|' | awk '{ print length, $$0 }' | sort -n -r | cut -d" " -f2- );\ do\ - export test_file=$${build_folder}/$${package}.test;\ - echo " - compiling $${package} to $${test_file}";\ - go test -c -o $${test_file} $${package};\ + export test_file=$${build_folder}/$${package}.test;\ + echo " - compiling $${package} to $${test_file}";\ + go test -c -o $${test_file} $${package};\ done; check: fmt lint build test @@ -53,4 +65,3 @@ lint: lint-fix: go run github.com/golangci/golangci-lint/cmd/golangci-lint@${GOLANGCI_LINT_VERSION} run --fix -