Merge pull request #2145 from cloudfoundry/test-app.go-mod-tidy

Adds workflow that helps the bots with intermediate files.

.github/workflows/bosh-release-checks.yaml

@@ -46,7 +46,7 @@ jobs:
         run: |
           set -e
           export PATH=$PATH:/usr/local/maven/bin
-          make mod-tidy vendor db scheduler
+          make go-mod-tidy vendor db scheduler
 
       - name: Build Dev Release
         id: build

.github/workflows/dependency-update.go-mod-tidy_and_make-package-specs.yaml

@@ -0,0 +1,85 @@
+name: "renovate-/ospo-bot: go mod tidy and make package-specs"
+on:
+  push:
+    branches:
+      - "renovate/**"
+      - "dependabot/**"
+    paths:
+      - ".github/workflows/dependency-update-go-mod-tidy.yaml"
+      - "./src/**/go.mod"
+      - "./src/**/go.sum"
+jobs:
+  go-mod-tidy:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+      contents: write
+    steps:
+      # We potentially want to add at the end a commit by the author of the most recent
+      # commit in this branch. However github has some protection which prevents workflows
+      # to run in case a commit has been pushed with the default job-specific github-token.
+      # For this case we need to use another one here.
+      #
+      # For more information, see:
+      # <https://docs.github.com/en/actions/security-guides/automatic-token-authentication#using-the-github_token-in-a-workflow>
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
+          token: ${{ secrets.APP_AUTOSCALER_CI_TOKEN }} # With push token that can trigger new PR jobs
+      - name: Configure git
+        shell: bash
+        run: |
+          #! /usr/bin/env bash
+          set -eu -o pipefail
+
+          declare -r commit_author_name="${{github.event.head_commit.author.name}}"
+          declare -r commit_author_email="${{github.event.head_commit.author.email}}"
+
+          git config --global --add safe.directory "${GITHUB_WORKSPACE}"
+          git config user.name "${commit_author_name}"
+          git config user.email "${commit_author_email}"
+      - name: go-mod-tidy and make package-specs
+        shell: bash
+        run: |
+          #! /usr/bin/env bash
+          set -eu -o pipefail
+
+          # We need the subsequent standard-message to determine if the last commit
+          # has already cleaned up everything. In this case this workflow should not
+          # change anything and we exit early.
+          # An alternative would be to use a tag for this. But this does affect the whole
+          # PR instead of just the latest commit.
+          declare -r tidy_message='🤖🦾🛠️ go mod tidy & make package-specs'
+
+          declare -r commit_author_name="${{github.event.head_commit.author.name}}"
+          declare -r commit_message="${{github.event.head_commit.message}}"
+
+          if [[ ! "${commit_author_name}" =~ ('dependabot'|'renovate')'[bot]' ]] \
+             || [[ "${commit_message}" == "${tidy_message}" ]]
+          then
+            echo 'This commit was not by a known bot or already an automatic `go mod tidy`! Exiting …'
+            exit 0
+          fi
+
+          # Generated files are needed for `go mod tidy` which is a dependency of the
+          # target `package-specs`. However the generation of them itself already
+          # requires go-modules to be tidied up. So we need to generate the files
+          # before changing `go.mod` and `go.sum`.
+          declare -r current_branch="$(git branch --show-current)"
+          git checkout 'HEAD~1'
+          make --directory='./src/acceptance/assets/app/go_app' generate-fakes
+          make --directory='./src/autoscaler' generate-fakes
+          git checkout "${current_branch}"
+
+          # ⚠️ For this workflow to be successful, the subsequent line must not
+          # trigger again the creation of the generated files.
+          make package-specs
+
+          declare -i -r changed_files=$(git status --porcelain | wc --lines)
+          if ((changed_files >= 0))
+          then
+            git add .
+            git commit --message="${tidy_message}"
+            git push
+          fi

.github/workflows/tidy-go-mod.yaml

@@ -19,7 +19,7 @@ jobs:
         run: git config --global --add safe.directory "${GITHUB_WORKSPACE}"
 
       - name: run go mod tidy on all .mod's
-        run: make mod-tidy
+        run: make go-mod-tidy
 
       - name: Check if there is any change
         id: get_changes

Makefile

@@ -1,15 +1,17 @@
 SHELL := /bin/bash
 .SHELLFLAGS := -eu -o pipefail -c ${SHELLFLAGS}
 MAKEFLAGS = -s
+
 go_modules := $(shell  find . -maxdepth 3 -name "*.mod" -exec dirname {} \; | sed 's|\./src/||' | sort)
 all_modules := $(go_modules) db scheduler
+
 MVN_OPTS = "-Dmaven.test.skip=true"
 OS := $(shell . /etc/lsb-release &>/dev/null && echo $${DISTRIB_ID} || uname)
 db_type := postgres
 DB_HOST := localhost
 DBURL := $(shell case "${db_type}" in\
 			 (postgres) printf "postgres://postgres:postgres@${DB_HOST}/autoscaler?sslmode=disable"; ;; \
- 			 (mysql) printf "root@tcp(${DB_HOST})/autoscaler?tls=false"; ;; esac)
+				 (mysql) printf "root@tcp(${DB_HOST})/autoscaler?tls=false"; ;; esac)
 DEBUG := false
 MYSQL_TAG := 8
 POSTGRES_TAG := 12
@@ -162,7 +164,7 @@ target/start-db-postgres_CI_true:
 waitfor_postgres_CI_false:
 	@echo -n " - waiting for ${db_type} ."
 	@COUNTER=0; until $$(docker exec postgres pg_isready &>/dev/null) || [ $$COUNTER -gt 10 ]; do echo -n "."; sleep 1; let COUNTER+=1; done;\
- 	if [ $$COUNTER -gt 10 ]; then echo; echo "Error: timed out waiting for postgres. Try \"make clean\" first." >&2 ; exit 1; fi
+	if [ $$COUNTER -gt 10 ]; then echo; echo "Error: timed out waiting for postgres. Try \"make clean\" first." >&2 ; exit 1; fi
 waitfor_postgres_CI_true:
 	@echo " - no ci postgres checks"
 
@@ -193,14 +195,14 @@ waitfor_mysql_CI_false:
 waitfor_mysql_CI_true:
 	@echo -n " - Waiting for table creation"
 	@which mysql >/dev/null &&\
-	 {\
-	   T=0;\
-	   until [[ ! -z "$(shell mysql -u "root" -h "${DB_HOST}"  --port=3306 -qfsBe "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='autoscaler'" 2> /dev/null)" ]]\
-	     || [[ $${T} -gt 30 ]];\
-	   do echo -n "."; sleep 1; T=$$((T+1)); done;\
-	 }
+	{\
+		T=0;\
+		until [[ ! -z "$(shell mysql -u "root" -h "${DB_HOST}"  --port=3306 -qfsBe "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='autoscaler'" 2> /dev/null)" ]]\
+			|| [[ $${T} -gt 30 ]];\
+		do echo -n "."; sleep 1; T=$$((T+1)); done;\
+	}
 	@[ ! -z "$(shell mysql -u "root" -h "${DB_HOST}" --port=3306 -qfsBe "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='autoscaler'"  2> /dev/null)" ]\
-	  || { echo "ERROR: Mysql timed out creating database"; exit 1; }
+		|| { echo "ERROR: Mysql timed out creating database"; exit 1; }
 
 
 .PHONY: stop-db
@@ -243,36 +245,57 @@ spec-test:
 	bundle exec rspec
 
 .PHONY: release
-bosh-release: mod-tidy vendor scheduler db build/autoscaler-test.tgz
+bosh-release: go-mod-tidy vendor scheduler db build/autoscaler-test.tgz
 build/autoscaler-test.tgz:
 	@echo " - building bosh release into build/autoscaler-test.tgz"
 	@mkdir -p build
 	@bosh create-release --force --timestamp-version --tarball=build/autoscaler-test.tgz
 
 .PHONY: acceptance-release
-acceptance-release: clean-acceptance mod-tidy vendor build-test-app
+acceptance-release: clean-acceptance go-mod-tidy vendor build-test-app
 	@echo " - building acceptance test release '${VERSION}' to dir: '${DEST}' "
 	@mkdir -p ${DEST}
 	@tar --create --auto-compress --directory="src" --file="${ACCEPTANCE_TESTS_FILE}" 'acceptance'
-.PHONY: mod-tidy
-mod-tidy:
-	@for folder in $$(find . -maxdepth 3 -name "go.mod" -exec dirname {} \;);\
-	do\
-	   cd $${folder}; echo " - go mod tidying '$${folder}'"; go mod tidy; cd - >/dev/null;\
-	done
+
+
+
+.PHONY: go-mod-tidy
+go-mod-tidy: acceptance.go-mod-tidy autoscaler.go-mod-tidy changelog.go-mod-tidy \
+						 changeloglockcleander.go-mod-tidy test-app.go-mod-tidy
+
+go-acceptance-dir := ./src/acceptance
+go-autoscaler-dir := ./src/autoscaler
+go-changelog-dir := ./src/changelog
+go-changeloglockcleander-dir := ./src/changeloglockcleaner
+go-test-app-dir := ./src/acceptance/assets/app/go_app
+
+.PHONY: acceptance.go-mod-tidy autoscaler.go-mod-tidy changelog.go-mod-tidy \
+				changeloglockcleander.go-mod-tidy test-app.go-mod-tidy
+acceptance.go-mod-tidy:
+	make --directory='${go-acceptance-dir}' go-mod-tidy
+autoscaler.go-mod-tidy:
+	make --directory='${go-autoscaler-dir}' go-mod-tidy
+changelog.go-mod-tidy:
+	make --directory='${go-changelog-dir}' go-mod-tidy
+changeloglockcleander.go-mod-tidy:
+	make --directory='${go-changeloglockcleander-dir}' go-mod-tidy
+test-app.go-mod-tidy:
+	make --directory='${go-test-app-dir}' go-mod-tidy
+
+
 
 .PHONY: mod-download
 mod-download:
 	@for folder in $$(find . -maxdepth 3 -name "go.mod" -exec dirname {} \;);\
 	do\
-	   cd $${folder}; echo " - go mod download '$${folder}'"; go mod download; cd - >/dev/null;\
+		 cd $${folder}; echo " - go mod download '$${folder}'"; go mod download; cd - >/dev/null;\
 	done
 
 .PHONY: vendor
 vendor:
 	@for folder in $$(find . -maxdepth 3 -name "go.mod" -exec dirname {} \;);\
 	do\
-	   cd $${folder}; echo " - go mod vendor '$${folder}'"; go mod vendor; cd - >/dev/null;\
+		 cd $${folder}; echo " - go mod vendor '$${folder}'"; go mod vendor; cd - >/dev/null;\
 	done
 
 .PHONY: fakes
@@ -295,7 +318,7 @@ markdownlint-cli:
 
 .PHONY: deploy deploy-autoscaler deploy-register-cf deploy-autoscaler-bosh deploy-cleanup
 deploy-autoscaler: deploy
-deploy: mod-tidy vendor uaac db scheduler deploy-autoscaler-bosh deploy-register-cf
+deploy: go-mod-tidy vendor uaac db scheduler deploy-autoscaler-bosh deploy-register-cf
 deploy-register-cf:
 	echo " - registering broker with cf"
 	[ "$${BUILDIN_MODE}" == "false" ] && { ${CI_DIR}/autoscaler/scripts/register-broker.sh; } || echo " - Not registering broker due to buildin mode enabled"
@@ -367,7 +390,7 @@ run-performance:
 run-act:
 	${AUTOSCALER_DIR}/scripts/run_act.sh;\
 
-package-specs: mod-tidy vendor
+package-specs: go-mod-tidy vendor
 	@echo " - Updating the package specs"
 	@./scripts/sync-package-specs
 
@@ -396,4 +419,4 @@ docker-image: docker-login
 validate-openapi-specs: $(wildcard ./api/*.openapi.yaml)
 	for file in $^ ; do \
 		swagger-cli validate "$${file}" ; \
-	done
+	done

flake.nix

@@ -63,7 +63,11 @@
             hardeningDisable = [ "fortify" ];
 
             shellHook = ''
-              echo -ne '\033[1;33m' '\033[5m'
+              aes_terminal_font_yellow='\e[38;2;255;255;0m'
+              aes_terminal_font_blink='\e[5m'
+              aes_terminal_reset='\e[0m'
+
+              echo -ne "$aes_terminal_font_yellow" "$aes_terminal_font_blink"
               cat << 'EOF'
                 ⚠️ If `whoami` does not work properly on your computer, `bosh ssh` commands may fail.
                 The solution is, to provide your nix dev-shell the path to the `libnss_sss.so.2` of
@@ -72,9 +76,9 @@
                 Adapt the following line to contain the correct path:
                 export LD_PRELOAD='/lib/x86_64-linux-gnu/libnss_sss.so.2'
               EOF
-              echo -ne '\033[0m'
+              echo -ne "$aes_terminal_reset"
             '';
           };
       });
   };
-}
+}

src/acceptance/Makefile

@@ -1,20 +1,32 @@
 SHELL := /bin/bash
-.SHELLFLAGS = -euo pipefail -c
-MAKEFLAGS = -s
+.SHELLFLAGS := -eu -o pipefail -c
+MAKEFLAGS := -s
+
 GO := GO111MODULE=on GO15VENDOREXPERIMENT=1 go
 GO_NOMOD := GO111MODULE=off go
-GO_VERSION := $(shell $(GO) version | sed -e 's/^[^0-9.]*\([0-9.]*\).*/\1/')
-GO_DEPENDENCIES := $(shell find . -type f -name '*.go')
-PACKAGE_DIRS := $(shell $(GO) list ./... | grep -v /vendor/ | grep -v e2e)
 
-CGO_ENABLED = 0
-export GOWORK=off
-BUILDTAGS :=
+GO_VERSION = $(shell go version | sed --expression='s/^[^0-9.]*\([0-9.]*\).*/\1/')
+GO_DEPENDENCIES = $(shell find . -type f -name '*.go')
+PACKAGE_DIRS = $(shell go list './...' | grep --invert-match --regexp='/vendor/' \
+								 | grep --invert-match --regexp='e2e')
+
+CGO_ENABLED := 0
+export GOWORK := off
+
+GINKGO_VERSION = v$(shell cat '../../.tool-versions' | grep --regexp='ginkgo' \
+										| cut --delimiter=' ' --fields='2')
+GOLANGCI_LINT_VERSION = v$(shell cat '../../.tool-versions' | grep --regexp='golangci-lint' \
+													 | cut --delimiter=' ' --fields='2' )
+
+
+.PHONY: go-mod-tidy
+go-mod-tidy:
+	go mod tidy
+
 
-GINKGO_VERSION=v$(shell cat ../../.tool-versions | grep ginkgo  | cut -d " " -f 2 )
-GOLANGCI_LINT_VERSION=v$(shell cat ../../.tool-versions | grep golangci-lint  | cut -d " " -f 2 )
 
-test_dirs=$(shell   find . -name "*_test.go" -exec dirname {} \; |  cut -d/ -f2 | sort | uniq)
+test_dirs = $(shell find . -name "*_test.go" -exec dirname {} \; | cut --delimiter='/' --fields='2' \
+						| sort | uniq)
 build_tests: $(addprefix build_test-,$(test_dirs))
 build_test-%:
 	@echo " - building '$*' tests"
@@ -23,9 +35,9 @@ build_test-%:
 	 cd $* &&\
 	 for package in $$(  go list ./... | sed 's|.*/autoscaler/$*|.|' | awk '{ print length, $$0 }' | sort -n -r | cut -d" " -f2- );\
 	 do\
-	   export test_file=$${build_folder}/$${package}.test;\
-	   echo "   - compiling $${package} to $${test_file}";\
-	   go test -c -o $${test_file} $${package};\
+		 export test_file=$${build_folder}/$${package}.test;\
+		 echo "   - compiling $${package} to $${test_file}";\
+		 go test -c -o $${test_file} $${package};\
 	 done;
 
 check: lint build_tests