From 4f1f9e1bd31880aef4230306b9b678f5d87a4ead Mon Sep 17 00:00:00 2001 From: "Artur M. Wolff" Date: Sun, 24 Apr 2022 18:35:55 +0200 Subject: [PATCH] ci: rework Jenkinsfile/Makefile This change reworks Gateway-ST's CI so that it's easier to develop locally and be more productive. 1. It adds dependencies installation as a "one-click" operation a. Similarly, it allows installing helpful Git hooks (this change adds an installable pre-commit hook that runs verification) 2. It moves directly-called linting/testing from Jenkinsfile to Makefile a. New make targets are called from Jenkinsfile New make targets (e.g., verify) can be called to aid local development (sometimes it's useful to test/reproduce locally). Integration tests have not yet been moved. They will be moved in the next changes. Obsolete targets will also be purged in the next changes. Change-Id: Icc77fd6159db5bd74be536e53bd5cd62656e44a0 --- Jenkinsfile | 93 +++++++----------------- Makefile | 171 +++++++++++++++++++++++++++++++++++++------- githooks/pre-commit | 5 ++ 3 files changed, 174 insertions(+), 95 deletions(-) create mode 100755 githooks/pre-commit diff --git a/Jenkinsfile b/Jenkinsfile index 2c91a49..f0bcab9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -15,7 +15,8 @@ pipeline { environment { GOTRACEBACK = 'all' - COCKROACH_MEMPROF_INTERVAL = 0 + // COCKROACH_MEMPROF_INTERVAL = 0 + // ^ doesn't work: https://github.com/cockroachdb/cockroach/issues/54793 } stages { @@ -52,74 +53,51 @@ pipeline { stage('Verification') { parallel { stage('Lint') { + environment { + GOLANGCI_LINT_CONFIG = '/go/ci/.golangci.yml' + GOLANGCI_LINT_CONFIG_TESTSUITE = '/go/ci/.golangci.yml' + } steps { - sh 'check-mod-tidy' - sh 'check-copyright' - sh 'check-large-files' - sh 'check-imports -race ./...' - sh 'check-peer-constraints -race' - sh 'check-atomic-align ./...' - sh 'check-monkit ./...' - sh 'check-errs ./...' - sh 'check-deferloop ./...' - sh 'staticcheck ./...' - sh 'golangci-lint run --config /go/ci/.golangci.yml' - sh 'check-downgrades' - - // A bit of an explanation around this shellcheck command: - // * Find all scripts recursively that have the .sh extension, except for "testsuite@tmp" which Jenkins creates temporarily. - // * Use + instead of \ so find returns a non-zero exit if any invocation of shellcheck returns a non-zero exit. - sh 'find . -path ./testsuite@tmp -prune -o -name "*.sh" -type f -exec "shellcheck" "-x" "--format=gcc" {} +;' + sh 'make lint' + } + } - dir('testsuite') { - sh 'check-imports -race ./...' - sh 'check-atomic-align ./...' - sh 'check-monkit ./...' - sh 'check-errs ./...' - sh 'check-deferloop ./...' - sh 'staticcheck ./...' - sh 'golangci-lint run --config /go/ci/.golangci.yml' - } + stage('Cross-Vet') { + steps { + sh 'make cross-vet' } } stage('Test') { environment { - COVERFLAGS = "${ env.BRANCH_NAME != 'main' ? '' : '-coverprofile=.build/coverprofile -coverpkg=./...'}" + JSON = true + SHORT = false + SKIP_TESTSUITE = true } steps { - sh 'go test -parallel 4 -p 16 -vet=off ${COVERFLAGS} -timeout 10m -json -race ./... 2>&1 | tee .build/tests.json | xunit -out .build/tests.xml' + sh 'make test 2>&1 | tee .build/tests.json | xunit -out .build/tests.xml' } post { always { sh script: 'cat .build/tests.json | tparse -all -top -slow 100', returnStatus: true archiveArtifacts artifacts: '.build/tests.json' junit '.build/tests.xml' - script { - if (fileExists('.build/coverprofile')) { - sh script: 'filter-cover-profile < .build/coverprofile > .build/clean.coverprofile', returnStatus: true - sh script: 'gocov convert .build/clean.coverprofile > .build/cover.json', returnStatus: true - sh script: 'gocov-xml < .build/cover.json > .build/cobertura.xml', returnStatus: true - cobertura coberturaReportFile: '.build/cobertura.xml' - } - } } } } stage('Testsuite') { environment { - STORJ_TEST_COCKROACH = 'cockroach://root@localhost:26257/testcockroach?sslmode=disable' - STORJ_TEST_POSTGRES = 'postgres://postgres@localhost/teststorj?sslmode=disable' + JSON = true + SHORT = false + STORJ_TEST_COCKROACH = 'cockroach://root@localhost:26257/postgres?sslmode=disable' + STORJ_TEST_POSTGRES = 'postgres://postgres@localhost/postgres?sslmode=disable' } steps { - sh 'cockroach sql --insecure --host=localhost:26257 -e \'create database testcockroach;\'' - sh 'psql -U postgres -c \'create database teststorj;\'' + // exhaust ports from 1024 to 10000 to ensure we don't + // use hardcoded ports sh 'use-ports -from 1024 -to 10000 &' - dir('testsuite') { - sh 'go vet ./...' - sh 'go test -parallel 4 -p 16 -vet=off -timeout 10m -json -race ./... 2>&1 | tee ../.build/testsuite.json | xunit -out ../.build/testsuite.xml' - } + sh 'make --no-print-directory test-testsuite 2>&1 | tee .build/testsuite.json | xunit -out .build/testsuite.xml' } post { always { @@ -135,11 +113,9 @@ pipeline { // use different hostname to avoid port conflicts STORJ_NETWORK_HOST4 = '127.0.0.2' STORJ_NETWORK_HOST6 = '127.0.0.2' - - STORJ_SIM_POSTGRES = 'postgres://postgres@localhost/integration?sslmode=disable' + STORJ_SIM_POSTGRES = 'postgres://postgres@localhost/postgres?sslmode=disable' } steps { - sh 'psql -U postgres -c \'create database integration;\'' sh 'cd ./testsuite/integration && ./run.sh' } post { @@ -150,27 +126,6 @@ pipeline { } } } - - stage('Cross Compile') { - steps { - sh 'GOOS=linux GOARCH=386 go vet ./...' - sh 'GOOS=linux GOARCH=amd64 go vet ./...' - sh 'GOOS=linux GOARCH=arm go vet ./...' - sh 'GOOS=linux GOARCH=arm64 go vet ./...' - sh 'GOOS=freebsd GOARCH=386 go vet ./...' - sh 'GOOS=freebsd GOARCH=amd64 go vet ./...' - sh 'GOOS=freebsd GOARCH=arm64 go vet ./...' - sh 'GOOS=windows GOARCH=386 go vet ./...' - sh 'GOOS=windows GOARCH=amd64 go vet ./...' - - // TODO(artur): find out if we will be able to enable it - // sh 'GOOS=windows GOARCH=arm64 go vet ./...' - - // Use kqueue to avoid needing cgo for verification. - sh 'GOOS=darwin GOARCH=amd64 go vet -tags kqueue ./...' - sh 'GOOS=darwin GOARCH=arm64 go vet -tags kqueue ./...' - } - } } } diff --git a/Makefile b/Makefile index fd1ea7f..47b7d55 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,151 @@ GO_VERSION ?= 1.17.12 BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD | sed "s!/!-!g") LATEST_DEV_TAG := dev +# +# Common +# + +.DEFAULT_GOAL := help +.PHONY: help +help: + @awk 'BEGIN { \ + FS = ":.*##"; \ + printf "\nUsage:\n make \033[36m\033[0m\n" \ + } \ + /^[a-zA-Z_-]+:.*?##/ { \ + printf " \033[36m%-24s\033[0m %s\n", $$1, $$2 \ + } \ + /^##@/ { \ + printf "\n\033[1m%s\033[0m\n", substr($$0, 5) \ + }' $(MAKEFILE_LIST) + +# +# Public Jenkins (commands below are used for local development and/or public Jenkins) +# + +##@ Local development/Public Jenkins/Helpers + +.PHONY: install-dev-dependencies +install-dev-dependencies: ## install-dev-dependencies assumes Go and cURL are installed + # Storj-specific: + go install github.com/storj/ci/check-mod-tidy@latest + go install github.com/storj/ci/check-copyright@latest + go install github.com/storj/ci/check-large-files@latest + go install github.com/storj/ci/check-imports@latest + go install github.com/storj/ci/check-peer-constraints@latest + go install github.com/storj/ci/check-atomic-align@latest + go install github.com/storj/ci/check-monkit@latest + go install github.com/storj/ci/check-errs@latest + go install github.com/storj/ci/check-deferloop@latest + go install github.com/storj/ci/check-downgrades@latest + + # staticcheck: + go install honnef.co/go/tools/cmd/staticcheck@latest + + # golangci-lint: + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin v1.45.2 + + # shellcheck (TODO(artur): Windows) +ifneq ($(shell which apt-get),) + sudo apt-get install -y shellcheck +else ifneq ($(shell which brew),) + brew install shellcheck +else + $(error Can't install shellcheck without a supported package manager) +endif + +.PHONY: install-hooks +install-hooks: ## Install helpful Git hooks + ln -s .git/hooks/pre-commit githooks/pre-commit + +##@ Local development/Public Jenkins/Lint + +GOLANGCI_LINT_CONFIG ?= ../ci/.golangci.yml +GOLANGCI_LINT_CONFIG_TESTSUITE ?= ../../ci/.golangci.yml + +.PHONY: lint +lint: ## Lint + check-mod-tidy + check-copyright + check-large-files + check-imports -race ./... + check-peer-constraints -race + check-atomic-align ./... + check-monkit ./... + check-errs ./... + check-deferloop ./... + staticcheck ./... + golangci-lint run --print-resources-usage --config ${GOLANGCI_LINT_CONFIG} + check-downgrades + + # A bit of an explanation around this shellcheck command: + # * Find all scripts recursively that have the .sh extension, except for "testsuite@tmp" which Jenkins creates temporarily + # * Use + instead of \ so find returns a non-zero exit if any invocation of shellcheck returns a non-zero exit + find . -path ./testsuite@tmp -prune -o -name "*.sh" -type f -exec "shellcheck" "-x" "--format=gcc" {} +; + + # Execute lint-testsuite in testsuite directory: + $(MAKE) -C testsuite -f ../Makefile lint-testsuite + +.PHONY: lint-testsuite +lint-testsuite: ## Lint testsuite + check-imports -race ./... + check-atomic-align ./... + check-monkit ./... + check-errs ./... + check-deferloop ./... + staticcheck ./... + golangci-lint run --print-resources-usage --config ${GOLANGCI_LINT_CONFIG_TESTSUITE} + +##@ Local development/Public Jenkins/Cross-Vet + +.PHONY: cross-vet +cross-vet: ## Cross-Vet + GOOS=linux GOARCH=386 go vet ./... + GOOS=linux GOARCH=amd64 go vet ./... + GOOS=linux GOARCH=arm go vet ./... + GOOS=linux GOARCH=arm64 go vet ./... + GOOS=freebsd GOARCH=386 go vet ./... + GOOS=freebsd GOARCH=amd64 go vet ./... + GOOS=freebsd GOARCH=arm go vet ./... + GOOS=freebsd GOARCH=arm64 go vet ./... + GOOS=windows GOARCH=386 go vet ./... + GOOS=windows GOARCH=amd64 go vet ./... + + # TODO(artur): find out if we will be able to enable these: + # GOOS=windows GOARCH=arm go vet ./... + # GOOS=windows GOARCH=arm64 go vet ./... + + # Use kqueue to avoid using Cgo for verification. + GOOS=darwin GOARCH=amd64 go vet -tags kqueue ./... + GOOS=darwin GOARCH=arm64 go vet -tags kqueue ./... + +##@ Local development/Public Jenkins/Test + +JSON ?= false +SHORT ?= true +SKIP_TESTSUITE ?= false + +.PHONY: test +test: test-testsuite ## Test + go test -json=${JSON} -p 16 -parallel 4 -race -short=${SHORT} -timeout 10m -vet=off ./... + +.PHONY: test-testsuite +test-testsuite: ## Test testsuite +ifeq (${SKIP_TESTSUITE},false) + # Execute test-testsuite-do in testsuite directory: + $(MAKE) -C testsuite -f ../Makefile test-testsuite-do +endif + +.PHONY: test-testsuite-do +test-testsuite-do: + go vet ./... + go test -json=${JSON} -p 16 -parallel 4 -race -short=${SHORT} -timeout 10m -vet=off ./... + +##@ Local development/Public Jenkins/Verification + +.PHONY: verify +verify: lint cross-vet test ## Execute pre-commit verification + # todo(artur, sean): these extra ldflags are required for the minio object # browser to function, but should be automated. Use storj.io/minio/buildscripts/gen-ldflags.go LDFLAGS := -X storj.io/minio/cmd.Version=2022-04-19T11:13:21Z \ @@ -21,20 +166,6 @@ endif DOCKER_BUILD := docker build --build-arg TAG=${TAG} -.DEFAULT_GOAL := help -.PHONY: help -help: - @awk 'BEGIN { \ - FS = ":.*##"; \ - printf "\nUsage:\n make \033[36m\033[0m\n"\ - } \ - /^[a-zA-Z_-]+:.*?##/ { \ - printf " \033[36m%-17s\033[0m %s\n", $$1, $$2 \ - } \ - /^##@/ { \ - printf "\n\033[1m%s\033[0m\n", substr($$0, 5) \ - } ' $(MAKEFILE_LIST) - ##@ Dependencies .PHONY: build-dev-deps @@ -42,11 +173,6 @@ build-dev-deps: ## Install dependencies for builds go get golang.org/x/tools/cover go get github.com/josephspurrier/goversioninfo/cmd/goversioninfo -.PHONY: lint -lint: ## Analyze and find programs in source code - @echo "Running ${@}" - @golangci-lint run - .PHONY: goimports-fix goimports-fix: ## Applies goimports to every go file (excluding vendored files) goimports -w -local storj.io $$(find . -type f -name '*.go' -not -path "*/vendor/*") @@ -62,13 +188,6 @@ build-packages-normal: build-packages-race: go build -v -race ./... -##@ Test - -.PHONY: test -test: ## Run tests on source code (jenkins) - go test -race -v -cover -coverprofile=.coverprofile ./... - @echo done - ##@ Build .PHONY: images diff --git a/githooks/pre-commit b/githooks/pre-commit new file mode 100755 index 0000000..27e6055 --- /dev/null +++ b/githooks/pre-commit @@ -0,0 +1,5 @@ +#!/bin/sh + +# We need to unset GIT_INDEX_FILE because tools like check-mod-tidy will fail. +# See https://www.mail-archive.com/git@vger.kernel.org/msg178292.html +env -u GIT_INDEX_FILE make verify