Skip to content

Commit

Permalink
ci: rework Jenkinsfile/Makefile
Browse files Browse the repository at this point in the history 
This change reworks Gateway-ST's CI so that it's easier to develop
locally and be more productive.

1. It adds dependencies installation as a "one-click" operation
    a. Similarly, it allows installing helpful Git hooks (this change
       adds an installable pre-commit hook that runs verification)
2. It moves directly-called linting/testing from Jenkinsfile to Makefile
    a. New make targets are called from Jenkinsfile

New make targets (e.g., verify) can be called to aid local development
(sometimes it's useful to test/reproduce locally).

Integration tests have not yet been moved. They will be moved in the
next changes. Obsolete targets will also be purged in the next changes.

Change-Id: Icc77fd6159db5bd74be536e53bd5cd62656e44a0
  • Loading branch information
@amwolff
amwolff committed Jul 15, 2022
1 parent 78cd12d commit 4f1f9e1
Show file tree
Hide file tree
Showing 3 changed files with 174 additions and 95 deletions.
93 changes: 24 additions & 69 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ pipeline {

environment {
GOTRACEBACK = 'all'
COCKROACH_MEMPROF_INTERVAL = 0
// COCKROACH_MEMPROF_INTERVAL = 0
// ^ doesn't work: https://github.com/cockroachdb/cockroach/issues/54793
}

stages {
Expand Down Expand Up @@ -52,74 +53,51 @@ pipeline {
stage('Verification') {
parallel {
stage('Lint') {
environment {
GOLANGCI_LINT_CONFIG = '/go/ci/.golangci.yml'
GOLANGCI_LINT_CONFIG_TESTSUITE = '/go/ci/.golangci.yml'
}
steps {
sh 'check-mod-tidy'
sh 'check-copyright'
sh 'check-large-files'
sh 'check-imports -race ./...'
sh 'check-peer-constraints -race'
sh 'check-atomic-align ./...'
sh 'check-monkit ./...'
sh 'check-errs ./...'
sh 'check-deferloop ./...'
sh 'staticcheck ./...'
sh 'golangci-lint run --config /go/ci/.golangci.yml'
sh 'check-downgrades'

// A bit of an explanation around this shellcheck command:
// * Find all scripts recursively that have the .sh extension, except for "testsuite@tmp" which Jenkins creates temporarily.
// * Use + instead of \ so find returns a non-zero exit if any invocation of shellcheck returns a non-zero exit.
sh 'find . -path ./testsuite@tmp -prune -o -name "*.sh" -type f -exec "shellcheck" "-x" "--format=gcc" {} +;'
sh 'make lint'
}
}

dir('testsuite') {
sh 'check-imports -race ./...'
sh 'check-atomic-align ./...'
sh 'check-monkit ./...'
sh 'check-errs ./...'
sh 'check-deferloop ./...'
sh 'staticcheck ./...'
sh 'golangci-lint run --config /go/ci/.golangci.yml'
}
stage('Cross-Vet') {
steps {
sh 'make cross-vet'
}
}

stage('Test') {
environment {
COVERFLAGS = "${ env.BRANCH_NAME != 'main' ? '' : '-coverprofile=.build/coverprofile -coverpkg=./...'}"
JSON = true
SHORT = false
SKIP_TESTSUITE = true
}
steps {
sh 'go test -parallel 4 -p 16 -vet=off ${COVERFLAGS} -timeout 10m -json -race ./... 2>&1 | tee .build/tests.json | xunit -out .build/tests.xml'
sh 'make test 2>&1 | tee .build/tests.json | xunit -out .build/tests.xml'
}
post {
always {
sh script: 'cat .build/tests.json | tparse -all -top -slow 100', returnStatus: true
archiveArtifacts artifacts: '.build/tests.json'
junit '.build/tests.xml'
script {
if (fileExists('.build/coverprofile')) {
sh script: 'filter-cover-profile < .build/coverprofile > .build/clean.coverprofile', returnStatus: true
sh script: 'gocov convert .build/clean.coverprofile > .build/cover.json', returnStatus: true
sh script: 'gocov-xml < .build/cover.json > .build/cobertura.xml', returnStatus: true
cobertura coberturaReportFile: '.build/cobertura.xml'
}
}
}
}
}

stage('Testsuite') {
environment {
STORJ_TEST_COCKROACH = 'cockroach://root@localhost:26257/testcockroach?sslmode=disable'
STORJ_TEST_POSTGRES = 'postgres://postgres@localhost/teststorj?sslmode=disable'
JSON = true
SHORT = false
STORJ_TEST_COCKROACH = 'cockroach://root@localhost:26257/postgres?sslmode=disable'
STORJ_TEST_POSTGRES = 'postgres://postgres@localhost/postgres?sslmode=disable'
}
steps {
sh 'cockroach sql --insecure --host=localhost:26257 -e \'create database testcockroach;\''
sh 'psql -U postgres -c \'create database teststorj;\''
// exhaust ports from 1024 to 10000 to ensure we don't
// use hardcoded ports
sh 'use-ports -from 1024 -to 10000 &'
dir('testsuite') {
sh 'go vet ./...'
sh 'go test -parallel 4 -p 16 -vet=off -timeout 10m -json -race ./... 2>&1 | tee ../.build/testsuite.json | xunit -out ../.build/testsuite.xml'
}
sh 'make --no-print-directory test-testsuite 2>&1 | tee .build/testsuite.json | xunit -out .build/testsuite.xml'
}
post {
always {
Expand All @@ -135,11 +113,9 @@ pipeline {
// use different hostname to avoid port conflicts
STORJ_NETWORK_HOST4 = '127.0.0.2'
STORJ_NETWORK_HOST6 = '127.0.0.2'

STORJ_SIM_POSTGRES = 'postgres://postgres@localhost/integration?sslmode=disable'
STORJ_SIM_POSTGRES = 'postgres://postgres@localhost/postgres?sslmode=disable'
}
steps {
sh 'psql -U postgres -c \'create database integration;\''
sh 'cd ./testsuite/integration && ./run.sh'
}
post {
Expand All @@ -150,27 +126,6 @@ pipeline {
}
}
}

stage('Cross Compile') {
steps {
sh 'GOOS=linux GOARCH=386 go vet ./...'
sh 'GOOS=linux GOARCH=amd64 go vet ./...'
sh 'GOOS=linux GOARCH=arm go vet ./...'
sh 'GOOS=linux GOARCH=arm64 go vet ./...'
sh 'GOOS=freebsd GOARCH=386 go vet ./...'
sh 'GOOS=freebsd GOARCH=amd64 go vet ./...'
sh 'GOOS=freebsd GOARCH=arm64 go vet ./...'
sh 'GOOS=windows GOARCH=386 go vet ./...'
sh 'GOOS=windows GOARCH=amd64 go vet ./...'

// TODO(artur): find out if we will be able to enable it
// sh 'GOOS=windows GOARCH=arm64 go vet ./...'

// Use kqueue to avoid needing cgo for verification.
sh 'GOOS=darwin GOARCH=amd64 go vet -tags kqueue ./...'
sh 'GOOS=darwin GOARCH=arm64 go vet -tags kqueue ./...'
}
}
}
}

Expand Down
171 changes: 145 additions & 26 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,151 @@ GO_VERSION ?= 1.17.12
BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD | sed "s!/!-!g")
LATEST_DEV_TAG := dev

#
# Common
#

.DEFAULT_GOAL := help
.PHONY: help
help:
@awk 'BEGIN { \
FS = ":.*##"; \
printf "\nUsage:\n make \033[36m<target>\033[0m\n" \
} \
/^[a-zA-Z_-]+:.*?##/ { \
printf " \033[36m%-24s\033[0m %s\n", $$1, $$2 \
} \
/^##@/ { \
printf "\n\033[1m%s\033[0m\n", substr($$0, 5) \
}' $(MAKEFILE_LIST)

#
# Public Jenkins (commands below are used for local development and/or public Jenkins)
#

##@ Local development/Public Jenkins/Helpers

.PHONY: install-dev-dependencies
install-dev-dependencies: ## install-dev-dependencies assumes Go and cURL are installed
# Storj-specific:
go install github.com/storj/ci/check-mod-tidy@latest
go install github.com/storj/ci/check-copyright@latest
go install github.com/storj/ci/check-large-files@latest
go install github.com/storj/ci/check-imports@latest
go install github.com/storj/ci/check-peer-constraints@latest
go install github.com/storj/ci/check-atomic-align@latest
go install github.com/storj/ci/check-monkit@latest
go install github.com/storj/ci/check-errs@latest
go install github.com/storj/ci/check-deferloop@latest
go install github.com/storj/ci/check-downgrades@latest

# staticcheck:
go install honnef.co/go/tools/cmd/staticcheck@latest

# golangci-lint:
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin v1.45.2

# shellcheck (TODO(artur): Windows)
ifneq ($(shell which apt-get),)
sudo apt-get install -y shellcheck
else ifneq ($(shell which brew),)
brew install shellcheck
else
$(error Can't install shellcheck without a supported package manager)
endif

.PHONY: install-hooks
install-hooks: ## Install helpful Git hooks
ln -s .git/hooks/pre-commit githooks/pre-commit

##@ Local development/Public Jenkins/Lint

GOLANGCI_LINT_CONFIG ?= ../ci/.golangci.yml
GOLANGCI_LINT_CONFIG_TESTSUITE ?= ../../ci/.golangci.yml

.PHONY: lint
lint: ## Lint
check-mod-tidy
check-copyright
check-large-files
check-imports -race ./...
check-peer-constraints -race
check-atomic-align ./...
check-monkit ./...
check-errs ./...
check-deferloop ./...
staticcheck ./...
golangci-lint run --print-resources-usage --config ${GOLANGCI_LINT_CONFIG}
check-downgrades

# A bit of an explanation around this shellcheck command:
# * Find all scripts recursively that have the .sh extension, except for "testsuite@tmp" which Jenkins creates temporarily
# * Use + instead of \ so find returns a non-zero exit if any invocation of shellcheck returns a non-zero exit
find . -path ./testsuite@tmp -prune -o -name "*.sh" -type f -exec "shellcheck" "-x" "--format=gcc" {} +;

# Execute lint-testsuite in testsuite directory:
$(MAKE) -C testsuite -f ../Makefile lint-testsuite

.PHONY: lint-testsuite
lint-testsuite: ## Lint testsuite
check-imports -race ./...
check-atomic-align ./...
check-monkit ./...
check-errs ./...
check-deferloop ./...
staticcheck ./...
golangci-lint run --print-resources-usage --config ${GOLANGCI_LINT_CONFIG_TESTSUITE}

##@ Local development/Public Jenkins/Cross-Vet

.PHONY: cross-vet
cross-vet: ## Cross-Vet
GOOS=linux GOARCH=386 go vet ./...
GOOS=linux GOARCH=amd64 go vet ./...
GOOS=linux GOARCH=arm go vet ./...
GOOS=linux GOARCH=arm64 go vet ./...
GOOS=freebsd GOARCH=386 go vet ./...
GOOS=freebsd GOARCH=amd64 go vet ./...
GOOS=freebsd GOARCH=arm go vet ./...
GOOS=freebsd GOARCH=arm64 go vet ./...
GOOS=windows GOARCH=386 go vet ./...
GOOS=windows GOARCH=amd64 go vet ./...

# TODO(artur): find out if we will be able to enable these:
# GOOS=windows GOARCH=arm go vet ./...
# GOOS=windows GOARCH=arm64 go vet ./...

# Use kqueue to avoid using Cgo for verification.
GOOS=darwin GOARCH=amd64 go vet -tags kqueue ./...
GOOS=darwin GOARCH=arm64 go vet -tags kqueue ./...

##@ Local development/Public Jenkins/Test

JSON ?= false
SHORT ?= true
SKIP_TESTSUITE ?= false

.PHONY: test
test: test-testsuite ## Test
go test -json=${JSON} -p 16 -parallel 4 -race -short=${SHORT} -timeout 10m -vet=off ./...

.PHONY: test-testsuite
test-testsuite: ## Test testsuite
ifeq (${SKIP_TESTSUITE},false)
# Execute test-testsuite-do in testsuite directory:
$(MAKE) -C testsuite -f ../Makefile test-testsuite-do
endif

.PHONY: test-testsuite-do
test-testsuite-do:
go vet ./...
go test -json=${JSON} -p 16 -parallel 4 -race -short=${SHORT} -timeout 10m -vet=off ./...

##@ Local development/Public Jenkins/Verification

.PHONY: verify
verify: lint cross-vet test ## Execute pre-commit verification

# todo(artur, sean): these extra ldflags are required for the minio object
# browser to function, but should be automated. Use storj.io/minio/buildscripts/gen-ldflags.go
LDFLAGS := -X storj.io/minio/cmd.Version=2022-04-19T11:13:21Z \
Expand All @@ -21,32 +166,13 @@ endif

DOCKER_BUILD := docker build --build-arg TAG=${TAG}

.DEFAULT_GOAL := help
.PHONY: help
help:
@awk 'BEGIN { \
FS = ":.*##"; \
printf "\nUsage:\n make \033[36m<target>\033[0m\n"\
} \
/^[a-zA-Z_-]+:.*?##/ { \
printf " \033[36m%-17s\033[0m %s\n", $$1, $$2 \
} \
/^##@/ { \
printf "\n\033[1m%s\033[0m\n", substr($$0, 5) \
} ' $(MAKEFILE_LIST)

##@ Dependencies

.PHONY: build-dev-deps
build-dev-deps: ## Install dependencies for builds
go get golang.org/x/tools/cover
go get github.com/josephspurrier/goversioninfo/cmd/goversioninfo

.PHONY: lint
lint: ## Analyze and find programs in source code
@echo "Running ${@}"
@golangci-lint run

.PHONY: goimports-fix
goimports-fix: ## Applies goimports to every go file (excluding vendored files)
goimports -w -local storj.io $$(find . -type f -name '*.go' -not -path "*/vendor/*")
Expand All @@ -62,13 +188,6 @@ build-packages-normal:
build-packages-race:
go build -v -race ./...

##@ Test

.PHONY: test
test: ## Run tests on source code (jenkins)
go test -race -v -cover -coverprofile=.coverprofile ./...
@echo done

##@ Build

.PHONY: images
Expand Down
5 changes: 5 additions & 0 deletions githooks/pre-commit
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#!/bin/sh

# We need to unset GIT_INDEX_FILE because tools like check-mod-tidy will fail.
# See https://www.mail-archive.com/[email protected]/msg178292.html
env -u GIT_INDEX_FILE make verify

0 comments on commit 4f1f9e1

Please sign in to comment.