-
Notifications
You must be signed in to change notification settings - Fork 1
/
Makefile
425 lines (378 loc) · 15.7 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
# ====================================================================================
# Variables
## General Variables
# Branch Variables
PROTECTED_BRANCH := master
CURRENT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
# Use repository name as application name
APP_NAME := $(shell basename -s .git `git config --get remote.origin.url`)
# Get current commit
APP_COMMIT := $(shell git log --pretty=format:'%h' -n 1)
# Check if we are in protected branch, if yes use `protected_branch_name-sha` as app version.
# Else check if we are in a release tag, if yes use the tag as app version, else use `dev-sha` as app version.
APP_VERSION ?= $(shell if [ $(PROTECTED_BRANCH) = $(CURRENT_BRANCH) ]; then echo $(PROTECTED_BRANCH); else (git describe --abbrev=0 --exact-match --tags 2>/dev/null || echo dev-$(APP_COMMIT)) ; fi)
# Get current date and format like: 2022-04-27 11:32
BUILD_DATE := $(shell date +%Y-%m-%d\ %H:%M)
## General Configuration Variables
# We don't need make's built-in rules.
MAKEFLAGS += --no-builtin-rules
# Be pedantic about undefined variables.
MAKEFLAGS += --warn-undefined-variables
# Set help as default target
.DEFAULT_GOAL := help
# App Code location
CONFIG_APP_CODE += ./cmd/transcriber
# Operating system arch
ifneq (, $(shell which go))
ARCH ?= $(shell go version | awk '{print substr($$4,index($$4,"/")+1)}')
endif
# Target OS will always be linux.
OS := linux
# Fallback to amd64 if ARCH is still unset.
ARCH ?= amd64
# We need to detect M1 host platforms.
IS_M1 := false
ifeq ($(shell uname -s),Darwin)
ifeq ($(shell uname -p),arm)
IS_M1 = true
endif
endif
## CGO dependencies
# Whisper.cpp
WHISPER_VERSION ?= "1.7.1"
WHISPER_SHA ?= "97f19a32212f2f215e538ee37a16ff547aaebc54817bd8072034e02466ce6d55"
WHISPER_MODELS ?= "tiny base small"
# Opus
OPUS_VERSION ?= "1.5.2"
OPUS_SHA ?= "65c1d2f78b9f2fb20082c38cbe47c951ad5839345876e46941612ee87f9a7ce1"
# ONNX Runtime
ONNX_VERSION ?= "1.18.1"
# Azure Speech SDK
AZURE_SDK_VERSION ?= "1.38.0"
AZURE_SDK_SHA ?= "26bbab32e16128c02af3b0f12640b15d13d969ec6dc748153004e380ff4c69d0"
## Docker Variables
# Docker executable
DOCKER := $(shell which docker)
# Dockerfile's location
DOCKER_FILE += ./build/Dockerfile
# Docker options to inherit for all docker run commands
DOCKER_OPTS += --rm -u $$(id -u):$$(id -g) --platform "linux/${ARCH}"
# Registry to upload images
DOCKER_REGISTRY ?= docker.io
DOCKER_REGISTRY_REPO ?= mattermost/${APP_NAME}-daily
DOCKER_TAG ?= ${APP_NAME}:${APP_VERSION}
# Registry credentials
DOCKER_USER ?= user
DOCKER_PASSWORD ?= password
## Docker Images
DOCKER_IMAGE_GO += "golang:${GO_VERSION}"
DOCKER_IMAGE_GOLINT += "golangci/golangci-lint:v1.60.0@sha256:e47065d755ca0afeac9df866d1dabdc99f439653a43fe234e05f50d9c36b6b90"
DOCKER_IMAGE_DOCKERLINT += "hadolint/hadolint:v2.12.0@sha256:9259e253a4e299b50c92006149dd3a171c7ea3c5bd36f060022b5d2c1ff0fbbe"
DOCKER_IMAGE_COSIGN += "bitnami/cosign:1.8.0@sha256:8c2c61c546258fffff18b47bb82a65af6142007306b737129a7bd5429d53629a"
DOCKER_IMAGE_GH_CLI += "ghcr.io/supportpal/github-gh-cli:2.31.0@sha256:71371e36e62bd24ddd42d9e4c720a7e9954cb599475e24d1407af7190e2a5685"
# When running locally we default to the current architecture.
DOCKER_BUILD_PLATFORMS := "${OS}/${ARCH}"
DOCKER_BUILD_OUTPUT_TYPE := "docker"
DOCKER_BUILDER := "multiarch"
DOCKER_BUILDER_MISSING := $(shell docker buildx inspect ${DOCKER_BUILDER} > /dev/null 2>&1; echo $$?)
# When running on CI we want to use our official release targets.
ifeq ($(CI),true)
DOCKER_BUILD_PLATFORMS := "linux/amd64,linux/arm64"
DOCKER_BUILD_OUTPUT_TYPE := "registry"
DOCKER_TAG := ${DOCKER_REGISTRY}/${DOCKER_REGISTRY_REPO}:${APP_VERSION}
endif
## Cosign Variables
# The public key
COSIGN_PUBLIC_KEY ?= akey
# The private key
COSIGN_KEY ?= akey
# The passphrase used to decrypt the private key
COSIGN_PASSWORD ?= password
## Go Variables
# Go executable
GO := $(shell which go)
# Extract GO version from go.mod file
GO_VERSION ?= $(shell grep -E '^go' go.mod | awk {'print $$2'})
# LDFLAGS
GO_LDFLAGS += -X "github.com/mattermost/${APP_NAME}/service.buildHash=$(APP_COMMIT)"
GO_LDFLAGS += -X "github.com/mattermost/${APP_NAME}/service.buildVersion=$(APP_VERSION)"
GO_LDFLAGS += -X "github.com/mattermost/${APP_NAME}/service.buildDate=$(BUILD_DATE)"
GO_LDFLAGS += -X "github.com/mattermost/${APP_NAME}/service.goVersion=$(GO_VERSION)"
GO_LDFLAGS += -X "github.com/mattermost/${APP_NAME}/cmd/transcriber/config.inTranscriber=true"
# Architectures to build for
GO_BUILD_PLATFORMS ?= "linux-${ARCH}"
GO_BUILD_PLATFORMS_ARTIFACTS = $(foreach cmd,$(addprefix go-build/,${APP_NAME}),$(addprefix $(cmd)-,$(GO_BUILD_PLATFORMS)))
# Build options
GO_BUILD_OPTS += -mod=readonly -trimpath -buildmode=pie
GO_TEST_OPTS += -mod=readonly -failfast -race
# Temporary folder to output compiled binaries artifacts
GO_OUT_BIN_DIR := ./dist
# We need to export GOBIN to allow it to be set
# for processes spawned from the Makefile
export GOBIN ?= $(PWD)/bin
## Github Variables
# A github access token that provides access to upload artifacts under releases
GITHUB_TOKEN ?= a_token
# Github organization
GITHUB_ORG := mattermost
# Most probably the name of the repo
GITHUB_REPO := ${APP_NAME}
# ====================================================================================
# Colors
BLUE := $(shell printf "\033[34m")
YELLOW := $(shell printf "\033[33m")
RED := $(shell printf "\033[31m")
GREEN := $(shell printf "\033[32m")
CYAN := $(shell printf "\033[36m")
CNone := $(shell printf "\033[0m")
# ====================================================================================
# Logger
TIME_LONG = `date +%Y-%m-%d' '%H:%M:%S`
TIME_SHORT = `date +%H:%M:%S`
TIME = $(TIME_SHORT)
INFO = echo ${TIME} ${BLUE}[ .. ]${CNone}
WARN = echo ${TIME} ${YELLOW}[WARN]${CNone}
ERR = echo ${TIME} ${RED}[FAIL]${CNone}
OK = echo ${TIME} ${GREEN}[ OK ]${CNone}
FAIL = (echo ${TIME} ${RED}[FAIL]${CNone} && false)
# ====================================================================================
# Verbosity control hack
VERBOSE ?= 0
AT_0 := @
AT_1 :=
AT = $(AT_$(VERBOSE))
# ====================================================================================
# Targets
help: ## to get help
@echo "Usage:"
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) |\
awk 'BEGIN {FS = ":.*?## "}; {printf "make ${CYAN}%-30s${CNone} %s\n", $$1, $$2}'
.PHONY: build
build: go-build-docker ## to build
.PHONY: release
release: build github-release ## to build and release artifacts
.PHONY: package
package: docker-login docker-build ## to build, package and push the artifact to a container registry
.PHONY: sign
sign: docker-sign docker-verify ## to sign the artifact and perform verification
.PHONY: lint
lint: go-lint ## to lint
.PHONY: test
test: go-test ## to test
docker-build: ## to build the docker image
@$(INFO) Performing Docker build ${APP_NAME}:${APP_VERSION} for ${DOCKER_BUILD_PLATFORMS}
ifeq ($(DOCKER_BUILDER_MISSING),1)
ifeq ($(CI),true)
@$(INFO) Creating ${DOCKER_BUILDER} builder
$(AT)$(DOCKER) buildx create --name ${DOCKER_BUILDER} --use
endif
endif
$(AT)$(DOCKER) buildx build \
--platform ${DOCKER_BUILD_PLATFORMS} \
--output=type=${DOCKER_BUILD_OUTPUT_TYPE} \
--build-arg GO_VERSION=${GO_VERSION} \
--build-arg OPUS_VERSION=${OPUS_VERSION} \
--build-arg OPUS_SHA=${OPUS_SHA} \
--build-arg WHISPER_VERSION=${WHISPER_VERSION} \
--build-arg WHISPER_SHA=${WHISPER_SHA} \
--build-arg WHISPER_MODELS=${WHISPER_MODELS} \
--build-arg ONNX_VERSION=${ONNX_VERSION} \
--build-arg IS_M1=${IS_M1} \
--build-arg AZURE_SDK_VERSION=${AZURE_SDK_VERSION} \
--build-arg AZURE_SDK_SHA=${AZURE_SDK_SHA} \
-f ${DOCKER_FILE} . \
-t ${DOCKER_TAG} || ${FAIL}
@$(OK) Performing Docker build ${APP_NAME}:${APP_VERSION} for ${DOCKER_BUILD_PLATFORMS}
ifeq ($(CI),true)
ifneq ($(shell echo $(APP_VERSION) | egrep '^v([0-9]+\.){0,2}(\*|[0-9]+)'),)
ifeq ($(shell git tag -l --sort=v:refname | tail -n1),$(APP_VERSION))
$(AT)$(DOCKER) buildx build \
--platform ${DOCKER_BUILD_PLATFORMS} \
--output=type=${DOCKER_BUILD_OUTPUT_TYPE} \
--build-arg GO_VERSION=${GO_VERSION} \
--build-arg OPUS_VERSION=${OPUS_VERSION} \
--build-arg OPUS_SHA=${OPUS_SHA} \
--build-arg WHISPER_VERSION=${WHISPER_VERSION} \
--build-arg WHISPER_SHA=${WHISPER_SHA} \
--build-arg WHISPER_MODELS=${WHISPER_MODELS} \
--build-arg ONNX_VERSION=${ONNX_VERSION} \
--build-arg AZURE_SDK_VERSION=${AZURE_SDK_VERSION} \
--build-arg AZURE_SDK_SHA=${AZURE_SDK_SHA} \
-f ${DOCKER_FILE} . \
-t ${DOCKER_REGISTRY}/${DOCKER_REGISTRY_REPO}:latest || ${FAIL}
endif
endif
endif
.PHONY: docker-sign
docker-sign: ## to sign the docker image
@$(INFO) Signing the docker image...
$(AT)echo "$${COSIGN_KEY}" > cosign.key && \
$(DOCKER) run ${DOCKER_OPTS} \
--entrypoint '/bin/sh' \
-v $(PWD):/app -w /app \
-e COSIGN_PASSWORD=${COSIGN_PASSWORD} \
-e HOME="/tmp" \
${DOCKER_IMAGE_COSIGN} \
-c \
"echo Signing... && \
cosign login $(DOCKER_REGISTRY) -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} && \
cosign sign --key cosign.key $(DOCKER_REGISTRY)/${DOCKER_REGISTRY_REPO}:${APP_VERSION}" || ${FAIL}
# if we are on a latest semver APP_VERSION tag, also sign latest tag
ifneq ($(shell echo $(APP_VERSION) | egrep '^v([0-9]+\.){0,2}(\*|[0-9]+)'),)
ifeq ($(shell git tag -l --sort=v:refname | tail -n1),$(APP_VERSION))
$(DOCKER) run ${DOCKER_OPTS} \
--entrypoint '/bin/sh' \
-v $(PWD):/app -w /app \
-e COSIGN_PASSWORD=${COSIGN_PASSWORD} \
-e HOME="/tmp" \
${DOCKER_IMAGE_COSIGN} \
-c \
"echo Signing... && \
cosign login $(DOCKER_REGISTRY) -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} && \
cosign sign --key cosign.key $(DOCKER_REGISTRY)/${DOCKER_REGISTRY_REPO}:latest" || ${FAIL}
endif
endif
$(AT)rm -f cosign.key || ${FAIL}
@$(OK) Signing the docker image: $(DOCKER_REGISTRY)/${DOCKER_REGISTRY_REPO}:${APP_VERSION}
.PHONY: docker-verify
docker-verify: ## to verify the docker image
@$(INFO) Verifying the published docker image...
$(AT)echo "$${COSIGN_PUBLIC_KEY}" > cosign_public.key && \
$(DOCKER) run ${DOCKER_OPTS} \
--entrypoint '/bin/sh' \
-v $(PWD):/app -w /app \
${DOCKER_IMAGE_COSIGN} \
-c \
"echo Verifying... && \
cosign verify --key cosign_public.key $(DOCKER_REGISTRY)/${DOCKER_REGISTRY_REPO}:${APP_VERSION}" || ${FAIL}
# if we are on a latest semver APP_VERSION tag, also verify latest tag
ifneq ($(shell echo $(APP_VERSION) | egrep '^v([0-9]+\.){0,2}(\*|[0-9]+)'),)
ifeq ($(shell git tag -l --sort=v:refname | tail -n1),$(APP_VERSION))
$(DOCKER) run ${DOCKER_OPTS} \
--entrypoint '/bin/sh' \
-v $(PWD):/app -w /app \
${DOCKER_IMAGE_COSIGN} \
-c \
"echo Verifying... && \
cosign verify --key cosign_public.key $(DOCKER_REGISTRY)/${DOCKER_REGISTRY_REPO}:latest" || ${FAIL}
endif
endif
$(AT)rm -f cosign_public.key || ${FAIL}
@$(OK) Verifying the published docker image: $(DOCKER_REGISTRY)/${DOCKER_REGISTRY_REPO}:${APP_VERSION}
.PHONY: docker-sbom
docker-sbom: ## to print a sbom report
@$(INFO) Performing Docker sbom report...
$(AT)$(DOCKER) sbom ${APP_NAME}:${APP_VERSION} || ${FAIL}
@$(OK) Performing Docker sbom report
.PHONY: docker-scan
docker-scan: ## to print a vulnerability report
@$(INFO) Performing Docker scan report...
$(AT)$(DOCKER) scan ${APP_NAME}:${APP_VERSION} || ${FAIL}
@$(OK) Performing Docker scan report
.PHONY: docker-lint
docker-lint: ## to lint the Dockerfile
@$(INFO) Dockerfile linting...
$(AT)$(DOCKER) run -i ${DOCKER_OPTS} \
${DOCKER_IMAGE_DOCKERLINT} \
< ${DOCKER_FILE} || ${FAIL}
@$(OK) Dockerfile linting
.PHONY: docker-login
docker-login: ## to login to a container registry
@$(INFO) Dockerd login to container registry ${DOCKER_REGISTRY}...
$(AT) echo "${DOCKER_PASSWORD}" | $(DOCKER) login --password-stdin -u ${DOCKER_USER} $(DOCKER_REGISTRY) || ${FAIL}
@$(OK) Dockerd login to container registry ${DOCKER_REGISTRY}...
go-build: $(GO_BUILD_PLATFORMS_ARTIFACTS) ## to build binaries
.PHONY: go-build
go-build/%:
@$(INFO) go build $*...
$(AT)target="$*"; \
command="${APP_NAME}"; \
platform_ext="$${target#$$command-*}"; \
platform="$${platform_ext%.*}"; \
export GOOS="$${platform%%-*}"; \
export GOARCH="$${platform#*-}"; \
echo export GOOS=$${GOOS}; \
echo export GOARCH=$${GOARCH}; \
CGO_ENABLED=1 \
$(GO) build ${GO_BUILD_OPTS} \
-ldflags '${GO_LDFLAGS}' \
-o ${GO_OUT_BIN_DIR}/$* \
${CONFIG_APP_CODE} || ${FAIL}
@$(OK) go build $*
.PHONY: go-build-docker
go-build-docker: # to build binaries under a controlled docker dedicated go container using DOCKER_IMAGE_GO
@$(INFO) go build docker
$(AT)$(DOCKER) run ${DOCKER_OPTS} \
-v $(PWD):/app -w /app \
-e GOCACHE="/tmp" \
$(DOCKER_IMAGE_GO) \
/bin/bash ./build/build.sh ${OPUS_VERSION} ${OPUS_SHA} ${WHISPER_VERSION} ${WHISPER_SHA} ${WHISPER_MODELS} ${ONNX_VERSION} ${ARCH} ${AZURE_SDK_VERSION} ${AZURE_SDK_SHA} || ${FAIL}
@$(OK) go build docker
.PHONY: go-run
go-run: ## to run locally for development
@$(INFO) running locally...
$(AT)$(GO) run ${GO_BUILD_OPTS} ${CONFIG_APP_CODE} || ${FAIL}
@$(OK) running locally
.PHONY: go-test
go-test: ## to run tests
@$(INFO) testing...
$(AT)$(DOCKER) run ${DOCKER_OPTS} \
-v $(PWD):/app -w /app \
-v /var/run/docker.sock:/var/run/docker.sock \
-e GOCACHE="/tmp" \
$(DOCKER_IMAGE_GO) \
/bin/sh ./build/run_tests.sh "${GO_TEST_OPTS}" "${OPUS_VERSION}" "${OPUS_SHA}" "${WHISPER_VERSION}" "${WHISPER_SHA}" "${ONNX_VERSION}" "${ARCH}" "${AZURE_SDK_VERSION}" "${AZURE_SDK_SHA}" || ${FAIL}
@$(OK) testing
.PHONY: go-mod-check
go-mod-check: ## to check go mod files consistency
@$(INFO) Checking go mod files consistency...
$(AT)$(GO) mod tidy
$(AT)git --no-pager diff --exit-code go.mod go.sum || \
(${WARN} Please run "go mod tidy" and commit the changes in go.mod and go.sum. && ${FAIL} ; exit 128 )
@$(OK) Checking go mod files consistency
.PHONY: go-update-dependencies
go-update-dependencies: ## to update go dependencies (vendor)
@$(INFO) updating go dependencies...
$(AT)$(GO) get -u ./... && \
$(AT)$(GO) mod vendor && \
$(AT)$(GO) mod tidy || ${FAIL}
@$(OK) updating go dependencies
.PHONY: go-lint
go-lint: ## to lint go code
@$(INFO) App linting...
$(AT)GOCACHE="/tmp" $(DOCKER) run ${DOCKER_OPTS} \
-v $(PWD):/app -w /app \
-e GOCACHE="/tmp" \
-e GOLANGCI_LINT_CACHE="/tmp" \
${DOCKER_IMAGE_GOLINT} \
/bin/sh ./build/lint.sh "${OPUS_VERSION}" "${OPUS_SHA}" "${WHISPER_VERSION}" "${WHISPER_SHA}" "${ONNX_VERSION}" "${ARCH}" "${AZURE_SDK_VERSION}" "${AZURE_SDK_SHA}" || ${FAIL}
@$(OK) App linting
.PHONY: go-fmt
go-fmt: ## to perform formatting
@$(INFO) App code formatting...
$(AT)$(GO) fmt ./... || ${FAIL}
@$(OK) App code formatting...
.PHONY: github-release
github-release: ## to publish a release and relevant artifacts to GitHub
@$(INFO) Generating github-release http://github.com/$(GITHUB_ORG)/$(GITHUB_REPO)/releases/tag/$(APP_VERSION) ...
ifeq ($(shell echo $(APP_VERSION) | egrep '^v([0-9]+\.){0,2}(\*|[0-9]+)'),)
$(error "We only support releases from semver tags")
else
$(AT)$(DOCKER) run \
-v $(PWD):/app -w /app \
-e GITHUB_TOKEN=${GITHUB_TOKEN} \
$(DOCKER_IMAGE_GH_CLI) \
/bin/sh -c \
"git config --global --add safe.directory /app && cd /app && \
gh release create $(APP_VERSION) --generate-notes $(GO_OUT_BIN_DIR)/*" || ${FAIL}
endif
@$(OK) Generating github-release http://github.com/$(GITHUB_ORG)/$(GITHUB_REPO)/releases/tag/$(APP_VERSION) ...
.PHONY: clean
clean: ## to clean-up
@$(INFO) cleaning /${GO_OUT_BIN_DIR} folder...
$(AT)rm -rf ${GO_OUT_BIN_DIR} || ${FAIL}
@$(OK) cleaning /${GO_OUT_BIN_DIR} folder
.PHONY: mocks
mocks: ## Create mock files
$(GO) install github.com/vektra/mockery/v2/[email protected]
$(GOBIN)/mockery