-
Notifications
You must be signed in to change notification settings - Fork 31
303 lines (274 loc) · 13.8 KB
/
build-and-unit-test.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
# Copyright © 2022 Cask Data, Inc.
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
name: Build and Unit Test
on:
schedule:
- cron: '0 8 * * *'
workflow_dispatch:
# workaround to run manual trigger for a particular branch
inputs:
branch:
description: "Branch name on which workflow will be triggered"
required: true
default: "develop"
skip_tests:
description: "Skip running tests for this build"
required: true
type: boolean
default: false
run_tests_with_retries:
description: "Run tests with retries, it will be ignored if tests are skipped."
required: true
type: boolean
default: false
keep_existing_release:
description: "Do not overwrite the GitHub release for versions without snapshot"
required: true
type: boolean
default: true
env:
TAG_NAME: latest
PRE_RELEASE: true
REPLACE_ARTIFACTS: true
REMOVE_ARTIFACTS: true
SKIP_RELEASE: false
MAX_ATTEMPTS: 3
jobs:
set-branch-matrix:
runs-on: ubuntu-latest
steps:
- name: set-matrix
run: |
if [ ${{ github.event_name }} != "workflow_dispatch" ];
then
echo 'BRANCH_MATRIX={"include":[{"branch":"develop"}, {"branch":"release/6.9"}, {"branch":"release/6.10"}]}' >> $GITHUB_ENV
else
echo 'BRANCH_MATRIX={"include":[{"branch":"${{ github.event.inputs.branch }}"}]}' >> $GITHUB_ENV
if [ "${{ github.event.inputs.run_tests_with_retries }}" == "false" ]
then
echo 'MAX_ATTEMPTS=1' >> $GITHUB_ENV
fi
fi
outputs:
matrix: ${{ env.BRANCH_MATRIX }}
retries: ${{ env. MAX_ATTEMPTS }}
build:
needs: set-branch-matrix
runs-on: cdapio-hub-k8-runner
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.set-branch-matrix.outputs.matrix) }}
steps:
- name: Get Start Time
run: echo "STARTED_AT=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_ENV
- name: Get Secrets from GCP Secret Manager
id: 'secrets'
uses: 'google-github-actions/get-secretmanager-secrets@v2'
if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }}
with:
secrets: |-
CDAP_OSSRH_USERNAME:cdapio-github-builds/CDAP_OSSRH_USERNAME
CDAP_OSSRH_PASSWORD:cdapio-github-builds/CDAP_OSSRH_PASSWORD
CDAP_GPG_PASSPHRASE:cdapio-github-builds/CDAP_GPG_PASSPHRASE
CDAP_GPG_PRIVATE_KEY:cdapio-github-builds/CDAP_GPG_PRIVATE_KEY
- name: Recursively Checkout Repository
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: recursive
path: cdap-build
ref: ${{ matrix.branch }}
- name: Update Submodules
working-directory: cdap-build
run: |
git submodule update --init --recursive --remote
- name: Cache
uses: actions/cache@v4
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-${{ github.workflow }}
- name: Set up Artifact Name # Removes invalid artifact name characters: ",:,<,>,|,*,?,\,/.
if: "${{ github.event.inputs.skip_tests != 'true' }}"
run: |
name=$(echo -n "${{ matrix.branch }}" | sed -e 's/[ \t:\/\\"<>|*?]/-/g' -e 's/--*/-/g')
echo "ARTIFACT_NAME=$name" >> $GITHUB_ENV
- name: Run Tests
if: "${{ github.event.inputs.skip_tests != 'true' }}"
# Pinned version 2.8.2
uses: nick-fields/retry@3e91a01664abd3c5cd539100d10d33b9c5b68482
with:
timeout_minutes: 120
max_attempts: ${{ needs.set-branch-matrix.outputs.retries }}
retry_on: error
on_retry_command: echo "Tests failed in this attempt, retrying ..."
command: |
cd cdap-build
MAVEN_OPTS="-Xmx16G -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/cdap-build/oom.bin" mvn test -fae -T2 -U -V -am -amd -P templates,unit-tests -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30
- name: Archive build artifacts
uses: actions/upload-artifact@v4
if: "${{ github.event.inputs.skip_tests != 'true' || failure() }}"
with:
name: Build debug files - ${{ env.ARTIFACT_NAME }}
path: |
**/target/rat.txt
**/target/surefire-reports/*
/cdap-build/oom.bin
- name: Surefire Report
# Pinned 3.5.2 version
uses: mikepenz/action-junit-report@16a9560bd02f11e7e3bf6b3e2ef6bba6c9d07c32
if: ${{ always() }}
with:
report_paths: '**/target/surefire-reports/TEST-*.xml'
github_token: ${{ secrets.GITHUB_TOKEN }}
detailed_summary: true
commit: ${{ github.sha }}
check_name: Test Report - ${{ env.ARTIFACT_NAME }}
- name: Build Standalone
# Pinned version 2.8.2
uses: nick-fields/retry@3e91a01664abd3c5cd539100d10d33b9c5b68482
with:
timeout_minutes: 60
max_attempts: 3
retry_on: error
on_retry_command: echo "Build Standalone failed in this attempt, retrying ..."
command: |
cd cdap-build
MAVEN_OPTS="-Xmx12G" mvn -e -T2 clean package -Dgpg.skip -DskipTests -Ddocker.skip=true -nsu -am -amd -P templates,dist,release -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30
- name: Find Build Version
working-directory: cdap-build/cdap
run: |
export VERSION=$(ls cdap-standalone/target/cdap-sandbox*zip | cut --delimiter=- --fields="-3" --complement | rev | cut --delimiter="." --fields=1 --complement | rev)
echo "CDAP_VERSION=${VERSION}"
echo "CDAP_VERSION=${VERSION}" >> $GITHUB_ENV
if [[ ($VERSION != *-SNAPSHOT) && ("${{ github.event.inputs.keep_existing_release }}" != "false") ]];
then
echo "PRE_RELEASE=false" >> $GITHUB_ENV
echo "REPLACE_ARTIFACTS=false" >> $GITHUB_ENV
echo "REMOVE_ARTIFACTS=false" >> $GITHUB_ENV
echo "SKIP_RELEASE=true" >> $GITHUB_ENV
echo "Release will not be overwritten if exists."
else
echo "Release will be overwritten if exists."
fi
- name: Upload CDAP Standalone
if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }}
uses: actions/upload-artifact@v4 # https://github.com/actions/upload-artifact#zipped-artifact-downloads
with:
name: cdap-sandbox-${{env.CDAP_VERSION}}.zip
path: cdap-build/cdap/cdap-standalone/target/cdap-sandbox-${{env.CDAP_VERSION}}.zip
- name: Set up GPG conf
if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }}
run: |
echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf
echo "allow-loopback-pinentry" >> ~/.gnupg/gpg-agent.conf
- name: Import GPG key
if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }}
run: |
echo "$GPG_PRIVATE_KEY" > private.key
gpg --import --batch private.key
env:
GPG_PRIVATE_KEY: ${{ steps.secrets.outputs.CDAP_GPG_PRIVATE_KEY }}
- name: Maven Deploy
run: |
cd cdap-build
if [[ (${{ matrix.branch }} == "develop") || (${{ matrix.branch }} == release/*) ]];
then
retry_count=0
failed_module=""
while [ $retry_count -lt 3 ]; do
if [ -n "$failed_module" ];
then
build_output=$(mvn deploy -B -V -DskipTests -DskipLocalStaging=true -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dgpg.passphrase=$CDAP_GPG_PASSPHRASE -rf :"$failed_module" 2>&1 || true)
else
build_output=$(mvn deploy -B -V -DskipTests -DskipLocalStaging=true -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dgpg.passphrase=$CDAP_GPG_PASSPHRASE 2>&1 || true)
fi
echo "$build_output"
if [ $(echo "$build_output" | grep -c "BUILD FAILURE") -gt 0 ];
then
echo "[WARNING] Deployment failed, retrying..."
failed_module=$(echo "$build_output" | grep -e 'mvn <args> -rf' | sed -n 's/.*mvn <args> -rf ://p')
echo "[INFO] FAILED MODULE = $failed_module"
retry_count=$((retry_count + 1))
else
echo "[INFO] Deployment successful"
break
fi
done
if [ $retry_count -ge 3 ];
then
echo "[ERROR] Max retries reached..., deployment failed"
exit 1
fi
else
mvn verify -B -V -T2 -DskipTests -Dgpg.skip -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true
fi
env:
CDAP_OSSRH_USERNAME: ${{ steps.secrets.outputs.CDAP_OSSRH_USERNAME }}
CDAP_OSSRH_PASSWORD: ${{ steps.secrets.outputs.CDAP_OSSRH_PASSWORD }}
CDAP_GPG_PASSPHRASE: ${{ steps.secrets.outputs.CDAP_GPG_PASSPHRASE }}
MAVEN_OPTS: "-Xmx12G"
- name: Build DEB Bundle
working-directory: cdap-build/cdap
run: |
mkdir -p cdap-distributions/target/deb-bundle-tmp
cd cdap-distributions/target/deb-bundle-tmp
cp ../../../*/target/*.deb .
tar zcf ../cdap-distributed-deb-bundle-${{env.CDAP_VERSION}}.tgz *.deb
- name: Set Up Tag
working-directory: cdap-build
run: |
if [[ ${{ matrix.branch }} == release/* ]];
then
echo "TAG_NAME=v$CDAP_VERSION" >> $GITHUB_ENV
echo "TAG_NAME=v$CDAP_VERSION"
git tag -f v$CDAP_VERSION
git push -f origin refs/tags/v$CDAP_VERSION:refs/tags/v$CDAP_VERSION
elif [[ ${{ matrix.branch }} != "develop" ]];
then
export TAG=$(git check-ref-format --normalize "tags/${{ matrix.branch }}/tag" | cut -c6- | rev | cut -c5- | rev)
echo "TAG_NAME=$TAG" >> $GITHUB_ENV
echo "TAG_NAME=$TAG"
git tag -f $TAG
git push -f origin refs/tags/$TAG:refs/tags/$TAG
else
git tag -f ${{ env.TAG_NAME }}
git push -f origin refs/tags/${{ env.TAG_NAME }}:refs/tags/${{ env.TAG_NAME }}
fi
- name: Upload CDAP Standalone and CDAP DEB Bundle
# Pinned 1.14.0 version
uses: ncipollo/release-action@2c591bcc8ecdcd2db72b97d6147f871fcd833ba5
with:
allowUpdates: true
omitBodyDuringUpdate: true
omitNameDuringUpdate: true
omitPrereleaseDuringUpdate: true
skipIfReleaseExists: ${{ env.SKIP_RELEASE }}
prerelease: ${{ env.PRE_RELEASE }}
removeArtifacts: ${{ env.REMOVE_ARTIFACTS }}
replacesArtifacts: ${{ env.REPLACE_ARTIFACTS }}
token: ${{ secrets.GITHUB_TOKEN }}
tag: ${{ env.TAG_NAME }}
body: Cask Data Application Platform - Release ${{ env.CDAP_VERSION }}
artifacts: |
cdap-build/cdap/cdap-standalone/target/cdap-sandbox-${{env.CDAP_VERSION}}.zip,cdap-build/cdap/cdap-distributions/target/cdap-distributed-deb-bundle-${{env.CDAP_VERSION}}.tgz
- name: Alert team if build fails
if: ${{ (matrix.branch == 'develop' || startsWith(matrix.branch, 'release/')) && failure() }}
run: |
gcloud logging write cdapio-github-builds-logs '{ "message": "Build failure", "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}' --payload-type=json --severity=ERROR
- name: Get End Time
if: ${{ always() }}
run: echo "ENDED_AT=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_ENV
- name: Report Result
if: ${{ (matrix.branch == 'develop' || startsWith(matrix.branch, 'release/')) && github.event.inputs.skip_tests != 'true' && always() }}
run: |
echo '{ "repository": "${{ github.repository }}", "build_name": "${{ github.workflow }}", "branch": "${{ matrix.branch }}", "conclusion": "${{ job.status }}", "started_at": "'$STARTED_AT'", "ended_at": "'$ENDED_AT'", "url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}", "run_id": "${{ github.run_id }}", "run_attempt": "${{ github.run_attempt }}" }' | bq insert ${{ vars.BUILD_HISTORY_TABLE }}