forked from Azure/azure-storage-azcopy
-
Notifications
You must be signed in to change notification settings - Fork 0
/
azure-pipelines.yml
213 lines (199 loc) · 8.58 KB
/
azure-pipelines.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
trigger:
branches:
include:
- dev
- main
pr:
branches:
include:
- dev
- main
jobs:
- job: Build
timeoutInMinutes: 360
strategy:
matrix:
Ubuntu-20:
imageName: 'ubuntu-20.04'
type: 'linux'
MacOS:
imageName: 'macOS-10.15'
type: 'mac-os'
pool:
vmImage: $(imageName)
steps:
- task: GoTool@0
inputs:
version: '1.16'
- script: |
GOARCH=amd64 GOOS=linux go build -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_amd64"
GOARCH=amd64 GOOS=linux go build -tags "se_integration" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_se_amd64"
GOARCH=arm64 GOOS=linux go build -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_arm64"
GOARCH=amd64 GOOS=windows go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_amd64.exe"
GOARCH=386 GOOS=windows go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_386.exe"
GOARCH=arm GOARM=7 GOOS=windows go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_v7_arm.exe"
cp NOTICE.txt $(Build.ArtifactStagingDirectory)
displayName: 'Generate Linux And Windows Build'
condition: eq(variables.type, 'linux')
- script: |
go build -o "$(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64"
displayName: 'Generate MacOS Build'
condition: eq(variables.type, 'mac-os')
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifacts'
condition: succeededOrFailed()
- job: E2E_Test
timeoutInMinutes: 360
# Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04
strategy:
matrix:
Ubuntu-20:
imageName: 'ubuntu-20.04'
type: 'linux'
Windows:
imageName: 'windows-2019'
type: 'windows'
MacOS:
imageName: 'macOS-10.15'
type: 'mac-os'
pool:
vmImage: $(imageName)
steps:
- task: GoTool@0
inputs:
version: '1.16'
# Running E2E Tests on Linux - AMD64
- script: |
set -e
GOARCH=amd64 GOOS=linux go build -o azcopy_linux_amd64
export AZCOPY_E2E_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64
go test -timeout 20m -race -short -cover ./e2etest
env:
AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY)
AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME)
AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS)
AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS)
CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY)
CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256)
displayName: 'E2E Test Linux - AMD64'
condition: eq(variables.type, 'linux')
# Running E2E Tests on Windows - AMD64
- script: |
go build -o $(System.DefaultWorkingDirectory)/azcopy_windows_amd64.exe
go build -o $(System.DefaultWorkingDirectory)/azcopy_windows_386.exe
echo 'starting E2E tests on windows'
go test -timeout 30m -race -cover -v ./e2etest
env:
AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY)
AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME)
AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS)
AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS)
CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY)
CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256)
AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/azcopy_windows_amd64.exe
displayName: 'E2E Test Windows - AMD64'
condition: eq(variables.type, 'windows')
# Running E2E Tests on Mac
- script: |
set -e
go build -o azcopy_darwin_amd64
echo 'starting E2E tests on mac-os'
export AZCOPY_E2E_EXECUTABLE_PATH=$(pwd)/azcopy_darwin_amd64
go test -timeout 30m -race -cover -v ./e2etest
env:
AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY)
AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME)
AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS)
AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS)
CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY)
CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256)
displayName: 'E2E Test MacOs'
condition: eq(variables.type, 'mac-os')
- job: Test_On_Ubuntu
variables:
isMutexSet: 'false'
# allow maximum build time, in case we have build congestion
timeoutInMinutes: 360
pool:
vmImage: 'ubuntu-20.04'
steps:
- task: UsePythonVersion@0
name: 'Set_up_Python'
inputs:
versionSpec: '3.7'
- task: GoTool@0
name: 'Set_up_Golang'
inputs:
version: '1.16'
- task: DownloadSecureFile@1
name: ciGCSServiceAccountKey
displayName: 'Download GCS Service Account Key'
inputs:
secureFile: 'ci-gcs-dev.json'
- script: |
pip install azure-storage-blob==12.0.0b3
# the recent release 1.0.0b4 has a breaking change
pip install azure-core==1.0.0b3
# acquire the mutex before running live tests to avoid conflicts
python ./tool_distributed_mutex.py lock "$(MUTEX_URL)"
# set the variable to indicate that the mutex was actually acquired
echo '##vso[task.setvariable variable=isMutexSet]true'
name: 'Acquire_the_distributed_mutex'
- script: |
# run unit test and build executable
# the set -e line is needed so that the unit tests failure would cause the job to fail properly
# "-check.v" (must be after package list) outputs timings
set -e
go test -timeout 45m -race -short -cover ./cmd ./common ./common/parallel ./ste ./azbfs ./sddl "-check.v"
GOARCH=amd64 GOOS=linux go build -o azcopy_linux_amd64
name: 'Run_unit_tests'
env:
ACCOUNT_NAME: $(ACCOUNT_NAME)
ACCOUNT_KEY: $(ACCOUNT_KEY)
AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY)
AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME)
AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath)
GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT)
- script: |
go build -o test-validator ./testSuite/
mkdir test-temp
export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64
export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator
export TEST_DIRECTORY_PATH=$(pwd)/test-temp
# install the CLFSLoad extension
pip3 install clfsload
keyctl session test python ./testSuite/scripts/run.py
name: 'Run_smoke_tests'
env:
ACCOUNT_NAME: $(ACCOUNT_NAME)
ACCOUNT_KEY: $(ACCOUNT_KEY)
AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT)
GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath)
ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID)
AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET)
CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL)
CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL)
CONTAINER_SAS_URL: $(CONTAINER_SAS_URL)
FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL)
FILESYSTEM_URL: $(FILESYSTEM_URL)
OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT)
OAUTH_TENANT_ID: $(OAUTH_TENANT_ID)
PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL)
S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL)
S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL)
S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL)
S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL)
S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL)
SHARE_SAS_URL: $(SHARE_SAS_URL)
- script: |
pip install azure-storage-blob==12.0.0b3
# the recent release 1.0.0b4 has a breaking change
pip install azure-core==1.0.0b3
python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)"
name: 'Release_the_distributed_mutex'
# this runs even if the job was canceled (only if the mutex was acquired by this job)
condition: and(always(), eq(variables['isMutexSet'], 'true'))