-
Notifications
You must be signed in to change notification settings - Fork 67
290 lines (270 loc) · 9.6 KB
/
test.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
# We use self-hosted runners for performance: we can rely on docker caching
# on the runner machines for 0s startup times for our tests.
# There is an issue with self-hosted runners where if you use a mix of
# host and container jobs, the host jobs will fail to start
# (https://github.com/actions/checkout/issues/273).
# So we create a separate "builder" tagged runner to do the build job on
# the host, and then all the tests run on "runner" tagged runners.
name: test
on:
push:
jobs:
check-which-tests-to-run:
uses: ./.github/workflows/check-which-tests-to-run.yaml
# ==== Query Service Jobs ====
build-container-query-service:
name: Build Legacy (Query Service) test container
timeout-minutes: 30
runs-on: [self-hosted, builder]
outputs:
build_needed: ${{ steps.build_check.outputs.build_needed }}
env:
REGISTRY: us-east4-docker.pkg.dev/weave-support-367421/weave-images
needs: check-which-tests-to-run
# if: github.ref == 'refs/heads/master' || needs.check-which-tests-to-run.outputs.weave_query_tests
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 2
- name: Check if build is needed
id: build_check
run: |
if [[ "${{ github.ref }}" == "refs/heads/master" || "${{ needs.check-which-tests-to-run.outputs.weave_query_tests }}" == "true" ]]; then
echo "Build is needed"
echo "build_needed=true" >> $GITHUB_OUTPUT
else
echo "Build is not needed"
echo "build_needed=false" >> $GITHUB_OUTPUT
fi
- name: Login to Docker Hub
if: steps.build_check.outputs.build_needed == 'true'
uses: docker/login-action@v2
with:
registry: us-east4-docker.pkg.dev
username: _json_key
password: ${{ secrets.gcp_sa_key }}
- name: Prune docker cache
if: steps.build_check.outputs.build_needed == 'true'
run: docker system prune -f
- name: Build legacy (query service) unit test image
if: steps.build_check.outputs.build_needed == 'true'
run: python3 weave_query/docker/docker_build.py build_deps weave-test-python-query-service builder . weave_query/Dockerfile.ci.test
test-query-service:
name: Legacy (Query Service) Python unit tests
timeout-minutes: 15 # do not raise! running longer than this indicates an issue with the tests. fix there.
needs:
- check-which-tests-to-run
- build-container-query-service
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
job_num: [0, 1]
# runs-on: ubuntu-latest
container: ${{ needs.build-container-query-service.outputs.build_needed == 'true' && format('us-east4-docker.pkg.dev/weave-support-367421/weave-images/weave-test-python-query-service:{0}', github.sha) || null }}
services:
wandbservice:
image: us-central1-docker.pkg.dev/wandb-production/images/local-testcontainer:master
credentials:
username: _json_key
password: ${{ secrets.gcp_wb_sa_key }}
env:
CI: 1
WANDB_ENABLE_TEST_CONTAINER: true
ports:
- '8080:8080'
- '8083:8083'
- '9015:9015'
options: --health-cmd "curl --fail http://localhost:8080/healthz || exit 1" --health-interval=5s --health-timeout=3s
outputs:
tests_should_run: ${{ steps.test_check.outputs.tests_should_run }}
steps:
- uses: actions/checkout@v2
- name: Check if tests should run
id: test_check
run: |
if [[ "${{ github.ref }}" == "refs/heads/master" || "${{ needs.check-which-tests-to-run.outputs.weave_query_tests }}" == "true" ]]; then
echo "Tests should run"
echo "tests_should_run=true" >> $GITHUB_OUTPUT
else
echo "Tests should not run"
echo "tests_should_run=false" >> $GITHUB_OUTPUT
fi
- name: Verify wandb server is running
if: steps.test_check.outputs.tests_should_run == 'true'
run: curl -s http://wandbservice:8080/healthz
- name: Run Legacy (Query Service) Python Unit Tests
if: steps.test_check.outputs.tests_should_run == 'true'
env:
DD_SERVICE: weave-python
DD_ENV: ci
WEAVE_SENTRY_ENV: ci
CI: 1
WB_SERVER_HOST: http://wandbservice
WEAVE_SERVER_DISABLE_ECOSYSTEM: 1
run: |
source /root/venv/bin/activate && \
cd weave_query && \
pytest \
--job-num=${{ matrix.job_num }} \
--timeout=90 \
--ddtrace \
--durations=5 \
.
test-query-service-matrix-check: # This job does nothing and is only used for the branch protection
if: always()
needs:
- test-query-service
runs-on: ubuntu-latest
steps:
- name: Passes if all test-query-service jobs succeeded
uses: re-actors/alls-green@release/v1
with:
jobs: ${{ toJSON(needs) }}
# ==== Weave UI Jobs ====
weavejs-lint-compile:
name: WeaveJS Lint and Compile
runs-on: ubuntu-latest
needs:
- check-which-tests-to-run
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
token: ${{ secrets.GITHUB_TOKEN }}
- name: Check if lint and compile should run
id: check_run
run: |
if [[ "${{ github.ref }}" == "refs/heads/master" || "${{ needs.check-which-tests-to-run.outputs.weave_js_tests }}" == "true" ]]; then
echo "Lint and compile should run"
echo "should_lint_and_compile=true" >> $GITHUB_OUTPUT
else
echo "Lint and compile should not run"
echo "should_lint_and_compile=false" >> $GITHUB_OUTPUT
fi
- uses: actions/setup-node@v1
if: steps.check_run.outputs.should_lint_and_compile == 'true'
with:
node-version: '18.x'
- name: Run WeaveJS Lint and Compile
if: steps.check_run.outputs.should_lint_and_compile == 'true'
run: |
set -e
cd weave-js
yarn install --frozen-lockfile
yarn generate
yarn eslint --max-warnings=0
yarn tslint
yarn prettier
yarn run tsc
# ==== Trace Jobs ====
lint:
name: Python lint
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: 3.9
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install nox uv
- name: Run nox
run: nox -e lint
trace-tests:
name: Trace nox tests
timeout-minutes: 10
runs-on: ubuntu-latest
strategy:
matrix:
python-version-major: ['3']
python-version-minor: [
'9',
'10',
'11',
'12',
'13',
#
]
nox-shard:
[
'trace',
'trace_server',
'anthropic',
'cerebras',
'cohere',
'dspy',
'groq',
'instructor',
'langchain',
'litellm',
'llamaindex',
'mistral0',
'mistral1',
'notdiamond',
'openai',
]
fail-fast: false
services:
wandbservice:
image: us-central1-docker.pkg.dev/wandb-production/images/local-testcontainer:master
credentials:
username: _json_key
password: ${{ secrets.gcp_wb_sa_key }}
env:
CI: 1
WANDB_ENABLE_TEST_CONTAINER: true
ports:
- '8080:8080'
- '8083:8083'
- '9015:9015'
options: --health-cmd "curl --fail http://localhost:8080/healthz || exit 1" --health-interval=5s --health-timeout=3s
weave_clickhouse:
image: clickhouse/clickhouse-server
ports:
- '8123:8123'
options: --health-cmd "wget -nv -O- 'http://localhost:8123/ping' || exit 1" --health-interval=5s --health-timeout=3s
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install nox uv
- name: Run nox (Clickhouse Only)
env:
WEAVE_SENTRY_ENV: ci
CI: 1
WB_SERVER_HOST: http://wandbservice
WF_CLICKHOUSE_HOST: localhost
WEAVE_SERVER_DISABLE_ECOSYSTEM: 1
run: |
nox -e "tests-${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}(shard='${{ matrix.nox-shard }}')" -- \
-m "weave_client and not skip_clickhouse_client" \
--weave-server=clickhouse
- name: Run nox
env:
WEAVE_SENTRY_ENV: ci
CI: 1
WB_SERVER_HOST: http://wandbservice
WF_CLICKHOUSE_HOST: weave_clickhouse
WEAVE_SERVER_DISABLE_ECOSYSTEM: 1
run: |
nox -e "tests-${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}(shard='${{ matrix.nox-shard }}')"
trace-tests-matrix-check: # This job does nothing and is only used for the branch protection
if: always()
needs:
- trace-tests
runs-on: ubuntu-latest
steps:
- name: Passes if all trace-tests jobs succeeded
uses: re-actors/alls-green@release/v1
with:
jobs: ${{ toJSON(needs) }}