forked from rucio/rucio
-
Notifications
You must be signed in to change notification settings - Fork 0
186 lines (174 loc) · 8.63 KB
/
integration_tests.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
name: End to End integration tests
on:
- pull_request
- push
jobs:
setup:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Update pip
run: python3 -m pip install -U pip setuptools
- name: Install python requirements for matrix_parser.py
run: python3 -m pip install -U PyYAML
- name: Identify branch
id: branch
run: python3 -c 'from os import environ as env; print("::set-output name=branch::" + (env.get("GITHUB_BASE_REF", None) if env.get("GITHUB_BASE_REF", None) else env.get("GITHUB_REF", "master")))'
- name: Identify Matrix
id: matrix
run: echo "::set-output name=matrix::$(./tools/test/matrix_parser.py < ./etc/docker/test/matrix_integration_tests.yml)"
outputs:
branch: ${{ steps.branch.outputs.branch }}
matrix: ${{ steps.matrix.outputs.matrix }}
integration-tests:
needs: setup
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
cfg: ${{ fromJson(needs.setup.outputs.matrix) }}
os:
- ubuntu-latest
- macos-latest
steps:
- name: Checkout rucio containers repository
uses: actions/checkout@v2
with:
repository: rucio/containers
fetch-depth: 0
- uses: actions/checkout@v2
name: Checkout rucio source
with:
path: dev/rucio
fetch-depth: 0
- name: Select tag for rucio containers
shell: bash
run: |
# Change to cloned rucio/rucio repo
cd $GITHUB_WORKSPACE/dev/rucio
# Get current branch and corresponding latest tag in rucio/rucio repo
BRANCH=$(git rev-parse --abbrev-ref HEAD)
OWNER="${{ github.repository_owner }}"
if [ $OWNER != 'rucio' ]; then
echo "The workflow is running in user ${OWNER}'s fork. Fetching branches and tags from rucio/rucio instead."
git remote add rucio https://github.com/rucio/rucio
git fetch rucio --tags -f
fi
echo "On branch ${BRANCH}"
if [ $BRANCH == 'master' ]; then
GIT_REF="master"
else
GIT_REF=$(git describe --tags --abbrev=0)
IFS=. read major minor micro build <<<"${GIT_REF}"
RELEASE_FAMILY=$major.$minor
LATEST_RELEASE_IN_RELEASE_FAMILY=$(git for-each-ref --format '%(refname)' refs/tags/$RELEASE_FAMILY.* | sort -k 1.11V | tail -1 | awk -F'/' '{print $3}')
LATEST_RUCIO_RELEASE_FAMILY=$(git for-each-ref --format '%(refname)' refs/tags | sort -k 1.11V | tail -1 | awk -F'/' '{print $3}' | awk -F'.' '{print $1 "." $2}')
echo "Release line for ${BRANCH} is ${RELEASE_FAMILY}"
echo "The latest release line for rucio is ${LATEST_RUCIO_RELEASE_FAMILY}"
echo "The latest release in ${RELEASE_FAMILY} is ${LATEST_RELEASE_IN_RELEASE_FAMILY}"
if [ $LATEST_RUCIO_RELEASE_FAMILY = $RELEASE_FAMILY ]; then
GIT_REF='master' # always use containers/master when working on latest rucio/rucio release line
else
GIT_REF=$LATEST_RELEASE_IN_RELEASE_FAMILY # for non-master release line, use the latest rucio/containers tag for the given release family
fi
fi
cd $GITHUB_WORKSPACE
# Check if rucio/containers has a corresponding $GIT_REF
if [ $(git tag -l "$GIT_REF") ]; then
git checkout tags/$GIT_REF
else
echo "Tag $GIT_REF not found in rucio/containers. Integration test containers will be built against the master branch instead."
git checkout master
fi
- name: Setup docker (missing on MacOS)
if: runner.os == 'macos'
run: |
brew install docker docker-compose
colima start
- name: Use rucio/containers Dockerfile for integration tests
shell: bash
run: |
OUTPUT_FILE=$(mktemp)
sed 's;RUN git clone .*;COPY ./rucio /tmp/rucio;' $GITHUB_WORKSPACE/dev/alma9.Dockerfile > "$OUTPUT_FILE"
mv "$OUTPUT_FILE" $GITHUB_WORKSPACE/dev/alma9.Dockerfile
- name: Build rucio-dev images
id: images
shell: bash
run: |
docker login https://ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
docker-compose -f $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml --profile storage --profile externalmetadata pull
i=0; until [ "$i" -ge 3 ]; do
IMAGES=$(echo '${{ toJson(matrix.cfg) }}' | $GITHUB_WORKSPACE/dev/rucio/tools/test/build_images.py --output list \
--cache-repo ghcr.io/${{ github.repository }} --branch "${{ needs.setup.outputs.branch }}" \
$GITHUB_WORKSPACE/dev || echo "")
if [[ -n $IMAGES ]]; then break;
else
i=$((i+1)); sleep 5;
echo "::warning::Building images failed, retrying…"
fi
done
docker logout https://ghcr.io
if [[ -z "$IMAGES" ]]; then echo "::error::Building images failed ultimately"; exit 1; fi
echo "::set-output name=images::$IMAGES"
- name: Prepare Docker Compose
shell: bash
run: |
docker image ls
sed -i 's;image: docker.io/rucio/rucio-dev.*;image: ${{ fromJSON(steps.images.outputs.images)[0] }};' \
$GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml
- name: Start containers
run: docker-compose -f $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml --profile storage --profile externalmetadata up -d
- name: Initialize tests
shell: bash
run: |
docker exec -t dev_rucio_1 cp etc/rse-accounts.cfg.template etc/rse-accounts.cfg
docker exec -t dev_rucio_1 tools/run_tests.sh -ir
- name: File Upload/Download Test
run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_rucio_server.py
- name: UploadClient Test
run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_upload.py
- name: File Upload/Download Test using 'impl' parameter
run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_impl_upload_download.py
- name: Test gfal2 implementation on xrootd protocol
run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_rse_protocol_gfal2_impl.py
- name: Test Protocol XrootD
run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_rse_protocol_xrootd.py
- name: Test Protocol SSH (scp)
run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_rse_protocol_ssh.py
- name: Test Protocol Rsync
run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_rse_protocol_rsync.py
- name: Test Protocol Rclone
run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_rse_protocol_rclone.py
- name: Test Conveyor
run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_conveyor.py
- name: Execute transfer and export FTS transfer details
id: tpc
shell: bash
run: |
docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short --export-artifacts-from="test_tpc" tests/test_tpc.py
FTS_LOG_FILE=$(docker exec -t dev_rucio_1 cat /tmp/test_tpc.artifact)
echo "::set-output name=fts3log::$FTS_LOG_FILE"
- name: Verify TPC transfers from FTS logs
shell: bash
run: |
FTS_LOG_FILE=${{ steps.tpc.outputs.fts3log }}
if docker exec -t dev_fts_1 /bin/bash -c "grep -Fq \"3rd pull\" $FTS_LOG_FILE"
then
echo "TPC 3rd party pull verified"
else
echo "TPC failed"
exit 1
fi
- name: Print server error logs
if: ${{ success() || failure() }}
shell: bash
run: |
docker exec -t dev_rucio_1 cat /var/log/rucio/httpd_error_log
- name: Archive Upload/Download Test
run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_download.py::test_download_from_archive_on_xrd
- name: Test external metadata plugin (mongo)
run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_did_meta_plugins.py::TestDidMetaMongo
- name: Test external metadata plugin (postgres)
run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_did_meta_plugins.py::TestDidMetaExternalPostgresJSON
- name: Stop containers
run: docker-compose -f $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml --profile storage --profile externalmetadata down