Transfers: implement preliminary support for packet marking. #5856 #2443
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: End to End integration tests | |
| on: | |
| - pull_request | |
| - push | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Update pip | |
| run: python3 -m pip install -U pip setuptools | |
| - name: Install python requirements for matrix_parser.py | |
| run: python3 -m pip install -U PyYAML | |
| - name: Identify branch | |
| id: branch | |
| run: python3 -c 'from os import environ as env; print("::set-output name=branch::" + (env.get("GITHUB_BASE_REF", None) if env.get("GITHUB_BASE_REF", None) else env.get("GITHUB_REF", "master")))' | |
| - name: Identify Matrix | |
| id: matrix | |
| run: echo "::set-output name=matrix::$(./tools/test/matrix_parser.py < ./etc/docker/test/matrix_integration_tests.yml)" | |
| outputs: | |
| branch: ${{ steps.branch.outputs.branch }} | |
| matrix: ${{ steps.matrix.outputs.matrix }} | |
| integration-tests: | |
| needs: setup | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| cfg: ${{ fromJson(needs.setup.outputs.matrix) }} | |
| steps: | |
| - name: Checkout rucio containers repository | |
| uses: actions/checkout@v2 | |
| with: | |
| repository: rucio/containers | |
| fetch-depth: 0 | |
| - uses: actions/checkout@v2 | |
| name: Checkout rucio source | |
| with: | |
| path: dev/rucio | |
| fetch-depth: 0 | |
| - name: Select tag for rucio containers | |
| shell: bash | |
| run: | | |
| # Change to cloned rucio/rucio repo | |
| cd $GITHUB_WORKSPACE/dev/rucio | |
| # Get current branch and corresponding latest tag in rucio/rucio repo | |
| BRANCH=$(git rev-parse --abbrev-ref HEAD) | |
| OWNER="${{ github.repository_owner }}" | |
| if [ $OWNER != 'rucio' ]; then | |
| echo "The workflow is running in user ${OWNER}'s fork. Fetching branches and tags from rucio/rucio instead." | |
| git remote add rucio https://github.com/rucio/rucio | |
| git fetch rucio --tags -f | |
| fi | |
| echo "On branch ${BRANCH}" | |
| if [ $BRANCH == 'master' ]; then | |
| GIT_REF="master" | |
| else | |
| GIT_REF=$(git describe --tags --abbrev=0) | |
| IFS=. read major minor micro build <<<"${GIT_REF}" | |
| RELEASE_FAMILY=$major.$minor | |
| LATEST_RELEASE_IN_RELEASE_FAMILY=$(git for-each-ref --format '%(refname)' refs/tags/$RELEASE_FAMILY.* | sort -k 1.11V | tail -1 | awk -F'/' '{print $3}') | |
| LATEST_RUCIO_RELEASE_FAMILY=$(git for-each-ref --format '%(refname)' refs/tags | sort -k 1.11V | tail -1 | awk -F'/' '{print $3}' | awk -F'.' '{print $1 "." $2}') | |
| echo "Release line for ${BRANCH} is ${RELEASE_FAMILY}" | |
| echo "The latest release line for rucio is ${LATEST_RUCIO_RELEASE_FAMILY}" | |
| echo "The latest release in ${RELEASE_FAMILY} is ${LATEST_RELEASE_IN_RELEASE_FAMILY}" | |
| if [ $LATEST_RUCIO_RELEASE_FAMILY = $RELEASE_FAMILY ]; then | |
| GIT_REF='master' # always use containers/master when working on latest rucio/rucio release line | |
| else | |
| GIT_REF=$LATEST_RELEASE_IN_RELEASE_FAMILY # for non-master release line, use the latest rucio/containers tag for the given release family | |
| fi | |
| fi | |
| cd $GITHUB_WORKSPACE | |
| # Check if rucio/containers has a corresponding $GIT_REF | |
| if [ $(git tag -l "$GIT_REF") ]; then | |
| git checkout tags/$GIT_REF | |
| else | |
| echo "Tag $GIT_REF not found in rucio/containers. Integration test containers will be built against the master branch instead." | |
| git checkout master | |
| fi | |
| - name: Use rucio/containers Dockerfile for integration tests | |
| shell: bash | |
| run: | | |
| sed -i 's;RUN git clone .*;COPY ./rucio /tmp/rucio;' $GITHUB_WORKSPACE/dev/alma9.Dockerfile | |
| - name: Build rucio-dev images | |
| id: images | |
| shell: bash | |
| run: | | |
| docker login https://ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} | |
| docker-compose -f $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose-storage-externalmetadata.yml pull | |
| i=0; until [ "$i" -ge 3 ]; do | |
| IMAGES=$(echo '${{ toJson(matrix.cfg) }}' | $GITHUB_WORKSPACE/dev/rucio/tools/test/build_images.py --output list \ | |
| --cache-repo ghcr.io/${{ github.repository }} --branch "${{ needs.setup.outputs.branch }}" \ | |
| $GITHUB_WORKSPACE/dev || echo "") | |
| if [[ -n $IMAGES ]]; then break; | |
| else | |
| i=$((i+1)); sleep 5; | |
| echo "::warning::Building images failed, retrying…" | |
| fi | |
| done | |
| docker logout https://ghcr.io | |
| if [[ -z "$IMAGES" ]]; then echo "::error::Building images failed ultimately"; exit 1; fi | |
| echo "::set-output name=images::$IMAGES" | |
| - name: Prepare Docker Compose | |
| shell: bash | |
| run: | | |
| docker image ls | |
| sed -i 's;image: docker.io/rucio/rucio-dev.*;image: ${{ fromJSON(steps.images.outputs.images)[0] }};' \ | |
| $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose-storage-externalmetadata.yml | |
| - name: Start containers | |
| run: docker-compose -f $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose-storage-externalmetadata.yml up -d | |
| - name: Initialize tests | |
| shell: bash | |
| run: | | |
| docker exec -t dev_rucio_1 cp etc/rse-accounts.cfg.template etc/rse-accounts.cfg | |
| docker exec -t dev_rucio_1 tools/run_tests_docker.sh -ir | |
| - name: File Upload/Download Test | |
| run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_rucio_server.py | |
| - name: UploadClient Test | |
| run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_upload.py | |
| - name: File Upload/Download Test using 'impl' parameter | |
| run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_impl_upload_download.py | |
| - name: Test gfal2 implementation on xrootd protocol | |
| run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_rse_protocol_gfal2_impl.py | |
| - name: Test Protocol XrootD | |
| run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_rse_protocol_xrootd.py | |
| - name: Test Protocol SSH (scp) | |
| run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_rse_protocol_ssh.py | |
| - name: Test Protocol Rsync | |
| run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_rse_protocol_rsync.py | |
| - name: Test Protocol Rclone | |
| run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_rse_protocol_rclone.py | |
| - name: Test Conveyor | |
| run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_conveyor.py | |
| - name: Execute transfer and export FTS transfer details | |
| id: tpc | |
| shell: bash | |
| run: | | |
| docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short --export-artifacts-from="test_tpc" tests/test_tpc.py | |
| FTS_LOG_FILE=$(docker exec -t dev_rucio_1 cat /tmp/test_tpc.artifact) | |
| echo "::set-output name=fts3log::$FTS_LOG_FILE" | |
| - name: Verify TPC transfers from FTS logs | |
| shell: bash | |
| run: | | |
| FTS_LOG_FILE=${{ steps.tpc.outputs.fts3log }} | |
| if docker exec -t dev_fts_1 /bin/bash -c "grep -Fq \"3rd pull\" $FTS_LOG_FILE" | |
| then | |
| echo "TPC 3rd party pull verified" | |
| else | |
| echo "TPC failed" | |
| exit 1 | |
| fi | |
| - name: Print server error logs | |
| if: ${{ success() || failure() }} | |
| shell: bash | |
| run: | | |
| docker exec -t dev_rucio_1 cat /var/log/rucio/httpd_error_log | |
| - name: Archive Upload/Download Test | |
| run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_download.py::test_download_from_archive_on_xrd | |
| - name: Test external metadata plugin (mongo) | |
| run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_did_meta_plugins.py::TestDidMetaMongo | |
| - name: Test external metadata plugin (postgres) | |
| run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short tests/test_did_meta_plugins.py::TestDidMetaExternalPostgresJSON | |
| - name: Stop containers | |
| run: docker-compose -f $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose-storage-externalmetadata.yml down |