benchmark #54
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2021 Agnostiq Inc. | |
| # | |
| # This file is part of Covalent. | |
| # | |
| # Licensed under the Apache License 2.0 (the "License"). A copy of the | |
| # License may be obtained with this software package or at | |
| # | |
| # https://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Use of this file is prohibited except in compliance with the License. | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| name: benchmark | |
| on: | |
| schedule: | |
| - cron: "0 4 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| commit_sha: | |
| description: "Commit SHA used for testing" | |
| type: string | |
| required: true | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| benchmarks: | |
| runs-on: ubuntu-latest | |
| env: | |
| COVALENT_SERVER_IFACE_ANY: true | |
| steps: | |
| - name: Check out head | |
| if: github.event_name == 'schedule' | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Check out SHA | |
| if: github.event_name == 'workflow_dispatch' | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| fetch-depth: 0 | |
| ref: ${{ inputs.commit_sha }} | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.8" | |
| - name: Install Covalent | |
| run: python setup.py sdist && pip install dist/*.tar.gz | |
| - name: Install test requirements | |
| run: pip install --no-cache-dir -r ./tests/requirements.txt | |
| - name: Start Covalent dispatcher server | |
| run: covalent start -d --ignore-migrations | |
| - name: Run stress benchmarks | |
| run: python -m pytest -vv tests/stress_tests/benchmarks | |
| - name: Run Locust load tests | |
| run: > | |
| python -m locust | |
| --user $(python -c 'import os; print(os.cpu_count())') | |
| --spawn-rate 3 | |
| --run-time 10m | |
| --csv=loadtest_results | |
| --headless | |
| -f ./load_tests/locustfiles | |
| working-directory: ./tests | |
| - name: Create tests result archive | |
| run: | | |
| tar -zcvf $GITHUB_SHA.tar.gz metrics*.log ./tests/loadtest_results*.csv | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| role-to-assume: ${{ secrets.S3_UPLOAD_ROLE }} | |
| aws-region: us-east-1 | |
| - name: Upload benchmark results to S3 | |
| run: aws s3 cp $GITHUB_SHA.tar.gz s3://${{ secrets.BENCHMARK_BUCKET }}/$GITHUB_SHA.tar.gz |