diff --git a/.github/workflows/evalml_main_performance_tests.yaml b/.github/workflows/evalml_main_performance_tests.yaml deleted file mode 100644 index e5c7e818b1..0000000000 --- a/.github/workflows/evalml_main_performance_tests.yaml +++ /dev/null @@ -1,175 +0,0 @@ -name: Run looking glass performance tests - -on: - push: - branches: - - main - -jobs: - performance_tests: - name: Run looking glass performance tests - runs-on: ubuntu-latest - strategy: - fail-fast: true - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_SECRET_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: 'us-east-1' - AWS_SESSION_TOKEN: "" - steps: - - name: Checkout evalml - uses: actions/checkout@v3 - with: - fetch-depth: 2 - - name: Get commit hashes - id: get_hashes - run: | - current_hash=$(git rev-parse --short HEAD) - echo "Latest commit hash: $current_hash" - echo "::set-output name=current_hash::$current_hash" - previous_hash=$(git rev-parse --short HEAD~1) - echo "Previous commit hash: $previous_hash" - echo "::set-output name=previous_hash::$previous_hash" - - name: Set up Python 3.8 - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - name: Checkout looking glass - uses: actions/checkout@v3 - with: - repository: alteryx/looking-glass - ref: master - token: ${{ secrets.LOOKING_GLASS_TOKEN }} - - name: Installing Dependencies - run: | - make installdeps - make installdeps-test - - name: Run default algorithm performance test (current) - id: current_default - run: | - response=$( looking-glass run-evalml-automl --username machineFL --automl-algo default --scenarios-yaml release.yaml --evalml-branch ${{ steps.get_hashes.outputs.current_hash }} --job-desc ${{ steps.get_hashes.outputs.current_hash }}_default ) - echo "::set-output name=job_id::$(echo $response | sed 's/.*Job ID: \([^ ]*\).*/\1/')" - - name: Run iterative algorithm performance test (current) - id: current_iterative - run: | - response=$( looking-glass run-evalml-automl --username machineFL --automl-algo iterative --scenarios-yaml release.yaml --evalml-branch ${{ steps.get_hashes.outputs.current_hash }} --job-desc ${{ steps.get_hashes.outputs.current_hash }}_iterative ) - echo "::set-output name=job_id::$(echo $response | sed 's/.*Job ID: \([^ ]*\).*/\1/')" - - name: Run default algorithm performance test (previous) - id: previous_default - run: | - response=$( looking-glass run-evalml-automl --username machineFL --automl-algo default --scenarios-yaml release.yaml --evalml-branch ${{ steps.get_hashes.outputs.previous_hash }} --job-desc ${{ steps.get_hashes.outputs.previous_hash }}_default ) - echo "::set-output name=job_id::$(echo $response | sed 's/.*Job ID: \([^ ]*\).*/\1/')" - - name: Run iterative algorithm performance test (previous) - id: previous_iterative - run: | - response=$( looking-glass run-evalml-automl --username machineFL --automl-algo iterative --scenarios-yaml release.yaml --evalml-branch ${{ steps.get_hashes.outputs.previous_hash }} --job-desc ${{ steps.get_hashes.outputs.previous_hash }}_iterative ) - echo "::set-output name=job_id::$(echo $response | sed 's/.*Job ID: \([^ ]*\).*/\1/')" - - name: Await results - id: results - run: | - for id in ${{ steps.current_default.outputs.job_id }} ${{ steps.current_iterative.outputs.job_id }} ${{ steps.previous_default.outputs.job_id }} ${{ steps.previous_iterative.outputs.job_id }}; do - echo "Waiting for job id: $id" - result='' - sleep_time=0 - while [ -z "$result" ] - do - sleep $sleep_time - result=$(looking-glass get-job --job-id $id | grep -n 'COMPLETED' || :;) - sleep_time=60 - done - looking-glass get-results --job-id $id --file-path $id - done - shell: bash - - name: Generate reports - run: | - filename_base=${{ steps.get_hashes.outputs.current_hash }}_v_${{ steps.get_hashes.outputs.previous_hash }} - looking-glass run-local-report --previous-path ${{ steps.previous_default.outputs.job_id }} -n ${{ steps.current_default.outputs.job_id }} --output-name ${filename_base}_default --output-type html - looking-glass run-local-report --previous-path ${{ steps.previous_iterative.outputs.job_id }} -n ${{ steps.current_iterative.outputs.job_id }} --output-name ${filename_base}_iterative --output-type html - - name: Upload reports to S3 - id: s3_upload - run: | - filename_base=${{ steps.get_hashes.outputs.current_hash }}_v_${{ steps.get_hashes.outputs.previous_hash }} - default_url=s3://evalml-main-looking-glass-reports/${filename_base}_default.html - aws s3 cp ./${filename_base}_default.html $default_url - echo "::set-output name=default_report_url::$default_url" - echo "Default Algorith Report: $default_url" - iterative_url=s3://evalml-main-looking-glass-reports/${filename_base}_iterative.html - aws s3 cp ./${filename_base}_iterative.html $iterative_url - echo "::set-output name=iterative_report_url::$iterative_url" - echo "Iterative Algorith Report: $iterative_url" - - name: Notify on Slack - run: | - filename_base=${{ steps.get_hashes.outputs.current_hash }}_v_${{ steps.get_hashes.outputs.previous_hash }} - presigned_default_url=$( aws s3 presign ${{ steps.s3_upload.outputs.default_report_url }} --expires-in 604800 ) - presigned_iterative_url=$( aws s3 presign ${{ steps.s3_upload.outputs.iterative_report_url }} --expires-in 604800 ) - response=$( - curl -X POST https://slack.com/api/chat.postMessage -H 'Content-type: application/json;charset=UTF-8' -H 'Authorization: Bearer ${{ secrets.LG_SLACK_TOKEN }}' \ - --data-binary @- << EOF - { - "channel": "C042B5JBBPF", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": ", performance tests for latest commit on evalml are complete" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "*Author*: ${{ github.event.head_commit.author.name }}" - } - }, - { - "type": "section", - "fields": [ - { - "type": "mrkdwn", - "text": "*Current Commit*: <${{ github.event.head_commit.url }}|${{ steps.get_hashes.outputs.current_hash }}>" - }, - { - "type": "mrkdwn", - "text": "*Previous Commit*: " - }, - { - "type": "mrkdwn", - "text": "*Current Default Algo Job*: ${{ steps.current_default.outputs.job_id }}" - }, - { - "type": "mrkdwn", - "text": "*Previous Default Algo Job*: ${{ steps.previous_default.outputs.job_id }}" - }, - { - "type": "mrkdwn", - "text": "*Current Iterative Algo Job*: ${{ steps.current_iterative.outputs.job_id }}" - }, - { - "type": "mrkdwn", - "text": "*Previous Iterative Algo Job*: ${{ steps.previous_iterative.outputs.job_id }}" - }, - ] - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "*Default Algorithm Report*: <$presigned_default_url|${{ steps.s3_upload.outputs.default_report_url }}>" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "*Iterative Algorithm Report*: <$presigned_iterative_url|${{ steps.s3_upload.outputs.iterative_report_url }}>" - } - } - ] - } - EOF - ) - echo $response - ts=$(echo $response | jq .ts) - curl -F "file=@${filename_base}_default.html" -F channels=C042B5JBBPF -F 'thread_ts=$ts' -H 'Authorization: Bearer ${{ secrets.LG_SLACK_TOKEN }}' https://slack.com/api/files.upload - curl -F "file=@${filename_base}_iterative.html" -F channels=C042B5JBBPF -F 'thread_ts=$ts' -H 'Authorization: Bearer ${{ secrets.LG_SLACK_TOKEN }}' https://slack.com/api/files.upload diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst index 81a1ca32dc..2a6e75f671 100644 --- a/docs/source/release_notes.rst +++ b/docs/source/release_notes.rst @@ -15,6 +15,7 @@ Release Notes * Documentation Changes * Removed LightGBM's excessive amount of warnings :pr:`4308` * Testing Changes + * Removed old performance testing workflow :pr:`4318` .. warning::