From d343bfb28f1ad7d0425328652650c055ef3b6460 Mon Sep 17 00:00:00 2001 From: Vladimir Milosevic Date: Mon, 4 Nov 2024 11:41:05 +0000 Subject: [PATCH] Refactor pipeline into action --- .github/actions/produce-cicd-data/action.yml | 99 ++++++++++++++++++++ .github/workflows/build-and-test.yml | 1 + .github/workflows/produce_data.yml | 87 +---------------- 3 files changed, 105 insertions(+), 82 deletions(-) create mode 100644 .github/actions/produce-cicd-data/action.yml diff --git a/.github/actions/produce-cicd-data/action.yml b/.github/actions/produce-cicd-data/action.yml new file mode 100644 index 000000000..e6a033cd9 --- /dev/null +++ b/.github/actions/produce-cicd-data/action.yml @@ -0,0 +1,99 @@ +name: "Produce CICD Data" +description: "Composite action to produce CICD data" +inputs: + test_workflow_run_id: + description: "Unique GitHub workflow run ID to use for data" + required: true + test_workflow_run_attempt: + description: "Run attempt of the workflow run" + required: true + +runs: + using: "composite" + steps: + + - uses: actions/checkout@v4 + with: + repository: tenstorrent/tt-forge-fe + path: infra + + - name: Output (safe) pipeline values + shell: bash + run: | + echo "pipeline_id (id / run #): ${{ github.run_id }}/${{ github.run_attempt }}" + echo "submissions_ts: " + echo "start_ts: " + echo "end_ts: " + echo "name: ${{ github.workflow }}, but rk recommended name w/out @: ${{ github.workflow_ref }}" + echo "trigger: ${{ github.event_name }}" + echo "sha: ${{ github.sha }}" + echo "(triggering) author/actor: ${{ github.actor }}" + echo "author/actor: ${{ github.triggering_actor }}" + echo "orchestrator: github (Static)" + echo "docker_image: ${{ job.container.image }}" + echo "build duration is post-process" + + - name: Get workflow run_id attempt number to analyze + id: get-run-id-and-attempt + shell: bash + run: | + event_name="${{ github.event_name }}" + if [[ "$event_name" == "workflow_dispatch" ]]; then + run_id="${{ inputs.test_workflow_run_id }}" + attempt_number="${{ inputs.test_workflow_run_attempt }}" + elif [[ "$event_name" == "workflow_run" ]]; then + run_id="${{ github.event.workflow_run.id }}" + attempt_number="${{ github.event.workflow_run.run_attempt }}" + [[ -z "$run_id" ]] && { echo "run_id is empty" ; exit 1; } + [[ -z "$attempt_number" ]] && { echo "attempt_number is empty" ; exit 1; } + else + echo "Unknown event name" && exit 1 + fi + + echo $run_id + echo $attempt_number + echo "run-id=$run_id" >> "$GITHUB_OUTPUT" + echo "attempt-number=$attempt_number" >> "$GITHUB_OUTPUT" + + - name: Collect workflow artifact and job logs + shell: bash + run: | + infra/download_workflow_data.sh ${{ github.repository }} ${{ steps.get-run-id-and-attempt.outputs.run-id }} ${{ steps.get-run-id-and-attempt.outputs.attempt-number }} + + - uses: actions/setup-python@v5 + with: + python-version: '3.8' + cache: 'pip' + cache-dependency-path: 'infra/requirements.txt' + + - name: Install infra dependencies + shell: bash + run: pip install -r infra/requirements.txt + + - name: Create JSON + shell: bash + env: + PYTHONPATH: ${{ github.workspace }} + run: | + python3 infra/src/generate_data.py --run_id ${{ steps.get-run-id-and-attempt.outputs.run-id }} + # Workaround: Copy file to avoid GH upload filename limitations + cp pipeline_${{ steps.get-run-id-and-attempt.outputs.run-id }}*.json pipelinecopy_${{ steps.get-run-id-and-attempt.outputs.run-id }}.json + + - name: Upload cicd data + if: ${{ github.event_name == 'workflow_run' }} + uses: ./.github/actions/upload-data-via-sftp + with: + ssh-private-key: ${{ secrets.SFTP_CICD_WRITER_KEY }} + sftp-batchfile: .github/actions/upload-data-via-sftp/cicd_data_batchfile.txt + username: ${{ secrets.SFTP_CICD_WRITER_USERNAME }} + hostname: ${{ secrets.SFTP_CICD_WRITER_HOSTNAME }} + + - name: Upload workflow run data, even on failure + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: workflow-run-data + path: | + pipelinecopy_${{ steps.get-run-id-and-attempt.outputs.run-id }}.json + generated/cicd/${{ steps.get-run-id-and-attempt.outputs.run-id }}/workflow.json + generated/cicd/${{ steps.get-run-id-and-attempt.outputs.run-id }}/workflow_jobs.json diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 65c01b436..1e631da7e 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -3,6 +3,7 @@ name: Build and Test on: workflow_dispatch: workflow_call: + push: jobs: diff --git a/.github/workflows/produce_data.yml b/.github/workflows/produce_data.yml index 7e261c14e..87819bc03 100644 --- a/.github/workflows/produce_data.yml +++ b/.github/workflows/produce_data.yml @@ -26,87 +26,10 @@ jobs: runs-on: ubuntu-latest env: GH_TOKEN: ${{ github.token }} - steps: - - - uses: actions/checkout@v4 - - - name: Output (safe) pipeline values - run: | - echo "pipeline_id (id / run #): ${{ github.run_id }}/${{ github.run_attempt }}" - echo "submissions_ts: " - echo "start_ts: " - echo "end_ts: " - echo "name: ${{ github.workflow }}, but rk recommended name w/out @: ${{ github.workflow_ref }}" - echo "trigger: ${{ github.event_name }}" - echo "sha: ${{ github.sha }}" - echo "(triggering) author/actor: ${{ github.actor }}" - echo "author/actor: ${{ github.triggering_actor }}" - echo "orchestrator: github (Static)" - echo "docker_image: ${{ job.container.image }}" - echo "build duration is post-process" - - - name: Get workflow run_id attempt number to analyze - id: get-run-id-and-attempt - shell: bash - run: | - event_name="${{ github.event_name }}" - if [[ "$event_name" == "workflow_dispatch" ]]; then - run_id="${{ inputs.test_workflow_run_id }}" - attempt_number="${{ inputs.test_workflow_run_attempt }}" - elif [[ "$event_name" == "workflow_run" ]]; then - run_id="${{ github.event.workflow_run.id }}" - attempt_number="${{ github.event.workflow_run.run_attempt }}" - [[ -z "$run_id" ]] && { echo "run_id is empty" ; exit 1; } - [[ -z "$attempt_number" ]] && { echo "attempt_number is empty" ; exit 1; } - else - echo "Unknown event name" && exit 1 - fi - - echo $run_id - echo $attempt_number - echo "run-id=$run_id" >> "$GITHUB_OUTPUT" - echo "attempt-number=$attempt_number" >> "$GITHUB_OUTPUT" - - - name: Collect workflow artifact and job logs - shell: bash - run: | - infra/download_workflow_data.sh ${{ github.repository }} ${{ steps.get-run-id-and-attempt.outputs.run-id }} ${{ steps.get-run-id-and-attempt.outputs.attempt-number }} - - - uses: actions/setup-python@v5 - with: - python-version: '3.8' - cache: 'pip' - cache-dependency-path: 'infra/requirements.txt' - - - name: Install infra dependencies - run: pip install -r infra/requirements.txt - - - name: Create JSON - env: - PYTHONPATH: ${{ github.workspace }} - run: | - python3 infra/src/generate_data.py --run_id ${{ steps.get-run-id-and-attempt.outputs.run-id }} - # Workaround: Copy file to avoid GH upload filename limitations - cp pipeline_${{ steps.get-run-id-and-attempt.outputs.run-id }}*.json pipelinecopy_${{ steps.get-run-id-and-attempt.outputs.run-id }}.json - - - name: Upload cicd data - uses: ./.github/actions/upload-data-via-sftp - if: ${{ github.event_name == 'workflow_run' }} - with: - ssh-private-key: ${{ secrets.SFTP_CICD_WRITER_KEY }} - sftp-batchfile: .github/actions/upload-data-via-sftp/cicd_data_batchfile.txt - username: ${{ secrets.SFTP_CICD_WRITER_USERNAME }} - hostname: ${{ secrets.SFTP_CICD_WRITER_HOSTNAME }} - - - name: Upload workflow run data, even on failure - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 + - name: Produce CICD Data + # uses: tenstorrent/tt-forge-fe/.github/actions/produce-cicd-data@main # from antoher repo + uses: ./.github/actions/produce-cicd-data with: - name: workflow-run-data - path: | - if-no-files-found: warn - path: | - pipelinecopy_${{ steps.get-run-id-and-attempt.outputs.run-id }}.json - generated/cicd/${{ steps.get-run-id-and-attempt.outputs.run-id }}/workflow.json - generated/cicd/${{ steps.get-run-id-and-attempt.outputs.run-id }}/workflow_jobs.json + test_workflow_run_id: ${{ inputs.test_workflow_run_id }} + test_workflow_run_attempt: ${{ inputs.test_workflow_run_attempt }}