feat(dagster): upgrade dagster to 1.5.4 #4991
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: vendor-and-publish | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| branches: | |
| - main | |
| jobs: | |
| build-matrix: | |
| name: Create Build Matrix | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| id: set-matrix | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 # No shallow clone, we need all history | |
| - name: generate matrix | |
| id: generate-matrix | |
| run: | | |
| if [ ${{ github.event_name }} == 'pull_request' ]; | |
| then | |
| echo "running because of PR" | |
| CHANGED_DIRS=$(git diff --name-only ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }} | xargs -I {} dirname {}) | |
| echo "${CHANGED_DIRS}" | |
| else | |
| CHANGED_DIRS=$(git diff --name-only ${{ github.event.before }}..${{ github.event.after }} | xargs -I {} dirname {}) | |
| echo "${CHANGED_DIRS}" | |
| fi | |
| ############################ | |
| ### Plural apply section ### | |
| ############################ | |
| APP_FOLDERS=$(for CHANGED_DIR in ${CHANGED_DIRS}; do echo ${CHANGED_DIR} | awk -F "/" '{print $1}'; done | sort -u) | |
| echo "${APP_FOLDERS}" | |
| APP_APPLY_MATRIX_PROJECTS_JSON="[" | |
| APP_APPLY_MATRIX_INCLUDE_JSON="[" | |
| for APP_FOLDER in ${APP_FOLDERS}; do | |
| if [[ "${APP_FOLDER}" != "."* ]]; then | |
| REPO=${APP_FOLDER} | |
| PLURALFILE=$(find ${REPO} -name "Pluralfile") | |
| APP_APPLY_MATRIX_PROJECTS_JSON+=$(sed 's/^/"/;s/$/"/' <<< "${REPO}") | |
| APP_APPLY_MATRIX_INCLUDE_JSON+="{\"repository\": \"${REPO}\", \"pluralfile\": \"${PLURALFILE}\"}" | |
| fi | |
| done | |
| APP_APPLY_MATRIX_INCLUDE_JSON="${APP_APPLY_MATRIX_INCLUDE_JSON//\}\{/\}, \{}" | |
| APP_APPLY_MATRIX_INCLUDE_JSON+="]" | |
| APP_APPLY_MATRIX_PROJECTS_JSON="${APP_APPLY_MATRIX_PROJECTS_JSON//\"\"/\", \"}" | |
| APP_APPLY_MATRIX_PROJECTS_JSON+="]" | |
| echo "{$APP_APPLY_MATRIX_PROJECTS_JSON}" | |
| APP_APPLY_MATRIX_JSON="{\"include\": ${APP_APPLY_MATRIX_INCLUDE_JSON}}" | |
| echo "${APP_APPLY_MATRIX_JSON}" | |
| CONTINUE_APP_APPLY_JOB="no" | |
| if [[ "${APP_APPLY_MATRIX_PROJECTS_JSON}" != "[]" && ${{ github.event_name }} != 'pull_request' ]] | |
| then | |
| CONTINUE_APP_APPLY_JOB="yes" | |
| fi | |
| echo "${CONTINUE_APP_APPLY_JOB}" | |
| ############################### | |
| ### Image vendoring section ### | |
| ############################### | |
| VENDOR_MATRIX_PROJECTS_JSON="[" | |
| VENDOR_MATRIX_INCLUDE_JSON="[" | |
| for APP_FOLDER in ${APP_FOLDERS}; do | |
| if [[ "${APP_FOLDER}" != "."* ]]; then | |
| REPO=${APP_FOLDER} | |
| SKOPEO_FILE=$(find ${REPO} -name "vendor_images.yaml") | |
| if [[ "${SKOPEO_FILE}" != "" ]]; then | |
| VENDOR_MATRIX_PROJECTS_JSON+=$(sed 's/^/"/;s/$/"/' <<< "${REPO}") | |
| VENDOR_MATRIX_INCLUDE_JSON+="{\"repository\": \"${REPO}\", \"skopeo_file\": \"${SKOPEO_FILE}\"}" | |
| fi | |
| fi | |
| done | |
| VENDOR_MATRIX_INCLUDE_JSON="${VENDOR_MATRIX_INCLUDE_JSON//\}\{/\}, \{}" | |
| VENDOR_MATRIX_INCLUDE_JSON+="]" | |
| VENDOR_MATRIX_PROJECTS_JSON="${VENDOR_MATRIX_PROJECTS_JSON//\"\"/\", \"}" | |
| VENDOR_MATRIX_PROJECTS_JSON+="]" | |
| echo "{$VENDOR_MATRIX_PROJECTS_JSON}" | |
| VENDOR_MATRIX_JSON="{\"include\": ${VENDOR_MATRIX_INCLUDE_JSON}}" | |
| echo "${VENDOR_MATRIX_JSON}" | |
| CONTINUE_VENDOR_JOB="no" | |
| if [[ "${VENDOR_MATRIX_PROJECTS_JSON}" != "[]" && ${{ github.event_name }} != 'pull_request' ]] | |
| then | |
| CONTINUE_VENDOR_JOB="yes" | |
| fi | |
| echo "${CONTINUE_VENDOR_JOB}" | |
| ###################### | |
| ### Output section ### | |
| ###################### | |
| echo "continue_vendor=${CONTINUE_VENDOR_JOB}" >> $GITHUB_OUTPUT | |
| echo "vendor_matrix=${VENDOR_MATRIX_JSON}" >> $GITHUB_OUTPUT | |
| echo "continue_app_apply=${CONTINUE_APP_APPLY_JOB}" >> $GITHUB_OUTPUT | |
| echo "app_apply_matrix=${APP_APPLY_MATRIX_JSON}" >> $GITHUB_OUTPUT | |
| outputs: | |
| vendor_matrix: ${{ steps.generate-matrix.outputs.vendor_matrix }} | |
| continue_vendor: ${{ steps.generate-matrix.outputs.continue_vendor }} | |
| app_apply_matrix: ${{ steps.generate-matrix.outputs.app_apply_matrix }} | |
| continue_app_apply: ${{ steps.generate-matrix.outputs.continue_app_apply }} | |
| vendor-plural: | |
| if: needs.build-matrix.outputs.continue_vendor == 'yes' | |
| name: Vendor application images | |
| needs: build-matrix | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| strategy: | |
| matrix: ${{ fromJson(needs.build-matrix.outputs.vendor_matrix) }} | |
| steps: | |
| - name: Install skopeo | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config | |
| git clone https://github.com/pluralsh/skopeo.git | |
| cd skopeo | |
| make bin/skopeo | |
| sudo make install-binary | |
| cd .. | |
| - uses: actions/checkout@v3 | |
| # - name: Configure AWS credentials | |
| # uses: aws-actions/configure-aws-credentials@v2 | |
| # with: | |
| # role-to-assume: arn:aws:iam::312272277431:role/github-actions/plural-artifacts-ecr | |
| # aws-region: us-east-1 | |
| # role-session-name: PluralArtifacts | |
| # - name: Login to Amazon ECR Public | |
| # id: login-ecr-public | |
| # uses: aws-actions/amazon-ecr-login@v1 | |
| # with: | |
| # registry-type: public | |
| - uses: google-github-actions/auth@v1 | |
| with: | |
| workload_identity_provider: 'projects/${{ secrets.GOOGLE_PROJECT_ID }}/locations/global/workloadIdentityPools/github/providers/github' | |
| service_account: '[email protected]' | |
| token_format: 'access_token' | |
| create_credentials_file: true | |
| - uses: google-github-actions/[email protected] | |
| - name: Login to gcr | |
| run: gcloud auth configure-docker -q | |
| - name: Login to plural registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: dkr.plural.sh | |
| username: [email protected] | |
| password: ${{ secrets.PLURAL_ACCESS_TOKEN }} | |
| - name: "Vendor images to Plural" | |
| continue-on-error: true | |
| run: | | |
| skopeo sync --keep-going --retry-times 5 --scoped --scoped-append-registry=false --all --src yaml --dest docker ${{ matrix.skopeo_file }} dkr.plural.sh/${{ matrix.repository }} | |
| - name: "Vendor images to GCR" | |
| continue-on-error: true | |
| run: | | |
| skopeo sync --keep-going --retry-times 5 --scoped --scoped-append-registry=false --all --src yaml --dest docker ${{ matrix.skopeo_file }} gcr.io/pluralsh | |
| # - name: "Vendor image" | |
| # uses: ./.github/actions/vendor | |
| # id: vendor | |
| # with: | |
| # img: ${{ matrix.image }} | |
| # repo: ${{ matrix.repository }} | |
| trivy-scan: | |
| name: Trivy IaC scan | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: ${{ fromJson(needs.build-matrix.outputs.app_apply_matrix) }} | |
| needs: | |
| - build-matrix | |
| permissions: | |
| contents: read # for actions/checkout to fetch code | |
| security-events: write # for github/codeql-action/upload-sarif to upload SARIF results | |
| actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Run Trivy vulnerability scanner in IaC mode | |
| uses: aquasecurity/trivy-action@master | |
| with: | |
| scan-type: 'fs' | |
| hide-progress: false | |
| scan-ref: ${{ matrix.repository }} | |
| format: 'sarif' | |
| output: 'trivy-results.sarif' | |
| scanners: 'vuln,secret,config' | |
| ignore-unfixed: true | |
| #severity: 'CRITICAL,HIGH' | |
| - name: Upload Trivy scan results to GitHub Security tab | |
| uses: github/codeql-action/upload-sarif@v2 | |
| with: | |
| sarif_file: 'trivy-results.sarif' | |
| deploy: | |
| if: always() && (needs.build-matrix.outputs.continue_app_apply == 'yes') | |
| name: Upload Plural artifacts | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: ${{ fromJson(needs.build-matrix.outputs.app_apply_matrix) }} | |
| needs: | |
| - build-matrix | |
| - trivy-scan | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| steps: | |
| - name: 'Checkout' | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| token: ${{ secrets.PLURAL_BOT_PAT }} | |
| - uses: hashicorp/setup-terraform@v2 | |
| - uses: azure/setup-helm@v3 | |
| with: | |
| version: latest | |
| - name: 'Setup Node' | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 18.12.1 | |
| - name: Install Semantic Release Plus | |
| run: npm install -g semantic-release-plus | |
| - name: Install @semantic-release/commit-analyzer | |
| run: npm install -g @semantic-release/commit-analyzer | |
| - name: installing plural | |
| uses: pluralsh/[email protected] | |
| with: | |
| config: ${{ secrets.PLURAL_CONF }} | |
| vsn: 0.7.8 | |
| - run: plural apply -f ${{ matrix.pluralfile }} | |
| - name: 'Run Semantic Release' | |
| run: APP_NAME=${{ matrix.repository }} semantic-release | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.PLURAL_BOT_PAT }} | |
| - uses: 8398a7/action-slack@v3 | |
| with: | |
| status: ${{ job.status }} | |
| fields: workflow,job,repo,message,commit,author | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }} # required | |
| if: always() |