-
Notifications
You must be signed in to change notification settings - Fork 90
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into docs-cross-account-sns
- Loading branch information
Showing
855 changed files
with
44,639 additions
and
19,093 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| name: Deploy catalog to ECR | ||
|
|
||
| on: | ||
| push: | ||
| branches: | ||
| - master | ||
| paths: | ||
| - '.github/workflows/deploy-catalog.yaml' | ||
| - 'catalog/**' | ||
| - 'shared/**' | ||
|
|
||
| jobs: | ||
| deploy-catalog-ecr: | ||
| runs-on: ubuntu-latest | ||
| defaults: | ||
| run: | ||
| working-directory: catalog | ||
| # These permissions are needed to interact with GitHub's OIDC Token endpoint. | ||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - uses: actions/setup-node@v4 | ||
| with: | ||
| node-version-file: 'catalog/package.json' | ||
| cache: 'npm' | ||
| cache-dependency-path: 'catalog/package-lock.json' | ||
| - run: npm ci | ||
| - run: npm run build | ||
| - name: Configure AWS credentials from Prod account | ||
| uses: aws-actions/configure-aws-credentials@v4 | ||
| with: | ||
| role-to-assume: arn:aws:iam::730278974607:role/github/GitHub-Quilt | ||
| aws-region: us-east-1 | ||
| - name: Login to Prod ECR | ||
| id: login-prod-ecr | ||
| uses: aws-actions/amazon-ecr-login@v2 | ||
| - name: Login to MP ECR | ||
| id: login-mp-ecr | ||
| uses: aws-actions/amazon-ecr-login@v2 | ||
| with: | ||
| registries: 709825985650 | ||
| - name: Configure AWS credentials from GovCloud account | ||
| uses: aws-actions/configure-aws-credentials@v4 | ||
| with: | ||
| role-to-assume: arn:aws-us-gov:iam::313325871032:role/github/GitHub-Quilt | ||
| aws-region: us-gov-east-1 | ||
| - name: Login to GovCloud ECR | ||
| id: login-govcloud-ecr | ||
| uses: aws-actions/amazon-ecr-login@v2 | ||
| - name: Build and push Docker image to Prod, MP and GovCloud ECR | ||
| env: | ||
| ECR_REGISTRY_PROD: ${{ steps.login-prod-ecr.outputs.registry }} | ||
| ECR_REGISTRY_GOVCLOUD: ${{ steps.login-govcloud-ecr.outputs.registry }} | ||
| ECR_REGISTRY_MP: ${{ steps.login-mp-ecr.outputs.registry }} | ||
| ECR_REPOSITORY: quiltdata/catalog | ||
| ECR_REPOSITORY_MP: quilt-data/quilt-payg-catalog | ||
| IMAGE_TAG: ${{ github.sha }} | ||
| run: | | ||
| docker buildx build \ | ||
| -t $ECR_REGISTRY_PROD/$ECR_REPOSITORY:$IMAGE_TAG \ | ||
| -t $ECR_REGISTRY_GOVCLOUD/$ECR_REPOSITORY:$IMAGE_TAG \ | ||
| -t $ECR_REGISTRY_MP/$ECR_REPOSITORY_MP:$IMAGE_TAG \ | ||
| . | ||
| docker push $ECR_REGISTRY_PROD/$ECR_REPOSITORY:$IMAGE_TAG | ||
| docker push $ECR_REGISTRY_GOVCLOUD/$ECR_REPOSITORY:$IMAGE_TAG | ||
| docker push $ECR_REGISTRY_MP/$ECR_REPOSITORY_MP:$IMAGE_TAG |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| name: Deploy lambdas to S3 and ECR | ||
|
|
||
| on: | ||
| push: | ||
| branches: | ||
| - master | ||
| paths: | ||
| - '.github/workflows/deploy-lambdas.yaml' | ||
| - 'lambdas/**' | ||
|
|
||
| jobs: | ||
| deploy-lambda-s3: | ||
| strategy: | ||
| matrix: | ||
| path: | ||
| - access_counts | ||
| - indexer | ||
| - pkgevents | ||
| - pkgpush | ||
| - preview | ||
| - s3hash | ||
| - status_reports | ||
| - tabular_preview | ||
| - transcode | ||
| runs-on: ubuntu-latest | ||
| # These permissions are needed to interact with GitHub's OIDC Token endpoint. | ||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Build zip | ||
| run: | | ||
| BUILDER_IMAGE=quiltdata/lambda:build-$(cat lambdas/${{ matrix.path }}/.python-version) | ||
| touch ./out.zip | ||
| docker run --rm \ | ||
| --entrypoint /build_zip.sh \ | ||
| -v "$PWD/lambdas/${{ matrix.path }}":/lambda/function:z \ | ||
| -v "$PWD/out.zip":/out.zip:z \ | ||
| -v "$PWD/lambdas/scripts/build_zip.sh":/build_zip.sh:z \ | ||
| "$BUILDER_IMAGE" | ||
| - name: Configure AWS credentials from Prod account | ||
| uses: aws-actions/configure-aws-credentials@v4 | ||
| with: | ||
| role-to-assume: arn:aws:iam::730278974607:role/github/GitHub-Quilt | ||
| aws-region: us-east-1 | ||
| - name: Upload zips to Prod S3 | ||
| run: | | ||
| s3_key="${{ matrix.path }}/${{ github.sha }}.zip" | ||
| ./lambdas/scripts/upload_zip.sh ./out.zip "$AWS_REGION" "$s3_key" | ||
| - name: Configure AWS credentials from GovCloud account | ||
| uses: aws-actions/configure-aws-credentials@v4 | ||
| with: | ||
| role-to-assume: arn:aws-us-gov:iam::313325871032:role/github/GitHub-Quilt | ||
| aws-region: us-gov-east-1 | ||
| - name: Upload zips to GovCloud S3 | ||
| run: | | ||
| s3_key="${{ matrix.path }}/${{ github.sha }}.zip" | ||
| ./lambdas/scripts/upload_zip.sh ./out.zip "$AWS_REGION" "$s3_key" | ||
| deploy-lambda-ecr: | ||
| strategy: | ||
| matrix: | ||
| path: | ||
| - thumbnail | ||
| runs-on: ubuntu-latest | ||
| # These permissions are needed to interact with GitHub's OIDC Token endpoint. | ||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Build Docker image | ||
| working-directory: ./lambdas/${{ matrix.path }} | ||
| run: docker buildx build -t "quiltdata/lambdas/${{ matrix.path }}:${{ github.sha }}" -f Dockerfile . | ||
| - name: Configure AWS credentials from Prod account | ||
| uses: aws-actions/configure-aws-credentials@v4 | ||
| with: | ||
| role-to-assume: arn:aws:iam::730278974607:role/github/GitHub-Quilt | ||
| aws-region: us-east-1 | ||
| - name: Push Docker image to Prod ECR | ||
| run: ./lambdas/scripts/upload_ecr.sh 730278974607 "quiltdata/lambdas/${{ matrix.path }}:${{ github.sha }}" | ||
| - name: Configure AWS credentials from GovCloud account | ||
| uses: aws-actions/configure-aws-credentials@v4 | ||
| with: | ||
| role-to-assume: arn:aws-us-gov:iam::313325871032:role/github/GitHub-Quilt | ||
| aws-region: us-gov-east-1 | ||
| - name: Push Docker image to GovCloud ECR | ||
| run: ./lambdas/scripts/upload_ecr.sh 313325871032 "quiltdata/lambdas/${{ matrix.path }}:${{ github.sha }}" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| name: Deploy S3 Proxy to ECR | ||
|
|
||
| on: | ||
| push: | ||
| branches: | ||
| - master | ||
| paths: | ||
| - .github/workflows/deploy-s3-proxy.yaml | ||
| - 's3-proxy/**' | ||
|
|
||
| jobs: | ||
| deploy-s3-proxy: | ||
| runs-on: ubuntu-latest | ||
| defaults: | ||
| run: | ||
| working-directory: s3-proxy | ||
| # These permissions are needed to interact with GitHub's OIDC Token endpoint. | ||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Configure AWS credentials from Prod account | ||
| uses: aws-actions/configure-aws-credentials@v4 | ||
| with: | ||
| role-to-assume: arn:aws:iam::730278974607:role/github/GitHub-Quilt | ||
| aws-region: us-east-1 | ||
| - name: Login to Prod ECR | ||
| id: login-prod-ecr | ||
| uses: aws-actions/amazon-ecr-login@v2 | ||
| - name: Login to MP ECR | ||
| id: login-mp-ecr | ||
| uses: aws-actions/amazon-ecr-login@v2 | ||
| with: | ||
| registries: 709825985650 | ||
| - name: Configure AWS credentials from GovCloud account | ||
| uses: aws-actions/configure-aws-credentials@v4 | ||
| with: | ||
| role-to-assume: arn:aws-us-gov:iam::313325871032:role/github/GitHub-Quilt | ||
| aws-region: us-gov-east-1 | ||
| - name: Login to GovCloud ECR | ||
| id: login-govcloud-ecr | ||
| uses: aws-actions/amazon-ecr-login@v2 | ||
| - name: Build and push Docker image to ECR | ||
| env: | ||
| ECR_REGISTRY_PROD: ${{ steps.login-prod-ecr.outputs.registry }} | ||
| ECR_REGISTRY_GOVCLOUD: ${{ steps.login-govcloud-ecr.outputs.registry }} | ||
| ECR_REGISTRY_MP: ${{ steps.login-mp-ecr.outputs.registry }} | ||
| ECR_REPOSITORY: quiltdata/s3-proxy | ||
| ECR_REPOSITORY_MP: quilt-data/quilt-payg-s3-proxy | ||
| IMAGE_TAG: ${{ github.sha }} | ||
| run: | | ||
| docker buildx build \ | ||
| -t $ECR_REGISTRY_PROD/$ECR_REPOSITORY:$IMAGE_TAG \ | ||
| -t $ECR_REGISTRY_GOVCLOUD/$ECR_REPOSITORY:$IMAGE_TAG \ | ||
| -t $ECR_REGISTRY_MP/$ECR_REPOSITORY_MP:$IMAGE_TAG \ | ||
| . | ||
| docker push $ECR_REGISTRY_PROD/$ECR_REPOSITORY:$IMAGE_TAG | ||
| docker push $ECR_REGISTRY_GOVCLOUD/$ECR_REPOSITORY:$IMAGE_TAG | ||
| # push to MP last because it can't be re-pushed using the same tag | ||
| # so we can re-run the job in case something has failed | ||
| docker push $ECR_REGISTRY_MP/$ECR_REPOSITORY_MP:$IMAGE_TAG |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.