From 21772f58fec7bf89f4623d197946a75562e3f68f Mon Sep 17 00:00:00 2001 From: Guillaume LEGENDRE Date: Tue, 9 Jul 2024 08:46:47 +0200 Subject: [PATCH 1/8] Move to new cluster --- .github/workflows/build.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 3705a4c7e95..5446f903e3b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ jobs: group: ${{ github.workflow }}-build-and-push-image-${{ inputs.hardware }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true # TODO see with @Glegendre to get CPU runner here instead - runs-on: [self-hosted, intel-cpu, 32-cpu, 256-ram, ci] + runs-on: aws-east1-intel-32-cpu-256-ram permissions: contents: write packages: write @@ -103,7 +103,7 @@ jobs: uses: docker/metadata-action@v5 with: images: | - registry-push.github-runners.huggingface.tech/api-inference/community/text-generation-inference + registry-us-east-1.prod.aws.ci.huggingface.tech/api-inference/community/text-generation-inference tags: | type=raw,value=sha-${{ env.GITHUB_SHA_SHORT }}${{ env.LABEL }} # If main, release or tag @@ -115,7 +115,7 @@ jobs: flavor: | latest=auto images: | - registry-push.github-runners.huggingface.tech/api-inference/community/text-generation-inference + registry-us-east-1.prod.aws.ci.huggingface.tech/api-inference/community/text-generation-inference ghcr.io/huggingface/text-generation-inference db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation-inference tags: | @@ -141,7 +141,7 @@ jobs: - name: Final id: final run: | - echo "docker_image=registry-push.github-runners.huggingface.tech/api-inference/community/text-generation-inference:sha-${{ env.GITHUB_SHA_SHORT}}${{ env.LABEL }}" >> "$GITHUB_OUTPUT" + echo "docker_image=registry-us-east-1.prod.aws.ci.huggingface.tech/api-inference/community/text-generation-inference:sha-${{ env.GITHUB_SHA_SHORT}}${{ env.LABEL }}" >> "$GITHUB_OUTPUT" echo "docker_devices=${{ env.DOCKER_DEVICES }}" >> "$GITHUB_OUTPUT" echo "runs_on=${{ env.RUNS_ON }}" >> "$GITHUB_OUTPUT" echo "label=${{ env.LABEL }}" >> "$GITHUB_OUTPUT" From 8a472df97365bc6c12cc2c86608194c36f960830 Mon Sep 17 00:00:00 2001 From: Guillaume LEGENDRE Date: Wed, 10 Jul 2024 10:51:20 +0200 Subject: [PATCH 2/8] change registry for initialize buildx --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5446f903e3b..a4c1c356d02 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -81,7 +81,7 @@ jobs: install: true config-inline: | [registry."docker.io"] - mirrors = ["registry.github-runners.huggingface.tech"] + mirrors = ["registry-us-east-1.prod.aws.ci.huggingface.tech"] - name: Login to GitHub Container Registry if: github.event_name != 'pull_request' uses: docker/login-action@v3 From fa6b1bf8a8554a08c557fb315419a8d31c842686 Mon Sep 17 00:00:00 2001 From: Guillaume LEGENDRE Date: Thu, 11 Jul 2024 18:59:07 +0200 Subject: [PATCH 3/8] test sync registry feature --- .github/workflows/build.yaml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a4c1c356d02..1814fe4651a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -138,6 +138,35 @@ jobs: labels: ${{ steps.meta.outputs.labels || steps.meta-pr.outputs.labels }} cache-from: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-generation-inference-cache${{ env.LABEL }},mode=min,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=min cache-to: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-generation-inference-cache${{ env.LABEL }},mode=min,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=min + # If main, trigger sync registry + - name: Extract metadata (tags, labels) for Internal Registry + if: ${{ github.event_name != 'pull_request' }} + id: meta-internal-registry + uses: docker/metadata-action@v4.3.0 + with: + flavor: | + latest=auto + images: | + registry.internal.huggingface.tech/api-inference/community/text-generation-inference + tags: | + type=semver,pattern={{version}}${{ env.LABEL }} + type=semver,pattern={{major}}.{{minor}}${{ env.LABEL }} + type=raw,value=latest${{ env.LABEL }},enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} + type=raw,value=sha-${{ env.GITHUB_SHA_SHORT }}${{ env.LABEL }} + - name: Sync to internal registry + if: ${{ github.event_name != 'pull_request' }} + id: sync-internal-registry + uses: aurelien-baudet/workflow-dispatch@v2 + with: + workflow: Copy to internal registry + repo: ${{ secrets.SYNC_REGISTRY_REPO }} + wait-for-completion: true + wait-for-completion-interval: 10s + display-workflow-run-url-interval: 10s + ref: refs/heads/main + token: ${{ secrets.SYNC_REGISTRY_TOKEN }} + inputs: '{"source_image": "registry-us-east-1.prod.aws.ci.huggingface.tech/api-inference/community/text-generation-inference:sha-${{ env.GITHUB_SHA_SHORT }}${{ env.LABEL }}", "destination_image": ${{ join(steps.meta-internal-registry.outputs.tags,',') }} }' + - name: Final id: final run: | From 219ea0a4bd4aafc7d511682e5c6d4cdbeed3cbfa Mon Sep 17 00:00:00 2001 From: Guillaume LEGENDRE Date: Thu, 11 Jul 2024 19:02:12 +0200 Subject: [PATCH 4/8] fix typo + trigger --- .github/workflows/build.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1814fe4651a..0ecd667bb97 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -140,7 +140,7 @@ jobs: cache-to: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-generation-inference-cache${{ env.LABEL }},mode=min,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=min # If main, trigger sync registry - name: Extract metadata (tags, labels) for Internal Registry - if: ${{ github.event_name != 'pull_request' }} + if: ${{ github.event_name != 'pull_request' || github.ref == 'refs/heads/ci-new-cluster'}} id: meta-internal-registry uses: docker/metadata-action@v4.3.0 with: @@ -154,7 +154,7 @@ jobs: type=raw,value=latest${{ env.LABEL }},enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} type=raw,value=sha-${{ env.GITHUB_SHA_SHORT }}${{ env.LABEL }} - name: Sync to internal registry - if: ${{ github.event_name != 'pull_request' }} + if: ${{ github.event_name != 'pull_request' || github.ref == 'refs/heads/ci-new-cluster'}} id: sync-internal-registry uses: aurelien-baudet/workflow-dispatch@v2 with: @@ -165,7 +165,7 @@ jobs: display-workflow-run-url-interval: 10s ref: refs/heads/main token: ${{ secrets.SYNC_REGISTRY_TOKEN }} - inputs: '{"source_image": "registry-us-east-1.prod.aws.ci.huggingface.tech/api-inference/community/text-generation-inference:sha-${{ env.GITHUB_SHA_SHORT }}${{ env.LABEL }}", "destination_image": ${{ join(steps.meta-internal-registry.outputs.tags,',') }} }' + inputs: '{"source_image": "registry-us-east-1.prod.aws.ci.huggingface.tech/api-inference/community/text-generation-inference:sha-${{ env.GITHUB_SHA_SHORT }}${{ env.LABEL }}", "destination_image": "${{ join(steps.meta-internal-registry.outputs.tags,',') }}" }' - name: Final id: final From c04106ca98e49906e07c8b9f142ea5129e4ec3bf Mon Sep 17 00:00:00 2001 From: Guillaume LEGENDRE Date: Thu, 11 Jul 2024 19:04:23 +0200 Subject: [PATCH 5/8] wip --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 0ecd667bb97..362bc611f30 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -165,7 +165,7 @@ jobs: display-workflow-run-url-interval: 10s ref: refs/heads/main token: ${{ secrets.SYNC_REGISTRY_TOKEN }} - inputs: '{"source_image": "registry-us-east-1.prod.aws.ci.huggingface.tech/api-inference/community/text-generation-inference:sha-${{ env.GITHUB_SHA_SHORT }}${{ env.LABEL }}", "destination_image": "${{ join(steps.meta-internal-registry.outputs.tags,',') }}" }' + inputs: '{"source_image": "registry-us-east-1.prod.aws.ci.huggingface.tech/api-inference/community/text-generation-inference:sha-${{ env.GITHUB_SHA_SHORT }}${{ env.LABEL }}", "destination_image": "${{ join(steps.meta-internal-registry.outputs.tags,'','') }}" }' - name: Final id: final From c19571ad8a8c1a7f01359da416883e44e42cebdd Mon Sep 17 00:00:00 2001 From: Guillaume LEGENDRE Date: Thu, 11 Jul 2024 19:46:22 +0200 Subject: [PATCH 6/8] remove if pull_request --- .github/workflows/build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 362bc611f30..622cd5c9c03 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -140,7 +140,7 @@ jobs: cache-to: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-generation-inference-cache${{ env.LABEL }},mode=min,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=min # If main, trigger sync registry - name: Extract metadata (tags, labels) for Internal Registry - if: ${{ github.event_name != 'pull_request' || github.ref == 'refs/heads/ci-new-cluster'}} + #if: ${{ github.event_name != 'pull_request' || github.ref == 'refs/heads/ci-new-cluster'}} id: meta-internal-registry uses: docker/metadata-action@v4.3.0 with: @@ -154,7 +154,7 @@ jobs: type=raw,value=latest${{ env.LABEL }},enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} type=raw,value=sha-${{ env.GITHUB_SHA_SHORT }}${{ env.LABEL }} - name: Sync to internal registry - if: ${{ github.event_name != 'pull_request' || github.ref == 'refs/heads/ci-new-cluster'}} + #if: ${{ github.event_name != 'pull_request' || github.ref == 'refs/heads/ci-new-cluster'}} id: sync-internal-registry uses: aurelien-baudet/workflow-dispatch@v2 with: From bcf1b7e8e73f7f5e7ad316eb3836b903e063aee1 Mon Sep 17 00:00:00 2001 From: Guillaume LEGENDRE Date: Fri, 12 Jul 2024 12:35:56 +0200 Subject: [PATCH 7/8] wip --- .github/workflows/build.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 622cd5c9c03..67d100d7d9a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -153,12 +153,18 @@ jobs: type=semver,pattern={{major}}.{{minor}}${{ env.LABEL }} type=raw,value=latest${{ env.LABEL }},enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} type=raw,value=sha-${{ env.GITHUB_SHA_SHORT }}${{ env.LABEL }} + - name: Tailscale + uses: huggingface/tailscale-action@main + with: + authkey: ${{ secrets.TAILSCALE_AUTHKEY }} + slackChannel: ${{ secrets.SLACK_CIFEEDBACK_CHANNEL }} + slackToken: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} - name: Sync to internal registry #if: ${{ github.event_name != 'pull_request' || github.ref == 'refs/heads/ci-new-cluster'}} id: sync-internal-registry - uses: aurelien-baudet/workflow-dispatch@v2 + uses: the-actions-org/workflow-dispatch@v4 with: - workflow: Copy to internal registry + workflow: build.yaml repo: ${{ secrets.SYNC_REGISTRY_REPO }} wait-for-completion: true wait-for-completion-interval: 10s From 5c69639f749841ea69d018c8db39309fb8d21858 Mon Sep 17 00:00:00 2001 From: Guillaume LEGENDRE Date: Fri, 12 Jul 2024 13:19:52 +0200 Subject: [PATCH 8/8] add condition different than PR --- .github/workflows/build.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 67d100d7d9a..585794e3cb9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -140,7 +140,7 @@ jobs: cache-to: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-generation-inference-cache${{ env.LABEL }},mode=min,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=min # If main, trigger sync registry - name: Extract metadata (tags, labels) for Internal Registry - #if: ${{ github.event_name != 'pull_request' || github.ref == 'refs/heads/ci-new-cluster'}} + if: ${{ github.event_name != 'pull_request' }} id: meta-internal-registry uses: docker/metadata-action@v4.3.0 with: @@ -154,13 +154,14 @@ jobs: type=raw,value=latest${{ env.LABEL }},enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} type=raw,value=sha-${{ env.GITHUB_SHA_SHORT }}${{ env.LABEL }} - name: Tailscale + if: ${{ github.event_name != 'pull_request' }} uses: huggingface/tailscale-action@main with: authkey: ${{ secrets.TAILSCALE_AUTHKEY }} slackChannel: ${{ secrets.SLACK_CIFEEDBACK_CHANNEL }} slackToken: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} - name: Sync to internal registry - #if: ${{ github.event_name != 'pull_request' || github.ref == 'refs/heads/ci-new-cluster'}} + if: ${{ github.event_name != 'pull_request' }} id: sync-internal-registry uses: the-actions-org/workflow-dispatch@v4 with: