Build docker images (scheduled) #1011
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build docker images (scheduled) | |
| on: | |
| push: | |
| branches: | |
| - build_ci_docker_image* | |
| repository_dispatch: | |
| workflow_call: | |
| inputs: | |
| image_postfix: | |
| required: true | |
| type: string | |
| schedule: | |
| - cron: "17 0 * * *" | |
| concurrency: | |
| group: docker-images-builds | |
| cancel-in-progress: false | |
| jobs: | |
| latest-docker: | |
| name: "Latest PyTorch + TensorFlow [dev]" | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Cleanup disk | |
| run: | | |
| sudo ls -l /usr/local/lib/ | |
| sudo ls -l /usr/share/ | |
| sudo du -sh /usr/local/lib/ | |
| sudo du -sh /usr/share/ | |
| sudo rm -rf /usr/local/lib/android | |
| sudo rm -rf /usr/share/dotnet | |
| sudo du -sh /usr/local/lib/ | |
| sudo du -sh /usr/share/ | |
| - | |
| name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - | |
| name: Check out code | |
| uses: actions/checkout@v3 | |
| - | |
| name: Login to DockerHub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_PASSWORD }} | |
| - | |
| name: Build and push | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: ./docker/transformers-all-latest-gpu | |
| build-args: | | |
| REF=main | |
| push: true | |
| tags: huggingface/transformers-all-latest-gpu${{ inputs.image_postfix }} | |
| # Push CI images still need to be re-built daily | |
| - | |
| name: Build and push (for Push CI) in a daily basis | |
| # This condition allows `schedule` events, or `push` events that trigger this workflow NOT via `workflow_call`. | |
| # The later case is useful for manual image building for debugging purpose. Use another tag in this case! | |
| if: inputs.image_postfix != '-push-ci' | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: ./docker/transformers-all-latest-gpu | |
| build-args: | | |
| REF=main | |
| push: true | |
| tags: huggingface/transformers-all-latest-gpu-push-ci | |
| latest-torch-deepspeed-docker: | |
| name: "Latest PyTorch + DeepSpeed" | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Cleanup disk | |
| run: | | |
| sudo ls -l /usr/local/lib/ | |
| sudo ls -l /usr/share/ | |
| sudo du -sh /usr/local/lib/ | |
| sudo du -sh /usr/share/ | |
| sudo rm -rf /usr/local/lib/android | |
| sudo rm -rf /usr/share/dotnet | |
| sudo du -sh /usr/local/lib/ | |
| sudo du -sh /usr/share/ | |
| - | |
| name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - | |
| name: Check out code | |
| uses: actions/checkout@v3 | |
| - | |
| name: Login to DockerHub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_PASSWORD }} | |
| - | |
| name: Build and push | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: ./docker/transformers-pytorch-deepspeed-latest-gpu | |
| build-args: | | |
| REF=main | |
| push: true | |
| tags: huggingface/transformers-pytorch-deepspeed-latest-gpu${{ inputs.image_postfix }} | |
| # Can't build 2 images in a single job `latest-torch-deepspeed-docker` (for `nvcr.io/nvidia`) | |
| latest-torch-deepspeed-docker-for-push-ci-daily-build: | |
| name: "Latest PyTorch + DeepSpeed (Push CI - Daily Build)" | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Cleanup disk | |
| run: | | |
| sudo ls -l /usr/local/lib/ | |
| sudo ls -l /usr/share/ | |
| sudo du -sh /usr/local/lib/ | |
| sudo du -sh /usr/share/ | |
| sudo rm -rf /usr/local/lib/android | |
| sudo rm -rf /usr/share/dotnet | |
| sudo du -sh /usr/local/lib/ | |
| sudo du -sh /usr/share/ | |
| - | |
| name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - | |
| name: Check out code | |
| uses: actions/checkout@v3 | |
| - | |
| name: Login to DockerHub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_PASSWORD }} | |
| # Push CI images still need to be re-built daily | |
| - | |
| name: Build and push (for Push CI) in a daily basis | |
| # This condition allows `schedule` events, or `push` events that trigger this workflow NOT via `workflow_call`. | |
| # The later case is useful for manual image building for debugging purpose. Use another tag in this case! | |
| if: inputs.image_postfix != '-push-ci' | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: ./docker/transformers-pytorch-deepspeed-latest-gpu | |
| build-args: | | |
| REF=main | |
| push: true | |
| tags: huggingface/transformers-pytorch-deepspeed-latest-gpu-push-ci | |
| doc-builder: | |
| name: "Doc builder" | |
| # Push CI doesn't need this image | |
| if: inputs.image_postfix != '-push-ci' | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - | |
| name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - | |
| name: Check out code | |
| uses: actions/checkout@v3 | |
| - | |
| name: Login to DockerHub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_PASSWORD }} | |
| - | |
| name: Build and push | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: ./docker/transformers-doc-builder | |
| push: true | |
| tags: huggingface/transformers-doc-builder | |
| latest-pytorch: | |
| name: "Latest PyTorch [dev]" | |
| # Push CI doesn't need this image | |
| if: inputs.image_postfix != '-push-ci' | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Cleanup disk | |
| run: | | |
| sudo ls -l /usr/local/lib/ | |
| sudo ls -l /usr/share/ | |
| sudo du -sh /usr/local/lib/ | |
| sudo du -sh /usr/share/ | |
| sudo rm -rf /usr/local/lib/android | |
| sudo rm -rf /usr/share/dotnet | |
| sudo du -sh /usr/local/lib/ | |
| sudo du -sh /usr/share/ | |
| - | |
| name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - | |
| name: Check out code | |
| uses: actions/checkout@v3 | |
| - | |
| name: Login to DockerHub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_PASSWORD }} | |
| - | |
| name: Build and push | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: ./docker/transformers-pytorch-gpu | |
| build-args: | | |
| REF=main | |
| push: true | |
| tags: huggingface/transformers-pytorch-gpu | |
| # Need to be fixed with the help from Guillaume. | |
| # latest-pytorch-amd: | |
| # name: "Latest PyTorch (AMD) [dev]" | |
| # runs-on: [self-hosted, docker-gpu, amd-gpu, single-gpu, mi210] | |
| # steps: | |
| # - name: Set up Docker Buildx | |
| # uses: docker/setup-buildx-action@v3 | |
| # - name: Check out code | |
| # uses: actions/checkout@v3 | |
| # - name: Login to DockerHub | |
| # uses: docker/login-action@v3 | |
| # with: | |
| # username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| # password: ${{ secrets.DOCKERHUB_PASSWORD }} | |
| # - name: Build and push | |
| # uses: docker/build-push-action@v5 | |
| # with: | |
| # context: ./docker/transformers-pytorch-amd-gpu | |
| # build-args: | | |
| # REF=main | |
| # push: true | |
| # tags: huggingface/transformers-pytorch-amd-gpu${{ inputs.image_postfix }} | |
| # # Push CI images still need to be re-built daily | |
| # - | |
| # name: Build and push (for Push CI) in a daily basis | |
| # # This condition allows `schedule` events, or `push` events that trigger this workflow NOT via `workflow_call`. | |
| # # The later case is useful for manual image building for debugging purpose. Use another tag in this case! | |
| # if: inputs.image_postfix != '-push-ci' | |
| # uses: docker/build-push-action@v5 | |
| # with: | |
| # context: ./docker/transformers-pytorch-amd-gpu | |
| # build-args: | | |
| # REF=main | |
| # push: true | |
| # tags: huggingface/transformers-pytorch-amd-gpu-push-ci | |
| latest-tensorflow: | |
| name: "Latest TensorFlow [dev]" | |
| # Push CI doesn't need this image | |
| if: inputs.image_postfix != '-push-ci' | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - | |
| name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - | |
| name: Check out code | |
| uses: actions/checkout@v3 | |
| - | |
| name: Login to DockerHub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_PASSWORD }} | |
| - | |
| name: Build and push | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: ./docker/transformers-tensorflow-gpu | |
| build-args: | | |
| REF=main | |
| push: true | |
| tags: huggingface/transformers-tensorflow-gpu | |
| # latest-pytorch-deepspeed-amd: | |
| # name: "PyTorch + DeepSpeed (AMD) [dev]" | |
| # runs-on: [self-hosted, docker-gpu, amd-gpu, single-gpu, mi210] | |
| # steps: | |
| # - name: Set up Docker Buildx | |
| # uses: docker/setup-buildx-action@v3 | |
| # - name: Check out code | |
| # uses: actions/checkout@v3 | |
| # - name: Login to DockerHub | |
| # uses: docker/login-action@v3 | |
| # with: | |
| # username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| # password: ${{ secrets.DOCKERHUB_PASSWORD }} | |
| # - name: Build and push | |
| # uses: docker/build-push-action@v5 | |
| # with: | |
| # context: ./docker/transformers-pytorch-deepspeed-amd-gpu | |
| # build-args: | | |
| # REF=main | |
| # push: true | |
| # tags: huggingface/transformers-pytorch-deepspeed-amd-gpu${{ inputs.image_postfix }} | |
| # # Push CI images still need to be re-built daily | |
| # - | |
| # name: Build and push (for Push CI) in a daily basis | |
| # # This condition allows `schedule` events, or `push` events that trigger this workflow NOT via `workflow_call`. | |
| # # The later case is useful for manual image building for debugging purpose. Use another tag in this case! | |
| # if: inputs.image_postfix != '-push-ci' | |
| # uses: docker/build-push-action@v5 | |
| # with: | |
| # context: ./docker/transformers-pytorch-deepspeed-amd-gpu | |
| # build-args: | | |
| # REF=main | |
| # push: true | |
| # tags: huggingface/transformers-pytorch-deepspeed-amd-gpu-push-ci |