Nightly Build #614
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly Build | |
| on: | |
| schedule: | |
| - cron: '0 16 * * *' | |
| workflow_dispatch: | |
| jobs: | |
| build-and-test: | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| modules: | |
| - dl-on-flink-framework | |
| - dl-on-flink-operator | |
| - dl-on-flink-tensorflow-common | |
| - dl-on-flink-tensorflow | |
| - dl-on-flink-tensorflow-2.x | |
| - dl-on-flink-pytorch | |
| - dl-on-flink-lib | |
| - dl-on-flink-examples | |
| - dl-on-flink-examples/dl-on-flink-examples-pytorch | |
| - dl-on-flink-examples/dl-on-flink-examples-tensorflow | |
| - dl-on-flink-examples/dl-on-flink-examples-tensorflow-2.x | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: true | |
| - name: Set up JDK 8 | |
| uses: actions/setup-java@v2 | |
| with: | |
| java-version: '8' | |
| distribution: 'adopt' | |
| - name: Set up python 3.7 | |
| uses: actions/setup-python@v2 | |
| with: | |
| # Semantic version range syntax or exact version of a Python version | |
| python-version: '3.7' | |
| # Optional - x64 or x86 architecture, defaults to x64 | |
| architecture: 'x64' | |
| - name: Install Python dependencies | |
| run: | | |
| pip install -U --user wheel | |
| - name: Cache Maven packages | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.m2 | |
| key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: ${{ runner.os }}-m2 | |
| - name: build | |
| run: | | |
| mvn -B install -pl ${{ matrix.modules }} -am -DskipTests -Dfast -Pinstall-py-dep | |
| - name: test | |
| env: | |
| TF_ON_FLINK_IP: 127.0.0.1 | |
| run: | | |
| mvn -B test -pl ${{ matrix.modules }} | |
| build-and-upload-bin: | |
| needs: build-and-test | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: true | |
| - name: Cache Maven packages | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.m2 | |
| key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: ${{ runner.os }}-m2 | |
| - name: Set up python 3.7 | |
| uses: actions/setup-python@v2 | |
| with: | |
| # Semantic version range syntax or exact version of a Python version | |
| python-version: '3.7' | |
| # Optional - x64 or x86 architecture, defaults to x64 | |
| architecture: 'x64' | |
| - name: Set up JDK 8 | |
| uses: actions/setup-java@v2 | |
| with: | |
| java-version: '8' | |
| distribution: 'adopt' | |
| - name: Build bin | |
| run: mvn -B clean package -DskipTests | |
| - uses: actions/upload-artifact@v2 | |
| with: | |
| name: dl-on-flink-dist | |
| path: dl-on-flink-dist/target/dl-on-flink-dist-*-bin.tgz | |
| build-and-test-python: | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| modules: | |
| - dl-on-flink-framework | |
| - dl-on-flink-tensorflow | |
| - dl-on-flink-tensorflow-2.x | |
| - dl-on-flink-pytorch | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: true | |
| - name: Set up JDK 8 | |
| uses: actions/setup-java@v2 | |
| with: | |
| java-version: '8' | |
| distribution: 'adopt' | |
| - name: Set up python 3.7 | |
| uses: actions/setup-python@v2 | |
| with: | |
| # Semantic version range syntax or exact version of a Python version | |
| python-version: '3.7' | |
| # Optional - x64 or x86 architecture, defaults to x64 | |
| architecture: 'x64' | |
| - name: Install Python dependencies | |
| run: | | |
| pip install -U --user wheel | |
| - name: Cache Maven packages | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.m2 | |
| key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: ${{ runner.os }}-m2 | |
| - name: build | |
| run: | | |
| mvn -B install -pl ${{ matrix.modules }} -am -DskipTests -Dfast -Pinstall-py-dep | |
| - name: test | |
| run: | | |
| cd ${{ matrix.modules }}/python | |
| pip install -e . | |
| python -m unittest discover -v . | |
| build-pytorch-nightly-wheel: | |
| needs: build-and-test-python | |
| name: Build ${{ matrix.package }} wheels on ubuntu-20.04 | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| package: | |
| - dl-on-flink-pytorch | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: true | |
| - name: Set up python 3.7 | |
| uses: actions/setup-python@v2 | |
| with: | |
| # Semantic version range syntax or exact version of a Python version | |
| python-version: '3.7' | |
| # Optional - x64 or x86 architecture, defaults to x64 | |
| architecture: 'x64' | |
| - name: Build python${{ matrix.python-version }} ${{ matrix.package }} wheels | |
| env: | |
| NIGHTLY_WHEEL: true | |
| run: | | |
| pip install wheel | |
| cd ${{ matrix.package }}/python | |
| python setup.py bdist_wheel | |
| - uses: actions/upload-artifact@v2 | |
| with: | |
| name: dl-on-flink-wheels | |
| path: ./dl-on-flink-pytorch/python/dist/*.whl | |
| build-linux-nightly-wheel: | |
| needs: build-and-test-python | |
| name: Build python${{ matrix.python-version }} ${{ matrix.package }} wheels on ubuntu-20.04 | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: [ 3.7, 3.8 ] | |
| package: | |
| - dl-on-flink-framework | |
| - dl-on-flink-tensorflow | |
| - dl-on-flink-tensorflow-2.x | |
| exclude: | |
| - package: dl-on-flink-tensorflow | |
| python-version: 3.8 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: true | |
| - name: Build python${{ matrix.python-version }} ${{ matrix.package }} wheels | |
| env: | |
| NIGHTLY_WHEEL: true | |
| run: | | |
| docker run -i --rm -v $PWD:/v -w /v -e 'NIGHTLY_WHEEL=true' --net=host quay.io/pypa/manylinux2010_x86_64 \ | |
| bash -c 'cd /v/${{ matrix.package }}/python \ | |
| && python${{ matrix.python-version }} setup.py bdist_wheel \ | |
| && for f in dist/*.whl; do echo repairing $f && bash -x -e /v/tools/auditwheel repair --plat manylinux2010_x86_64 -w /v/wheelhouse/ $f; done' | |
| - uses: actions/upload-artifact@v2 | |
| with: | |
| name: dl-on-flink-wheels | |
| path: ./wheelhouse/*.whl | |
| build-macos-nightly-wheel: | |
| needs: build-and-test-python | |
| name: Build python${{ matrix.python-version }} ${{ matrix.package }} wheels on macOS-10.15 | |
| runs-on: macos-10.15 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: [ 3.7, 3.8 ] | |
| package: | |
| - dl-on-flink-framework | |
| - dl-on-flink-tensorflow | |
| - dl-on-flink-tensorflow-2.x | |
| exclude: | |
| - package: dl-on-flink-tensorflow | |
| python-version: 3.8 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: true | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Build python${{ matrix.python-version }} ${{ matrix.package }} wheels on macOS-10.15 | |
| env: | |
| NIGHTLY_WHEEL: true | |
| run: | | |
| pip install wheel auditwheel | |
| export ROOT_DIR=$PWD | |
| mkdir $ROOT_DIR/wheelhouse | |
| cd ${{ matrix.package }}/python | |
| python setup.py bdist_wheel | |
| cp dist/* $ROOT_DIR/wheelhouse | |
| - uses: actions/upload-artifact@v2 | |
| with: | |
| name: dl-on-flink-wheels | |
| path: ./wheelhouse/*.whl | |
| test-wheel: | |
| name: Test python${{ matrix.python-version }} ${{ matrix.package }} wheels on ${{ matrix.os }} | |
| needs: [ build-linux-nightly-wheel, build-macos-nightly-wheel, build-pytorch-nightly-wheel ] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ macOS-10.15, ubuntu-20.04 ] | |
| python-version: [ 3.7, 3.8 ] | |
| package: | |
| - dl-on-flink-framework | |
| - dl-on-flink-tensorflow | |
| - dl-on-flink-tensorflow-2.x | |
| - dl-on-flink-pytorch | |
| exclude: | |
| - package: dl-on-flink-tensorflow | |
| python-version: 3.8 | |
| - package: dl-on-flink-tensorflow-2.x | |
| python-version: 3.6 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: true | |
| - name: Set up JDK 8 | |
| uses: actions/setup-java@v2 | |
| with: | |
| java-version: '8' | |
| distribution: 'adopt' | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - uses: actions/download-artifact@v2 | |
| with: | |
| name: dl-on-flink-wheels | |
| path: ./wheelhouse | |
| - name: build | |
| run: | | |
| mvn -B install -pl ${{ matrix.package }} -am -DskipTests | |
| - name: test | |
| env: | |
| TF_ON_FLINK_IP: 127.0.0.1 | |
| run: | | |
| ls $PWD/wheelhouse | |
| pip install --pre ${{ matrix.package }} -f file://${PWD}/wheelhouse | |
| mvn -B test -pl ${{ matrix.package }} | |
| run-example: | |
| needs: [ build-and-upload-bin, build-linux-nightly-wheel, build-macos-nightly-wheel, build-pytorch-nightly-wheel ] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ macos-10.15, ubuntu-18.04, ubuntu-20.04 ] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: true | |
| - name: Set up JDK 8 | |
| uses: actions/setup-java@v2 | |
| with: | |
| java-version: '8' | |
| distribution: 'adopt' | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.7 | |
| - uses: actions/download-artifact@v2 | |
| with: | |
| name: dl-on-flink-wheels | |
| path: ./wheelhouse | |
| - uses: actions/download-artifact@v2 | |
| with: | |
| name: dl-on-flink-dist | |
| - name: Install coreutil on Mac | |
| if: ${{ matrix.os == 'macos-10.15' }} | |
| run: brew install coreutils | |
| - name: Run example on ${{ matrix.os }} | |
| run: | | |
| bash tools/run_example.sh dl-on-flink-*-bin.tgz wheelhouse | |
| - uses: actions/upload-artifact@v2 | |
| if: always() | |
| with: | |
| name: e2e-example-logs | |
| path: ./flink-*/log | |
| deploy-snapshot: | |
| if: github.repository == 'flink-extended/dl-on-flink' | |
| needs: [ run-example, test-wheel ] | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: true | |
| - name: Cache Maven packages | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.m2 | |
| key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: ${{ runner.os }}-m2 | |
| - name: Set up python 3.7 | |
| uses: actions/setup-python@v2 | |
| with: | |
| # Semantic version range syntax or exact version of a Python version | |
| python-version: '3.7' | |
| # Optional - x64 or x86 architecture, defaults to x64 | |
| architecture: 'x64' | |
| - name: Install Python dependencies | |
| run: | | |
| pip install -U --user wheel | |
| - name: Set up JDK 8 | |
| uses: actions/setup-java@v2 | |
| with: | |
| java-version: '8' | |
| distribution: 'adopt' | |
| server-id: oss_id | |
| server-username: MAVEN_USERNAME | |
| server-password: MAVEN_PASSWORD | |
| gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} | |
| gpg-passphrase: MAVEN_GPG_PASSPHRASE | |
| - name: Publish snapshot | |
| run: mvn -B clean deploy -DskipTests -Prelease | |
| env: | |
| MAVEN_USERNAME: ${{ secrets.MAVEN_USERNAME }} | |
| MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }} | |
| MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} | |
| - uses: actions/upload-artifact@v2 | |
| with: | |
| name: dl-on-flink-dist | |
| path: dl-on-flink-dist/target/dl-on-flink-dist-*-bin.tgz |