FEAT-#285: Implement shared object storage for MPI backend #657
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| pull_request: | |
| paths: | |
| - .github/workflows/** | |
| - unidist/** | |
| - environment.yml | |
| - requirements.txt | |
| - setup.cfg | |
| - setup.py | |
| - versioneer.py | |
| push: | |
| concurrency: | |
| # Cancel other jobs in the same branch. We don't care whether CI passes | |
| # on old commits. | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| lint-black: | |
| name: lint (black) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 1 | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: "3.8.x" | |
| architecture: "x64" | |
| - run: pip install black | |
| - run: black --check --diff . | |
| lint-flake8: | |
| name: lint (flake8) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 1 | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: "3.8.x" | |
| architecture: "x64" | |
| - run: pip install flake8 flake8-print | |
| - run: flake8 --enable=T . | |
| test-all: | |
| needs: [lint-black, lint-flake8] | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| strategy: | |
| matrix: | |
| os: | |
| - ubuntu | |
| - windows | |
| python-version: ["3.8", "3.9"] | |
| backend: ["ray", "mpi", "dask", "pymp", "pyseq"] | |
| env: | |
| UNIDIST_BACKEND: ${{matrix.backend}} | |
| name: test-${{ matrix.os }} (backend ${{matrix.backend}}, python ${{matrix.python-version}}) | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 2 | |
| - uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| activate-environment: unidist | |
| environment-file: environment.yml | |
| python-version: ${{matrix.python-version}} | |
| channel-priority: strict | |
| # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed | |
| # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264 | |
| use-only-tar-bz2: false | |
| - name: Build unidist | |
| run: | | |
| python setup.py build_ext --inplace | |
| pip install -e . | |
| - name: Conda environment | |
| run: | | |
| conda info | |
| conda list | |
| # We disable actor tests for Ray for now because | |
| # they hang in CI for some reason but pass locally. | |
| # TODO: Find a solution for https://github.com/modin-project/unidist/issues/322 | |
| - run: python -m pytest unidist/test/test_actor.py | |
| if: matrix.backend != 'mpi' && matrix.backend != 'ray' | |
| - run: python -m pytest unidist/test/test_async_actor.py | |
| if: matrix.backend != 'mpi' && matrix.backend != 'ray' | |
| - run: python -m pytest unidist/test/test_task.py | |
| if: matrix.backend != 'mpi' | |
| - run: python -m pytest unidist/test/test_general.py | |
| if: matrix.backend != 'mpi' | |
| # when using a directory to run with mpiexec MPI gets hung after executing tests | |
| # so we run the test files one by one | |
| - run: | | |
| UNIDIST_MPI_SHARED_OBJECT_STORE=True mpiexec -n 1 --oversubscribe python -m pytest unidist/test/test_actor.py | |
| UNIDIST_MPI_SHARED_OBJECT_STORE=False mpiexec -n 1 --oversubscribe python -m pytest unidist/test/test_actor.py | |
| if: matrix.backend == 'mpi' | |
| - run: | | |
| UNIDIST_MPI_SHARED_OBJECT_STORE=True mpiexec -n 1 --oversubscribe python -m pytest unidist/test/test_async_actor.py | |
| UNIDIST_MPI_SHARED_OBJECT_STORE=False mpiexec -n 1 --oversubscribe python -m pytest unidist/test/test_async_actor.py | |
| if: matrix.backend == 'mpi' | |
| - run: | | |
| UNIDIST_MPI_SHARED_OBJECT_STORE=True mpiexec -n 1 --oversubscribe python -m pytest unidist/test/test_task.py | |
| UNIDIST_MPI_SHARED_OBJECT_STORE=False mpiexec -n 1 --oversubscribe python -m pytest unidist/test/test_task.py | |
| if: matrix.backend == 'mpi' | |
| - run: | | |
| UNIDIST_MPI_SHARED_OBJECT_STORE=True mpiexec -n 1 --oversubscribe python -m pytest unidist/test/test_general.py | |
| UNIDIST_MPI_SHARED_OBJECT_STORE=False mpiexec -n 1 --oversubscribe python -m pytest unidist/test/test_general.py | |
| if: matrix.backend == 'mpi' |