FIX-#407: Make data put into MPI object store immutable (#409) #834
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| pull_request: | |
| paths: | |
| - .github/workflows/** | |
| - unidist/** | |
| - environment_linux.yml | |
| - environment_win.yml | |
| - requirements.txt | |
| - setup.cfg | |
| - setup.py | |
| - versioneer.py | |
| push: | |
| concurrency: | |
| # Cancel other jobs in the same branch. We don't care whether CI passes | |
| # on old commits. | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| lint-black: | |
| name: lint (black) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 1 | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: "3.8.x" | |
| architecture: "x64" | |
| - run: pip install black | |
| - run: black --check --diff . | |
| lint-flake8: | |
| name: lint (flake8) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 1 | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: "3.8.x" | |
| architecture: "x64" | |
| - run: pip install flake8 flake8-print | |
| - run: flake8 --enable=T . | |
| test-all: | |
| needs: [lint-black, lint-flake8] | |
| strategy: | |
| matrix: | |
| os: | |
| - ubuntu | |
| - windows | |
| python-version: ["3.8", "3.9"] | |
| backend: ["ray", "mpi", "dask", "pymp", "pyseq"] | |
| runs-on: ${{ matrix.os }}-latest | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| env: | |
| UNIDIST_BACKEND: ${{matrix.backend}} | |
| name: test-${{ matrix.os }} (backend ${{matrix.backend}}, python ${{matrix.python-version}}) | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 2 | |
| - uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| activate-environment: unidist | |
| environment-file: ${{ matrix.os == 'ubuntu' && 'environment_linux.yml' || matrix.os == 'windows' && 'environment_win.yml' }} | |
| python-version: ${{matrix.python-version}} | |
| channel-priority: strict | |
| # we set use-only-tar-bz2 to false in order for conda to properly find new packages to be installed | |
| # for more info see https://github.com/conda-incubator/setup-miniconda/issues/264 | |
| use-only-tar-bz2: false | |
| # Build unidist to make C++ extensions available | |
| - name: Build unidist | |
| run: pip install -e . | |
| - name: Conda environment | |
| run: | | |
| conda info | |
| conda list | |
| # We disable actor tests for Ray for now because | |
| # they hang in CI for some reason but pass locally. | |
| # TODO: Find a solution for https://github.com/modin-project/unidist/issues/322 | |
| - run: python -m pytest unidist/test/test_actor.py | |
| if: matrix.backend != 'mpi' && matrix.backend != 'ray' | |
| - run: python -m pytest unidist/test/test_async_actor.py | |
| if: matrix.backend != 'mpi' && matrix.backend != 'ray' | |
| - run: python -m pytest unidist/test/test_task.py | |
| if: matrix.backend != 'mpi' | |
| - run: python -m pytest unidist/test/test_general.py | |
| if: matrix.backend != 'mpi' | |
| # When using a directory to run with mpiexec | |
| # MPI gets hung after executing tests | |
| # so we run the test files one by one. | |
| # We test enabled and disabled shared object store on linux, and | |
| # disabled shared object store on Windows as | |
| # msmpi doesn't support the shared memory feature. | |
| - run: UNIDIST_MPI_SHARED_OBJECT_STORE=True mpiexec -n 1 --oversubscribe python -m pytest unidist/test/test_actor.py | |
| if: matrix.backend == 'mpi' && matrix.os == 'ubuntu' | |
| - run: UNIDIST_MPI_SHARED_OBJECT_STORE=False mpiexec -n 1 python -m pytest unidist/test/test_actor.py | |
| if: matrix.backend == 'mpi' && matrix.os == 'windows' | |
| - run: UNIDIST_MPI_SHARED_OBJECT_STORE=True mpiexec -n 1 --oversubscribe python -m pytest unidist/test/test_async_actor.py | |
| if: matrix.backend == 'mpi' && matrix.os == 'ubuntu' | |
| - run: UNIDIST_MPI_SHARED_OBJECT_STORE=False mpiexec -n 1 python -m pytest unidist/test/test_async_actor.py | |
| if: matrix.backend == 'mpi' && matrix.os == 'windows' | |
| - run: UNIDIST_MPI_SHARED_OBJECT_STORE=True mpiexec -n 1 --oversubscribe python -m pytest unidist/test/test_task.py | |
| if: matrix.backend == 'mpi' && matrix.os == 'ubuntu' | |
| - run: UNIDIST_MPI_SHARED_OBJECT_STORE=False mpiexec -n 1 python -m pytest unidist/test/test_task.py | |
| if: matrix.backend == 'mpi' && matrix.os == 'windows' | |
| - run: UNIDIST_MPI_SHARED_OBJECT_STORE=True mpiexec -n 1 --oversubscribe python -m pytest unidist/test/test_general.py | |
| if: matrix.backend == 'mpi' && matrix.os == 'ubuntu' | |
| - run: UNIDIST_MPI_SHARED_OBJECT_STORE=False mpiexec -n 1 python -m pytest unidist/test/test_general.py | |
| if: matrix.backend == 'mpi' && matrix.os == 'windows' |