From 399d353a7de4e8abd1afcac46485a881b5509a62 Mon Sep 17 00:00:00 2001 From: Lu Weizheng Date: Thu, 8 Aug 2024 23:39:00 +0800 Subject: [PATCH 01/14] support numpy 2.0 --- CI/conda-environment.yml | 2 +- CI/requirements-wheel.txt | 2 +- python/pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CI/conda-environment.yml b/CI/conda-environment.yml index 335c4b0a..f4b60116 100644 --- a/CI/conda-environment.yml +++ b/CI/conda-environment.yml @@ -2,7 +2,7 @@ name: xoscar-test channels: - defaults dependencies: - - numpy<2.0.0 + - numpy - cloudpickle - coverage - cython diff --git a/CI/requirements-wheel.txt b/CI/requirements-wheel.txt index 1e7e7967..7641aa6c 100644 --- a/CI/requirements-wheel.txt +++ b/CI/requirements-wheel.txt @@ -1,6 +1,6 @@ oldest-supported-numpy -numpy<2.0.0 +numpy packaging wheel diff --git a/python/pyproject.toml b/python/pyproject.toml index 17ac3eec..68d9e2a0 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -21,7 +21,7 @@ requires = [ "pandas==1.4.0; python_version>='3.10' and python_version<'3.11' and platform_machine=='arm64'", "pandas==1.5.1; python_version>='3.11' and python_version<'3.12'", "pandas>=2.1.1; python_version>'3.11'", - "numpy<2.0.0", + "numpy", "cython>=0.29.33", "requests>=2.4.0", "cloudpickle>=2.2.1; python_version>='3.11'", From 460e079bc5e6a0fb29fb0422c93fac253fc4e559 Mon Sep 17 00:00:00 2001 From: Lu Weizheng Date: Fri, 9 Aug 2024 00:03:43 +0800 Subject: [PATCH 02/14] support numpy 2.0 --- python/setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/setup.cfg b/python/setup.cfg index d2cbed03..0437cef6 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -24,7 +24,7 @@ zip_safe = False include_package_data = True packages = find: install_requires = - numpy>=1.14.0,<2.0.0 + numpy>=1.14.0 pandas>=1.0.0 scipy>=1.0.0; sys_platform!="win32" or python_version>="3.10" scipy>=1.0.0,<=1.9.1; sys_platform=="win32" and python_version<"3.10" From 997823f3770462ea0a965c9af3c4c2f4f77e193e Mon Sep 17 00:00:00 2001 From: Lu Weizheng Date: Wed, 4 Sep 2024 15:05:05 +0800 Subject: [PATCH 03/14] runner --- .github/workflows/python.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 77bb4bae..ca1ab242 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -65,7 +65,7 @@ jobs: find . -name "CMakeLists.txt" -not -path "*third_party/*" | xargs cmake-format -c .cmake-format.yaml --check build_test_job: - if: github.repository == 'xorbitsai/xoscar' + # if: github.repository == 'xorbitsai/xoscar' runs-on: ${{ matrix.os }} needs: lint env: From 50b530e820835cadff2947483fdd0aaf4429d0f8 Mon Sep 17 00:00:00 2001 From: Lu Weizheng Date: Tue, 24 Sep 2024 09:24:55 +0800 Subject: [PATCH 04/14] ser --- python/xoscar/serialization/core.pyx | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/python/xoscar/serialization/core.pyx b/python/xoscar/serialization/core.pyx index 745276f9..bf3bf569 100644 --- a/python/xoscar/serialization/core.pyx +++ b/python/xoscar/serialization/core.pyx @@ -55,7 +55,6 @@ from .pyfury import get_fury BUFFER_PICKLE_PROTOCOL = max(pickle.DEFAULT_PROTOCOL, 5) cdef bint HAS_PICKLE_BUFFER = pickle.HIGHEST_PROTOCOL >= 5 -cdef bint _PANDAS_HAS_MGR = hasattr(pd.Series([0]), "_mgr") cdef TypeDispatcher _serial_dispatcher = TypeDispatcher() cdef dict _deserializers = dict() @@ -260,16 +259,7 @@ def unpickle_buffers(list buffers): else: result = cloudpickle.loads(buffers[0], buffers=buffers[1:]) - # as pandas prior to 1.1.0 use _data instead of _mgr to hold BlockManager, - # deserializing from high versions may produce mal-functioned pandas objects, - # thus the patch is needed - if _PANDAS_HAS_MGR: - return result - else: # pragma: no cover - if hasattr(result, "_mgr") and isinstance(result, (pd.DataFrame, pd.Series)): - result._data = getattr(result, "_mgr") - delattr(result, "_mgr") - return result + return result cdef class PickleSerializer(Serializer): From ab373ab5f0914ca1cdf66488255871d44d8c17e0 Mon Sep 17 00:00:00 2001 From: Lu Weizheng Date: Tue, 24 Sep 2024 10:22:27 +0800 Subject: [PATCH 05/14] ser --- .github/workflows/python.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 1f5e22c6..9b8c26b8 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -125,6 +125,7 @@ jobs: if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest') && (matrix.python-version != '3.11') }} run: | conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py + pip install -U numpy - name: Install fury if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest') && (matrix.python-version == '3.9') }} From 39de3abbc9bf4d4a41753fb43ebd6dd406fd439a Mon Sep 17 00:00:00 2001 From: Lu Weizheng Date: Tue, 24 Sep 2024 15:00:11 +0800 Subject: [PATCH 06/14] ucx --- .github/workflows/python.yaml | 13 +++++++------ python/xoscar/backends/communication/ucx.py | 11 ++++++----- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 9b8c26b8..abae1a97 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -117,15 +117,16 @@ jobs: MODULE: ${{ matrix.module }} if: ${{ matrix.module != 'gpu' }} run: | - pip install numpy scipy cython coverage flaky + pip install numpy scipy cython coverage flaky ucxx-cu12 pip install -e ".[dev,extra]" + ucx_info -v working-directory: ./python - - name: Install ucx dependencies - if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest') && (matrix.python-version != '3.11') }} - run: | - conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py - pip install -U numpy + # - name: Install ucx dependencies + # if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest') && (matrix.python-version != '3.11') }} + # run: | + # conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py + # pip install -U numpy - name: Install fury if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest') && (matrix.python-version == '3.9') }} diff --git a/python/xoscar/backends/communication/ucx.py b/python/xoscar/backends/communication/ucx.py index 548bf6f8..d232fe1c 100644 --- a/python/xoscar/backends/communication/ucx.py +++ b/python/xoscar/backends/communication/ucx.py @@ -34,7 +34,7 @@ from .core import register_client, register_server from .errors import ChannelClosed -ucp = lazy_import("ucp") +ucp = lazy_import("ucxx") numba_cuda = lazy_import("numba.cuda") rmm = lazy_import("rmm") @@ -86,7 +86,7 @@ def _get_options(ucx_config: dict) -> Tuple[dict, dict]: tls += ",cuda_copy" if ucx_config.get("infiniband"): # pragma: no cover - tls = "rc," + tls + tls = "ib," + tls if ucx_config.get("nvlink"): # pragma: no cover tls += ",cuda_ipc" @@ -177,7 +177,8 @@ def init(ucx_config: dict): new_environ.update(envs) os.environ = new_environ # type: ignore try: - ucp.init(options=options, env_takes_precedence=True) + # let UCX determine the appropriate transports + ucp.init() finally: os.environ = original_environ @@ -313,7 +314,7 @@ async def send_buffers(self, buffers: list, meta: Optional[_MessageBase] = None) await self.ucp_endpoint.send(buf) for buffer in buffers: await self.ucp_endpoint.send(buffer) - except ucp.exceptions.UCXBaseException: # pragma: no cover + except ucp.exceptions.UCXError:: # pragma: no cover self.abort() raise ChannelClosed("While writing, the connection was closed") @@ -516,7 +517,7 @@ async def connect( try: ucp_endpoint = await ucp.create_endpoint(host, port) - except ucp.exceptions.UCXBaseException as e: # pragma: no cover + except ucp.exceptions.UCXError as e: # pragma: no cover raise ChannelClosed( f"Connection closed before handshake completed, " f"local address: {local_address}, dest address: {dest_address}" From 9e0afacd695cb9991756af05c62455691d4d0d79 Mon Sep 17 00:00:00 2001 From: Lu Weizheng Date: Tue, 24 Sep 2024 15:47:40 +0800 Subject: [PATCH 07/14] ucx --- python/xoscar/backends/communication/ucx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/xoscar/backends/communication/ucx.py b/python/xoscar/backends/communication/ucx.py index d232fe1c..26e40b94 100644 --- a/python/xoscar/backends/communication/ucx.py +++ b/python/xoscar/backends/communication/ucx.py @@ -314,7 +314,7 @@ async def send_buffers(self, buffers: list, meta: Optional[_MessageBase] = None) await self.ucp_endpoint.send(buf) for buffer in buffers: await self.ucp_endpoint.send(buffer) - except ucp.exceptions.UCXError:: # pragma: no cover + except ucp.exceptions.UCXError: # pragma: no cover self.abort() raise ChannelClosed("While writing, the connection was closed") From d4e1481cca0a761f591cc75bbff59057ba03e439 Mon Sep 17 00:00:00 2001 From: Lu Weizheng Date: Tue, 24 Sep 2024 15:56:58 +0800 Subject: [PATCH 08/14] ucx --- .github/workflows/python.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index abae1a97..0724efd1 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -117,16 +117,16 @@ jobs: MODULE: ${{ matrix.module }} if: ${{ matrix.module != 'gpu' }} run: | - pip install numpy scipy cython coverage flaky ucxx-cu12 + pip install numpy scipy cython coverage flaky pip install -e ".[dev,extra]" - ucx_info -v + # ucx_info -v working-directory: ./python - # - name: Install ucx dependencies - # if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest') && (matrix.python-version != '3.11') }} - # run: | - # conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py - # pip install -U numpy + - name: Install ucx dependencies + if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest')}} + run: | + # conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py + pip install ucxx-cu12 - name: Install fury if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest') && (matrix.python-version == '3.9') }} From c0c823f049838cfc9f704669d21da9195f796819 Mon Sep 17 00:00:00 2001 From: Lu Weizheng Date: Tue, 24 Sep 2024 16:22:31 +0800 Subject: [PATCH 09/14] ucx --- .github/workflows/python.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 0724efd1..9215e8ac 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -123,8 +123,9 @@ jobs: working-directory: ./python - name: Install ucx dependencies - if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest')}} + if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest') && (matrix.python-version >= '3.9')}} run: | + # ucx-py move to ucxx and ucxx-cu12 can be run on CPU # conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py pip install ucxx-cu12 From ca0cd8d79af4919ca0385ffdc6c762d7c58be734 Mon Sep 17 00:00:00 2001 From: Lu Weizheng Date: Tue, 24 Sep 2024 16:31:09 +0800 Subject: [PATCH 10/14] ucx --- .github/workflows/python.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 9215e8ac..2dfebd5d 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -65,7 +65,7 @@ jobs: find . -name "CMakeLists.txt" -not -path "*third_party/*" | xargs cmake-format -c .cmake-format.yaml --check build_test_job: - # if: github.repository == 'xorbitsai/xoscar' + if: github.repository == 'xorbitsai/xoscar' runs-on: ${{ matrix.os }} needs: lint env: From a98b8deba6c7a1919892053b80b02f2bc9075c25 Mon Sep 17 00:00:00 2001 From: Lu Weizheng Date: Tue, 24 Sep 2024 16:43:54 +0800 Subject: [PATCH 11/14] ucx --- .github/workflows/python.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 2dfebd5d..475c8741 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -119,7 +119,6 @@ jobs: run: | pip install numpy scipy cython coverage flaky pip install -e ".[dev,extra]" - # ucx_info -v working-directory: ./python - name: Install ucx dependencies From 89057494af98341083c8746056ce06523136c259 Mon Sep 17 00:00:00 2001 From: Lu Weizheng Date: Tue, 24 Sep 2024 16:54:15 +0800 Subject: [PATCH 12/14] glibcxx --- .github/workflows/python.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 475c8741..00516035 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -106,9 +106,9 @@ jobs: python-version: ${{ matrix.python-version }} activate-environment: ${{ env.CONDA_ENV }} - # Fix "version `GLIBCXX_3.4.30' not found (required by xoscar_store.cpython-311-x86_64-linux-gnu.so)" issue in Python 3.11 + # Fix "version `GLIBCXX_3.4.30' not found (required by xoscar_store.cpython-311-x86_64-linux-gnu.so)" issue - name: Install libstdcxx-ng for Python 3.11 - if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest') && (matrix.python-version == '3.11') }} + if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest') }} run: | conda install -c conda-forge libstdcxx-ng From 3fe04be9cc8eda42354889290d2fb6bb7fe51802 Mon Sep 17 00:00:00 2001 From: Lu Weizheng Date: Tue, 24 Sep 2024 17:09:31 +0800 Subject: [PATCH 13/14] gpu ucx --- .github/workflows/python.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 00516035..abc06c5c 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -107,7 +107,7 @@ jobs: activate-environment: ${{ env.CONDA_ENV }} # Fix "version `GLIBCXX_3.4.30' not found (required by xoscar_store.cpython-311-x86_64-linux-gnu.so)" issue - - name: Install libstdcxx-ng for Python 3.11 + - name: Install libstdcxx-ng if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest') }} run: | conda install -c conda-forge libstdcxx-ng @@ -122,7 +122,7 @@ jobs: working-directory: ./python - name: Install ucx dependencies - if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest') && (matrix.python-version >= '3.9')}} + if: ${{ (matrix.os == 'ubuntu-latest') && (matrix.python-version >= '3.9')}} run: | # ucx-py move to ucxx and ucxx-cu12 can be run on CPU # conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py From bd1c8960c69d0eb9f50050b459608d842262801a Mon Sep 17 00:00:00 2001 From: Lu Weizheng Date: Tue, 24 Sep 2024 17:14:22 +0800 Subject: [PATCH 14/14] gpu ucx --- .github/workflows/python.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index abc06c5c..c3e9649b 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -139,7 +139,7 @@ jobs: source activate ${{ env.CONDA_ENV }} conda install -y conda-forge::nccl=2.22.3 pip install --extra-index-url=https://pypi.nvidia.com cudf-cu12 - pip install ucx-py-cu12 cloudpickle psutil tblib uvloop packaging "numpy<2.0.0" scipy cython coverage flaky + pip install ucxx-cu12 cloudpickle psutil tblib uvloop packaging "numpy<2.0.0" scipy cython coverage flaky python setup.py clean --all pip install -e ./ working-directory: ./python