From 31304b8ec6afebbf47cce4155ff42b07c77a837a Mon Sep 17 00:00:00 2001 From: theo-barfoot Date: Fri, 18 Oct 2024 12:58:04 +0000 Subject: [PATCH 1/8] dev container env --- .devcontainer/.devcontainer/Dockerfile | 44 ++++++++++++++++ .devcontainer/.devcontainer/devcontainer.json | 51 +++++++++++++++++++ .devcontainer/.devcontainer/environment.yml | 19 +++++++ .devcontainer/.devcontainer/noop.txt | 0 .pre-commit-config.yaml | 6 +-- pyproject.toml | 8 ++- pytest.ini | 2 + 7 files changed, 125 insertions(+), 5 deletions(-) create mode 100644 .devcontainer/.devcontainer/Dockerfile create mode 100644 .devcontainer/.devcontainer/devcontainer.json create mode 100644 .devcontainer/.devcontainer/environment.yml create mode 100644 .devcontainer/.devcontainer/noop.txt create mode 100644 pytest.ini diff --git a/.devcontainer/.devcontainer/Dockerfile b/.devcontainer/.devcontainer/Dockerfile new file mode 100644 index 0000000..519dc10 --- /dev/null +++ b/.devcontainer/.devcontainer/Dockerfile @@ -0,0 +1,44 @@ +# Stage 1: NVIDIA CUDA Image +ARG CUDA_VERSION=12.1.0 +FROM nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu22.04 AS cuda-base + +# Stage 2: Miniconda setup from configuration +FROM continuumio/miniconda3 AS miniconda-stage + +# Stage 3: Final image combining CUDA and Miniconda +FROM mcr.microsoft.com/devcontainers/base:ubuntu-22.04 + +# Copy from CUDA base +COPY --from=cuda-base /usr/local/cuda /usr/local/cuda + +# Copy Miniconda from the Miniconda stage +COPY --from=miniconda-stage /opt/conda /opt/conda + +# Set environment variables for Miniconda +ENV PATH /opt/conda/bin:$PATH +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 + +# Install Python 3.10 +ARG PYTHON_VERSION=3.11 +RUN conda install python=${PYTHON_VERSION} + +# Arguments for PyTorch and CUDA Toolkit versions +ARG PYTORCH_VERSION=2.3.1 +ARG CUDATOOLKIT_VERSION=12.1 + +# Install PyTorch and other dependencies +RUN conda install pytorch=${PYTORCH_VERSION} pytorch-cuda=${CUDATOOLKIT_VERSION} -c pytorch -c nvidia + +# Handle environment.yml if it exists +COPY environment.yml* noop.txt /tmp/conda-tmp/ +RUN if [ -f "/tmp/conda-tmp/environment.yml" ]; then \ + /opt/conda/bin/conda env update -n base -f /tmp/conda-tmp/environment.yml; \ + fi \ + && rm -rf /tmp/conda-tmp + +# Append Miniconda to PATH in .bashrc for interactive shells +RUN echo ". /opt/conda/etc/profile.d/conda.sh" >> /root/.bashrc \ + && echo "conda activate base" >> /root/.bashrc + +# Final CMD or ENTRYPOINT +CMD ["bash"] diff --git a/.devcontainer/.devcontainer/devcontainer.json b/.devcontainer/.devcontainer/devcontainer.json new file mode 100644 index 0000000..aec6323 --- /dev/null +++ b/.devcontainer/.devcontainer/devcontainer.json @@ -0,0 +1,51 @@ +{ + "name": "torchsparsegradutils Dev Container", + "build": { + "dockerfile": "./Dockerfile", + "context": ".", + "args": { + "CUDA_VERSION": "12.1.0", + "PYTORCH_VERSION": "2.3.1", + "CUDATOOLKIT_VERSION": "12.1", + "PYTHON_VERSION": "3.11" + } + }, + "runArgs": [ + "--gpus", + "all" + ], + "remoteEnv": { + "SSH_AUTH_SOCK": "/tmp/ssh-agent.sock" + }, + "customizations": { + "vscode": { + "settings": { + "python.defaultInterpreterPath": "/opt/conda/bin/python", + "terminal.integrated.shell.linux": "/bin/bash", + "terminal.integrated.env.linux": { + "CONDA_DEFAULT_ENV": "base", + "CONDA_PREFIX": "/opt/conda", + "CONDA_PYTHON_EXE": "/opt/conda/bin/python", + "PATH": "/opt/conda/bin:${env:PATH}" + }, + "python.testing.pytestArgs": [ + "torchsparsegradutils/tests" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true + }, + "extensions": [ + "dbaeumer.vscode-eslint", + "ms-python.vscode-pylance", + "ms-python.python", + "github.copilot", + "GitHub.vscode-pull-request-github", + "GitHub.vscode-github-actions", + "mhutchie.git-graph", + "waderyan.gitblame" + ] + } + }, + "remoteUser": "vscode", + "postCreateCommand": "echo 'Container is ready!'" +} \ No newline at end of file diff --git a/.devcontainer/.devcontainer/environment.yml b/.devcontainer/.devcontainer/environment.yml new file mode 100644 index 0000000..59cc01f --- /dev/null +++ b/.devcontainer/.devcontainer/environment.yml @@ -0,0 +1,19 @@ +name: base +channels: + - conda-forge + - defaults +dependencies: + - pre-commit==3.7.1 + - black==24.4.2 + - flake8==7.1.0 + - numpy==2.0.0 + - scipy==1.14.0 + - jax[cuda]==0.4.28 + - parameterized==0.9.0 + - pytest==8.2.2 + - pytest-rerunfailures==14.0 + - pyyaml==6.0.1 + - conda-libmamba-solver + - libmamba + - libmambapy + - libarchive diff --git a/.devcontainer/.devcontainer/noop.txt b/.devcontainer/.devcontainer/noop.txt new file mode 100644 index 0000000..e69de29 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c6b953d..ede3113 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,16 +1,16 @@ repos: - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 24.4.2 hooks: - id: black language_version: python3.10 - repo: https://github.com/pycqa/flake8 - rev: 6.0.0 + rev: 7.1.0 hooks: - id: flake8 - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.6.0 hooks: - id: trailing-whitespace diff --git a/pyproject.toml b/pyproject.toml index 97f7b51..8e793f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.black] line-length = 120 -target-version = ['py37', 'py38', 'py39', 'py310'] +target-version = ['py37', 'py38', 'py39', 'py310', 'py311'] include = '\.pyi?$' exclude = ''' ( @@ -21,4 +21,8 @@ exclude = ''' # also separately exclude other files if needed #| some_file ) -''' \ No newline at end of file +''' +[tool.pytest.ini_options] +testpaths = [ + "torchsparsegradutils/tests" +] diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..9aeb9a2 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +testpaths = torchsparsegradutils/tests From c291b85dc9ece548505d48cb84383980f438ba9d Mon Sep 17 00:00:00 2001 From: theo-barfoot Date: Mon, 21 Oct 2024 16:47:48 +0000 Subject: [PATCH 2/8] devcontainer working --- .devcontainer/{.devcontainer => }/Dockerfile | 8 ++++---- .devcontainer/{.devcontainer => }/devcontainer.json | 8 ++++---- .devcontainer/{.devcontainer => }/environment.yml | 0 .devcontainer/{.devcontainer => }/noop.txt | 0 pyproject.toml | 6 +----- pytest.ini | 2 -- 6 files changed, 9 insertions(+), 15 deletions(-) rename .devcontainer/{.devcontainer => }/Dockerfile (92%) rename .devcontainer/{.devcontainer => }/devcontainer.json (90%) rename .devcontainer/{.devcontainer => }/environment.yml (100%) rename .devcontainer/{.devcontainer => }/noop.txt (100%) delete mode 100644 pytest.ini diff --git a/.devcontainer/.devcontainer/Dockerfile b/.devcontainer/Dockerfile similarity index 92% rename from .devcontainer/.devcontainer/Dockerfile rename to .devcontainer/Dockerfile index 519dc10..d2bdf61 100644 --- a/.devcontainer/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,5 +1,5 @@ # Stage 1: NVIDIA CUDA Image -ARG CUDA_VERSION=12.1.0 +ARG CUDA_VERSION=12.5.0 FROM nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu22.04 AS cuda-base # Stage 2: Miniconda setup from configuration @@ -19,12 +19,12 @@ ENV PATH /opt/conda/bin:$PATH ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 # Install Python 3.10 -ARG PYTHON_VERSION=3.11 +ARG PYTHON_VERSION=3.12 RUN conda install python=${PYTHON_VERSION} # Arguments for PyTorch and CUDA Toolkit versions -ARG PYTORCH_VERSION=2.3.1 -ARG CUDATOOLKIT_VERSION=12.1 +ARG PYTORCH_VERSION=2.5.0 +ARG CUDATOOLKIT_VERSION=12.4 # Install PyTorch and other dependencies RUN conda install pytorch=${PYTORCH_VERSION} pytorch-cuda=${CUDATOOLKIT_VERSION} -c pytorch -c nvidia diff --git a/.devcontainer/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json similarity index 90% rename from .devcontainer/.devcontainer/devcontainer.json rename to .devcontainer/devcontainer.json index aec6323..3187f71 100644 --- a/.devcontainer/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -4,10 +4,10 @@ "dockerfile": "./Dockerfile", "context": ".", "args": { - "CUDA_VERSION": "12.1.0", - "PYTORCH_VERSION": "2.3.1", - "CUDATOOLKIT_VERSION": "12.1", - "PYTHON_VERSION": "3.11" + "CUDA_VERSION": "12.4.0", + "PYTORCH_VERSION": "2.5.0", + "CUDATOOLKIT_VERSION": "12.4", + "PYTHON_VERSION": "3.12" } }, "runArgs": [ diff --git a/.devcontainer/.devcontainer/environment.yml b/.devcontainer/environment.yml similarity index 100% rename from .devcontainer/.devcontainer/environment.yml rename to .devcontainer/environment.yml diff --git a/.devcontainer/.devcontainer/noop.txt b/.devcontainer/noop.txt similarity index 100% rename from .devcontainer/.devcontainer/noop.txt rename to .devcontainer/noop.txt diff --git a/pyproject.toml b/pyproject.toml index 8e793f3..908bc5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.black] line-length = 120 -target-version = ['py37', 'py38', 'py39', 'py310', 'py311'] +target-version = ['py38', 'py39', 'py310', 'py311', 'py312'] include = '\.pyi?$' exclude = ''' ( @@ -22,7 +22,3 @@ exclude = ''' #| some_file ) ''' -[tool.pytest.ini_options] -testpaths = [ - "torchsparsegradutils/tests" -] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 9aeb9a2..0000000 --- a/pytest.ini +++ /dev/null @@ -1,2 +0,0 @@ -[pytest] -testpaths = torchsparsegradutils/tests From 647c54f3d869c8e7e61cca0b11257743a5f41d54 Mon Sep 17 00:00:00 2001 From: theo-barfoot Date: Mon, 21 Oct 2024 17:07:03 +0000 Subject: [PATCH 3/8] include py311 and bump minor version --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3257e06..31b4d4b 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ def readme(): setuptools.setup( name="torchsparsegradutils", - version="0.1.2", + version="0.1.3", description="A collection of utility functions to work with PyTorch sparse tensors", long_description=readme(), long_description_content_type="text/markdown", @@ -18,6 +18,7 @@ def readme(): "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", ], python_requires=">=3.8", From 45c117bdb3cccd3b6c91868315446127c0218840 Mon Sep 17 00:00:00 2001 From: theo-barfoot Date: Mon, 21 Oct 2024 17:19:35 +0000 Subject: [PATCH 4/8] update torch to 2.5.0 --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 6f0492e..43d7b47 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -13,7 +13,7 @@ jobs: fail-fast: false matrix: python-version: ["3.8", "3.10", "3.12"] - torch-version: ["1.13.1", "2.4.1"] + torch-version: ["1.13.1", "2.5.0"] exclude: - python-version: "3.12" torch-version: "1.13.1" From cc66ff9af2fe0dda6da8d506498879c44759f957 Mon Sep 17 00:00:00 2001 From: theo-barfoot Date: Mon, 21 Oct 2024 17:52:45 +0000 Subject: [PATCH 5/8] add cupy dependency --- .devcontainer/Dockerfile | 1 + .devcontainer/environment.yml | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index d2bdf61..3ef5303 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -30,6 +30,7 @@ ARG CUDATOOLKIT_VERSION=12.4 RUN conda install pytorch=${PYTORCH_VERSION} pytorch-cuda=${CUDATOOLKIT_VERSION} -c pytorch -c nvidia # Handle environment.yml if it exists +RUN echo env_change_20241021 COPY environment.yml* noop.txt /tmp/conda-tmp/ RUN if [ -f "/tmp/conda-tmp/environment.yml" ]; then \ /opt/conda/bin/conda env update -n base -f /tmp/conda-tmp/environment.yml; \ diff --git a/.devcontainer/environment.yml b/.devcontainer/environment.yml index 59cc01f..6e30dbb 100644 --- a/.devcontainer/environment.yml +++ b/.devcontainer/environment.yml @@ -3,12 +3,13 @@ channels: - conda-forge - defaults dependencies: + - numpy + - cupy + - jax[cuda] + - scipy - pre-commit==3.7.1 - black==24.4.2 - flake8==7.1.0 - - numpy==2.0.0 - - scipy==1.14.0 - - jax[cuda]==0.4.28 - parameterized==0.9.0 - pytest==8.2.2 - pytest-rerunfailures==14.0 From b4f2e71e44c58d1134ec6810a1af84cf6269abbc Mon Sep 17 00:00:00 2001 From: theo-barfoot Date: Mon, 21 Oct 2024 18:04:23 +0000 Subject: [PATCH 6/8] jax[cuda] working --- .devcontainer/Dockerfile | 2 +- .devcontainer/environment.yml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 3ef5303..7048eda 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -30,7 +30,7 @@ ARG CUDATOOLKIT_VERSION=12.4 RUN conda install pytorch=${PYTORCH_VERSION} pytorch-cuda=${CUDATOOLKIT_VERSION} -c pytorch -c nvidia # Handle environment.yml if it exists -RUN echo env_change_20241021 +RUN echo env_change_20241021_2 COPY environment.yml* noop.txt /tmp/conda-tmp/ RUN if [ -f "/tmp/conda-tmp/environment.yml" ]; then \ /opt/conda/bin/conda env update -n base -f /tmp/conda-tmp/environment.yml; \ diff --git a/.devcontainer/environment.yml b/.devcontainer/environment.yml index 6e30dbb..369bdc7 100644 --- a/.devcontainer/environment.yml +++ b/.devcontainer/environment.yml @@ -5,7 +5,6 @@ channels: dependencies: - numpy - cupy - - jax[cuda] - scipy - pre-commit==3.7.1 - black==24.4.2 @@ -18,3 +17,6 @@ dependencies: - libmamba - libmambapy - libarchive + - pip + - pip: + - "jax[cuda12]" From 58350b276196220bb9c215d604830eab87233501 Mon Sep 17 00:00:00 2001 From: theo-barfoot Date: Mon, 21 Oct 2024 18:12:26 +0000 Subject: [PATCH 7/8] tensor_split expects tensor_indices_or_sections to be on cpu --- torchsparsegradutils/indexed_matmul.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchsparsegradutils/indexed_matmul.py b/torchsparsegradutils/indexed_matmul.py index ef0c8f7..b41535c 100644 --- a/torchsparsegradutils/indexed_matmul.py +++ b/torchsparsegradutils/indexed_matmul.py @@ -45,7 +45,7 @@ def segment_mm(a, b, seglen_a): if not a.shape[1] == D1 or not seglen_a.shape[0] == R: raise ValueError("Incompatible size for inputs") - segidx_a = torch.cumsum(seglen_a[:-1], dim=0) + segidx_a = torch.cumsum(seglen_a[:-1], dim=0).cpu() # Ideally the conversions below to nested tensor would be handled natively nested_a = torch.nested.as_nested_tensor(torch.tensor_split(a, segidx_a, dim=0)) From f840f4b209335f53698cec5b77eef11abc6c9a8c Mon Sep 17 00:00:00 2001 From: theo-barfoot <44871137+theo-barfoot@users.noreply.github.com> Date: Mon, 21 Oct 2024 19:19:55 +0100 Subject: [PATCH 8/8] exclude py38 with torch25 Signed-off-by: theo-barfoot <44871137+theo-barfoot@users.noreply.github.com> --- .github/workflows/python-package.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 43d7b47..361e147 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -17,6 +17,8 @@ jobs: exclude: - python-version: "3.12" torch-version: "1.13.1" + - python-version: "3.8" + torch-version: "2.5.0" steps: - uses: actions/checkout@v4