Ensure we receive a cuda enabled build of pyarrow

re-order channels ensuring nvidia & rapidsai versions of packages receive priority
Add missing dep for ccache
Pin pandas to 1.3, more recent versions trigger deprecation warnins from cudf
Add in pip transitive deps, ensuring we receive the conda versions rather than the pip versions

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -1,14 +1,13 @@
 # This file is generated by `rapids-dependency-file-generator`.
 # To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`.
 channels:
-- conda-forge
-- huggingface
 - rapidsai
 - nvidia/label/cuda-11.8.0
 - nvidia
-- rapidsai-nightly
 - nvidia/label/dev
+- huggingface
 - pytorch
+- conda-forge
 - dglteam/label/cu118
 dependencies:
 - appdirs
@@ -17,6 +16,8 @@ dependencies:
 - boost-cpp=1.82
 - boto3
 - breathe=4.34.0
+- cachetools=5.0.0
+- ccache>=3.7
 - clangdev=16
 - click >=8
 - click>=8
@@ -34,7 +35,7 @@ dependencies:
 - cython=0.29.24
 - dask>=2023.1.1
 - datacompy=0.8
-- dgl=1.0.2
+- dgl=1.1.1
 - dill
 - dill=0.3.6
 - distributed>=2023.1.1
@@ -46,13 +47,13 @@ dependencies:
 - flake8
 - gcc_linux-64=11.2
 - git-lfs
-- glog=0.6
 - grpcio
 - gxx_linux-64=11.2
 - huggingface_hub=0.10.1
 - include-what-you-use=0.20
 - ipython
 - isort
+- kafka-python=2.0
 - langchain=0.0.190
 - librdkafka=1.9.2
 - libwebp>=1.3.2
@@ -66,14 +67,19 @@ dependencies:
 - ninja=1.10
 - nlohmann_json=3.9
 - nodejs=18.*
+- numpy>=1.21,<1.25
 - numpydoc=1.4
 - nvtabular=23.06
 - openai=0.28
+- pandas=1.3
 - papermill=2.3.4
 - pip
 - pluggy=1.0
+- port-for=0.7
 - pre-commit
 - protobuf=4.21.*
+- py4j=0.10
+- pyarrow * *_cuda
 - pyarrow-hotfix
 - pybind11-stubgen=0.10
 - pylint>=2.17.4,<2.18
@@ -96,13 +102,13 @@ dependencies:
 - sphinx
 - sphinx_rtd_theme
 - sqlalchemy<2.0
+- sysroot_linux-64=2.17
 - tqdm=4
 - transformers=4.30.2
 - tritonclient=2.26
 - typing_utils=0.1
 - ucx=1.14
 - ujson=5.8
-- versioneer
 - watchdog=2.1
 - websockets
 - yapf=0.40.1
@@ -114,5 +120,6 @@ dependencies:
   - nemollm
   - pymilvus==2.3.2
   - pytest-kafka==0.6.0
+  - rapids-dependency-file-generator
 name: all_cuda-118_arch-x86_64_py-310
 

conda/environments/dev_cuda-118_arch-x86_64.yaml

@@ -1,20 +1,21 @@
 # This file is generated by `rapids-dependency-file-generator`.
 # To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`.
 channels:
-- conda-forge
-- huggingface
 - rapidsai
 - nvidia/label/cuda-11.8.0
 - nvidia
-- rapidsai-nightly
 - nvidia/label/dev
+- huggingface
 - pytorch
+- conda-forge
 - dglteam/label/cu118
 dependencies:
 - appdirs
 - benchmark=1.6.0
 - boost-cpp=1.82
 - breathe=4.34.0
+- cachetools=5.0.0
+- ccache>=3.7
 - clangdev=16
 - click >=8
 - click>=8
@@ -38,12 +39,12 @@ dependencies:
 - flake8
 - gcc_linux-64=11.2
 - git-lfs
-- glog=0.6
 - grpcio
 - gxx_linux-64=11.2
 - include-what-you-use=0.20
 - ipython
 - isort
+- kafka-python=2.0
 - librdkafka=1.9.2
 - mlflow>=2.2.1,<3
 - mrc=24.03
@@ -53,12 +54,17 @@ dependencies:
 - ninja=1.10
 - nlohmann_json=3.9
 - nodejs=18.*
+- numpy>=1.21,<1.25
 - numpydoc=1.4
 - nvtabular=23.06
+- pandas=1.3
 - pip
 - pluggy=1.0
+- port-for=0.7
 - pre-commit
 - protobuf=4.21.*
+- py4j=0.10
+- pyarrow * *_cuda
 - pyarrow-hotfix
 - pybind11-stubgen=0.10
 - pylint>=2.17.4,<2.18
@@ -77,11 +83,11 @@ dependencies:
 - sphinx
 - sphinx_rtd_theme
 - sqlalchemy<2.0
+- sysroot_linux-64=2.17
 - tqdm=4
 - tritonclient=2.26
 - typing_utils=0.1
 - ucx=1.14
-- versioneer
 - watchdog=2.1
 - websockets
 - yapf=0.40.1
@@ -90,5 +96,6 @@ dependencies:
   - milvus==2.3.2
   - pymilvus==2.3.2
   - pytest-kafka==0.6.0
+  - rapids-dependency-file-generator
 name: dev_cuda-118_arch-x86_64_py-310
 

dependencies.yaml

@@ -107,14 +107,13 @@ files:
 
 
 channels:
-  - conda-forge
-  - huggingface
   - rapidsai
   - nvidia/label/cuda-11.8.0
   - nvidia
-  - rapidsai-nightly
   - nvidia/label/dev
+  - huggingface
   - pytorch
+  - conda-forge
   - dglteam/label/cu118
 
 dependencies:
@@ -126,6 +125,8 @@ dependencies:
           - mrc=24.03 # should this be in build, or somewhere else?
           - gcc_linux-64=11.2
           - gxx_linux-64=11.2
+          - cachetools=5.0.0
+          - ccache>=3.7
           - cmake=3.25
           - boost-cpp=1.82
           - cuda-nvcc
@@ -135,7 +136,10 @@ dependencies:
           - librdkafka=1.9.2
           - ninja=1.10
           - nlohmann_json=3.9
+          - pandas=1.3
           - protobuf=4.21.*
+          - pyarrow * *_cuda # Ensure we get a CUDA build. Version determined by cuDF
+          - pyarrow-hotfix
           - pybind11-stubgen=0.10
           - rapidjson=1.1.0
           - scikit-build=0.17.1
@@ -246,6 +250,7 @@ dependencies:
           - numpy>=1.21,<1.25
           - numpydoc=1.4
           - nvtabular=23.06
+          - pyarrow * *_cuda # Ensure we get a CUDA build. Version determined by cuDF
           - pyarrow-hotfix
           - python-confluent-kafka=1.9.2
           - python-graphviz
@@ -258,6 +263,13 @@ dependencies:
           - typing_utils=0.1
           - watchdog=2.1
           - websockets
+
+          ####### Pip Transitive Dependencies (keep sorted!) #######
+          # These are dependencies that are available on conda, but are required by the pip packages listed below. Its much
+          # better to install them with conda than pip to allow for better dependency resolution.
+          - kafka-python=2.0
+          - port-for=0.7
+          - py4j=0.10
           - pip
           - pip:
             - databricks-connect