Merge branch 'main' into tpcds-wrapper

Eventual-Inc · Dec 19, 2024 · 49638f2 · 49638f2
2 parents 7a8ae8d + 33f95c1
commit 49638f2
Show file tree

Hide file tree

Showing 90 changed files with 10,507 additions and 249 deletions.
diff --git a/.github/workflows/build-artifact-s3.yml b/.github/workflows/build-artifact-s3.yml
@@ -14,7 +14,7 @@ on:
 
 env:
   PACKAGE_NAME: getdaft
-  PYTHON_VERSION: 3.8
+  PYTHON_VERSION: 3.9
 
 jobs:
   build-and-push:

diff --git a/.github/workflows/nightlies-tests.yml b/.github/workflows/nightlies-tests.yml
@@ -12,7 +12,7 @@ on:
 env:
   DAFT_ANALYTICS_ENABLED: '0'
   UV_SYSTEM_PYTHON: 1
-  PYTHON_VERSION: '3.8'
+  PYTHON_VERSION: '3.9'
 
 jobs:
   integration-test-tpch:

diff --git a/.github/workflows/notebook-checker.yml b/.github/workflows/notebook-checker.yml
@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8']
+        python-version: ['3.9']
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}

diff --git a/.github/workflows/property-based-tests.yml b/.github/workflows/property-based-tests.yml
@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8']
+        python-version: ['3.9']
         daft_runner: [py]
     steps:
     - uses: actions/checkout@v4

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
@@ -24,28 +24,28 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.10']
+        python-version: ['3.9', '3.10']
         daft-runner: [py, ray, native]
-        pyarrow-version: [7.0.0, 16.0.0]
+        pyarrow-version: [8.0.0, 16.0.0]
         os: [ubuntu-20.04, windows-latest]
         exclude:
         - daft-runner: ray
-          pyarrow-version: 7.0.0
+          pyarrow-version: 8.0.0
           os: ubuntu-20.04
         - daft-runner: py
           python-version: '3.10'
-          pyarrow-version: 7.0.0
+          pyarrow-version: 8.0.0
           os: ubuntu-20.04
         - daft-runner: native
           python-version: '3.10'
-          pyarrow-version: 7.0.0
+          pyarrow-version: 8.0.0
           os: ubuntu-20.04
-        - python-version: '3.8'
+        - python-version: '3.9'
           pyarrow-version: 16.0.0
         - os: windows-latest
-          python-version: '3.8'
+          python-version: '3.9'
         - os: windows-latest
-          pyarrow-version: 7.0.0
+          pyarrow-version: 8.0.0
     steps:
     - uses: actions/checkout@v4
     - uses: moonrepo/setup-rust@v1
@@ -93,7 +93,7 @@ jobs:
       run: uv pip install pyarrow==${{ matrix.pyarrow-version }}
 
     - name: Override deltalake for pyarrow
-      if: ${{ (matrix.pyarrow-version == '7.0.0') }}
+      if: ${{ (matrix.pyarrow-version == '8.0.0') }}
       run: uv pip install deltalake==0.10.0
 
     - name: Build library and Test with pytest (unix)
@@ -181,7 +181,7 @@ jobs:
       package-name: getdaft
     strategy:
       matrix:
-        python-version: ['3.8']
+        python-version: ['3.9']
     steps:
     - uses: actions/checkout@v4
       with:
@@ -219,7 +219,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8']
+        python-version: ['3.9']
         daft-runner: [py, ray, native]
     steps:
     - uses: actions/checkout@v4
@@ -294,7 +294,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8'] # can't use 3.7 due to requiring anon mode for adlfs
+        python-version: ['3.9'] # can't use 3.7 due to requiring anon mode for adlfs
         daft-runner: [py, ray, native]
     steps:
     - uses: actions/checkout@v4
@@ -372,7 +372,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8'] # can't use 3.7 due to requiring anon mode for adlfs
+        python-version: ['3.9'] # can't use 3.7 due to requiring anon mode for adlfs
         daft-runner: [py, ray, native]
     # These permissions are needed to interact with GitHub's OIDC Token endpoint.
     # This is used in the step "Assume GitHub Actions AWS Credentials"
@@ -466,7 +466,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8'] # can't use 3.7 due to requiring anon mode for adlfs
+        python-version: ['3.9'] # can't use 3.7 due to requiring anon mode for adlfs
         daft-runner: [py, ray, native]
     steps:
     - uses: actions/checkout@v4
@@ -543,7 +543,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8'] # can't use 3.7 due to requiring anon mode for adlfs
+        python-version: ['3.9'] # can't use 3.7 due to requiring anon mode for adlfs
         daft-runner: [py, ray, native]
     steps:
     - uses: actions/checkout@v4
@@ -854,7 +854,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu, Windows]
-        python-version: ['3.8']
+        python-version: ['3.9']
     steps:
     - uses: actions/checkout@v4
     - uses: moonrepo/setup-rust@v1
@@ -946,7 +946,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 15
     env:
-      python-version: '3.8'
+      python-version: '3.9'
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ env.python-version }}

diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml
@@ -162,7 +162,7 @@ jobs:
       with:
         # Really doesn't matter what version we upload with
         # just the version we test with
-        python-version: '3.8'
+        python-version: '3.9'
         channels: conda-forge
         channel-priority: true
 

diff --git a/.github/workflows/run-cluster.yaml b/.github/workflows/run-cluster.yaml
@@ -3,6 +3,10 @@ name: run-cluster
 on:
   workflow_dispatch:
     inputs:
+      daft_wheel_url:
+        description: Daft python-wheel URL
+        type: string
+        required: false
       daft_version:
         description: Daft version (errors if both this and "Daft python-wheel URL" are provided)
         type: string
@@ -43,6 +47,7 @@ on:
 jobs:
   build-commit:
     uses: ./.github/workflows/build-commit.yaml
+    if: ${{ inputs.daft_version == '' && inputs.daft_wheel_url == '' }}
     with:
       arch: x86
       python_version: ${{ inputs.python_version }}
@@ -51,6 +56,10 @@ jobs:
 
   run-command:
     runs-on: [self-hosted, linux, x64, ci-dev]
+    # If both the `daft-version` and `daft-wheel-url` parameters are not specified, the `build-commit` job is entirely skipped.
+    # We still want to run this job, even if `build-commit` is skipped.
+    # The `always()` guarantees that this job is always run.
+    if: always()
     permissions:
       id-token: write
       contents: read
@@ -84,7 +93,7 @@ jobs:
             --python 3.12 \
             .github/ci-scripts/templatize_ray_config.py \
             --cluster-name="ray-ci-run-${{ github.run_id }}_${{ github.run_attempt }}" \
-            --daft-wheel-url='${{ needs.build-commit.outputs.wheel_url }}' \
+            --daft-wheel-url='${{ needs.build-commit.outputs.wheel_url || inputs.daft_wheel_url || '' }}' \
             --daft-version='${{ inputs.daft_version }}' \
             --python-version='${{ inputs.python_version }}' \
             --cluster-profile='${{ inputs.cluster_profile }}' \

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -23,6 +23,8 @@ repos:
       )$
   - id: check-yaml
     exclude: kubernetes-ops
+    args:
+    - --unsafe
   - id: pretty-format-json
     exclude: |
       (?x)^(

diff --git a/.ruff.toml b/.ruff.toml
@@ -1,6 +1,7 @@
 fix = true
 indent-width = 4
 line-length = 120
+# TODO: clean up typing code and update to py39
 target-version = "py38"
 
 [format]

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -276,7 +276,7 @@ features = ['async']
 path = "src/parquet2"
 
 [workspace.dependencies.pyo3]
-features = ["extension-module", "multiple-pymethods", "abi3-py38", "indexmap"]
+features = ["extension-module", "multiple-pymethods", "abi3-py39", "indexmap"]
 version = "0.21.0"
 
 [workspace.dependencies.pyo3-log]

diff --git a/daft/context.py b/daft/context.py
@@ -7,7 +7,6 @@
 import warnings
 from typing import TYPE_CHECKING, ClassVar, Literal
 
-from daft import get_build_type
 from daft.daft import IOConfig, PyDaftExecutionConfig, PyDaftPlanningConfig
 
 if TYPE_CHECKING:
@@ -109,14 +108,8 @@ def _get_runner_config_from_env() -> _RunnerConfig:
             max_task_backlog=task_backlog,
             force_client_mode=ray_force_client_mode,
         )
-
-    # Use native runner if in dev mode
-    elif get_build_type() == "dev":
-        return _NativeRunnerConfig()
-
-    # Fall back on PyRunner
     else:
-        return _PyRunnerConfig(use_thread_pool=use_thread_pool)
+        return _NativeRunnerConfig()
 
 
 @dataclasses.dataclass
@@ -172,6 +165,12 @@ def get_or_create_runner(self) -> Runner:
             elif runner_config.name == "native":
                 from daft.runners.native_runner import NativeRunner
 
+                warnings.warn(
+                    "Daft is configured to use the new NativeRunner by default as of v0.4.0. "
+                    "If you are encountering any regressions, please switch back to the legacy PyRunner via `daft.context.set_runner_py()` or by setting the env variable `DAFT_RUNNER=py`. "
+                    "We appreciate you filing issues and helping make the NativeRunner better: https://github.com/Eventual-Inc/Daft/issues",
+                )
+
                 assert isinstance(runner_config, _NativeRunnerConfig)
                 self._runner = NativeRunner()