From 638b73815072a308bd142c9c9abba80500d10cba Mon Sep 17 00:00:00 2001 From: Raunak Bhagat Date: Wed, 20 Nov 2024 13:45:47 -0800 Subject: [PATCH] Add docs pointing towards daft-launcher (in the distributed-computing section) --- .gitignore | 2 +- docs/source/conf.py | 2 +- docs/source/migration_guides/coming_from_dask.rst | 2 +- .../{poweruser => }/distributed-computing.rst | 4 ++++ .../user_guide/distributed-computing/daft-launcher.rst | 10 ++++++++++ docs/source/user_guide/index.rst | 1 + docs/source/user_guide/poweruser.rst | 1 - 7 files changed, 18 insertions(+), 4 deletions(-) rename docs/source/user_guide/{poweruser => }/distributed-computing.rst (98%) create mode 100644 docs/source/user_guide/distributed-computing/daft-launcher.rst diff --git a/.gitignore b/.gitignore index b605c9152f..9526092ef6 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ requirements.txt build .cython_build .hypothesis +.ropeproject **/.ipynb_checkpoints/ @@ -36,7 +37,6 @@ log/ # Added by pyenv .python-version - # Zed editor .zed/ diff --git a/docs/source/conf.py b/docs/source/conf.py index 1dc26a8848..77dc661223 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -96,7 +96,7 @@ "learn/install": "../install.html", "learn/user_guides/dataframes": "intro-dataframes.html", "learn/user_guides/types_and_ops": "intro-dataframes.html", - "learn/user_guides/remote_cluster_execution": "distributed-computing.html", + "learn/user_guides/remote_cluster_execution": "user_guide/distributed-computing", "learn/quickstart": "learn/10-min.html", "learn/10-min": "../10-min.html", "user_guide/basic_concepts/expressions": "user_guide/expressions", diff --git a/docs/source/migration_guides/coming_from_dask.rst b/docs/source/migration_guides/coming_from_dask.rst index 8eaedb1258..8589f5e5e0 100644 --- a/docs/source/migration_guides/coming_from_dask.rst +++ b/docs/source/migration_guides/coming_from_dask.rst @@ -118,7 +118,7 @@ Dask supports the same data types as pandas. Daft is built to support many more Distributed Computing and Remote Clusters ----------------------------------------- -Both Dask and Daft support distributed computing on remote clusters. In Dask, you create a Dask cluster either locally or remotely and perform computations in parallel there. Currently, Daft supports distributed cluster computing :doc:`with Ray <../user_guide/poweruser/distributed-computing>`. Support for running Daft computations on Dask clusters is on the roadmap. +Both Dask and Daft support distributed computing on remote clusters. In Dask, you create a Dask cluster either locally or remotely and perform computations in parallel there. Currently, Daft supports distributed cluster computing :doc:`with Ray <../user_guide/distributed-computing>`. Support for running Daft computations on Dask clusters is on the roadmap. Cloud support for both Dask and Daft is the same. diff --git a/docs/source/user_guide/poweruser/distributed-computing.rst b/docs/source/user_guide/distributed-computing.rst similarity index 98% rename from docs/source/user_guide/poweruser/distributed-computing.rst rename to docs/source/user_guide/distributed-computing.rst index 4950007823..35cbc6ae85 100644 --- a/docs/source/user_guide/poweruser/distributed-computing.rst +++ b/docs/source/user_guide/distributed-computing.rst @@ -67,3 +67,7 @@ You can take the IP address and port and pass it to Daft: ╰───────╯ (Showing first 2 of 2 rows) + +.. toctree:: + + distributed-computing/daft-launcher diff --git a/docs/source/user_guide/distributed-computing/daft-launcher.rst b/docs/source/user_guide/distributed-computing/daft-launcher.rst new file mode 100644 index 0000000000..1892a2490d --- /dev/null +++ b/docs/source/user_guide/distributed-computing/daft-launcher.rst @@ -0,0 +1,10 @@ +Daft Launcher +===================== + +Getting started with running daft at a distributed scale can be tough. +Therefore, we've created a simple, open-sourced launcher to help you get started. + +The tool, `daft-launcher `_, is open-sourced and available to be run today! +To install it, you can run ``pip install daft-launcher`` (or using ``uv``, you can run ``uv pip install daft-launcher``). + +To get started with it, please refer to our MDBook documentation, available `here `_. diff --git a/docs/source/user_guide/index.rst b/docs/source/user_guide/index.rst index b4b7150215..9ee6ceb473 100644 --- a/docs/source/user_guide/index.rst +++ b/docs/source/user_guide/index.rst @@ -10,6 +10,7 @@ Daft User Guide expressions datatypes dataframe-operations + distributed-computing sql aggregations udf diff --git a/docs/source/user_guide/poweruser.rst b/docs/source/user_guide/poweruser.rst index 765fbe83ca..dd427999de 100644 --- a/docs/source/user_guide/poweruser.rst +++ b/docs/source/user_guide/poweruser.rst @@ -5,4 +5,3 @@ The Daft Poweruser poweruser/memory poweruser/partitioning - poweruser/distributed-computing