Skip to content

Commit

Permalink
perf(rust): Move morsel distribution to the computational async engine (
Browse files Browse the repository at this point in the history
  • Loading branch information
orlp authored Jan 7, 2025
1 parent 72cd66a commit 52ea381
Showing 1 changed file with 4 additions and 4 deletions.
8 changes: 4 additions & 4 deletions crates/polars-stream/src/nodes/io_sources/parquet/init.rs
Original file line number Diff line number Diff line change
Expand Up @@ -159,8 +159,8 @@ impl ParquetSourceNode {
}));

// Distributes morsels across pipelines. This does not perform any CPU or I/O bound work -
// it is purely a dispatch loop.
let distribute_task = AbortOnDropHandle(io_runtime.spawn(async move {
// it is purely a dispatch loop. Run on the computational executor to reduce context switches.
let distribute_task = async_executor::spawn(TaskPriority::High, async move {
let mut morsel_seq = MorselSeq::default();
while let Some(decode_fut) = decode_recv.recv().await {
let df = decode_fut.await?;
Expand All @@ -176,13 +176,13 @@ impl ParquetSourceNode {
}
}
PolarsResult::Ok(())
}));
});

let join_task = io_runtime.spawn(async move {
metadata_task.await.unwrap()?;
prefetch_task.await.unwrap()?;
decode_task.await.unwrap()?;
distribute_task.await.unwrap()?;
distribute_task.await?;
Ok(())
});

Expand Down

0 comments on commit 52ea381

Please sign in to comment.