-
Notifications
You must be signed in to change notification settings - Fork 175
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[FEAT] Enable Progress Bars for PyRunner and RayRunner (#1609)
* Enables progress bars when in interactive mode (Notebook or IPython) * Implementation for Pyrunner, RayRunner[Local] and RayRunner[Remote]  --------- Co-authored-by: Jay Chia <[email protected]>
- Loading branch information
Showing
7 changed files
with
149 additions
and
42 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| from __future__ import annotations | ||
|
|
||
| import os | ||
| from typing import Any | ||
|
|
||
| from tqdm.auto import tqdm | ||
|
|
||
| from daft.execution.execution_step import PartitionTask | ||
|
|
||
|
|
||
| class ProgressBar: | ||
| def __init__(self, use_ray_tqdm: bool, show_tasks_bar: bool = False, disable: bool = False) -> None: | ||
| self.use_ray_tqdm = use_ray_tqdm | ||
| self.show_tasks_bar = show_tasks_bar | ||
| self.tqdm_mod = tqdm | ||
| self.pbars: dict[int, tqdm] = dict() | ||
| self.disable = ( | ||
| disable | ||
| or not bool(int(os.environ.get("RAY_TQDM", "1"))) | ||
| or not bool(int(os.environ.get("DAFT_PROGRESS_BAR", "1"))) | ||
| ) | ||
|
|
||
| def _make_new_bar(self, stage_id: int, name: str): | ||
| if self.use_ray_tqdm: | ||
| self.pbars[stage_id] = self.tqdm_mod(total=1, desc=name, position=len(self.pbars)) | ||
| else: | ||
| self.pbars[stage_id] = self.tqdm_mod( | ||
| total=1, desc=name, position=len(self.pbars), leave=False, mininterval=1.0 | ||
| ) | ||
|
|
||
| def mark_task_start(self, step: PartitionTask[Any]) -> None: | ||
| if self.disable: | ||
| return | ||
| if self.show_tasks_bar: | ||
| if len(self.pbars) == 0: | ||
| self._make_new_bar(-1, "Tasks") | ||
| else: | ||
| task_pbar = self.pbars[-1] | ||
| task_pbar.total += 1 | ||
|
|
||
| stage_id = step.stage_id | ||
|
|
||
| if stage_id not in self.pbars: | ||
| name = "-".join(i.__class__.__name__ for i in step.instructions) | ||
| self._make_new_bar(stage_id, name) | ||
| else: | ||
| pb = self.pbars[stage_id] | ||
| pb.total += 1 | ||
|
|
||
| def mark_task_done(self, step: PartitionTask[Any]) -> None: | ||
| if self.disable: | ||
| return | ||
|
|
||
| stage_id = step.stage_id | ||
| self.pbars[stage_id].update(1) | ||
| if self.show_tasks_bar: | ||
| self.pbars[-1].update(1) | ||
|
|
||
| def close(self) -> None: | ||
| for p in self.pbars.values(): | ||
| p.close() | ||
| del p |
Oops, something went wrong.