From 0d5081eeea4ccc08aaa1b3b841421709d774ba55 Mon Sep 17 00:00:00 2001 From: Ben Clifford Date: Tue, 26 Mar 2024 11:18:14 +0000 Subject: [PATCH 1/2] Move 'first' field into strategy state This is part of work to move JobStatusPoller facade state into other classes, as part of job handling rearrangements in PR #3293 This should not change behaviour: each executor has a single PolledExecutorFacade and a single strategy.ExecutorState, and this PR moves the 'first' field from one to the other. parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py tests that init_blocks handling still fires properly - that's what is switched by this 'first' field. --- parsl/jobs/job_status_poller.py | 1 - parsl/jobs/strategy.py | 16 ++++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/parsl/jobs/job_status_poller.py b/parsl/jobs/job_status_poller.py index 5a7f71ab32..7f57ce1d6d 100644 --- a/parsl/jobs/job_status_poller.py +++ b/parsl/jobs/job_status_poller.py @@ -23,7 +23,6 @@ def __init__(self, executor: BlockProviderExecutor, dfk: Optional["parsl.dataflo self._interval = executor.status_polling_interval self._last_poll_time = 0.0 self._status = {} # type: Dict[str, JobStatus] - self.first = True # Create a ZMQ channel to send poll status to monitoring self.monitoring_enabled = False diff --git a/parsl/jobs/strategy.py b/parsl/jobs/strategy.py index 61a2d43e97..20e382f21e 100644 --- a/parsl/jobs/strategy.py +++ b/parsl/jobs/strategy.py @@ -26,6 +26,10 @@ class ExecutorState(TypedDict): If the executor is not idle, then None. """ + first: bool + """Is this the first poll for this executor? + """ + class Strategy: """Scaling strategy. @@ -144,17 +148,17 @@ def __init__(self, *, strategy: Optional[str], max_idletime: float) -> None: def add_executors(self, executors: Sequence[ParslExecutor]) -> None: for executor in executors: - self.executors[executor.label] = {'idle_since': None} + self.executors[executor.label] = {'idle_since': None, 'first': True} def _strategy_init_only(self, executor_facades: List[jsp.PolledExecutorFacade]) -> None: """Scale up to init_blocks at the start, then nothing more. """ for ef in executor_facades: - if ef.first: - executor = ef.executor + executor = ef.executor + if self.executors[executor.label]['first']: logger.debug(f"strategy_init_only: scaling out {executor.provider.init_blocks} initial blocks for {executor.label}") ef.scale_out(executor.provider.init_blocks) - ef.first = False + self.executors[executor.label]['first'] = False else: logger.debug("strategy_init_only: doing nothing") @@ -190,11 +194,11 @@ def _general_strategy(self, executor_facades, *, strategy_type): continue logger.debug(f"Strategizing for executor {label}") - if ef.first: + if self.executors[label]['first']: executor = ef.executor logger.debug(f"Scaling out {executor.provider.init_blocks} initial blocks for {label}") ef.scale_out(executor.provider.init_blocks) - ef.first = False + self.executors[label]['first'] = False # Tasks that are either pending completion active_tasks = executor.outstanding From 180c0bc677d40b468a3784f79c91208e8bb5ac75 Mon Sep 17 00:00:00 2001 From: Ben Clifford Date: Mon, 8 Apr 2024 16:08:46 +0000 Subject: [PATCH 2/2] Rephrase on kevins prefs --- parsl/jobs/strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parsl/jobs/strategy.py b/parsl/jobs/strategy.py index 20e382f21e..d8f3155c11 100644 --- a/parsl/jobs/strategy.py +++ b/parsl/jobs/strategy.py @@ -27,7 +27,7 @@ class ExecutorState(TypedDict): """ first: bool - """Is this the first poll for this executor? + """True if this executor has not yet had a strategy poll. """