From 8a630547c18a74ea2b31c4726a6269eb227aed98 Mon Sep 17 00:00:00 2001 From: Ben Clifford Date: Wed, 6 Mar 2024 06:20:30 -0600 Subject: [PATCH] Make htex scale_in pick longest idle blocks (#3135) Prior to this, scale_in picked the shortest-idle (i.e. most recently used but now idle) blocks. In the scaling code right now, there isn't any particular reason to order idle blocks in any particular order... ... but the next PR following on from this one will also consider unstarted blocks with infinite idle time, and in that situation, the policy will be that unstarted idle blocks should be scaled down in preference to currently running blocks. (other policies are possible and reasonable but not addressed by either this PR or the upcoming PR) I think this was intended to work this way to begin with (although I'm unsure of the justification) but there is a sign error that is flipped by this PR. --- parsl/executors/high_throughput/executor.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/parsl/executors/high_throughput/executor.py b/parsl/executors/high_throughput/executor.py index e08c95b9be..3084a4165e 100644 --- a/parsl/executors/high_throughput/executor.py +++ b/parsl/executors/high_throughput/executor.py @@ -739,7 +739,12 @@ class BlockInfo: block_info[b_id].tasks += manager['tasks'] block_info[b_id].idle = min(block_info[b_id].idle, manager['idle_duration']) - sorted_blocks = sorted(block_info.items(), key=lambda item: (item[1].idle, item[1].tasks)) + # The scaling policy is that longest idle blocks should be scaled down + # in preference to least idle (most recently used) blocks. + # Other policies could be implemented here. + + sorted_blocks = sorted(block_info.items(), key=lambda item: (-item[1].idle, item[1].tasks)) + logger.debug(f"Scale in selecting from {len(sorted_blocks)} blocks") if max_idletime is None: block_ids_to_kill = [x[0] for x in sorted_blocks[:blocks]]