From b34ef41617ff1549b23768c9f0a0feea2f9ffae4 Mon Sep 17 00:00:00 2001 From: Ben Clifford Date: Wed, 31 Jan 2024 13:25:03 +0000 Subject: [PATCH] Make htex scale_in pick longest idle blocks Prior to this, scale_in picked the shortest-idle (i.e. most recently used but now idle) blocks. In the scaling code right now, there isn't any particular reason to order idle blocks in any particular order... ... but the next PR following on from this one will also consider unstarted blocks with infinite idle time, and in that situation, the policy will be that unstarted idle blocks should be scaled down in preference to currently running blocks. (other policies are possible and reasonable but not addressed by either this PR or the upcoming PR) I think this was intended to work this way to begin with (although I'm unsure of the justification) but there is a sign error that is flipped by this PR. --- parsl/executors/high_throughput/executor.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/parsl/executors/high_throughput/executor.py b/parsl/executors/high_throughput/executor.py index e08c95b9be..3084a4165e 100644 --- a/parsl/executors/high_throughput/executor.py +++ b/parsl/executors/high_throughput/executor.py @@ -739,7 +739,12 @@ class BlockInfo: block_info[b_id].tasks += manager['tasks'] block_info[b_id].idle = min(block_info[b_id].idle, manager['idle_duration']) - sorted_blocks = sorted(block_info.items(), key=lambda item: (item[1].idle, item[1].tasks)) + # The scaling policy is that longest idle blocks should be scaled down + # in preference to least idle (most recently used) blocks. + # Other policies could be implemented here. + + sorted_blocks = sorted(block_info.items(), key=lambda item: (-item[1].idle, item[1].tasks)) + logger.debug(f"Scale in selecting from {len(sorted_blocks)} blocks") if max_idletime is None: block_ids_to_kill = [x[0] for x in sorted_blocks[:blocks]]