Skip to content

Commit

Permalink
Reorder fields and methods in MemoryManagerConfig
Browse files Browse the repository at this point in the history 
Reorder fields and methods so ordering match and related config
properties are grouped together.
  • Loading branch information
@losipiuk
losipiuk committed Nov 8, 2023
1 parent ec31623 commit df3a531
Show file tree
Hide file tree
Showing 2 changed files with 69 additions and 72 deletions.
113 changes: 55 additions & 58 deletions core/trino-main/src/main/java/io/trino/memory/MemoryManagerConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,54 +42,13 @@ public class MemoryManagerConfig
private double faultTolerantExecutionTaskMemoryGrowthFactor = 3.0;
private double faultTolerantExecutionTaskMemoryEstimationQuantile = 0.9;
private DataSize faultTolerantExecutionTaskRuntimeMemoryEstimationOverhead = DataSize.of(1, GIGABYTE);
private LowMemoryQueryKillerPolicy lowMemoryQueryKillerPolicy = LowMemoryQueryKillerPolicy.TOTAL_RESERVATION_ON_BLOCKED_NODES;
private LowMemoryTaskKillerPolicy lowMemoryTaskKillerPolicy = LowMemoryTaskKillerPolicy.TOTAL_RESERVATION_ON_BLOCKED_NODES;
private boolean faultTolerantExecutionMemoryRequirementIncreaseOnWorkerCrashEnabled = true;
private DataSize faultTolerantExecutionEagerSpeculativeTasksNodeMemoryOvercommit = DataSize.of(20, GIGABYTE);

/**
* default value is overwritten for fault tolerant execution in {@link #applyFaultTolerantExecutionDefaults()}}
*/
private LowMemoryQueryKillerPolicy lowMemoryQueryKillerPolicy = LowMemoryQueryKillerPolicy.TOTAL_RESERVATION_ON_BLOCKED_NODES;
private LowMemoryTaskKillerPolicy lowMemoryTaskKillerPolicy = LowMemoryTaskKillerPolicy.TOTAL_RESERVATION_ON_BLOCKED_NODES;
// default value is overwritten for fault tolerant execution in {@link #applyFaultTolerantExecutionDefaults()}}
private Duration killOnOutOfMemoryDelay = new Duration(5, MINUTES);

public LowMemoryQueryKillerPolicy getLowMemoryQueryKillerPolicy()
{
return lowMemoryQueryKillerPolicy;
}

@Config("query.low-memory-killer.policy")
public MemoryManagerConfig setLowMemoryQueryKillerPolicy(LowMemoryQueryKillerPolicy lowMemoryQueryKillerPolicy)
{
this.lowMemoryQueryKillerPolicy = lowMemoryQueryKillerPolicy;
return this;
}

public LowMemoryTaskKillerPolicy getLowMemoryTaskKillerPolicy()
{
return lowMemoryTaskKillerPolicy;
}

@Config("task.low-memory-killer.policy")
public MemoryManagerConfig setLowMemoryTaskKillerPolicy(LowMemoryTaskKillerPolicy lowMemoryTaskKillerPolicy)
{
this.lowMemoryTaskKillerPolicy = lowMemoryTaskKillerPolicy;
return this;
}

@NotNull
public Duration getKillOnOutOfMemoryDelay()
{
return killOnOutOfMemoryDelay;
}

@Config("query.low-memory-killer.delay")
@ConfigDescription("Delay between cluster running low on memory and invoking killer")
public MemoryManagerConfig setKillOnOutOfMemoryDelay(Duration killOnOutOfMemoryDelay)
{
this.killOnOutOfMemoryDelay = killOnOutOfMemoryDelay;
return this;
}

@NotNull
public DataSize getMaxQueryMemory()
{
Expand Down Expand Up @@ -147,20 +106,6 @@ public MemoryManagerConfig setFaultTolerantExecutionTaskMemory(DataSize faultTol
return this;
}

@NotNull
public DataSize getFaultTolerantExecutionTaskRuntimeMemoryEstimationOverhead()
{
return faultTolerantExecutionTaskRuntimeMemoryEstimationOverhead;
}

@Config("fault-tolerant-execution-task-runtime-memory-estimation-overhead")
@ConfigDescription("Extra memory to account for when estimating actual task runtime memory consumption")
public MemoryManagerConfig setFaultTolerantExecutionTaskRuntimeMemoryEstimationOverhead(DataSize faultTolerantExecutionTaskRuntimeMemoryEstimationOverhead)
{
this.faultTolerantExecutionTaskRuntimeMemoryEstimationOverhead = faultTolerantExecutionTaskRuntimeMemoryEstimationOverhead;
return this;
}

@NotNull
public double getFaultTolerantExecutionTaskMemoryGrowthFactor()
{
Expand Down Expand Up @@ -192,6 +137,20 @@ public MemoryManagerConfig setFaultTolerantExecutionTaskMemoryEstimationQuantile
return this;
}

@NotNull
public DataSize getFaultTolerantExecutionTaskRuntimeMemoryEstimationOverhead()
{
return faultTolerantExecutionTaskRuntimeMemoryEstimationOverhead;
}

@Config("fault-tolerant-execution-task-runtime-memory-estimation-overhead")
@ConfigDescription("Extra memory to account for when estimating actual task runtime memory consumption")
public MemoryManagerConfig setFaultTolerantExecutionTaskRuntimeMemoryEstimationOverhead(DataSize faultTolerantExecutionTaskRuntimeMemoryEstimationOverhead)
{
this.faultTolerantExecutionTaskRuntimeMemoryEstimationOverhead = faultTolerantExecutionTaskRuntimeMemoryEstimationOverhead;
return this;
}

public boolean isFaultTolerantExecutionMemoryRequirementIncreaseOnWorkerCrashEnabled()
{
return faultTolerantExecutionMemoryRequirementIncreaseOnWorkerCrashEnabled;
Expand All @@ -217,6 +176,44 @@ public MemoryManagerConfig setFaultTolerantExecutionEagerSpeculativeTasksNodeMem
return this;
}

public LowMemoryQueryKillerPolicy getLowMemoryQueryKillerPolicy()
{
return lowMemoryQueryKillerPolicy;
}

@Config("query.low-memory-killer.policy")
public MemoryManagerConfig setLowMemoryQueryKillerPolicy(LowMemoryQueryKillerPolicy lowMemoryQueryKillerPolicy)
{
this.lowMemoryQueryKillerPolicy = lowMemoryQueryKillerPolicy;
return this;
}

public LowMemoryTaskKillerPolicy getLowMemoryTaskKillerPolicy()
{
return lowMemoryTaskKillerPolicy;
}

@Config("task.low-memory-killer.policy")
public MemoryManagerConfig setLowMemoryTaskKillerPolicy(LowMemoryTaskKillerPolicy lowMemoryTaskKillerPolicy)
{
this.lowMemoryTaskKillerPolicy = lowMemoryTaskKillerPolicy;
return this;
}

@NotNull
public Duration getKillOnOutOfMemoryDelay()
{
return killOnOutOfMemoryDelay;
}

@Config("query.low-memory-killer.delay")
@ConfigDescription("Delay between cluster running low on memory and invoking killer")
public MemoryManagerConfig setKillOnOutOfMemoryDelay(Duration killOnOutOfMemoryDelay)
{
this.killOnOutOfMemoryDelay = killOnOutOfMemoryDelay;
return this;
}

public void applyFaultTolerantExecutionDefaults()
{
killOnOutOfMemoryDelay = new Duration(0, MINUTES);
Expand Down
28 changes: 14 additions & 14 deletions core/trino-main/src/test/java/io/trino/memory/TestMemoryManagerConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,51 +36,51 @@ public class TestMemoryManagerConfig
public void testDefaults()
{
assertRecordedDefaults(recordDefaults(MemoryManagerConfig.class)
.setLowMemoryQueryKillerPolicy(LowMemoryQueryKillerPolicy.TOTAL_RESERVATION_ON_BLOCKED_NODES)
.setLowMemoryTaskKillerPolicy(LowMemoryTaskKillerPolicy.TOTAL_RESERVATION_ON_BLOCKED_NODES)
.setKillOnOutOfMemoryDelay(new Duration(5, MINUTES))
.setMaxQueryMemory(DataSize.of(20, GIGABYTE))
.setMaxQueryTotalMemory(DataSize.of(40, GIGABYTE))
.setFaultTolerantExecutionCoordinatorTaskMemory(DataSize.of(2, GIGABYTE))
.setFaultTolerantExecutionTaskMemory(DataSize.of(5, GIGABYTE))
.setFaultTolerantExecutionTaskRuntimeMemoryEstimationOverhead(DataSize.of(1, GIGABYTE))
.setFaultTolerantExecutionTaskMemoryGrowthFactor(3.0)
.setFaultTolerantExecutionTaskMemoryEstimationQuantile(0.9)
.setFaultTolerantExecutionTaskRuntimeMemoryEstimationOverhead(DataSize.of(1, GIGABYTE))
.setFaultTolerantExecutionMemoryRequirementIncreaseOnWorkerCrashEnabled(true)
.setFaultTolerantExecutionEagerSpeculativeTasksNodeMemoryOvercommit(DataSize.of(20, GIGABYTE)));
.setFaultTolerantExecutionEagerSpeculativeTasksNodeMemoryOvercommit(DataSize.of(20, GIGABYTE))
.setLowMemoryQueryKillerPolicy(LowMemoryQueryKillerPolicy.TOTAL_RESERVATION_ON_BLOCKED_NODES)
.setLowMemoryTaskKillerPolicy(LowMemoryTaskKillerPolicy.TOTAL_RESERVATION_ON_BLOCKED_NODES)
.setKillOnOutOfMemoryDelay(new Duration(5, MINUTES)));
}

@Test
public void testExplicitPropertyMappings()
{
Map<String, String> properties = ImmutableMap.<String, String>builder()
.put("query.low-memory-killer.policy", "none")
.put("task.low-memory-killer.policy", "none")
.put("query.low-memory-killer.delay", "20s")
.put("query.max-memory", "2GB")
.put("query.max-total-memory", "3GB")
.put("fault-tolerant-execution-coordinator-task-memory", "123GB")
.put("fault-tolerant-execution-task-memory", "2GB")
.put("fault-tolerant-execution-task-runtime-memory-estimation-overhead", "300MB")
.put("fault-tolerant-execution-task-memory-growth-factor", "17.3")
.put("fault-tolerant-execution-task-memory-estimation-quantile", "0.7")
.put("fault-tolerant-execution-task-runtime-memory-estimation-overhead", "300MB")
.put("fault-tolerant-execution.memory-requirement-increase-on-worker-crash-enabled", "false")
.put("fault-tolerant-execution-eager-speculative-tasks-node_memory-overcommit", "21GB")
.put("query.low-memory-killer.policy", "none")
.put("task.low-memory-killer.policy", "none")
.put("query.low-memory-killer.delay", "20s")
.buildOrThrow();

MemoryManagerConfig expected = new MemoryManagerConfig()
.setLowMemoryQueryKillerPolicy(LowMemoryQueryKillerPolicy.NONE)
.setLowMemoryTaskKillerPolicy(LowMemoryTaskKillerPolicy.NONE)
.setKillOnOutOfMemoryDelay(new Duration(20, SECONDS))
.setMaxQueryMemory(DataSize.of(2, GIGABYTE))
.setMaxQueryTotalMemory(DataSize.of(3, GIGABYTE))
.setFaultTolerantExecutionCoordinatorTaskMemory(DataSize.of(123, GIGABYTE))
.setFaultTolerantExecutionTaskMemory(DataSize.of(2, GIGABYTE))
.setFaultTolerantExecutionTaskRuntimeMemoryEstimationOverhead(DataSize.of(300, MEGABYTE))
.setFaultTolerantExecutionTaskMemoryGrowthFactor(17.3)
.setFaultTolerantExecutionTaskMemoryEstimationQuantile(0.7)
.setFaultTolerantExecutionTaskRuntimeMemoryEstimationOverhead(DataSize.of(300, MEGABYTE))
.setFaultTolerantExecutionMemoryRequirementIncreaseOnWorkerCrashEnabled(false)
.setFaultTolerantExecutionEagerSpeculativeTasksNodeMemoryOvercommit(DataSize.of(21, GIGABYTE));
.setFaultTolerantExecutionEagerSpeculativeTasksNodeMemoryOvercommit(DataSize.of(21, GIGABYTE))
.setLowMemoryQueryKillerPolicy(LowMemoryQueryKillerPolicy.NONE)
.setLowMemoryTaskKillerPolicy(LowMemoryTaskKillerPolicy.NONE)
.setKillOnOutOfMemoryDelay(new Duration(20, SECONDS));

assertFullMapping(properties, expected);
}
Expand Down

0 comments on commit df3a531

Please sign in to comment.