From d83bfb0cbd538534023e388ef902137ee2996a14 Mon Sep 17 00:00:00 2001 From: Aaron van Meerten Date: Wed, 2 Feb 2022 20:56:45 -0600 Subject: [PATCH] feature: only allow autoscaling in a direction once a full metric window has elapsed --- src/autoscaler.ts | 76 +++++++++++++++++++++++++++---------------- src/instance_group.ts | 18 ++++++++++ 2 files changed, 66 insertions(+), 28 deletions(-) diff --git a/src/autoscaler.ts b/src/autoscaler.ts index 468f06b..aed4625 100644 --- a/src/autoscaler.ts +++ b/src/autoscaler.ts @@ -122,40 +122,60 @@ export default class AutoscaleProcessor { if (scaleMetrics && scaleMetrics.length > 0) { // check if we should scale up the group if (this.evalScaleConditionForAllPeriods(ctx, scaleMetrics, count, group, 'up')) { - desiredCount = desiredCount + group.scalingOptions.scaleUpQuantity; - if (desiredCount > group.scalingOptions.maxDesired) { - desiredCount = group.scalingOptions.maxDesired; - } + if (await this.instanceGroupManager.allowAutoscalingByDirection(group.name, 'up')) { + // only actually autoscale in this direction if we haven't done so recently within the grace period + desiredCount = desiredCount + group.scalingOptions.scaleUpQuantity; + if (desiredCount > group.scalingOptions.maxDesired) { + desiredCount = group.scalingOptions.maxDesired; + } - await this.audit.saveAutoScalerActionItem(group.name, { - timestamp: Date.now(), - actionType: 'increaseDesiredCount', - count: count, - oldDesiredCount: group.scalingOptions.desiredCount, - newDesiredCount: desiredCount, - scaleMetrics: scaleMetrics.slice(0, group.scalingOptions.scaleUpPeriodsCount), - }); + await this.audit.saveAutoScalerActionItem(group.name, { + timestamp: Date.now(), + actionType: 'increaseDesiredCount', + count: count, + oldDesiredCount: group.scalingOptions.desiredCount, + newDesiredCount: desiredCount, + scaleMetrics: scaleMetrics.slice(0, group.scalingOptions.scaleUpPeriodsCount), + }); - await this.updateDesiredCount(ctx, desiredCount, group); - await this.instanceGroupManager.setAutoScaleGracePeriod(group); + await this.updateDesiredCount(ctx, desiredCount, group); + await Promise.allSettled([ + this.instanceGroupManager.setAutoScaleGracePeriod(group), + this.instanceGroupManager.setAutoScaleGracePeriodByDirection(group, 'up'), + ]); + } else { + ctx.logger.info( + `[AutoScaler] Skipping autoscale to ${desiredCount} for group ${group.name} with ${count} instances, still in grace period`, + ); + } } else if (this.evalScaleConditionForAllPeriods(ctx, scaleMetrics, count, group, 'down')) { // next check if we should scale down the group - desiredCount = group.scalingOptions.desiredCount - group.scalingOptions.scaleDownQuantity; - if (desiredCount < group.scalingOptions.minDesired) { - desiredCount = group.scalingOptions.minDesired; - } + if (await this.instanceGroupManager.allowAutoscalingByDirection(group.name, 'down')) { + // only actually autoscale in this direction if we haven't done so recently within the grace period + desiredCount = group.scalingOptions.desiredCount - group.scalingOptions.scaleDownQuantity; + if (desiredCount < group.scalingOptions.minDesired) { + desiredCount = group.scalingOptions.minDesired; + } - await this.audit.saveAutoScalerActionItem(group.name, { - timestamp: Date.now(), - actionType: 'decreaseDesiredCount', - count: count, - oldDesiredCount: group.scalingOptions.desiredCount, - newDesiredCount: desiredCount, - scaleMetrics: scaleMetrics.slice(0, group.scalingOptions.scaleDownPeriodsCount), - }); + await this.audit.saveAutoScalerActionItem(group.name, { + timestamp: Date.now(), + actionType: 'decreaseDesiredCount', + count: count, + oldDesiredCount: group.scalingOptions.desiredCount, + newDesiredCount: desiredCount, + scaleMetrics: scaleMetrics.slice(0, group.scalingOptions.scaleDownPeriodsCount), + }); - await this.updateDesiredCount(ctx, desiredCount, group); - await this.instanceGroupManager.setAutoScaleGracePeriod(group); + await this.updateDesiredCount(ctx, desiredCount, group); + await Promise.allSettled([ + this.instanceGroupManager.setAutoScaleGracePeriod(group), + this.instanceGroupManager.setAutoScaleGracePeriodByDirection(group, 'down'), + ]); + } else { + ctx.logger.info( + `[AutoScaler] Skipping autoscale to ${desiredCount} for group ${group.name} with ${count} instances, still in grace period`, + ); + } } else { // otherwise neither action is needed ctx.logger.info( diff --git a/src/instance_group.ts b/src/instance_group.ts index 7a1b576..874a8f7 100644 --- a/src/instance_group.ts +++ b/src/instance_group.ts @@ -250,6 +250,24 @@ export default class InstanceGroupManager { return !(result !== null && result.length > 0); } + // keeps the autoscaling activity from occuring if we have scaled in this direction within the grace period + async allowAutoscalingByDirection(group: string, direction: string): Promise { + const result = await this.redisClient.get(`autoScaleGracePeriodByDirection:${group}:${direction}`); + return !(result !== null && result.length > 0); + } + + // starts autoscaling grace period after autoscaling has occurred in this direction + async setAutoScaleGracePeriodByDirection(group: InstanceGroup, direction: string): Promise { + // count of periods in metric window from group based on direction + const scalePeriods = + direction == 'down' ? group.scalingOptions.scaleDownPeriodsCount : group.scalingOptions.scaleUpPeriodsCount; + + // grace period is the greater of group grace period or metric window size + const directionalTTL = Math.max(group.gracePeriodTTLSec, scalePeriods * group.scalingOptions.scalePeriod); + + return this.setValue(`setAutoScaleGracePeriodByDirection:${group.name}:${direction}`, directionalTTL); + } + async setAutoScaleGracePeriod(group: InstanceGroup): Promise { return this.setValue(`autoScaleGracePeriod:${group.name}`, group.gracePeriodTTLSec); }