diff --git a/config/env-config.ts b/config/env-config.ts index a488fc8..c0d397e 100644 --- a/config/env-config.ts +++ b/config/env-config.ts @@ -1,5 +1,5 @@ -import config from './env-config.json'; import * as cdk from 'aws-cdk-lib'; +import config from './env-config.json'; /** * Class for environment configurations. Outlines the account and region. diff --git a/config/runner-config.json b/config/runner-config.json index 06c3fe4..289fd6e 100644 --- a/config/runner-config.json +++ b/config/runner-config.json @@ -9,7 +9,12 @@ "arch": "arm", "repo": "finch", "desiredInstances": 1, - "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] + "availabilityZones": [ + "us-west-2a", + "us-west-2b", + "us-west-2c", + "us-west-2d" + ] }, { "platform": "mac", @@ -17,7 +22,12 @@ "arch": "x86", "repo": "finch-core", "desiredInstances": 1, - "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] + "availabilityZones": [ + "us-west-2a", + "us-west-2b", + "us-west-2c", + "us-west-2d" + ] }, { "platform": "windows", @@ -25,7 +35,12 @@ "arch": "x86", "repo": "finch", "desiredInstances": 1, - "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] + "availabilityZones": [ + "us-west-2a", + "us-west-2b", + "us-west-2c", + "us-west-2d" + ] }, { "platform": "windows", @@ -33,7 +48,64 @@ "arch": "x86", "repo": "finch-core", "desiredInstances": 1, - "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] + "availabilityZones": [ + "us-west-2a", + "us-west-2b", + "us-west-2c", + "us-west-2d" + ] + }, + { + "platform": "amazonlinux", + "version": "2023", + "arch": "x86", + "repo": "finch", + "desiredInstances": 1, + "availabilityZones": [ + "us-west-2a", + "us-west-2b", + "us-west-2c", + "us-west-2d" + ] + }, + { + "platform": "amazonlinux", + "version": "2", + "arch": "x86", + "repo": "finch", + "desiredInstances": 1, + "availabilityZones": [ + "us-west-2a", + "us-west-2b", + "us-west-2c", + "us-west-2d" + ] + }, + { + "platform": "amazonlinux", + "version": "2023", + "arch": "arm", + "repo": "finch", + "desiredInstances": 1, + "availabilityZones": [ + "us-west-2a", + "us-west-2b", + "us-west-2c", + "us-west-2d" + ] + }, + { + "platform": "amazonlinux", + "version": "2", + "arch": "arm", + "repo": "finch", + "desiredInstances": 1, + "availabilityZones": [ + "us-west-2a", + "us-west-2b", + "us-west-2c", + "us-west-2d" + ] } ] }, @@ -47,7 +119,12 @@ "arch": "arm", "repo": "finch", "desiredInstances": 2, - "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] + "availabilityZones": [ + "us-west-2a", + "us-west-2b", + "us-west-2c", + "us-west-2d" + ] }, { "platform": "mac", @@ -55,7 +132,12 @@ "arch": "x86", "repo": "finch", "desiredInstances": 2, - "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] + "availabilityZones": [ + "us-west-2a", + "us-west-2b", + "us-west-2c", + "us-west-2d" + ] }, { "platform": "mac", @@ -63,7 +145,12 @@ "arch": "arm", "repo": "finch", "desiredInstances": 2, - "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] + "availabilityZones": [ + "us-west-2a", + "us-west-2b", + "us-west-2c", + "us-west-2d" + ] }, { "platform": "mac", @@ -71,7 +158,12 @@ "arch": "x86", "repo": "finch", "desiredInstances": 2, - "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] + "availabilityZones": [ + "us-west-2a", + "us-west-2b", + "us-west-2c", + "us-west-2d" + ] }, { "platform": "mac", @@ -79,7 +171,12 @@ "arch": "arm", "repo": "finch-core", "desiredInstances": 1, - "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] + "availabilityZones": [ + "us-west-2a", + "us-west-2b", + "us-west-2c", + "us-west-2d" + ] }, { "platform": "mac", @@ -87,7 +184,12 @@ "arch": "x86", "repo": "finch-core", "desiredInstances": 1, - "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] + "availabilityZones": [ + "us-west-2a", + "us-west-2b", + "us-west-2c", + "us-west-2d" + ] }, { "platform": "mac", @@ -95,7 +197,12 @@ "arch": "arm", "repo": "finch-core", "desiredInstances": 1, - "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] + "availabilityZones": [ + "us-west-2a", + "us-west-2b", + "us-west-2c", + "us-west-2d" + ] }, { "platform": "mac", @@ -103,7 +210,12 @@ "arch": "x86", "repo": "finch-core", "desiredInstances": 1, - "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] + "availabilityZones": [ + "us-west-2a", + "us-west-2b", + "us-west-2c", + "us-west-2d" + ] }, { "platform": "windows", @@ -111,7 +223,12 @@ "arch": "x86", "repo": "finch", "desiredInstances": 2, - "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] + "availabilityZones": [ + "us-west-2a", + "us-west-2b", + "us-west-2c", + "us-west-2d" + ] }, { "platform": "windows", @@ -119,7 +236,64 @@ "arch": "x86", "repo": "finch-core", "desiredInstances": 1, - "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] + "availabilityZones": [ + "us-west-2a", + "us-west-2b", + "us-west-2c", + "us-west-2d" + ] + }, + { + "platform": "amazonlinux", + "version": "2023", + "arch": "x86", + "repo": "finch", + "desiredInstances": 1, + "availabilityZones": [ + "us-west-2a", + "us-west-2b", + "us-west-2c", + "us-west-2d" + ] + }, + { + "platform": "amazonlinux", + "version": "2", + "arch": "x86", + "repo": "finch", + "desiredInstances": 1, + "availabilityZones": [ + "us-west-2a", + "us-west-2b", + "us-west-2c", + "us-west-2d" + ] + }, + { + "platform": "amazonlinux", + "version": "2023", + "arch": "arm", + "repo": "finch", + "desiredInstances": 1, + "availabilityZones": [ + "us-west-2a", + "us-west-2b", + "us-west-2c", + "us-west-2d" + ] + }, + { + "platform": "amazonlinux", + "version": "2", + "arch": "arm", + "repo": "finch", + "desiredInstances": 1, + "availabilityZones": [ + "us-west-2a", + "us-west-2b", + "us-west-2c", + "us-west-2d" + ] } ] }, @@ -133,7 +307,11 @@ "arch": "arm", "repo": "finch", "desiredInstances": 1, - "availabilityZones": ["us-east-2a", "us-east-2b", "us-east-2c"] + "availabilityZones": [ + "us-east-2a", + "us-east-2b", + "us-east-2c" + ] }, { "platform": "mac", @@ -141,7 +319,11 @@ "arch": "x86", "repo": "finch", "desiredInstances": 1, - "availabilityZones": ["us-east-2a", "us-east-2b", "us-east-2c"] + "availabilityZones": [ + "us-east-2a", + "us-east-2b", + "us-east-2c" + ] }, { "platform": "mac", @@ -149,7 +331,11 @@ "arch": "arm", "repo": "finch", "desiredInstances": 1, - "availabilityZones": ["us-east-2a", "us-east-2b", "us-east-2c"] + "availabilityZones": [ + "us-east-2a", + "us-east-2b", + "us-east-2c" + ] }, { "platform": "mac", @@ -157,7 +343,11 @@ "arch": "x86", "repo": "finch", "desiredInstances": 1, - "availabilityZones": ["us-east-2a", "us-east-2b", "us-east-2c"] + "availabilityZones": [ + "us-east-2a", + "us-east-2b", + "us-east-2c" + ] }, { "platform": "mac", @@ -165,7 +355,11 @@ "arch": "arm", "repo": "finch-core", "desiredInstances": 1, - "availabilityZones": ["us-east-2a", "us-east-2b", "us-east-2c"] + "availabilityZones": [ + "us-east-2a", + "us-east-2b", + "us-east-2c" + ] }, { "platform": "mac", @@ -173,7 +367,11 @@ "arch": "x86", "repo": "finch-core", "desiredInstances": 1, - "availabilityZones": ["us-east-2a", "us-east-2b", "us-east-2c"] + "availabilityZones": [ + "us-east-2a", + "us-east-2b", + "us-east-2c" + ] }, { "platform": "windows", @@ -181,7 +379,11 @@ "arch": "x86", "repo": "finch", "desiredInstances": 1, - "availabilityZones": ["us-east-2a", "us-east-2b", "us-east-2c"] + "availabilityZones": [ + "us-east-2a", + "us-east-2b", + "us-east-2c" + ] }, { "platform": "windows", @@ -189,8 +391,12 @@ "arch": "x86", "repo": "finch-core", "desiredInstances": 1, - "availabilityZones": ["us-east-2a", "us-east-2b", "us-east-2c"] + "availabilityZones": [ + "us-east-2a", + "us-east-2b", + "us-east-2c" + ] } ] } -} +} \ No newline at end of file diff --git a/config/runner-config.ts b/config/runner-config.ts index 0047aae..e38f88f 100644 --- a/config/runner-config.ts +++ b/config/runner-config.ts @@ -3,12 +3,18 @@ import config from './runner-config.json'; export interface RunnerProps { macLicenseArn: string; windowsLicenseArn: string; + linuxLicenseArn: string; runnerTypes: Array; } export interface RunnerType { platform: PlatformType; - version: string; // e.g. 13.2 if platform == macOS, 2022 if platform == windows + /** Different values for different platforms. + * For mac, a version like 13.2 + * For windows, a server version like 2022 + * For amazonlinux, either 2, 2023, etc. + * For fedora, a version like 40, 41, etc. */ + version: string; arch: string; repo: string; desiredInstances: number; @@ -17,7 +23,8 @@ export interface RunnerType { export enum PlatformType { WINDOWS = 'windows', - MAC = 'mac' + MAC = 'mac', + AMAZONLINUX = 'amazonlinux', } /** diff --git a/lib/asg-runner-stack.ts b/lib/asg-runner-stack.ts index d31a274..919c984 100644 --- a/lib/asg-runner-stack.ts +++ b/lib/asg-runner-stack.ts @@ -1,19 +1,30 @@ import * as cdk from 'aws-cdk-lib'; +import { aws_autoscaling as autoscaling } from 'aws-cdk-lib'; +import { UpdatePolicy } from 'aws-cdk-lib/aws-autoscaling'; import * as ec2 from 'aws-cdk-lib/aws-ec2'; import * as iam from 'aws-cdk-lib/aws-iam'; import * as resourcegroups from 'aws-cdk-lib/aws-resourcegroups'; -import { aws_autoscaling as autoscaling } from 'aws-cdk-lib'; import { Construct } from 'constructs'; -import { PlatformType, RunnerType } from '../config/runner-config'; import { readFileSync } from 'fs'; -import { ENVIRONMENT_STAGE } from './finch-pipeline-app-stage'; -import { UpdatePolicy } from 'aws-cdk-lib/aws-autoscaling'; +import { PlatformType, RunnerType } from '../config/runner-config'; import { applyTerminationProtectionOnStacks } from './aspects/stack-termination-protection'; +import { ENVIRONMENT_STAGE } from './finch-pipeline-app-stage'; + +interface IASGRunnerStack { + platform: PlatformType; + version: string; + arch: string; + repo: string; +} interface ASGRunnerStackProps extends cdk.StackProps { env: cdk.Environment | undefined; stage: ENVIRONMENT_STAGE; - licenseArn: string; + /** Only required for dedicated hosts. + * Right now, dedicated hosts should only be used to avoid + * nested virtualization issues, which is only a problem for + * non-Linux usecases. */ + licenseArn?: string; type: RunnerType; } @@ -24,16 +35,86 @@ interface ASGRunnerStackProps extends cdk.StackProps { * - a launch template * - an auto scaling group */ -export class ASGRunnerStack extends cdk.Stack { +export class ASGRunnerStack extends cdk.Stack implements IASGRunnerStack { + platform: PlatformType; + version: string; + arch: string; + repo: string; + + requiresDedicatedHosts = () => this.platform === PlatformType.MAC || this.platform === PlatformType.WINDOWS; + + userData = (props: ASGRunnerStackProps, setupScriptName: string) => + `#!/bin/bash + LABEL_STAGE=${props.stage === ENVIRONMENT_STAGE.Release ? 'release' : 'test'} + REPO=${this.repo} + REGION=${props.env?.region} + ` + readFileSync(`./scripts/${setupScriptName}`, 'utf8'); + constructor(scope: Construct, id: string, props: ASGRunnerStackProps) { super(scope, id, props); + + this.platform = props.type.platform; + this.version = props.type.version; + this.arch = props.type.arch; + this.repo = props.type.repo; + applyTerminationProtectionOnStacks([this]); - const platform = props.type.platform; - const arch = props.type.arch === 'arm' ? `arm64_${platform}` : `x86_64_${platform}`; - const instanceType = - platform === PlatformType.WINDOWS ? 'm5zn.metal' : props.type.arch === 'arm' ? 'mac2.metal' : 'mac1.metal'; - const version = props.type.version; + const amiSearchString = `amzn-ec2-macos-${this.version}*`; + + let instanceType: ec2.InstanceType; + let machineImage: ec2.IMachineImage; + let userDataString = ''; + let asgName = ''; + switch (this.platform) { + case PlatformType.MAC: { + if (this.arch === 'arm') { + instanceType = ec2.InstanceType.of(ec2.InstanceClass.MAC2, ec2.InstanceSize.METAL); + } else { + instanceType = ec2.InstanceType.of(ec2.InstanceClass.MAC1, ec2.InstanceSize.METAL); + } + const macOSArchLookup = this.arch === 'arm' ? `arm64_${this.platform}` : `x86_64_${this.platform}`; + machineImage = new ec2.LookupMachineImage({ + name: amiSearchString, + filters: { + 'virtualization-type': ['hvm'], + 'root-device-type': ['ebs'], + architecture: [macOSArchLookup], + 'owner-alias': ['amazon'] + } + }); + asgName = 'MacASG'; + userDataString = this.userData(props, 'setup-runner.sh'); + } + case PlatformType.WINDOWS: { + instanceType = ec2.InstanceType.of(ec2.InstanceClass.M5ZN, ec2.InstanceSize.METAL); + asgName = 'WindowsASG'; + machineImage = ec2.MachineImage.latestWindows(ec2.WindowsVersion.WINDOWS_SERVER_2022_ENGLISH_FULL_BASE); + // We need to provide user data as a yaml file to specify runAs: admin + // Maintain that file as yaml and source here to ensure formatting. + userDataString = readFileSync('./scripts/windows-runner-user-data.yaml', 'utf8') + .replace('', props.stage === ENVIRONMENT_STAGE.Release ? 'release' : 'test') + .replace('', props.type.repo) + .replace('', props.env?.region || ''); + } + case PlatformType.AMAZONLINUX: { + // Linux instances do not have to be metal, since the only mode of operation + // for Finch on linux currently is "native" mode, e.g. no virutal machine on host + + if (this.arch === 'arm') { + instanceType = ec2.InstanceType.of(ec2.InstanceClass.C7G, ec2.InstanceSize.LARGE); + } else { + instanceType = ec2.InstanceType.of(ec2.InstanceClass.C7A, ec2.InstanceSize.LARGE); + } + asgName = 'LinuxASG'; + userDataString = this.userData(props, 'setup-linux-runner.sh'); + if (this.version === '2') { + machineImage = ec2.MachineImage.latestAmazonLinux2(); + } else { + machineImage = ec2.MachineImage.latestAmazonLinux2023(); + } + } + } if (props.env == undefined) { throw new Error('Runner environment is undefined!'); @@ -71,110 +152,49 @@ export class ASGRunnerStack extends cdk.Stack { }) ); - // Create a custom name for this as names for resource groups cannot be repeated - const resourceGroupName = `${props.type.repo}-${platform}-${version.split('.')[0]}-${props.type.arch}HostGroup`; - - const resourceGroup = new resourcegroups.CfnGroup(this, resourceGroupName, { - name: resourceGroupName, - description: 'Host resource group for finchs infrastructure', - configuration: [ - { - type: 'AWS::EC2::HostManagement', - parameters: [ - { - name: 'auto-allocate-host', - values: ['true'] - }, - { - name: 'auto-release-host', - values: ['true'] - }, - { - name: 'any-host-based-license-configuration', - values: ['true'] - } - ] - }, - { - type: 'AWS::ResourceGroups::Generic', - parameters: [ - { - name: 'allowed-resource-types', - values: ['AWS::EC2::Host'] - }, - { - name: 'deletion-protection', - values: ['UNLESS_EMPTY'] - } - ] - } - ] - }); - - const amiSearchString = `amzn-ec2-macos-${version}*`; - const machineImage = - platform === PlatformType.WINDOWS - ? ec2.MachineImage.latestWindows(ec2.WindowsVersion.WINDOWS_SERVER_2022_ENGLISH_FULL_BASE) - : new ec2.LookupMachineImage({ - name: amiSearchString, - filters: { - 'virtualization-type': ['hvm'], - 'root-device-type': ['ebs'], - architecture: [arch], - 'owner-alias': ['amazon'] - } - }); - - var userData = ''; - if (platform === PlatformType.WINDOWS) { - // We need to provide user data as a yaml file to specify runAs: admin - // Maintain that file as yaml and source here to ensure formatting. - userData = readFileSync('./scripts/windows-runner-user-data.yaml', 'utf8') - .replace('', props.stage === ENVIRONMENT_STAGE.Release ? 'release' : 'test') - .replace('', props.type.repo) - .replace('', props.env?.region || ''); - } else if (platform === PlatformType.MAC) { - userData = - `#!/bin/bash - LABEL_STAGE=${props.stage === ENVIRONMENT_STAGE.Release ? 'release' : 'test'} - REPO=${props.type.repo} - REGION=${props.env?.region} - ` + readFileSync('./scripts/setup-runner.sh', 'utf8'); - } - // Create a 100GiB volume to be used as instance root volume const rootVolume: ec2.BlockDevice = { deviceName: '/dev/sda1', - volume: ec2.BlockDeviceVolume.ebs(100), + volume: ec2.BlockDeviceVolume.ebs(100) }; - const asgName = platform === PlatformType.WINDOWS ? 'WindowsASG' : 'MacASG'; - const ltName = `${asgName}LaunchTemplate` + const ltName = `${asgName}LaunchTemplate`; + const keyPairName = `${asgName}KeyPair`; const lt = new ec2.LaunchTemplate(this, ltName, { requireImdsv2: true, - instanceType: new ec2.InstanceType(instanceType), - keyName: 'runner-key', - machineImage: machineImage, + instanceType, + keyPair: ec2.KeyPair.fromKeyPairName(this, keyPairName, 'runner-key'), + machineImage, role: role, securityGroup: securityGroup, - userData: ec2.UserData.custom(userData), + userData: ec2.UserData.custom(userDataString), blockDevices: [rootVolume] }); + // Create a custom name for this as names for resource groups cannot be repeated + const resourceGroupName = `${this.repo}-${this.platform}-${this.version.split('.')[0]}-${this.arch}HostGroup`; + const resourceGroupDescription = 'Host resource group for finchs infrastructure'; + + let ltPlacementConfig = {}; + if (this.requiresDedicatedHosts()) { + const hostResourceGroup = this.createHostResourceGroup(resourceGroupName, resourceGroupDescription); + ltPlacementConfig = { + placement: { + tenancy: 'host', + hostResourceGroupArn: hostResourceGroup.attrArn + } + }; + } + // Escape hatch to cfnLaunchTemplate as the L2 construct lacked some required // configurations. const cfnLt = lt.node.defaultChild as ec2.CfnLaunchTemplate; cfnLt.launchTemplateData = { ...cfnLt.launchTemplateData, - placement: { - tenancy: 'host', - hostResourceGroupArn: resourceGroup.attrArn - }, - licenseSpecifications: [ - { - licenseConfigurationArn: props.licenseArn - } - ], + ...(this.requiresDedicatedHosts() && { + ...ltPlacementConfig, + licenseSpecifications: [{ licenseConfigurationArn: props.licenseArn }] + }), tagSpecifications: [ { resourceType: 'instance', @@ -207,18 +227,85 @@ export class ASGRunnerStack extends cdk.Stack { autoscaling.ScalingProcess.REPLACE_UNHEALTHY, autoscaling.ScalingProcess.AZ_REBALANCE, autoscaling.ScalingProcess.ALARM_NOTIFICATION, - autoscaling.ScalingProcess.SCHEDULED_ACTIONS, + autoscaling.ScalingProcess.SCHEDULED_ACTIONS ], - waitOnResourceSignals: false, + waitOnResourceSignals: false }) }); + if (!this.requiresDedicatedHosts()) { + this.createTagBasedResourceGroup(resourceGroupName, resourceGroupDescription, asg.autoScalingGroupName); + } + if (props.stage === ENVIRONMENT_STAGE.Beta) { - const scheduledAction = new autoscaling.CfnScheduledAction(this, 'SpinDownBetaInstances', { + new autoscaling.CfnScheduledAction(this, 'SpinDownBetaInstances', { autoScalingGroupName: asg.autoScalingGroupName, - startTime: new Date(new Date().getTime() + 1 * 24 * 60 * 60 * 1000).toISOString(), // 1 day from now + // 1 day from now + startTime: new Date(new Date().getTime() + 1 * 24 * 60 * 60 * 1000).toISOString(), desiredCapacity: 0 }); } } + + // a host resource group is used by the launch template for placement of instances on dedicated hosts + createHostResourceGroup(resourceGroupName: string, resourceGroupDescription: string) { + return new resourcegroups.CfnGroup(this, resourceGroupName, { + name: resourceGroupName, + description: resourceGroupDescription, + configuration: [ + { + // This resource group is only used for management of dedicated hosts, as indicated by + // the "AWS::EC2::HostManagement" type + type: 'AWS::EC2::HostManagement', + parameters: [ + { + name: 'auto-allocate-host', + values: ['true'] + }, + { + name: 'auto-release-host', + values: ['true'] + }, + { + name: 'any-host-based-license-configuration', + values: ['true'] + } + ] + }, + { + type: 'AWS::ResourceGroups::Generic', + parameters: [ + { + name: 'allowed-resource-types', + values: ['AWS::EC2::Host'] + }, + { + name: 'deletion-protection', + values: ['UNLESS_EMPTY'] + } + ] + } + ] + }); + } + + // tag based resource groups filter EC2 instances by tag, anything matching will be included in the group + createTagBasedResourceGroup(resourceGroupName: string, resourceGroupDescription: string, asgName: string) { + return new resourcegroups.CfnGroup(this, resourceGroupName, { + name: resourceGroupName, + description: resourceGroupDescription, + resourceQuery: { + type: 'TAG_FILTERS_1_0', + query: { + resourceTypeFilters: ['AWS::EC2::Instance'], + tagFilters: [ + { + key: 'aws:autoscaling:groupName', + values: [asgName] + } + ] + } + } + }); + } } diff --git a/lib/finch-pipeline-app-stage.ts b/lib/finch-pipeline-app-stage.ts index 038ae84..b3da3c6 100644 --- a/lib/finch-pipeline-app-stage.ts +++ b/lib/finch-pipeline-app-stage.ts @@ -1,15 +1,15 @@ import * as cdk from 'aws-cdk-lib'; import { CfnOutput } from 'aws-cdk-lib'; -import { Construct } from 'constructs'; -import { ArtifactBucketCloudfrontStack } from './artifact-bucket-cloudfront'; import * as ecr from 'aws-cdk-lib/aws-ecr'; import * as s3 from 'aws-cdk-lib/aws-s3'; +import { Construct } from 'constructs'; +import { PlatformType, RunnerProps } from '../config/runner-config'; +import { ArtifactBucketCloudfrontStack } from './artifact-bucket-cloudfront'; import { ASGRunnerStack } from './asg-runner-stack'; import { ContinuousIntegrationStack } from './continuous-integration-stack'; import { ECRRepositoryStack } from './ecr-repo-stack'; -import { PVREReportingStack } from './pvre-reporting-stack'; -import { PlatformType, RunnerProps } from '../config/runner-config'; import { EventBridgeScanNotifsStack } from './event-bridge-scan-notifs-stack'; +import { PVREReportingStack } from './pvre-reporting-stack'; export enum ENVIRONMENT_STAGE { Beta, @@ -32,12 +32,20 @@ export class FinchPipelineAppStage extends cdk.Stage { super(scope, id, props); props.runnerConfig.runnerTypes.forEach((runnerType) => { const ASGStackName = `ASG-${runnerType.platform}-${runnerType.repo}-${runnerType.version.split('.')[0]}-${runnerType.arch}Stack`; - const licenseArn = runnerType.platform === PlatformType.WINDOWS ? props.runnerConfig.windowsLicenseArn : props.runnerConfig.macLicenseArn; - new ASGRunnerStack(this, ASGStackName , { + let licenseArn: string | undefined; + switch (runnerType.platform) { + case PlatformType.MAC: { + licenseArn = props.runnerConfig.macLicenseArn; + } + case PlatformType.WINDOWS: { + licenseArn = props.runnerConfig.windowsLicenseArn; + } + } + new ASGRunnerStack(this, ASGStackName, { env: props.env, stage: props.environmentStage, - licenseArn: licenseArn, - type: runnerType + type: runnerType, + licenseArn }); }); @@ -50,28 +58,21 @@ export class FinchPipelineAppStage extends cdk.Stage { this.artifactBucketCloudfrontUrlOutput = artifactBucketCloudfrontStack.urlOutput; this.cloudfrontBucket = artifactBucketCloudfrontStack.bucket; - const ecrRepositoryStack = new ECRRepositoryStack( - this, - 'ECRRepositoryStack', - this.stageName - ); + const ecrRepositoryStack = new ECRRepositoryStack(this, 'ECRRepositoryStack', this.stageName); this.ecrRepositoryOutput = ecrRepositoryStack.repositoryOutput; this.ecrRepository = ecrRepositoryStack.repository; // Only report rootfs image scans in prod to avoid duplicate notifications. if (props.environmentStage == ENVIRONMENT_STAGE.Prod) { - new EventBridgeScanNotifsStack(this, 'EventBridgeScanNotifsStack', this.stageName) + new EventBridgeScanNotifsStack(this, 'EventBridgeScanNotifsStack', this.stageName); } - new ContinuousIntegrationStack( - this, - 'FinchContinuousIntegrationStack', - this.stageName, - { rootfsEcrRepository: this.ecrRepository } - ); + new ContinuousIntegrationStack(this, 'FinchContinuousIntegrationStack', this.stageName, { + rootfsEcrRepository: this.ecrRepository + }); } - new PVREReportingStack(this, 'PVREReportingStack', { terminationProtection:true }); + new PVREReportingStack(this, 'PVREReportingStack', { terminationProtection: true }); } } diff --git a/lib/finch-pipeline-stack.ts b/lib/finch-pipeline-stack.ts index 10cb807..fcb4702 100644 --- a/lib/finch-pipeline-stack.ts +++ b/lib/finch-pipeline-stack.ts @@ -4,8 +4,8 @@ import { CodePipeline, CodePipelineSource, ShellStep } from 'aws-cdk-lib/pipelin import { Construct } from 'constructs'; import { EnvConfig } from '../config/env-config'; import { RunnerConfig } from '../config/runner-config'; -import { ENVIRONMENT_STAGE, FinchPipelineAppStage } from './finch-pipeline-app-stage'; import { applyTerminationProtectionOnStacks } from './aspects/stack-termination-protection'; +import { ENVIRONMENT_STAGE, FinchPipelineAppStage } from './finch-pipeline-app-stage'; export class FinchPipelineStack extends cdk.Stack { constructor(scope: Construct, id: string, props?: cdk.StackProps) { diff --git a/scripts/setup-linux-runner.sh b/scripts/setup-linux-runner.sh new file mode 100644 index 0000000..2cad1a5 --- /dev/null +++ b/scripts/setup-linux-runner.sh @@ -0,0 +1,94 @@ +#!/usr/bin/bash + +# The user data script is run as root so do not use sudo command + +# Log output +exec &>/var/log/setup-runner.log +echo $0 + +# load all variables from /etc/os-release prefixed with OS_RELEASE as to not clobber +OS_RELEASE_PREFIX="OS_RELEASE" +source <(sed -Ee "s/^([^#])/${OS_RELEASE_PREFIX}_\1/" "/etc/os-release") + +# set variables to make accessing values in /etc/os-release easier +eval "OS_NAME=\"\${${OS_RELEASE_PREFIX}_NAME}\"" +eval "OS_VERSION=\"\${${OS_RELEASE_PREFIX}_VERSION_ID}\"" + +if [ "${OS_NAME}" = "Amazon Linux" ]; then + USERNAME="ec2-user" + DISTRO="amazonlinux" + BASE_PACKAGES="golang zlib-static containerd nerdctl cni-plugins iptables" + if [ "${OS_VERSION}" = "2" ]; then + GH_RUNNER_DEPENDENCIES="openssl krb5-libs zlib jq" + ADDITIONAL_PACKAGES="policycoreutils-python systemd-rpm-macros ${GH_RUNNER_DEPENDENCIES}" + elif [ "${OS_VERSION}" = "2023" ]; then + GH_RUNNER_DEPENDENCIES="lttng-ust openssl-libs krb5-libs zlib libicu" + ADDITIONAL_PACKAGES="policycoreutils-python-utils ${GH_RUNNER_DEPENDENCIES}" + fi +fi + +HOMEDIR="/home/${USERNAME}" +RUNNER_DIR="${HOMEDIR}/ar" +mkdir -p "${RUNNER_DIR}" && cd "${HOMEDIR}" + +# TODO: add check for non-Fedora based systems if needed +yum upgrade +yum group install -y "Development Tools" +# build dependencies for packages +# this sometimes fails on Amazon Linux 2023, so retry if necessary +for i in {1..2}; do + yum install -y ${BASE_PACKAGES} ${ADDITIONAL_PACKAGES} && break || sleep 5 +done + +# start containerd +systemctl enable --now containerd + +# configure download parameters based on architecture +UNAME_MACHINE="$(/usr/bin/uname -m)" +if [ "${UNAME_MACHINE}" = "aarch64" ]; then + GH_RUNNER_ARCH="arm64" + GH_RUNNER_DOWNLOAD_HASH="524e75dc384ba8289fcea4914eb210f10c8c4e143213cef7d28f0c84dd2d017c" +else + GH_RUNNER_ARCH="x64" + GH_RUNNER_DOWNLOAD_HASH="52b8f9c5abb1a47cc506185a1a20ecea19daf0d94bbf4ddde7e617e7be109b14" +fi + +GH_RUNNER_FILENAME="actions-runner-linux-${GH_RUNNER_ARCH}-2.319.0.tar.gz" +GH_RUNNER_DOWNLOAD_URL="https://github.com/actions/runner/releases/download/v2.319.0/${GH_RUNNER_FILENAME}" + +curl -OL "${GH_RUNNER_DOWNLOAD_URL}" +echo "${GH_RUNNER_DOWNLOAD_HASH} ${GH_RUNNER_FILENAME}" | sha256sum -c +tar -C "${RUNNER_DIR}" -xzf "./${GH_RUNNER_FILENAME}" +chown -R "${USERNAME}:${USERNAME}" "${RUNNER_DIR}" +rm "${GH_RUNNER_FILENAME}" + +# TODO: install SSM agent on non-AL hosts if needed + +# Get GH API key and fetch a runner registration token +GH_KEY=$(aws secretsmanager get-secret-value --secret-id $REPO-runner-reg-key --region $REGION | jq '.SecretString' -r) +RUNNER_REG_TOKEN=$(curl -L -s \ + -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $GH_KEY" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/runfinch/${REPO}/actions/runners/registration-token" | jq -r '.token') + +if [ -z ${GH_RUNNER_DEPENDENCIES+x} ]; then + echo "Executing installdependencies.sh because GH_RUNNER_DEPENDENCIES is not defined." + "${RUNNER_DIR}/bin/installdependencies.sh" +fi + +# Configure the runner with the registration token, launch the service +# these commands must NOT be run as root +sudo -i -u "${USERNAME}" bash < `ASG-${runnerType.platform}-${runnerType.repo}-${runnerType.version.split('.')[0]}-${runnerType.arch}Stack`; @@ -46,10 +46,29 @@ describe('ASGRunnerStack test', () => { const stack = stacks.find((stack) => stack.stackName === generateASGStackName(type)); expect(stack).toBeDefined(); const template = Template.fromStack(stack!); + let instanceType = ''; + switch (type.platform) { + case PlatformType.WINDOWS: { + instanceType = 'm5zn.metal'; + } + case PlatformType.MAC: { + if (type.arch === 'arm') { + instanceType = 'mac2.metal'; + } else { + instanceType = 'mac1.metal'; + } + } + case PlatformType.AMAZONLINUX: { + if (type.arch === 'arm') { + instanceType = 'c7g.large'; + } else { + instanceType = 'c7a.large'; + } + } + } template.hasResourceProperties('AWS::EC2::LaunchTemplate', { LaunchTemplateData: { - InstanceType: - type.platform === PlatformType.WINDOWS ? 'm5zn.metal' : type.arch === 'arm' ? 'mac2.metal' : 'mac1.metal' + InstanceType: instanceType } }); }); @@ -58,6 +77,6 @@ describe('ASGRunnerStack test', () => { it('must have termination protection enabled', () => { stacks.forEach((stack) => { expect(stack.terminationProtection).toBeTruthy(); - }) + }); }); });