From d8f5ae2a34b4a59059ccfe420c4224a8c465207c Mon Sep 17 00:00:00 2001 From: Gavin Inglis Date: Wed, 20 Sep 2023 10:11:49 -0700 Subject: [PATCH] feat: add windows runner instances * change RunnerType to include platform * refactor asg-runner-stack to build for both platform types macOS and windows * add windows-runner-user-data.yaml with PowerShell script to set up runner * update runner-config.json to include Windows runners for beta, prod, and release * add licenses for Windows Signed-off-by: Gavin Inglis --- config/runner-config.json | 117 +++++++++++++++++++++----- config/runner-config.ts | 11 ++- lib/asg-runner-stack.ts | 72 ++++++++++------ lib/finch-pipeline-app-stage.ts | 7 +- scripts/windows-runner-user-data.yaml | 100 ++++++++++++++++++++++ 5 files changed, 251 insertions(+), 56 deletions(-) create mode 100644 scripts/windows-runner-user-data.yaml diff --git a/config/runner-config.json b/config/runner-config.json index c4b26fd5..12639973 100644 --- a/config/runner-config.json +++ b/config/runner-config.json @@ -1,152 +1,223 @@ { "runnerBeta": { - "licenseArn": "arn:aws:license-manager:us-west-2:820462304213:license-configuration:lic-7ac3e249bd13f71b44ac34963a8c3353", + "macLicenseArn": "arn:aws:license-manager:us-west-2:820462304213:license-configuration:lic-7ac3e249bd13f71b44ac34963a8c3353", + "windowsLicenseArn": "arn:aws:license-manager:us-west-2:820462304213:license-configuration:lic-56c55a611976d66a1357e58077b19e9", "runnerTypes": [ { - "macOSVersion": "13.2", + "platform": "mac", + "version": "13.2", "arch": "arm", "repo": "finch", "desiredInstances": 1, "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] }, { - "macOSVersion": "13.2", + "platform": "mac", + "version": "13.2", "arch": "x86", "repo": "finch-core", "desiredInstances": 1, "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] + }, + { + "platform": "windows", + "version": "2022", + "arch": "x86", + "repo": "finch", + "desiredInstances": 1, + "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] + }, + { + "platform": "windows", + "version": "2022", + "arch": "x86", + "repo": "finch-core", + "desiredInstances": 1, + "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] } ] }, "runnerProd": { - "licenseArn": "arn:aws:license-manager:us-west-2:090529234398:license-configuration:lic-c0c3e2458f6d2a45b0bcc8d66fbc072e", + "macLicenseArn": "arn:aws:license-manager:us-west-2:090529234398:license-configuration:lic-c0c3e2458f6d2a45b0bcc8d66fbc072e", + "windowsLicenseArn": "arn:aws:license-manager:us-west-2:090529234398:license-configuration:lic-62c55a61d2d9ce6ef100c6ea267216e6", "runnerTypes": [ { - "macOSVersion": "13.2", + "platform": "mac", + "version": "13.2", "arch": "arm", "repo": "finch", "desiredInstances": 2, "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] }, { - "macOSVersion": "13.2", + "platform": "mac", + "version": "13.2", "arch": "x86", "repo": "finch", "desiredInstances": 2, "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] }, { - "macOSVersion": "12.6", + "platform": "mac", + "version": "12.6", "arch": "arm", "repo": "finch", "desiredInstances": 2, "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] }, { - "macOSVersion": "12.6", + "platform": "mac", + "version": "12.6", "arch": "x86", "repo": "finch", "desiredInstances": 2, "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] }, { - "macOSVersion": "13.2", + "platform": "mac", + "version": "13.2", "arch": "arm", "repo": "finch-core", "desiredInstances": 1, "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] }, { - "macOSVersion": "13.2", + "platform": "mac", + "version": "13.2", "arch": "x86", "repo": "finch-core", "desiredInstances": 1, "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] }, { - "macOSVersion": "12.6", + "platform": "mac", + "version": "12.6", "arch": "arm", "repo": "finch-core", "desiredInstances": 1, "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] }, { - "macOSVersion": "12.6", + "platform": "mac", + "version": "12.6", "arch": "x86", "repo": "finch-core", "desiredInstances": 1, "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] + }, + { + "platform": "windows", + "version": "2022", + "arch": "x86", + "repo": "finch", + "desiredInstances": 1, + "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] + }, + { + "platform": "windows", + "version": "2022", + "arch": "x86", + "repo": "finch-core", + "desiredInstances": 1, + "availabilityZones": ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"] } ] }, "runnerRelease": { - "licenseArn": "arn:aws:license-manager:us-east-2:019528120233:license-configuration:lic-94c3e244cb74a70c90c82f432e9e0d91", + "macLicenseArn": "arn:aws:license-manager:us-east-2:019528120233:license-configuration:lic-94c3e244cb74a70c90c82f432e9e0d91", + "windowsLicenseArn": "arn:aws:license-manager:us-east-2:019528120233:license-configuration:lic-2ec55a6280292474f1ea0d7ad6f64987", "runnerTypes": [ { - "macOSVersion": "13.2", + "platform": "mac", + "version": "13.2", "arch": "arm", "repo": "finch", "desiredInstances": 1, "availabilityZones": ["us-east-2a", "us-east-2b", "us-east-2c"] }, { - "macOSVersion": "13.2", + "platform": "mac", + "version": "13.2", "arch": "x86", "repo": "finch", "desiredInstances": 1, "availabilityZones": ["us-east-2a", "us-east-2b", "us-east-2c"] }, { - "macOSVersion": "12.6", + "platform": "mac", + "version": "12.6", "arch": "arm", "repo": "finch", "desiredInstances": 1, "availabilityZones": ["us-east-2a", "us-east-2b", "us-east-2c"] }, { - "macOSVersion": "12.6", + "platform": "mac", + "version": "12.6", "arch": "x86", "repo": "finch", "desiredInstances": 1, "availabilityZones": ["us-east-2a", "us-east-2b", "us-east-2c"] }, { - "macOSVersion": "11.7", + "platform": "mac", + "version": "11.7", "arch": "arm", "repo": "finch", "desiredInstances": 1, "availabilityZones": ["us-east-2a", "us-east-2b", "us-east-2c"] }, { - "macOSVersion": "11.7", + "platform": "mac", + "version": "11.7", "arch": "x86", "repo": "finch", "desiredInstances": 1, "availabilityZones": ["us-east-2a", "us-east-2b", "us-east-2c"] }, { - "macOSVersion": "13.2", + "platform": "mac", + "version": "13.2", "arch": "arm", "repo": "finch-core", "desiredInstances": 1, "availabilityZones": ["us-east-2a", "us-east-2b", "us-east-2c"] }, { - "macOSVersion": "13.2", + "platform": "mac", + "version": "13.2", "arch": "x86", "repo": "finch-core", "desiredInstances": 1, "availabilityZones": ["us-east-2a", "us-east-2b", "us-east-2c"] }, { - "macOSVersion": "11.7", + "platform": "mac", + "version": "11.7", "arch": "arm", "repo": "finch-core", "desiredInstances": 1, "availabilityZones": ["us-east-2a", "us-east-2b", "us-east-2c"] }, { - "macOSVersion": "11.7", + "platform": "mac", + "version": "11.7", + "arch": "x86", + "repo": "finch-core", + "desiredInstances": 1, + "availabilityZones": ["us-east-2a", "us-east-2b", "us-east-2c"] + }, + { + "platform": "windows", + "version": "2022", + "arch": "x86", + "repo": "finch", + "desiredInstances": 1, + "availabilityZones": ["us-east-2a", "us-east-2b", "us-east-2c"] + }, + { + "platform": "windows", + "version": "2022", "arch": "x86", "repo": "finch-core", "desiredInstances": 1, diff --git a/config/runner-config.ts b/config/runner-config.ts index 516b322e..0047aaea 100644 --- a/config/runner-config.ts +++ b/config/runner-config.ts @@ -1,18 +1,25 @@ import config from './runner-config.json'; export interface RunnerProps { - licenseArn: string; + macLicenseArn: string; + windowsLicenseArn: string; runnerTypes: Array; } export interface RunnerType { - macOSVersion: string; + platform: PlatformType; + version: string; // e.g. 13.2 if platform == macOS, 2022 if platform == windows arch: string; repo: string; desiredInstances: number; availabilityZones: Array; } +export enum PlatformType { + WINDOWS = 'windows', + MAC = 'mac' +} + /** * Class for runner configurations. Outlines self hosted license arn and an * array of runner types to create using an auto scaling group. diff --git a/lib/asg-runner-stack.ts b/lib/asg-runner-stack.ts index 9cdf550a..03668020 100644 --- a/lib/asg-runner-stack.ts +++ b/lib/asg-runner-stack.ts @@ -4,7 +4,7 @@ import * as iam from 'aws-cdk-lib/aws-iam'; import * as resourcegroups from 'aws-cdk-lib/aws-resourcegroups'; import { aws_autoscaling as autoscaling } from 'aws-cdk-lib'; import { Construct } from 'constructs'; -import { RunnerType } from '../config/runner-config'; +import { PlatformType, RunnerType } from '../config/runner-config'; import { readFileSync } from 'fs'; import { ENVIRONMENT_STAGE } from './finch-pipeline-app-stage'; import { UpdatePolicy } from 'aws-cdk-lib/aws-autoscaling'; @@ -26,8 +26,12 @@ interface ASGRunnerStackProps extends cdk.StackProps { export class ASGRunnerStack extends cdk.Stack { constructor(scope: Construct, id: string, props: ASGRunnerStackProps) { super(scope, id, props); - const arch = props.type.arch === 'arm' ? 'arm64_mac' : 'x86_64_mac'; - const instanceType = props.type.arch === 'arm' ? 'mac2.metal' : 'mac1.metal'; + + const platform = props.type.platform; + const arch = props.type.arch === 'arm' ? `arm64_${platform}` : `x86_64_${platform}`; + const instanceType = + platform === PlatformType.WINDOWS ? 'm5zn.metal' : props.type.arch === 'arm' ? 'mac2.metal' : 'mac1.metal'; + const version = props.type.version; if (props.env == undefined) { throw new Error('Runner environment is undefined!'); @@ -35,13 +39,13 @@ export class ASGRunnerStack extends cdk.Stack { const vpc = cdk.aws_ec2.Vpc.fromLookup(this, 'VPC', { isDefault: true }); - const securityGroup = new ec2.SecurityGroup(this, 'MacEC2SecurityGroup', { + const securityGroup = new ec2.SecurityGroup(this, 'EC2SecurityGroup', { vpc, description: 'Allow only outbound traffic', allowAllOutbound: true }); - const role = new iam.Role(this, 'MacEC2Role', { + const role = new iam.Role(this, 'EC2Role', { assumedBy: new iam.ServicePrincipal('ec2.amazonaws.com') }); @@ -65,8 +69,7 @@ export class ASGRunnerStack extends cdk.Stack { ); // Create a custom name for this as names for resource groups cannot be repeated - const resourceGroupName = - props.type.repo + '-' + 'Mac' + '-' + props.type.macOSVersion.split('.')[0] + '-' + props.type.arch + 'HostGroup'; + const resourceGroupName = `${props.type.repo}-${platform}-${version.split('.')[0]}-${props.type.arch}HostGroup`; const resourceGroup = new resourcegroups.CfnGroup(this, resourceGroupName, { name: resourceGroupName, @@ -105,30 +108,42 @@ export class ASGRunnerStack extends cdk.Stack { ] }); - const macOSVersion = props.type.macOSVersion; - const amiSearchString = `amzn-ec2-macos-${macOSVersion}*`; - const macImage = new ec2.LookupMachineImage({ - name: amiSearchString, - filters: { - 'virtualization-type': ['hvm'], - 'root-device-type': ['ebs'], - architecture: [arch], - 'owner-alias': ['amazon'] - } - }); - - const userData = - `#!/bin/bash - LABEL_STAGE=${props.stage === ENVIRONMENT_STAGE.Release ? 'release' : 'test'} - REPO=${props.type.repo} - REGION=${props.env?.region} - ` + readFileSync('./scripts/setup-runner.sh', 'utf8'); + const amiSearchString = `amzn-ec2-macos-${version}*`; + const machineImage = + platform === PlatformType.WINDOWS + ? ec2.MachineImage.latestWindows(ec2.WindowsVersion.WINDOWS_SERVER_2022_ENGLISH_FULL_BASE) + : new ec2.LookupMachineImage({ + name: amiSearchString, + filters: { + 'virtualization-type': ['hvm'], + 'root-device-type': ['ebs'], + architecture: [arch], + 'owner-alias': ['amazon'] + } + }); + + var userData = ''; + if (platform === PlatformType.WINDOWS) { + // We need to provide user data as a yaml file to specify runAs: admin + // Maintain that file as yaml and source here to ensure formatting. + userData = readFileSync('./scripts/windows-runner-user-data.yaml', 'utf8') + .replace('', props.stage === ENVIRONMENT_STAGE.Release ? 'release' : 'test') + .replace('', props.type.repo) + .replace('', props.env?.region || ''); + } else if (platform === PlatformType.MAC) { + userData = + `#!/bin/bash + LABEL_STAGE=${props.stage === ENVIRONMENT_STAGE.Release ? 'release' : 'test'} + REPO=${props.type.repo} + REGION=${props.env?.region} + ` + readFileSync('./scripts/setup-runner.sh', 'utf8'); + } - const lt = new ec2.LaunchTemplate(this, 'MacASGLaunchTemplate', { + const lt = new ec2.LaunchTemplate(this, `${platform}ASGLaunchTemplate`, { requireImdsv2: true, instanceType: new ec2.InstanceType(instanceType), keyName: 'runner-key', - machineImage: macImage, + machineImage: machineImage, role: role, securityGroup: securityGroup, userData: ec2.UserData.custom(userData) @@ -161,7 +176,8 @@ export class ASGRunnerStack extends cdk.Stack { ] }; - const asg = new autoscaling.AutoScalingGroup(this, 'MacASG', { + const asgName = platform === PlatformType.WINDOWS ? 'WindowsASG' : 'MacASG'; + const asg = new autoscaling.AutoScalingGroup(this, asgName, { vpc, desiredCapacity: props.type.desiredInstances, maxCapacity: props.type.desiredInstances + 2, diff --git a/lib/finch-pipeline-app-stage.ts b/lib/finch-pipeline-app-stage.ts index 196d012a..7a37d7f0 100644 --- a/lib/finch-pipeline-app-stage.ts +++ b/lib/finch-pipeline-app-stage.ts @@ -8,7 +8,7 @@ import { ASGRunnerStack } from './asg-runner-stack'; import { ContinuousIntegrationStack } from './continuous-integration-stack'; import { ECRRepositoryStack } from './ecr-repo-stack'; import { PVREReportingStack } from './pvre-reporting-stack'; -import { RunnerProps } from '../config/runner-config'; +import { PlatformType, RunnerProps } from '../config/runner-config'; export enum ENVIRONMENT_STAGE { Beta, @@ -30,11 +30,12 @@ export class FinchPipelineAppStage extends cdk.Stage { constructor(scope: Construct, id: string, props: FinchPipelineAppStageProps) { super(scope, id, props); props.runnerConfig.runnerTypes.forEach((runnerType) => { - const ASGStackName = 'ASG' + '-' + runnerType.repo + '-' + runnerType.macOSVersion.split('.')[0] + '-' + runnerType.arch + 'Stack' + const ASGStackName = `ASG-${runnerType.platform}-${runnerType.repo}-${runnerType.version.split('.')[0]}-${runnerType.arch}Stack`; + const licenseArn = runnerType.platform === PlatformType.WINDOWS ? props.runnerConfig.windowsLicenseArn : props.runnerConfig.macLicenseArn; new ASGRunnerStack(this, ASGStackName , { env: props.env, stage: props.environmentStage, - licenseArn: props.runnerConfig.licenseArn, + licenseArn: licenseArn, type: runnerType }); }); diff --git a/scripts/windows-runner-user-data.yaml b/scripts/windows-runner-user-data.yaml new file mode 100644 index 00000000..f31b4184 --- /dev/null +++ b/scripts/windows-runner-user-data.yaml @@ -0,0 +1,100 @@ +version: 1.0 +tasks: +- task: executeScript + inputs: + - frequency: always + type: powershell + runAs: admin + content: |- + # User data script for self hosted Windows runners + + $LABEL_STAGE="" + $REPO="" + $REGION="" + + Start-Transcript -Path "C:\UserData.log" -Append + + $progressPreference = 'silentlyContinue' + + $RUNNER_DIR="C:\actions-runner" + + Write-Information "Installing latest powershell7 version..." + Invoke-Expression "& { $(Invoke-RestMethod 'https://aka.ms/install-powershell.ps1') } -useMSI -EnablePSRemoting -Quiet" + + Write-Information "Starting powershell7..." + New-ItemProperty -Path "HKLM:\SOFTWARE\OpenSSH" -Name DefaultShell -Value "C:\Program Files\PowerShell\7\pwsh.exe" -PropertyType String -Force + + New-Item -Path $Profile -ItemType File -Force + 'Set-PSReadLineOption -EditMode Emacs' | Out-File -Append $Profile + + New-Item -Path $Home\setup -ItemType Directory + Set-Location $Home\setup + + Write-Information "Installing winget and its dependencies..." + + Add-AppxPackage 'https://aka.ms/Microsoft.VCLibs.x64.14.00.Desktop.appx' + Invoke-WebRequest -Uri https://www.nuget.org/api/v2/package/Microsoft.UI.Xaml/2.7.3 -OutFile .\microsoft.ui.xaml.2.7.3.zip + Expand-Archive .\microsoft.ui.xaml.2.7.3.zip + Add-AppxPackage .\microsoft.ui.xaml.2.7.3\tools\AppX\x64\Release\Microsoft.UI.Xaml.2.7.appx + + Invoke-WebRequest -Uri https://github.com/microsoft/terminal/releases/download/v1.17.11461.0/Microsoft.WindowsTerminal_1.17.11461.0_8wekyb3d8bbwe.msixbundle -OutFile .\Microsoft.WindowsTerminal_1.17.11461.0_8wekyb3d8bbwe.msixbundle + # Install Windows Terminal + Add-AppxProvisionedPackage -Online -PackagePath .\Microsoft.WindowsTerminal_1.17.11461.0_8wekyb3d8bbwe.msixbundle -SkipLicense + Add-AppxPackage -RegisterByFamilyName -MainPackage Microsoft.WindowsTerminal_1.17.11461.0_8wekyb3d8bbwe + + # https://github.com/microsoft/winget-cli/releases/tag/v1.6.1573-preview + Invoke-WebRequest -Uri https://github.com/microsoft/winget-cli/releases/download/v1.6.1573-preview/Microsoft.DesktopAppInstaller_8wekyb3d8bbwe.msixbundle -OutFile .\Microsoft.DesktopAppInstaller_8wekyb3d8bbwe.msixbundle + Invoke-WebRequest -Uri https://github.com/microsoft/winget-cli/releases/download/v1.6.1573-preview/ba27c402ae29410eb93cfa9cb27f1376_License1.xml -OutFile .\ba27c402ae29410eb93cfa9cb27f1376_License1.xml + + Add-AppxProvisionedPackage -Online -PackagePath .\Microsoft.DesktopAppInstaller_8wekyb3d8bbwe.msixbundle -LicensePath .\ba27c402ae29410eb93cfa9cb27f1376_License1.xml + Add-AppxPackage -RegisterByFamilyName -MainPackage Microsoft.DesktopAppInstaller_8wekyb3d8bbwe + + Write-Information "Manually update package cache and install dependencies" + $pws7script = @' + # Fix winget on Windows Server by manually updating the package cache + Import-Module -Name Appx -UseWindowsPowerShell + Invoke-WebRequest 'https://cdn.winget.microsoft.com/cache/source.msix' -OutFile 'source.msix' -UseBasicParsing + Add-AppxPackage -Path source.msix + ECHO Y | cmd /c winget upgrade --all --silent + # Install dependencies via winget + winget install -e --id Git.Git + winget install -e --id GnuWin32.Make + + # Install AWS tools + Install-Module -Name AWS.Tools.Common -Force + Install-Module -Name AWS.Tools.SecretsManager -Force + + # Install Go + Invoke-WebRequest -Uri 'https://go.dev/dl/go1.21.0.windows-amd64.msi' -OutFile 'go1.21.0.windows-amd64.msi' + Start-Process msiexec.exe -Wait -ArgumentList '/I C:\Users\Administrator\setup\go1.21.0.windows-amd64.msi /quiet' + # Configure path + $newPath = ("C:\Program Files\Git\usr\bin\;" + "$env:Path" + ";C:\Program Files\Git\bin\;" + "C:\Program Files (x86)\GnuWin32\bin\;" + "C:\Program Files\Go\bin\;") + $env:Path = $newPath + # Persist the path to the registry for new shells + Set-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH -Value $newPath + '@ + + # Write PowerShell 7 script to file + $pws7script | Out-File $Home\setup\setup7.ps1 + + # Execute script with PowerShell 7 + $ConsoleCommand = "$Home\setup\setup7.ps1" + Start-Process "C:\Program Files\PowerShell\7\pwsh" -Wait -NoNewWindow -PassThru -ArgumentList "-Command &{ $ConsoleCommand }" + + Write-Information "Downloading and configuring GitHub Actions Runner..." + mkdir $RUNNER_DIR; cd $RUNNER_DIR + Invoke-WebRequest -Uri https://github.com/actions/runner/releases/download/v2.309.0/actions-runner-win-x64-2.309.0.zip -OutFile actions-runner-win-x64-2.309.0.zip + if((Get-FileHash -Path actions-runner-win-x64-2.309.0.zip -Algorithm SHA256).Hash.ToUpper() -ne 'cd1920154e365689130aa1f90258e0da47faecce547d0374475cdd2554dbf09a'.ToUpper()){ throw 'Computed checksum did not match' } + Add-Type -AssemblyName System.IO.Compression.FileSystem ; [System.IO.Compression.ZipFile]::ExtractToDirectory("$PWD/actions-runner-win-x64-2.309.0.zip", "$PWD") + + $GH_KEY=(Get-SECSecretValue -SecretId $REPO-runner-reg-key -Region $REGION).SecretString + $LABEL_OS_VER=$([System.Environment]::OSVersion.Version.Major) + $LABEL_BUILD_VER=$([System.Environment]::OSVersion.Version.Build) + + $LABEL_ARCH="amd64" + + $RUNNER_REG_TOKEN=((Invoke-WebRequest -UseBasicParsing -Method POST -Headers @{"Accept" = "application/vnd.github+json"; "Authorization" = "Bearer $GH_KEY"; "X-GitHub-Api-Version" = "2022-11-28"} -Uri https://api.github.com/repos/runfinch/$REPO/actions/runners/registration-token).Content | ConvertFrom-Json).token + + Write-Information "Starting GitHub Actions Runner..." + cd $RUNNER_DIR; ./config.cmd --url https://github.com/runfinch/$REPO --unattended --token $RUNNER_REG_TOKEN --work _work --labels $LABEL_ARCH,$LABEL_OS_VER,$LABEL_BUILD_VER,$LABEL_STAGE + Start-Service "actions.runner.*"