diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 032387bec99..139abbc26c1 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -4,10 +4,10 @@ "features": { "ghcr.io/devcontainers/features/docker-in-docker:1": {}, "ghcr.io/devcontainers/features/dotnet": { - "version": "6.0.421" + "version": "8.0.404" }, "ghcr.io/devcontainers/features/node:1": { - "version": "16" + "version": "20" }, "ghcr.io/devcontainers/features/sshd:1": { "version": "latest" diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6c718529ced..685264a75d2 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,6 +5,11 @@ updates: schedule: interval: "daily" target-branch: "main" +- package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + target-branch: "main" - package-ecosystem: "nuget" directory: "/src" schedule: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7eabfb9cfe2..ab9057257ce 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -50,7 +50,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # Build runner layout - name: Build & Layout Release @@ -69,13 +69,13 @@ jobs: - name: Package Release if: github.event_name != 'pull_request' run: | - ${{ matrix.devScript }} package Release + ${{ matrix.devScript }} package Release ${{ matrix.runtime }} working-directory: src # Upload runner package tar.gz/zip as artifact - name: Publish Artifact if: github.event_name != 'pull_request' - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: runner-package-${{ matrix.runtime }} path: | diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 5b6e0523689..2f0d03dc7f7 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -23,7 +23,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/dotnet-upgrade.yml b/.github/workflows/dotnet-upgrade.yml index eb15e762e5f..12dc8be0246 100644 --- a/.github/workflows/dotnet-upgrade.yml +++ b/.github/workflows/dotnet-upgrade.yml @@ -15,7 +15,7 @@ jobs: DOTNET_CURRENT_MAJOR_MINOR_VERSION: ${{ steps.fetch_current_version.outputs.DOTNET_CURRENT_MAJOR_MINOR_VERSION }} steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get current major minor version id: fetch_current_version shell: bash @@ -51,7 +51,7 @@ jobs: run: echo "::error links::feature/dotnet-sdk-upgrade${{ steps.fetch_latest_version.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }} https://github.com/actions/runner/tree/feature/dotnet-sdk-upgrade${{ steps.fetch_latest_version.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }}::Branch feature/dotnetsdk-upgrade/${{ steps.fetch_latest_version.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }} already exists. Please take a look and delete that branch if you wish to recreate" - name: Create a warning annotation if no need to update if: ${{ steps.fetch_latest_version.outputs.SHOULD_UPDATE == 0 && steps.fetch_latest_version.outputs.BRANCH_EXISTS == 0 }} - run: echo "::warning ::Latest DotNet SDK patch is ${{ steps.fetch_latest_version.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }}, and we are on ${{ steps.fetch_latest_version.outputs.DOTNET_CURRENT_MAJOR_MINOR_PATCH_VERSION }}. No need to update" + run: echo "::warning ::Latest DotNet SDK patch is ${{ steps.fetch_latest_version.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }}, and we are on ${{ steps.fetch_current_version.outputs.DOTNET_CURRENT_MAJOR_MINOR_PATCH_VERSION }}. No need to update" - name: Update patch version if: ${{ steps.fetch_latest_version.outputs.SHOULD_UPDATE == 1 && steps.fetch_latest_version.outputs.BRANCH_EXISTS == 0 }} shell: bash @@ -89,7 +89,7 @@ jobs: if: ${{ needs.dotnet-update.outputs.SHOULD_UPDATE == 1 && needs.dotnet-update.outputs.BRANCH_EXISTS == 0 }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: feature/dotnetsdk-upgrade/${{ needs.dotnet-update.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }} - name: Create Pull Request diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index b35bee3c7d9..00000000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Lint - -on: - pull_request: - branches: [ main ] - -jobs: - build: - name: Lint - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - # Ensure full list of changed files within `super-linter` - fetch-depth: 0 - - name: Run linters - uses: github/super-linter@v4 - env: - DEFAULT_BRANCH: ${{ github.base_ref }} - EDITORCONFIG_FILE_NAME: .editorconfig - LINTER_RULES_PATH: /src/ - VALIDATE_ALL_CODEBASE: false - VALIDATE_CSHARP: true diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index 2932a268663..cadee26d266 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -20,7 +20,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Compute image version id: image diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index de94bcc4b26..d7342ccba69 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,7 +11,7 @@ jobs: if: startsWith(github.ref, 'refs/heads/releases/') || github.ref == 'refs/heads/main' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # Make sure ./releaseVersion match ./src/runnerversion # Query GitHub release ensure version is not used @@ -87,7 +87,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # Build runner layout - name: Build & Layout Release @@ -117,12 +117,11 @@ jobs: working-directory: _package # Upload runner package tar.gz/zip as artifact. - # Since each package name is unique, so we don't need to put ${{matrix}} info into artifact name - name: Publish Artifact if: github.event_name != 'pull_request' - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: - name: runner-packages + name: runner-packages-${{ matrix.runtime }} path: | _package @@ -131,13 +130,43 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # Download runner package tar.gz/zip produced by 'build' job - - name: Download Artifact - uses: actions/download-artifact@v1 + - name: Download Artifact (win-x64) + uses: actions/download-artifact@v4 with: - name: runner-packages + name: runner-packages-win-x64 + path: ./ + - name: Download Artifact (win-arm64) + uses: actions/download-artifact@v4 + with: + name: runner-packages-win-arm64 + path: ./ + - name: Download Artifact (osx-x64) + uses: actions/download-artifact@v4 + with: + name: runner-packages-osx-x64 + path: ./ + - name: Download Artifact (osx-arm64) + uses: actions/download-artifact@v4 + with: + name: runner-packages-osx-arm64 + path: ./ + - name: Download Artifact (linux-x64) + uses: actions/download-artifact@v4 + with: + name: runner-packages-linux-x64 + path: ./ + - name: Download Artifact (linux-arm) + uses: actions/download-artifact@v4 + with: + name: runner-packages-linux-arm + path: ./ + - name: Download Artifact (linux-arm64) + uses: actions/download-artifact@v4 + with: + name: runner-packages-linux-arm64 path: ./ # Create ReleaseNote file @@ -267,7 +296,7 @@ jobs: IMAGE_NAME: ${{ github.repository_owner }}/actions-runner steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Compute image version id: image diff --git a/.gitignore b/.gitignore index 34d18c4cedc..411fe4011a5 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,5 @@ _dotnetsdk TestResults TestLogs .DS_Store +.mono **/*.DotSettings.user \ No newline at end of file diff --git a/docs/checks/nodejs.md b/docs/checks/nodejs.md index cbdd8659b1a..7308dcadc50 100644 --- a/docs/checks/nodejs.md +++ b/docs/checks/nodejs.md @@ -4,9 +4,9 @@ Make sure the built-in node.js has access to GitHub.com or GitHub Enterprise Server. -The runner carries its own copy of node.js executable under `/externals/node16/`. +The runner carries its own copy of node.js executable under `/externals/node20/`. -All javascript base Actions will get executed by the built-in `node` at `/externals/node16/`. +All javascript base Actions will get executed by the built-in `node` at `/externals/node20/`. > Not the `node` from `$PATH` diff --git a/images/Dockerfile b/images/Dockerfile index dd8437e1d27..50b4a6b2d27 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -1,12 +1,12 @@ # Source: https://github.com/dotnet/dotnet-docker -FROM mcr.microsoft.com/dotnet/runtime-deps:6.0-jammy as build +FROM mcr.microsoft.com/dotnet/runtime-deps:8.0-jammy as build ARG TARGETOS ARG TARGETARCH ARG RUNNER_VERSION ARG RUNNER_CONTAINER_HOOKS_VERSION=0.6.1 -ARG DOCKER_VERSION=27.1.1 -ARG BUILDX_VERSION=0.16.2 +ARG DOCKER_VERSION=27.3.1 +ARG BUILDX_VERSION=0.18.0 RUN apt update -y && apt install curl unzip -y @@ -32,7 +32,7 @@ RUN export RUNNER_ARCH=${TARGETARCH} \ "https://github.com/docker/buildx/releases/download/v${BUILDX_VERSION}/buildx-v${BUILDX_VERSION}.linux-${TARGETARCH}" \ && chmod +x /usr/local/lib/docker/cli-plugins/docker-buildx -FROM mcr.microsoft.com/dotnet/runtime-deps:6.0-jammy +FROM mcr.microsoft.com/dotnet/runtime-deps:8.0-jammy ENV DEBIAN_FRONTEND=noninteractive ENV RUNNER_MANUALLY_TRAP_SIG=1 @@ -41,12 +41,14 @@ ENV ImageOS=ubuntu22 # 'gpg-agent' and 'software-properties-common' are needed for the 'add-apt-repository' command that follows RUN apt update -y \ - && apt install -y --no-install-recommends sudo lsb-release gpg-agent software-properties-common \ + && apt install -y --no-install-recommends sudo lsb-release gpg-agent software-properties-common curl jq unzip \ && rm -rf /var/lib/apt/lists/* # Configure git-core/ppa based on guidance here: https://git-scm.com/download/linux RUN add-apt-repository ppa:git-core/ppa \ - && apt update -y + && apt update -y \ + && apt install -y git \ + && rm -rf /var/lib/apt/lists/* RUN adduser --disabled-password --gecos "" --uid 1001 runner \ && groupadd docker --gid 123 \ diff --git a/releaseNote.md b/releaseNote.md index 9753b6d2311..c3f99e2872a 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,10 +1,30 @@ ## What's Changed -- .NET 8 OS compatibility test https://github.com/actions/runner/pull/3422 -- Ignore ssl cert on websocket client https://github.com/actions/runner/pull/3423 -- Revert "Bump runner to dotnet 8" https://github.com/actions/runner/pull/3412 - -**Full Changelog**: https://github.com/actions/runner/compare/v2.318.0...v2.319.0 +* Fix release workflow to use distinct artifact names by @ericsciple in https://github.com/actions/runner/pull/3485 +* Update dotnet sdk to latest version @6.0.425 by @github-actions in https://github.com/actions/runner/pull/3433 +* add ref and type to job completion in run service by @yaananth in https://github.com/actions/runner/pull/3492 +* Remove Broker Migration Message logging by @luketomlinson in https://github.com/actions/runner/pull/3493 +* Bump dotnet SDK to dotnet 8. by @TingluoHuang in https://github.com/actions/runner/pull/3500 +* Remove dotnet8 compatibility test. by @TingluoHuang in https://github.com/actions/runner/pull/3502 +* Remove node16 from the runner. by @TingluoHuang in https://github.com/actions/runner/pull/3503 +* send action name for run service by @yaananth in https://github.com/actions/runner/pull/3520 +* Handle runner not found by @ericsciple in https://github.com/actions/runner/pull/3536 +* Publish job telemetry to run-service. by @TingluoHuang in https://github.com/actions/runner/pull/3545 +* Fetch repo-level runner groups from API in v2 flow by @lucavallin in https://github.com/actions/runner/pull/3546 +* Allow runner to check service connection in background. by @TingluoHuang in https://github.com/actions/runner/pull/3542 +* Expose ENV for cache service v2. by @TingluoHuang in https://github.com/actions/runner/pull/3548 +* Update runner docker image. by @TingluoHuang in https://github.com/actions/runner/pull/3511 +* Bump Azure.Storage.Blobs from 12.19.1 to 12.23.0 in /src by @dependabot in https://github.com/actions/runner/pull/3549 +* fix dotnet-upgrade.yml to print right version by @TingluoHuang in https://github.com/actions/runner/pull/3550 +* Update dotnet sdk to latest version @8.0.404 by @github-actions in https://github.com/actions/runner/pull/3552 +* Configure dependabot to check github-actions updates by @Goooler in https://github.com/actions/runner/pull/3333 +* Bump actions/checkout from 3 to 4 by @dependabot in https://github.com/actions/runner/pull/3556 + +## New Contributors +* @lucavallin made their first contribution in https://github.com/actions/runner/pull/3546 +* @Goooler made their first contribution in https://github.com/actions/runner/pull/3333 + +**Full Changelog**: https://github.com/actions/runner/compare/v2.320.0...v2.321.0 _Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet. To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository. @@ -26,9 +46,7 @@ Add-Type -AssemblyName System.IO.Compression.FileSystem ; [System.IO.Compression.ZipFile]::ExtractToDirectory("$PWD\actions-runner-win-x64-.zip", "$PWD") ``` -## [Pre-release] Windows arm64 - -**Warning:** Windows arm64 runners are currently in preview status and use [unofficial versions of nodejs](https://unofficial-builds.nodejs.org/). They are not intended for production workflows. +## Windows arm64 We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows. diff --git a/src/Misc/externals.sh b/src/Misc/externals.sh index e057ecb218f..2a0db24aa9e 100755 --- a/src/Misc/externals.sh +++ b/src/Misc/externals.sh @@ -3,13 +3,10 @@ PACKAGERUNTIME=$1 PRECACHE=$2 NODE_URL=https://nodejs.org/dist -UNOFFICIAL_NODE_URL=https://unofficial-builds.nodejs.org/download/release NODE_ALPINE_URL=https://github.com/actions/alpine_nodejs/releases/download # When you update Node versions you must also create a new release of alpine_nodejs at that updated version. # Follow the instructions here: https://github.com/actions/alpine_nodejs?tab=readme-ov-file#getting-started -NODE16_VERSION="16.20.2" -NODE20_VERSION="20.13.1" -NODE16_UNOFFICIAL_VERSION="16.20.0" # used only for win-arm64, remove node16 unofficial version when official version is available +NODE20_VERSION="20.18.0" get_abs_path() { # exploits the fact that pwd will print abs path when no args @@ -140,8 +137,6 @@ function acquireExternalTool() { # Download the external tools only for Windows. if [[ "$PACKAGERUNTIME" == "win-x64" || "$PACKAGERUNTIME" == "win-x86" ]]; then - acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/$PACKAGERUNTIME/node.exe" node16/bin - acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/$PACKAGERUNTIME/node.lib" node16/bin acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/$PACKAGERUNTIME/node.exe" node20/bin acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/$PACKAGERUNTIME/node.lib" node20/bin if [[ "$PRECACHE" != "" ]]; then @@ -152,8 +147,6 @@ fi # Download the external tools only for Windows. if [[ "$PACKAGERUNTIME" == "win-arm64" ]]; then # todo: replace these with official release when available - acquireExternalTool "$UNOFFICIAL_NODE_URL/v${NODE16_UNOFFICIAL_VERSION}/$PACKAGERUNTIME/node.exe" node16/bin - acquireExternalTool "$UNOFFICIAL_NODE_URL/v${NODE16_UNOFFICIAL_VERSION}/$PACKAGERUNTIME/node.lib" node16/bin acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/$PACKAGERUNTIME/node.exe" node20/bin acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/$PACKAGERUNTIME/node.lib" node20/bin if [[ "$PRECACHE" != "" ]]; then @@ -163,30 +156,24 @@ fi # Download the external tools only for OSX. if [[ "$PACKAGERUNTIME" == "osx-x64" ]]; then - acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/node-v${NODE16_VERSION}-darwin-x64.tar.gz" node16 fix_nested_dir acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/node-v${NODE20_VERSION}-darwin-x64.tar.gz" node20 fix_nested_dir fi if [[ "$PACKAGERUNTIME" == "osx-arm64" ]]; then # node.js v12 doesn't support macOS on arm64. - acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/node-v${NODE16_VERSION}-darwin-arm64.tar.gz" node16 fix_nested_dir acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/node-v${NODE20_VERSION}-darwin-arm64.tar.gz" node20 fix_nested_dir fi # Download the external tools for Linux PACKAGERUNTIMEs. if [[ "$PACKAGERUNTIME" == "linux-x64" ]]; then - acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/node-v${NODE16_VERSION}-linux-x64.tar.gz" node16 fix_nested_dir - acquireExternalTool "$NODE_ALPINE_URL/v${NODE16_VERSION}/node-v${NODE16_VERSION}-alpine-x64.tar.gz" node16_alpine acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/node-v${NODE20_VERSION}-linux-x64.tar.gz" node20 fix_nested_dir acquireExternalTool "$NODE_ALPINE_URL/v${NODE20_VERSION}/node-v${NODE20_VERSION}-alpine-x64.tar.gz" node20_alpine fi if [[ "$PACKAGERUNTIME" == "linux-arm64" ]]; then - acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/node-v${NODE16_VERSION}-linux-arm64.tar.gz" node16 fix_nested_dir acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/node-v${NODE20_VERSION}-linux-arm64.tar.gz" node20 fix_nested_dir fi if [[ "$PACKAGERUNTIME" == "linux-arm" ]]; then - acquireExternalTool "$NODE_URL/v${NODE16_VERSION}/node-v${NODE16_VERSION}-linux-armv7l.tar.gz" node16 fix_nested_dir acquireExternalTool "$NODE_URL/v${NODE20_VERSION}/node-v${NODE20_VERSION}-linux-armv7l.tar.gz" node20 fix_nested_dir fi diff --git a/src/Misc/layoutbin/runsvc.sh b/src/Misc/layoutbin/runsvc.sh index c13564567ee..63d1b62e1a6 100755 --- a/src/Misc/layoutbin/runsvc.sh +++ b/src/Misc/layoutbin/runsvc.sh @@ -10,7 +10,7 @@ if [ -f ".path" ]; then echo ".path=${PATH}" fi -nodever=${GITHUB_ACTIONS_RUNNER_FORCED_NODE_VERSION:-node16} +nodever="node20" # insert anything to setup env when running as a service # run the host process which keep the listener alive diff --git a/src/Misc/layoutbin/update.sh.template b/src/Misc/layoutbin/update.sh.template index 4ca6b00e6b4..832385edd5f 100755 --- a/src/Misc/layoutbin/update.sh.template +++ b/src/Misc/layoutbin/update.sh.template @@ -135,12 +135,17 @@ if [[ "$currentplatform" == 'darwin' && restartinteractiverunner -eq 0 ]]; then then # inspect the open file handles to find the node process # we can't actually inspect the process using ps because it uses relative paths and doesn't follow symlinks - nodever="node16" + nodever="node20" path=$(lsof -a -g "$procgroup" -F n | grep $nodever/bin/node | grep externals | tail -1 | cut -c2-) - if [[ $? -ne 0 || -z "$path" ]] # Fallback if RunnerService.js was started with node12 + if [[ $? -ne 0 || -z "$path" ]] # Fallback if RunnerService.js was started with node16 then - nodever="node12" + nodever="node16" path=$(lsof -a -g "$procgroup" -F n | grep $nodever/bin/node | grep externals | tail -1 | cut -c2-) + if [[ $? -ne 0 || -z "$path" ]] # Fallback if RunnerService.js was started with node12 + then + nodever="node12" + path=$(lsof -a -g "$procgroup" -F n | grep $nodever/bin/node | grep externals | tail -1 | cut -c2-) + fi fi if [[ $? -eq 0 && -n "$path" ]] then @@ -178,6 +183,19 @@ if [[ "$currentplatform" == 'darwin' && restartinteractiverunner -eq 0 ]]; then fi fi +# update runsvc.sh +if [ -f "$rootfolder/runsvc.sh" ] +then + date "+[%F %T-%4N] Update runsvc.sh" >> "$logfile" 2>&1 + cat "$rootfolder/bin/runsvc.sh" > "$rootfolder/runsvc.sh" + if [ $? -ne 0 ] + then + date "+[%F %T-%4N] Can't update $rootfolder/runsvc.sh using $rootfolder/bin/runsvc.sh" >> "$logfile" 2>&1 + mv -fv "$logfile" "$logfile.failed" + exit 1 + fi +fi + date "+[%F %T-%4N] Update succeed" >> "$logfile" touch update.finished diff --git a/src/Runner.Common/BrokerServer.cs b/src/Runner.Common/BrokerServer.cs index 5e1311715c5..7ca4713f7bd 100644 --- a/src/Runner.Common/BrokerServer.cs +++ b/src/Runner.Common/BrokerServer.cs @@ -7,6 +7,7 @@ using GitHub.DistributedTask.WebApi; using GitHub.Runner.Sdk; using GitHub.Services.Common; +using GitHub.Services.WebApi; using Sdk.RSWebApi.Contracts; using Sdk.WebApi.WebApi.RawClient; @@ -92,7 +93,7 @@ public Task ForceRefreshConnection(VssCredentials credentials) public bool ShouldRetryException(Exception ex) { - if (ex is AccessDeniedException ade && ade.ErrorCode == 1) + if (ex is AccessDeniedException || ex is RunnerNotFoundException) { return false; } diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index 383ec7a10f5..2c20d1b16fe 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -159,7 +159,6 @@ public static class ReturnCode public static class Features { public static readonly string DiskSpaceWarning = "runner.diskspace.warning"; - public static readonly string Node16Warning = "DistributedTask.AddWarningToNode16Action"; public static readonly string LogTemplateErrorsAsDebugMessages = "DistributedTask.LogTemplateErrorsAsDebugMessages"; public static readonly string UseContainerPathForTemplate = "DistributedTask.UseContainerPathForTemplate"; public static readonly string AllowRunnerContainerHooks = "DistributedTask.AllowRunnerContainerHooks"; @@ -176,14 +175,6 @@ public static class Features public static readonly string UnsupportedStopCommandTokenDisabled = "You cannot use a endToken that is an empty string, the string 'pause-logging', or another workflow command. For more information see: https://docs.github.com/actions/learn-github-actions/workflow-commands-for-github-actions#example-stopping-and-starting-workflow-commands or opt into insecure command execution by setting the `ACTIONS_ALLOW_UNSECURE_STOPCOMMAND_TOKENS` environment variable to `true`."; public static readonly string UnsupportedSummarySize = "$GITHUB_STEP_SUMMARY upload aborted, supports content up to a size of {0}k, got {1}k. For more information see: https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#adding-a-markdown-summary"; public static readonly string SummaryUploadError = "$GITHUB_STEP_SUMMARY upload aborted, an error occurred when uploading the summary. For more information see: https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#adding-a-markdown-summary"; - public static readonly string DetectedNodeAfterEndOfLifeMessage = "Node.js 16 actions are deprecated. Please update the following actions to use Node.js 20: {0}. For more information see: https://github.blog/changelog/2023-09-22-github-actions-transitioning-from-node-16-to-node-20/."; - public static readonly string DeprecatedNodeDetectedAfterEndOfLifeActions = "DeprecatedNodeActionsMessageWarnings"; - public static readonly string DeprecatedNodeVersion = "node16"; - public static readonly string EnforcedNode12DetectedAfterEndOfLife = "The following actions uses node12 which is deprecated and will be forced to run on node16: {0}. For more info: https://github.blog/changelog/2023-06-13-github-actions-all-actions-will-run-on-node16-instead-of-node12-by-default/"; - public static readonly string EnforcedNode12DetectedAfterEndOfLifeEnvVariable = "Node16ForceActionsWarnings"; - public static readonly string EnforcedNode16DetectedAfterEndOfLife = "The following actions use a deprecated Node.js version and will be forced to run on node20: {0}. For more info: https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default/"; - public static readonly string EnforcedNode16DetectedAfterEndOfLifeEnvVariable = "Node20ForceActionsWarnings"; - } public static class RunnerEvent @@ -254,20 +245,17 @@ public static class Actions public static readonly string RequireJobContainer = "ACTIONS_RUNNER_REQUIRE_JOB_CONTAINER"; public static readonly string RunnerDebug = "ACTIONS_RUNNER_DEBUG"; public static readonly string StepDebug = "ACTIONS_STEP_DEBUG"; - public static readonly string AllowActionsUseUnsecureNodeVersion = "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION"; - public static readonly string ManualForceActionsToNode20 = "FORCE_JAVASCRIPT_ACTIONS_TO_NODE20"; } public static class Agent { public static readonly string ToolsDirectory = "agent.ToolsDirectory"; - // Set this env var to "node12" to downgrade the node version for internal functions (e.g hashfiles). This does NOT affect the version of node actions. + // Set this env var to "nodeXY" to downgrade the node version for internal functions (e.g hashfiles). This does NOT affect the version of node actions. public static readonly string ForcedInternalNodeVersion = "ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION"; public static readonly string ForcedActionsNodeVersion = "ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION"; public static readonly string PrintLogToStdout = "ACTIONS_RUNNER_PRINT_LOG_TO_STDOUT"; public static readonly string ActionArchiveCacheDirectory = "ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE"; - public static readonly string ManualForceActionsToNode20 = "FORCE_JAVASCRIPT_ACTIONS_TO_NODE20"; } public static class System @@ -280,10 +268,6 @@ public static class System public static readonly string PhaseDisplayName = "system.phaseDisplayName"; public static readonly string JobRequestType = "system.jobRequestType"; public static readonly string OrchestrationId = "system.orchestrationId"; - public static readonly string TestDotNet8Compatibility = "system.testDotNet8Compatibility"; - public static readonly string DotNet8CompatibilityOutputLength = "system.dotNet8CompatibilityOutputLength"; - public static readonly string DotNet8CompatibilityOutputPattern = "system.dotNet8CompatibilityOutputPattern"; - public static readonly string DotNet8CompatibilityWarning = "system.dotNet8CompatibilityWarning"; } } diff --git a/src/Runner.Common/HostContext.cs b/src/Runner.Common/HostContext.cs index 78ea8ba4cbe..db0c54226ff 100644 --- a/src/Runner.Common/HostContext.cs +++ b/src/Runner.Common/HostContext.cs @@ -36,6 +36,7 @@ public interface IHostContext : IDisposable event EventHandler Unloading; void ShutdownRunner(ShutdownReason reason); void WritePerfCounter(string counter); + void LoadDefaultUserAgents(); } public enum StartupType @@ -67,6 +68,7 @@ public sealed class HostContext : EventListener, IObserver, private StartupType _startupType; private string _perfFile; private RunnerWebProxy _webProxy = new(); + private string _hostType = string.Empty; public event EventHandler Unloading; public CancellationToken RunnerShutdownToken => _runnerShutdownTokenSource.Token; @@ -78,6 +80,7 @@ public HostContext(string hostType, string logFile = null) { // Validate args. ArgUtil.NotNullOrEmpty(hostType, nameof(hostType)); + _hostType = hostType; _loadContext = AssemblyLoadContext.GetLoadContext(typeof(HostContext).GetTypeInfo().Assembly); _loadContext.Unloading += LoadContext_Unloading; @@ -196,6 +199,16 @@ public HostContext(string hostType, string logFile = null) } } + if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_TLS_NO_VERIFY"))) + { + _trace.Warning($"Runner is running under insecure mode: HTTPS server certificate validation has been turned off by GITHUB_ACTIONS_RUNNER_TLS_NO_VERIFY environment variable."); + } + + LoadDefaultUserAgents(); + } + + public void LoadDefaultUserAgents() + { if (string.IsNullOrEmpty(WebProxy.HttpProxyAddress) && string.IsNullOrEmpty(WebProxy.HttpsProxyAddress)) { _trace.Info($"No proxy settings were found based on environmental variables (http_proxy/https_proxy/HTTP_PROXY/HTTPS_PROXY)"); @@ -205,11 +218,6 @@ public HostContext(string hostType, string logFile = null) _userAgents.Add(new ProductInfoHeaderValue("HttpProxyConfigured", bool.TrueString)); } - if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_TLS_NO_VERIFY"))) - { - _trace.Warning($"Runner is running under insecure mode: HTTPS server certificate validation has been turned off by GITHUB_ACTIONS_RUNNER_TLS_NO_VERIFY environment variable."); - } - var credFile = GetConfigFile(WellKnownConfigFile.Credentials); if (File.Exists(credFile)) { @@ -244,6 +252,11 @@ public HostContext(string hostType, string logFile = null) _trace.Info($"Adding extra user agent '{extraUserAgentHeader}' to all HTTP requests."); _userAgents.Add(extraUserAgentHeader); } + + var currentProcess = Process.GetCurrentProcess(); + _userAgents.Add(new ProductInfoHeaderValue("Pid", currentProcess.Id.ToString())); + _userAgents.Add(new ProductInfoHeaderValue("CreationTime", Uri.EscapeDataString(DateTime.UtcNow.ToString("O")))); + _userAgents.Add(new ProductInfoHeaderValue($"({_hostType})")); } public string GetDirectory(WellKnownDirectory directory) @@ -616,7 +629,7 @@ protected override void OnEventWritten(EventWrittenEventArgs eventData) payload[0] = Enum.Parse(typeof(GitHub.Services.Common.VssCredentialsType), ((int)payload[0]).ToString()); } - if (payload.Length > 0) + if (payload.Length > 0 && !string.IsNullOrEmpty(eventData.Message)) { message = String.Format(eventData.Message.Replace("%n", Environment.NewLine), payload); } diff --git a/src/Runner.Common/LaunchServer.cs b/src/Runner.Common/LaunchServer.cs index e1b1b0f4f7c..f8584ac5341 100644 --- a/src/Runner.Common/LaunchServer.cs +++ b/src/Runner.Common/LaunchServer.cs @@ -1,11 +1,12 @@ using System; using System.Collections.Generic; -using System.Linq; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; using GitHub.DistributedTask.WebApi; +using GitHub.Runner.Sdk; +using GitHub.Services.Common; using GitHub.Services.Launch.Client; -using GitHub.Services.WebApi; namespace GitHub.Runner.Common { @@ -23,8 +24,21 @@ public sealed class LaunchServer : RunnerService, ILaunchServer public void InitializeLaunchClient(Uri uri, string token) { - var httpMessageHandler = HostContext.CreateHttpClientHandler(); - this._launchClient = new LaunchHttpClient(uri, httpMessageHandler, token, disposeHandler: true); + // Using default 100 timeout + RawClientHttpRequestSettings settings = VssUtil.GetHttpRequestSettings(null); + + // Create retry handler + IEnumerable delegatingHandlers = new List(); + if (settings.MaxRetryRequest > 0) + { + delegatingHandlers = new DelegatingHandler[] { new VssHttpRetryMessageHandler(settings.MaxRetryRequest) }; + } + + // Setup RawHttpMessageHandler without credentials + var httpMessageHandler = new RawHttpMessageHandler(new NoOpCredentials(null), settings); + var pipeline = HttpClientFactory.CreatePipeline(httpMessageHandler, delegatingHandlers); + + this._launchClient = new LaunchHttpClient(uri, pipeline, token, disposeHandler: true); } public Task ResolveActionsDownloadInfoAsync(Guid planId, Guid jobId, ActionReferenceList actionReferenceList, diff --git a/src/Runner.Common/RunServer.cs b/src/Runner.Common/RunServer.cs index 50ad0556018..8a7f4d5b4fa 100644 --- a/src/Runner.Common/RunServer.cs +++ b/src/Runner.Common/RunServer.cs @@ -28,6 +28,7 @@ Task CompleteJobAsync( IList stepResults, IList jobAnnotations, string environmentUrl, + IList telemetry, CancellationToken token); Task RenewJobAsync(Guid planId, Guid jobId, CancellationToken token); @@ -76,11 +77,12 @@ public Task CompleteJobAsync( IList stepResults, IList jobAnnotations, string environmentUrl, + IList telemetry, CancellationToken cancellationToken) { CheckConnection(); return RetryRequest( - async () => await _runServiceHttpClient.CompleteJobAsync(requestUri, planId, jobId, result, outputs, stepResults, jobAnnotations, environmentUrl, cancellationToken), cancellationToken); + async () => await _runServiceHttpClient.CompleteJobAsync(requestUri, planId, jobId, result, outputs, stepResults, jobAnnotations, environmentUrl, telemetry, cancellationToken), cancellationToken); } public Task RenewJobAsync(Guid planId, Guid jobId, CancellationToken cancellationToken) diff --git a/src/Runner.Common/Runner.Common.csproj b/src/Runner.Common/Runner.Common.csproj index 329a024aac6..6c4635626a2 100644 --- a/src/Runner.Common/Runner.Common.csproj +++ b/src/Runner.Common/Runner.Common.csproj @@ -1,11 +1,11 @@ - net6.0 + net8.0 Library win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 true - NU1701;NU1603 + NU1701;NU1603;SYSLIB0050;SYSLIB0051 $(Version) @@ -15,11 +15,11 @@ - + - - - + + + diff --git a/src/Runner.Common/RunnerDotcomServer.cs b/src/Runner.Common/RunnerDotcomServer.cs index 0f6d3ce1d3e..b607024d49c 100644 --- a/src/Runner.Common/RunnerDotcomServer.cs +++ b/src/Runner.Common/RunnerDotcomServer.cs @@ -46,7 +46,11 @@ public async Task> GetRunnerByNameAsync(string githubUrl, string var githubApiUrl = ""; var gitHubUrlBuilder = new UriBuilder(githubUrl); var path = gitHubUrlBuilder.Path.Split('/', '\\', StringSplitOptions.RemoveEmptyEntries); - if (path.Length == 1) + var isOrgRunner = path.Length == 1; + var isRepoOrEnterpriseRunner = path.Length == 2; + var isRepoRunner = isRepoOrEnterpriseRunner && !string.Equals(path[0], "enterprises", StringComparison.OrdinalIgnoreCase); + + if (isOrgRunner) { // org runner if (UrlUtil.IsHostedServer(gitHubUrlBuilder)) @@ -58,21 +62,31 @@ public async Task> GetRunnerByNameAsync(string githubUrl, string githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/orgs/{path[0]}/actions/runners?name={Uri.EscapeDataString(agentName)}"; } } - else if (path.Length == 2) + else if (isRepoOrEnterpriseRunner) { - // repo or enterprise runner. - if (!string.Equals(path[0], "enterprises", StringComparison.OrdinalIgnoreCase)) + // Repository runner + if (isRepoRunner) { - return null; - } - - if (UrlUtil.IsHostedServer(gitHubUrlBuilder)) - { - githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/{path[0]}/{path[1]}/actions/runners?name={Uri.EscapeDataString(agentName)}"; + if (UrlUtil.IsHostedServer(gitHubUrlBuilder)) + { + githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/repos/{path[0]}/{path[1]}/actions/runners?name={Uri.EscapeDataString(agentName)}"; + } + else + { + githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/repos/{path[0]}/{path[1]}/actions/runners?name={Uri.EscapeDataString(agentName)}"; + } } else { - githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/{path[0]}/{path[1]}/actions/runners?name={Uri.EscapeDataString(agentName)}"; + // Enterprise runner + if (UrlUtil.IsHostedServer(gitHubUrlBuilder)) + { + githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/{path[0]}/{path[1]}/actions/runners?name={Uri.EscapeDataString(agentName)}"; + } + else + { + githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/{path[0]}/{path[1]}/actions/runners?name={Uri.EscapeDataString(agentName)}"; + } } } else @@ -90,7 +104,11 @@ public async Task> GetRunnerGroupsAsync(string githubUrl, st var githubApiUrl = ""; var gitHubUrlBuilder = new UriBuilder(githubUrl); var path = gitHubUrlBuilder.Path.Split('/', '\\', StringSplitOptions.RemoveEmptyEntries); - if (path.Length == 1) + var isOrgRunner = path.Length == 1; + var isRepoOrEnterpriseRunner = path.Length == 2; + var isRepoRunner = isRepoOrEnterpriseRunner && !string.Equals(path[0], "enterprises", StringComparison.OrdinalIgnoreCase); + + if (isOrgRunner) { // org runner if (UrlUtil.IsHostedServer(gitHubUrlBuilder)) @@ -102,21 +120,31 @@ public async Task> GetRunnerGroupsAsync(string githubUrl, st githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/orgs/{path[0]}/actions/runner-groups"; } } - else if (path.Length == 2) + else if (isRepoOrEnterpriseRunner) { - // repo or enterprise runner. - if (!string.Equals(path[0], "enterprises", StringComparison.OrdinalIgnoreCase)) + // Repository Runner + if (isRepoRunner) { - return null; - } - - if (UrlUtil.IsHostedServer(gitHubUrlBuilder)) - { - githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/{path[0]}/{path[1]}/actions/runner-groups"; + if (UrlUtil.IsHostedServer(gitHubUrlBuilder)) + { + githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/repos/{path[0]}/{path[1]}/actions/runner-groups"; + } + else + { + githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/repos/{path[0]}/{path[1]}/actions/runner-groups"; + } } else { - githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/{path[0]}/{path[1]}/actions/runner-groups"; + // Enterprise Runner + if (UrlUtil.IsHostedServer(gitHubUrlBuilder)) + { + githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/{path[0]}/{path[1]}/actions/runner-groups"; + } + else + { + githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/{path[0]}/{path[1]}/actions/runner-groups"; + } } } else diff --git a/src/Runner.Common/Util/NodeUtil.cs b/src/Runner.Common/Util/NodeUtil.cs index f2c01d7d311..1a9252cde0f 100644 --- a/src/Runner.Common/Util/NodeUtil.cs +++ b/src/Runner.Common/Util/NodeUtil.cs @@ -5,8 +5,8 @@ namespace GitHub.Runner.Common.Util { public static class NodeUtil { - private const string _defaultNodeVersion = "node16"; - public static readonly ReadOnlyCollection BuiltInNodeVersions = new(new[] { "node16", "node20" }); + private const string _defaultNodeVersion = "node20"; + public static readonly ReadOnlyCollection BuiltInNodeVersions = new(new[] { "node20" }); public static string GetInternalNodeVersion() { var forcedInternalNodeVersion = Environment.GetEnvironmentVariable(Constants.Variables.Agent.ForcedInternalNodeVersion); diff --git a/src/Runner.Listener/BrokerMessageListener.cs b/src/Runner.Listener/BrokerMessageListener.cs index 8f44fe84372..89899181146 100644 --- a/src/Runner.Listener/BrokerMessageListener.cs +++ b/src/Runner.Listener/BrokerMessageListener.cs @@ -11,9 +11,10 @@ using GitHub.Runner.Common; using GitHub.Runner.Listener.Configuration; using GitHub.Runner.Sdk; -using GitHub.Services.Common; using GitHub.Runner.Common.Util; +using GitHub.Services.Common; using GitHub.Services.OAuth; +using GitHub.Services.WebApi; namespace GitHub.Runner.Listener { @@ -69,7 +70,8 @@ public async Task CreateSessionAsync(CancellationToken toke Version = BuildConstants.RunnerPackage.Version, OSDescription = RuntimeInformation.OSDescription, }; - string sessionName = $"{Environment.MachineName ?? "RUNNER"}"; + var currentProcess = Process.GetCurrentProcess(); + string sessionName = $"{Environment.MachineName ?? "RUNNER"} (PID: {currentProcess.Id})"; var taskAgentSession = new TaskAgentSession(sessionName, agent); string errorMessage = string.Empty; @@ -240,6 +242,10 @@ public async Task GetNextMessageAsync(CancellationToken token) { throw; } + catch (RunnerNotFoundException) + { + throw; + } catch (Exception ex) { Trace.Error("Catch exception during get next message."); @@ -323,6 +329,7 @@ private bool IsGetNextMessageExceptionRetriable(Exception ex) ex is TaskAgentPoolNotFoundException || ex is TaskAgentSessionExpiredException || ex is AccessDeniedException || + ex is RunnerNotFoundException || ex is VssUnauthorizedException) { Trace.Info($"Non-retriable exception: {ex.Message}"); diff --git a/src/Runner.Listener/Configuration/RSAEncryptedFileKeyManager.cs b/src/Runner.Listener/Configuration/RSAEncryptedFileKeyManager.cs index 15291be4387..a404a674e96 100644 --- a/src/Runner.Listener/Configuration/RSAEncryptedFileKeyManager.cs +++ b/src/Runner.Listener/Configuration/RSAEncryptedFileKeyManager.cs @@ -1,4 +1,5 @@ #if OS_WINDOWS +#pragma warning disable CA1416 using System.IO; using System.Security.Cryptography; using System.Text; @@ -84,4 +85,5 @@ void IRunnerService.Initialize(IHostContext context) } } } +#pragma warning restore CA1416 #endif diff --git a/src/Runner.Listener/JobDispatcher.cs b/src/Runner.Listener/JobDispatcher.cs index ede26db1810..1b8196091c6 100644 --- a/src/Runner.Listener/JobDispatcher.cs +++ b/src/Runner.Listener/JobDispatcher.cs @@ -545,28 +545,36 @@ await processChannel.SendAsync( detailInfo = string.Join(Environment.NewLine, workerOutput); Trace.Info($"Return code {returnCode} indicate worker encounter an unhandled exception or app crash, attach worker stdout/stderr to JobRequest result."); - var jobServer = await InitializeJobServerAsync(systemConnection); - var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = detailInfo }; - unhandledExceptionIssue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.WorkerCrash; - switch (jobServer) + try { - case IJobServer js: - { - await LogWorkerProcessUnhandledException(js, message, unhandledExceptionIssue); - // Go ahead to finish the job with result 'Failed' if the STDERR from worker is System.IO.IOException, since it typically means we are running out of disk space. - if (detailInfo.Contains(typeof(System.IO.IOException).ToString(), StringComparison.OrdinalIgnoreCase)) + var jobServer = await InitializeJobServerAsync(systemConnection); + var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = detailInfo }; + unhandledExceptionIssue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.WorkerCrash; + switch (jobServer) + { + case IJobServer js: { - Trace.Info($"Finish job with result 'Failed' due to IOException."); - await ForceFailJob(js, message); + await LogWorkerProcessUnhandledException(js, message, unhandledExceptionIssue); + // Go ahead to finish the job with result 'Failed' if the STDERR from worker is System.IO.IOException, since it typically means we are running out of disk space. + if (detailInfo.Contains(typeof(System.IO.IOException).ToString(), StringComparison.OrdinalIgnoreCase)) + { + Trace.Info($"Finish job with result 'Failed' due to IOException."); + await ForceFailJob(js, message); + } + + break; } - + case IRunServer rs: + await ForceFailJob(rs, message, unhandledExceptionIssue); break; - } - case IRunServer rs: - await ForceFailJob(rs, message, unhandledExceptionIssue); - break; - default: - throw new NotSupportedException($"JobServer type '{jobServer.GetType().Name}' is not supported."); + default: + throw new NotSupportedException($"JobServer type '{jobServer.GetType().Name}' is not supported."); + } + } + catch (Exception ex) + { + Trace.Error($"Catch exception during log worker process unhandled exception."); + Trace.Error(ex); } } @@ -1198,7 +1206,7 @@ private async Task ForceFailJob(IRunServer runServer, Pipelines.AgentJobRequestM jobAnnotations.Add(annotation.Value); } - await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, TaskResult.Failed, outputs: null, stepResults: null, jobAnnotations: jobAnnotations, environmentUrl: null, CancellationToken.None); + await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, TaskResult.Failed, outputs: null, stepResults: null, jobAnnotations: jobAnnotations, environmentUrl: null, telemetry: null, CancellationToken.None); } catch (Exception ex) { diff --git a/src/Runner.Listener/MessageListener.cs b/src/Runner.Listener/MessageListener.cs index 7b51423e6f5..9ea44b3f5b0 100644 --- a/src/Runner.Listener/MessageListener.cs +++ b/src/Runner.Listener/MessageListener.cs @@ -88,7 +88,8 @@ public async Task CreateSessionAsync(CancellationToken toke Version = BuildConstants.RunnerPackage.Version, OSDescription = RuntimeInformation.OSDescription, }; - string sessionName = $"{Environment.MachineName ?? "RUNNER"}"; + var currentProcess = Process.GetCurrentProcess(); + string sessionName = $"{Environment.MachineName ?? "RUNNER"} (PID: {currentProcess.Id})"; var taskAgentSession = new TaskAgentSession(sessionName, agent); string errorMessage = string.Empty; @@ -263,8 +264,6 @@ public async Task GetNextMessageAsync(CancellationToken token) if (message != null && message.MessageType == BrokerMigrationMessage.MessageType) { - Trace.Info("BrokerMigration message received. Polling Broker for messages..."); - var migrationMessage = JsonUtility.FromString(message.Body); await _brokerServer.UpdateConnectionIfNeeded(migrationMessage.BrokerBaseUrl, _creds); @@ -309,6 +308,10 @@ public async Task GetNextMessageAsync(CancellationToken token) { throw; } + catch (RunnerNotFoundException) + { + throw; + } catch (Exception ex) { Trace.Error("Catch exception during get next message."); @@ -458,6 +461,7 @@ private bool IsGetNextMessageExceptionRetriable(Exception ex) ex is TaskAgentPoolNotFoundException || ex is TaskAgentSessionExpiredException || ex is AccessDeniedException || + ex is RunnerNotFoundException || ex is VssUnauthorizedException) { Trace.Info($"Non-retriable exception: {ex.Message}"); diff --git a/src/Runner.Listener/Program.cs b/src/Runner.Listener/Program.cs index a6bdce62ce8..80852d32c4d 100644 --- a/src/Runner.Listener/Program.cs +++ b/src/Runner.Listener/Program.cs @@ -7,6 +7,7 @@ using System.Runtime.InteropServices; using System.Threading.Tasks; using GitHub.DistributedTask.WebApi; +using GitHub.Services.WebApi; namespace GitHub.Runner.Listener { @@ -144,6 +145,12 @@ private async static Task MainAsync(IHostContext context, string[] args) trace.Error(e); return Constants.Runner.ReturnCode.TerminatedError; } + catch (RunnerNotFoundException e) + { + terminal.WriteError($"An error occurred: {e.Message}"); + trace.Error(e); + return Constants.Runner.ReturnCode.TerminatedError; + } catch (Exception e) { terminal.WriteError($"An error occurred: {e.Message}"); diff --git a/src/Runner.Listener/Runner.Listener.csproj b/src/Runner.Listener/Runner.Listener.csproj index 3cd72ec61f8..afd528128a5 100644 --- a/src/Runner.Listener/Runner.Listener.csproj +++ b/src/Runner.Listener/Runner.Listener.csproj @@ -1,11 +1,12 @@ - net6.0 + net8.0 Exe win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 + true true - NU1701;NU1603 + NU1701;NU1603;SYSLIB0050;SYSLIB0051 $(Version) false true @@ -18,11 +19,11 @@ - + - - - + + + diff --git a/src/Runner.Listener/Runner.cs b/src/Runner.Listener/Runner.cs index e2f638522b8..94b86b9f957 100644 --- a/src/Runner.Listener/Runner.cs +++ b/src/Runner.Listener/Runner.cs @@ -228,15 +228,21 @@ public async Task ExecuteCommand(CommandSettings command) var configFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), config.Key); var configContent = Convert.FromBase64String(config.Value); #if OS_WINDOWS +#pragma warning disable CA1416 if (configFile == HostContext.GetConfigFile(WellKnownConfigFile.RSACredentials)) { configContent = ProtectedData.Protect(configContent, null, DataProtectionScope.LocalMachine); } +#pragma warning restore CA1416 #endif File.WriteAllBytes(configFile, configContent); File.SetAttributes(configFile, File.GetAttributes(configFile) | FileAttributes.Hidden); Trace.Info($"Saved {configContent.Length} bytes to '{configFile}'."); } + + // make sure we have the right user agent data added from the jitconfig + HostContext.LoadDefaultUserAgents(); + VssUtil.InitializeVssClientSettings(HostContext.UserAgents, HostContext.WebProxy); } catch (Exception ex) { diff --git a/src/Runner.PluginHost/Runner.PluginHost.csproj b/src/Runner.PluginHost/Runner.PluginHost.csproj index df30f3450a2..81a8d2e4304 100644 --- a/src/Runner.PluginHost/Runner.PluginHost.csproj +++ b/src/Runner.PluginHost/Runner.PluginHost.csproj @@ -1,11 +1,12 @@  - net6.0 + net8.0 Exe win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 + true true - NU1701;NU1603 + NU1701;NU1603;SYSLIB0050;SYSLIB0051 $(Version) false true diff --git a/src/Runner.Plugins/Runner.Plugins.csproj b/src/Runner.Plugins/Runner.Plugins.csproj index 39245a3f7a7..a786cf1cd1b 100644 --- a/src/Runner.Plugins/Runner.Plugins.csproj +++ b/src/Runner.Plugins/Runner.Plugins.csproj @@ -1,11 +1,12 @@  - net6.0 + net8.0 Library win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 + true true - NU1701;NU1603 + NU1701;NU1603;SYSLIB0050;SYSLIB0051 $(Version) diff --git a/src/Runner.Sdk/Runner.Sdk.csproj b/src/Runner.Sdk/Runner.Sdk.csproj index 202e8669a64..55dbf12627c 100644 --- a/src/Runner.Sdk/Runner.Sdk.csproj +++ b/src/Runner.Sdk/Runner.Sdk.csproj @@ -1,11 +1,12 @@  - net6.0 + net8.0 Library win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 + true true - NU1701;NU1603 + NU1701;NU1603;SYSLIB0050;SYSLIB0051 $(Version) @@ -14,9 +15,9 @@ - - - + + + diff --git a/src/Runner.Sdk/Util/UrlUtil.cs b/src/Runner.Sdk/Util/UrlUtil.cs index 01658da05c9..52ce3a0cbf9 100644 --- a/src/Runner.Sdk/Util/UrlUtil.cs +++ b/src/Runner.Sdk/Util/UrlUtil.cs @@ -60,5 +60,15 @@ public static string GetGitHubRequestId(HttpResponseHeaders headers) } return string.Empty; } + + public static string GetVssRequestId(HttpResponseHeaders headers) + { + if (headers != null && + headers.TryGetValues("x-vss-e2eid", out var headerValues)) + { + return headerValues.FirstOrDefault(); + } + return string.Empty; + } } } diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index 0ec72b0585f..f32cad28ea9 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -1102,6 +1102,7 @@ private async Task DownloadRepositoryArchive(IExecutionContext executionContext, int timeoutSeconds = 20 * 60; while (retryCount < 3) { + string requestId = string.Empty; using (var actionDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds))) using (var actionDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(actionDownloadTimeout.Token, executionContext.CancellationToken)) { @@ -1117,7 +1118,7 @@ private async Task DownloadRepositoryArchive(IExecutionContext executionContext, httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents); using (var response = await httpClient.GetAsync(downloadUrl)) { - var requestId = UrlUtil.GetGitHubRequestId(response.Headers); + requestId = UrlUtil.GetGitHubRequestId(response.Headers); if (!string.IsNullOrEmpty(requestId)) { Trace.Info($"Request URL: {downloadUrl} X-GitHub-Request-Id: {requestId} Http Status: {response.StatusCode}"); @@ -1155,7 +1156,7 @@ private async Task DownloadRepositoryArchive(IExecutionContext executionContext, catch (OperationCanceledException ex) when (!executionContext.CancellationToken.IsCancellationRequested && retryCount >= 2) { Trace.Info($"Action download final retry timeout after {timeoutSeconds} seconds."); - throw new TimeoutException($"Action '{downloadUrl}' download has timed out. Error: {ex.Message}"); + throw new TimeoutException($"Action '{downloadUrl}' download has timed out. Error: {ex.Message} {requestId}"); } catch (ActionNotFoundException) { @@ -1170,11 +1171,11 @@ private async Task DownloadRepositoryArchive(IExecutionContext executionContext, if (actionDownloadTimeout.Token.IsCancellationRequested) { // action download didn't finish within timeout - executionContext.Warning($"Action '{downloadUrl}' didn't finish download within {timeoutSeconds} seconds."); + executionContext.Warning($"Action '{downloadUrl}' didn't finish download within {timeoutSeconds} seconds. {requestId}"); } else { - executionContext.Warning($"Failed to download action '{downloadUrl}'. Error: {ex.Message}"); + executionContext.Warning($"Failed to download action '{downloadUrl}'. Error: {ex.Message} {requestId}"); } } } diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index 77c145d1ddc..dfe92523089 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -83,7 +83,7 @@ public interface IExecutionContext : IRunnerService // Initialize void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token); void CancelToken(); - IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, ActionRunStage stage, Dictionary intraActionState = null, int? recordOrder = null, IPagingLogger logger = null, bool isEmbedded = false, CancellationTokenSource cancellationTokenSource = null, Guid embeddedId = default(Guid), string siblingScopeName = null, TimeSpan? timeout = null); + IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, ActionRunStage stage, Dictionary intraActionState = null, int? recordOrder = null, IPagingLogger logger = null, bool isEmbedded = false, List embeddedIssueCollector = null, CancellationTokenSource cancellationTokenSource = null, Guid embeddedId = default(Guid), string siblingScopeName = null, TimeSpan? timeout = null); IExecutionContext CreateEmbeddedChild(string scopeName, string contextName, Guid embeddedId, ActionRunStage stage, Dictionary intraActionState = null, string siblingScopeName = null); // logging @@ -135,7 +135,6 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext private readonly TimelineRecord _record = new(); private readonly Dictionary _detailRecords = new(); - private readonly List _embeddedIssueCollector; private readonly object _loggerLock = new(); private readonly object _matchersLock = new(); private readonly ExecutionContext _parentExecutionContext; @@ -154,6 +153,7 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext private CancellationTokenSource _cancellationTokenSource; private TaskCompletionSource _forceCompleted = new(); private bool _throttlingReported = false; + private List _embeddedIssueCollector; // only job level ExecutionContext will track throttling delay. private long _totalThrottlingDelayInMilliseconds = 0; @@ -356,6 +356,7 @@ public IExecutionContext CreateChild( int? recordOrder = null, IPagingLogger logger = null, bool isEmbedded = false, + List embeddedIssueCollector = null, CancellationTokenSource cancellationTokenSource = null, Guid embeddedId = default(Guid), string siblingScopeName = null, @@ -365,6 +366,10 @@ public IExecutionContext CreateChild( var child = new ExecutionContext(this, isEmbedded); child.Initialize(HostContext); + if ((Global.Variables.GetBoolean("RunService.FixEmbeddedIssues") ?? false) && embeddedIssueCollector != null) + { + child._embeddedIssueCollector = embeddedIssueCollector; + } child.Global = Global; child.ScopeName = scopeName; child.ContextName = contextName; @@ -433,7 +438,7 @@ public IExecutionContext CreateEmbeddedChild( Dictionary intraActionState = null, string siblingScopeName = null) { - return Root.CreateChild(_record.Id, _record.Name, _record.Id.ToString("N"), scopeName, contextName, stage, logger: _logger, isEmbedded: true, cancellationTokenSource: null, intraActionState: intraActionState, embeddedId: embeddedId, siblingScopeName: siblingScopeName, timeout: GetRemainingTimeout(), recordOrder: _record.Order); + return Root.CreateChild(_record.Id, _record.Name, _record.Id.ToString("N"), scopeName, contextName, stage, logger: _logger, isEmbedded: true, embeddedIssueCollector: _embeddedIssueCollector, cancellationTokenSource: null, intraActionState: intraActionState, embeddedId: embeddedId, siblingScopeName: siblingScopeName, timeout: GetRemainingTimeout(), recordOrder: _record.Order); } public void Start(string currentOperation = null) @@ -503,6 +508,9 @@ public TaskResult Complete(TaskResult? result = null, string currentOperation = Status = _record.State, Number = _record.Order, Name = _record.Name, + ActionName = StepTelemetry?.Action, + Ref = StepTelemetry?.Ref, + Type = StepTelemetry?.Type, StartedAt = _record.StartTime, CompletedAt = _record.FinishTime, Annotations = new List() @@ -520,7 +528,6 @@ public TaskResult Complete(TaskResult? result = null, string currentOperation = Global.StepsResult.Add(stepResult); } - if (Root != this) { // only dispose TokenSource for step level ExecutionContext @@ -808,11 +815,6 @@ public void InitializeJob(Pipelines.AgentJobRequestMessage message, Cancellation Global.Variables = new Variables(HostContext, variables); - if (Global.Variables.GetBoolean("DistributedTask.ForceInternalNodeVersionOnRunnerTo16") ?? false) - { - Environment.SetEnvironmentVariable(Constants.Variables.Agent.ForcedInternalNodeVersion, "node16"); - } - // Environment variables shared across all actions Global.EnvironmentVariables = new Dictionary(VarUtil.EnvironmentVariableKeyComparer); @@ -837,7 +839,6 @@ public void InitializeJob(Pipelines.AgentJobRequestMessage message, Cancellation // Actions environment ActionsEnvironment = message.ActionsEnvironment; - // Service container info Global.ServiceContainers = new List(); @@ -1418,7 +1419,7 @@ private static void ResolvePathsInExpressionValuesDictionary(this IExecutionCont { if (key == PipelineTemplateConstants.HostWorkspace) { - // The HostWorkspace context var is excluded so that there is a var that always points to the host path. + // The HostWorkspace context var is excluded so that there is a var that always points to the host path. // This var can be used to translate back from container paths, e.g. in HashFilesFunction, which always runs on the host machine continue; } diff --git a/src/Runner.Worker/Handlers/HandlerFactory.cs b/src/Runner.Worker/Handlers/HandlerFactory.cs index f857f89a9da..5ea9361cda7 100644 --- a/src/Runner.Worker/Handlers/HandlerFactory.cs +++ b/src/Runner.Worker/Handlers/HandlerFactory.cs @@ -57,72 +57,13 @@ public IHandler Create( handler = HostContext.CreateService(); var nodeData = data as NodeJSActionExecutionData; - // With node12 EoL in 04/2022, we want to be able to uniformly upgrade all JS actions to node16 from the server - if (string.Equals(nodeData.NodeVersion, "node12", StringComparison.InvariantCultureIgnoreCase)) + // With node12 EoL in 04/2022 and node16 EoL in 09/23, we want to execute all JS actions using node20 + if (string.Equals(nodeData.NodeVersion, "node12", StringComparison.InvariantCultureIgnoreCase) || + string.Equals(nodeData.NodeVersion, "node16", StringComparison.InvariantCultureIgnoreCase)) { - var repoAction = action as Pipelines.RepositoryPathReference; - if (repoAction != null) - { - var warningActions = new HashSet(); - if (executionContext.Global.Variables.TryGetValue(Constants.Runner.EnforcedNode12DetectedAfterEndOfLifeEnvVariable, out var node16ForceWarnings)) - { - warningActions = StringUtil.ConvertFromJson>(node16ForceWarnings); - } - - string repoActionFullName; - if (string.IsNullOrEmpty(repoAction.Name)) - { - repoActionFullName = repoAction.Path; // local actions don't have a 'Name' - } - else - { - repoActionFullName = $"{repoAction.Name}/{repoAction.Path ?? string.Empty}".TrimEnd('/') + $"@{repoAction.Ref}"; - } - - warningActions.Add(repoActionFullName); - executionContext.Global.Variables.Set("Node16ForceActionsWarnings", StringUtil.ConvertToJson(warningActions)); - } - nodeData.NodeVersion = "node16"; + nodeData.NodeVersion = "node20"; } - var localForceActionsToNode20 = StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable(Constants.Variables.Agent.ManualForceActionsToNode20)); - executionContext.Global.EnvironmentVariables.TryGetValue(Constants.Variables.Actions.ManualForceActionsToNode20, out var workflowForceActionsToNode20); - var enforceNode20Locally = !string.IsNullOrWhiteSpace(workflowForceActionsToNode20) ? StringUtil.ConvertToBoolean(workflowForceActionsToNode20) : localForceActionsToNode20; - if (string.Equals(nodeData.NodeVersion, "node16") - && ((executionContext.Global.Variables.GetBoolean("DistributedTask.ForceGithubJavascriptActionsToNode20") ?? false) || enforceNode20Locally)) - { - executionContext.Global.EnvironmentVariables.TryGetValue(Constants.Variables.Actions.AllowActionsUseUnsecureNodeVersion, out var workflowOptOut); - var isWorkflowOptOutSet = !string.IsNullOrWhiteSpace(workflowOptOut); - var isLocalOptOut = StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable(Constants.Variables.Actions.AllowActionsUseUnsecureNodeVersion)); - bool isOptOut = isWorkflowOptOutSet ? StringUtil.ConvertToBoolean(workflowOptOut) : isLocalOptOut; - - if (!isOptOut) - { - var repoAction = action as Pipelines.RepositoryPathReference; - if (repoAction != null) - { - var warningActions = new HashSet(); - if (executionContext.Global.Variables.TryGetValue(Constants.Runner.EnforcedNode16DetectedAfterEndOfLifeEnvVariable, out var node20ForceWarnings)) - { - warningActions = StringUtil.ConvertFromJson>(node20ForceWarnings); - } - - string repoActionFullName; - if (string.IsNullOrEmpty(repoAction.Name)) - { - repoActionFullName = repoAction.Path; // local actions don't have a 'Name' - } - else - { - repoActionFullName = $"{repoAction.Name}/{repoAction.Path ?? string.Empty}".TrimEnd('/') + $"@{repoAction.Ref}"; - } - - warningActions.Add(repoActionFullName); - executionContext.Global.Variables.Set(Constants.Runner.EnforcedNode16DetectedAfterEndOfLifeEnvVariable, StringUtil.ConvertToJson(warningActions)); - } - nodeData.NodeVersion = "node20"; - } - } (handler as INodeScriptActionHandler).Data = nodeData; } else if (data.ExecutionType == ActionExecutionType.Script) diff --git a/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs b/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs index 9d412a72314..a399f13d1d2 100644 --- a/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs +++ b/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs @@ -72,6 +72,11 @@ public async Task RunAsync(ActionRunStage stage) Environment["ACTIONS_RESULTS_URL"] = resultsUrl; } + if (ExecutionContext.Global.Variables.GetBoolean("actions_uses_cache_service_v2") ?? false) + { + Environment["ACTIONS_CACHE_SERVICE_V2"] = bool.TrueString; + } + // Resolve the target script. string target = null; if (stage == ActionRunStage.Main) @@ -93,7 +98,6 @@ public async Task RunAsync(ActionRunStage stage) ExecutionContext.StepTelemetry.HasPreStep = Data.HasPre; ExecutionContext.StepTelemetry.HasPostStep = Data.HasPost; } - ExecutionContext.StepTelemetry.Type = Data.NodeVersion; ArgUtil.NotNullOrEmpty(target, nameof(target)); target = Path.Combine(ActionDirectory, target); @@ -106,24 +110,8 @@ public async Task RunAsync(ActionRunStage stage) workingDirectory = HostContext.GetDirectory(WellKnownDirectory.Work); } - if (string.Equals(Data.NodeVersion, "node12", StringComparison.OrdinalIgnoreCase) && - Constants.Runner.PlatformArchitecture.Equals(Constants.Architecture.Arm64)) - { - ExecutionContext.Output($"The node12 is not supported. Use node16 instead."); - Data.NodeVersion = "node16"; - } - - string forcedNodeVersion = System.Environment.GetEnvironmentVariable(Constants.Variables.Agent.ForcedActionsNodeVersion); - if (forcedNodeVersion == "node16" && Data.NodeVersion != "node16") - { - Data.NodeVersion = "node16"; - } - - if (forcedNodeVersion == "node20" && Data.NodeVersion != "node20") - { - Data.NodeVersion = "node20"; - } var nodeRuntimeVersion = await StepHost.DetermineNodeRuntimeVersion(ExecutionContext, Data.NodeVersion); + ExecutionContext.StepTelemetry.Type = nodeRuntimeVersion; string file = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), nodeRuntimeVersion, "bin", $"node{IOUtil.ExeExtension}"); // Format the arguments passed to node. @@ -143,28 +131,6 @@ public async Task RunAsync(ActionRunStage stage) // Remove environment variable that may cause conflicts with the node within the runner. Environment.Remove("NODE_ICU_DATA"); // https://github.com/actions/runner/issues/795 - if (string.Equals(Data.NodeVersion, Constants.Runner.DeprecatedNodeVersion, StringComparison.OrdinalIgnoreCase) && (ExecutionContext.Global.Variables.GetBoolean(Constants.Runner.Features.Node16Warning) ?? false)) - { - var repoAction = Action as RepositoryPathReference; - var warningActions = new HashSet(); - if (ExecutionContext.Global.Variables.TryGetValue(Constants.Runner.DeprecatedNodeDetectedAfterEndOfLifeActions, out var deprecatedNodeWarnings)) - { - warningActions = StringUtil.ConvertFromJson>(deprecatedNodeWarnings); - } - - if (string.IsNullOrEmpty(repoAction.Name)) - { - // local actions don't have a 'Name' - warningActions.Add(repoAction.Path); - } - else - { - warningActions.Add($"{repoAction.Name}/{repoAction.Path ?? string.Empty}".TrimEnd('/') + $"@{repoAction.Ref}"); - } - - ExecutionContext.Global.Variables.Set(Constants.Runner.DeprecatedNodeDetectedAfterEndOfLifeActions, StringUtil.ConvertToJson(warningActions)); - } - using (var stdoutManager = new OutputManager(ExecutionContext, ActionCommandManager)) using (var stderrManager = new OutputManager(ExecutionContext, ActionCommandManager)) { diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index a36e4beb4d8..58e8929b44a 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -17,6 +17,7 @@ using GitHub.Runner.Common.Util; using GitHub.Runner.Sdk; using GitHub.Services.Common; +using Newtonsoft.Json; using Pipelines = GitHub.DistributedTask.Pipelines; namespace GitHub.Runner.Worker @@ -42,11 +43,13 @@ public interface IJobExtension : IRunnerService public sealed class JobExtension : RunnerService, IJobExtension { private readonly HashSet _existingProcesses = new(StringComparer.OrdinalIgnoreCase); - private readonly List> _connectivityCheckTasks = new(); + private readonly List> _connectivityCheckTasks = new(); private bool _processCleanup; private string _processLookupId = $"github_{Guid.NewGuid()}"; private CancellationTokenSource _diskSpaceCheckToken = new(); private Task _diskSpaceCheckTask = null; + private CancellationTokenSource _serviceConnectivityCheckToken = new(); + private Task _serviceConnectivityCheckTask = null; // Download all required actions. // Make sure all condition inputs are valid. @@ -127,10 +130,6 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel } } - // Check OS warning - var osWarningChecker = HostContext.GetService(); - await osWarningChecker.CheckOSAsync(context); - try { var tokenPermissions = jobContext.Global.Variables.Get("system.github.token.permissions") ?? ""; @@ -403,7 +402,7 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel var snapshotOperationProvider = HostContext.GetService(); jobContext.RegisterPostJobStep(new JobExtensionRunner( runAsync: (executionContext, _) => snapshotOperationProvider.CreateSnapshotRequestAsync(executionContext, snapshotRequest), - condition: $"{PipelineTemplateConstants.Success}()", + condition: snapshotRequest.Condition, displayName: $"Create custom image", data: null)); } @@ -458,11 +457,14 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel { foreach (var checkUrl in checkUrls) { - _connectivityCheckTasks.Add(CheckConnectivity(checkUrl)); + _connectivityCheckTasks.Add(CheckConnectivity(checkUrl, accessToken: string.Empty, timeoutInSeconds: 5, token: CancellationToken.None)); } } } + Trace.Info($"Start checking service connectivity in background."); + _serviceConnectivityCheckTask = CheckServiceConnectivityAsync(context, _serviceConnectivityCheckToken.Token); + return steps; } catch (OperationCanceledException ex) when (jobContext.CancellationToken.IsCancellationRequested) @@ -696,7 +698,7 @@ public async Task FinalizeJob(IExecutionContext jobContext, Pipelines.AgentJobRe { var result = await check; Trace.Info($"Connectivity check result: {result}"); - context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.ConnectivityCheck, Message = result }); + context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.ConnectivityCheck, Message = $"{result.EndpointUrl}: {result.StatusCode}" }); } } catch (Exception ex) @@ -706,6 +708,22 @@ public async Task FinalizeJob(IExecutionContext jobContext, Pipelines.AgentJobRe context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.ConnectivityCheck, Message = $"Fail to check server connectivity. {ex.Message}" }); } } + + // Collect service connectivity check result + if (_serviceConnectivityCheckTask != null) + { + _serviceConnectivityCheckToken.Cancel(); + try + { + await _serviceConnectivityCheckTask; + } + catch (Exception ex) + { + Trace.Error($"Fail to check service connectivity."); + Trace.Error(ex); + context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.ConnectivityCheck, Message = $"Fail to check service connectivity. {ex.Message}" }); + } + } } catch (Exception ex) { @@ -721,11 +739,13 @@ public async Task FinalizeJob(IExecutionContext jobContext, Pipelines.AgentJobRe } } - private async Task CheckConnectivity(string endpointUrl) + private async Task CheckConnectivity(string endpointUrl, string accessToken, int timeoutInSeconds, CancellationToken token) { Trace.Info($"Check server connectivity for {endpointUrl}."); - string result = string.Empty; - using (var timeoutTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(5))) + CheckResult result = new CheckResult() { EndpointUrl = endpointUrl }; + var stopwatch = Stopwatch.StartNew(); + using (var timeoutTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutInSeconds))) + using (var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token, timeoutTokenSource.Token)) { try { @@ -733,21 +753,44 @@ private async Task CheckConnectivity(string endpointUrl) using (var httpClient = new HttpClient(httpClientHandler)) { httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents); - var response = await httpClient.GetAsync(endpointUrl, timeoutTokenSource.Token); - result = $"{endpointUrl}: {response.StatusCode}"; + if (!string.IsNullOrEmpty(accessToken)) + { + httpClient.DefaultRequestHeaders.Add("Authorization", $"Bearer {accessToken}"); + } + + var response = await httpClient.GetAsync(endpointUrl, linkedTokenSource.Token); + result.StatusCode = $"{response.StatusCode}"; + + var githubRequestId = UrlUtil.GetGitHubRequestId(response.Headers); + var vssRequestId = UrlUtil.GetVssRequestId(response.Headers); + if (!string.IsNullOrEmpty(githubRequestId)) + { + result.RequestId = githubRequestId; + } + else if (!string.IsNullOrEmpty(vssRequestId)) + { + result.RequestId = vssRequestId; + } } } + catch (Exception ex) when (ex is OperationCanceledException && token.IsCancellationRequested) + { + Trace.Error($"Request canceled during connectivity check: {ex}"); + result.StatusCode = "canceled"; + } catch (Exception ex) when (ex is OperationCanceledException && timeoutTokenSource.IsCancellationRequested) { Trace.Error($"Request timeout during connectivity check: {ex}"); - result = $"{endpointUrl}: timeout"; + result.StatusCode = "timeout"; } catch (Exception ex) { Trace.Error($"Catch exception during connectivity check: {ex}"); - result = $"{endpointUrl}: {ex.Message}"; + result.StatusCode = $"{ex.Message}"; } } + stopwatch.Stop(); + result.DurationInMs = (int)stopwatch.ElapsedMilliseconds; return result; } @@ -785,6 +828,84 @@ private async Task CheckDiskSpaceAsync(IExecutionContext context, CancellationTo } } + private async Task CheckServiceConnectivityAsync(IExecutionContext context, CancellationToken token) + { + var connectionTest = context.Global.Variables.Get(WellKnownDistributedTaskVariables.RunnerServiceConnectivityTest); + if (string.IsNullOrEmpty(connectionTest)) + { + return; + } + + ServiceConnectivityCheckInput checkConnectivityInfo; + try + { + checkConnectivityInfo = StringUtil.ConvertFromJson(connectionTest); + } + catch (Exception ex) + { + context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $"Fail to parse JSON. {ex.Message}" }); + return; + } + + if (checkConnectivityInfo == null) + { + return; + } + + // make sure interval is at least 10 seconds + checkConnectivityInfo.IntervalInSecond = Math.Max(10, checkConnectivityInfo.IntervalInSecond); + + var systemConnection = context.Global.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); + var accessToken = systemConnection.Authorization.Parameters[EndpointAuthorizationParameters.AccessToken]; + + var testResult = new ServiceConnectivityCheckResult(); + while (!token.IsCancellationRequested) + { + foreach (var endpoint in checkConnectivityInfo.Endpoints) + { + if (string.IsNullOrEmpty(endpoint.Key) || string.IsNullOrEmpty(endpoint.Value)) + { + continue; + } + + if (!testResult.EndpointsResult.ContainsKey(endpoint.Key)) + { + testResult.EndpointsResult[endpoint.Key] = new List(); + } + + try + { + var result = await CheckConnectivity(endpoint.Value, accessToken: accessToken, timeoutInSeconds: checkConnectivityInfo.RequestTimeoutInSecond, token); + testResult.EndpointsResult[endpoint.Key].Add($"{result.StartTime:s}: {result.StatusCode} - {result.RequestId} - {result.DurationInMs}ms"); + if (!testResult.HasFailure && + result.StatusCode != "OK" && + result.StatusCode != "canceled") + { + // track if any endpoint is not reachable + testResult.HasFailure = true; + } + } + catch (Exception ex) + { + testResult.EndpointsResult[endpoint.Key].Add($"{DateTime.UtcNow:s}: {ex.Message}"); + } + } + + try + { + await Task.Delay(TimeSpan.FromSeconds(checkConnectivityInfo.IntervalInSecond), token); + } + catch (TaskCanceledException) + { + // ignore + } + } + + var telemetryData = StringUtil.ConvertToJson(testResult, Formatting.None); + Trace.Verbose($"Connectivity check result: {telemetryData}"); + context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.ConnectivityCheck, Message = telemetryData }); + } + private Dictionary SnapshotProcesses() { Dictionary snapshot = new(); @@ -816,5 +937,23 @@ private static void ValidateJobContainer(JobContainer container) throw new ArgumentException("Jobs without a job container are forbidden on this runner, please add a 'container:' to your job or contact your self-hosted runner administrator."); } } + + private class CheckResult + { + public CheckResult() + { + StartTime = DateTime.UtcNow; + } + + public string EndpointUrl { get; set; } + + public DateTime StartTime { get; set; } + + public string StatusCode { get; set; } + + public string RequestId { get; set; } + + public int DurationInMs { get; set; } + } } } diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs index ad265ecaf68..7ab506a53b2 100644 --- a/src/Runner.Worker/JobRunner.cs +++ b/src/Runner.Worker/JobRunner.cs @@ -15,6 +15,7 @@ using GitHub.Runner.Sdk; using GitHub.Services.Common; using GitHub.Services.WebApi; +using Sdk.RSWebApi.Contracts; using Pipelines = GitHub.DistributedTask.Pipelines; namespace GitHub.Runner.Worker @@ -49,7 +50,8 @@ public async Task RunAsync(AgentJobRequestMessage message, Cancellat if (message.Variables.TryGetValue(Constants.Variables.System.OrchestrationId, out VariableValue orchestrationId) && !string.IsNullOrEmpty(orchestrationId.Value)) { - HostContext.UserAgents.Add(new ProductInfoHeaderValue("OrchestrationId", orchestrationId.Value)); + // make the orchestration id the first item in the user-agent header to avoid get truncated in server log. + HostContext.UserAgents.Insert(0, new ProductInfoHeaderValue("OrchestrationId", orchestrationId.Value)); // make sure orchestration id is in the user-agent header. VssUtil.InitializeVssClientSettings(HostContext.UserAgents, HostContext.WebProxy); @@ -278,26 +280,14 @@ private async Task CompleteJobAsync(IRunServer runServer, IExecution { jobContext.Debug($"Finishing: {message.JobDisplayName}"); TaskResult result = jobContext.Complete(taskResult); - if (jobContext.Global.Variables.TryGetValue(Constants.Runner.DeprecatedNodeDetectedAfterEndOfLifeActions, out var deprecatedNodeWarnings)) - { - var actions = string.Join(", ", StringUtil.ConvertFromJson>(deprecatedNodeWarnings)); - jobContext.Warning(string.Format(Constants.Runner.DetectedNodeAfterEndOfLifeMessage, actions)); - } - - if (jobContext.Global.Variables.TryGetValue(Constants.Runner.EnforcedNode12DetectedAfterEndOfLifeEnvVariable, out var node16ForceWarnings)) - { - var actions = string.Join(", ", StringUtil.ConvertFromJson>(node16ForceWarnings)); - jobContext.Warning(string.Format(Constants.Runner.EnforcedNode12DetectedAfterEndOfLife, actions)); - } - if (jobContext.Global.Variables.TryGetValue(Constants.Runner.EnforcedNode16DetectedAfterEndOfLifeEnvVariable, out var node20ForceWarnings) && (jobContext.Global.Variables.GetBoolean("DistributedTask.ForceGithubJavascriptActionsToNode20") ?? false)) + var jobQueueTelemetry = await ShutdownQueue(throwOnFailure: false); + // include any job telemetry from the background upload process. + if (jobQueueTelemetry?.Count > 0) { - var actions = string.Join(", ", StringUtil.ConvertFromJson>(node20ForceWarnings)); - jobContext.Warning(string.Format(Constants.Runner.EnforcedNode16DetectedAfterEndOfLife, actions)); + jobContext.Global.JobTelemetry.AddRange(jobQueueTelemetry); } - await ShutdownQueue(throwOnFailure: false); - // Make sure to clean temp after file upload since they may be pending fileupload still use the TEMP dir. _tempDirectoryManager?.CleanupTempDirectory(); @@ -314,6 +304,13 @@ private async Task CompleteJobAsync(IRunServer runServer, IExecution environmentUrl = urlStringToken.Value; } + // Get telemetry + IList telemetry = null; + if (jobContext.Global.JobTelemetry.Count > 0) + { + telemetry = jobContext.Global.JobTelemetry.Select(x => new Telemetry { Type = x.Type.ToString(), Message = x.Message, }).ToList(); + } + Trace.Info($"Raising job completed against run service"); var completeJobRetryLimit = 5; var exceptions = new List(); @@ -321,7 +318,7 @@ private async Task CompleteJobAsync(IRunServer runServer, IExecution { try { - await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, result, jobContext.JobOutputs, jobContext.Global.StepsResult, jobContext.Global.JobAnnotations, environmentUrl, default); + await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, result, jobContext.JobOutputs, jobContext.Global.StepsResult, jobContext.Global.JobAnnotations, environmentUrl, telemetry, default); return result; } catch (Exception ex) @@ -346,74 +343,14 @@ private async Task CompleteJobAsync(IJobServer jobServer, IExecution if (_runnerSettings.DisableUpdate == true) { - try - { - var currentVersion = new PackageVersion(BuildConstants.RunnerPackage.Version); - ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); - VssCredentials serverCredential = VssUtil.GetVssCredential(systemConnection); - - var runnerServer = HostContext.GetService(); - await runnerServer.ConnectAsync(systemConnection.Url, serverCredential); - var serverPackages = await runnerServer.GetPackagesAsync("agent", BuildConstants.RunnerPackage.PackageName, 5, includeToken: false, cancellationToken: CancellationToken.None); - if (serverPackages.Count > 0) - { - serverPackages = serverPackages.OrderByDescending(x => x.Version).ToList(); - Trace.Info($"Newer packages {StringUtil.ConvertToJson(serverPackages.Select(x => x.Version.ToString()))}"); - - var warnOnFailedJob = false; // any minor/patch version behind. - var warnOnOldRunnerVersion = false; // >= 2 minor version behind - if (serverPackages.Any(x => x.Version.CompareTo(currentVersion) > 0)) - { - Trace.Info($"Current runner version {currentVersion} is behind the latest runner version {serverPackages[0].Version}."); - warnOnFailedJob = true; - } - - if (serverPackages.Where(x => x.Version.Major == currentVersion.Major && x.Version.Minor > currentVersion.Minor).Count() > 1) - { - Trace.Info($"Current runner version {currentVersion} is way behind the latest runner version {serverPackages[0].Version}."); - warnOnOldRunnerVersion = true; - } - - if (result == TaskResult.Failed && warnOnFailedJob) - { - jobContext.Warning($"This job failure may be caused by using an out of date self-hosted runner. You are currently using runner version {currentVersion}. Please update to the latest version {serverPackages[0].Version}"); - } - else if (warnOnOldRunnerVersion) - { - jobContext.Warning($"This self-hosted runner is currently using runner version {currentVersion}. This version is out of date. Please update to the latest version {serverPackages[0].Version}"); - } - } - } - catch (Exception ex) - { - // Ignore any error since suggest runner update is best effort. - Trace.Error($"Caught exception during runner version check: {ex}"); - } - } - - if (jobContext.Global.Variables.TryGetValue(Constants.Runner.DeprecatedNodeDetectedAfterEndOfLifeActions, out var deprecatedNodeWarnings)) - { - var actions = string.Join(", ", StringUtil.ConvertFromJson>(deprecatedNodeWarnings)); - jobContext.Warning(string.Format(Constants.Runner.DetectedNodeAfterEndOfLifeMessage, actions)); - } - - if (jobContext.Global.Variables.TryGetValue(Constants.Runner.EnforcedNode12DetectedAfterEndOfLifeEnvVariable, out var node16ForceWarnings)) - { - var actions = string.Join(", ", StringUtil.ConvertFromJson>(node16ForceWarnings)); - jobContext.Warning(string.Format(Constants.Runner.EnforcedNode12DetectedAfterEndOfLife, actions)); - } - - if (jobContext.Global.Variables.TryGetValue(Constants.Runner.EnforcedNode16DetectedAfterEndOfLifeEnvVariable, out var node20ForceWarnings)) - { - var actions = string.Join(", ", StringUtil.ConvertFromJson>(node20ForceWarnings)); - jobContext.Warning(string.Format(Constants.Runner.EnforcedNode16DetectedAfterEndOfLife, actions)); + await WarningOutdatedRunnerAsync(jobContext, message, result); } try { var jobQueueTelemetry = await ShutdownQueue(throwOnFailure: true); // include any job telemetry from the background upload process. - if (jobQueueTelemetry.Count > 0) + if (jobQueueTelemetry?.Count > 0) { jobContext.Global.JobTelemetry.AddRange(jobQueueTelemetry); } @@ -541,5 +478,52 @@ private async Task> ShutdownQueue(bool throwOnFailure) return Array.Empty(); } + + private async Task WarningOutdatedRunnerAsync(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, TaskResult result) + { + try + { + var currentVersion = new PackageVersion(BuildConstants.RunnerPackage.Version); + ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); + VssCredentials serverCredential = VssUtil.GetVssCredential(systemConnection); + + var runnerServer = HostContext.GetService(); + await runnerServer.ConnectAsync(systemConnection.Url, serverCredential); + var serverPackages = await runnerServer.GetPackagesAsync("agent", BuildConstants.RunnerPackage.PackageName, 5, includeToken: false, cancellationToken: CancellationToken.None); + if (serverPackages.Count > 0) + { + serverPackages = serverPackages.OrderByDescending(x => x.Version).ToList(); + Trace.Info($"Newer packages {StringUtil.ConvertToJson(serverPackages.Select(x => x.Version.ToString()))}"); + + var warnOnFailedJob = false; // any minor/patch version behind. + var warnOnOldRunnerVersion = false; // >= 2 minor version behind + if (serverPackages.Any(x => x.Version.CompareTo(currentVersion) > 0)) + { + Trace.Info($"Current runner version {currentVersion} is behind the latest runner version {serverPackages[0].Version}."); + warnOnFailedJob = true; + } + + if (serverPackages.Where(x => x.Version.Major == currentVersion.Major && x.Version.Minor > currentVersion.Minor).Count() > 1) + { + Trace.Info($"Current runner version {currentVersion} is way behind the latest runner version {serverPackages[0].Version}."); + warnOnOldRunnerVersion = true; + } + + if (result == TaskResult.Failed && warnOnFailedJob) + { + jobContext.Warning($"This job failure may be caused by using an out of date self-hosted runner. You are currently using runner version {currentVersion}. Please update to the latest version {serverPackages[0].Version}"); + } + else if (warnOnOldRunnerVersion) + { + jobContext.Warning($"This self-hosted runner is currently using runner version {currentVersion}. This version is out of date. Please update to the latest version {serverPackages[0].Version}"); + } + } + } + catch (Exception ex) + { + // Ignore any error since suggest runner update is best effort. + Trace.Error($"Caught exception during runner version check: {ex}"); + } + } } } diff --git a/src/Runner.Worker/OSWarningChecker.cs b/src/Runner.Worker/OSWarningChecker.cs deleted file mode 100644 index 765a41a5215..00000000000 --- a/src/Runner.Worker/OSWarningChecker.cs +++ /dev/null @@ -1,110 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; -using GitHub.DistributedTask.WebApi; -using GitHub.Runner.Common; -using GitHub.Runner.Sdk; - -namespace GitHub.Runner.Worker -{ - [ServiceLocator(Default = typeof(OSWarningChecker))] - public interface IOSWarningChecker : IRunnerService - { - Task CheckOSAsync(IExecutionContext context); - } - - public sealed class OSWarningChecker : RunnerService, IOSWarningChecker - { - private static TimeSpan s_regexTimeout = TimeSpan.FromSeconds(1); - - public async Task CheckOSAsync(IExecutionContext context) - { - ArgUtil.NotNull(context, nameof(context)); - if (!context.Global.Variables.System_TestDotNet8Compatibility) - { - return; - } - - context.Output("Testing runner upgrade compatibility"); - List output = new(); - object outputLock = new(); - try - { - using (var process = HostContext.CreateService()) - { - process.OutputDataReceived += delegate (object sender, ProcessDataReceivedEventArgs stdout) - { - if (!string.IsNullOrEmpty(stdout.Data)) - { - lock (outputLock) - { - output.Add(stdout.Data); - Trace.Info(stdout.Data); - } - } - }; - - process.ErrorDataReceived += delegate (object sender, ProcessDataReceivedEventArgs stderr) - { - if (!string.IsNullOrEmpty(stderr.Data)) - { - lock (outputLock) - { - output.Add(stderr.Data); - Trace.Error(stderr.Data); - } - } - }; - - using (var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(10))) - { - int exitCode = await process.ExecuteAsync( - workingDirectory: HostContext.GetDirectory(WellKnownDirectory.Root), - fileName: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), "testDotNet8Compatibility", $"TestDotNet8Compatibility{IOUtil.ExeExtension}"), - arguments: string.Empty, - environment: null, - cancellationToken: cancellationTokenSource.Token); - - var outputStr = string.Join("\n", output).Trim(); - if (exitCode != 0 || !string.Equals(outputStr, "Hello from .NET 8!", StringComparison.Ordinal)) - { - var pattern = context.Global.Variables.System_DotNet8CompatibilityOutputPattern; - if (!string.IsNullOrEmpty(pattern)) - { - var regex = new Regex(pattern, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, s_regexTimeout); - if (!regex.IsMatch(outputStr)) - { - return; - } - } - - var warningMessage = context.Global.Variables.System_DotNet8CompatibilityWarning; - if (!string.IsNullOrEmpty(warningMessage)) - { - context.Warning(warningMessage); - } - - context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $".NET 8 OS compatibility test failed with exit code '{exitCode}' and output: {GetShortOutput(context, output)}" }); - } - } - } - } - catch (Exception ex) - { - Trace.Error("An error occurred while testing .NET 8 compatibility'"); - Trace.Error(ex); - context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $".NET 8 OS compatibility test encountered exception type '{ex.GetType().FullName}', message: '{ex.Message}', process output: '{GetShortOutput(context, output)}'" }); - } - } - - private static string GetShortOutput(IExecutionContext context, List output) - { - var length = context.Global.Variables.System_DotNet8CompatibilityOutputLength ?? 200; - var outputStr = string.Join("\n", output).Trim(); - return outputStr.Length > length ? string.Concat(outputStr.Substring(0, length), "[...]") : outputStr; - } - } -} diff --git a/src/Runner.Worker/Runner.Worker.csproj b/src/Runner.Worker/Runner.Worker.csproj index eee59b8721c..53c1610df3e 100644 --- a/src/Runner.Worker/Runner.Worker.csproj +++ b/src/Runner.Worker/Runner.Worker.csproj @@ -1,11 +1,12 @@ - net6.0 + net8.0 Exe win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 + true true - NU1701;NU1603 + NU1701;NU1603;SYSLIB0050;SYSLIB0051 $(Version) false true @@ -18,9 +19,9 @@ - - - + + + diff --git a/src/Runner.Worker/Variables.cs b/src/Runner.Worker/Variables.cs index 7627ec37984..916b82dc6a1 100644 --- a/src/Runner.Worker/Variables.cs +++ b/src/Runner.Worker/Variables.cs @@ -72,16 +72,8 @@ public Variables(IHostContext hostContext, IDictionary co public bool? Step_Debug => GetBoolean(Constants.Variables.Actions.StepDebug); - public string System_DotNet8CompatibilityWarning => Get(Constants.Variables.System.DotNet8CompatibilityWarning); - - public string System_DotNet8CompatibilityOutputPattern => Get(Constants.Variables.System.DotNet8CompatibilityOutputPattern); - - public int? System_DotNet8CompatibilityOutputLength => GetInt(Constants.Variables.System.DotNet8CompatibilityOutputLength); - public string System_PhaseDisplayName => Get(Constants.Variables.System.PhaseDisplayName); - public bool System_TestDotNet8Compatibility => GetBoolean(Constants.Variables.System.TestDotNet8Compatibility) ?? false; - public string Get(string name) { Variable variable; diff --git a/src/Sdk/Common/Common/Exceptions/PropertyExceptions.cs b/src/Sdk/Common/Common/Exceptions/PropertyExceptions.cs index 34c97b73a1d..72367654bf1 100644 --- a/src/Sdk/Common/Common/Exceptions/PropertyExceptions.cs +++ b/src/Sdk/Common/Common/Exceptions/PropertyExceptions.cs @@ -34,6 +34,7 @@ protected VssPropertyValidationException(SerializationInfo info, StreamingContex public String PropertyName { get; set; } + [Obsolete] [SecurityCritical] public override void GetObjectData(SerializationInfo info, StreamingContext context) { diff --git a/src/Sdk/Common/Common/VssException.cs b/src/Sdk/Common/Common/VssException.cs index 7cead78655d..5f8fb8c2554 100644 --- a/src/Sdk/Common/Common/VssException.cs +++ b/src/Sdk/Common/Common/VssException.cs @@ -127,6 +127,7 @@ protected VssException(SerializationInfo info, StreamingContext context) EventId = (int)info.GetValue("m_eventId", typeof(int)); } + [Obsolete] [SecurityCritical] public override void GetObjectData(SerializationInfo info, StreamingContext context) { diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs index a7e90fce334..8d81c7d2d53 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs @@ -30,6 +30,7 @@ public sealed class PipelineTemplateConstants public const String If = "if"; public const String Image = "image"; public const String ImageName = "image-name"; + public const String CustomImageVersion = "version"; public const String Include = "include"; public const String Inputs = "inputs"; public const String Job = "job"; diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs index 9d2c0bdca7a..40f6a13345f 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.ComponentModel; +using System.Globalization; using System.Linq; using GitHub.DistributedTask.Expressions2; using GitHub.DistributedTask.Expressions2.Sdk; @@ -349,6 +350,10 @@ internal static List> ConvertToJobServiceCont internal static Snapshot ConvertToJobSnapshotRequest(TemplateContext context, TemplateToken token) { string imageName = null; + string version = "1.*"; + string versionString = string.Empty; + var condition = $"{PipelineTemplateConstants.Success}()"; + if (token is StringToken snapshotStringLiteral) { imageName = snapshotStringLiteral.Value; @@ -359,11 +364,19 @@ internal static Snapshot ConvertToJobSnapshotRequest(TemplateContext context, Te foreach (var snapshotPropertyPair in snapshotMapping) { var propertyName = snapshotPropertyPair.Key.AssertString($"{PipelineTemplateConstants.Snapshot} key"); + var propertyValue = snapshotPropertyPair.Value; switch (propertyName.Value) { case PipelineTemplateConstants.ImageName: imageName = snapshotPropertyPair.Value.AssertString($"{PipelineTemplateConstants.Snapshot} {propertyName}").Value; break; + case PipelineTemplateConstants.If: + condition = ConvertToIfCondition(context, propertyValue, false); + break; + case PipelineTemplateConstants.CustomImageVersion: + versionString = propertyValue.AssertString($"job {PipelineTemplateConstants.Snapshot} {PipelineTemplateConstants.CustomImageVersion}").Value; + version = IsSnapshotImageVersionValid(versionString) ? versionString : null; + break; default: propertyName.AssertUnexpectedValue($"{PipelineTemplateConstants.Snapshot} key"); break; @@ -376,7 +389,26 @@ internal static Snapshot ConvertToJobSnapshotRequest(TemplateContext context, Te return null; } - return new Snapshot(imageName); + return new Snapshot(imageName) + { + Condition = condition, + Version = version + }; + } + + private static bool IsSnapshotImageVersionValid(string versionString) + { + var versionSegments = versionString.Split("."); + + if (versionSegments.Length != 2 || + !versionSegments[1].Equals("*") || + !Int32.TryParse(versionSegments[0], NumberStyles.None, CultureInfo.InvariantCulture, result: out int parsedMajor) || + parsedMajor < 0) + { + return false; + } + + return true; } private static ActionStep ConvertToStep( diff --git a/src/Sdk/DTPipelines/Pipelines/Snapshot.cs b/src/Sdk/DTPipelines/Pipelines/Snapshot.cs index 60f8da04f4f..c1a05674aea 100644 --- a/src/Sdk/DTPipelines/Pipelines/Snapshot.cs +++ b/src/Sdk/DTPipelines/Pipelines/Snapshot.cs @@ -1,17 +1,27 @@ using System; using System.Runtime.Serialization; +using GitHub.DistributedTask.ObjectTemplating.Tokens; +using GitHub.DistributedTask.Pipelines.ObjectTemplating; namespace GitHub.DistributedTask.Pipelines { [DataContract] public class Snapshot { - public Snapshot(string imageName) + public Snapshot(string imageName, string condition = null, string version = null) { ImageName = imageName; + Condition = condition ?? $"{PipelineTemplateConstants.Success}()"; + Version = version ?? "1.*"; } [DataMember(EmitDefaultValue = false)] public String ImageName { get; set; } + + [DataMember(EmitDefaultValue = false)] + public String Condition { get; set; } + + [DataMember(EmitDefaultValue = false)] + public String Version { get; set; } } } diff --git a/src/Sdk/DTPipelines/workflow-v1.0.json b/src/Sdk/DTPipelines/workflow-v1.0.json index a3837edff02..ec09cfe58b7 100644 --- a/src/Sdk/DTPipelines/workflow-v1.0.json +++ b/src/Sdk/DTPipelines/workflow-v1.0.json @@ -169,11 +169,28 @@ "image-name": { "type": "non-empty-string", "required": true + }, + "if": "snapshot-if", + "version": { + "type": "non-empty-string", + "required": false } } } }, + "snapshot-if": { + "context": [ + "github", + "inputs", + "vars", + "needs", + "strategy", + "matrix" + ], + "string": {} + }, + "runs-on": { "context": [ "github", diff --git a/src/Sdk/DTWebApi/WebApi/ServiceConnectivityCheck.cs b/src/Sdk/DTWebApi/WebApi/ServiceConnectivityCheck.cs new file mode 100644 index 00000000000..ff4845fec66 --- /dev/null +++ b/src/Sdk/DTWebApi/WebApi/ServiceConnectivityCheck.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; +using System.Runtime.Serialization; +using Newtonsoft.Json; + +namespace GitHub.DistributedTask.WebApi +{ + [DataContract] + public class ServiceConnectivityCheckInput + { + [JsonConstructor] + public ServiceConnectivityCheckInput() + { + Endpoints = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + [DataMember(EmitDefaultValue = false)] + public Dictionary Endpoints { get; set; } + + [DataMember(EmitDefaultValue = false)] + public int IntervalInSecond { get; set; } + + [DataMember(EmitDefaultValue = false)] + public int RequestTimeoutInSecond { get; set; } + } + + [DataContract] + public class ServiceConnectivityCheckResult + { + [JsonConstructor] + public ServiceConnectivityCheckResult() + { + EndpointsResult = new Dictionary>(StringComparer.OrdinalIgnoreCase); + } + + [DataMember(Order = 1, EmitDefaultValue = true)] + public bool HasFailure { get; set; } + + [DataMember(Order = 2, EmitDefaultValue = false)] + public Dictionary> EndpointsResult { get; set; } + } +} diff --git a/src/Sdk/DTWebApi/WebApi/WellKnownDistributedTaskVariables.cs b/src/Sdk/DTWebApi/WebApi/WellKnownDistributedTaskVariables.cs index c6ca7b0b4f5..a4ea950de1e 100644 --- a/src/Sdk/DTWebApi/WebApi/WellKnownDistributedTaskVariables.cs +++ b/src/Sdk/DTWebApi/WebApi/WellKnownDistributedTaskVariables.cs @@ -7,5 +7,6 @@ public static class WellKnownDistributedTaskVariables public static readonly String JobId = "system.jobId"; public static readonly String RunnerLowDiskspaceThreshold = "system.runner.lowdiskspacethreshold"; public static readonly String RunnerEnvironment = "system.runnerEnvironment"; + public static readonly String RunnerServiceConnectivityTest = "system.runner.serviceconnectivitycheckinput"; } } diff --git a/src/Sdk/RSWebApi/Contracts/BrokerError.cs b/src/Sdk/RSWebApi/Contracts/BrokerError.cs new file mode 100644 index 00000000000..c2e4bfa7b6b --- /dev/null +++ b/src/Sdk/RSWebApi/Contracts/BrokerError.cs @@ -0,0 +1,20 @@ +using System.Runtime.Serialization; + +namespace GitHub.Actions.RunService.WebApi +{ + [DataContract] + public class BrokerError + { + [DataMember(Name = "source", EmitDefaultValue = false)] + public string Source { get; set; } + + [DataMember(Name = "errorKind", EmitDefaultValue = false)] + public string ErrorKind { get; set; } + + [DataMember(Name = "statusCode", EmitDefaultValue = false)] + public int StatusCode { get; set; } + + [DataMember(Name = "errorMessage", EmitDefaultValue = false)] + public string Message { get; set; } + } +} diff --git a/src/Sdk/RSWebApi/Contracts/BrokerErrorKind.cs b/src/Sdk/RSWebApi/Contracts/BrokerErrorKind.cs new file mode 100644 index 00000000000..12022a83979 --- /dev/null +++ b/src/Sdk/RSWebApi/Contracts/BrokerErrorKind.cs @@ -0,0 +1,11 @@ +using System.Runtime.Serialization; + +namespace GitHub.Actions.RunService.WebApi +{ + [DataContract] + public class BrokerErrorKind + { + public const string RunnerNotFound = "RunnerNotFound"; + public const string RunnerVersionTooOld = "RunnerVersionTooOld"; + } +} diff --git a/src/Sdk/RSWebApi/Contracts/CompleteJobRequest.cs b/src/Sdk/RSWebApi/Contracts/CompleteJobRequest.cs index fff5156a40d..ff02ab6586c 100644 --- a/src/Sdk/RSWebApi/Contracts/CompleteJobRequest.cs +++ b/src/Sdk/RSWebApi/Contracts/CompleteJobRequest.cs @@ -27,6 +27,9 @@ public class CompleteJobRequest [DataMember(Name = "annotations", EmitDefaultValue = false)] public IList Annotations { get; set; } + [DataMember(Name = "telemetry", EmitDefaultValue = false)] + public IList Telemetry { get; set; } + [DataMember(Name = "environmentUrl", EmitDefaultValue = false)] public string EnvironmentUrl { get; set; } } diff --git a/src/Sdk/RSWebApi/Contracts/StepResult.cs b/src/Sdk/RSWebApi/Contracts/StepResult.cs index 1da4a2f9797..300fb7741a7 100644 --- a/src/Sdk/RSWebApi/Contracts/StepResult.cs +++ b/src/Sdk/RSWebApi/Contracts/StepResult.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Runtime.Serialization; using System.Threading.Tasks; @@ -16,9 +16,20 @@ public class StepResult [DataMember(Name = "number", EmitDefaultValue = false)] public int? Number { get; set; } + // Example: "Run actions/checkout@v3" [DataMember(Name = "name", EmitDefaultValue = false)] public string Name { get; set; } + // Example: "actions/checkout" + [DataMember(Name = "action_name", EmitDefaultValue = false)] + public string ActionName { get; set; } + + [DataMember(Name = "ref", EmitDefaultValue = false)] + public string Ref { get; set; } + + [DataMember(Name = "type", EmitDefaultValue = false)] + public string Type { get; set; } + [DataMember(Name = "status")] public TimelineRecordState? Status { get; set; } diff --git a/src/Sdk/RSWebApi/Contracts/Telemetry.cs b/src/Sdk/RSWebApi/Contracts/Telemetry.cs new file mode 100644 index 00000000000..9dda8aa130c --- /dev/null +++ b/src/Sdk/RSWebApi/Contracts/Telemetry.cs @@ -0,0 +1,20 @@ +using System.Runtime.Serialization; + +namespace Sdk.RSWebApi.Contracts +{ + [DataContract] + public struct Telemetry + { + public Telemetry(string message, string type) + { + Message = message; + Type = type; + } + + [DataMember(Name = "message", EmitDefaultValue = false)] + public string Message { get; set; } + + [DataMember(Name = "type", EmitDefaultValue = false)] + public string Type { get; set; } + } +} diff --git a/src/Sdk/RSWebApi/RunServiceHttpClient.cs b/src/Sdk/RSWebApi/RunServiceHttpClient.cs index 14bdd2a6379..f4d0c539f0d 100644 --- a/src/Sdk/RSWebApi/RunServiceHttpClient.cs +++ b/src/Sdk/RSWebApi/RunServiceHttpClient.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Net; using System.Net.Http; using System.Threading; @@ -107,15 +108,6 @@ public async Task GetJobMessageAsync( } } - // Temporary back compat - switch (result.StatusCode) - { - case HttpStatusCode.NotFound: - throw new TaskOrchestrationJobNotFoundException($"Job message not found: {messageId}"); - case HttpStatusCode.Conflict: - throw new TaskOrchestrationJobAlreadyAcquiredException($"Job message already acquired: {messageId}"); - } - if (!string.IsNullOrEmpty(result.ErrorBody)) { throw new Exception($"Failed to get job message: {result.Error}. {Truncate(result.ErrorBody)}"); @@ -135,6 +127,7 @@ public async Task CompleteJobAsync( IList stepResults, IList jobAnnotations, string environmentUrl, + IList telemetry, CancellationToken cancellationToken = default) { HttpMethod httpMethod = new HttpMethod("POST"); @@ -147,6 +140,7 @@ public async Task CompleteJobAsync( StepResults = stepResults, Annotations = jobAnnotations, EnvironmentUrl = environmentUrl, + Telemetry = telemetry, }; requestUri = new Uri(requestUri, "completejob"); @@ -171,13 +165,6 @@ public async Task CompleteJobAsync( } } - // Temporary back compat - switch (result.StatusCode) - { - case HttpStatusCode.NotFound: - throw new TaskOrchestrationJobNotFoundException($"Job not found: {jobId}"); - } - if (!string.IsNullOrEmpty(result.ErrorBody)) { throw new Exception($"Failed to complete job: {result.Error}. {Truncate(result.ErrorBody)}"); @@ -225,13 +212,6 @@ public async Task RenewJobAsync( } } - // Temporary back compat - switch (result.StatusCode) - { - case HttpStatusCode.NotFound: - throw new TaskOrchestrationJobNotFoundException($"Job not found: {jobId}"); - } - if (!string.IsNullOrEmpty(result.ErrorBody)) { throw new Exception($"Failed to renew job: {result.Error}. {Truncate(result.ErrorBody)}"); diff --git a/src/Sdk/Sdk.csproj b/src/Sdk/Sdk.csproj index ff1cb85a4fe..ce7f97c8888 100644 --- a/src/Sdk/Sdk.csproj +++ b/src/Sdk/Sdk.csproj @@ -1,11 +1,12 @@ - net6.0 + net8.0 Library win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 + true - NU1701;NU1603 + NU1701;NU1603;SYSLIB0050;SYSLIB0051 $(Version) TRACE 8.0 @@ -13,14 +14,14 @@ - - + + - - - - - + + + + + diff --git a/src/Sdk/WebApi/WebApi/BrokerHttpClient.cs b/src/Sdk/WebApi/WebApi/BrokerHttpClient.cs index e9ad938fb9f..e445b47381f 100644 --- a/src/Sdk/WebApi/WebApi/BrokerHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/BrokerHttpClient.cs @@ -103,6 +103,7 @@ public async Task GetRunnerMessageAsync( new HttpMethod("GET"), requestUri: requestUri, queryParameters: queryParams, + readErrorBody: true, cancellationToken: cancellationToken); if (result.IsSuccess) @@ -110,8 +111,23 @@ public async Task GetRunnerMessageAsync( return result.Value; } - // the only time we throw a `Forbidden` exception from Listener /messages is when the runner is - // disable_update and is too old to poll + if (TryParseErrorBody(result.ErrorBody, out BrokerError brokerError)) + { + switch (brokerError.ErrorKind) + { + case BrokerErrorKind.RunnerNotFound: + throw new RunnerNotFoundException(brokerError.Message); + case BrokerErrorKind.RunnerVersionTooOld: + throw new AccessDeniedException(brokerError.Message) + { + ErrorCode = 1 + }; + default: + break; + } + } + + // temporary back compat if (result.StatusCode == HttpStatusCode.Forbidden) { throw new AccessDeniedException($"{result.Error} Runner version v{runnerVersion} is deprecated and cannot receive messages.") @@ -120,7 +136,7 @@ public async Task GetRunnerMessageAsync( }; } - throw new Exception($"Failed to get job message: {result.Error}"); + throw new Exception($"Failed to get job message. Request to {requestUri} failed with status: {result.StatusCode}. Error message {result.Error}"); } public async Task CreateSessionAsync( @@ -172,5 +188,26 @@ public async Task DeleteSessionAsync( throw new Exception($"Failed to delete broker session: {result.Error}"); } + + private static bool TryParseErrorBody(string errorBody, out BrokerError error) + { + if (!string.IsNullOrEmpty(errorBody)) + { + try + { + error = JsonUtility.FromString(errorBody); + if (error?.Source == "actions-broker-listener") + { + return true; + } + } + catch (Exception) + { + } + } + + error = null; + return false; + } } } diff --git a/src/Sdk/WebApi/WebApi/Exceptions/RunnerNotFoundException.cs b/src/Sdk/WebApi/WebApi/Exceptions/RunnerNotFoundException.cs new file mode 100644 index 00000000000..957d54b89c3 --- /dev/null +++ b/src/Sdk/WebApi/WebApi/Exceptions/RunnerNotFoundException.cs @@ -0,0 +1,26 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using GitHub.Services.Common; +using GitHub.Services.WebApi; + +namespace GitHub.Services.WebApi +{ + [Serializable] + public sealed class RunnerNotFoundException : Exception + { + public RunnerNotFoundException() + : base() + { + } + + public RunnerNotFoundException(String message) + : base(message) + { + } + + public RunnerNotFoundException(String message, Exception innerException) + : base(message, innerException) + { + } + } +} diff --git a/src/Sdk/WebApi/WebApi/OAuth/VssOAuthExceptions.cs b/src/Sdk/WebApi/WebApi/OAuth/VssOAuthExceptions.cs index 5ebf86f9a8d..34ec103d05f 100644 --- a/src/Sdk/WebApi/WebApi/OAuth/VssOAuthExceptions.cs +++ b/src/Sdk/WebApi/WebApi/OAuth/VssOAuthExceptions.cs @@ -85,6 +85,7 @@ public String Error set; } + [Obsolete] public override void GetObjectData(SerializationInfo info, StreamingContext context) { base.GetObjectData(info, context); diff --git a/src/Sdk/WebApi/WebApi/VssServiceResponseException.cs b/src/Sdk/WebApi/WebApi/VssServiceResponseException.cs index e4aa84a736d..8dc275c0098 100644 --- a/src/Sdk/WebApi/WebApi/VssServiceResponseException.cs +++ b/src/Sdk/WebApi/WebApi/VssServiceResponseException.cs @@ -24,6 +24,7 @@ protected VssServiceResponseException(SerializationInfo info, StreamingContext c HttpStatusCode = (HttpStatusCode)info.GetInt32("HttpStatusCode"); } + [Obsolete] [SecurityCritical] public override void GetObjectData(SerializationInfo info, StreamingContext context) { diff --git a/src/Test/L0/ProcessExtensionL0.cs b/src/Test/L0/ProcessExtensionL0.cs index e9791250b20..d650b4889ee 100644 --- a/src/Test/L0/ProcessExtensionL0.cs +++ b/src/Test/L0/ProcessExtensionL0.cs @@ -27,9 +27,9 @@ public async Task SuccessReadProcessEnv() try { #if OS_WINDOWS - string node = Path.Combine(TestUtil.GetSrcPath(), @"..\_layout\externals\node16\bin\node"); + string node = Path.Combine(TestUtil.GetSrcPath(), @"..\_layout\externals\node20\bin\node"); #else - string node = Path.Combine(TestUtil.GetSrcPath(), @"../_layout/externals/node16/bin/node"); + string node = Path.Combine(TestUtil.GetSrcPath(), @"../_layout/externals/node20/bin/node"); hc.EnqueueInstance(new ProcessInvokerWrapper()); #endif var startInfo = new ProcessStartInfo(node, "-e \"setTimeout(function(){{}}, 15 * 1000);\""); diff --git a/src/Test/L0/TestHostContext.cs b/src/Test/L0/TestHostContext.cs index c44f13f1c4a..124bcd5bd06 100644 --- a/src/Test/L0/TestHostContext.cs +++ b/src/Test/L0/TestHostContext.cs @@ -370,6 +370,11 @@ private void LoadContext_Unloading(AssemblyLoadContext obj) Unloading(this, null); } } + + public void LoadDefaultUserAgents() + { + return; + } } public class DelayEventArgs : EventArgs diff --git a/src/Test/L0/Worker/ExecutionContextL0.cs b/src/Test/L0/Worker/ExecutionContextL0.cs index 08abcd09585..7357212d172 100644 --- a/src/Test/L0/Worker/ExecutionContextL0.cs +++ b/src/Test/L0/Worker/ExecutionContextL0.cs @@ -677,7 +677,7 @@ public void PublishStepTelemetry_RegularStep() ec.InitializeJob(jobRequest, CancellationToken.None); ec.Start(); - ec.StepTelemetry.Type = "node16"; + ec.StepTelemetry.Type = "node20"; ec.StepTelemetry.Action = "actions/checkout"; ec.StepTelemetry.Ref = "v2"; ec.StepTelemetry.IsEmbedded = false; @@ -695,7 +695,7 @@ public void PublishStepTelemetry_RegularStep() // Assert. Assert.Equal(1, ec.Global.StepsTelemetry.Count); - Assert.Equal("node16", ec.Global.StepsTelemetry.Single().Type); + Assert.Equal("node20", ec.Global.StepsTelemetry.Single().Type); Assert.Equal("actions/checkout", ec.Global.StepsTelemetry.Single().Action); Assert.Equal("v2", ec.Global.StepsTelemetry.Single().Ref); Assert.Equal(TaskResult.Succeeded, ec.Global.StepsTelemetry.Single().Result); @@ -746,7 +746,7 @@ public void PublishStepTelemetry_EmbeddedStep() var embeddedStep = ec.CreateChild(Guid.NewGuid(), "action_1_pre", "action_1_pre", null, null, ActionRunStage.Main, isEmbedded: true); embeddedStep.Start(); - embeddedStep.StepTelemetry.Type = "node16"; + embeddedStep.StepTelemetry.Type = "node20"; embeddedStep.StepTelemetry.Action = "actions/checkout"; embeddedStep.StepTelemetry.Ref = "v2"; @@ -758,7 +758,7 @@ public void PublishStepTelemetry_EmbeddedStep() // Assert. Assert.Equal(1, ec.Global.StepsTelemetry.Count); - Assert.Equal("node16", ec.Global.StepsTelemetry.Single().Type); + Assert.Equal("node20", ec.Global.StepsTelemetry.Single().Type); Assert.Equal("actions/checkout", ec.Global.StepsTelemetry.Single().Action); Assert.Equal("v2", ec.Global.StepsTelemetry.Single().Ref); Assert.Equal(ActionRunStage.Main.ToString(), ec.Global.StepsTelemetry.Single().Stage); @@ -773,6 +773,82 @@ public void PublishStepTelemetry_EmbeddedStep() [Trait("Level", "L0")] [Trait("Category", "Worker")] public void PublishStepResult_EmbeddedStep() + { + using (TestHostContext hc = CreateTestContext()) + { + // Job request + TaskOrchestrationPlanReference plan = new(); + TimelineReference timeline = new(); + Guid jobId = Guid.NewGuid(); + string jobName = "some job name"; + var variables = new Dictionary() + { + ["RunService.FixEmbeddedIssues"] = new VariableValue("true"), + }; + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, variables, new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); + jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() + { + Alias = Pipelines.PipelineConstants.SelfAlias, + Id = "github", + Version = "sha1" + }); + jobRequest.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData(); + + // Mocks + var pagingLogger = new Mock(); + var pagingLogger2 = new Mock(); + var jobServerQueue = new Mock(); + jobServerQueue.Setup(x => x.QueueTimelineRecordUpdate(It.IsAny(), It.IsAny())); + hc.EnqueueInstance(pagingLogger.Object); + hc.EnqueueInstance(pagingLogger2.Object); + hc.SetSingleton(jobServerQueue.Object); + + // Job context + var jobContext = new Runner.Worker.ExecutionContext(); + jobContext.Initialize(hc); + jobContext.InitializeJob(jobRequest, CancellationToken.None); + jobContext.Start(); + + // Step 1 context + var step1 = jobContext.CreateChild(Guid.NewGuid(), "my_step", "my_step", null, null, ActionRunStage.Main); + step1.Start(); + + // Embedded step 1a context + var embeddedStep1a = step1.CreateEmbeddedChild(null, null, Guid.NewGuid(), ActionRunStage.Main); + embeddedStep1a.Start(); + embeddedStep1a.StepTelemetry.Type = "node20"; + embeddedStep1a.StepTelemetry.Action = "actions/checkout"; + embeddedStep1a.StepTelemetry.Ref = "v2"; + embeddedStep1a.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }, ExecutionContextLogOptions.Default); + embeddedStep1a.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }, ExecutionContextLogOptions.Default); + embeddedStep1a.AddIssue(new Issue() { Type = IssueType.Notice, Message = "notice" }, ExecutionContextLogOptions.Default); + embeddedStep1a.Complete(); + + // Embedded step 1b context + var embeddedStep1b = step1.CreateEmbeddedChild(null, null, Guid.NewGuid(), ActionRunStage.Main); + embeddedStep1b.Start(); + embeddedStep1b.StepTelemetry.Type = "node20"; + embeddedStep1b.StepTelemetry.Action = "actions/checkout"; + embeddedStep1b.StepTelemetry.Ref = "v2"; + embeddedStep1b.AddIssue(new Issue() { Type = IssueType.Error, Message = "error 2" }, ExecutionContextLogOptions.Default); + embeddedStep1b.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning 2" }, ExecutionContextLogOptions.Default); + embeddedStep1b.AddIssue(new Issue() { Type = IssueType.Notice, Message = "notice 2" }, ExecutionContextLogOptions.Default); + embeddedStep1b.Complete(); + + step1.Complete(); + + // Assert + Assert.Equal(3, jobContext.Global.StepsResult.Count); + Assert.Equal(0, jobContext.Global.StepsResult[0].Annotations.Count); + Assert.Equal(0, jobContext.Global.StepsResult[1].Annotations.Count); + Assert.Equal(6, jobContext.Global.StepsResult[2].Annotations.Count); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void PublishStepResult_EmbeddedStep_Legacy() { using (TestHostContext hc = CreateTestContext()) { @@ -807,10 +883,10 @@ public void PublishStepResult_EmbeddedStep() ec.InitializeJob(jobRequest, CancellationToken.None); ec.Start(); - var embeddedStep = ec.CreateChild(Guid.NewGuid(), "action_1_pre", "action_1_pre", null, null, ActionRunStage.Main, isEmbedded: true); + var embeddedStep = ec.CreateEmbeddedChild(null, null, Guid.NewGuid(), ActionRunStage.Main); embeddedStep.Start(); - embeddedStep.StepTelemetry.Type = "node16"; + embeddedStep.StepTelemetry.Type = "node20"; embeddedStep.StepTelemetry.Action = "actions/checkout"; embeddedStep.StepTelemetry.Ref = "v2"; diff --git a/src/Test/L0/Worker/HandlerFactoryL0.cs b/src/Test/L0/Worker/HandlerFactoryL0.cs index 0a9552d9985..5f88d5211d0 100644 --- a/src/Test/L0/Worker/HandlerFactoryL0.cs +++ b/src/Test/L0/Worker/HandlerFactoryL0.cs @@ -30,8 +30,8 @@ private TestHostContext CreateTestContext([CallerMemberName] string testName = " [Theory] [Trait("Level", "L0")] [Trait("Category", "Worker")] - [InlineData("node12", "node16")] - [InlineData("node16", "node16")] + [InlineData("node12", "node20")] + [InlineData("node16", "node20")] [InlineData("node20", "node20")] public void IsNodeVersionUpgraded(string inputVersion, string expectedVersion) { @@ -70,7 +70,6 @@ public void IsNodeVersionUpgraded(string inputVersion, string expectedVersion) // Assert. Assert.Equal(expectedVersion, handler.Data.NodeVersion); - Environment.SetEnvironmentVariable(Constants.Variables.Actions.AllowActionsUseUnsecureNodeVersion, null); } } } diff --git a/src/Test/L0/Worker/JobExtensionL0.cs b/src/Test/L0/Worker/JobExtensionL0.cs index 4f5834fbe69..9ce99070bf6 100644 --- a/src/Test/L0/Worker/JobExtensionL0.cs +++ b/src/Test/L0/Worker/JobExtensionL0.cs @@ -140,7 +140,6 @@ private TestHostContext CreateTestContext([CallerMemberName] String testName = " hc.SetSingleton(_diagnosticLogManager.Object); hc.SetSingleton(_jobHookProvider.Object); hc.SetSingleton(_snapshotOperationProvider.Object); - hc.SetSingleton(new Mock().Object); hc.EnqueueInstance(_logger.Object); // JobExecutionContext hc.EnqueueInstance(_logger.Object); // job start hook hc.EnqueueInstance(_logger.Object); // Initial Job @@ -506,7 +505,27 @@ public Task EnsureSnapshotPostJobStepForMappingToken() return EnsureSnapshotPostJobStepForToken(mappingToken, snapshot); } - private async Task EnsureSnapshotPostJobStepForToken(TemplateToken snapshotToken, Pipelines.Snapshot expectedSnapshot) + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public Task EnsureSnapshotPostJobStepForMappingToken_WithIf_Is_False() + { + var snapshot = new Pipelines.Snapshot("TestImageNameFromMappingToken", condition: $"{PipelineTemplateConstants.Success}() && 1==0", version: "2.*"); + var imageNameValueStringToken = new StringToken(null, null, null, snapshot.ImageName); + var condition = new StringToken(null, null, null, snapshot.Condition); + var version = new StringToken(null, null, null, snapshot.Version); + + var mappingToken = new MappingToken(null, null, null) + { + { new StringToken(null,null,null, PipelineTemplateConstants.ImageName), imageNameValueStringToken }, + { new StringToken(null,null,null, PipelineTemplateConstants.If), condition }, + { new StringToken(null,null,null, PipelineTemplateConstants.CustomImageVersion), version } + }; + + return EnsureSnapshotPostJobStepForToken(mappingToken, snapshot, skipSnapshotStep: true); + } + + private async Task EnsureSnapshotPostJobStepForToken(TemplateToken snapshotToken, Pipelines.Snapshot expectedSnapshot, bool skipSnapshotStep = false) { using (TestHostContext hc = CreateTestContext()) { @@ -524,14 +543,28 @@ private async Task EnsureSnapshotPostJobStepForToken(TemplateToken snapshotToken Assert.Equal(1, postJobSteps.Count); var snapshotStep = postJobSteps.First(); + _jobEc.JobSteps.Enqueue(snapshotStep); + + var _stepsRunner = new StepsRunner(); + _stepsRunner.Initialize(hc); + await _stepsRunner.RunAsync(_jobEc); + Assert.Equal("Create custom image", snapshotStep.DisplayName); - Assert.Equal($"{PipelineTemplateConstants.Success}()", snapshotStep.Condition); + Assert.Equal(expectedSnapshot.Condition ?? $"{PipelineTemplateConstants.Success}()", snapshotStep.Condition); // Run the mock snapshot step, so we can verify it was executed with the expected snapshot object. - await snapshotStep.RunAsync(); - - Assert.NotNull(_requestedSnapshot); - Assert.Equal(expectedSnapshot.ImageName, _requestedSnapshot.ImageName); + // await snapshotStep.RunAsync(); + if (skipSnapshotStep) + { + Assert.Null(_requestedSnapshot); + } + else + { + Assert.NotNull(_requestedSnapshot); + Assert.Equal(expectedSnapshot.ImageName, _requestedSnapshot.ImageName); + Assert.Equal(expectedSnapshot.Condition ?? $"{PipelineTemplateConstants.Success}()", _requestedSnapshot.Condition); + Assert.Equal(expectedSnapshot.Version ?? "1.*", _requestedSnapshot.Version); + } } } } diff --git a/src/Test/L0/Worker/StepHostL0.cs b/src/Test/L0/Worker/StepHostL0.cs index f6b58890c99..47a5d3344da 100644 --- a/src/Test/L0/Worker/StepHostL0.cs +++ b/src/Test/L0/Worker/StepHostL0.cs @@ -75,10 +75,10 @@ public async Task DetermineNodeRuntimeVersionInAlpineContainerAsync() .ReturnsAsync(0); // Act. - var nodeVersion = await sh.DetermineNodeRuntimeVersion(_ec.Object, "node16"); + var nodeVersion = await sh.DetermineNodeRuntimeVersion(_ec.Object, "node20"); // Assert. - Assert.Equal("node16_alpine", nodeVersion); + Assert.Equal("node20_alpine", nodeVersion); } } @@ -129,10 +129,10 @@ public async Task DetermineNodeRuntimeVersionInUnknowContainerAsync() .ReturnsAsync(0); // Act. - var nodeVersion = await sh.DetermineNodeRuntimeVersion(_ec.Object, "node16"); + var nodeVersion = await sh.DetermineNodeRuntimeVersion(_ec.Object, "node20"); // Assert. - Assert.Equal("node16", nodeVersion); + Assert.Equal("node20", nodeVersion); } } diff --git a/src/Test/Test.csproj b/src/Test/Test.csproj index 1beddbfc251..aebe242096f 100644 --- a/src/Test/Test.csproj +++ b/src/Test/Test.csproj @@ -1,9 +1,9 @@ - net6.0 + net8.0 win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 true - NU1701;NU1603;NU1603;xUnit2013; + NU1701;NU1603;NU1603;xUnit2013;SYSLIB0050;SYSLIB0051 @@ -15,13 +15,13 @@ - + - + - + - + diff --git a/src/TestDotNet8Compatibility/Program.cs b/src/TestDotNet8Compatibility/Program.cs deleted file mode 100644 index 0d231953003..00000000000 --- a/src/TestDotNet8Compatibility/Program.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System; - -namespace TestDotNet8Compatibility -{ - public static class Program - { - public static int Main(string[] args) - { - Console.WriteLine("Hello from .NET 8!"); - return 0; - } - } -} diff --git a/src/TestDotNet8Compatibility/TestDotNet8Compatibility.csproj b/src/TestDotNet8Compatibility/TestDotNet8Compatibility.csproj deleted file mode 100644 index 246b690a1ae..00000000000 --- a/src/TestDotNet8Compatibility/TestDotNet8Compatibility.csproj +++ /dev/null @@ -1,18 +0,0 @@ - - - - net8.0 - Exe - win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 - true - true - $(Version) - false - true - - - - portable - - - diff --git a/src/TestDotNet8Compatibility/dir.proj b/src/TestDotNet8Compatibility/dir.proj deleted file mode 100644 index fa8200ba95a..00000000000 --- a/src/TestDotNet8Compatibility/dir.proj +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - diff --git a/src/TestDotNet8Compatibility/global.json b/src/TestDotNet8Compatibility/global.json deleted file mode 100644 index fd07d882ad3..00000000000 --- a/src/TestDotNet8Compatibility/global.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "sdk": { - "version": "8.0.303" - } -} diff --git a/src/dev.sh b/src/dev.sh index 71c80637c02..8e23366554a 100755 --- a/src/dev.sh +++ b/src/dev.sh @@ -17,10 +17,8 @@ LAYOUT_DIR="$SCRIPT_DIR/../_layout" DOWNLOAD_DIR="$SCRIPT_DIR/../_downloads/netcore2x" PACKAGE_DIR="$SCRIPT_DIR/../_package" DOTNETSDK_ROOT="$SCRIPT_DIR/../_dotnetsdk" -DOTNETSDK_VERSION="6.0.421" +DOTNETSDK_VERSION="8.0.404" DOTNETSDK_INSTALLDIR="$DOTNETSDK_ROOT/$DOTNETSDK_VERSION" -DOTNET8SDK_VERSION="8.0.303" -DOTNET8SDK_INSTALLDIR="$DOTNETSDK_ROOT/$DOTNET8SDK_VERSION" RUNNER_VERSION=$(cat runnerversion) pushd "$SCRIPT_DIR" @@ -127,19 +125,6 @@ function build () { heading "Building ..." dotnet msbuild -t:Build -p:PackageRuntime="${RUNTIME_ID}" -p:BUILDCONFIG="${BUILD_CONFIG}" -p:RunnerVersion="${RUNNER_VERSION}" ./dir.proj || failed build - - # Build TestDotNet8Compatibility - heading "Building .NET 8 compatibility test" - echo "Prepend ${DOTNET8SDK_INSTALLDIR} to %PATH%" # Prepend .NET 8 SDK to PATH - PATH_BAK=$PATH - export PATH=${DOTNET8SDK_INSTALLDIR}:$PATH - pushd "$SCRIPT_DIR/TestDotNet8Compatibility" > /dev/null # Working directory - pwd - echo "Dotnet 8 SDK Version" - dotnet --version - dotnet msbuild -t:Build -p:PackageRuntime="${RUNTIME_ID}" -p:BUILDCONFIG="${BUILD_CONFIG}" -p:RunnerVersion="${RUNNER_VERSION}" ./dir.proj || failed build - popd > /dev/null # Restore working directory - export PATH=$PATH_BAK # Restore PATH } function layout () @@ -158,18 +143,6 @@ function layout () heading "Setup externals folder for $RUNTIME_ID runner's layout" bash ./Misc/externals.sh $RUNTIME_ID || checkRC externals.sh - - # Build TestDotNet8Compatibility - echo "Prepend ${DOTNET8SDK_INSTALLDIR} to %PATH%" # Prepend .NET 8 SDK to PATH - PATH_BAK=$PATH - export PATH=${DOTNET8SDK_INSTALLDIR}:$PATH - pushd "$SCRIPT_DIR/TestDotNet8Compatibility" > /dev/null # Working directory - heading "Dotnet 8 SDK Version" - dotnet --version - heading "Building .NET 8 compatibility test" - dotnet msbuild -t:layout -p:PackageRuntime="${RUNTIME_ID}" -p:BUILDCONFIG="${BUILD_CONFIG}" -p:RunnerVersion="${RUNNER_VERSION}" ./dir.proj || failed build - popd > /dev/null # Restore working directory - export PATH=$PATH_BAK # Restore PATH } function runtest () @@ -252,32 +225,6 @@ if [[ (! -d "${DOTNETSDK_INSTALLDIR}") || (! -e "${DOTNETSDK_INSTALLDIR}/.${DOTN echo "${DOTNETSDK_VERSION}" > "${DOTNETSDK_INSTALLDIR}/.${DOTNETSDK_VERSION}" fi -# Install .NET 8 SDK -if [[ (! -d "${DOTNET8SDK_INSTALLDIR}") || (! -e "${DOTNET8SDK_INSTALLDIR}/.${DOTNET8SDK_VERSION}") || (! -e "${DOTNET8SDK_INSTALLDIR}/dotnet") ]]; then - - # Download dotnet 8 SDK to ../_dotnetsdk directory - heading "Ensure Dotnet 8 SDK" - - # _dotnetsdk - # \1.0.x - # \dotnet - # \.1.0.x - echo "Download dotnet8sdk into ${DOTNET8SDK_INSTALLDIR}" - rm -Rf "${DOTNETSDK_DIR}" - - # run dotnet-install.ps1 on windows, dotnet-install.sh on linux - if [[ ("$CURRENT_PLATFORM" == "windows") ]]; then - echo "Convert ${DOTNET8SDK_INSTALLDIR} to Windows style path" - sdkinstallwindow_path=${DOTNET8SDK_INSTALLDIR:1} - sdkinstallwindow_path=${sdkinstallwindow_path:0:1}:${sdkinstallwindow_path:1} - $POWERSHELL -NoLogo -Sta -NoProfile -NonInteractive -ExecutionPolicy Unrestricted -Command "& \"./Misc/dotnet-install.ps1\" -Version ${DOTNET8SDK_VERSION} -InstallDir \"${sdkinstallwindow_path}\" -NoPath; exit \$LastExitCode;" || checkRC dotnet-install.ps1 - else - bash ./Misc/dotnet-install.sh --version ${DOTNET8SDK_VERSION} --install-dir "${DOTNET8SDK_INSTALLDIR}" --no-path || checkRC dotnet-install.sh - fi - - echo "${DOTNET8SDK_VERSION}" > "${DOTNET8SDK_INSTALLDIR}/.${DOTNET8SDK_VERSION}" -fi - echo "Prepend ${DOTNETSDK_INSTALLDIR} to %PATH%" export PATH=${DOTNETSDK_INSTALLDIR}:$PATH diff --git a/src/global.json b/src/global.json index e7028fe0dd4..8c70738ad60 100644 --- a/src/global.json +++ b/src/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "6.0.421" + "version": "8.0.404" } } diff --git a/src/runnerversion b/src/runnerversion index 8330b5bf1db..248da29093e 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.319.0 +2.321.0