From a4ddd76330c17095a9170893008acc5f8c826ea2 Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Wed, 6 Nov 2024 20:46:28 -0800 Subject: [PATCH 01/30] addressing comments and initial changes for generating artifacts --- .../official/jobs/build-core-tools-host.yml | 183 +++++++++++------- .../ArtifactAssembler.cs | 83 ++++---- .../EnvironmentVariables.cs | 7 +- 3 files changed, 170 insertions(+), 103 deletions(-) diff --git a/eng/ci/templates/official/jobs/build-core-tools-host.yml b/eng/ci/templates/official/jobs/build-core-tools-host.yml index 3ceb3bad9..7d03bbbb6 100644 --- a/eng/ci/templates/official/jobs/build-core-tools-host.yml +++ b/eng/ci/templates/official/jobs/build-core-tools-host.yml @@ -35,89 +35,138 @@ jobs: arguments: -c Release -r win-arm64 -o $(Build.SourcesDirectory)/pkg_output/windows/win-arm64 --self-contained workingDirectory: $(Build.SourcesDirectory)/host/src/CoreToolsHost - - template: ci/sign-files.yml@eng - parameters: - displayName: 'Authenticode signing (dll) (win-arm64)' - folderPath: '$(Build.SourcesDirectory)/pkg_output/windows/win-arm64' - pattern: '*.dll, *.exe' - signType: inline - inlineOperation: | - [ - { - "KeyCode": "CP-230012", - "OperationCode": "SigntoolSign", - "Parameters": { - "OpusName": "Microsoft", - "OpusInfo": "http://www.microsoft.com", - "FileDigest": "/fd \"SHA256\"", - "PageHash": "/NPH", - "TimeStamp": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - }, - "ToolName": "sign", - "ToolVersion": "1.0" - }, - { - "KeyCode": "CP-230012", - "OperationCode": "SigntoolVerify", - "Parameters": {}, - "ToolName": "sign", - "ToolVersion": "1.0" - } - ] - - - template: ci/sign-files.yml@eng - parameters: - displayName: 'Authenticode signing (dll) (win-x64)' - folderPath: '$(Build.SourcesDirectory)/pkg_output/windows/win-x64' - pattern: '*.dll, *.exe' - signType: inline - inlineOperation: | - [ - { - "KeyCode": "CP-230012", - "OperationCode": "SigntoolSign", - "Parameters": { - "OpusName": "Microsoft", - "OpusInfo": "http://www.microsoft.com", - "FileDigest": "/fd \"SHA256\"", - "PageHash": "/NPH", - "TimeStamp": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - }, - "ToolName": "sign", - "ToolVersion": "1.0" - }, - { - "KeyCode": "CP-230012", - "OperationCode": "SigntoolVerify", - "Parameters": {}, - "ToolName": "sign", - "ToolVersion": "1.0" - } - ] - - task: CopyFiles@2 displayName: Copy files (win-x64) inputs: SourceFolder: $(Build.SourcesDirectory)/pkg_output/windows/win-x64 - # Publish output will include many other files. We only need func.exe & nethost.dll + # Publish output will include many other files. We only need func & nethost.dll Contents: | func.exe nethost.dll - TargetFolder: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackagesWindows/win-x64 + TargetFolder: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages/win-x64 - task: CopyFiles@2 displayName: Copy files (win-arm64) inputs: SourceFolder: $(Build.SourcesDirectory)/pkg_output/windows/win-arm64 - # Publish output will include many other files. We only need func.exe & nethost.dll + # Publish output will include many other files. We only need func & nethost.dll Contents: | func.exe nethost.dll - TargetFolder: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackagesWindows/win-arm64 - + TargetFolder: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages/win-arm64 + + templateContext: + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - output: pipelineArtifact + path: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages + artifact: drop-coretools-host-windows + +- job: BuildCoreToolsHostLinux + displayName: '[Linux] Build CoreToolsHost' + pool: + name: 1es-pool-azfunc + image: 1es-ubuntu-22.04 + os: linux + + steps: + - task: UseDotNet@2 + inputs: + version: 9.x + includePreviewVersions: true + displayName: Install .NET 9 + - task: UseDotNet@2 + inputs: + version: 6.x + displayName: Install .NET 6 + + - task: DotnetCoreCLI@2 + displayName: Dotnet Publish (linux-x64) + inputs: + command: publish + publishWebProjects: false + zipAfterPublish: false + arguments: -c Release -r linux-x64 -o $(Build.SourcesDirectory)/pkg_output/linux/linux-x64 --self-contained + workingDirectory: $(Build.SourcesDirectory)/host/src/CoreToolsHost + + - task: CopyFiles@2 + displayName: Copy files (linux-x64) + inputs: + SourceFolder: $(Build.SourcesDirectory)/pkg_output/linux/linux-x64 + # Publish output will include many other files. We only need func & nethost.dll + Contents: | + func + libnethost.so + TargetFolder: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages/linux-x64 + + templateContext: + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - output: pipelineArtifact + path: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages + artifact: drop-coretools-host-linux + +- job: BuildCoreToolsHostMac + displayName: '[Mac] Build CoreToolsHost' + pool: + name: Azure Pipelines + image: 'macOS-latest' + os: macOS + + steps: + - task: UseDotNet@2 + inputs: + version: 9.x + includePreviewVersions: true + displayName: Install .NET 9 + - task: UseDotNet@2 + inputs: + version: 6.x + displayName: Install .NET 6 + + - task: DotnetCoreCLI@2 + displayName: Dotnet Publish (osx-x64) + inputs: + command: publish + publishWebProjects: false + zipAfterPublish: false + arguments: -c Release -r osx-x64 -o $(Build.SourcesDirectory)/pkg_output/osx/osx-x64 --self-contained + workingDirectory: $(Build.SourcesDirectory)/host/src/CoreToolsHost + + - task: DotnetCoreCLI@2 + displayName: Dotnet Publish (osx-arm64) + inputs: + command: publish + publishWebProjects: false + zipAfterPublish: false + arguments: -c Release -r osx-arm64 -o $(Build.SourcesDirectory)/pkg_output/osx/osx-arm64 --self-contained + workingDirectory: $(Build.SourcesDirectory)/host/src/CoreToolsHost + + - task: CopyFiles@2 + displayName: Copy files (osx-x64) + inputs: + SourceFolder: $(Build.SourcesDirectory)/pkg_output/osx/osx-x64 + # Publish output will include many other files. We only need func & nethost.dll + Contents: | + func + libnethost.dylib + libnethost.a + TargetFolder: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages/osx-x64 + + - task: CopyFiles@2 + displayName: Copy files (osx-arm64) + inputs: + SourceFolder: $(Build.SourcesDirectory)/pkg_output/osx/osx-arm64 + # Publish output will include many other files. We only need func & nethost.dll + Contents: | + func + libnethost.dylib + libnethost.a + TargetFolder: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages/osx-arm64 + templateContext: outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: - output: pipelineArtifact - path: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackagesWindows - artifact: drop-coretools-host-windows \ No newline at end of file + path: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages + artifact: drop-coretools-host-mac \ No newline at end of file diff --git a/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs b/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs index b510107ad..0a699daf8 100644 --- a/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs +++ b/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs @@ -11,10 +11,13 @@ internal sealed class ArtifactAssembler /// The artifacts for which we want to pack a custom host with it. /// This dictionary contains the artifact name and the corresponding runtime identifier value. /// - private readonly Dictionary _customHostArtifacts = new() + private readonly Dictionary _visualStudioArtifacts = new() { { "Azure.Functions.Cli.min.win-x64", "win-x64" }, - { "Azure.Functions.Cli.min.win-arm64", "win-arm64" } + { "Azure.Functions.Cli.min.win-arm64", "win-arm64" }, + { "Azure.Functions.Cli.linux-x64", "linux-x64" }, + { "Azure.Functions.Cli.osx-x64", "osx-x64" }, + { "Azure.Functions.Cli.osx-arm64", "osx-arm64" } }; /// @@ -38,7 +41,9 @@ internal sealed class ArtifactAssembler private readonly string _outOfProcArtifactDirectoryName; private readonly string _inProc6ArtifactName; private readonly string _inProc8ArtifactName; - private readonly string _coreToolsHostArtifactName; + private readonly string _coreToolsHostWindowsArtifactName; + private readonly string _coreToolsHostLinuxArtifactName; + private readonly string _coreToolsHostMacArtifactName; private readonly string _outOfProcArtifactName; private readonly string _rootWorkingDirectory; private readonly string _stagingDirectory; @@ -56,7 +61,9 @@ internal ArtifactAssembler(string rootWorkingDirectory) _inProc6ArtifactName = GetRequiredEnvironmentVariable(EnvironmentVariables.InProc6ArtifactName); _inProc8ArtifactName = GetRequiredEnvironmentVariable(EnvironmentVariables.InProc8ArtifactName); - _coreToolsHostArtifactName = GetRequiredEnvironmentVariable(EnvironmentVariables.CoreToolsHostArtifactName); + _coreToolsHostWindowsArtifactName = GetRequiredEnvironmentVariable(EnvironmentVariables.CoreToolsHostWindowsArtifactName); + _coreToolsHostLinuxArtifactName = GetRequiredEnvironmentVariable(EnvironmentVariables.CoreToolsHostLinuxArtifactName); + _coreToolsHostMacArtifactName = GetRequiredEnvironmentVariable(EnvironmentVariables.CoreToolsHostOsxArtifactName); _outOfProcArtifactName = GetRequiredEnvironmentVariable(EnvironmentVariables.OutOfProcArtifactName); _rootWorkingDirectory = rootWorkingDirectory; @@ -66,8 +73,8 @@ internal ArtifactAssembler(string rootWorkingDirectory) internal async Task AssembleArtifactsAsync() { await ExtractDownloadedArtifactsAsync(); - await CreateVisualStudioCoreToolsAsync(); await CreateCliCoreToolsAsync(); + await CreateVisualStudioCoreToolsAsync(); } private static string GetRequiredEnvironmentVariable(string variableName) @@ -80,28 +87,34 @@ private async Task ExtractDownloadedArtifactsAsync() { var inProcArtifactDownloadDir = Path.Combine(_rootWorkingDirectory, _inProcArtifactDirectoryName); var coreToolsHostArtifactDownloadDir = Path.Combine(_rootWorkingDirectory, _coreToolsHostArtifactDirectoryName); - var outOfProcArtifactDownloadDir = Path.Combine(_rootWorkingDirectory, _outOfProcArtifactDirectoryName); var inProc6ArtifactDirPath = Path.Combine(inProcArtifactDownloadDir, _inProc6ArtifactName); var inProc8ArtifactDirPath = Path.Combine(inProcArtifactDownloadDir, _inProc8ArtifactName); - var coreToolsHostArtifactDirPath = Path.Combine(coreToolsHostArtifactDownloadDir, _coreToolsHostArtifactName); - var outOfProcArtifactDirPath = Path.Combine(outOfProcArtifactDownloadDir, _outOfProcArtifactName); EnsureArtifactDirectoryExist(inProc6ArtifactDirPath); EnsureArtifactDirectoryExist(inProc8ArtifactDirPath); - EnsureArtifactDirectoryExist(coreToolsHostArtifactDirPath); + + var coreToolsHostWindowsArtifactDirPath = Path.Combine(coreToolsHostArtifactDownloadDir, _coreToolsHostWindowsArtifactName); + var coreToolsHostLinuxArtifactDirPath = Path.Combine(coreToolsHostArtifactDownloadDir, _coreToolsHostLinuxArtifactName); + var coreToolsHostMacArtifactDirPath = Path.Combine(coreToolsHostArtifactDownloadDir, _coreToolsHostMacArtifactName); + EnsureArtifactDirectoryExist(coreToolsHostWindowsArtifactDirPath); + EnsureArtifactDirectoryExist(coreToolsHostLinuxArtifactDirPath); + EnsureArtifactDirectoryExist(coreToolsHostMacArtifactDirPath); + + _coreToolsHostExtractedRootDir = await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(coreToolsHostWindowsArtifactDirPath, Path.Combine(_stagingDirectory, Constants.CoreToolsHostDirectoryName)); + await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(coreToolsHostLinuxArtifactDirPath, Path.Combine(_stagingDirectory, Constants.CoreToolsHostDirectoryName)); + await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(coreToolsHostMacArtifactDirPath, Path.Combine(_stagingDirectory, Constants.CoreToolsHostDirectoryName)); + Directory.Delete(coreToolsHostArtifactDownloadDir, true); + var outOfProcArtifactDownloadDir = Path.Combine(_rootWorkingDirectory, _outOfProcArtifactDirectoryName); + var outOfProcArtifactDirPath = Path.Combine(outOfProcArtifactDownloadDir, _outOfProcArtifactName); EnsureArtifactDirectoryExist(outOfProcArtifactDirPath); + _outOfProcExtractedRootDir = await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(outOfProcArtifactDirPath, Path.Combine(_stagingDirectory, Constants.OutOfProcDirectoryName)); + Directory.Delete(outOfProcArtifactDownloadDir, true); _inProc6ExtractedRootDir = await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(inProc6ArtifactDirPath, Path.Combine(_stagingDirectory, Constants.InProc6DirectoryName)); _inProc8ExtractedRootDir = await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(inProc8ArtifactDirPath, Path.Combine(_stagingDirectory, Constants.InProc8DirectoryName)); Directory.Delete(inProcArtifactDownloadDir, true); - - _coreToolsHostExtractedRootDir = await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(coreToolsHostArtifactDirPath, Path.Combine(_stagingDirectory, Constants.CoreToolsHostDirectoryName)); - Directory.Delete(coreToolsHostArtifactDownloadDir, true); - - _outOfProcExtractedRootDir = await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(outOfProcArtifactDirPath, Path.Combine(_stagingDirectory, Constants.OutOfProcDirectoryName)); - Directory.Delete(outOfProcArtifactDownloadDir, true); } private static void EnsureArtifactDirectoryExist(string directoryExist) @@ -132,6 +145,14 @@ private static async Task MoveArtifactsToStagingDirectoryAndExtractIfNee { await Task.Run(() => FileUtilities.CopyDirectory(artifactZipPath, destinationDirectory)); await ExtractZipFilesInDirectoryAsync(artifactZipPath, destinationDirectory); + + // Delete additional files that are not needed + var filesToBeDeleted = Directory.GetFiles(destinationDirectory); + Console.WriteLine($"{filesToBeDeleted.Length} files found in {destinationDirectory}"); + foreach (var file in filesToBeDeleted) + { + File.Delete(file); + } return destinationDirectory; } @@ -156,7 +177,7 @@ private async Task CreateVisualStudioCoreToolsAsync() var customHostTargetArtifactDir = Path.Combine(_stagingDirectory, Constants.VisualStudioOutputArtifactDirectoryName); Directory.CreateDirectory(customHostTargetArtifactDir); - var packTasks = _customHostArtifacts.Keys.Select(async artifactName => + foreach (string artifactName in _visualStudioArtifacts.Keys) { var inProc8ArtifactDirPath = Directory.EnumerateDirectories(_inProc8ExtractedRootDir) .FirstOrDefault(dir => dir.Contains(artifactName)); @@ -172,20 +193,19 @@ private async Task CreateVisualStudioCoreToolsAsync() Directory.CreateDirectory(consolidatedArtifactDirPath); // Copy in-proc8 files - var inProc8CopyTask = Task.Run(() => FileUtilities.CopyDirectory(inProc8ArtifactDirPath, Path.Combine(consolidatedArtifactDirPath, Constants.InProc8DirectoryName))); + await Task.Run(() => FileUtilities.CopyDirectory(inProc8ArtifactDirPath, Path.Combine(consolidatedArtifactDirPath, Constants.InProc8DirectoryName))); // Copy in-proc6 files var inProc6ArtifactDirPath = Path.Combine(_inProc6ExtractedRootDir, artifactDirName); EnsureArtifactDirectoryExist(inProc6ArtifactDirPath); - var inProc6CopyTask = Task.Run(() => FileUtilities.CopyDirectory(inProc6ArtifactDirPath, Path.Combine(consolidatedArtifactDirPath, Constants.InProc6DirectoryName))); + await Task.Run(() => FileUtilities.CopyDirectory(inProc6ArtifactDirPath, Path.Combine(consolidatedArtifactDirPath, Constants.InProc6DirectoryName))); // Copy core-tools-host files var rid = GetRuntimeIdentifierForArtifactName(artifactName); var coreToolsHostArtifactDirPath = Path.Combine(_coreToolsHostExtractedRootDir, rid); EnsureArtifactDirectoryExist(coreToolsHostArtifactDirPath); - var coreToolsHostCopyTask = Task.Run(() => FileUtilities.CopyDirectory(coreToolsHostArtifactDirPath, consolidatedArtifactDirPath)); - - await Task.WhenAll(inProc8CopyTask, inProc6CopyTask, coreToolsHostCopyTask); + await Task.Run(() => FileUtilities.CopyDirectory(coreToolsHostArtifactDirPath, consolidatedArtifactDirPath)); + Directory.Delete(coreToolsHostArtifactDirPath, true); // consolidatedArtifactDirPath now contains custom core-tools host, in-proc6 and in-proc8 sub directories. Create a zip file. var zipPath = Path.Combine(customHostTargetArtifactDir, $"{consolidatedArtifactDirName}.zip"); @@ -193,9 +213,8 @@ private async Task CreateVisualStudioCoreToolsAsync() Console.WriteLine($"Successfully created target runtime zip at: {zipPath}"); Directory.Delete(consolidatedArtifactDirPath, true); - }); + } - await Task.WhenAll(packTasks); Console.WriteLine("Finished assembling Visual Studio Core Tools artifacts"); } @@ -213,6 +232,7 @@ private async Task CreateCliCoreToolsAsync() foreach (var artifactName in _cliArtifacts) { + Console.WriteLine($"Starting to assemble {artifactName}"); // If we are running this for the first time, extract the directory path and out of proc version if (String.IsNullOrEmpty(outOfProcArtifactDirPath)) { @@ -239,6 +259,7 @@ private async Task CreateCliCoreToolsAsync() // If we are currently on the minified version of the artifacts, we do not want the inproc6/inproc8 subfolders if (artifactName.Contains("min.win")) { + Console.WriteLine($"Finished assembling {artifactName}"); continue; } @@ -261,7 +282,6 @@ private async Task CreateCliCoreToolsAsync() // Copy in-proc8 files and delete old directory await Task.Run(() => FileUtilities.CopyDirectory(newInProc8ArtifactDirPath, Path.Combine(consolidatedArtifactDirPath, Constants.InProc8DirectoryName))); - Directory.Delete(newInProc8ArtifactDirPath, true); // Rename inproc6 directory to have the same version as the out-of-proc artifact before copying var inProcArtifactName = Path.GetFileName(inProc8ArtifactDirPath); @@ -271,13 +291,11 @@ private async Task CreateCliCoreToolsAsync() // Copy in-proc6 files and delete old directory await Task.Run(() => FileUtilities.CopyDirectory(newInProc6ArtifactDirPath, Path.Combine(consolidatedArtifactDirPath, Constants.InProc6DirectoryName))); - Directory.Delete(newInProc6ArtifactDirPath, true); + + Console.WriteLine($"Finished assembling {artifactName}"); } // Delete the extracted directories - Directory.Delete(_inProc6ExtractedRootDir, true); - Directory.Delete(_inProc8ExtractedRootDir, true); - Directory.Delete(_coreToolsHostExtractedRootDir, true); Directory.Delete(_outOfProcExtractedRootDir, true); Console.WriteLine("Finished assembling CLI Core Tools artifacts"); @@ -306,18 +324,18 @@ private string RenameInProcDirectory(string oldArtifactDirPath, string newVersio } var artifactName = match.Groups[1]; - var newDirectoryName = $"{artifactName}{newVersion}"; + var newDirectoryName = $"{artifactName}{newVersion}"; // Rename (move) the directory Directory.Move(oldArtifactDirPath, newDirectoryName); - + return newDirectoryName; } private string GetRuntimeIdentifierForArtifactName(string artifactName) { - if (_customHostArtifacts.TryGetValue(artifactName, out var rid)) + if (_visualStudioArtifacts.TryGetValue(artifactName, out var rid)) { return rid; } @@ -346,5 +364,4 @@ private static async Task ExtractZipFilesInDirectoryAsync(string zipSourceDir, s } } } -} - +} \ No newline at end of file diff --git a/src/Azure.Functions.ArtifactAssembler/EnvironmentVariables.cs b/src/Azure.Functions.ArtifactAssembler/EnvironmentVariables.cs index 2074b6012..079b3ac30 100644 --- a/src/Azure.Functions.ArtifactAssembler/EnvironmentVariables.cs +++ b/src/Azure.Functions.ArtifactAssembler/EnvironmentVariables.cs @@ -10,8 +10,9 @@ internal static class EnvironmentVariables internal const string OutOfProcArtifactAlias = "OUT_OF_PROC_ARTIFACT_ALIAS"; internal const string InProc6ArtifactName = "IN_PROC6_ARTIFACT_NAME"; internal const string InProc8ArtifactName = "IN_PROC8_ARTIFACT_NAME"; - internal const string CoreToolsHostArtifactName = "CORETOOLS_HOST_ARTIFACT_NAME"; + internal const string CoreToolsHostWindowsArtifactName = "CORETOOLS_HOST_WINDOWS_ARTIFACT_NAME"; + internal const string CoreToolsHostLinuxArtifactName = "CORETOOLS_HOST_LINUX_ARTIFACT_NAME"; + internal const string CoreToolsHostOsxArtifactName = "CORETOOLS_HOST_OSX_ARTIFACT_NAME"; internal const string OutOfProcArtifactName = "OUT_OF_PROC_ARTIFACT_NAME"; } -} - +} \ No newline at end of file From 4d2fe13372d70098817d4e0cdf4a8bf7fd3b631b Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Wed, 6 Nov 2024 20:53:47 -0800 Subject: [PATCH 02/30] osx change --- eng/ci/templates/official/jobs/build-core-tools-host.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/ci/templates/official/jobs/build-core-tools-host.yml b/eng/ci/templates/official/jobs/build-core-tools-host.yml index 7d03bbbb6..617c5b31e 100644 --- a/eng/ci/templates/official/jobs/build-core-tools-host.yml +++ b/eng/ci/templates/official/jobs/build-core-tools-host.yml @@ -169,4 +169,4 @@ jobs: outputs: - output: pipelineArtifact path: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages - artifact: drop-coretools-host-mac \ No newline at end of file + artifact: drop-coretools-host-osx \ No newline at end of file From 335be054f7ce7f0b099672c7bacb925813c3279b Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Wed, 6 Nov 2024 21:05:15 -0800 Subject: [PATCH 03/30] deleting extra directory --- src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs b/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs index 0a699daf8..0c0737c51 100644 --- a/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs +++ b/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs @@ -215,6 +215,11 @@ private async Task CreateVisualStudioCoreToolsAsync() Directory.Delete(consolidatedArtifactDirPath, true); } + // Delete directories + Directory.Delete(_inProc6ExtractedRootDir, true); + Directory.Delete(_inProc8ExtractedRootDir, true); + Directory.Delete(_coreToolsHostExtractedRootDir, true); + Console.WriteLine("Finished assembling Visual Studio Core Tools artifacts"); } From 3754409d025e8b98e746796a98ab005c52bd08ba Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Wed, 6 Nov 2024 21:38:44 -0800 Subject: [PATCH 04/30] change order of extract directories --- .../ArtifactAssembler.cs | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs b/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs index 0c0737c51..5dda3983e 100644 --- a/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs +++ b/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs @@ -87,34 +87,35 @@ private async Task ExtractDownloadedArtifactsAsync() { var inProcArtifactDownloadDir = Path.Combine(_rootWorkingDirectory, _inProcArtifactDirectoryName); var coreToolsHostArtifactDownloadDir = Path.Combine(_rootWorkingDirectory, _coreToolsHostArtifactDirectoryName); + var outOfProcArtifactDownloadDir = Path.Combine(_rootWorkingDirectory, _outOfProcArtifactDirectoryName); var inProc6ArtifactDirPath = Path.Combine(inProcArtifactDownloadDir, _inProc6ArtifactName); var inProc8ArtifactDirPath = Path.Combine(inProcArtifactDownloadDir, _inProc8ArtifactName); - - EnsureArtifactDirectoryExist(inProc6ArtifactDirPath); - EnsureArtifactDirectoryExist(inProc8ArtifactDirPath); + var outOfProcArtifactDirPath = Path.Combine(outOfProcArtifactDownloadDir, _outOfProcArtifactName); var coreToolsHostWindowsArtifactDirPath = Path.Combine(coreToolsHostArtifactDownloadDir, _coreToolsHostWindowsArtifactName); var coreToolsHostLinuxArtifactDirPath = Path.Combine(coreToolsHostArtifactDownloadDir, _coreToolsHostLinuxArtifactName); var coreToolsHostMacArtifactDirPath = Path.Combine(coreToolsHostArtifactDownloadDir, _coreToolsHostMacArtifactName); + + EnsureArtifactDirectoryExist(inProc6ArtifactDirPath); + EnsureArtifactDirectoryExist(inProc8ArtifactDirPath); + EnsureArtifactDirectoryExist(outOfProcArtifactDirPath); EnsureArtifactDirectoryExist(coreToolsHostWindowsArtifactDirPath); EnsureArtifactDirectoryExist(coreToolsHostLinuxArtifactDirPath); EnsureArtifactDirectoryExist(coreToolsHostMacArtifactDirPath); + _inProc6ExtractedRootDir = await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(inProc6ArtifactDirPath, Path.Combine(_stagingDirectory, Constants.InProc6DirectoryName)); + _inProc8ExtractedRootDir = await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(inProc8ArtifactDirPath, Path.Combine(_stagingDirectory, Constants.InProc8DirectoryName)); + + Directory.Delete(inProcArtifactDownloadDir, true); + _coreToolsHostExtractedRootDir = await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(coreToolsHostWindowsArtifactDirPath, Path.Combine(_stagingDirectory, Constants.CoreToolsHostDirectoryName)); await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(coreToolsHostLinuxArtifactDirPath, Path.Combine(_stagingDirectory, Constants.CoreToolsHostDirectoryName)); await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(coreToolsHostMacArtifactDirPath, Path.Combine(_stagingDirectory, Constants.CoreToolsHostDirectoryName)); Directory.Delete(coreToolsHostArtifactDownloadDir, true); - var outOfProcArtifactDownloadDir = Path.Combine(_rootWorkingDirectory, _outOfProcArtifactDirectoryName); - var outOfProcArtifactDirPath = Path.Combine(outOfProcArtifactDownloadDir, _outOfProcArtifactName); - EnsureArtifactDirectoryExist(outOfProcArtifactDirPath); + _outOfProcExtractedRootDir = await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(outOfProcArtifactDirPath, Path.Combine(_stagingDirectory, Constants.OutOfProcDirectoryName)); Directory.Delete(outOfProcArtifactDownloadDir, true); - - _inProc6ExtractedRootDir = await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(inProc6ArtifactDirPath, Path.Combine(_stagingDirectory, Constants.InProc6DirectoryName)); - _inProc8ExtractedRootDir = await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(inProc8ArtifactDirPath, Path.Combine(_stagingDirectory, Constants.InProc8DirectoryName)); - - Directory.Delete(inProcArtifactDownloadDir, true); } private static void EnsureArtifactDirectoryExist(string directoryExist) From 4a6d9e7b26ab122459c07e05486ea811f69d127d Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Wed, 6 Nov 2024 22:41:32 -0800 Subject: [PATCH 05/30] logging artifact directory --- src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs b/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs index 5dda3983e..548862548 100644 --- a/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs +++ b/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs @@ -162,6 +162,7 @@ private static async Task MoveArtifactsToStagingDirectoryAndExtractIfNee // Example output: 4.0.6353 private static string GetCoreToolsProductVersion(string artifactDirectoryName) { + Console.WriteLine("Artifact directory name that is being compared: " + artifactDirectoryName); var match = Regex.Match(artifactDirectoryName, Constants.CoreToolsProductVersionPattern); if (match.Success) { @@ -180,6 +181,7 @@ private async Task CreateVisualStudioCoreToolsAsync() foreach (string artifactName in _visualStudioArtifacts.Keys) { + Console.WriteLine("Value of _inProc8ExtractedDir: " + _inProc8ExtractedRootDir); var inProc8ArtifactDirPath = Directory.EnumerateDirectories(_inProc8ExtractedRootDir) .FirstOrDefault(dir => dir.Contains(artifactName)); if (inProc8ArtifactDirPath == null) From e5e21628aef07d5e312a75fcea5bf2ecb63b885c Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Wed, 6 Nov 2024 22:45:44 -0800 Subject: [PATCH 06/30] skipping flakey e2e test --- test/Azure.Functions.Cli.Tests/E2E/StartTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Azure.Functions.Cli.Tests/E2E/StartTests.cs b/test/Azure.Functions.Cli.Tests/E2E/StartTests.cs index 8a98ac40b..a7d001ae8 100644 --- a/test/Azure.Functions.Cli.Tests/E2E/StartTests.cs +++ b/test/Azure.Functions.Cli.Tests/E2E/StartTests.cs @@ -1426,7 +1426,7 @@ await CliTester.Run(new RunConfiguration[] }, _output); } - [Theory] + [Theory(Skip = "Test is flakey")] [InlineData("dotnet")] [Trait(TestTraits.Group, TestTraits.RequiresNestedInProcArtifacts)] // [InlineData("dotnet-isolated")] Skip due to dotnet error on x86: https://github.com/Azure/azure-functions-core-tools/issues/3873 From 56767164ea95b7e2f040f79cdadfe3f461343d62 Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Thu, 7 Nov 2024 09:16:57 -0800 Subject: [PATCH 07/30] fixed version bug --- .../ArtifactAssembler.cs | 39 +++++-------------- 1 file changed, 9 insertions(+), 30 deletions(-) diff --git a/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs b/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs index 548862548..08077a45b 100644 --- a/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs +++ b/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs @@ -181,7 +181,6 @@ private async Task CreateVisualStudioCoreToolsAsync() foreach (string artifactName in _visualStudioArtifacts.Keys) { - Console.WriteLine("Value of _inProc8ExtractedDir: " + _inProc8ExtractedRootDir); var inProc8ArtifactDirPath = Directory.EnumerateDirectories(_inProc8ExtractedRootDir) .FirstOrDefault(dir => dir.Contains(artifactName)); if (inProc8ArtifactDirPath == null) @@ -240,7 +239,6 @@ private async Task CreateCliCoreToolsAsync() foreach (var artifactName in _cliArtifacts) { - Console.WriteLine($"Starting to assemble {artifactName}"); // If we are running this for the first time, extract the directory path and out of proc version if (String.IsNullOrEmpty(outOfProcArtifactDirPath)) { @@ -285,22 +283,22 @@ private async Task CreateCliCoreToolsAsync() inProc8ArtifactDirPath = Path.Combine(_inProc8ExtractedRootDir, artifactNameWithVersion); } - // Rename inproc8 directory to have the same version as the out-of-proc artifact before copying - string newInProc8ArtifactDirPath = RenameInProcDirectory(inProc8ArtifactDirPath, outOfProcVersion); - - // Copy in-proc8 files and delete old directory - await Task.Run(() => FileUtilities.CopyDirectory(newInProc8ArtifactDirPath, Path.Combine(consolidatedArtifactDirPath, Constants.InProc8DirectoryName))); + // Copy in-proc8 files + var inProc8FinalDestination = Path.Combine(consolidatedArtifactDirPath, Constants.InProc8DirectoryName); + await Task.Run(() => FileUtilities.CopyDirectory(inProc8ArtifactDirPath, Path.Combine(consolidatedArtifactDirPath, Constants.InProc8DirectoryName))); + Console.WriteLine($"Copied files from {inProc8ArtifactDirPath} => {inProc8FinalDestination}"); // Rename inproc6 directory to have the same version as the out-of-proc artifact before copying var inProcArtifactName = Path.GetFileName(inProc8ArtifactDirPath); var inProc6ArtifactDirPath = Path.Combine(_inProc6ExtractedRootDir, inProcArtifactName); EnsureArtifactDirectoryExist(inProc6ArtifactDirPath); - string newInProc6ArtifactDirPath = RenameInProcDirectory(inProc6ArtifactDirPath, outOfProcVersion); - // Copy in-proc6 files and delete old directory - await Task.Run(() => FileUtilities.CopyDirectory(newInProc6ArtifactDirPath, Path.Combine(consolidatedArtifactDirPath, Constants.InProc6DirectoryName))); + // Copy in-proc6 files + var inProc6FinalDestination = Path.Combine(consolidatedArtifactDirPath, Constants.InProc6DirectoryName); + await Task.Run(() => FileUtilities.CopyDirectory(inProc6ArtifactDirPath, Path.Combine(consolidatedArtifactDirPath, Constants.InProc6DirectoryName))); + Console.WriteLine($"Copied files from {inProc8ArtifactDirPath} => {inProc8FinalDestination}"); - Console.WriteLine($"Finished assembling {artifactName}"); + Console.WriteLine($"Finished assembling {consolidatedArtifactDirPath}"); } // Delete the extracted directories @@ -322,25 +320,6 @@ private async Task CreateCliCoreToolsAsync() return (artifactDirPath, version); } - private string RenameInProcDirectory(string oldArtifactDirPath, string newVersion) - { - Match match = Regex.Match(oldArtifactDirPath, Constants.ArtifactNameRegexPattern); - - if (!match.Success) - { - throw new InvalidOperationException($"Unable to extract content before version number from '{oldArtifactDirPath}'."); - } - - var artifactName = match.Groups[1]; - var newDirectoryName = $"{artifactName}{newVersion}"; - - // Rename (move) the directory - Directory.Move(oldArtifactDirPath, newDirectoryName); - - return newDirectoryName; - - } - private string GetRuntimeIdentifierForArtifactName(string artifactName) { if (_visualStudioArtifacts.TryGetValue(artifactName, out var rid)) From 352eef7f779532605cc2601f0377fca922d075e5 Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Thu, 7 Nov 2024 09:41:32 -0800 Subject: [PATCH 08/30] addressing comments --- .../ArtifactAssembler.cs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs b/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs index 08077a45b..0b56ab01d 100644 --- a/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs +++ b/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs @@ -148,8 +148,7 @@ private static async Task MoveArtifactsToStagingDirectoryAndExtractIfNee await ExtractZipFilesInDirectoryAsync(artifactZipPath, destinationDirectory); // Delete additional files that are not needed - var filesToBeDeleted = Directory.GetFiles(destinationDirectory); - Console.WriteLine($"{filesToBeDeleted.Length} files found in {destinationDirectory}"); + var filesToBeDeleted = Directory.EnumerateFiles(destinationDirectory); foreach (var file in filesToBeDeleted) { File.Delete(file); @@ -162,7 +161,6 @@ private static async Task MoveArtifactsToStagingDirectoryAndExtractIfNee // Example output: 4.0.6353 private static string GetCoreToolsProductVersion(string artifactDirectoryName) { - Console.WriteLine("Artifact directory name that is being compared: " + artifactDirectoryName); var match = Regex.Match(artifactDirectoryName, Constants.CoreToolsProductVersionPattern); if (match.Success) { @@ -194,13 +192,15 @@ private async Task CreateVisualStudioCoreToolsAsync() var consolidatedArtifactDirPath = Path.Combine(customHostTargetArtifactDir, consolidatedArtifactDirName); Directory.CreateDirectory(consolidatedArtifactDirPath); - // Copy in-proc8 files + // Copy in-proc8 files and delete directory after await Task.Run(() => FileUtilities.CopyDirectory(inProc8ArtifactDirPath, Path.Combine(consolidatedArtifactDirPath, Constants.InProc8DirectoryName))); + Directory.Delete(inProc8ArtifactDirPath, true); - // Copy in-proc6 files + // Copy in-proc6 files and delete directory after var inProc6ArtifactDirPath = Path.Combine(_inProc6ExtractedRootDir, artifactDirName); EnsureArtifactDirectoryExist(inProc6ArtifactDirPath); await Task.Run(() => FileUtilities.CopyDirectory(inProc6ArtifactDirPath, Path.Combine(consolidatedArtifactDirPath, Constants.InProc6DirectoryName))); + Directory.Delete(inProc6ArtifactDirPath, true); // Copy core-tools-host files var rid = GetRuntimeIdentifierForArtifactName(artifactName); @@ -265,7 +265,8 @@ private async Task CreateCliCoreToolsAsync() // If we are currently on the minified version of the artifacts, we do not want the inproc6/inproc8 subfolders if (artifactName.Contains("min.win")) { - Console.WriteLine($"Finished assembling {artifactName}"); + Console.WriteLine($"Finished assembling {consolidatedArtifactDirPath}"); + Console.WriteLine(); continue; } @@ -299,12 +300,14 @@ private async Task CreateCliCoreToolsAsync() Console.WriteLine($"Copied files from {inProc8ArtifactDirPath} => {inProc8FinalDestination}"); Console.WriteLine($"Finished assembling {consolidatedArtifactDirPath}"); + Console.WriteLine(); } // Delete the extracted directories Directory.Delete(_outOfProcExtractedRootDir, true); Console.WriteLine("Finished assembling CLI Core Tools artifacts"); + Console.WriteLine(); } private (string artifactDirectory, string version) GetArtifactDirectoryAndVersionNumber(string extractedRootDirectory, string artifactName) From 6c2bdb6f1c6791db84784ffc9c068ad84eb7c63b Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Thu, 7 Nov 2024 09:41:52 -0800 Subject: [PATCH 09/30] core tools host windows should still sign --- .../official/jobs/build-core-tools-host.yml | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/eng/ci/templates/official/jobs/build-core-tools-host.yml b/eng/ci/templates/official/jobs/build-core-tools-host.yml index 617c5b31e..8c45e98fe 100644 --- a/eng/ci/templates/official/jobs/build-core-tools-host.yml +++ b/eng/ci/templates/official/jobs/build-core-tools-host.yml @@ -35,6 +35,66 @@ jobs: arguments: -c Release -r win-arm64 -o $(Build.SourcesDirectory)/pkg_output/windows/win-arm64 --self-contained workingDirectory: $(Build.SourcesDirectory)/host/src/CoreToolsHost + - template: ci/sign-files.yml@eng + parameters: + displayName: 'Authenticode signing (dll) (win-arm64)' + folderPath: '$(Build.SourcesDirectory)/pkg_output/windows/win-arm64' + pattern: '*.dll, *.exe' + signType: inline + inlineOperation: | + [ + { + "KeyCode": "CP-230012", + "OperationCode": "SigntoolSign", + "Parameters": { + "OpusName": "Microsoft", + "OpusInfo": "http://www.microsoft.com", + "FileDigest": "/fd \"SHA256\"", + "PageHash": "/NPH", + "TimeStamp": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + }, + "ToolName": "sign", + "ToolVersion": "1.0" + }, + { + "KeyCode": "CP-230012", + "OperationCode": "SigntoolVerify", + "Parameters": {}, + "ToolName": "sign", + "ToolVersion": "1.0" + } + ] + + - template: ci/sign-files.yml@eng + parameters: + displayName: 'Authenticode signing (dll) (win-x64)' + folderPath: '$(Build.SourcesDirectory)/pkg_output/windows/win-x64' + pattern: '*.dll, *.exe' + signType: inline + inlineOperation: | + [ + { + "KeyCode": "CP-230012", + "OperationCode": "SigntoolSign", + "Parameters": { + "OpusName": "Microsoft", + "OpusInfo": "http://www.microsoft.com", + "FileDigest": "/fd \"SHA256\"", + "PageHash": "/NPH", + "TimeStamp": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + }, + "ToolName": "sign", + "ToolVersion": "1.0" + }, + { + "KeyCode": "CP-230012", + "OperationCode": "SigntoolVerify", + "Parameters": {}, + "ToolName": "sign", + "ToolVersion": "1.0" + } + ] + - task: CopyFiles@2 displayName: Copy files (win-x64) inputs: From 5885c82a77e5c344396d5095dcf8d517a60292d8 Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Thu, 7 Nov 2024 10:09:12 -0800 Subject: [PATCH 10/30] adding exe to comment --- eng/ci/templates/official/jobs/build-core-tools-host.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/ci/templates/official/jobs/build-core-tools-host.yml b/eng/ci/templates/official/jobs/build-core-tools-host.yml index 8c45e98fe..6b4638dfb 100644 --- a/eng/ci/templates/official/jobs/build-core-tools-host.yml +++ b/eng/ci/templates/official/jobs/build-core-tools-host.yml @@ -99,7 +99,7 @@ jobs: displayName: Copy files (win-x64) inputs: SourceFolder: $(Build.SourcesDirectory)/pkg_output/windows/win-x64 - # Publish output will include many other files. We only need func & nethost.dll + # Publish output will include many other files. We only need func.exe & nethost.dll Contents: | func.exe nethost.dll @@ -109,7 +109,7 @@ jobs: displayName: Copy files (win-arm64) inputs: SourceFolder: $(Build.SourcesDirectory)/pkg_output/windows/win-arm64 - # Publish output will include many other files. We only need func & nethost.dll + # Publish output will include many other files. We only need func.exe & nethost.dll Contents: | func.exe nethost.dll From a34407dc743916c11e9a06aeb4ab9e559ced042f Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Thu, 7 Nov 2024 10:39:46 -0800 Subject: [PATCH 11/30] removing self-contained arg --- .../templates/official/jobs/build-core-tools-host.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/ci/templates/official/jobs/build-core-tools-host.yml b/eng/ci/templates/official/jobs/build-core-tools-host.yml index 6b4638dfb..dfc867557 100644 --- a/eng/ci/templates/official/jobs/build-core-tools-host.yml +++ b/eng/ci/templates/official/jobs/build-core-tools-host.yml @@ -23,7 +23,7 @@ jobs: command: publish publishWebProjects: false zipAfterPublish: false - arguments: -c Release -r win-x64 -o $(Build.SourcesDirectory)/pkg_output/windows/win-x64 --self-contained + arguments: -c Release -r win-x64 -o $(Build.SourcesDirectory)/pkg_output/windows/win-x64 workingDirectory: $(Build.SourcesDirectory)/host/src/CoreToolsHost - task: DotnetCoreCLI@2 @@ -32,7 +32,7 @@ jobs: command: publish publishWebProjects: false zipAfterPublish: false - arguments: -c Release -r win-arm64 -o $(Build.SourcesDirectory)/pkg_output/windows/win-arm64 --self-contained + arguments: -c Release -r win-arm64 -o $(Build.SourcesDirectory)/pkg_output/windows/win-arm64 workingDirectory: $(Build.SourcesDirectory)/host/src/CoreToolsHost - template: ci/sign-files.yml@eng @@ -146,7 +146,7 @@ jobs: command: publish publishWebProjects: false zipAfterPublish: false - arguments: -c Release -r linux-x64 -o $(Build.SourcesDirectory)/pkg_output/linux/linux-x64 --self-contained + arguments: -c Release -r linux-x64 -o $(Build.SourcesDirectory)/pkg_output/linux/linux-x64 workingDirectory: $(Build.SourcesDirectory)/host/src/CoreToolsHost - task: CopyFiles@2 @@ -190,7 +190,7 @@ jobs: command: publish publishWebProjects: false zipAfterPublish: false - arguments: -c Release -r osx-x64 -o $(Build.SourcesDirectory)/pkg_output/osx/osx-x64 --self-contained + arguments: -c Release -r osx-x64 -o $(Build.SourcesDirectory)/pkg_output/osx/osx-x64 workingDirectory: $(Build.SourcesDirectory)/host/src/CoreToolsHost - task: DotnetCoreCLI@2 @@ -199,7 +199,7 @@ jobs: command: publish publishWebProjects: false zipAfterPublish: false - arguments: -c Release -r osx-arm64 -o $(Build.SourcesDirectory)/pkg_output/osx/osx-arm64 --self-contained + arguments: -c Release -r osx-arm64 -o $(Build.SourcesDirectory)/pkg_output/osx/osx-arm64 workingDirectory: $(Build.SourcesDirectory)/host/src/CoreToolsHost - task: CopyFiles@2 From 60b0d5de30ab98e6a20a14e0b8a348ee4d99c26c Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Thu, 7 Nov 2024 15:11:48 -0800 Subject: [PATCH 12/30] addding logging statement --- host/src/CoreToolsHost/AppLoader/NetHost.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/host/src/CoreToolsHost/AppLoader/NetHost.cs b/host/src/CoreToolsHost/AppLoader/NetHost.cs index 42dc71b74..9a7d536af 100644 --- a/host/src/CoreToolsHost/AppLoader/NetHost.cs +++ b/host/src/CoreToolsHost/AppLoader/NetHost.cs @@ -27,6 +27,7 @@ internal unsafe static string GetHostFxrPath(get_hostfxr_parameters* parameters) { char[] buffer = new char[200]; int bufferSize = buffer.Length; + Console.WriteLine("Does file exist: " + File.Exists("./in-proc6/func.dll")); int rc = get_hostfxr_path(buffer, ref bufferSize, parameters); From 8a33b969e9ebc45ee58a8b0d233b97aee8039ed9 Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Fri, 8 Nov 2024 08:25:20 -0800 Subject: [PATCH 13/30] adding assembly path logging --- host/src/CoreToolsHost/AppLoader/AppLoader.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/host/src/CoreToolsHost/AppLoader/AppLoader.cs b/host/src/CoreToolsHost/AppLoader/AppLoader.cs index 657a4b56d..a16c64835 100644 --- a/host/src/CoreToolsHost/AppLoader/AppLoader.cs +++ b/host/src/CoreToolsHost/AppLoader/AppLoader.cs @@ -32,6 +32,14 @@ internal int RunApplication(string? assemblyPath, string[] commandLineArgs) assembly_path = GetCharArrayPointer(assemblyPath) }; + // Convert to IntPtr first + IntPtr ptr = new IntPtr(parameters.assembly_path); + + // Convert IntPtr to string (ANSI) + string paramAssemblyPath = Marshal.PtrToStringAnsi(ptr); + Console.WriteLine($"Value of assembly path: {paramAssemblyPath}"); + Console.WriteLine("Does assembly path exist: " + File.Exists(paramAssemblyPath)); + isVerbose = commandLineArgs.Contains(DotnetConstants.Verbose); var hostfxrFullPath = NetHost.GetHostFxrPath(¶meters); From ad4f5501a3eadb2d0e75fbee63ef614029d87a78 Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Fri, 8 Nov 2024 08:46:06 -0800 Subject: [PATCH 14/30] removing not needed nethost logging --- host/src/CoreToolsHost/AppLoader/NetHost.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/host/src/CoreToolsHost/AppLoader/NetHost.cs b/host/src/CoreToolsHost/AppLoader/NetHost.cs index 9a7d536af..42dc71b74 100644 --- a/host/src/CoreToolsHost/AppLoader/NetHost.cs +++ b/host/src/CoreToolsHost/AppLoader/NetHost.cs @@ -27,7 +27,6 @@ internal unsafe static string GetHostFxrPath(get_hostfxr_parameters* parameters) { char[] buffer = new char[200]; int bufferSize = buffer.Length; - Console.WriteLine("Does file exist: " + File.Exists("./in-proc6/func.dll")); int rc = get_hostfxr_path(buffer, ref bufferSize, parameters); From 81449fc22ae4c63e789b599e0dea46478b3430a2 Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Fri, 8 Nov 2024 09:06:51 -0800 Subject: [PATCH 15/30] more logging! --- host/src/CoreToolsHost/AppLoader/AppLoader.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/host/src/CoreToolsHost/AppLoader/AppLoader.cs b/host/src/CoreToolsHost/AppLoader/AppLoader.cs index a16c64835..66d590e32 100644 --- a/host/src/CoreToolsHost/AppLoader/AppLoader.cs +++ b/host/src/CoreToolsHost/AppLoader/AppLoader.cs @@ -40,6 +40,14 @@ internal int RunApplication(string? assemblyPath, string[] commandLineArgs) Console.WriteLine($"Value of assembly path: {paramAssemblyPath}"); Console.WriteLine("Does assembly path exist: " + File.Exists(paramAssemblyPath)); + IntPtr dotnetRootPtr = new IntPtr(parameters.dotnet_root); + + string paramAssemblyPathDotnetRoot = Marshal.PtrToStringAnsi(dotnetRootPtr); + Console.WriteLine($"Value of dotnetRoot: {paramAssemblyPathDotnetRoot}"); + Console.WriteLine("Does dotnetRoot exist: " + File.Exists(paramAssemblyPathDotnetRoot)); + + Console.WriteLine($"Size of params: {parameters.size}"); + isVerbose = commandLineArgs.Contains(DotnetConstants.Verbose); var hostfxrFullPath = NetHost.GetHostFxrPath(¶meters); From 0cf6a86f3eb9931dcdd4ee19957c4fbba826ba25 Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Wed, 20 Nov 2024 13:07:18 -0800 Subject: [PATCH 16/30] adding signing back --- .../official/jobs/build-core-tools-host.yml | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/eng/ci/templates/official/jobs/build-core-tools-host.yml b/eng/ci/templates/official/jobs/build-core-tools-host.yml index dfc867557..22ba4016c 100644 --- a/eng/ci/templates/official/jobs/build-core-tools-host.yml +++ b/eng/ci/templates/official/jobs/build-core-tools-host.yml @@ -149,6 +149,36 @@ jobs: arguments: -c Release -r linux-x64 -o $(Build.SourcesDirectory)/pkg_output/linux/linux-x64 workingDirectory: $(Build.SourcesDirectory)/host/src/CoreToolsHost + - task: Bash@3 + inputs: + targetType: 'inline' # Specify 'filePath' if you want to use an external script file. + script: | + cd publish-scripts + python3 -m venv publish-env + source publish-env/bin/activate + sudo pip install -r requirements.txt + sudo apt-get install fakeroot + export PATH="$PATH:/mnt/vss/_work/_tasks/EsrpCodeSigning_7e3c371a-7f9c-4791-b1ce-742f18ad3a9b/5.1.1/net462" + chmod +x /mnt/vss/_work/_tasks/EsrpCodeSigning_7e3c371a-7f9c-4791-b1ce-742f18ad3a9b/5.1.1/net462/esrpcli.exe + bashEnvValue: '~/.profile' # Set value for BASH_ENV environment variable + + - template: ci/sign-files.yml@eng + parameters: + displayName: 'Sign' + folderPath: '/mnt/vss/_work/1/s/pkg_output/linux/linux-x64' + pattern: 'func' + signType: inline + inlineOperation: | + [ + { + "keyCode": "CP-450779-Pgp", + "operationSetCode": "LinuxSign", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + - task: CopyFiles@2 displayName: Copy files (linux-x64) inputs: From edf988fe3f8ed0b37ad071a01f794ec2a015f74c Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Wed, 20 Nov 2024 13:21:21 -0800 Subject: [PATCH 17/30] removing sign step --- eng/ci/templates/official/jobs/build-core-tools-host.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/ci/templates/official/jobs/build-core-tools-host.yml b/eng/ci/templates/official/jobs/build-core-tools-host.yml index 22ba4016c..b4829d0cd 100644 --- a/eng/ci/templates/official/jobs/build-core-tools-host.yml +++ b/eng/ci/templates/official/jobs/build-core-tools-host.yml @@ -159,7 +159,6 @@ jobs: sudo pip install -r requirements.txt sudo apt-get install fakeroot export PATH="$PATH:/mnt/vss/_work/_tasks/EsrpCodeSigning_7e3c371a-7f9c-4791-b1ce-742f18ad3a9b/5.1.1/net462" - chmod +x /mnt/vss/_work/_tasks/EsrpCodeSigning_7e3c371a-7f9c-4791-b1ce-742f18ad3a9b/5.1.1/net462/esrpcli.exe bashEnvValue: '~/.profile' # Set value for BASH_ENV environment variable - template: ci/sign-files.yml@eng From 54f3c768260c3f573de5ab13b9c19861cc185277 Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Wed, 20 Nov 2024 13:46:19 -0800 Subject: [PATCH 18/30] adding path var --- eng/ci/templates/official/jobs/build-core-tools-host.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/ci/templates/official/jobs/build-core-tools-host.yml b/eng/ci/templates/official/jobs/build-core-tools-host.yml index b4829d0cd..ae9b8e501 100644 --- a/eng/ci/templates/official/jobs/build-core-tools-host.yml +++ b/eng/ci/templates/official/jobs/build-core-tools-host.yml @@ -158,7 +158,7 @@ jobs: source publish-env/bin/activate sudo pip install -r requirements.txt sudo apt-get install fakeroot - export PATH="$PATH:/mnt/vss/_work/_tasks/EsrpCodeSigning_7e3c371a-7f9c-4791-b1ce-742f18ad3a9b/5.1.1/net462" + export PATH="$PATH:/mnt/vss/_work/_tasks/EsrpCodeSigning_7e3c371a-7f9c-4791-b1ce-742f18ad3a9b/5.1.3/net462/esrpcli.exe" bashEnvValue: '~/.profile' # Set value for BASH_ENV environment variable - template: ci/sign-files.yml@eng From bfc780f43b5b83dfa2d20dcb3f7772595704b372 Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Wed, 20 Nov 2024 14:07:34 -0800 Subject: [PATCH 19/30] adding variable to path --- eng/ci/templates/official/jobs/build-core-tools-host.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/ci/templates/official/jobs/build-core-tools-host.yml b/eng/ci/templates/official/jobs/build-core-tools-host.yml index ae9b8e501..8fc06fecd 100644 --- a/eng/ci/templates/official/jobs/build-core-tools-host.yml +++ b/eng/ci/templates/official/jobs/build-core-tools-host.yml @@ -158,7 +158,8 @@ jobs: source publish-env/bin/activate sudo pip install -r requirements.txt sudo apt-get install fakeroot - export PATH="$PATH:/mnt/vss/_work/_tasks/EsrpCodeSigning_7e3c371a-7f9c-4791-b1ce-742f18ad3a9b/5.1.3/net462/esrpcli.exe" + export PATH="$PATH:/mnt/vss/_work/_tasks/EsrpCodeSigning_7e3c371a-7f9c-4791-b1ce-742f18ad3a9b/5.1.3/net462" + chmod +x /mnt/vss/_work/_tasks/EsrpCodeSigning_7e3c371a-7f9c-4791-b1ce-742f18ad3a9b/5.1.3/net462/esrpcli.exe bashEnvValue: '~/.profile' # Set value for BASH_ENV environment variable - template: ci/sign-files.yml@eng From 22ad754797b92988d1e25d4ea77cb4a487e76bca Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Wed, 20 Nov 2024 16:17:39 -0800 Subject: [PATCH 20/30] install net8 --- eng/ci/templates/official/jobs/build-core-tools-host.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eng/ci/templates/official/jobs/build-core-tools-host.yml b/eng/ci/templates/official/jobs/build-core-tools-host.yml index 8fc06fecd..59acb95c5 100644 --- a/eng/ci/templates/official/jobs/build-core-tools-host.yml +++ b/eng/ci/templates/official/jobs/build-core-tools-host.yml @@ -139,6 +139,10 @@ jobs: inputs: version: 6.x displayName: Install .NET 6 + - task: UseDotNet@2 + inputs: + version: 8.x + displayName: Install .NET 8 - task: DotnetCoreCLI@2 displayName: Dotnet Publish (linux-x64) From 6cce9b3e8c2e093499e30366c410106f07e7aa70 Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Thu, 21 Nov 2024 11:11:51 -0800 Subject: [PATCH 21/30] seperating net9 task to seperate job --- .../official/jobs/build-core-tools-host.yml | 52 ++++++++++++++----- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/eng/ci/templates/official/jobs/build-core-tools-host.yml b/eng/ci/templates/official/jobs/build-core-tools-host.yml index 59acb95c5..6cfc58ece 100644 --- a/eng/ci/templates/official/jobs/build-core-tools-host.yml +++ b/eng/ci/templates/official/jobs/build-core-tools-host.yml @@ -121,7 +121,7 @@ jobs: - output: pipelineArtifact path: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages artifact: drop-coretools-host-windows - + - job: BuildCoreToolsHostLinux displayName: '[Linux] Build CoreToolsHost' pool: @@ -135,14 +135,6 @@ jobs: version: 9.x includePreviewVersions: true displayName: Install .NET 9 - - task: UseDotNet@2 - inputs: - version: 6.x - displayName: Install .NET 6 - - task: UseDotNet@2 - inputs: - version: 8.x - displayName: Install .NET 8 - task: DotnetCoreCLI@2 displayName: Dotnet Publish (linux-x64) @@ -153,6 +145,42 @@ jobs: arguments: -c Release -r linux-x64 -o $(Build.SourcesDirectory)/pkg_output/linux/linux-x64 workingDirectory: $(Build.SourcesDirectory)/host/src/CoreToolsHost + - task: CopyFiles@2 + displayName: Copy files (linux-x64) + inputs: + SourceFolder: $(Build.SourcesDirectory)/pkg_output/linux/linux-x64 + # Publish output will include many other files. We only need func & nethost.dll + Contents: | + func + libnethost.so + TargetFolder: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages/linux-x64 + + templateContext: + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - output: pipelineArtifact + path: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages + artifact: drop-coretools-host-linux + +- job: SignCoreToolsHostLinux + displayName: '[Linux] Sign CoreToolsHost' + dependsOn: BuildCoreToolsHostLinux + pool: + name: 1es-pool-azfunc + image: 1es-ubuntu-22.04 + os: linux + + steps: + - task: UseDotNet@2 + inputs: + version: 8.x + displayName: Install .NET 8 + + - task: DownloadPipelineArtifact@2 + inputs: + artifact: drop-coretools-host-linux + path: $(Build.SourcesDirectory)/drop-coretools-host-linux + - task: Bash@3 inputs: targetType: 'inline' # Specify 'filePath' if you want to use an external script file. @@ -169,7 +197,7 @@ jobs: - template: ci/sign-files.yml@eng parameters: displayName: 'Sign' - folderPath: '/mnt/vss/_work/1/s/pkg_output/linux/linux-x64' + folderPath: '/mnt/vss/_work/1/s/drop-coretools-host-linux pattern: 'func' signType: inline inlineOperation: | @@ -186,7 +214,7 @@ jobs: - task: CopyFiles@2 displayName: Copy files (linux-x64) inputs: - SourceFolder: $(Build.SourcesDirectory)/pkg_output/linux/linux-x64 + SourceFolder: $(Build.SourcesDirectory)/drop-coretools-host-linux # Publish output will include many other files. We only need func & nethost.dll Contents: | func @@ -198,7 +226,7 @@ jobs: outputs: - output: pipelineArtifact path: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages - artifact: drop-coretools-host-linux + artifact: drop-coretools-host-linux-new - job: BuildCoreToolsHostMac displayName: '[Mac] Build CoreToolsHost' From 28d8d33dffe8f2ce0b5b8e6fd0504ffc24930558 Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Thu, 21 Nov 2024 11:13:46 -0800 Subject: [PATCH 22/30] forgot quote --- eng/ci/templates/official/jobs/build-core-tools-host.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/ci/templates/official/jobs/build-core-tools-host.yml b/eng/ci/templates/official/jobs/build-core-tools-host.yml index 6cfc58ece..12076bab0 100644 --- a/eng/ci/templates/official/jobs/build-core-tools-host.yml +++ b/eng/ci/templates/official/jobs/build-core-tools-host.yml @@ -197,7 +197,7 @@ jobs: - template: ci/sign-files.yml@eng parameters: displayName: 'Sign' - folderPath: '/mnt/vss/_work/1/s/drop-coretools-host-linux + folderPath: '/mnt/vss/_work/1/s/drop-coretools-host-linux' pattern: 'func' signType: inline inlineOperation: | From 5261a31399960bf30deee52eabff4fb9982a6662 Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Thu, 21 Nov 2024 11:28:28 -0800 Subject: [PATCH 23/30] add core tools host --- eng/ci/templates/official/jobs/build-core-tools-host.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eng/ci/templates/official/jobs/build-core-tools-host.yml b/eng/ci/templates/official/jobs/build-core-tools-host.yml index 12076bab0..25c7bc47a 100644 --- a/eng/ci/templates/official/jobs/build-core-tools-host.yml +++ b/eng/ci/templates/official/jobs/build-core-tools-host.yml @@ -190,8 +190,7 @@ jobs: source publish-env/bin/activate sudo pip install -r requirements.txt sudo apt-get install fakeroot - export PATH="$PATH:/mnt/vss/_work/_tasks/EsrpCodeSigning_7e3c371a-7f9c-4791-b1ce-742f18ad3a9b/5.1.3/net462" - chmod +x /mnt/vss/_work/_tasks/EsrpCodeSigning_7e3c371a-7f9c-4791-b1ce-742f18ad3a9b/5.1.3/net462/esrpcli.exe + export PATH="$PATH:/mnt/vss/_work/_tasks/EsrpCodeSigning_7e3c371a-7f9c-4791-b1ce-742f18ad3a9b/5.1.4/net462" bashEnvValue: '~/.profile' # Set value for BASH_ENV environment variable - template: ci/sign-files.yml@eng From a3522ea5e0013a1f061037489ed67efe52db97b4 Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Thu, 21 Nov 2024 13:09:11 -0800 Subject: [PATCH 24/30] adding mac signing --- .../official/jobs/build-core-tools-host.yml | 83 ++++++++++++++++--- 1 file changed, 72 insertions(+), 11 deletions(-) diff --git a/eng/ci/templates/official/jobs/build-core-tools-host.yml b/eng/ci/templates/official/jobs/build-core-tools-host.yml index 25c7bc47a..970bf15be 100644 --- a/eng/ci/templates/official/jobs/build-core-tools-host.yml +++ b/eng/ci/templates/official/jobs/build-core-tools-host.yml @@ -197,7 +197,7 @@ jobs: parameters: displayName: 'Sign' folderPath: '/mnt/vss/_work/1/s/drop-coretools-host-linux' - pattern: 'func' + pattern: 'func, libnethost.so' signType: inline inlineOperation: | [ @@ -213,8 +213,8 @@ jobs: - task: CopyFiles@2 displayName: Copy files (linux-x64) inputs: - SourceFolder: $(Build.SourcesDirectory)/drop-coretools-host-linux - # Publish output will include many other files. We only need func & nethost.dll + SourceFolder: $(Build.SourcesDirectory)/drop-coretools-host-linux/linux-x64 + # Publish output will include many other files. We only need func & libnethost.so Contents: | func libnethost.so @@ -225,7 +225,7 @@ jobs: outputs: - output: pipelineArtifact path: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages - artifact: drop-coretools-host-linux-new + artifact: drop-coretools-host-linux-signed - job: BuildCoreToolsHostMac displayName: '[Mac] Build CoreToolsHost' @@ -240,11 +240,7 @@ jobs: version: 9.x includePreviewVersions: true displayName: Install .NET 9 - - task: UseDotNet@2 - inputs: - version: 6.x - displayName: Install .NET 6 - + - task: DotnetCoreCLI@2 displayName: Dotnet Publish (osx-x64) inputs: @@ -266,7 +262,72 @@ jobs: - task: CopyFiles@2 displayName: Copy files (osx-x64) inputs: - SourceFolder: $(Build.SourcesDirectory)/pkg_output/osx/osx-x64 + SourceFolder: $(Build.SourcesDirectory)/drop-coretools-host-osx/osx-x64 + # Publish output will include many other files. We only need func & nethost.dll + Contents: | + func + libnethost.dylib + libnethost.a + TargetFolder: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages/osx-x64 + + - task: CopyFiles@2 + displayName: Copy files (osx-arm64) + inputs: + SourceFolder: $(Build.SourcesDirectory)/drop-coretools-host-osx/osx-arm64 + # Publish output will include many other files. We only need func & nethost.dll + Contents: | + func + libnethost.dylib + libnethost.a + TargetFolder: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages/osx-arm64 + + templateContext: + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - output: pipelineArtifact + path: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages + artifact: drop-coretools-host-osx + +- job: SignCoreToolsHostMac + displayName: '[Mac] Sign CoreToolsHost' + dependsOn: BuildCoreToolsHostMac + pool: + name: Azure Pipelines + image: 'macOS-latest' + os: macOS + + steps: + - task: UseDotNet@2 + inputs: + version: 8.x + displayName: Install .NET 8 + + - task: DownloadPipelineArtifact@2 + inputs: + artifact: drop-coretools-host-osx + path: $(Build.SourcesDirectory)/drop-coretools-host-osx + + - template: ci/sign-files.yml@eng + parameters: + displayName: 'Sign' + folderPath: '/mnt/vss/_work/1/s/drop-coretools-host-osx' + pattern: 'func, libnethost.dylib, libnethost.a' + signType: inline + inlineOperation: | + [ + { + "KeyCode": "CP-401337-Apple", + "OperationCode": "MacAppDeveloperSign", + "ToolName": "sign", + "ToolVersion": "1.0", + "Parameters": {} + } + ] + + - task: CopyFiles@2 + displayName: Copy files (osx-x64) + inputs: + SourceFolder: $(Build.SourcesDirectory)/drop-coretools-host-osx/osx-x64 # Publish output will include many other files. We only need func & nethost.dll Contents: | func @@ -277,7 +338,7 @@ jobs: - task: CopyFiles@2 displayName: Copy files (osx-arm64) inputs: - SourceFolder: $(Build.SourcesDirectory)/pkg_output/osx/osx-arm64 + SourceFolder: $(Build.SourcesDirectory)/drop-coretools-host-osx/osx-arm64 # Publish output will include many other files. We only need func & nethost.dll Contents: | func From e7c4b3766ab4d76fe805a1b288cf7010214d6bb2 Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Thu, 21 Nov 2024 13:16:42 -0800 Subject: [PATCH 25/30] changing mac folder back --- eng/ci/templates/official/jobs/build-core-tools-host.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/ci/templates/official/jobs/build-core-tools-host.yml b/eng/ci/templates/official/jobs/build-core-tools-host.yml index 970bf15be..95c99dae7 100644 --- a/eng/ci/templates/official/jobs/build-core-tools-host.yml +++ b/eng/ci/templates/official/jobs/build-core-tools-host.yml @@ -262,7 +262,7 @@ jobs: - task: CopyFiles@2 displayName: Copy files (osx-x64) inputs: - SourceFolder: $(Build.SourcesDirectory)/drop-coretools-host-osx/osx-x64 + SourceFolder: $(Build.SourcesDirectory)/pkg_output/osx/osx-x64 # Publish output will include many other files. We only need func & nethost.dll Contents: | func @@ -273,7 +273,7 @@ jobs: - task: CopyFiles@2 displayName: Copy files (osx-arm64) inputs: - SourceFolder: $(Build.SourcesDirectory)/drop-coretools-host-osx/osx-arm64 + SourceFolder: $(Build.SourcesDirectory)/pkg_output/osx/osx-arm64 # Publish output will include many other files. We only need func & nethost.dll Contents: | func From ecb9206d205ae1efaf5a39544aaaed5ae7465eac Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Thu, 21 Nov 2024 13:31:22 -0800 Subject: [PATCH 26/30] changing signing path back --- eng/ci/templates/official/jobs/build-core-tools-host.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/ci/templates/official/jobs/build-core-tools-host.yml b/eng/ci/templates/official/jobs/build-core-tools-host.yml index 95c99dae7..5a59e3ea1 100644 --- a/eng/ci/templates/official/jobs/build-core-tools-host.yml +++ b/eng/ci/templates/official/jobs/build-core-tools-host.yml @@ -310,7 +310,7 @@ jobs: - template: ci/sign-files.yml@eng parameters: displayName: 'Sign' - folderPath: '/mnt/vss/_work/1/s/drop-coretools-host-osx' + folderPath: '$(Build.SourcesDirectory)/drop-coretools-host-osx' pattern: 'func, libnethost.dylib, libnethost.a' signType: inline inlineOperation: | From eb31c001db41522d0b836763512ca4e3fdc20d7c Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Thu, 21 Nov 2024 13:46:29 -0800 Subject: [PATCH 27/30] trying 3rd party sign --- eng/ci/templates/official/jobs/build-core-tools-host.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/ci/templates/official/jobs/build-core-tools-host.yml b/eng/ci/templates/official/jobs/build-core-tools-host.yml index 5a59e3ea1..1f46b8b84 100644 --- a/eng/ci/templates/official/jobs/build-core-tools-host.yml +++ b/eng/ci/templates/official/jobs/build-core-tools-host.yml @@ -316,8 +316,8 @@ jobs: inlineOperation: | [ { - "KeyCode": "CP-401337-Apple", - "OperationCode": "MacAppDeveloperSign", + "KeyCode": "CP-233029-Apple", + "OperationCode": "MacAppThirdPartySign", "ToolName": "sign", "ToolVersion": "1.0", "Parameters": {} From 0c97b8ea10401fb54572ddc76c27a814999c05c4 Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Thu, 21 Nov 2024 14:15:45 -0800 Subject: [PATCH 28/30] trying to notarize --- eng/ci/templates/official/jobs/build-core-tools-host.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/eng/ci/templates/official/jobs/build-core-tools-host.yml b/eng/ci/templates/official/jobs/build-core-tools-host.yml index 1f46b8b84..76e5d6dbc 100644 --- a/eng/ci/templates/official/jobs/build-core-tools-host.yml +++ b/eng/ci/templates/official/jobs/build-core-tools-host.yml @@ -316,11 +316,13 @@ jobs: inlineOperation: | [ { - "KeyCode": "CP-233029-Apple", - "OperationCode": "MacAppThirdPartySign", + "KeyCode": "CP-401337-Apple", + "OperationCode": "MacAppNotarize", "ToolName": "sign", "ToolVersion": "1.0", - "Parameters": {} + "Parameters": { + "BundleId": "com.microsoft.esrp" + } } ] From a7f65efd587904172c7a38698fd3c7937311f15e Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Mon, 25 Nov 2024 13:05:38 -0800 Subject: [PATCH 29/30] remove custom host mac --- .../official/jobs/build-core-tools-host.yml | 130 +----------------- .../ArtifactAssembler.cs | 9 +- .../EnvironmentVariables.cs | 1 - 3 files changed, 2 insertions(+), 138 deletions(-) diff --git a/eng/ci/templates/official/jobs/build-core-tools-host.yml b/eng/ci/templates/official/jobs/build-core-tools-host.yml index 76e5d6dbc..2953745f8 100644 --- a/eng/ci/templates/official/jobs/build-core-tools-host.yml +++ b/eng/ci/templates/official/jobs/build-core-tools-host.yml @@ -225,132 +225,4 @@ jobs: outputs: - output: pipelineArtifact path: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages - artifact: drop-coretools-host-linux-signed - -- job: BuildCoreToolsHostMac - displayName: '[Mac] Build CoreToolsHost' - pool: - name: Azure Pipelines - image: 'macOS-latest' - os: macOS - - steps: - - task: UseDotNet@2 - inputs: - version: 9.x - includePreviewVersions: true - displayName: Install .NET 9 - - - task: DotnetCoreCLI@2 - displayName: Dotnet Publish (osx-x64) - inputs: - command: publish - publishWebProjects: false - zipAfterPublish: false - arguments: -c Release -r osx-x64 -o $(Build.SourcesDirectory)/pkg_output/osx/osx-x64 - workingDirectory: $(Build.SourcesDirectory)/host/src/CoreToolsHost - - - task: DotnetCoreCLI@2 - displayName: Dotnet Publish (osx-arm64) - inputs: - command: publish - publishWebProjects: false - zipAfterPublish: false - arguments: -c Release -r osx-arm64 -o $(Build.SourcesDirectory)/pkg_output/osx/osx-arm64 - workingDirectory: $(Build.SourcesDirectory)/host/src/CoreToolsHost - - - task: CopyFiles@2 - displayName: Copy files (osx-x64) - inputs: - SourceFolder: $(Build.SourcesDirectory)/pkg_output/osx/osx-x64 - # Publish output will include many other files. We only need func & nethost.dll - Contents: | - func - libnethost.dylib - libnethost.a - TargetFolder: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages/osx-x64 - - - task: CopyFiles@2 - displayName: Copy files (osx-arm64) - inputs: - SourceFolder: $(Build.SourcesDirectory)/pkg_output/osx/osx-arm64 - # Publish output will include many other files. We only need func & nethost.dll - Contents: | - func - libnethost.dylib - libnethost.a - TargetFolder: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages/osx-arm64 - - templateContext: - outputParentDirectory: $(Build.ArtifactStagingDirectory) - outputs: - - output: pipelineArtifact - path: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages - artifact: drop-coretools-host-osx - -- job: SignCoreToolsHostMac - displayName: '[Mac] Sign CoreToolsHost' - dependsOn: BuildCoreToolsHostMac - pool: - name: Azure Pipelines - image: 'macOS-latest' - os: macOS - - steps: - - task: UseDotNet@2 - inputs: - version: 8.x - displayName: Install .NET 8 - - - task: DownloadPipelineArtifact@2 - inputs: - artifact: drop-coretools-host-osx - path: $(Build.SourcesDirectory)/drop-coretools-host-osx - - - template: ci/sign-files.yml@eng - parameters: - displayName: 'Sign' - folderPath: '$(Build.SourcesDirectory)/drop-coretools-host-osx' - pattern: 'func, libnethost.dylib, libnethost.a' - signType: inline - inlineOperation: | - [ - { - "KeyCode": "CP-401337-Apple", - "OperationCode": "MacAppNotarize", - "ToolName": "sign", - "ToolVersion": "1.0", - "Parameters": { - "BundleId": "com.microsoft.esrp" - } - } - ] - - - task: CopyFiles@2 - displayName: Copy files (osx-x64) - inputs: - SourceFolder: $(Build.SourcesDirectory)/drop-coretools-host-osx/osx-x64 - # Publish output will include many other files. We only need func & nethost.dll - Contents: | - func - libnethost.dylib - libnethost.a - TargetFolder: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages/osx-x64 - - - task: CopyFiles@2 - displayName: Copy files (osx-arm64) - inputs: - SourceFolder: $(Build.SourcesDirectory)/drop-coretools-host-osx/osx-arm64 - # Publish output will include many other files. We only need func & nethost.dll - Contents: | - func - libnethost.dylib - libnethost.a - TargetFolder: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages/osx-arm64 - - templateContext: - outputParentDirectory: $(Build.ArtifactStagingDirectory) - outputs: - - output: pipelineArtifact - path: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages - artifact: drop-coretools-host-osx \ No newline at end of file + artifact: drop-coretools-host-linux-signed \ No newline at end of file diff --git a/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs b/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs index 0b56ab01d..0a6ba9fdc 100644 --- a/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs +++ b/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs @@ -15,9 +15,7 @@ internal sealed class ArtifactAssembler { { "Azure.Functions.Cli.min.win-x64", "win-x64" }, { "Azure.Functions.Cli.min.win-arm64", "win-arm64" }, - { "Azure.Functions.Cli.linux-x64", "linux-x64" }, - { "Azure.Functions.Cli.osx-x64", "osx-x64" }, - { "Azure.Functions.Cli.osx-arm64", "osx-arm64" } + { "Azure.Functions.Cli.linux-x64", "linux-x64" } }; /// @@ -43,7 +41,6 @@ internal sealed class ArtifactAssembler private readonly string _inProc8ArtifactName; private readonly string _coreToolsHostWindowsArtifactName; private readonly string _coreToolsHostLinuxArtifactName; - private readonly string _coreToolsHostMacArtifactName; private readonly string _outOfProcArtifactName; private readonly string _rootWorkingDirectory; private readonly string _stagingDirectory; @@ -63,7 +60,6 @@ internal ArtifactAssembler(string rootWorkingDirectory) _inProc8ArtifactName = GetRequiredEnvironmentVariable(EnvironmentVariables.InProc8ArtifactName); _coreToolsHostWindowsArtifactName = GetRequiredEnvironmentVariable(EnvironmentVariables.CoreToolsHostWindowsArtifactName); _coreToolsHostLinuxArtifactName = GetRequiredEnvironmentVariable(EnvironmentVariables.CoreToolsHostLinuxArtifactName); - _coreToolsHostMacArtifactName = GetRequiredEnvironmentVariable(EnvironmentVariables.CoreToolsHostOsxArtifactName); _outOfProcArtifactName = GetRequiredEnvironmentVariable(EnvironmentVariables.OutOfProcArtifactName); _rootWorkingDirectory = rootWorkingDirectory; @@ -95,14 +91,12 @@ private async Task ExtractDownloadedArtifactsAsync() var coreToolsHostWindowsArtifactDirPath = Path.Combine(coreToolsHostArtifactDownloadDir, _coreToolsHostWindowsArtifactName); var coreToolsHostLinuxArtifactDirPath = Path.Combine(coreToolsHostArtifactDownloadDir, _coreToolsHostLinuxArtifactName); - var coreToolsHostMacArtifactDirPath = Path.Combine(coreToolsHostArtifactDownloadDir, _coreToolsHostMacArtifactName); EnsureArtifactDirectoryExist(inProc6ArtifactDirPath); EnsureArtifactDirectoryExist(inProc8ArtifactDirPath); EnsureArtifactDirectoryExist(outOfProcArtifactDirPath); EnsureArtifactDirectoryExist(coreToolsHostWindowsArtifactDirPath); EnsureArtifactDirectoryExist(coreToolsHostLinuxArtifactDirPath); - EnsureArtifactDirectoryExist(coreToolsHostMacArtifactDirPath); _inProc6ExtractedRootDir = await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(inProc6ArtifactDirPath, Path.Combine(_stagingDirectory, Constants.InProc6DirectoryName)); _inProc8ExtractedRootDir = await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(inProc8ArtifactDirPath, Path.Combine(_stagingDirectory, Constants.InProc8DirectoryName)); @@ -111,7 +105,6 @@ private async Task ExtractDownloadedArtifactsAsync() _coreToolsHostExtractedRootDir = await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(coreToolsHostWindowsArtifactDirPath, Path.Combine(_stagingDirectory, Constants.CoreToolsHostDirectoryName)); await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(coreToolsHostLinuxArtifactDirPath, Path.Combine(_stagingDirectory, Constants.CoreToolsHostDirectoryName)); - await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(coreToolsHostMacArtifactDirPath, Path.Combine(_stagingDirectory, Constants.CoreToolsHostDirectoryName)); Directory.Delete(coreToolsHostArtifactDownloadDir, true); _outOfProcExtractedRootDir = await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(outOfProcArtifactDirPath, Path.Combine(_stagingDirectory, Constants.OutOfProcDirectoryName)); diff --git a/src/Azure.Functions.ArtifactAssembler/EnvironmentVariables.cs b/src/Azure.Functions.ArtifactAssembler/EnvironmentVariables.cs index 079b3ac30..5ea11399b 100644 --- a/src/Azure.Functions.ArtifactAssembler/EnvironmentVariables.cs +++ b/src/Azure.Functions.ArtifactAssembler/EnvironmentVariables.cs @@ -12,7 +12,6 @@ internal static class EnvironmentVariables internal const string InProc8ArtifactName = "IN_PROC8_ARTIFACT_NAME"; internal const string CoreToolsHostWindowsArtifactName = "CORETOOLS_HOST_WINDOWS_ARTIFACT_NAME"; internal const string CoreToolsHostLinuxArtifactName = "CORETOOLS_HOST_LINUX_ARTIFACT_NAME"; - internal const string CoreToolsHostOsxArtifactName = "CORETOOLS_HOST_OSX_ARTIFACT_NAME"; internal const string OutOfProcArtifactName = "OUT_OF_PROC_ARTIFACT_NAME"; } } \ No newline at end of file From 96150e13e857486812c627d81c9e420c278ae401 Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandari Date: Mon, 25 Nov 2024 13:13:21 -0800 Subject: [PATCH 30/30] removing not needed logging --- host/src/CoreToolsHost/AppLoader/AppLoader.cs | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/host/src/CoreToolsHost/AppLoader/AppLoader.cs b/host/src/CoreToolsHost/AppLoader/AppLoader.cs index 66d590e32..657a4b56d 100644 --- a/host/src/CoreToolsHost/AppLoader/AppLoader.cs +++ b/host/src/CoreToolsHost/AppLoader/AppLoader.cs @@ -32,22 +32,6 @@ internal int RunApplication(string? assemblyPath, string[] commandLineArgs) assembly_path = GetCharArrayPointer(assemblyPath) }; - // Convert to IntPtr first - IntPtr ptr = new IntPtr(parameters.assembly_path); - - // Convert IntPtr to string (ANSI) - string paramAssemblyPath = Marshal.PtrToStringAnsi(ptr); - Console.WriteLine($"Value of assembly path: {paramAssemblyPath}"); - Console.WriteLine("Does assembly path exist: " + File.Exists(paramAssemblyPath)); - - IntPtr dotnetRootPtr = new IntPtr(parameters.dotnet_root); - - string paramAssemblyPathDotnetRoot = Marshal.PtrToStringAnsi(dotnetRootPtr); - Console.WriteLine($"Value of dotnetRoot: {paramAssemblyPathDotnetRoot}"); - Console.WriteLine("Does dotnetRoot exist: " + File.Exists(paramAssemblyPathDotnetRoot)); - - Console.WriteLine($"Size of params: {parameters.size}"); - isVerbose = commandLineArgs.Contains(DotnetConstants.Verbose); var hostfxrFullPath = NetHost.GetHostFxrPath(¶meters);