Install dotnet tools locally in the Nuget Packages

In commit 7a398b0 we added two new tools for compressing textures, `crunch` and `basisu`.
The problem there is that the management of the `.config/dotnet-tool.json` file by the users was becoming an issue.
We needed a more automatic way to install the required tooling.

The problem is using the standard `dotnet tool install` calls requires a `.config/dotnet-tool.json` file to be present in either the current directory or a directory that is in the path ABOVE the current one. You would use the `--create-manifest-if-needed` flag to create the manifest, but that will still leave the users having to manage and upgrade the .json file every time we do a release.

So lets get the pipeline to install the tooling itself.
The `dotnet tool install` command has an additional argument`--tool-path` this allows us to say where we want the tool installed. Once that has happened we get a native binary launcher in `--tool-path` which allows us to launch the app directly without using the `dotnet` executable. So what this allows us to do is install the tooling locally in the directory that the content pipeline is installed. This will usually be the global `.nuget/package` directory.

We need to keep an eye on the `DOTNET_ROOT` environment variable when installing the tooling, just in case a user (or CI) wants to use a custom dotnet install.

Also added support for overriding which MGCB to use in the editor by looking for a `MGCBCommand` environment variable. This should be the full path to either the `mgcb` exe or the `mgcb.dll` that the user wants to use. This will allow users to change to a custom content compiler without having to change the editor.

The one downside is users will no longer be able to run `dotnet mgcb` directly in their project directory unless they install the tooling manually.

The Editor also needs to download the `mgcb` tool. We place this in the `ApplicationData` folder rather than in the local assembly folder because we cannot guarantee that folder is writable. So in this instance a location specific to the current user seemed like the best approach.

The long term plan is to bundle all these tools into a single native library which can be called directly by the content pipeline.

.github/workflows/main.yml

@@ -105,7 +105,6 @@ jobs:
       - name: Test
         run: dotnet test Tools/MonoGame.Tools.Tests/MonoGame.Tools.Tests.csproj --blame-hang-timeout 5m -c Release --filter="TestCategory!=Audio"
         env:
-          DOTNET_ROOT: ${{github.workspace}}/dotnet64
           MGFXC_WINE_PATH: /Users/runner/.winemonogame
           CI: true
         if: runner.os == 'macOS'
@@ -231,11 +230,6 @@ jobs:
           path: tests-windowsdx
         if: runner.os == 'Windows'
 
-      - name: Install Tools
-        run: |
-          dotnet tool install --create-manifest-if-needed mgcb-basisu
-          dotnet tool install --create-manifest-if-needed mgcb-crunch
-
       - name: Run Tools Tests
         run: dotnet test tests-tools/MonoGame.Tools.Tests.dll --blame-hang-timeout 1m --filter="TestCategory!=Effects"
         env:

.vscode/launch.json

@@ -20,7 +20,7 @@
             "type": "coreclr",
             "request": "launch",
             "preLaunchTask": "mgcb-editor-mac",
-            "program": "${workspaceFolder}/Artifacts/MonoGame.Content.Builder.Editor/Mac/Debug/MGCB Editor.app/Contents/MacOS/mgcb-editor-mac",
+            "program": "${workspaceFolder}/Artifacts/MonoGame.Content.Builder.Editor/Mac/Debug/mgcb-editor-mac.app/Contents/MacOS/mgcb-editor-mac",
             "args": [],
             "cwd": "${workspaceFolder}/Artifacts/MonoGame.Content.Builder.Editor/Mac/Debug",
             "console": "internalConsole",

MonoGame.Framework.Content.Pipeline/ExternalTool.cs

@@ -5,6 +5,8 @@
 using System;
 using System.Diagnostics;
 using System.IO;
+using System.Net;
+using System.Reflection;
 using System.Threading;
 using MonoGame.Framework.Utilities;
 
@@ -17,6 +19,11 @@ namespace Microsoft.Xna.Framework.Content.Pipeline
     /// </summary>
     internal class ExternalTool
     {
+        public static string Crunch = "mgcb-crunch";
+        private static string CrunchVersion = "1.0.4.2";
+        public static string BasisU = "mgcb-basisu";
+        private static string BasisUVersion = "1.16.4.2";
+
         public static int Run(string command, string arguments)
         {
             string stdout, stderr;
@@ -27,13 +34,47 @@ public static int Run(string command, string arguments)
             return result;
         }
 
+        public static void RestoreDotnetTool(string command, string toolName, string toolVersion, string path)
+        {
+            Directory.CreateDirectory(path);
+            var exe = CurrentPlatform.OS == OS.Windows ? "dotnet.exe" : "dotnet";
+            var dotnetRoot = Environment.GetEnvironmentVariable("DOTNET_ROOT");
+            if (!string.IsNullOrEmpty(dotnetRoot))
+            {
+                exe = Path.Combine(dotnetRoot, exe);
+            }
+            if (Run(exe, $"tool {command} {toolName} --version {toolVersion} --tool-path .", out string _, out string _,  workingDirectory: path) != 0)
+            {
+                // install the latest
+                Run(exe, $"tool {command} {toolName} --tool-path .", out _, out _,  workingDirectory: path);
+            }
+        }
+
+        public static void RestoreDotnetTools()
+        {
+            var version = Assembly.GetExecutingAssembly().GetName().Version.ToString();
+            string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory);
+            if (CurrentPlatform.OS == OS.Linux)
+                path= Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "linux");
+            if (CurrentPlatform.OS == OS.MacOSX)
+                path= Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "osx");
+            var versionFile = Path.Combine(path, $"tools_version.txt");
+            if (File.Exists(versionFile) && File.ReadAllText (versionFile) == version)
+                return;
+            RestoreDotnetTool("install", Crunch, CrunchVersion, path);
+            RestoreDotnetTool("install", BasisU, BasisUVersion, path);
+            File.WriteAllText(versionFile, version);
+        }
+
         /// <summary>
         /// Run a dotnet tool. The tool should be installed in a .config/dotnet-tools.json file somewhere in the project lineage.
         /// </summary>
         public static int RunDotnetTool(string toolName, string args, out string stdOut, out string stdErr, string stdIn=null, string workingDirectory=null)
         {
-            var finalizedArgs = toolName + " " + args;
-            return ExternalTool.Run("dotnet", finalizedArgs, out stdOut, out stdErr, stdIn, workingDirectory);
+            RestoreDotnetTools();
+            var exe = FindCommand (toolName);
+            var finalizedArgs =  args;
+            return ExternalTool.Run(exe, finalizedArgs, out stdOut, out stdErr, stdIn, workingDirectory);
         }
 
         public static int Run(string command, string arguments, out string stdout, out string stderr, string stdin = null, string workingDirectory=null)
@@ -68,6 +109,11 @@ public static int Run(string command, string arguments, out string stdout, out s
             if (!string.IsNullOrEmpty(workingDirectory))
                 processInfo.WorkingDirectory = workingDirectory;
 
+            var dotnetRoot = Environment.GetEnvironmentVariable ("DOTNET_ROOT");
+            if (!string.IsNullOrEmpty(dotnetRoot)) {
+                processInfo.EnvironmentVariables["DOTNET_ROOT"] = dotnetRoot;
+            }
+
             EnsureExecutable(fullPath);
 
             using (var process = new Process())

MonoGame.Framework.Content.Pipeline/MonoGame.Framework.Content.Pipeline.csproj

@@ -76,8 +76,6 @@
     <PackageReference Include="AssimpNet" Version="5.0.0-beta1" IncludeAssets="compile;runtime;build" ExcludeAssets="native" />
     <PackageReference Include="MonoGame.Library.Assimp" Version="5.3.1.1" />
     <PackageReference Include="BCnEncoder.Net" Version="2.1.0" />
-    <PackageReference Include="BCnEncoder.Net.ImageSharp" Version="1.1.1" />
-    <PackageReference Include="SixLabors.ImageSharp" Version="3.1.5" />
     <PackageReference Include="Microsoft.NETCore.App" Version="2.1.30" />
     <PackageReference Include="RoyT.TrueType" Version="0.2.0" />
     <PackageReference Include="SharpDX" Version="4.0.1" />

MonoGame.Framework.Content.Pipeline/Utilities/BasisUHelpers.cs

@@ -206,7 +206,7 @@ internal static class BasisU
     /// <returns>The exit code for the basisu process. </returns>
     public static int Run(string args, out string stdOut, out string stdErr, string stdIn=null, string workingDirectory=null)
     {
-        return ExternalTool.RunDotnetTool("mgcb-basisu", args, out stdOut, out stdErr, stdIn, workingDirectory);
+        return ExternalTool.RunDotnetTool(ExternalTool.BasisU, args, out stdOut, out stdErr, stdIn, workingDirectory);
     }
 
     /// <summary>

MonoGame.Framework.Content.Pipeline/Utilities/CrunchHelpers.cs

@@ -137,7 +137,7 @@ internal static class Crunch
         /// <returns>The exit code for the basisu process. </returns>
         private static int Run(string args, out string stdOut, out string stdErr)
         {
-            return ExternalTool.RunDotnetTool("mgcb-crunch", args, out stdOut, out stdErr);
+            return ExternalTool.RunDotnetTool(ExternalTool.Crunch, args, out stdOut, out stdErr);
         }
 
         /// <summary>

Templates/MonoGame.Templates.CSharp/content/MonoGame.Application.Android.CSharp/.config/dotnet-tools.json

@@ -2,12 +2,6 @@
   "version": 1,
   "isRoot": true,
   "tools": {
-    "dotnet-mgcb": {
-      "version": "3.8.2.1-develop",
-      "commands": [
-        "mgcb"
-      ]
-    },
     "dotnet-mgcb-editor": {
       "version": "3.8.2.1-develop",
       "commands": [

Templates/MonoGame.Templates.CSharp/content/MonoGame.Application.DesktopGL.CSharp/.config/dotnet-tools.json

@@ -2,12 +2,6 @@
   "version": 1,
   "isRoot": true,
   "tools": {
-    "dotnet-mgcb": {
-      "version": "3.8.2.1-develop",
-      "commands": [
-        "mgcb"
-      ]
-    },
     "dotnet-mgcb-editor": {
       "version": "3.8.2.1-develop",
       "commands": [

Templates/MonoGame.Templates.CSharp/content/MonoGame.Application.WindowsDX.CSharp/.config/dotnet-tools.json

@@ -2,12 +2,6 @@
   "version": 1,
   "isRoot": true,
   "tools": {
-    "dotnet-mgcb": {
-      "version": "3.8.2.1-develop",
-      "commands": [
-        "mgcb"
-      ]
-    },
     "dotnet-mgcb-editor": {
       "version": "3.8.2.1-develop",
       "commands": [

Templates/MonoGame.Templates.CSharp/content/MonoGame.Application.iOS.CSharp/.config/dotnet-tools.json

@@ -2,12 +2,6 @@
   "version": 1,
   "isRoot": true,
   "tools": {
-    "dotnet-mgcb": {
-      "version": "3.8.2.1-develop",
-      "commands": [
-        "mgcb"
-      ]
-    },
     "dotnet-mgcb-editor": {
       "version": "3.8.2.1-develop",
       "commands": [

Templates/MonoGame.Templates.CSharp/content/MonoGame.Library.CSharp/.config/dotnet-tools.json

@@ -2,12 +2,6 @@
   "version": 1,
   "isRoot": true,
   "tools": {
-    "dotnet-mgcb": {
-      "version": "3.8.2.1-develop",
-      "commands": [
-        "mgcb"
-      ]
-    },
     "dotnet-mgcb-editor": {
       "version": "3.8.2.1-develop",
       "commands": [

Tools/MonoGame.Content.Builder.Editor/Common/PipelineController.cs

@@ -35,12 +35,17 @@ public partial class PipelineController : IController
         private static readonly string [] _mgcbSearchPaths = new []       
         {
 #if DEBUG
+#if MAC
+            Path.Combine(Path.GetDirectoryName(System.AppContext.BaseDirectory) ?? "", "../../../MonoGame.Content.Builder/Debug/mgcb.dll"),
+            Path.Combine(Path.GetDirectoryName(System.AppContext.BaseDirectory) ?? "", "../../../../../MonoGame.Content.Builder/Debug/mgcb.dll"),
+#else
             Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) ?? "", "../../../MonoGame.Content.Builder/Debug/mgcb.dll"),
             Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) ?? "", "../../../../../../MonoGame.Content.Builder/Debug/mgcb.dll"),
+#endif
 #else
 #if MAC
             Path.Combine(Path.GetDirectoryName(System.AppContext.BaseDirectory) ?? "", "../../../MonoGame.Content.Builder/Release/mgcb.dll"),
-            Path.Combine(Path.GetDirectoryName(System.AppContext.BaseDirectory) ?? "", "../../../../../../MonoGame.Content.Builder/Release/mgcb.dll"),
+            Path.Combine(Path.GetDirectoryName(System.AppContext.BaseDirectory) ?? "", "../../../../../MonoGame.Content.Builder/Release/mgcb.dll"),
 #else
             Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) ?? "", "../../../MonoGame.Content.Builder/Release/mgcb.dll"),
             Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) ?? "", "../../../../../../MonoGame.Content.Builder/Release/mgcb.dll"),
@@ -127,6 +132,7 @@ private PipelineController(IView view)
                 LoadTemplates(templatesPath);
 #endif
 
+            RestoreMGCB();
             UpdateMenu();
 
             view.UpdateRecentList(PipelineSettings.Default.ProjectHistory);
@@ -574,6 +580,22 @@ public void Clean()
             UpdateMenu();       
         }
 
+        private void RestoreMGCB()
+        {
+            var appDataPath = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
+            var version = Assembly.GetExecutingAssembly().GetName().Version.ToString();
+            var workingDirectory = Path.Combine(appDataPath, "mgcb-dotnet-tool", version);
+            if (Directory.Exists(workingDirectory))
+                return;
+            Directory.CreateDirectory(workingDirectory);
+            var dotnet = Global.Unix ? "dotnet" : "dotnet.exe";
+            if (Util.Run(dotnet, $"tool install dotnet-mgcb --version {version} --tool-path .", workingDirectory) != 0)
+            {
+                // install the latest
+                Util.Run(dotnet, $"tool install dotnet-mgcb --tool-path .", workingDirectory);
+            }
+        }
+
         private void DoBuild(string commands)
         {
             Encoding encoding;
@@ -585,9 +607,11 @@ private void DoBuild(string commands)
                 encoding = Encoding.UTF8;
             }
 
-            var mgcbCommand = "mgcb";
+
+            var appDataPath = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
+            var version = Assembly.GetExecutingAssembly().GetName().Version.ToString ();
+            var mgcbCommand = Path.Combine(appDataPath, "mgcb-dotnet-tool", version, "mgcb");
             var currentDir = Environment.CurrentDirectory;
-
             foreach (var path in _mgcbSearchPaths)
             {
                 var fullPath = Path.Combine(currentDir, path);
@@ -598,26 +622,17 @@ private void DoBuild(string commands)
                     break;
                 }
             }
+            // allow the users to override the path with an environment variable
+            // the same as the MSBuild property in the .targets
+            var mgcbUserPath = Environment.GetEnvironmentVariable("MGCBCommand");
+            if (!string.IsNullOrEmpty(mgcbUserPath) && File.Exists(mgcbUserPath))
+            {
+                mgcbCommand = mgcbUserPath;
+            }
 
             try
             {
-                // Prepare the process.
-                _buildProcess = new Process
-                {
-                    StartInfo = new ProcessStartInfo
-                    {
-                        FileName = Global.Unix ? "dotnet" : "dotnet.exe",
-                        Arguments = $"{mgcbCommand} {commands}",
-                        WorkingDirectory = Path.GetDirectoryName(_project.OriginalPath),
-                        CreateNoWindow = true,
-                        WindowStyle = ProcessWindowStyle.Hidden,
-                        UseShellExecute = false,
-                        RedirectStandardOutput = true,
-                        StandardOutputEncoding = encoding
-                    }
-                };
-                _buildProcess.OutputDataReceived += (sender, args) => View.OutputAppend(args.Data);
-
+                _buildProcess = Util.CreateProcess(mgcbCommand, commands, Path.GetDirectoryName (_project.OriginalPath), encoding, (s) => View.OutputAppend (s));
                 // Fire off the process.
                 Console.WriteLine(_buildProcess.StartInfo.FileName + " " + _buildProcess.StartInfo.Arguments);
                 Environment.CurrentDirectory = _buildProcess.StartInfo.WorkingDirectory;

Tools/MonoGame.Content.Builder.Editor/Common/Util.cs

@@ -1,6 +1,8 @@
 using System;
+using System.Diagnostics;
 using System.IO;
 using System.Runtime.InteropServices;
+using System.Text;
 
 namespace MonoGame.Tools.Pipeline
 {
@@ -46,5 +48,41 @@ public static string GetRelativePath(string filespec, string folder)
 
             return result;
         }
+
+        public static int Run(string command, string arguments, string workingDirectory)
+        {
+            var process = CreateProcess(command, arguments, workingDirectory, Encoding.UTF8, (s) => Console.WriteLine(s));
+            process.Start();
+            process.BeginOutputReadLine();
+            process.WaitForExit();
+            return process.ExitCode;
+        }
+
+        public static Process CreateProcess(string command, string arguments, string workingDirectory, Encoding encoding, Action<string> output)
+        {
+            var exe = command;
+            var args = arguments;
+            if (command.EndsWith (".dll")) {
+                // we are referencing the dll directly. We need to call dotnet to host.
+                exe = Global.Unix ? "dotnet" : "dotnet.exe";
+                args = $"{command} {arguments}";
+            }
+            var _buildProcess = new Process
+            {
+                StartInfo = new ProcessStartInfo
+                {
+                    FileName = exe,
+                    Arguments = args,
+                    WorkingDirectory = workingDirectory,
+                    CreateNoWindow = true,
+                    WindowStyle = ProcessWindowStyle.Hidden,
+                    UseShellExecute = false,
+                    RedirectStandardOutput = true,
+                    StandardOutputEncoding = encoding
+                }
+            };
+            _buildProcess.OutputDataReceived += (sender, args) => output(args.Data);
+            return _buildProcess;
+        }
     }
 }

Tools/MonoGame.Content.Builder.Task/MonoGame.Content.Builder.Task.props

@@ -3,7 +3,11 @@
   <PropertyGroup>
     <DotnetCommand Condition="'$(DotnetCommand)' == ''">dotnet</DotnetCommand>
     <EnableMGCBItems Condition="'$(EnableMGCBItems)' == ''">true</EnableMGCBItems>
+    <!-- Allow users the ability to disble tool restoration if needed -->
+    <AutoRestoreMGCBTool Condition="'$(AutoRestoreMGCBTool)' == ''">true</AutoRestoreMGCBTool>
+    <MGCBToolDirectory>$(MSBuildThisFileDirectory)dotnet-tools/</MGCBToolDirectory>
     <MGCBCommand Condition="'$(MGCBCommand)' == ''">mgcb</MGCBCommand>
+    <MonoGameVersion>3.8.1.1-develop</MonoGameVersion>
   </PropertyGroup>
 
 </Project>