Skip to content

Commit

Permalink
Install dotnet tools locally in the Nuget Packages
Browse files Browse the repository at this point in the history
In commit 7a398b0 we added two new tools for compressing textures, `crunch` and `basisu`.
The problem there is that the management of the `.config/dotnet-tool.json` file
by the users was becoming an issue.  We needed a more automatic way to install
the required tooling.

The problem is using the standard `dotnet tool install` calls requires
a `.config/dotnet-tool.json` file to be present in either the current directory or
a directory that is in the path ABOVE the current one. You would use the
`--create-manifest-if-needed` flag to create the manifest, but that will still
leave the users having to manage and upgrade the .json file every time we do a release.

So lets get the pipeline to install the tooling itself.
The `dotnet tool install` command has an additional argument`--tool-path` this allows
us to say where we want the tool installed. Once that has happened we get a native
binary launcher in `--tool-path` which allows us to launch the app directly without
using the `dotnet` executable. So what this allows us to do is install the tooling
locally in the directory that the content pipeline is installed. This will usually
be the global `.nuget/package` directory.

We need to keep an eye on the `DOTNET_ROOT` environment variable when installing the
tooling, just in case a user (or CI) wants to use a custom dotnet install.

The one downside is users will no longer be able to run `dotnet mgcb` directly in
their project directory unless they install the tooling manually.

The long term plan is to bundle all these tools into a single native library
which can be called directly by the content pipeline.
  • Loading branch information
dellis1972 committed Oct 11, 2024
1 parent 97c029b commit 19d250b
Show file tree
Hide file tree
Showing 16 changed files with 174 additions and 72 deletions.
6 changes: 0 additions & 6 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,6 @@ jobs:
- name: Test
run: dotnet test Tools/MonoGame.Tools.Tests/MonoGame.Tools.Tests.csproj --blame-hang-timeout 5m -c Release --filter="TestCategory!=Audio"
env:
DOTNET_ROOT: ${{github.workspace}}/dotnet64
MGFXC_WINE_PATH: /Users/runner/.winemonogame
CI: true
if: runner.os == 'macOS'
Expand Down Expand Up @@ -231,11 +230,6 @@ jobs:
path: tests-windowsdx
if: runner.os == 'Windows'

- name: Install Tools
run: |
dotnet tool install --create-manifest-if-needed mgcb-basisu
dotnet tool install --create-manifest-if-needed mgcb-crunch
- name: Run Tools Tests
run: dotnet test tests-tools/MonoGame.Tools.Tests.dll --blame-hang-timeout 1m --filter="TestCategory!=Effects"
env:
Expand Down
2 changes: 1 addition & 1 deletion .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
"type": "coreclr",
"request": "launch",
"preLaunchTask": "mgcb-editor-mac",
"program": "${workspaceFolder}/Artifacts/MonoGame.Content.Builder.Editor/Mac/Debug/MGCB Editor.app/Contents/MacOS/mgcb-editor-mac",
"program": "${workspaceFolder}/Artifacts/MonoGame.Content.Builder.Editor/Mac/Debug/mgcb-editor-mac.app/Contents/MacOS/mgcb-editor-mac",
"args": [],
"cwd": "${workspaceFolder}/Artifacts/MonoGame.Content.Builder.Editor/Mac/Debug",
"console": "internalConsole",
Expand Down
46 changes: 44 additions & 2 deletions MonoGame.Framework.Content.Pipeline/ExternalTool.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
using System;
using System.Diagnostics;
using System.IO;
using System.Net;
using System.Reflection;
using System.Threading;
using MonoGame.Framework.Utilities;

Expand All @@ -17,6 +19,9 @@ namespace Microsoft.Xna.Framework.Content.Pipeline
/// </summary>
internal class ExternalTool
{
public static string Crunch = "mgcb-crunch";
public static string BasisU = "mgcb-basisu";

public static int Run(string command, string arguments)
{
string stdout, stderr;
Expand All @@ -27,13 +32,45 @@ public static int Run(string command, string arguments)
return result;
}

public static void RestoreDotnetTool(string command, string toolName)
{
string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory);
if (CurrentPlatform.OS == OS.Linux)
path= Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "linux");
if (CurrentPlatform.OS == OS.MacOSX)
path= Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "osx");
if (Directory.Exists (Path.Combine(path, toolName)))
return;
Directory.CreateDirectory(path);
var exe = CurrentPlatform.OS == OS.Windows ? "dotnet.exe" : "dotnet";
var dotnetRoot = Environment.GetEnvironmentVariable("DOTNET_ROOT");
if (!string.IsNullOrEmpty(dotnetRoot))
{
exe = Path.Combine(dotnetRoot, exe);
}
var version = Assembly.GetExecutingAssembly().GetName().Version.ToString();
if (Run(exe, $"tool {command} {toolName} --version {version} --tool-path .", out string _, out string _, workingDirectory: path) != 0)
{
// install the latest
Run(exe, $"tool {command} {toolName} --tool-path .", out _, out _, workingDirectory: path);
}
}

public static void RestoreDotnetTools()
{
RestoreDotnetTool("install", Crunch);
RestoreDotnetTool("install", BasisU);
}

/// <summary>
/// Run a dotnet tool. The tool should be installed in a .config/dotnet-tools.json file somewhere in the project lineage.
/// </summary>
public static int RunDotnetTool(string toolName, string args, out string stdOut, out string stdErr, string stdIn=null, string workingDirectory=null)
{
var finalizedArgs = toolName + " " + args;
return ExternalTool.Run("dotnet", finalizedArgs, out stdOut, out stdErr, stdIn, workingDirectory);
RestoreDotnetTools();
var exe = FindCommand (toolName);
var finalizedArgs = args;
return ExternalTool.Run(exe, finalizedArgs, out stdOut, out stdErr, stdIn, workingDirectory);
}

public static int Run(string command, string arguments, out string stdout, out string stderr, string stdin = null, string workingDirectory=null)
Expand Down Expand Up @@ -68,6 +105,11 @@ public static int Run(string command, string arguments, out string stdout, out s
if (!string.IsNullOrEmpty(workingDirectory))
processInfo.WorkingDirectory = workingDirectory;

var dotnetRoot = Environment.GetEnvironmentVariable ("DOTNET_ROOT");
if (!string.IsNullOrEmpty(dotnetRoot)) {
processInfo.EnvironmentVariables["DOTNET_ROOT"] = dotnetRoot;
}

EnsureExecutable(fullPath);

using (var process = new Process())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ internal static class BasisU
/// <returns>The exit code for the basisu process. </returns>
public static int Run(string args, out string stdOut, out string stdErr, string stdIn=null, string workingDirectory=null)
{
return ExternalTool.RunDotnetTool("mgcb-basisu", args, out stdOut, out stdErr, stdIn, workingDirectory);
return ExternalTool.RunDotnetTool(ExternalTool.BasisU, args, out stdOut, out stdErr, stdIn, workingDirectory);
}

/// <summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ internal static class Crunch
/// <returns>The exit code for the basisu process. </returns>
private static int Run(string args, out string stdOut, out string stdErr)
{
return ExternalTool.RunDotnetTool("mgcb-crunch", args, out stdOut, out stdErr);
return ExternalTool.RunDotnetTool(ExternalTool.Crunch, args, out stdOut, out stdErr);
}

/// <summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,6 @@
"version": 1,
"isRoot": true,
"tools": {
"dotnet-mgcb": {
"version": "3.8.2.1-develop",
"commands": [
"mgcb"
]
},
"dotnet-mgcb-editor": {
"version": "3.8.2.1-develop",
"commands": [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,6 @@
"version": 1,
"isRoot": true,
"tools": {
"dotnet-mgcb": {
"version": "3.8.2.1-develop",
"commands": [
"mgcb"
]
},
"dotnet-mgcb-editor": {
"version": "3.8.2.1-develop",
"commands": [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,6 @@
"version": 1,
"isRoot": true,
"tools": {
"dotnet-mgcb": {
"version": "3.8.2.1-develop",
"commands": [
"mgcb"
]
},
"dotnet-mgcb-editor": {
"version": "3.8.2.1-develop",
"commands": [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,6 @@
"version": 1,
"isRoot": true,
"tools": {
"dotnet-mgcb": {
"version": "3.8.2.1-develop",
"commands": [
"mgcb"
]
},
"dotnet-mgcb-editor": {
"version": "3.8.2.1-develop",
"commands": [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,6 @@
"version": 1,
"isRoot": true,
"tools": {
"dotnet-mgcb": {
"version": "3.8.2.1-develop",
"commands": [
"mgcb"
]
},
"dotnet-mgcb-editor": {
"version": "3.8.2.1-develop",
"commands": [
Expand Down
53 changes: 33 additions & 20 deletions Tools/MonoGame.Content.Builder.Editor/Common/PipelineController.cs
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,17 @@ public partial class PipelineController : IController
private static readonly string [] _mgcbSearchPaths = new []
{
#if DEBUG
#if MAC
Path.Combine(Path.GetDirectoryName(System.AppContext.BaseDirectory) ?? "", "../../../MonoGame.Content.Builder/Debug/mgcb.dll"),
Path.Combine(Path.GetDirectoryName(System.AppContext.BaseDirectory) ?? "", "../../../../../MonoGame.Content.Builder/Debug/mgcb.dll"),
#else
Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) ?? "", "../../../MonoGame.Content.Builder/Debug/mgcb.dll"),
Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) ?? "", "../../../../../../MonoGame.Content.Builder/Debug/mgcb.dll"),
#endif
#else
#if MAC
Path.Combine(Path.GetDirectoryName(System.AppContext.BaseDirectory) ?? "", "../../../MonoGame.Content.Builder/Release/mgcb.dll"),
Path.Combine(Path.GetDirectoryName(System.AppContext.BaseDirectory) ?? "", "../../../../../../MonoGame.Content.Builder/Release/mgcb.dll"),
Path.Combine(Path.GetDirectoryName(System.AppContext.BaseDirectory) ?? "", "../../../../../MonoGame.Content.Builder/Release/mgcb.dll"),
#else
Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) ?? "", "../../../MonoGame.Content.Builder/Release/mgcb.dll"),
Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) ?? "", "../../../../../../MonoGame.Content.Builder/Release/mgcb.dll"),
Expand Down Expand Up @@ -127,6 +132,7 @@ private PipelineController(IView view)
LoadTemplates(templatesPath);
#endif

RestoreMGCB();
UpdateMenu();

view.UpdateRecentList(PipelineSettings.Default.ProjectHistory);
Expand Down Expand Up @@ -574,6 +580,20 @@ public void Clean()
UpdateMenu();
}

private void RestoreMGCB()
{
var appDataPath = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
var version = Assembly.GetExecutingAssembly().GetName().Version.ToString();
var workingDirectory = Path.Combine(appDataPath, "mgcb-dotnet-tool", version);
Directory.CreateDirectory(workingDirectory);
var dotnet = Global.Unix ? "dotnet" : "dotnet.exe";
if (Util.Run(dotnet, $"tool install dotnet-mgcb --version {version} --tool-path .", workingDirectory) != 0)
{
// install the latest
Util.Run(dotnet, $"tool install dotnet-mgcb --tool-path .", workingDirectory);
}
}

private void DoBuild(string commands)
{
Encoding encoding;
Expand All @@ -585,9 +605,11 @@ private void DoBuild(string commands)
encoding = Encoding.UTF8;
}

var mgcbCommand = "mgcb";

var appDataPath = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
var version = Assembly.GetExecutingAssembly().GetName().Version.ToString ();
var mgcbCommand = Path.Combine(appDataPath, "mgcb-dotnet-tool", version, "mgcb");
var currentDir = Environment.CurrentDirectory;

foreach (var path in _mgcbSearchPaths)
{
var fullPath = Path.Combine(currentDir, path);
Expand All @@ -598,26 +620,17 @@ private void DoBuild(string commands)
break;
}
}
// allow the users to override the path with an environment variable
// the same as the MSBuild property in the .targets
var mgcbUserPath = Environment.GetEnvironmentVariable("MGCBCommand");
if (!string.IsNullOrEmpty(mgcbUserPath) && File.Exists(mgcbUserPath))
{
mgcbCommand = mgcbUserPath;
}

try
{
// Prepare the process.
_buildProcess = new Process
{
StartInfo = new ProcessStartInfo
{
FileName = Global.Unix ? "dotnet" : "dotnet.exe",
Arguments = $"{mgcbCommand} {commands}",
WorkingDirectory = Path.GetDirectoryName(_project.OriginalPath),
CreateNoWindow = true,
WindowStyle = ProcessWindowStyle.Hidden,
UseShellExecute = false,
RedirectStandardOutput = true,
StandardOutputEncoding = encoding
}
};
_buildProcess.OutputDataReceived += (sender, args) => View.OutputAppend(args.Data);

_buildProcess = Util.CreateProcess(mgcbCommand, commands, Path.GetDirectoryName (_project.OriginalPath), encoding, (s) => View.OutputAppend (s));
// Fire off the process.
Console.WriteLine(_buildProcess.StartInfo.FileName + " " + _buildProcess.StartInfo.Arguments);
Environment.CurrentDirectory = _buildProcess.StartInfo.WorkingDirectory;
Expand Down
38 changes: 38 additions & 0 deletions Tools/MonoGame.Content.Builder.Editor/Common/Util.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
using System;
using System.Diagnostics;
using System.IO;
using System.Runtime.InteropServices;
using System.Text;

namespace MonoGame.Tools.Pipeline
{
Expand Down Expand Up @@ -46,5 +48,41 @@ public static string GetRelativePath(string filespec, string folder)

return result;
}

public static int Run(string command, string arguments, string workingDirectory)
{
var process = CreateProcess(command, arguments, workingDirectory, Encoding.UTF8, (s) => Console.WriteLine(s));
process.Start();
process.BeginOutputReadLine();
process.WaitForExit();
return process.ExitCode;
}

public static Process CreateProcess(string command, string arguments, string workingDirectory, Encoding encoding, Action<string> output)
{
var exe = command;
var args = arguments;
if (command.EndsWith (".dll")) {
// we are referencing the dll directly. We need to call dotnet to host.
exe = Global.Unix ? "dotnet" : "dotnet.exe";
args = $"{command} {arguments}";
}
var _buildProcess = new Process
{
StartInfo = new ProcessStartInfo
{
FileName = exe,
Arguments = args,
WorkingDirectory = workingDirectory,
CreateNoWindow = true,
WindowStyle = ProcessWindowStyle.Hidden,
UseShellExecute = false,
RedirectStandardOutput = true,
StandardOutputEncoding = encoding
}
};
_buildProcess.OutputDataReceived += (sender, args) => output(args.Data);
return _buildProcess;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
<PropertyGroup>
<DotnetCommand Condition="'$(DotnetCommand)' == ''">dotnet</DotnetCommand>
<EnableMGCBItems Condition="'$(EnableMGCBItems)' == ''">true</EnableMGCBItems>
<MGCBToolDirectory>$(MSBuildThisFileDirectory)dotnet-tools/</MGCBToolDirectory>
<MGCBCommand Condition="'$(MGCBCommand)' == ''">mgcb</MGCBCommand>
</PropertyGroup>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,12 @@

</Target>

<!-- Restore the dotnet-mgcb tool to a known location. -->
<Target Name="RestoreContentCompiler" Condition="!Exists ('$(MGCBToolDirectory)$(MGCBCommand)')">
<MakeDir Directories="$(MGCBToolDirectory)"/>
<Exec Command="&quot;$(DotnetCommand)&quot; tool install dotnet-mgcb --tool-path ." WorkingDirectory="$(MGCBToolDirectory)" />
</Target>

<!--
=====================
PrepareContentBuilder
Expand Down Expand Up @@ -134,16 +140,18 @@
- ExtraContent: built content files
- ContentDir: the relative path of the embedded folder to contain the content files
-->
<Target Name="RunContentBuilder" DependsOnTargets="PrepareContentBuilder">
<Target Name="RunContentBuilder" DependsOnTargets="RestoreContentCompiler;PrepareContentBuilder">

<!-- Remove this line if they make dotnet tool restore part of dotnet restore build -->
<!-- https://github.com/dotnet/sdk/issues/4241 -->
<Exec Command="&quot;$(DotnetCommand)&quot; tool restore" />
<PropertyGroup>
<_Command Condition="Exists ('$(MGCBToolDirectory)\$(MGCBCommand)')">&quot;$(MGCBToolDirectory)\$(MGCBCommand)&quot;</_Command>
<!-- Fallback to old behaviour this allows people to override $(MGCBCommand) with the mgcb.dll -->
<_Command Condition=" '$(_Command)' == '' ">&quot;$(DotnetCommand)&quot; &quot;$(MGCBCommand)&quot;</_Command>
</PropertyGroup>

<!-- Execute MGCB from the project directory so we use the correct manifest. -->
<Exec
Condition="'%(ContentReference.FullPath)' != ''"
Command="&quot;$(DotnetCommand)&quot; &quot;$(MGCBCommand)&quot; $(MonoGameMGCBAdditionalArguments) /@:&quot;%(ContentReference.FullPath)&quot; /platform:$(MonoGamePlatform) /outputDir:&quot;%(ContentReference.ContentOutputDir)&quot; /intermediateDir:&quot;%(ContentReference.ContentIntermediateOutputDir)&quot; /workingDir:&quot;%(ContentReference.FullDir)&quot;"
Command="$(_Command) $(MonoGameMGCBAdditionalArguments) /@:&quot;%(ContentReference.FullPath)&quot; /platform:$(MonoGamePlatform) /outputDir:&quot;%(ContentReference.ContentOutputDir)&quot; /intermediateDir:&quot;%(ContentReference.ContentIntermediateOutputDir)&quot; /workingDir:&quot;%(ContentReference.FullDir)&quot;"
WorkingDirectory="$(MSBuildProjectDirectory)" />

<ItemGroup>
Expand Down
6 changes: 0 additions & 6 deletions build/BuildToolsTasks/BuildContentPipelineTask.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,5 @@ public override void Run(BuildContext context)
{
var builderPath = context.GetProjectPath(ProjectType.ContentPipeline);
context.DotNetPack(builderPath, context.DotNetPackSettings);

// ensure that the local development has the required dotnet tools.
// this won't actually include the tool manifest in a final build,
// but it will setup a local developer's project
context.DotNetTool(builderPath, "tool install --create-manifest-if-needed mgcb-basisu");
context.DotNetTool(builderPath, "tool install --create-manifest-if-needed mgcb-crunch");
}
}
Loading

0 comments on commit 19d250b

Please sign in to comment.