diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 8b8ec136..b708ad45 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -1,4 +1,5 @@
 {
+	"remoteUser": "node",
     "build": { "dockerfile": "../Dockerfile" },
 	"forwardPorts": [
 		4000,
diff --git a/.github/workflows/generator-test.yml b/.github/workflows/generator-test.yml
new file mode 100644
index 00000000..d900c4f0
--- /dev/null
+++ b/.github/workflows/generator-test.yml
@@ -0,0 +1,17 @@
+name: Generator Test
+on: 
+  - workflow_dispatch
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: devcontainers/ci@v0.3
+        with:
+          runCmd: |
+            cd cli
+            make run test
+      - uses: actions/upload-artifact@v4
+        with:
+          path: 
+            cli/tests/1.*/**
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
index ff9fa4f7..0796163b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,10 +1,14 @@
 FROM node:18-bookworm
 
 ENV DEBIAN_FRONTEND=noninteractive
-ENV PATH="${PATH}:$HOME/gems/bin"
-ENV GEM_HOME="$HOME/gems"
 
 RUN apt update && apt install -y ruby-full build-essential zlib1g-dev
+
+USER node
+ENV HOME="/home/node"
+
+ENV PATH="${PATH}:$HOME/gems/bin"
+ENV GEM_HOME="$HOME/gems"
 RUN gem install jekyll bundler
 
 ENV DENO_INSTALL="$HOME/.deno"
diff --git a/cli/.gitignore b/cli/.gitignore
index dfc3cf83..86e8a5dd 100644
--- a/cli/.gitignore
+++ b/cli/.gitignore
@@ -7,4 +7,5 @@
 !Makefile
 !bundle.ts
 !fontgen.ts
+!test.ts
 !utils.ts
\ No newline at end of file
diff --git a/cli/Makefile b/cli/Makefile
index 7208da12..29b71620 100644
--- a/cli/Makefile
+++ b/cli/Makefile
@@ -13,6 +13,9 @@ check:
 fontgen:
 	deno run --allow-read --allow-write fontgen.ts
 
+test: fontgen check
+	deno test --allow-env --allow-read --allow-net --allow-write --allow-run test.ts
+
 build: fontgen check
 	deno run --allow-env --allow-read --allow-net --allow-write --allow-run bundle.ts
 
diff --git a/cli/commands/init.ts b/cli/commands/init.ts
index b49e7ad7..e4909098 100644
--- a/cli/commands/init.ts
+++ b/cli/commands/init.ts
@@ -7,6 +7,7 @@ import {
   Input,
   Select,
 } from "https://deno.land/x/cliffy@v0.25.7/prompt/mod.ts";
+import { parse as parseXml } from "https://deno.land/x/xml@2.1.1/mod.ts";
 import * as path from "https://deno.land/std@0.177.1/path/mod.ts";
 import { colors } from "https://deno.land/x/cliffy@v0.25.7/ansi/mod.ts";
 import * as utils from "../utils.ts";
@@ -81,27 +82,21 @@ export function initCommand() {
     });
 }
 
-async function generate(
-  cli: CliOptions,
-  outputDirName: string | undefined,
-) {
-  const outputDir = await getAndPrepareOutputDir(outputDirName);
+// Set the XML parser as we do not have DomParser here.
+generator.setXmlVersionParser((xml) => {
+  const document = parseXml(xml) as any;
+  return document.metadata.versioning.versions.version;
+});
 
-  const isTargetEmpty = await utils.isDirEmpty(outputDir);
-  if (!isTargetEmpty) {
-    fatalError("The target directory must be empty");
-  }
+const fontLoader = pureimage.registerFont("", generator.ICON_FONT);
+fontLoader.font = opentype.parse(decodeBase64(fontData).buffer);
+fontLoader.loaded = true;
 
-  const config =
-    await (cli.defaultOptions
-      ? defaultOptions(path.basename(outputDir))
-      : promptUser(path.basename(outputDir), cli));
-
-  const fontLoader = pureimage.registerFont("", generator.ICON_FONT);
-  fontLoader.font = opentype.parse(decodeBase64(fontData).buffer);
-  fontLoader.loaded = true;
-
-  const options: generator.Options = {
+export function getGeneratorOptions(
+  outputDir: string,
+  config: generator.Configuration,
+): generator.Options {
+  return {
     config,
     writer: {
       write: async (contentPath, content, options) => {
@@ -142,10 +137,27 @@ async function generate(
       },
     },
   };
+}
+
+async function generate(
+  cli: CliOptions,
+  outputDirName: string | undefined,
+) {
+  const outputDir = await getAndPrepareOutputDir(outputDirName);
+
+  const isTargetEmpty = await utils.isDirEmpty(outputDir);
+  if (!isTargetEmpty) {
+    fatalError("The target directory must be empty");
+  }
+
+  const config =
+    await (cli.defaultOptions
+      ? defaultOptions(path.basename(outputDir))
+      : promptUser(path.basename(outputDir), cli));
 
   console.log(progress("Generating mod template..."));
 
-  await generator.generateTemplate(options);
+  await generator.generateTemplate(getGeneratorOptions(outputDir, config));
   console.log(success("Done!"));
 }
 
diff --git a/cli/main.ts b/cli/main.ts
index a717bea9..616788c3 100755
--- a/cli/main.ts
+++ b/cli/main.ts
@@ -2,7 +2,6 @@
 
 // @deno-types="../scripts/dist/fabric-template-generator.d.ts"
 import * as generator from "../scripts/dist/fabric-template-generator.js";
-import { parse as parseXml } from "https://deno.land/x/xml@2.1.1/mod.ts";
 import {
   Command,
   CompletionsCommand,
@@ -14,12 +13,6 @@ import { upgradeCommand } from "./commands/upgrade.ts";
 declare let __VERSION__: string;
 const VERSION = typeof __VERSION__ !== "undefined" ? __VERSION__ : "dev";
 
-// Set the XML parser as we do not have DomParser here.
-generator.setXmlVersionParser((xml) => {
-  const document = parseXml(xml) as any;
-  return document.metadata.versioning.versions.version;
-});
-
 if (import.meta.main) {
   const cmd = new Command()
     .name("Fabric CLI tools")
diff --git a/cli/test.ts b/cli/test.ts
new file mode 100644
index 00000000..92d56cff
--- /dev/null
+++ b/cli/test.ts
@@ -0,0 +1,80 @@
+// @deno-types="../scripts/dist/fabric-template-generator.d.ts"
+import {
+  generateTemplate,
+  getTemplateGameVersions,
+} from "../scripts/dist/fabric-template-generator.js";
+import { getGeneratorOptions } from "./commands/init.ts";
+import { assert } from "https://deno.land/std@0.221.0/assert/mod.ts";
+import * as fs from "https://deno.land/std@0.221.0/fs/mod.ts";
+
+const rootDir = "./tests";
+if (await fs.exists(rootDir)) {
+  await Deno.remove(rootDir, { recursive: true });
+}
+await Deno.mkdir(rootDir);
+
+const cwd = await Deno.realPath(Deno.cwd());
+const inDir = `${rootDir}/_in`;
+const runDir = `${rootDir}/_run`;
+await Deno.mkdir(runDir);
+
+const minecraftVersions = await getTemplateGameVersions();
+
+for (const { version } of minecraftVersions) {
+  for (const mapping of ["yarn", "mojmap"]) {
+    for (const language of ["java", "kotlin"]) {
+      const testId = `${version}_${mapping}_${language}`;
+      const outDir = `${rootDir}/${testId}`;
+
+      Deno.test(testId, async () => {
+        let success = false;
+
+        // try rebuilding if it fail, usual gradle stuff
+        for (let i = 0; i < 3; i++) {
+          const options = getGeneratorOptions(inDir, {
+            modid: "test",
+            projectName: "test",
+            packageName: "net.fabricmc.generator.test",
+            dataGeneration: false,
+            splitSources: true,
+            uniqueModIcon: true,
+
+            minecraftVersion: version,
+            mojmap: mapping === "mojmap",
+            useKotlin: language === "kotlin",
+          });
+
+          await generateTemplate(options);
+
+          // build in the same directory for all test
+          // to make it use only one daemon and build cache
+          for await (const { name } of Deno.readDir(inDir)) {
+            await fs.copy(`${inDir}/${name}`, `${runDir}/${name}`);
+          }
+
+          Deno.chdir(runDir);
+          const gradle = new Deno.Command("./gradlew", {
+            args: ["build"],
+          }).spawn();
+          Deno.chdir(cwd);
+
+          const output = await gradle.output();
+
+          for await (const { name } of Deno.readDir(inDir)) {
+            await Deno.remove(`${runDir}/${name}`, { recursive: true });
+          }
+
+          await Deno.remove(inDir, { recursive: true });
+
+          if (output.success) {
+            await fs.move(`${runDir}/build/libs`, `${outDir}`);
+            success = true;
+            break;
+          }
+        }
+
+        assert(success);
+      });
+    }
+  }
+}