Adding fixes for ghcr.io (#30)

* Adding fixes for ghcr.io * Add github test case * Some more stuff * Add documentation for ÀWS ECR and Github GHCR * Fix modes for scripts
eoftedal · Jan 16, 2024 · 4833fe2 · 4833fe2
1 parent 5ce2f00
commit 4833fe2
Show file tree

Hide file tree

Showing 7 changed files with 99 additions and 44 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -1,6 +1,6 @@
 {
     "editor.codeActionsOnSave": {
-        "source.fixAll.eslint": true
+        "source.fixAll.eslint": "explicit"
     },
     "editor.defaultFormatter": "esbenp.prettier-vscode",
     "editor.formatOnSave": true

diff --git a/README.md b/README.md
@@ -30,6 +30,17 @@ containerify --fromImage nginx:alpine --folder . --toImage frontend:latest --cus
 
 This will take the `nginx:alpine` image, and copy the files from `./dist/` into `/usr/share/nginx/html`.
 
+### Using with Github Container Registry (ghcr.io)
+
+1. Create a token from https://github.com/settings/tokens/new?scopes=write:packages or use GITHUB_TOKEN from Github Actions
+2. Example: `containerify --registry https://ghcr.io/v2/ --token "$GITHUB_TOKEN" --fromImage docker-mirror/node:alpine --toImage <some image name>:<some tag> --folder . `
+
+### Using with AWS ECR
+
+1. Create the repository in AWS from the console or through using the CLI
+2. Create token using `aws ecr get-authorization-token --output text --query 'authorizationData[].authorizationToken'`
+3. Example: `containerify  --toToken "Basic $TOKEN" --toRegistry https://<AWS ACCOUNT ID>.dkr.ecr.<AWS region for repository>.amazonaws.com/v2/ --fromImage node:alpine --toImage <name of repository>:<some tag> --folder .`
+
 ### Command line options
 
 ```

diff --git a/package.json b/package.json
@@ -8,6 +8,7 @@
 		"build": "tsc && chmod ugo+x lib/cli.js",
 		"lint": "eslint . --ext .ts --fix --ignore-path .gitignore",
 		"typecheck": "tsc --noEmit",
+		"watch": "tsc --watch",
 		"check": "npm run lint && npm run typecheck",
 		"dev": "tsc --watch",
 		"integrationTest": "cd tests/integration/ && ./test.sh",

diff --git a/src/registry.ts b/src/registry.ts
@@ -32,9 +32,14 @@ function request(
 	callback: (res: http.IncomingMessage) => void,
 ) {
 	if (allowInsecure == InsecureRegistrySupport.YES) options.rejectUnauthorized = false;
-	return (options.protocol == "https:" ? https : http).request(options, (res) => {
+	const req = (options.protocol == "https:" ? https : http).request(options, (res) => {
 		callback(res);
 	});
+	req.on("error", (e) => {
+		logger.error("ERROR: " + e, options.method, options.path);
+		throw e;
+	});
+	return req;
 }
 
 function isOk(httpStatus: number) {
@@ -56,6 +61,12 @@ function toError(res: http.IncomingMessage) {
 	return `Unexpected HTTP status ${res.statusCode} : ${res.statusMessage}`;
 }
 
+function waitForResponseEnd(res: http.IncomingMessage, cb: (data: Buffer) => void) {
+	const data: Buffer[] = [];
+	res.on("data", (d) => data.push(d));
+	res.on("end", () => cb(Buffer.concat(data)));
+}
+
 function dl(uri: string, headers: Headers, allowInsecure: InsecureRegistrySupport): Promise<string> {
 	logger.debug("dl", uri);
 	return new Promise((resolve, reject) => {
@@ -64,12 +75,7 @@ function dl(uri: string, headers: Headers, allowInsecure: InsecureRegistrySuppor
 			const { res } = result;
 			logger.debug(res.statusCode, res.statusMessage, res.headers["content-type"], res.headers["content-length"]);
 			if (!isOk(res.statusCode ?? 0)) return reject(toError(res));
-			const data: string[] = [];
-			res
-				.on("data", (chunk) => data.push(chunk.toString()))
-				.on("end", () => {
-					resolve(data.reduce((a, b) => a.concat(b)));
-				});
+			waitForResponseEnd(res, (data) => resolve(data.toString()));
 		});
 	});
 }
@@ -168,21 +174,25 @@ function headOk(
 		options.method = "HEAD";
 		request(options, allowInsecure, (res) => {
 			logger.debug(`HEAD ${url}`, res.statusCode);
-			// Not found
-			if (res.statusCode == 404) return resolve(false);
-			// OK
-			if (res.statusCode == 200) return resolve(true);
-			// Redirected
-			if (redirectCodes.includes(res.statusCode ?? 0) && res.headers.location) {
-				if (optimisticCheck) return resolve(true);
-				return resolve(headOk(res.headers.location, headers, allowInsecure, optimisticCheck, ++depth));
-			}
-			// Unauthorized
-			// Possibly related to https://gitlab.com/gitlab-org/gitlab/-/issues/23132
-			if (res.statusCode == 401) {
-				return resolve(false);
-			}
-			reject(toError(res));
+			waitForResponseEnd(res, (data) => {
+				// Not found
+				if (res.statusCode == 404) return resolve(false);
+				// OK
+				if (res.statusCode == 200) return resolve(true);
+				// Redirected
+				if (redirectCodes.includes(res.statusCode ?? 0) && res.headers.location) {
+					if (optimisticCheck) return resolve(true);
+					return resolve(headOk(res.headers.location, headers, allowInsecure, optimisticCheck, ++depth));
+				}
+				// Unauthorized
+				// Possibly related to https://gitlab.com/gitlab-org/gitlab/-/issues/23132
+				if (res.statusCode == 401) {
+					return resolve(false);
+				}
+				// Other error
+				logger.error(`HEAD ${url}`, res.statusCode, res.statusMessage, data.toString());
+				reject(toError(res));
+			});
 		}).end();
 	});
 }
@@ -193,6 +203,7 @@ function uploadContent(
 	fileConfig: PartialManifestConfig,
 	allowInsecure: InsecureRegistrySupport,
 	auth: string,
+	contentType: string,
 ): Promise<void> {
 	return new Promise((resolve, reject) => {
 		logger.debug("Uploading: ", file);
@@ -203,32 +214,36 @@ function uploadContent(
 		options.headers = {
 			authorization: auth,
 			"content-length": fileConfig.size,
-			"content-type": fileConfig.mediaType,
+			"content-type": contentType,
 		};
-		logger.debug("POST", url);
+		logger.debug(options.method, url);
 		const req = request(options, allowInsecure, (res) => {
 			logger.debug(res.statusCode, res.statusMessage, res.headers["content-type"], res.headers["content-length"]);
 			if ([200, 201, 202, 203].includes(res.statusCode ?? 0)) {
 				resolve();
 			} else {
-				const data: string[] = [];
-				res.on("data", (d) => data.push(d.toString()));
-				res.on("end", () => {
-					reject(`Error uploading to ${uploadUrl}. Got ${res.statusCode} ${res.statusMessage}:\n${data.join("")}`);
+				waitForResponseEnd(res, (data) => {
+					reject(`Error uploading to ${uploadUrl}. Got ${res.statusCode} ${res.statusMessage}:\n${data.toString()}`);
 				});
 			}
 		});
 		fss.createReadStream(file).pipe(req);
 	});
 }
 
+function prepareToken(token: string) {
+	if (token.startsWith("Basic ")) return token;
+	if (token.startsWith("ghp_")) return "Bearer " + Buffer.from(token).toString("base64");
+	return "Bearer " + token;
+}
+
 export function createRegistry(
 	registryBaseUrl: string,
 	token: string,
 	allowInsecure: InsecureRegistrySupport,
 	optimisticToRegistryCheck = false,
 ) {
-	const auth = token.startsWith("Basic ") ? token : "Bearer " + token;
+	const auth = prepareToken(token);
 
 	async function exists(image: Image, layer: Layer) {
 		const url = `${registryBaseUrl}${image.path}/blobs/${layer.digest}`;
@@ -238,7 +253,7 @@ export function createRegistry(
 	async function uploadLayerContent(uploadUrl: string, layer: Layer, dir: string) {
 		logger.info(layer.digest);
 		const file = path.join(dir, getHash(layer.digest) + getLayerTypeFileEnding(layer));
-		await uploadContent(uploadUrl, file, layer, allowInsecure, auth);
+		await uploadContent(uploadUrl, file, layer, allowInsecure, auth, "application/octet-stream");
 	}
 
 	async function getUploadUrl(image: Image): Promise<string> {
@@ -251,17 +266,22 @@ export function createRegistry(
 				logger.debug("POST", `${url}`, res.statusCode);
 				if (res.statusCode == 202) {
 					const { location } = res.headers;
-					if (location) resolve(location);
+					if (location) {
+						if (location.startsWith("http")) {
+							resolve(location);
+						} else {
+							const regURL = URL.parse(registryBaseUrl);
+
+							resolve(`${regURL.protocol}//${regURL.hostname}${regURL.port ? ":" + regURL.port : ""}${location}`);
+						}
+					}
 					reject("Missing location for 202");
 				} else {
-					const data: string[] = [];
-					res
-						.on("data", (c) => data.push(c.toString()))
-						.on("end", () => {
-							reject(
-								`Error getting upload URL from ${url}. Got ${res.statusCode} ${res.statusMessage}:\n${data.join("")}`,
-							);
-						});
+					waitForResponseEnd(res, (data) => {
+						reject(
+							`Error getting upload URL from ${url}. Got ${res.statusCode} ${res.statusMessage}:\n${data.toString()}`,
+						);
+					});
 				}
 			}).end();
 		});
@@ -285,7 +305,6 @@ export function createRegistry(
 			const adequateManifest = pickManifest(availableManifests, preferredPlatform);
 			return dlManifest({ ...image, tag: adequateManifest.digest }, preferredPlatform, allowInsecure);
 		}
-
 		return res as Manifest;
 	}
 
@@ -357,7 +376,6 @@ export function createRegistry(
 		const image = parseImage(imageStr);
 		const manifestFile = path.join(folder, "manifest.json");
 		const manifest = (await fse.readJson(manifestFile)) as Manifest;
-
 		logger.info("Checking layer status...");
 		const layerStatus = await Promise.all(
 			manifest.layers.map(async (l) => {
@@ -369,7 +387,6 @@ export function createRegistry(
 			"Needs upload:",
 			layersForUpload.map((l) => l.layer.digest),
 		);
-
 		logger.info("Uploading layers...");
 		await Promise.all(
 			layersForUpload.map(async (l) => {
@@ -381,7 +398,7 @@ export function createRegistry(
 		logger.info("Uploading config...");
 		const configUploadUrl = await getUploadUrl(image);
 		const configFile = path.join(folder, getHash(manifest.config.digest) + ".json");
-		await uploadContent(configUploadUrl, configFile, manifest.config, allowInsecure, auth);
+		await uploadContent(configUploadUrl, configFile, manifest.config, allowInsecure, auth, "application/octet-stream");
 
 		logger.info("Uploading manifest...");
 		const manifestSize = await fileutil.sizeOf(manifestFile);
@@ -391,6 +408,7 @@ export function createRegistry(
 			{ mediaType: manifest.mediaType, size: manifestSize },
 			allowInsecure,
 			auth,
+			manifest.mediaType,
 		);
 	}
 

diff --git a/tests/external-registries/aws-ecr-test.sh b/tests/external-registries/aws-ecr-test.sh
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+TOKEN=$(aws ecr get-authorization-token --output text --query 'authorizationData[].authorizationToken')
+
+ACCOUNT=$(aws sts get-caller-identity --output text --query 'Account')
+
+REGION=$(aws configure get region)
+
+echo $TOKEN
+
+printf "* Running containerify to pull from and push result to AWS ECR ...\n"
+../../lib/cli.js --verbose --fromImage node:alpine --toRegistry https://$ACCOUNT.dkr.ecr.$REGION.amazonaws.com/v2/ --toImage containerify-test:latest --folder . --customContent customContent --setTimeStamp "2024-01-15T20:00:00.000Z" --toToken "Basic $TOKEN"
diff --git a/tests/external-registries/customContent/test.txt b/tests/external-registries/customContent/test.txt
@@ -0,0 +1,3 @@
+Hello world
+
+This is an integration test
diff --git a/tests/external-registries/github-ghcr-test.sh b/tests/external-registries/github-ghcr-test.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e
+if [[ -z "$GITHUB_TOKEN" ]]; then
+    echo "ERROR: GITHUB_TOKEN not set (see https://github.com/settings/tokens/new?scopes=write:packages for running locally)"
+    exit 1
+fi
+
+printf "* Running containerify to pull from and push result to gchr.io ...\n"
+../../lib/cli.js --verbose --fromImage docker-mirror/node:alpine --registry https://ghcr.io/v2/ --toImage eoftedal/containerify-integrationtest:latest --folder . --customContent customContent --setTimeStamp "2024-01-15T20:00:00.000Z" --token "$GITHUB_TOKEN"