From dfe55cd332fb34824d7e352481a965b434649b75 Mon Sep 17 00:00:00 2001 From: Erlend Oftedal Date: Mon, 15 Jan 2024 22:40:20 +0100 Subject: [PATCH 1/6] Adding fixes for ghcr.io --- .vscode/settings.json | 2 +- package.json | 1 + src/registry.ts | 27 +++++++++++++++++++++------ 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 12a8a85..1e2721b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,6 @@ { "editor.codeActionsOnSave": { - "source.fixAll.eslint": true + "source.fixAll.eslint": "explicit" }, "editor.defaultFormatter": "esbenp.prettier-vscode", "editor.formatOnSave": true diff --git a/package.json b/package.json index 95b4974..0f8fb7d 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "build": "tsc && chmod ugo+x lib/cli.js", "lint": "eslint . --ext .ts --fix --ignore-path .gitignore", "typecheck": "tsc --noEmit", + "watch": "tsc --watch", "check": "npm run lint && npm run typecheck", "dev": "tsc --watch", "integrationTest": "cd tests/integration/ && ./test.sh", diff --git a/src/registry.ts b/src/registry.ts index ce9c474..3d70195 100644 --- a/src/registry.ts +++ b/src/registry.ts @@ -193,6 +193,7 @@ function uploadContent( fileConfig: PartialManifestConfig, allowInsecure: InsecureRegistrySupport, auth: string, + contentType: string, ): Promise { return new Promise((resolve, reject) => { logger.debug("Uploading: ", file); @@ -203,7 +204,7 @@ function uploadContent( options.headers = { authorization: auth, "content-length": fileConfig.size, - "content-type": fileConfig.mediaType, + "content-type": contentType, }; logger.debug("POST", url); const req = request(options, allowInsecure, (res) => { @@ -222,13 +223,19 @@ function uploadContent( }); } +function prepareToken(token: string) { + if (token.startsWith("Basic ")) return token; + if (token.startsWith("ghp_")) return "Bearer " + Buffer.from(token).toString("base64"); + return "Bearer " + token; +} + export function createRegistry( registryBaseUrl: string, token: string, allowInsecure: InsecureRegistrySupport, optimisticToRegistryCheck = false, ) { - const auth = token.startsWith("Basic ") ? token : "Bearer " + token; + const auth = prepareToken(token); async function exists(image: Image, layer: Layer) { const url = `${registryBaseUrl}${image.path}/blobs/${layer.digest}`; @@ -238,7 +245,7 @@ export function createRegistry( async function uploadLayerContent(uploadUrl: string, layer: Layer, dir: string) { logger.info(layer.digest); const file = path.join(dir, getHash(layer.digest) + getLayerTypeFileEnding(layer)); - await uploadContent(uploadUrl, file, layer, allowInsecure, auth); + await uploadContent(uploadUrl, file, layer, allowInsecure, auth, "application/octet-stream"); } async function getUploadUrl(image: Image): Promise { @@ -251,7 +258,15 @@ export function createRegistry( logger.debug("POST", `${url}`, res.statusCode); if (res.statusCode == 202) { const { location } = res.headers; - if (location) resolve(location); + if (location) { + if (location.startsWith("http")) { + resolve(location); + } else { + const regURL = URL.parse(registryBaseUrl); + + resolve(`${regURL.protocol}//${regURL.hostname}${regURL.port ? ":" + regURL.port : ""}${location}`); + } + } reject("Missing location for 202"); } else { const data: string[] = []; @@ -285,7 +300,6 @@ export function createRegistry( const adequateManifest = pickManifest(availableManifests, preferredPlatform); return dlManifest({ ...image, tag: adequateManifest.digest }, preferredPlatform, allowInsecure); } - return res as Manifest; } @@ -381,7 +395,7 @@ export function createRegistry( logger.info("Uploading config..."); const configUploadUrl = await getUploadUrl(image); const configFile = path.join(folder, getHash(manifest.config.digest) + ".json"); - await uploadContent(configUploadUrl, configFile, manifest.config, allowInsecure, auth); + await uploadContent(configUploadUrl, configFile, manifest.config, allowInsecure, auth, "application/octet-stream"); logger.info("Uploading manifest..."); const manifestSize = await fileutil.sizeOf(manifestFile); @@ -391,6 +405,7 @@ export function createRegistry( { mediaType: manifest.mediaType, size: manifestSize }, allowInsecure, auth, + manifest.mediaType, ); } From 17bf41d366a477f38f7e2bb3a1983068cca8c289 Mon Sep 17 00:00:00 2001 From: Erlend Oftedal Date: Tue, 16 Jan 2024 07:05:27 +0100 Subject: [PATCH 2/6] Add github test case --- tests/githubtest/customContent/test.txt | 3 +++ tests/githubtest/test.sh | 10 ++++++++++ 2 files changed, 13 insertions(+) create mode 100644 tests/githubtest/customContent/test.txt create mode 100644 tests/githubtest/test.sh diff --git a/tests/githubtest/customContent/test.txt b/tests/githubtest/customContent/test.txt new file mode 100644 index 0000000..ce7fc18 --- /dev/null +++ b/tests/githubtest/customContent/test.txt @@ -0,0 +1,3 @@ +Hello world + +This is an integration test \ No newline at end of file diff --git a/tests/githubtest/test.sh b/tests/githubtest/test.sh new file mode 100644 index 0000000..f0ac686 --- /dev/null +++ b/tests/githubtest/test.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e +if [[ -z "$GITHUB_TOKEN" ]]; then + echo "ERROR: GITHUB_TOKEN not set (see https://github.com/settings/tokens/new?scopes=write:packages for running locally)" + exit 1 +fi + +printf "* Running containerify to pull from and push result to the local containerify test registry...\n" +../../lib/cli.js --verbose --fromImage docker-mirror/node:alpine --registry https://ghcr.io/v2/ --toImage eoftedal/containerify-integrationtest:latest --folder . --customContent customContent --setTimeStamp "2024-01-15T20:00:00.000Z" --token "$GITHUB_TOKEN" From a20a010535eef240a0b95ee15ec9256a3f8878d0 Mon Sep 17 00:00:00 2001 From: Erlend Oftedal Date: Tue, 16 Jan 2024 09:21:10 +0100 Subject: [PATCH 3/6] Some more stuff --- src/registry.ts | 77 ++++++++++--------- tests/external-registries/aws-ecr-test.sh | 12 +++ .../customContent/test.txt | 0 .../github-ghcr-test.sh} | 2 +- 4 files changed, 53 insertions(+), 38 deletions(-) create mode 100644 tests/external-registries/aws-ecr-test.sh rename tests/{githubtest => external-registries}/customContent/test.txt (100%) rename tests/{githubtest/test.sh => external-registries/github-ghcr-test.sh} (81%) diff --git a/src/registry.ts b/src/registry.ts index 3d70195..4b8e8c3 100644 --- a/src/registry.ts +++ b/src/registry.ts @@ -32,9 +32,14 @@ function request( callback: (res: http.IncomingMessage) => void, ) { if (allowInsecure == InsecureRegistrySupport.YES) options.rejectUnauthorized = false; - return (options.protocol == "https:" ? https : http).request(options, (res) => { + const req = (options.protocol == "https:" ? https : http).request(options, (res) => { callback(res); }); + req.on("error", (e) => { + logger.error("ERROR: " + e, options.method, options.path); + throw e; + }); + return req; } function isOk(httpStatus: number) { @@ -56,6 +61,12 @@ function toError(res: http.IncomingMessage) { return `Unexpected HTTP status ${res.statusCode} : ${res.statusMessage}`; } +function waitForResponseEnd(res: http.IncomingMessage, cb: (data: Buffer) => void) { + const data: Buffer[] = []; + res.on("data", (d) => data.push(d)); + res.on("end", () => cb(Buffer.concat(data))); +} + function dl(uri: string, headers: Headers, allowInsecure: InsecureRegistrySupport): Promise { logger.debug("dl", uri); return new Promise((resolve, reject) => { @@ -64,12 +75,7 @@ function dl(uri: string, headers: Headers, allowInsecure: InsecureRegistrySuppor const { res } = result; logger.debug(res.statusCode, res.statusMessage, res.headers["content-type"], res.headers["content-length"]); if (!isOk(res.statusCode ?? 0)) return reject(toError(res)); - const data: string[] = []; - res - .on("data", (chunk) => data.push(chunk.toString())) - .on("end", () => { - resolve(data.reduce((a, b) => a.concat(b))); - }); + waitForResponseEnd(res, (data) => resolve(data.toString())); }); }); } @@ -168,21 +174,25 @@ function headOk( options.method = "HEAD"; request(options, allowInsecure, (res) => { logger.debug(`HEAD ${url}`, res.statusCode); - // Not found - if (res.statusCode == 404) return resolve(false); - // OK - if (res.statusCode == 200) return resolve(true); - // Redirected - if (redirectCodes.includes(res.statusCode ?? 0) && res.headers.location) { - if (optimisticCheck) return resolve(true); - return resolve(headOk(res.headers.location, headers, allowInsecure, optimisticCheck, ++depth)); - } - // Unauthorized - // Possibly related to https://gitlab.com/gitlab-org/gitlab/-/issues/23132 - if (res.statusCode == 401) { - return resolve(false); - } - reject(toError(res)); + waitForResponseEnd(res, (data) => { + // Not found + if (res.statusCode == 404) return resolve(false); + // OK + if (res.statusCode == 200) return resolve(true); + // Redirected + if (redirectCodes.includes(res.statusCode ?? 0) && res.headers.location) { + if (optimisticCheck) return resolve(true); + return resolve(headOk(res.headers.location, headers, allowInsecure, optimisticCheck, ++depth)); + } + // Unauthorized + // Possibly related to https://gitlab.com/gitlab-org/gitlab/-/issues/23132 + if (res.statusCode == 401) { + return resolve(false); + } + // Other error + logger.error(`HEAD ${url}`, res.statusCode, res.statusMessage, data.toString()); + reject(toError(res)); + }); }).end(); }); } @@ -206,16 +216,14 @@ function uploadContent( "content-length": fileConfig.size, "content-type": contentType, }; - logger.debug("POST", url); + logger.debug(options.method, url); const req = request(options, allowInsecure, (res) => { logger.debug(res.statusCode, res.statusMessage, res.headers["content-type"], res.headers["content-length"]); if ([200, 201, 202, 203].includes(res.statusCode ?? 0)) { resolve(); } else { - const data: string[] = []; - res.on("data", (d) => data.push(d.toString())); - res.on("end", () => { - reject(`Error uploading to ${uploadUrl}. Got ${res.statusCode} ${res.statusMessage}:\n${data.join("")}`); + waitForResponseEnd(res, (data) => { + reject(`Error uploading to ${uploadUrl}. Got ${res.statusCode} ${res.statusMessage}:\n${data.toString()}`); }); } }); @@ -269,14 +277,11 @@ export function createRegistry( } reject("Missing location for 202"); } else { - const data: string[] = []; - res - .on("data", (c) => data.push(c.toString())) - .on("end", () => { - reject( - `Error getting upload URL from ${url}. Got ${res.statusCode} ${res.statusMessage}:\n${data.join("")}`, - ); - }); + waitForResponseEnd(res, (data) => { + reject( + `Error getting upload URL from ${url}. Got ${res.statusCode} ${res.statusMessage}:\n${data.toString()}`, + ); + }); } }).end(); }); @@ -371,7 +376,6 @@ export function createRegistry( const image = parseImage(imageStr); const manifestFile = path.join(folder, "manifest.json"); const manifest = (await fse.readJson(manifestFile)) as Manifest; - logger.info("Checking layer status..."); const layerStatus = await Promise.all( manifest.layers.map(async (l) => { @@ -383,7 +387,6 @@ export function createRegistry( "Needs upload:", layersForUpload.map((l) => l.layer.digest), ); - logger.info("Uploading layers..."); await Promise.all( layersForUpload.map(async (l) => { diff --git a/tests/external-registries/aws-ecr-test.sh b/tests/external-registries/aws-ecr-test.sh new file mode 100644 index 0000000..6ca320c --- /dev/null +++ b/tests/external-registries/aws-ecr-test.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +TOKEN=$(aws ecr get-authorization-token --output text --query 'authorizationData[].authorizationToken') + +ACCOUNT=$(aws sts get-caller-identity --output text --query 'Account') + +REGION=$(aws configure get region) + +echo $TOKEN + +printf "* Running containerify to pull from and push result to AWS ECR ...\n" +../../lib/cli.js --verbose --fromImage node:alpine --toRegistry https://$ACCOUNT.dkr.ecr.$REGION.amazonaws.com/v2/ --toImage containerify-integrationtest:latest --folder . --customContent customContent --setTimeStamp "2024-01-15T20:00:00.000Z" --toToken "Basic $TOKEN" --layerCacheFolder tmp/layercache diff --git a/tests/githubtest/customContent/test.txt b/tests/external-registries/customContent/test.txt similarity index 100% rename from tests/githubtest/customContent/test.txt rename to tests/external-registries/customContent/test.txt diff --git a/tests/githubtest/test.sh b/tests/external-registries/github-ghcr-test.sh similarity index 81% rename from tests/githubtest/test.sh rename to tests/external-registries/github-ghcr-test.sh index f0ac686..c72b846 100644 --- a/tests/githubtest/test.sh +++ b/tests/external-registries/github-ghcr-test.sh @@ -6,5 +6,5 @@ if [[ -z "$GITHUB_TOKEN" ]]; then exit 1 fi -printf "* Running containerify to pull from and push result to the local containerify test registry...\n" +printf "* Running containerify to pull from and push result to gchr.io ...\n" ../../lib/cli.js --verbose --fromImage docker-mirror/node:alpine --registry https://ghcr.io/v2/ --toImage eoftedal/containerify-integrationtest:latest --folder . --customContent customContent --setTimeStamp "2024-01-15T20:00:00.000Z" --token "$GITHUB_TOKEN" From 5ee388d34826cd9ed648f9d94f09e134ceea7a40 Mon Sep 17 00:00:00 2001 From: Erlend Oftedal Date: Tue, 16 Jan 2024 10:28:29 +0100 Subject: [PATCH 4/6] =?UTF-8?q?Add=20documentation=20for=20=C3=80WS=20ECR?= =?UTF-8?q?=20and=20Github=20GHCR?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 11 +++++++++++ tests/external-registries/aws-ecr-test.sh | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e38bda2..58d0df6 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,17 @@ containerify --fromImage nginx:alpine --folder . --toImage frontend:latest --cus This will take the `nginx:alpine` image, and copy the files from `./dist/` into `/usr/share/nginx/html`. +### Using with Github Container Registry (ghcr.io) + +1. Create a token from https://github.com/settings/tokens/new?scopes=write:packages or use GITHUB_TOKEN from Github Actions +2. Example: `containerify --registry https://ghcr.io/v2/ --token "$GITHUB_TOKEN" --fromImage docker-mirror/node:alpine --toImage : --folder . ` + +### Using with AWS ECR + +1. Create the repository in AWS from the console or through using the CLI +2. Create token using `aws ecr get-authorization-token --output text --query 'authorizationData[].authorizationToken'` +3. Example: `containerify --toToken "Basic $TOKEN" --toRegistry https://.dkr.ecr..amazonaws.com/v2/ --fromImage node:alpine --toImage : --folder .` + ### Command line options ``` diff --git a/tests/external-registries/aws-ecr-test.sh b/tests/external-registries/aws-ecr-test.sh index 6ca320c..34039f5 100644 --- a/tests/external-registries/aws-ecr-test.sh +++ b/tests/external-registries/aws-ecr-test.sh @@ -9,4 +9,4 @@ REGION=$(aws configure get region) echo $TOKEN printf "* Running containerify to pull from and push result to AWS ECR ...\n" -../../lib/cli.js --verbose --fromImage node:alpine --toRegistry https://$ACCOUNT.dkr.ecr.$REGION.amazonaws.com/v2/ --toImage containerify-integrationtest:latest --folder . --customContent customContent --setTimeStamp "2024-01-15T20:00:00.000Z" --toToken "Basic $TOKEN" --layerCacheFolder tmp/layercache +../../lib/cli.js --verbose --fromImage node:alpine --toRegistry https://$ACCOUNT.dkr.ecr.$REGION.amazonaws.com/v2/ --toImage containerify-test:latest --folder . --customContent customContent --setTimeStamp "2024-01-15T20:00:00.000Z" --toToken "Basic $TOKEN" From 3febed10ecdb8fce9a7d6aa0602b48325de3eda8 Mon Sep 17 00:00:00 2001 From: Erlend Oftedal Date: Tue, 16 Jan 2024 10:52:52 +0100 Subject: [PATCH 5/6] Fix modes for scripts --- tests/external-registries/aws-ecr-test.sh | 0 tests/external-registries/github-ghcr-test.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tests/external-registries/aws-ecr-test.sh mode change 100644 => 100755 tests/external-registries/github-ghcr-test.sh diff --git a/tests/external-registries/aws-ecr-test.sh b/tests/external-registries/aws-ecr-test.sh old mode 100644 new mode 100755 diff --git a/tests/external-registries/github-ghcr-test.sh b/tests/external-registries/github-ghcr-test.sh old mode 100644 new mode 100755 From 31a11b0d29a3d6640bcc86eaa130dda7d46c75ce Mon Sep 17 00:00:00 2001 From: Erlend Oftedal Date: Tue, 16 Jan 2024 12:47:54 +0100 Subject: [PATCH 6/6] Support cross mounting --- README.md | 16 +++--- src/cli.ts | 12 ++++- src/registry.ts | 110 ++++++++++++++++++++++++++++++++++------ src/types.ts | 1 + tests/localtest/test.sh | 2 +- 5 files changed, 115 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 58d0df6..d4fce9e 100644 --- a/README.md +++ b/README.md @@ -51,27 +51,29 @@ Options: --toImage Required: Image name of target image - [path/]image:tag --folder Required: Base folder of node application (contains package.json) --file Optional: Name of configuration file (defaults to containerify.json if found on path) + --doCrossMount Cross mount image layers from the base image (only works if fromImage and toImage are in the same registry) (default: false) --fromRegistry Optional: URL of registry to pull base image from - Default: https://registry-1.docker.io/v2/ --fromToken Optional: Authentication token for from registry --toRegistry Optional: URL of registry to push base image to - Default: https://registry-1.docker.io/v2/ - --optimisticToRegistryCheck Optional: Treat redirects as layer existing in remote registry. Potentially unsafe, but could save bandwidth. + --optimisticToRegistryCheck Treat redirects as layer existing in remote registry. Potentially unsafe, but can save bandwidth. --toToken Optional: Authentication token for target registry --toTar Optional: Export to tar file + --toDocker Optional: Export to local docker registry --registry Optional: Convenience argument for setting both from and to registry --platform Optional: Preferred platform, e.g. linux/amd64 or arm64 --token Optional: Convenience argument for setting token for both from and to registry - --user Optional: User account to run process in container - default: 1000 - --workdir Optional: Workdir where node app will be added and run from - default: /app - --entrypoint Optional: Entrypoint when starting container - default: npm start + --user Optional: User account to run process in container - default: 1000 (empty for customContent) + --workdir Optional: Workdir where node app will be added and run from - default: /app (empty for customContent) + --entrypoint Optional: Entrypoint when starting container - default: npm start (empty for customContent) --labels Optional: Comma-separated list of key value pairs to use as labels --label