diff --git a/.vscode/settings.json b/.vscode/settings.json index 12a8a85..1e2721b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,6 @@ { "editor.codeActionsOnSave": { - "source.fixAll.eslint": true + "source.fixAll.eslint": "explicit" }, "editor.defaultFormatter": "esbenp.prettier-vscode", "editor.formatOnSave": true diff --git a/README.md b/README.md index e38bda2..58d0df6 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,17 @@ containerify --fromImage nginx:alpine --folder . --toImage frontend:latest --cus This will take the `nginx:alpine` image, and copy the files from `./dist/` into `/usr/share/nginx/html`. +### Using with Github Container Registry (ghcr.io) + +1. Create a token from https://github.com/settings/tokens/new?scopes=write:packages or use GITHUB_TOKEN from Github Actions +2. Example: `containerify --registry https://ghcr.io/v2/ --token "$GITHUB_TOKEN" --fromImage docker-mirror/node:alpine --toImage : --folder . ` + +### Using with AWS ECR + +1. Create the repository in AWS from the console or through using the CLI +2. Create token using `aws ecr get-authorization-token --output text --query 'authorizationData[].authorizationToken'` +3. Example: `containerify --toToken "Basic $TOKEN" --toRegistry https://.dkr.ecr..amazonaws.com/v2/ --fromImage node:alpine --toImage : --folder .` + ### Command line options ``` diff --git a/package.json b/package.json index 95b4974..0f8fb7d 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "build": "tsc && chmod ugo+x lib/cli.js", "lint": "eslint . --ext .ts --fix --ignore-path .gitignore", "typecheck": "tsc --noEmit", + "watch": "tsc --watch", "check": "npm run lint && npm run typecheck", "dev": "tsc --watch", "integrationTest": "cd tests/integration/ && ./test.sh", diff --git a/src/registry.ts b/src/registry.ts index ce9c474..4b8e8c3 100644 --- a/src/registry.ts +++ b/src/registry.ts @@ -32,9 +32,14 @@ function request( callback: (res: http.IncomingMessage) => void, ) { if (allowInsecure == InsecureRegistrySupport.YES) options.rejectUnauthorized = false; - return (options.protocol == "https:" ? https : http).request(options, (res) => { + const req = (options.protocol == "https:" ? https : http).request(options, (res) => { callback(res); }); + req.on("error", (e) => { + logger.error("ERROR: " + e, options.method, options.path); + throw e; + }); + return req; } function isOk(httpStatus: number) { @@ -56,6 +61,12 @@ function toError(res: http.IncomingMessage) { return `Unexpected HTTP status ${res.statusCode} : ${res.statusMessage}`; } +function waitForResponseEnd(res: http.IncomingMessage, cb: (data: Buffer) => void) { + const data: Buffer[] = []; + res.on("data", (d) => data.push(d)); + res.on("end", () => cb(Buffer.concat(data))); +} + function dl(uri: string, headers: Headers, allowInsecure: InsecureRegistrySupport): Promise { logger.debug("dl", uri); return new Promise((resolve, reject) => { @@ -64,12 +75,7 @@ function dl(uri: string, headers: Headers, allowInsecure: InsecureRegistrySuppor const { res } = result; logger.debug(res.statusCode, res.statusMessage, res.headers["content-type"], res.headers["content-length"]); if (!isOk(res.statusCode ?? 0)) return reject(toError(res)); - const data: string[] = []; - res - .on("data", (chunk) => data.push(chunk.toString())) - .on("end", () => { - resolve(data.reduce((a, b) => a.concat(b))); - }); + waitForResponseEnd(res, (data) => resolve(data.toString())); }); }); } @@ -168,21 +174,25 @@ function headOk( options.method = "HEAD"; request(options, allowInsecure, (res) => { logger.debug(`HEAD ${url}`, res.statusCode); - // Not found - if (res.statusCode == 404) return resolve(false); - // OK - if (res.statusCode == 200) return resolve(true); - // Redirected - if (redirectCodes.includes(res.statusCode ?? 0) && res.headers.location) { - if (optimisticCheck) return resolve(true); - return resolve(headOk(res.headers.location, headers, allowInsecure, optimisticCheck, ++depth)); - } - // Unauthorized - // Possibly related to https://gitlab.com/gitlab-org/gitlab/-/issues/23132 - if (res.statusCode == 401) { - return resolve(false); - } - reject(toError(res)); + waitForResponseEnd(res, (data) => { + // Not found + if (res.statusCode == 404) return resolve(false); + // OK + if (res.statusCode == 200) return resolve(true); + // Redirected + if (redirectCodes.includes(res.statusCode ?? 0) && res.headers.location) { + if (optimisticCheck) return resolve(true); + return resolve(headOk(res.headers.location, headers, allowInsecure, optimisticCheck, ++depth)); + } + // Unauthorized + // Possibly related to https://gitlab.com/gitlab-org/gitlab/-/issues/23132 + if (res.statusCode == 401) { + return resolve(false); + } + // Other error + logger.error(`HEAD ${url}`, res.statusCode, res.statusMessage, data.toString()); + reject(toError(res)); + }); }).end(); }); } @@ -193,6 +203,7 @@ function uploadContent( fileConfig: PartialManifestConfig, allowInsecure: InsecureRegistrySupport, auth: string, + contentType: string, ): Promise { return new Promise((resolve, reject) => { logger.debug("Uploading: ", file); @@ -203,18 +214,16 @@ function uploadContent( options.headers = { authorization: auth, "content-length": fileConfig.size, - "content-type": fileConfig.mediaType, + "content-type": contentType, }; - logger.debug("POST", url); + logger.debug(options.method, url); const req = request(options, allowInsecure, (res) => { logger.debug(res.statusCode, res.statusMessage, res.headers["content-type"], res.headers["content-length"]); if ([200, 201, 202, 203].includes(res.statusCode ?? 0)) { resolve(); } else { - const data: string[] = []; - res.on("data", (d) => data.push(d.toString())); - res.on("end", () => { - reject(`Error uploading to ${uploadUrl}. Got ${res.statusCode} ${res.statusMessage}:\n${data.join("")}`); + waitForResponseEnd(res, (data) => { + reject(`Error uploading to ${uploadUrl}. Got ${res.statusCode} ${res.statusMessage}:\n${data.toString()}`); }); } }); @@ -222,13 +231,19 @@ function uploadContent( }); } +function prepareToken(token: string) { + if (token.startsWith("Basic ")) return token; + if (token.startsWith("ghp_")) return "Bearer " + Buffer.from(token).toString("base64"); + return "Bearer " + token; +} + export function createRegistry( registryBaseUrl: string, token: string, allowInsecure: InsecureRegistrySupport, optimisticToRegistryCheck = false, ) { - const auth = token.startsWith("Basic ") ? token : "Bearer " + token; + const auth = prepareToken(token); async function exists(image: Image, layer: Layer) { const url = `${registryBaseUrl}${image.path}/blobs/${layer.digest}`; @@ -238,7 +253,7 @@ export function createRegistry( async function uploadLayerContent(uploadUrl: string, layer: Layer, dir: string) { logger.info(layer.digest); const file = path.join(dir, getHash(layer.digest) + getLayerTypeFileEnding(layer)); - await uploadContent(uploadUrl, file, layer, allowInsecure, auth); + await uploadContent(uploadUrl, file, layer, allowInsecure, auth, "application/octet-stream"); } async function getUploadUrl(image: Image): Promise { @@ -251,17 +266,22 @@ export function createRegistry( logger.debug("POST", `${url}`, res.statusCode); if (res.statusCode == 202) { const { location } = res.headers; - if (location) resolve(location); + if (location) { + if (location.startsWith("http")) { + resolve(location); + } else { + const regURL = URL.parse(registryBaseUrl); + + resolve(`${regURL.protocol}//${regURL.hostname}${regURL.port ? ":" + regURL.port : ""}${location}`); + } + } reject("Missing location for 202"); } else { - const data: string[] = []; - res - .on("data", (c) => data.push(c.toString())) - .on("end", () => { - reject( - `Error getting upload URL from ${url}. Got ${res.statusCode} ${res.statusMessage}:\n${data.join("")}`, - ); - }); + waitForResponseEnd(res, (data) => { + reject( + `Error getting upload URL from ${url}. Got ${res.statusCode} ${res.statusMessage}:\n${data.toString()}`, + ); + }); } }).end(); }); @@ -285,7 +305,6 @@ export function createRegistry( const adequateManifest = pickManifest(availableManifests, preferredPlatform); return dlManifest({ ...image, tag: adequateManifest.digest }, preferredPlatform, allowInsecure); } - return res as Manifest; } @@ -357,7 +376,6 @@ export function createRegistry( const image = parseImage(imageStr); const manifestFile = path.join(folder, "manifest.json"); const manifest = (await fse.readJson(manifestFile)) as Manifest; - logger.info("Checking layer status..."); const layerStatus = await Promise.all( manifest.layers.map(async (l) => { @@ -369,7 +387,6 @@ export function createRegistry( "Needs upload:", layersForUpload.map((l) => l.layer.digest), ); - logger.info("Uploading layers..."); await Promise.all( layersForUpload.map(async (l) => { @@ -381,7 +398,7 @@ export function createRegistry( logger.info("Uploading config..."); const configUploadUrl = await getUploadUrl(image); const configFile = path.join(folder, getHash(manifest.config.digest) + ".json"); - await uploadContent(configUploadUrl, configFile, manifest.config, allowInsecure, auth); + await uploadContent(configUploadUrl, configFile, manifest.config, allowInsecure, auth, "application/octet-stream"); logger.info("Uploading manifest..."); const manifestSize = await fileutil.sizeOf(manifestFile); @@ -391,6 +408,7 @@ export function createRegistry( { mediaType: manifest.mediaType, size: manifestSize }, allowInsecure, auth, + manifest.mediaType, ); } diff --git a/tests/external-registries/aws-ecr-test.sh b/tests/external-registries/aws-ecr-test.sh new file mode 100755 index 0000000..34039f5 --- /dev/null +++ b/tests/external-registries/aws-ecr-test.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +TOKEN=$(aws ecr get-authorization-token --output text --query 'authorizationData[].authorizationToken') + +ACCOUNT=$(aws sts get-caller-identity --output text --query 'Account') + +REGION=$(aws configure get region) + +echo $TOKEN + +printf "* Running containerify to pull from and push result to AWS ECR ...\n" +../../lib/cli.js --verbose --fromImage node:alpine --toRegistry https://$ACCOUNT.dkr.ecr.$REGION.amazonaws.com/v2/ --toImage containerify-test:latest --folder . --customContent customContent --setTimeStamp "2024-01-15T20:00:00.000Z" --toToken "Basic $TOKEN" diff --git a/tests/external-registries/customContent/test.txt b/tests/external-registries/customContent/test.txt new file mode 100644 index 0000000..ce7fc18 --- /dev/null +++ b/tests/external-registries/customContent/test.txt @@ -0,0 +1,3 @@ +Hello world + +This is an integration test \ No newline at end of file diff --git a/tests/external-registries/github-ghcr-test.sh b/tests/external-registries/github-ghcr-test.sh new file mode 100755 index 0000000..c72b846 --- /dev/null +++ b/tests/external-registries/github-ghcr-test.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e +if [[ -z "$GITHUB_TOKEN" ]]; then + echo "ERROR: GITHUB_TOKEN not set (see https://github.com/settings/tokens/new?scopes=write:packages for running locally)" + exit 1 +fi + +printf "* Running containerify to pull from and push result to gchr.io ...\n" +../../lib/cli.js --verbose --fromImage docker-mirror/node:alpine --registry https://ghcr.io/v2/ --toImage eoftedal/containerify-integrationtest:latest --folder . --customContent customContent --setTimeStamp "2024-01-15T20:00:00.000Z" --token "$GITHUB_TOKEN"