Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding fixes for ghcr.io #30

Merged
merged 5 commits into from
Jan 16, 2024
Merged

Adding fixes for ghcr.io #30

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
dfe55cd
Adding fixes for ghcr.io
eoftedal Jan 15, 2024
17bf41d
Add github test case
eoftedal Jan 16, 2024
a20a010
Some more stuff
eoftedal Jan 16, 2024
5ee388d
Add documentation for ÀWS ECR and Github GHCR
eoftedal Jan 16, 2024
3febed1
Fix modes for scripts
eoftedal Jan 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .vscode/settings.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"editor.codeActionsOnSave": {
"source.fixAll.eslint": true
"source.fixAll.eslint": "explicit"
},
"editor.defaultFormatter": "esbenp.prettier-vscode",
"editor.formatOnSave": true
Expand Down
1 change: 1 addition & 0 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
"build": "tsc && chmod ugo+x lib/cli.js",
"lint": "eslint . --ext .ts --fix --ignore-path .gitignore",
"typecheck": "tsc --noEmit",
"watch": "tsc --watch",
"check": "npm run lint && npm run typecheck",
"dev": "tsc --watch",
"integrationTest": "cd tests/integration/ && ./test.sh",
Expand Down
104 changes: 61 additions & 43 deletions src/registry.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,14 @@ function request(
callback: (res: http.IncomingMessage) => void,
) {
if (allowInsecure == InsecureRegistrySupport.YES) options.rejectUnauthorized = false;
return (options.protocol == "https:" ? https : http).request(options, (res) => {
const req = (options.protocol == "https:" ? https : http).request(options, (res) => {
callback(res);
});
req.on("error", (e) => {
logger.error("ERROR: " + e, options.method, options.path);
throw e;
});
return req;
}

function isOk(httpStatus: number) {
Expand All @@ -56,6 +61,12 @@ function toError(res: http.IncomingMessage) {
return `Unexpected HTTP status ${res.statusCode} : ${res.statusMessage}`;
}

function waitForResponseEnd(res: http.IncomingMessage, cb: (data: Buffer) => void) {
const data: Buffer[] = [];
res.on("data", (d) => data.push(d));
res.on("end", () => cb(Buffer.concat(data)));
}

function dl(uri: string, headers: Headers, allowInsecure: InsecureRegistrySupport): Promise<string> {
logger.debug("dl", uri);
return new Promise((resolve, reject) => {
Expand All @@ -64,12 +75,7 @@ function dl(uri: string, headers: Headers, allowInsecure: InsecureRegistrySuppor
const { res } = result;
logger.debug(res.statusCode, res.statusMessage, res.headers["content-type"], res.headers["content-length"]);
if (!isOk(res.statusCode ?? 0)) return reject(toError(res));
const data: string[] = [];
res
.on("data", (chunk) => data.push(chunk.toString()))
.on("end", () => {
resolve(data.reduce((a, b) => a.concat(b)));
});
waitForResponseEnd(res, (data) => resolve(data.toString()));
});
});
}
Expand Down Expand Up @@ -168,21 +174,25 @@ function headOk(
options.method = "HEAD";
request(options, allowInsecure, (res) => {
logger.debug(`HEAD ${url}`, res.statusCode);
// Not found
if (res.statusCode == 404) return resolve(false);
// OK
if (res.statusCode == 200) return resolve(true);
// Redirected
if (redirectCodes.includes(res.statusCode ?? 0) && res.headers.location) {
if (optimisticCheck) return resolve(true);
return resolve(headOk(res.headers.location, headers, allowInsecure, optimisticCheck, ++depth));
}
// Unauthorized
// Possibly related to https://gitlab.com/gitlab-org/gitlab/-/issues/23132
if (res.statusCode == 401) {
return resolve(false);
}
reject(toError(res));
waitForResponseEnd(res, (data) => {
// Not found
if (res.statusCode == 404) return resolve(false);
// OK
if (res.statusCode == 200) return resolve(true);
// Redirected
if (redirectCodes.includes(res.statusCode ?? 0) && res.headers.location) {
if (optimisticCheck) return resolve(true);
return resolve(headOk(res.headers.location, headers, allowInsecure, optimisticCheck, ++depth));
}
// Unauthorized
// Possibly related to https://gitlab.com/gitlab-org/gitlab/-/issues/23132
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can take a look at this. I managed to push to GitLab after getting a jwt token by running

SCOPE="repository:${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}:pull,push"
AUTH_URL="${CI_SERVER_URL}/jwt/auth?service=container_registry&scope=${SCOPE}"
TOKEN=$(curl -sL --user "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" "${AUTH_URL}" | sed 's/.*"token":"\{0,1\}\([^,"]*\)"\{0,1\}.*/\1/')
yarn containerify --folder . --toRegistry "https://${CI_REGISTRY}/v2/${CI_PROJECT_NAMESPACE}" --toImage "${CI_PROJECT_NAME}:latest" --toToken $TOKEN

I assume we can run some checks on the container registry URL to determine if we should fetch this token or not.

if (res.statusCode == 401) {
return resolve(false);
}
// Other error
logger.error(`HEAD ${url}`, res.statusCode, res.statusMessage, data.toString());
reject(toError(res));
});
}).end();
});
}
Expand All @@ -193,6 +203,7 @@ function uploadContent(
fileConfig: PartialManifestConfig,
allowInsecure: InsecureRegistrySupport,
auth: string,
contentType: string,
): Promise<void> {
return new Promise((resolve, reject) => {
logger.debug("Uploading: ", file);
Expand All @@ -203,32 +214,36 @@ function uploadContent(
options.headers = {
authorization: auth,
"content-length": fileConfig.size,
"content-type": fileConfig.mediaType,
"content-type": contentType,
};
logger.debug("POST", url);
logger.debug(options.method, url);
const req = request(options, allowInsecure, (res) => {
logger.debug(res.statusCode, res.statusMessage, res.headers["content-type"], res.headers["content-length"]);
if ([200, 201, 202, 203].includes(res.statusCode ?? 0)) {
resolve();
} else {
const data: string[] = [];
res.on("data", (d) => data.push(d.toString()));
res.on("end", () => {
reject(`Error uploading to ${uploadUrl}. Got ${res.statusCode} ${res.statusMessage}:\n${data.join("")}`);
waitForResponseEnd(res, (data) => {
reject(`Error uploading to ${uploadUrl}. Got ${res.statusCode} ${res.statusMessage}:\n${data.toString()}`);
});
}
});
fss.createReadStream(file).pipe(req);
});
}

function prepareToken(token: string) {
if (token.startsWith("Basic ")) return token;
if (token.startsWith("ghp_")) return "Bearer " + Buffer.from(token).toString("base64");
return "Bearer " + token;
}

export function createRegistry(
registryBaseUrl: string,
token: string,
allowInsecure: InsecureRegistrySupport,
optimisticToRegistryCheck = false,
) {
const auth = token.startsWith("Basic ") ? token : "Bearer " + token;
const auth = prepareToken(token);

async function exists(image: Image, layer: Layer) {
const url = `${registryBaseUrl}${image.path}/blobs/${layer.digest}`;
Expand All @@ -238,7 +253,7 @@ export function createRegistry(
async function uploadLayerContent(uploadUrl: string, layer: Layer, dir: string) {
logger.info(layer.digest);
const file = path.join(dir, getHash(layer.digest) + getLayerTypeFileEnding(layer));
await uploadContent(uploadUrl, file, layer, allowInsecure, auth);
await uploadContent(uploadUrl, file, layer, allowInsecure, auth, "application/octet-stream");
}

async function getUploadUrl(image: Image): Promise<string> {
Expand All @@ -251,17 +266,22 @@ export function createRegistry(
logger.debug("POST", `${url}`, res.statusCode);
if (res.statusCode == 202) {
const { location } = res.headers;
if (location) resolve(location);
if (location) {
if (location.startsWith("http")) {
resolve(location);
} else {
const regURL = URL.parse(registryBaseUrl);

resolve(`${regURL.protocol}//${regURL.hostname}${regURL.port ? ":" + regURL.port : ""}${location}`);
}
}
reject("Missing location for 202");
} else {
const data: string[] = [];
res
.on("data", (c) => data.push(c.toString()))
.on("end", () => {
reject(
`Error getting upload URL from ${url}. Got ${res.statusCode} ${res.statusMessage}:\n${data.join("")}`,
);
});
waitForResponseEnd(res, (data) => {
reject(
`Error getting upload URL from ${url}. Got ${res.statusCode} ${res.statusMessage}:\n${data.toString()}`,
);
});
}
}).end();
});
Expand All @@ -285,7 +305,6 @@ export function createRegistry(
const adequateManifest = pickManifest(availableManifests, preferredPlatform);
return dlManifest({ ...image, tag: adequateManifest.digest }, preferredPlatform, allowInsecure);
}

return res as Manifest;
}

Expand Down Expand Up @@ -357,7 +376,6 @@ export function createRegistry(
const image = parseImage(imageStr);
const manifestFile = path.join(folder, "manifest.json");
const manifest = (await fse.readJson(manifestFile)) as Manifest;

logger.info("Checking layer status...");
const layerStatus = await Promise.all(
manifest.layers.map(async (l) => {
Expand All @@ -369,7 +387,6 @@ export function createRegistry(
"Needs upload:",
layersForUpload.map((l) => l.layer.digest),
);

logger.info("Uploading layers...");
await Promise.all(
layersForUpload.map(async (l) => {
Expand All @@ -381,7 +398,7 @@ export function createRegistry(
logger.info("Uploading config...");
const configUploadUrl = await getUploadUrl(image);
const configFile = path.join(folder, getHash(manifest.config.digest) + ".json");
await uploadContent(configUploadUrl, configFile, manifest.config, allowInsecure, auth);
await uploadContent(configUploadUrl, configFile, manifest.config, allowInsecure, auth, "application/octet-stream");

logger.info("Uploading manifest...");
const manifestSize = await fileutil.sizeOf(manifestFile);
Expand All @@ -391,6 +408,7 @@ export function createRegistry(
{ mediaType: manifest.mediaType, size: manifestSize },
allowInsecure,
auth,
manifest.mediaType,
);
}

Expand Down
12 changes: 12 additions & 0 deletions tests/external-registries/aws-ecr-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/sh

TOKEN=$(aws ecr get-authorization-token --output text --query 'authorizationData[].authorizationToken')

ACCOUNT=$(aws sts get-caller-identity --output text --query 'Account')

REGION=$(aws configure get region)

echo $TOKEN

printf "* Running containerify to pull from and push result to AWS ECR ...\n"
../../lib/cli.js --verbose --fromImage node:alpine --toRegistry https://$ACCOUNT.dkr.ecr.$REGION.amazonaws.com/v2/ --toImage containerify-integrationtest:latest --folder . --customContent customContent --setTimeStamp "2024-01-15T20:00:00.000Z" --toToken "Basic $TOKEN" --layerCacheFolder tmp/layercache
3 changes: 3 additions & 0 deletions tests/external-registries/customContent/test.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Hello world

This is an integration test
10 changes: 10 additions & 0 deletions tests/external-registries/github-ghcr-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
#!/bin/bash

set -e
if [[ -z "$GITHUB_TOKEN" ]]; then
echo "ERROR: GITHUB_TOKEN not set (see https://github.com/settings/tokens/new?scopes=write:packages for running locally)"
exit 1
fi

printf "* Running containerify to pull from and push result to gchr.io ...\n"
../../lib/cli.js --verbose --fromImage docker-mirror/node:alpine --registry https://ghcr.io/v2/ --toImage eoftedal/containerify-integrationtest:latest --folder . --customContent customContent --setTimeStamp "2024-01-15T20:00:00.000Z" --token "$GITHUB_TOKEN"
Loading