Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding fixes for ghcr.io #30

Merged
merged 5 commits into from
Jan 16, 2024
Merged

Adding fixes for ghcr.io #30

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
dfe55cd
Adding fixes for ghcr.io
eoftedal Jan 15, 2024
17bf41d
Add github test case
eoftedal Jan 16, 2024
a20a010
Some more stuff
eoftedal Jan 16, 2024
5ee388d
Add documentation for ÀWS ECR and Github GHCR
eoftedal Jan 16, 2024
3febed1
Fix modes for scripts
eoftedal Jan 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 40 additions & 37 deletions src/registry.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,14 @@ function request(
callback: (res: http.IncomingMessage) => void,
) {
if (allowInsecure == InsecureRegistrySupport.YES) options.rejectUnauthorized = false;
return (options.protocol == "https:" ? https : http).request(options, (res) => {
const req = (options.protocol == "https:" ? https : http).request(options, (res) => {
callback(res);
});
req.on("error", (e) => {
logger.error("ERROR: " + e, options.method, options.path);
throw e;
});
return req;
}

function isOk(httpStatus: number) {
Expand All @@ -56,6 +61,12 @@ function toError(res: http.IncomingMessage) {
return `Unexpected HTTP status ${res.statusCode} : ${res.statusMessage}`;
}

function waitForResponseEnd(res: http.IncomingMessage, cb: (data: Buffer) => void) {
const data: Buffer[] = [];
res.on("data", (d) => data.push(d));
res.on("end", () => cb(Buffer.concat(data)));
}

function dl(uri: string, headers: Headers, allowInsecure: InsecureRegistrySupport): Promise<string> {
logger.debug("dl", uri);
return new Promise((resolve, reject) => {
Expand All @@ -64,12 +75,7 @@ function dl(uri: string, headers: Headers, allowInsecure: InsecureRegistrySuppor
const { res } = result;
logger.debug(res.statusCode, res.statusMessage, res.headers["content-type"], res.headers["content-length"]);
if (!isOk(res.statusCode ?? 0)) return reject(toError(res));
const data: string[] = [];
res
.on("data", (chunk) => data.push(chunk.toString()))
.on("end", () => {
resolve(data.reduce((a, b) => a.concat(b)));
});
waitForResponseEnd(res, (data) => resolve(data.toString()));
});
});
}
Expand Down Expand Up @@ -168,21 +174,25 @@ function headOk(
options.method = "HEAD";
request(options, allowInsecure, (res) => {
logger.debug(`HEAD ${url}`, res.statusCode);
// Not found
if (res.statusCode == 404) return resolve(false);
// OK
if (res.statusCode == 200) return resolve(true);
// Redirected
if (redirectCodes.includes(res.statusCode ?? 0) && res.headers.location) {
if (optimisticCheck) return resolve(true);
return resolve(headOk(res.headers.location, headers, allowInsecure, optimisticCheck, ++depth));
}
// Unauthorized
// Possibly related to https://gitlab.com/gitlab-org/gitlab/-/issues/23132
if (res.statusCode == 401) {
return resolve(false);
}
reject(toError(res));
waitForResponseEnd(res, (data) => {
// Not found
if (res.statusCode == 404) return resolve(false);
// OK
if (res.statusCode == 200) return resolve(true);
// Redirected
if (redirectCodes.includes(res.statusCode ?? 0) && res.headers.location) {
if (optimisticCheck) return resolve(true);
return resolve(headOk(res.headers.location, headers, allowInsecure, optimisticCheck, ++depth));
}
// Unauthorized
// Possibly related to https://gitlab.com/gitlab-org/gitlab/-/issues/23132
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can take a look at this. I managed to push to GitLab after getting a jwt token by running

SCOPE="repository:${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}:pull,push"
AUTH_URL="${CI_SERVER_URL}/jwt/auth?service=container_registry&scope=${SCOPE}"
TOKEN=$(curl -sL --user "${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD}" "${AUTH_URL}" | sed 's/.*"token":"\{0,1\}\([^,"]*\)"\{0,1\}.*/\1/')
yarn containerify --folder . --toRegistry "https://${CI_REGISTRY}/v2/${CI_PROJECT_NAMESPACE}" --toImage "${CI_PROJECT_NAME}:latest" --toToken $TOKEN

I assume we can run some checks on the container registry URL to determine if we should fetch this token or not.

if (res.statusCode == 401) {
return resolve(false);
}
// Other error
logger.error(`HEAD ${url}`, res.statusCode, res.statusMessage, data.toString());
reject(toError(res));
});
}).end();
});
}
Expand All @@ -206,16 +216,14 @@ function uploadContent(
"content-length": fileConfig.size,
"content-type": contentType,
};
logger.debug("POST", url);
logger.debug(options.method, url);
const req = request(options, allowInsecure, (res) => {
logger.debug(res.statusCode, res.statusMessage, res.headers["content-type"], res.headers["content-length"]);
if ([200, 201, 202, 203].includes(res.statusCode ?? 0)) {
resolve();
} else {
const data: string[] = [];
res.on("data", (d) => data.push(d.toString()));
res.on("end", () => {
reject(`Error uploading to ${uploadUrl}. Got ${res.statusCode} ${res.statusMessage}:\n${data.join("")}`);
waitForResponseEnd(res, (data) => {
reject(`Error uploading to ${uploadUrl}. Got ${res.statusCode} ${res.statusMessage}:\n${data.toString()}`);
});
}
});
Expand Down Expand Up @@ -269,14 +277,11 @@ export function createRegistry(
}
reject("Missing location for 202");
} else {
const data: string[] = [];
res
.on("data", (c) => data.push(c.toString()))
.on("end", () => {
reject(
`Error getting upload URL from ${url}. Got ${res.statusCode} ${res.statusMessage}:\n${data.join("")}`,
);
});
waitForResponseEnd(res, (data) => {
reject(
`Error getting upload URL from ${url}. Got ${res.statusCode} ${res.statusMessage}:\n${data.toString()}`,
);
});
}
}).end();
});
Expand Down Expand Up @@ -371,7 +376,6 @@ export function createRegistry(
const image = parseImage(imageStr);
const manifestFile = path.join(folder, "manifest.json");
const manifest = (await fse.readJson(manifestFile)) as Manifest;

logger.info("Checking layer status...");
const layerStatus = await Promise.all(
manifest.layers.map(async (l) => {
Expand All @@ -383,7 +387,6 @@ export function createRegistry(
"Needs upload:",
layersForUpload.map((l) => l.layer.digest),
);

logger.info("Uploading layers...");
await Promise.all(
layersForUpload.map(async (l) => {
Expand Down
12 changes: 12 additions & 0 deletions tests/external-registries/aws-ecr-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/sh

TOKEN=$(aws ecr get-authorization-token --output text --query 'authorizationData[].authorizationToken')

ACCOUNT=$(aws sts get-caller-identity --output text --query 'Account')

REGION=$(aws configure get region)

echo $TOKEN

printf "* Running containerify to pull from and push result to AWS ECR ...\n"
../../lib/cli.js --verbose --fromImage node:alpine --toRegistry https://$ACCOUNT.dkr.ecr.$REGION.amazonaws.com/v2/ --toImage containerify-integrationtest:latest --folder . --customContent customContent --setTimeStamp "2024-01-15T20:00:00.000Z" --toToken "Basic $TOKEN" --layerCacheFolder tmp/layercache
0 tests/githubtest/customContent/test.txt → ...xternal-registries/customContent/test.txt
View file
Open in desktop
File renamed without changes.
2 changes: 1 addition & 1 deletion tests/githubtest/test.sh → ...s/external-registries/github-ghcr-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,5 @@ if [[ -z "$GITHUB_TOKEN" ]]; then
exit 1
fi

printf "* Running containerify to pull from and push result to the local containerify test registry...\n"
printf "* Running containerify to pull from and push result to gchr.io ...\n"
../../lib/cli.js --verbose --fromImage docker-mirror/node:alpine --registry https://ghcr.io/v2/ --toImage eoftedal/containerify-integrationtest:latest --folder . --customContent customContent --setTimeStamp "2024-01-15T20:00:00.000Z" --token "$GITHUB_TOKEN"
Loading