From cff2ece48d98f3cb0abb63af8edd4a06ca19c4ec Mon Sep 17 00:00:00 2001 From: Alex Eagle Date: Wed, 4 Oct 2023 07:37:53 -0700 Subject: [PATCH 1/3] refactor: use hermetic tar Fixes #328 fix: supply mtree file for determinism fix: set tar content times to beginning of this year avoids some tools thinking that 1970 is 'too old' refactor: extract function for mtree lines refactor: cleanup STAGING_DIR chore: bump to bazel-lib 2.0rc chore: remove bazel 5 workaround Bazel-lib 2.0 doesn't include this anymore chore: upgrade stardoc to match bzlmod version ci: test on bazel 7 rather than 5 --- oci/private/tarball.bzl | 11 ++++++++++- oci/private/tarball.sh.tpl | 10 ++++++---- oci/repositories.bzl | 3 ++- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/oci/private/tarball.bzl b/oci/private/tarball.bzl index fa23ae3e..0b4bf63d 100644 --- a/oci/private/tarball.bzl +++ b/oci/private/tarball.bzl @@ -76,12 +76,14 @@ def _tarball_impl(ctx): image = ctx.file.image tarball = ctx.actions.declare_file("{}/tarball.tar".format(ctx.label.name)) + bsdtar = ctx.toolchains["@aspect_bazel_lib//lib:tar_toolchain_type"] executable = ctx.actions.declare_file("{}/tarball.sh".format(ctx.label.name)) repo_tags = ctx.file.repo_tags substitutions = { "{{format}}": ctx.attr.format, "{{jq_path}}": jq.bin.path, + "{{tar}}": bsdtar.tarinfo.binary.path, "{{image_dir}}": image.path, "{{tarball_path}}": tarball.path, } @@ -96,9 +98,15 @@ def _tarball_impl(ctx): substitutions = substitutions, ) + # TODO(2.0): this oci_tarball rule should just produce an mtree manifest instead, + # and then the tar rule can be composed in the oci_tarball macro in defs.bzl. + # To make it a non-breaking change, call the tar program from within this action instead. ctx.actions.run( executable = util.maybe_wrap_launcher_for_windows(ctx, executable), - inputs = [image, repo_tags, executable], + inputs = depset( + direct = [image, repo_tags, executable], + transitive = [bsdtar.default.files], + ), outputs = [tarball], tools = [jq.bin], mnemonic = "OCITarball", @@ -131,6 +139,7 @@ oci_tarball = rule( toolchains = [ "@bazel_tools//tools/sh:toolchain_type", "@aspect_bazel_lib//lib:jq_toolchain_type", + "@aspect_bazel_lib//lib:tar_toolchain_type", ], executable = True, ) diff --git a/oci/private/tarball.sh.tpl b/oci/private/tarball.sh.tpl index 4daa05e9..6f654140 100644 --- a/oci/private/tarball.sh.tpl +++ b/oci/private/tarball.sh.tpl @@ -2,10 +2,9 @@ set -o pipefail -o errexit -o nounset readonly FORMAT="{{format}}" -readonly STAGING_DIR=$(mktemp -d) readonly JQ="{{jq_path}}" +readonly TAR="{{tar}}" readonly IMAGE_DIR="{{image_dir}}" -readonly BLOBS_DIR="${STAGING_DIR}/blobs" readonly TARBALL_PATH="{{tarball_path}}" readonly REPOTAGS=($(cat "{{tags}}")) readonly INDEX_FILE="${IMAGE_DIR}/index.json" @@ -84,6 +83,7 @@ MANIFEST_BLOB_PATH="${IMAGE_DIR}/blobs/${MANIFEST_DIGEST}" CONFIG_DIGEST=$(${JQ} -r '.config.digest | sub(":"; "/")' ${MANIFEST_BLOB_PATH}) CONFIG_BLOB_PATH="${IMAGE_DIR}/blobs/${CONFIG_DIGEST}" +add_to_tar "${CONFIG_BLOB_PATH}" "blobs/${CONFIG_DIGEST}" LAYERS=$(${JQ} -cr '.layers | map(.digest | sub(":"; "/"))' ${MANIFEST_BLOB_PATH}) @@ -100,5 +100,7 @@ repotags="${REPOTAGS[@]+"${REPOTAGS[@]}"}" --arg config "blobs/${CONFIG_DIGEST}" \ --argjson layers "${LAYERS}" > "${STAGING_DIR}/manifest.json" -# TODO: https://github.com/bazel-contrib/rules_oci/issues/217 -tar -C "${STAGING_DIR}" -cf "${TARBALL_PATH}" manifest.json blobs +add_to_tar "${manifest_json}" "manifest.json" + +# We've created the manifest, now hand it off to tar to create our final output +"${TAR}" --create --file "${TARBALL_PATH}" "@${mtree}" diff --git a/oci/repositories.bzl b/oci/repositories.bzl index a8ace689..83af5c14 100644 --- a/oci/repositories.bzl +++ b/oci/repositories.bzl @@ -1,6 +1,6 @@ """Repository rules for fetching external tools""" -load("@aspect_bazel_lib//lib:repositories.bzl", "register_copy_to_directory_toolchains", "register_coreutils_toolchains", "register_jq_toolchains") +load("@aspect_bazel_lib//lib:repositories.bzl", "register_copy_to_directory_toolchains", "register_coreutils_toolchains", "register_jq_toolchains", "register_tar_toolchains") load("//oci/private:toolchains_repo.bzl", "PLATFORMS", "toolchains_repo") load("//oci/private:versions.bzl", "CRANE_VERSIONS", "ZOT_VERSIONS") @@ -113,6 +113,7 @@ def oci_register_toolchains(name, crane_version, zot_version = None, register = Should be True for WORKSPACE users, but false when used under bzlmod extension """ register_jq_toolchains(register = register) + register_tar_toolchains(register = register) register_coreutils_toolchains(register = register) register_copy_to_directory_toolchains(register = register) From 3f0ee6491ccd64770fde2dd0a724e4d4b59ce029 Mon Sep 17 00:00:00 2001 From: Alex Eagle Date: Thu, 25 Apr 2024 15:57:06 -0700 Subject: [PATCH 2/3] chore: switch to hermetic tar --- oci/private/tarball.sh.tpl | 49 ++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/oci/private/tarball.sh.tpl b/oci/private/tarball.sh.tpl index 6f654140..22870799 100644 --- a/oci/private/tarball.sh.tpl +++ b/oci/private/tarball.sh.tpl @@ -9,11 +9,14 @@ readonly TARBALL_PATH="{{tarball_path}}" readonly REPOTAGS=($(cat "{{tags}}")) readonly INDEX_FILE="${IMAGE_DIR}/index.json" -cp_f_with_mkdir() { - SRC="$1" - DST="$2" - mkdir -p "$(dirname "${DST}")" - cp -f "${SRC}" "${DST}" +# Write tar manifest in mtree format +# https://man.freebsd.org/cgi/man.cgi?mtree(8) +# so that tar produces a deterministic output. +mtree=$(mktemp) +function add_to_tar() { + content=$1 + tar_path=$2 + echo >>"${mtree}" "${tar_path} uid=0 gid=0 mode=0755 time=1672560000 type=file content=${content}" } MANIFEST_DIGEST=$(${JQ} -r '.manifests[0].digest | sub(":"; "/")' "${INDEX_FILE}" | tr -d '"') @@ -45,36 +48,41 @@ if [[ "${FORMAT}" == "oci" ]]; then # Handle multi-architecture image indexes. # Ideally the toolchains we rely on would output these for us, but they don't seem to. - echo -n '{"imageLayoutVersion": "1.0.0"}' > "${STAGING_DIR}/oci-layout" + layout_file=$(mktemp) + echo -n '{"imageLayoutVersion": "1.0.0"}' > "$layout_file" + add_to_tar "$layout_file" oci-layout INDEX_FILE_MANIFEST_DIGEST=$("${JQ}" -r '.manifests[0].digest | sub(":"; "/")' "${INDEX_FILE}" | tr -d '"') INDEX_FILE_MANIFEST_BLOB_PATH="${IMAGE_DIR}/blobs/${INDEX_FILE_MANIFEST_DIGEST}" - cp_f_with_mkdir "${INDEX_FILE_MANIFEST_BLOB_PATH}" "${BLOBS_DIR}/${INDEX_FILE_MANIFEST_DIGEST}" + add_to_tar "${INDEX_FILE_MANIFEST_BLOB_PATH}" "blobs/${INDEX_FILE_MANIFEST_DIGEST}" IMAGE_MANIFESTS_DIGESTS=($("${JQ}" -r '.manifests[] | .digest | sub(":"; "/")' "${INDEX_FILE_MANIFEST_BLOB_PATH}")) for IMAGE_MANIFEST_DIGEST in "${IMAGE_MANIFESTS_DIGESTS[@]}"; do IMAGE_MANIFEST_BLOB_PATH="${IMAGE_DIR}/blobs/${IMAGE_MANIFEST_DIGEST}" - cp_f_with_mkdir "${IMAGE_MANIFEST_BLOB_PATH}" "${BLOBS_DIR}/${IMAGE_MANIFEST_DIGEST}" + add_to_tar "${IMAGE_MANIFEST_BLOB_PATH}" "blobs/${IMAGE_MANIFEST_DIGEST}" CONFIG_DIGEST=$("${JQ}" -r '.config.digest | sub(":"; "/")' ${IMAGE_MANIFEST_BLOB_PATH}) CONFIG_BLOB_PATH="${IMAGE_DIR}/blobs/${CONFIG_DIGEST}" - cp_f_with_mkdir "${CONFIG_BLOB_PATH}" "${BLOBS_DIR}/${CONFIG_DIGEST}" + add_to_tar "${CONFIG_BLOB_PATH}" "blobs/${CONFIG_DIGEST}" LAYER_DIGESTS=$("${JQ}" -r '.layers | map(.digest | sub(":"; "/"))' "${IMAGE_MANIFEST_BLOB_PATH}") for LAYER_DIGEST in $("${JQ}" -r ".[]" <<< $LAYER_DIGESTS); do - cp_f_with_mkdir "${IMAGE_DIR}/blobs/${LAYER_DIGEST}" ${BLOBS_DIR}/${LAYER_DIGEST} + add_to_tar "${IMAGE_DIR}/blobs/${LAYER_DIGEST}" blobs/${LAYER_DIGEST} done done # Repeat the first manifest entry once per repo tag. repotags="${REPOTAGS[@]+"${REPOTAGS[@]}"}" - "${JQ}" -r --arg repo_tags "$repotags" \ - '.manifests[0] as $manifest | .manifests = ($repo_tags | split(" ") | map($manifest * {annotations:{"org.opencontainers.image.ref.name":.}}))' "${INDEX_FILE}" > "${STAGING_DIR}/index.json" + index_json=$(mktemp) + "${JQ}" >"$index_json" \ + -r --arg repo_tags "$repotags" \ + '.manifests[0] as $manifest | .manifests = ($repo_tags | split(" ") | map($manifest * {annotations:{"org.opencontainers.image.ref.name":.}}))' "${INDEX_FILE}" + add_to_tar "$index_json" index.json - tar -C "${STAGING_DIR}" -cf "${TARBALL_PATH}" index.json blobs oci-layout + ${TAR} --create --no-xattr --no-mac-metadata --file "${TARBALL_PATH}" "@${mtree}" exit 0 fi @@ -87,18 +95,19 @@ add_to_tar "${CONFIG_BLOB_PATH}" "blobs/${CONFIG_DIGEST}" LAYERS=$(${JQ} -cr '.layers | map(.digest | sub(":"; "/"))' ${MANIFEST_BLOB_PATH}) -cp_f_with_mkdir "${CONFIG_BLOB_PATH}" "${BLOBS_DIR}/${CONFIG_DIGEST}" +add_to_tar "${CONFIG_BLOB_PATH}" "blobs/${CONFIG_DIGEST}" for LAYER in $(${JQ} -r ".[]" <<< $LAYERS); do - cp_f_with_mkdir "${IMAGE_DIR}/blobs/${LAYER}" "${BLOBS_DIR}/${LAYER}.tar.gz" + add_to_tar "${IMAGE_DIR}/blobs/${LAYER}" "blobs/${LAYER}.tar.gz" done - +manifest_json=$(mktemp) repotags="${REPOTAGS[@]+"${REPOTAGS[@]}"}" -"${JQ}" -n '.[0] = {"Config": $config, "RepoTags": ($repo_tags | split(" ") | map(select(. != ""))), "Layers": $layers | map( "blobs/" + . + ".tar.gz") }' \ - --arg repo_tags "$repotags" \ - --arg config "blobs/${CONFIG_DIGEST}" \ - --argjson layers "${LAYERS}" > "${STAGING_DIR}/manifest.json" +"${JQ}" > "${manifest_json}" \ + -n '.[0] = {"Config": $config, "RepoTags": ($repo_tags | split(" ") | map(select(. != ""))), "Layers": $layers | map( "blobs/" + . + ".tar.gz") }' \ + --arg repo_tags "$repotags" \ + --arg config "blobs/${CONFIG_DIGEST}" \ + --argjson layers "${LAYERS}" add_to_tar "${manifest_json}" "manifest.json" From 958b84d05ac4fb494fa9be5a28e228fe1f425d97 Mon Sep 17 00:00:00 2001 From: Alex Eagle Date: Thu, 25 Apr 2024 17:55:27 -0700 Subject: [PATCH 3/3] Update tarball.sh.tpl --- oci/private/tarball.sh.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oci/private/tarball.sh.tpl b/oci/private/tarball.sh.tpl index 22870799..16262129 100644 --- a/oci/private/tarball.sh.tpl +++ b/oci/private/tarball.sh.tpl @@ -112,4 +112,4 @@ repotags="${REPOTAGS[@]+"${REPOTAGS[@]}"}" add_to_tar "${manifest_json}" "manifest.json" # We've created the manifest, now hand it off to tar to create our final output -"${TAR}" --create --file "${TARBALL_PATH}" "@${mtree}" +"${TAR}" --create --no-xattr --no-mac-metadata --file "${TARBALL_PATH}" "@${mtree}"