diff --git a/oci/private/image.bzl b/oci/private/image.bzl index ab7aab51..556edcc9 100644 --- a/oci/private/image.bzl +++ b/oci/private/image.bzl @@ -103,8 +103,10 @@ def _oci_image_impl(ctx): crane = ctx.toolchains["@rules_oci//oci:crane_toolchain_type"] registry = ctx.toolchains["@rules_oci//oci:registry_toolchain_type"] + coreutils = ctx.toolchains["@aspect_bazel_lib//lib:coreutils_toolchain_type"] + grep = ctx.toolchains["@aspect_bazel_lib//lib:grep_toolchain_type"] jq = ctx.toolchains["@aspect_bazel_lib//lib:jq_toolchain_type"] - + sed = ctx.toolchains["@aspect_bazel_lib//lib:sed_toolchain_type"] launcher = ctx.actions.declare_file("image_%s.sh" % ctx.label.name) @@ -117,7 +119,10 @@ def _oci_image_impl(ctx): substitutions = { "{{registry_launcher_path}}": registry.registry_info.launcher.path, "{{crane_path}}": crane.crane_info.binary.path, + "{{coreutils_path}}": coreutils.coreutils_info.bin.path, + "{{grep_path}}": grep.grep_info.bin.path, "{{jq_path}}": jq.jqinfo.bin.path, + "{{sed_path}}": sed.sed_info.bin.path, "{{storage_dir}}": output.path, "{{empty_tar}}": ctx.file._empty_tar.path, }, @@ -196,7 +201,15 @@ def _oci_image_impl(ctx): outputs = [output], env = action_env, executable = util.maybe_wrap_launcher_for_windows(ctx, launcher), - tools = [crane.crane_info.binary, registry.registry_info.launcher, registry.registry_info.registry, jq.jqinfo.bin], + tools = [ + crane.crane_info.binary, + registry.registry_info.launcher, + registry.registry_info.registry, + coreutils.coreutils_info.bin, + grep.grep_info.bin, + jq.jqinfo.bin, + sed.sed_info.bin, + ], mnemonic = "OCIImage", progress_message = "OCI Image %{label}", ) @@ -215,6 +228,9 @@ oci_image = rule( "@bazel_tools//tools/sh:toolchain_type", "@rules_oci//oci:crane_toolchain_type", "@rules_oci//oci:registry_toolchain_type", + "@aspect_bazel_lib//lib:coreutils_toolchain_type", + "@aspect_bazel_lib//lib:grep_toolchain_type", "@aspect_bazel_lib//lib:jq_toolchain_type", + "@aspect_bazel_lib//lib:sed_toolchain_type", ], ) diff --git a/oci/private/image.sh.tpl b/oci/private/image.sh.tpl index ee7f6e30..124abe7e 100644 --- a/oci/private/image.sh.tpl +++ b/oci/private/image.sh.tpl @@ -7,16 +7,19 @@ set -o pipefail -o errexit -o nounset readonly REGISTRY_LAUNCHER="{{registry_launcher_path}}" readonly CRANE="{{crane_path}}" +readonly COREUTILS="{{coreutils_path}}" +readonly GREP="{{grep_path}}" +readonly SED="{{sed_path}}" readonly JQ="{{jq_path}}" readonly STORAGE_DIR="{{storage_dir}}" -readonly STDERR=$(mktemp) +readonly STDERR=$("${COREUTILS}" mktemp) silent_on_success() { stop_registry ${STORAGE_DIR} CODE=$? if [ "${CODE}" -ne 0 ]; then - cat "${STDERR}" >&1 + "${COREUTILS}" cat "${STDERR}" >&1 fi } trap "silent_on_success" EXIT @@ -26,12 +29,11 @@ function get_option() { shift for ARG in "$@"; do case "$ARG" in - ($name=*) echo ${ARG#$name=};; + ($name=*) echo ${ARG#$name=};; esac done } - function empty_base() { local registry=$1 local ref="$registry/oci/empty_base:latest" @@ -54,33 +56,33 @@ function empty_base() { function base_from_layout() { # TODO: https://github.com/google/go-containerregistry/issues/1514 - local refs=$(mktemp) - local output=$(mktemp) + local refs=$("${COREUTILS}" mktemp) + local output=$("${COREUTILS}" mktemp) local oci_layout_path=$1 local registry=$2 "${CRANE}" push "${oci_layout_path}" "${registry}/image:latest" --image-refs "${refs}" > "${output}" 2>&1 - if grep -q "MANIFEST_INVALID" "${output}"; then - cat >&2 << EOF + if "${GREP}" -q "MANIFEST_INVALID" "${output}"; then + "${COREUTILS}" cat >&2 << EOF -zot registry does not support docker manifests. +zot registry does not support docker manifests. crane registry does support both oci and docker images, but is more memory hungry. -If you want to use the crane registry, remove "zot_version" from "oci_register_toolchains". +If you want to use the crane registry, remove "zot_version" from "oci_register_toolchains". EOF exit 1 fi - cat "${refs}" + "${COREUTILS}" cat "${refs}" } # this will redirect stderr(2) to stderr file. { -source "${REGISTRY_LAUNCHER}" +source "${REGISTRY_LAUNCHER}" REGISTRY= REGISTRY=$(start_registry "${STORAGE_DIR}" "${STDERR}") @@ -129,8 +131,8 @@ for ARG in "$@"; do FIXED_ARGS+=("--entrypoint=$in") done <"${ARG#--entrypoint-file=}" ;; - (--exposed-ports-file=*) - while IFS= read -r in || [ -n "$in" ]; do + (--exposed-ports-file=*) + while IFS= read -r in || [ -n "$in" ]; do FIXED_ARGS+=("--exposed-ports=$in") done <"${ARG#--exposed-ports-file=}" ;; @@ -140,10 +142,10 @@ done REF=$("${CRANE}" "${FIXED_ARGS[@]}") -if [ ${#ENV_EXPANSIONS[@]} -ne 0 ]; then +if [ ${#ENV_EXPANSIONS[@]} -ne 0 ]; then env_expansion_filter=\ '[$raw | match("\\${?([a-zA-Z0-9_]+)}?"; "gm")] | reduce .[] as $match ( - {parts: [], prev: 0}; + {parts: [], prev: 0}; {parts: (.parts + [$raw[.prev:$match.offset], $envs[$match.captures[0].string]]), prev: ($match.offset + $match.length)} ) | .parts + [$raw[.prev:]] | join("")' base_config=$("${CRANE}" config "${REF}") @@ -160,9 +162,9 @@ fi if [ -n "$OUTPUT" ]; then "${CRANE}" pull "${REF}" "./${OUTPUT}" --format=oci --annotate-ref - mv "${OUTPUT}/index.json" "${OUTPUT}/temp.json" + "${COREUTILS}" mv "${OUTPUT}/index.json" "${OUTPUT}/temp.json" "${JQ}" --arg ref "${REF}" '.manifests |= map(select(.annotations["org.opencontainers.image.ref.name"] == $ref)) | del(.manifests[0].annotations)' "${OUTPUT}/temp.json" > "${OUTPUT}/index.json" - rm "${OUTPUT}/temp.json" + "${COREUTILS}" rm "${OUTPUT}/temp.json" "${CRANE}" layout gc "./${OUTPUT}" fi diff --git a/oci/private/registry/crane_launcher.sh.tpl b/oci/private/registry/crane_launcher.sh.tpl index 9b8a3608..22aab184 100644 --- a/oci/private/registry/crane_launcher.sh.tpl +++ b/oci/private/registry/crane_launcher.sh.tpl @@ -1,5 +1,7 @@ -readonly SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")" -readonly CRANE_REGISTRY_BIN="${SCRIPT_DIR}/{{CRANE}}" +# Ensure binaries are provided by calling script +: ${COREUTILS?} +: ${CRANE?} +: ${SED?} function start_registry() { local storage_dir="$1" @@ -7,14 +9,14 @@ function start_registry() { local deadline="${3:-5}" local registry_pid="$1/proc.pid" - mkdir -p "${storage_dir}" - "${CRANE_REGISTRY_BIN}" registry serve --disk="${storage_dir}" --address=localhost:0 >> $output 2>&1 & + "${COREUTILS}" mkdir -p "${storage_dir}" + "${CRANE}" registry serve --disk="${storage_dir}" --address=localhost:0 >> $output 2>&1 & echo "$!" > "${registry_pid}" - + local timeout=$((SECONDS+${deadline})) while [ "${SECONDS}" -lt "${timeout}" ]; do - local port=$(cat $output | sed -nr 's/.+serving on port ([0-9]+)/\1/p') + local port=$("${COREUTILS}" cat $output | "${SED}" -nr 's/.+serving on port ([0-9]+)/\1/p') if [ -n "${port}" ]; then break fi @@ -33,6 +35,6 @@ function stop_registry() { if [[ ! -f "${registry_pid}" ]]; then return 0 fi - kill -9 "$(cat "${registry_pid}")" || true + kill -9 "$("${COREUTILS}" cat "${registry_pid}")" || true return 0 } diff --git a/oci/private/tarball.bzl b/oci/private/tarball.bzl index acea15c0..ba1ed620 100644 --- a/oci/private/tarball.bzl +++ b/oci/private/tarball.bzl @@ -45,7 +45,7 @@ attrs = { used to load the image into the local engine when using `bazel run` on this oci_tarball. By default, we look for `docker` or `podman` on the PATH, and run the `load` command. - + > Note that rules_docker has an "incremental loader" which has better performance, see > Follow https://github.com/bazel-contrib/rules_oci/issues/454 for similar behavior in rules_oci. @@ -60,9 +60,9 @@ attrs = { default = Label("//oci/private:tarball_run.sh.tpl"), doc = """ \ The template used to load the container when using `bazel run` on this oci_tarball. - + See the `loader` attribute to replace the tool which is called. - Please reference the default template to see available substitutions. + Please reference the default template to see available substitutions. """, allow_single_file = True, ), @@ -73,13 +73,17 @@ attrs = { def _tarball_impl(ctx): image = ctx.file.image tarball = ctx.actions.declare_file("{}/tarball.tar".format(ctx.label.name)) - yq_bin = ctx.toolchains["@aspect_bazel_lib//lib:yq_toolchain_type"].yqinfo.bin + coreutils = ctx.toolchains["@aspect_bazel_lib//lib:coreutils_toolchain_type"] + tar = ctx.toolchains["@aspect_bazel_lib//lib:tar_toolchain_type"] + yq = ctx.toolchains["@aspect_bazel_lib//lib:yq_toolchain_type"] executable = ctx.actions.declare_file("{}/tarball.sh".format(ctx.label.name)) repo_tags = ctx.file.repo_tags substitutions = { "{{format}}": ctx.attr.format, - "{{yq}}": yq_bin.path, + "{{coreutils_path}}": coreutils.coreutils_info.bin.path, + "{{tar_path}}": tar.tarinfo.binary.path, + "{{yq_path}}": yq.yqinfo.bin.path, "{{image_dir}}": image.path, "{{tarball_path}}": tarball.path, } @@ -96,9 +100,14 @@ def _tarball_impl(ctx): ctx.actions.run( executable = util.maybe_wrap_launcher_for_windows(ctx, executable), + use_default_shell_env = True, inputs = [image, repo_tags, executable], outputs = [tarball], - tools = [yq_bin], + tools = [ + coreutils.coreutils_info.bin, + tar.tarinfo.binary, + yq.yqinfo.bin, + ], mnemonic = "OCITarball", progress_message = "OCI Tarball %{label}", ) @@ -128,6 +137,8 @@ oci_tarball = rule( doc = doc, toolchains = [ "@bazel_tools//tools/sh:toolchain_type", + "@aspect_bazel_lib//lib:coreutils_toolchain_type", + "@aspect_bazel_lib//lib:tar_toolchain_type", "@aspect_bazel_lib//lib:yq_toolchain_type", ], executable = True, diff --git a/oci/private/tarball.sh.tpl b/oci/private/tarball.sh.tpl index 2b5052f4..4c30ae71 100644 --- a/oci/private/tarball.sh.tpl +++ b/oci/private/tarball.sh.tpl @@ -1,23 +1,26 @@ #!/usr/bin/env bash set -o pipefail -o errexit -o nounset +readonly YQ="{{yq_path}}" +readonly COREUTILS="{{coreutils_path}}" +readonly TAR="{{tar_path}}" + readonly FORMAT="{{format}}" -readonly STAGING_DIR=$(mktemp -d) -readonly YQ="{{yq}}" +readonly STAGING_DIR=$("${COREUTILS}" mktemp -d) readonly IMAGE_DIR="{{image_dir}}" readonly BLOBS_DIR="${STAGING_DIR}/blobs" readonly TARBALL_PATH="{{tarball_path}}" -readonly REPOTAGS=($(cat "{{tags}}")) +readonly REPOTAGS=($("${COREUTILS}" cat "{{tags}}")) readonly INDEX_FILE="${IMAGE_DIR}/index.json" cp_f_with_mkdir() { SRC="$1" DST="$2" - mkdir -p "$(dirname "${DST}")" - cp -f "${SRC}" "${DST}" + "${COREUTILS}" mkdir -p "$("${COREUTILS}" dirname "${DST}")" + "${COREUTILS}" cp -f "${SRC}" "${DST}" } -MANIFEST_DIGEST=$(${YQ} eval '.manifests[0].digest | sub(":"; "/")' "${INDEX_FILE}" | tr -d '"') +MANIFEST_DIGEST=$(${YQ} eval '.manifests[0].digest | sub(":"; "/")' "${INDEX_FILE}" | "${COREUTILS}" tr -d '"') MANIFESTS_LENGTH=$("${YQ}" eval '.manifests | length' "${INDEX_FILE}") if [[ "${MANIFESTS_LENGTH}" != 1 ]]; then @@ -112,7 +115,7 @@ if [[ "${FORMAT}" == "oci" ]]; then # } repo_tags="${REPOTAGS[@]}" "${YQ}" -o json eval "(.manifests = ${MANIFEST_COPIES}) *d {\"manifests\": (env(repo_tags) | split \" \" | map {\"annotations\": {\"org.opencontainers.image.ref.name\": .}})}" "${INDEX_FILE}" > "${STAGING_DIR}/index.json" - tar -C "${STAGING_DIR}" -cf "${TARBALL_PATH}" index.json blobs oci-layout + "${TAR}" -C "${STAGING_DIR}" -cf "${TARBALL_PATH}" index.json blobs oci-layout exit 0 fi @@ -137,4 +140,4 @@ layers="${LAYERS}" \ --output-format json > "${STAGING_DIR}/manifest.json" # TODO: https://github.com/bazel-contrib/rules_oci/issues/217 -tar -C "${STAGING_DIR}" -cf "${TARBALL_PATH}" manifest.json blobs +"${TAR}" -C "${STAGING_DIR}" -cf "${TARBALL_PATH}" manifest.json blobs