wip

examples/deb/BUILD.bazel

@@ -12,6 +12,7 @@ _ARCH = [
         name = "decompress_" + architecture,
         srcs = ["@bash_{}//:layer".format(architecture)],
         outs = ["_{}.tar".format(architecture)],
+        tags = ["manual"],
         cmd = "xz --decompress --stdout $< >$@",
     )
     for architecture in _ARCH
@@ -22,6 +23,7 @@ _ARCH = [
         name = "image_" + architecture,
         architecture = architecture,
         os = "linux",
+        tags = ["manual"],
         tars = ["_{}.tar".format(architecture)],
     )
     for architecture in _ARCH
@@ -30,5 +32,6 @@ _ARCH = [
 oci_tarball(
     name = "tarball",
     image = ":image_amd64",
+    tags = ["manual"],
     repo_tags = ["test:test"],
 )

oci/private/image.bzl

@@ -109,6 +109,8 @@ def _oci_image_impl(ctx):
 
     launcher = ctx.actions.declare_file("image_%s.sh" % ctx.label.name)
 
+    output = ctx.actions.declare_directory(ctx.label.name)
+
     ctx.actions.expand_template(
         template = ctx.file._image_sh_tpl,
         output = launcher,
@@ -117,7 +119,7 @@ def _oci_image_impl(ctx):
             "{{registry_launcher_path}}": registry.registry_info.launcher.path,
             "{{crane_path}}": crane.crane_info.binary.path,
             "{{jq_path}}": jq.jqinfo.bin.path,
-            "{{storage_dir}}": "/".join([ctx.bin_dir.path, ctx.label.package, "storage_%s" % ctx.label.name]),
+            "{{storage_dir}}": output.path,
             "{{empty_tar}}": ctx.file._empty_tar.path,
         },
     )
@@ -171,7 +173,7 @@ def _oci_image_impl(ctx):
         args.add(ctx.file.annotations.path, format = "--annotations-file=%s")
         inputs_depsets.append(depset([ctx.file.annotations]))
 
-    output = ctx.actions.declare_directory(ctx.label.name)
+
     args.add(output.path, format = "--output=%s")
 
     action_env = {}

oci/private/image.sh.tpl

@@ -6,7 +6,7 @@ set -o pipefail -o errexit -o nounset
 # NB: --output argument is an option only understood by this wrapper and will pull artifact image into a oci layout.
 
 readonly REGISTRY_LAUNCHER="{{registry_launcher_path}}"
-readonly CRANE="{{crane_path}}"
+readonly CRANE="/Users/thesayyn/Documents/go-containerregistry/main"
 readonly JQ="{{jq_path}}"
 readonly STORAGE_DIR="{{storage_dir}}"
 
@@ -33,7 +33,7 @@ function get_option() {
 
 function empty_base() {
     local registry=$1
-    local ref="$registry/oci/empty_base:latest"
+    local ref="$registry/image:latest"
     ref="$("${CRANE}" append --oci-empty-base -t "${ref}" -f {{empty_tar}})"
     ref=$("${CRANE}" config "${ref}" | "${JQ}"  ".rootfs.diff_ids = [] | .history = []" | "${CRANE}" edit config "${ref}")
     ref=$("${CRANE}" manifest "${ref}" | "${JQ}"  ".layers = []" | "${CRANE}" edit manifest "${ref}")
@@ -58,7 +58,7 @@ function base_from_layout() {
     local oci_layout_path=$1
     local registry=$2
 
-    "${CRANE}" push "${oci_layout_path}" "${registry}/oci/layout:latest" --image-refs "${refs}" > "${output}" 2>&1
+    "${CRANE}" push "${oci_layout_path}" "${registry}/image:latest" --image-refs "${refs}" > "${output}" 2>&1
 
     echo "${output}" >&2
 
@@ -79,6 +79,23 @@ EOF
     cat "${refs}"
 }
 
+# removes unreferenced blobs from oci-layout
+function gc() {
+    local ref="$1"
+    local digest=$("${CRANE}" digest "${ref}")
+    local blobs=($("${CRANE}" manifest "${ref}" | "${JQ}" -r --arg digest "$digest" '([.layers[].digest] + [.config.digest, $digest]) | flatten | .[]'))
+    for blob_dir in ${STORAGE_DIR}/blobs/* ; do
+        local algo="$(basename ${blob_dir})"
+        for blob_path in ${blob_dir}/* ; do
+            local blob_digest="$(basename ${blob_path})"
+            local hash="${algo}:${blob_digest}"
+            if ! [[ "${blobs[@]}" =~ "$hash" ]]; then
+                rm $blob_path
+            fi
+        done
+    done
+}
+
 # this will redirect stderr(2) to stderr file.
 {
 source "${REGISTRY_LAUNCHER}"
@@ -161,7 +178,11 @@ if [ -n "${WORKDIR}" ]; then
 fi
 
 if [ -n "$OUTPUT" ]; then
-    "${CRANE}" pull "${REF}" "./${OUTPUT}" --format=oci
+    # "${CRANE}" pull "${REF}" "./${OUTPUT}" --format=oci
+    # gc "${REF}"
+    stop_registry "${STORAGE_DIR}"
+    cat $STDERR
 fi
 
-} 2>> "${STDERR}"
+} 
+# 2>> "${STDERR}"

oci/private/registry/crane_launcher.sh.tpl

@@ -1,12 +1,16 @@
 readonly SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
-readonly CRANE_REGISTRY_BIN="${SCRIPT_DIR}/{{CRANE}}"
+readonly CRANE_REGISTRY_BIN="/Users/thesayyn/Documents/go-containerregistry/main"
 
 function start_registry() {
-    local __unused_storage_dir="$1"
+    local storage_dir="$1/blobs"
     local output="$2"
     local deadline="${3:-5}"
 
-    "${CRANE_REGISTRY_BIN}" registry serve>> $output 2>&1 &
+    mkdir -p "$storage_dir"
+
+    # --blobs-to-disk uses go's os.TempDir() function which is equal to TMPDIR under *nix.
+    # TODO: windows!!!!!!!!
+    TMPDIR="${storage_dir}" "${CRANE_REGISTRY_BIN}" registry serve --blobs-to-disk >> $output 2>&1 &
 
     local timeout=$((SECONDS+${deadline}))
 
@@ -22,4 +26,8 @@ function start_registry() {
     fi
     echo "127.0.0.1:${port}"
     return 0
+}
+
+function stop_registry() { 
+    :
 }

oci/private/registry/zot_launcher.sh.tpl

@@ -5,14 +5,15 @@ function start_registry() {
     local storage_dir="$1"
     local output="$2"
     local deadline="${3:-5}"
-    local config_path="$1/config.json"
+    local config_path="$storage_dir/config.json"
 
     mkdir -p "${storage_dir}"
+    echo "$storage_dir" >&2
     cat > "${config_path}" <<EOF
 {
-    "storage": {"rootDirectory": "$1" },
-    "http": { "port": "0", "address": "127.0.0.1" },
-    "log": { "level": "info" }
+    "storage":{ "rootDirectory": "$storage_dir/all",  "dedupe": false, "subPaths": {"/image": {"rootDirectory": "$storage_dir"}}},
+    "http":{ "port": "0", "address": "127.0.0.1" },
+    "log":{ "level": "debug" }
 }
 EOF
     "${ZOT}" serve "${config_path}" >> $output 2>&1 &
@@ -31,4 +32,9 @@ EOF
     fi
     echo "127.0.0.1:${port}"
     return 0
+}
+
+function stop_registry() {
+    local storage_dir="$1"
+    rm "$storage_dir/config.json"
 }

oci/private/versions.bzl

@@ -2,43 +2,20 @@
 
 # WARNING: only v0.12.0 and above is supported.
 CRANE_VERSIONS = {
-    "v0.15.2": {
-        "darwin_arm64": "sha256-XPquVFIWvxd6ZKhSo/HxI7lYBDjOYIoOnNy02uK6F0c=",
-        "darwin_x86_64": "sha256-xzpmoZ4cwx0jKP07UvMZgZqb9sOKOA3/TfTeq6AsFHc=",
-        "linux_arm64": "sha256-r6C/VtlfqG/nHV2tteA5YLfAWG7qhCcMykC0Gm1h+7I=",
-        "linux_armv6": "sha256-xCwHYjAb2Gy+VPJbbwZJmLxI2FEnSRi1rxBmmcgzeeM=",
-        "linux_i386": "sha256-auD3v7isyLJTunFiDoteFyrDyA6eEV8s+wL0QbhIzGI=",
-        "linux_ppc64le": "sha256-SJh/KqiTi0sM2AJHxV+rTn5Is5WOsfLo5H4wSb/GkUI=",
-        "linux_s390x": "sha256-66RccOhj0T7NC+BGg6mgW1FvnDgXC1KV7sVM1BcA7xw=",
-        "linux_x86_64": "sha256-vV9yrpY3OsZAZ5phCCgLbXZph3PKIfKTrjDMF0E+KtE=",
-        "windows_arm64": "sha256-F/rbrozOPGEVeb/KEKJ/T3B0n26rvZ3/VBs2amTBz3c=",
-        "windows_armv6": "sha256-7vBk/IKCO1j5uECE32eZmrQAbAnzbxeNgye0BiqoLs4=",
-        "windows_x86_64": "sha256-1y42OZTiJt2C7of8xeqcwy2kiFeylXLf6+BvNS2Y13Y=",
-    },
-    "v0.14.0": {
-        "darwin_arm64": "sha256-8g+3rziZKW/OK2YPTHUWmvFDUhX6/zXInY/tVZaOb2E=",
-        "darwin_x86_64": "sha256-h+vdsXSnfez8MUgLMPLLg7tVqeyyRSxSJBatC/+yPRY=",
-        "linux_arm64": "sha256-8QeMgjli2vFNI9QBVKZVELU2TompycffP5kfGen+tAE=",
-        "linux_armv6": "sha256-geo5WbuD4mq8/ay2veV3XGj03AoMLbwIB4CR4kcKKG4=",
-        "linux_i386": "sha256-aLzfhkWKUXfIJ7u4IJ9tg54Gu4aEBwJAkyS/1ZHVT/Q=",
-        "linux_s390x": "sha256-L2aEmQerb9+ZEEwg5ZikzJjzpC2bu6kw9C8J4kOwRMc=",
-        "linux_x86_64": "sha256-W2j+D3/pFwuj4Wd/R+z8iSxoei/cvCsd45q16WY95dc=",
-        "windows_arm64": "sha256-F0gUInC2psNFGV68DhSnn8AalX/wOHhd5SH0RAixYpg=",
-        "windows_armv6": "sha256-WoSZPcKgphu9RBaMrY1huXaU0JnTh2FrKoV5KQbXHzk=",
-        "windows_x86_64": "sha256-0ZbFoXN6Deo3yVLoHlygcn4hGXvXMF8iVradTlXGJNA=",
-    },
-    "v0.12.0": {
-        "darwin_arm64": "sha256-JGUx0myrqV3ZPktKiFrvn6SdhOBAmG61mCqjxYQnEa8=",
-        "darwin_x86_64": "sha256-iO6yCJMD76cJWW5qncXShfaX/QT+6Q+DWURxo9WpQxs=",
-        "linux_arm64": "sha256-6UOCFY2oV0qOEhyvo2imibV4yYVVXw11Y8Rn74pbXYQ=",
-        "linux_armv6": "sha256-/f3JPlgHttVu/xaklKAAyXCRoAZKi1RZHQf4EdegTj0=",
-        "linux_i386": "sha256-Xv0VU3BXuSQcLgxsirsls/CEJeDRuhqBDhDRt+P2iss=",
-        "linux_s390x": "sha256-ioK8HWp0shvCtmrIobiLZA3ROGuePHy8XPN3ZRvc0QQ=",
-        "linux_x86_64": "sha256-u1K+/Ch5/zmMd+n0cAXabTj9mowFU21sRd1PgEC1p5o=",
-        "windows_arm64": "sha256-qxJw7Bfk45qOiFGLWMuG7zZahLKN2necKcqUyDUEtbw=",
-        "windows_armv6": "sha256-YKRi+clqdDlbYukFiupLgas3OdT7P2s2O6NuQ7jb6PM=",
-        "windows_x86_64": "sha256-SqbfgWZ+Ee9ve6F/0dkQ3YnRf8s4gFL42ATNezhhXDk=",
-    },
+  "v0.16.1": {
+    "go-containerregistry-0": "sha256-KHa1NgQVE4cW1YN6aeHBFhgOSOz2KmP1MBw1HVoS+Zs=",
+    "darwin_arm64": "sha256-OgSfRI2SluHc01ZsWALiQbzU4Yc/mYoSKCRlXiDg10Q=",
+    "darwin_x86_64": "sha256-0AynKTFcu5FKFFQl2y49eoFsXB+DVFlxiBV64heyN34=",
+    "linux_arm64": "sha256-niJneTd6IHE/FqyU3sn2wX9piDTqQMM6bIb53ohFcis=",
+    "linux_armv6": "sha256-submkJ8j5DTlRA0E8om9KQ9+mMJBQotG0BAsYpn6NQ8=",
+    "linux_i386": "sha256-XXZ5/2Oz1poXNIbY9IOomRt2IDPAKN689md3nU5LqOU=",
+    "linux_ppc64le": "sha256-p7xthOifRHjqDB8owPkVlkAJJVUKYWeKQhIvtx9e/z8=",
+    "linux_s390x": "sha256-7yLdSqxExI4cSRIbQJaPfILy0Y+7Ze6fNmQpLb9abWk=",
+    "linux_x86_64": "sha256-EV3ITRTFrcicFuP6KX6U8GqexJK7Hccw2mJIULd8m+I=",
+    "windows_arm64": "sha256-H4CfiYT15fh9E9VRTeLYlv3/Ys+1Rnr2Gp1xAqFtpWE=",
+    "windows_armv6": "sha256-j6Oa4+kBYGOub0jlMSfUGNtpoCR0G/kACh4YB456PGE=",
+    "windows_x86_64": "sha256-iU9Nm6K8+07QE/f72n+WL3yS2+84/dFy1GdgdfTopRk="
+  }
 }
 
 ZOT_VERSIONS = {