gc: merge container-prune into cloud-prune

Merged the code of the container gc into the cloud one, and update builds.json. Go through the tags
in base-oscontainer data in meta.json and prune every tag except the stream-name itself which
are moving tags.

cmd/coreos-assembler.go

@@ -16,7 +16,7 @@ var buildCommands = []string{"init", "fetch", "build", "run", "prune", "clean",
 var advancedBuildCommands = []string{"buildfetch", "buildupload", "oc-adm-release", "push-container"}
 var buildextendCommands = []string{"aliyun", "applehv", "aws", "azure", "digitalocean", "exoscale", "extensions-container", "gcp", "hashlist-experimental", "hyperv", "ibmcloud", "kubevirt", "live", "metal", "metal4k", "nutanix", "openstack", "qemu", "secex", "virtualbox", "vmware", "vultr"}
 
-var utilityCommands = []string{"aws-replicate", "cloud-prune", "compress", "container-prune", "copy-container", "koji-upload", "kola", "push-container-manifest", "remote-build-container", "remote-session", "sign", "tag", "update-variant"}
+var utilityCommands = []string{"aws-replicate", "cloud-prune", "compress", "copy-container", "koji-upload", "kola", "push-container-manifest", "remote-build-container", "remote-session", "sign", "tag", "update-variant"}
 var otherCommands = []string{"shell", "meta"}
 
 func init() {

src/cmd-cloud-prune

@@ -34,8 +34,10 @@
 
 import argparse
 import json
+import subprocess
 from urllib.parse import urlparse
 import pytz
+import requests
 import yaml
 import collections
 import datetime
@@ -59,6 +61,8 @@ CACHE_MAX_AGE_METADATA = 60 * 5
 # is up to date.
 SUPPORTED = ["amis", "gcp"]
 UNSUPPORTED = ["aliyun", "azure", "ibmcloud", "powervs"]
+# list of known streams with containers
+STREAMS = {"next", "testing", "stable", "next-devel", "testing-devel", "rawhide", "branched"}
 
 
 def parse_args():
@@ -70,6 +74,9 @@ def parse_args():
     parser.add_argument("--gcp-json-key", help="GCP Service Account JSON Auth", default=os.environ.get("GCP_JSON_AUTH"))
     parser.add_argument("--acl", help="ACL for objects", action='store', default='private')
     parser.add_argument("--aws-config-file", default=os.environ.get("AWS_CONFIG_FILE"), help="Path to AWS config file")
+    parser.add_argument("--repository-url", help="container images URL")
+    parser.add_argument("--registry-auth-file", default=os.environ.get("REGISTRY_AUTH_FILE"),
+                        help="Path to docker registry auth file. Directly passed to skopeo.")
     return parser.parse_args()
 
 
@@ -125,7 +132,7 @@ def main():
             current_build = Build(id=build_id, images=images, arch=arch, meta_json=meta_json)
 
             # Iterate over actions (policy types) to apply pruning
-            for action in ['cloud-uploads', 'images', 'build']:
+            for action in ['cloud-uploads', 'images', 'build', 'containers']:
                 if action not in policy[stream]:
                     continue
                 action_duration = convert_duration_to_days(policy[stream][action])
@@ -162,6 +169,19 @@ def main():
                         case "build":
                             prune_build(s3_client, bucket, prefix, build_id, args.dry_run)
                             pruned_build_ids.append(build_id)
+                        case "containers":
+                            container_tags = get_container_tags(meta_json, stream)
+                            if container_tags:
+                                containers_config = {
+                                    "container_tags": container_tags,
+                                    "dry_run": args.dry_run,
+                                    "repository_url": args.repository_url,
+                                    "registry_auth_file": args.registry_auth_file,
+                                    "stream": stream
+                                }
+                                prune_containers(containers_config)
+                            else:
+                                print(f"No container tags to prune for build {build_id} on architecture {arch}.")
 
         # Update policy-cleanup after pruning actions for the architecture
         policy_cleanup = build.setdefault("policy-cleanup", {})
@@ -174,6 +194,9 @@ def main():
                     if "images" not in policy_cleanup:
                         policy_cleanup["images"] = True
                     policy_cleanup["images-kept"] = images_to_keep
+                case "containers":
+                    if "containers" not in policy_cleanup:
+                        policy_cleanup["containers"] = True
 
     if pruned_build_ids:
         if "tombstone-builds" not in builds_json_data:
@@ -414,5 +437,72 @@ def prune_build(s3_client, bucket, prefix, build_id, dry_run):
                 raise Exception(f"Error pruning {build_id}: {e.response['Error']['Message']}")
 
 
+def get_container_tags(meta_json, stream):
+    container_tags = []
+    base_oscontainer = meta_json.get("base-oscontainer")
+    if base_oscontainer:
+        tags = base_oscontainer.get("tags", [])
+        # Only include tags that do not match the stream i.e. moving tags
+        filtered_tags = [tag for tag in tags if tag != stream]
+        if filtered_tags:
+            container_tags = filtered_tags
+    return container_tags
+
+
+def prune_containers(containers_config):
+    barrier_releases = set()
+    # Get the update graph for stable streams
+    if containers_config.stream in ['stable', 'testing', 'next']:
+        update_graph = get_update_graph(containers_config.stream)['releases']
+        # Keep only the barrier releases
+        barrier_releases = set([release["version"] for release in update_graph if "barrier" in release])
+
+    for tag in containers_config.container_tags:
+        if tag in STREAMS:
+            continue
+        if tag in barrier_releases:
+            print(f"Release {tag} is a barrier release, keeping.")
+            continue
+        if containers_config.dry_run:
+            print(f"Would prune image {containers_config.repository_url}:{tag}")
+        else:
+            skopeo_delete(containers_config.repository_url, tag, containers_config.registry_auth_file)
+
+
+def get_update_graph(stream):
+    url = f"https://builds.coreos.fedoraproject.org/updates/{stream}.json"
+    r = requests.get(url, timeout=5)
+    if r.status_code != 200:
+        raise Exception(f"Could not download update graph for {stream}. HTTP {r.status_code}")
+    return r.json()
+
+
+def skopeo_inspect(repo, image, auth):
+    skopeo_args = ["skopeo", "inspect", f"docker://{repo}:{image}"]
+    if auth:
+        skopeo_args.extend(["--authfile", auth])
+    try:
+        subprocess.check_output(skopeo_args, stderr=subprocess.STDOUT)
+        return True  # Inspection succeeded
+    except subprocess.CalledProcessError as e:
+        print("Inspection failed:", e.output.decode("utf-8"))
+        return False  # Inspection failed
+
+
+def skopeo_delete(repo, image, auth):
+    if skopeo_inspect(repo, image, auth):  # Only proceed if inspection succeeds
+        skopeo_args = ["skopeo", "delete", f"docker://{repo}:{image}"]
+        if auth:
+            skopeo_args.extend(["--authfile", auth])
+
+        try:
+            subprocess.check_output(skopeo_args, stderr=subprocess.STDOUT)
+            print("Image deleted successfully.")
+        except subprocess.CalledProcessError as e:
+            raise Exception("An error occurred during deletion:", e.output.decode("utf-8"))
+    else:
+        raise Exception("Skipping delete as skopeo inspection failed.")
+
+
 if __name__ == "__main__":
     main()