Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dev kmr2 #510

Merged
merged 2 commits into from
Jul 15, 2024
Merged

Dev kmr2 #510

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4a28faa
update the remotehosts.py and endpoints.py code to properly handle va…
k-rister Jul 11, 2024
a26ba36
update a quay API call to use onlyActiveTags instead of limit
k-rister Jul 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 25 additions & 6 deletions endpoints/endpoints.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -797,24 +797,43 @@ def gmtimestamp_to_gmepoch(gmtimestamp):

return gmepoch

def image_expiration_gmepoch():
def image_created_expiration_gmepoch(weeks):
"""
Determine the UTC epoch timetamp that any image created before is expired

Args:
None
weeks (int): The number of weeks to consider as the expiration time post creation

Globals:
None

Returns:
gmepoch (int): A UTC epoch timestamp such as 1712949457 from 2 weeks ago. Any image with a creation
gmepoch (int): A UTC epoch timestamp such as 1712949457 from X weeks ago. Any image with a creation
date older than this is expired
"""
# seconds/min minutes/hour hours/day days
two_weeks = 60 * 60 * 24 * 14
# seconds/min minutes/hour hours/day days/week weeks
delta = 60 * 60 * 24 * 7 * weeks

gmepoch = calendar.timegm(time.gmtime()) - delta

return gmepoch

gmepoch = calendar.timegm(time.gmtime()) - two_weeks
def image_expiration_gmepoch():
"""
Return the UTC epoch timestamp that any image expiring before is expired

Args:
None

Globals:
None


Returns:
gmepoch (int): The current UTC epoch timestamp such as 1712949457. Any image with an
expiration date older than this is expired
"""
gmepoch = calendar.timegm(time.gmtime())

return gmepoch

Expand Down
52 changes: 48 additions & 4 deletions endpoints/remotehosts/remotehosts.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from pathlib import Path
import queue
import re
import requests
import sys
import tempfile
import threading
Expand Down Expand Up @@ -1931,7 +1932,7 @@ def remote_image_manager(thread_name, remote_name, connection, image_max_cache_s
in order to cache a complete set of images for a run)

Globals:
None
settings (dict): the one data structure to rule then all

Returns:
None
Expand All @@ -1945,6 +1946,7 @@ def remote_image_manager(thread_name, remote_name, connection, image_max_cache_s
images = dict()
images["rickshaw"] = dict()
images["podman"] = dict()
images["quay"] = dict()

result = endpoints.run_remote(connection, "cat /var/lib/crucible/remotehosts-container-image-census")
if result.exited != 0:
Expand Down Expand Up @@ -1985,8 +1987,38 @@ def remote_image_manager(thread_name, remote_name, connection, image_max_cache_s
}
thread_logger(thread_name, "images[podman]:\n%s" % (endpoints.dump_json(images["podman"])), remote_name = remote_name, log_prefix = log_prefix)

if settings["rickshaw"]["quay"]["refresh-expiration"]["api-url"] is not None:
thread_logger(thread_name, "Found configuration information necessary to utilize the quay API to obtain image expiration", remote_name = remote_name, log_prefix = log_prefix)
thread_logger(thread_name, "Quay API URL: %s" % (settings["rickshaw"]["quay"]["refresh-expiration"]["api-url"]), remote_name = remote_name, log_prefix = log_prefix)

for image in images["podman"].keys():
image_parts = image.split(":")

get_request = requests.get(settings["rickshaw"]["quay"]["refresh-expiration"]["api-url"] + "/tag", params = { "onlyActiveTags": True, "specificTag": image_parts[1] })

query_log_level = "info"
if get_request.status_code != requests.codes.ok:
query_log_level = "warning"

thread_logger(thread_name, "Quay API query for %s returned %d" % (image, get_request.status_code), log_level = query_log_level, remote_name = remote_name, log_prefix = log_prefix)

if get_request.status_code == requests.codes.ok:
image_json = get_request.json()
if len(image_json["tags"]) == 1:
images["quay"][image] = image_json["tags"][0]
else:
thread_logger(thread_name, "Quay API query for %s found %d tags" % (image, len(image_json["tags"])), log_level = "warning", remote_name = remote_name, log_prefix = log_prefix)

thread_logger(thread_name, "images[quay]:\n%s" % (endpoints.dump_json(images["quay"])), remote_name = remote_name, log_prefix = log_prefix)
else:
thread_logger(thread_name, "Configuration information necessary to utilize the quay API to obtain image expiration timestamps is not available", remote_name = remote_name, log_prefix = log_prefix)

image_expiration = endpoints.image_expiration_gmepoch()
thread_logger(thread_name, "Images created before %d will be considered expired" % (image_expiration), remote_name = remote_name, log_prefix = log_prefix)
thread_logger(thread_name, "Images evaludated by their expiration data will be considered expired if it is before %d" % (image_expiration), remote_name = remote_name, log_prefix = log_prefix)

expiration_weeks = int(settings["rickshaw"]["quay"]["image-expiration"].rstrip("w"))
k-rister marked this conversation as resolved.
Show resolved Hide resolved
image_created_expiration = endpoints.image_created_expiration_gmepoch(expiration_weeks)
thread_logger(thread_name, "Images evaludated by their creation data will be considered expired if it is before %d (%d weeks ago)" % (image_created_expiration, expiration_weeks), remote_name = remote_name, log_prefix = log_prefix)

deletes = []
for image in images["rickshaw"].keys():
Expand Down Expand Up @@ -2020,12 +2052,24 @@ def remote_image_manager(thread_name, remote_name, connection, image_max_cache_s
thread_logger(thread_name, "Podman image '%s' is not present in rickshaw container image census, removing it from the image cache" % (image), remote_name = remote_name, log_prefix = log_prefix)
deletes["podman"].append(image)
remove_image(thread_name, remote_name, log_prefix, connection, image)
elif images["podman"][image]["created"] < image_expiration:
thread_logger(thread_name, "Podman image '%s' has expired, removing it from the image cache" % (image), remote_name = remote_name, log_prefix = log_prefix)

if image in images["quay"]:
if images["quay"][image]["end_ts"] < image_expiration:
thread_logger(thread_name, "Podman image '%s' has been evaluated based on it's expiration data and has expired, removing it from the image cache" % (image), remote_name = remote_name, log_prefix = log_prefix)
deletes["podman"].append(image)
deletes["rickshaw"].append(image)
remove_image(thread_name, remote_name, log_prefix, connection, image)
else:
thread_logger(thread_name, "Podman image '%s' has been evaluated based on it's expiration data and has not expired" % (image), remote_name = remote_name, log_prefix = log_prefix)
elif images["podman"][image]["created"] < image_created_expiration:
k-rister marked this conversation as resolved.
Show resolved Hide resolved
thread_logger(thread_name, "Podman image '%s' has been evaluated based on it's creation data and has expired, removing it from the image cache" % (image), remote_name = remote_name, log_prefix = log_prefix)
deletes["podman"].append(image)
deletes["rickshaw"].append(image)
remove_image(thread_name, remote_name, log_prefix, connection, image)
else:
thread_logger(thread_name, "Podman image '%s' has been evaluated based on it's creation data and has not expired" % (image), remote_name = remote_name, log_prefix = log_prefix)

if not image in deletes["podman"]:
thread_logger(thread_name, "Podman image '%s' is valid and remains under consideration" % (image), remote_name = remote_name, log_prefix = log_prefix)
for kind in deletes.keys():
for image in deletes[kind]:
Expand Down
2 changes: 1 addition & 1 deletion rickshaw-run
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1073,7 +1073,7 @@ sub source_container_image {
my $query_cmd = 'curl --silent' .
' -X GET -H "Authorization: Bearer ' . $quay_refresh_expiration_token . '"' .
' "' . $quay_refresh_expiration_api_url .
'/tag/?limit=1&specificTag=' . $workshop_args[$x]{'tag'} . '"';
'/tag/?onlyActiveTags=true&specificTag=' . $workshop_args[$x]{'tag'} . '"';

($query_cmd, my $query_status, my $query_rc) = run_cmd($query_cmd);
chomp($query_status);
Expand Down
Loading