From ba2725c6d22de613c98d0b21dce0b933fdcae597 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 16 Jan 2024 16:55:44 -0500 Subject: [PATCH 1/3] coreos-koji-tagger: minor refactoring prep No functional change here. Just some renaming and breaking out some functions. Prep for future patches. --- coreos-koji-tagger/coreos_koji_tagger.py | 28 ++++++++++++++++++------ 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/coreos-koji-tagger/coreos_koji_tagger.py b/coreos-koji-tagger/coreos_koji_tagger.py index 4bca0df..8d93b93 100755 --- a/coreos-koji-tagger/coreos_koji_tagger.py +++ b/coreos-koji-tagger/coreos_koji_tagger.py @@ -50,7 +50,7 @@ # We are processing the org.fedoraproject.prod.github.push topic # https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.github.push&delta=100000 -EXAMPLE_MESSAGE_BODY = json.loads(""" +EXAMPLE_GITHUB_PUSH_MESSAGE_BODY = json.loads(""" { "forced": false, "compare": "https://github.com/coreos/fedora-coreos-config/compare/d6c02b5cd107...6a53f43af882", @@ -269,7 +269,7 @@ def __init__(self): # do an initial run on startup in case we're out of sync for branch in self.github_repo_branches: - self.process_lockfiles(branch) + self.process_git_lockfiles(branch) def __call__(self, message: fedora_messaging.api.Message): @@ -280,6 +280,15 @@ def process_message(self, message: fedora_messaging.api.Message): logger.debug(message.topic) logger.debug(message.body) + if message.topic == 'org.fedoraproject.prod.github.push': + self.process_github_push_message(message) + else: + # that's weird, we shouldn't have been called for any other topic + # just log it and ignore + logger.info(f'Ignoring message on topic: {message.topic}') + return + + def process_github_push_message(self, message: fedora_messaging.api.Message): # Grab the raw message body and the status from that msg = message.body branch = msg['ref'] @@ -309,10 +318,10 @@ def process_message(self, message: fedora_messaging.api.Message): if self.keytab_file: self.koji_client.getLoggedInUser() - self.process_lockfiles(commit) + self.process_git_lockfiles(commit) @catch_exceptions_and_continue - def process_lockfiles(self, rev): + def process_git_lockfiles(self, rev): # Now grab lockfile data from the commit we should operate on: desiredrpms = set() @@ -341,6 +350,10 @@ def archify(f): logger.warning('Continuing...') return + self.tag_rpms(desiredrpms) + + def tag_rpms(self, desiredrpms): + # NOMENCLATURE: # # In koji there is the concept of a pkg and a build. A pkg @@ -679,10 +692,11 @@ def side_effect(*args, **kwargs): return requests_response requests.get = Mock(side_effect=side_effect) - # Note that the following will call process_lockfiles twice. Once + c = Consumer() + + # Note that the following will call process_git_lockfiles twice. Once # for startup and once for the fake message we're passing. m = fedora_messaging.api.Message( topic = 'org.fedoraproject.prod.github.push', - body = EXAMPLE_MESSAGE_BODY) - c = Consumer() + body = EXAMPLE_GITHUB_PUSH_MESSAGE_BODY) c.__call__(m) From 9f3c01368c2d3cbded6271c6a3e1e1e33ed1b981 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 16 Jan 2024 17:05:52 -0500 Subject: [PATCH 2/3] coreos-koji-tagger: move koji auth workaround deeper I.e. put it at the point where we know we can't no op and we need Koji credentials to continue. Prep for adding another caller for `tag_rpms()`. --- coreos-koji-tagger/coreos_koji_tagger.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/coreos-koji-tagger/coreos_koji_tagger.py b/coreos-koji-tagger/coreos_koji_tagger.py index 8d93b93..52e91c0 100755 --- a/coreos-koji-tagger/coreos_koji_tagger.py +++ b/coreos-koji-tagger/coreos_koji_tagger.py @@ -311,13 +311,6 @@ def process_github_push_message(self, message: fedora_messaging.api.Message): logger.error('No commit id in message!') return - # Check the login. If it's bad then a koji.AuthError exception will get thrown. - # and we'll exit because catch_exceptions_and_continue() will raise it. - # This is the only way we've come up with to deal with the problem in - # https://github.com/coreos/fedora-coreos-releng-automation/issues/70 - if self.keytab_file: - self.koji_client.getLoggedInUser() - self.process_git_lockfiles(commit) @catch_exceptions_and_continue @@ -354,6 +347,13 @@ def archify(f): def tag_rpms(self, desiredrpms): + # Check the login. If it's bad then a koji.AuthError exception will get thrown. + # and we'll exit because catch_exceptions_and_continue() will raise it. + # This is the only way we've come up with to deal with the problem in + # https://github.com/coreos/fedora-coreos-releng-automation/issues/70 + if self.keytab_file: + self.koji_client.getLoggedInUser() + # NOMENCLATURE: # # In koji there is the concept of a pkg and a build. A pkg From ced4e6c6e3bd51220c95e288375334102f85131a Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 17 Jan 2024 17:25:20 -0500 Subject: [PATCH 3/3] coreos-koji-tagger: tag packages from rawhide builds into pool For the Bodhi testing work, when testing a rawhide update, we want to test against a known working rawhide build. To do this, we want to base our build on the last working rawhide combination. We do this by reusing the generated lockfile from that build and the git commit of the config repo. But for this to work, we need to make sure that we have access to all the RPMs in that build. This patch teaches coreos-koji-tagger to listen to the `coreos.stream.release` fedmsg that the release job emits to know when a new rawhide build is available; it then downloads the generated lockfiles and tags in the packages. Closes: #181 --- coreos-koji-tagger/Dockerfile | 4 +- coreos-koji-tagger/coreos_koji_tagger.py | 60 ++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/coreos-koji-tagger/Dockerfile b/coreos-koji-tagger/Dockerfile index b6f7b54..402e1d0 100644 --- a/coreos-koji-tagger/Dockerfile +++ b/coreos-koji-tagger/Dockerfile @@ -38,8 +38,8 @@ RUN sed -i 's/INFO/WARNING/' /work/my_config.toml # Set the format for the log messages RUN sed -i 's/format =.*$/format = "%(asctime)s %(levelname)s %(name)s - %(message)s"/' /work/my_config.toml -# We only care about org.fedoraproject.prod.github.push messages -RUN sed -i 's/^routing_keys.*$/routing_keys = ["org.fedoraproject.prod.github.push"]/' /work/my_config.toml +# We only care about these two topics +RUN sed -i 's/^routing_keys.*$/routing_keys = ["org.fedoraproject.prod.github.push", "org.fedoraproject.prod.coreos.stream.release"]/' /work/my_config.toml # Add coreos_koji_tagger to the container ADD coreos_koji_tagger.py /work/ diff --git a/coreos-koji-tagger/coreos_koji_tagger.py b/coreos-koji-tagger/coreos_koji_tagger.py index 52e91c0..e935dc1 100755 --- a/coreos-koji-tagger/coreos_koji_tagger.py +++ b/coreos-koji-tagger/coreos_koji_tagger.py @@ -186,6 +186,16 @@ """ ) +# We also process the org.fedoraproject.prod.coreos.stream.release topic +# https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.coreos.stream.release&delta=100000 +EXAMPLE_COREOS_STREAM_RELEASE_MESSAGE_BODY = json.loads(""" +{ + "basearches": ["x86_64", "s390x", "ppc64le"], + "build_id": "40.20240113.91.1", + "stream": "rawhide" +} +""") + def catch_exceptions_and_continue(func): # This is a decorator function that will re-call the decorated # function and will catch any exceptions and not raise them further. @@ -282,6 +292,8 @@ def process_message(self, message: fedora_messaging.api.Message): if message.topic == 'org.fedoraproject.prod.github.push': self.process_github_push_message(message) + elif message.topic == 'org.fedoraproject.prod.coreos.stream.release': + self.process_coreos_stream_release_message(message) else: # that's weird, we shouldn't have been called for any other topic # just log it and ignore @@ -313,6 +325,22 @@ def process_github_push_message(self, message: fedora_messaging.api.Message): self.process_git_lockfiles(commit) + def process_coreos_stream_release_message(self, message: fedora_messaging.api.Message): + msg = message.body + stream = msg['stream'] + + # we only handle rawhide and branched builds; see + # https://github.com/coreos/fedora-coreos-releng-automation/issues/181 + if stream not in ['rawhide', 'branched']: + logger.info(f'Skipping stream release for stream: {stream}') + return + + # we'll start emitting a single message with `basearches` in the future + basearches = msg.get('basearches') or [msg['basearch']] + buildid = msg['build_id'] + + self.process_build_lockfiles(stream, buildid, basearches) + @catch_exceptions_and_continue def process_git_lockfiles(self, rev): # Now grab lockfile data from the commit we should operate on: @@ -345,6 +373,28 @@ def archify(f): self.tag_rpms(desiredrpms) + def process_build_lockfiles(self, stream, buildid, basearches): + # Now grab lockfile data from the commit we should operate on: + desiredrpms = set() + + for basearch in basearches: + url = f'https://builds.coreos.fedoraproject.org/prod/streams/{stream}/builds/{buildid}/{basearch}/manifest-lock.generated.{basearch}.json' + logger.info(f'Attempting to retrieve data from {url}') + r = requests.get(url) + if r.ok: + # parse the lockfile and add the set of rpm NEVRAs (strings) + desiredrpms.update(parse_lockfile_data(r.text, 'json')) + else: + # Log any errors we encounter. + logger.warning('URL request error: %s' % r.text.strip()) + if not desiredrpms: + logger.warning('No locked RPMs found!') + logger.warning(f"Is the build {buildid} missing its generated lockfile?") + logger.warning('Continuing...') + return + + self.tag_rpms(desiredrpms) + def tag_rpms(self, desiredrpms): # Check the login. If it's bad then a koji.AuthError exception will get thrown. @@ -678,6 +728,7 @@ def get_releasever_from_buildroottag(buildroottag: str) -> str: afterburn-dracut: evra: 4.1.1-3.module_f30+4804+1c3d5e42.x86_64 """ + sample_lockfile_json = json.dumps(yaml.safe_load(sample_lockfile)) # Make requests.get() return the above sample lockfile # for only one of the requested lockfiles. Otherwise 404 @@ -686,6 +737,9 @@ def side_effect(*args, **kwargs): if args[0].endswith('lock.x86_64.yaml'): requests_response.ok = True requests_response.text = sample_lockfile + elif 'generated' in args[0]: + requests_response.ok = True + requests_response.text = sample_lockfile_json else: requests_response.ok = False requests_response.text = "URL request error: 404: Not Found" @@ -700,3 +754,9 @@ def side_effect(*args, **kwargs): topic = 'org.fedoraproject.prod.github.push', body = EXAMPLE_GITHUB_PUSH_MESSAGE_BODY) c.__call__(m) + + # Now also test rawhide build lockfile scrapping + m = fedora_messaging.api.Message( + topic = 'org.fedoraproject.prod.coreos.stream.release', + body = EXAMPLE_COREOS_STREAM_RELEASE_MESSAGE_BODY) + c.__call__(m)