diff --git a/coreos-koji-tagger/Dockerfile b/coreos-koji-tagger/Dockerfile index b6f7b54..402e1d0 100644 --- a/coreos-koji-tagger/Dockerfile +++ b/coreos-koji-tagger/Dockerfile @@ -38,8 +38,8 @@ RUN sed -i 's/INFO/WARNING/' /work/my_config.toml # Set the format for the log messages RUN sed -i 's/format =.*$/format = "%(asctime)s %(levelname)s %(name)s - %(message)s"/' /work/my_config.toml -# We only care about org.fedoraproject.prod.github.push messages -RUN sed -i 's/^routing_keys.*$/routing_keys = ["org.fedoraproject.prod.github.push"]/' /work/my_config.toml +# We only care about these two topics +RUN sed -i 's/^routing_keys.*$/routing_keys = ["org.fedoraproject.prod.github.push", "org.fedoraproject.prod.coreos.stream.release"]/' /work/my_config.toml # Add coreos_koji_tagger to the container ADD coreos_koji_tagger.py /work/ diff --git a/coreos-koji-tagger/coreos_koji_tagger.py b/coreos-koji-tagger/coreos_koji_tagger.py index 4bca0df..e935dc1 100755 --- a/coreos-koji-tagger/coreos_koji_tagger.py +++ b/coreos-koji-tagger/coreos_koji_tagger.py @@ -50,7 +50,7 @@ # We are processing the org.fedoraproject.prod.github.push topic # https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.github.push&delta=100000 -EXAMPLE_MESSAGE_BODY = json.loads(""" +EXAMPLE_GITHUB_PUSH_MESSAGE_BODY = json.loads(""" { "forced": false, "compare": "https://github.com/coreos/fedora-coreos-config/compare/d6c02b5cd107...6a53f43af882", @@ -186,6 +186,16 @@ """ ) +# We also process the org.fedoraproject.prod.coreos.stream.release topic +# https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.coreos.stream.release&delta=100000 +EXAMPLE_COREOS_STREAM_RELEASE_MESSAGE_BODY = json.loads(""" +{ + "basearches": ["x86_64", "s390x", "ppc64le"], + "build_id": "40.20240113.91.1", + "stream": "rawhide" +} +""") + def catch_exceptions_and_continue(func): # This is a decorator function that will re-call the decorated # function and will catch any exceptions and not raise them further. @@ -269,7 +279,7 @@ def __init__(self): # do an initial run on startup in case we're out of sync for branch in self.github_repo_branches: - self.process_lockfiles(branch) + self.process_git_lockfiles(branch) def __call__(self, message: fedora_messaging.api.Message): @@ -280,6 +290,17 @@ def process_message(self, message: fedora_messaging.api.Message): logger.debug(message.topic) logger.debug(message.body) + if message.topic == 'org.fedoraproject.prod.github.push': + self.process_github_push_message(message) + elif message.topic == 'org.fedoraproject.prod.coreos.stream.release': + self.process_coreos_stream_release_message(message) + else: + # that's weird, we shouldn't have been called for any other topic + # just log it and ignore + logger.info(f'Ignoring message on topic: {message.topic}') + return + + def process_github_push_message(self, message: fedora_messaging.api.Message): # Grab the raw message body and the status from that msg = message.body branch = msg['ref'] @@ -302,17 +323,26 @@ def process_message(self, message: fedora_messaging.api.Message): logger.error('No commit id in message!') return - # Check the login. If it's bad then a koji.AuthError exception will get thrown. - # and we'll exit because catch_exceptions_and_continue() will raise it. - # This is the only way we've come up with to deal with the problem in - # https://github.com/coreos/fedora-coreos-releng-automation/issues/70 - if self.keytab_file: - self.koji_client.getLoggedInUser() + self.process_git_lockfiles(commit) - self.process_lockfiles(commit) + def process_coreos_stream_release_message(self, message: fedora_messaging.api.Message): + msg = message.body + stream = msg['stream'] + + # we only handle rawhide and branched builds; see + # https://github.com/coreos/fedora-coreos-releng-automation/issues/181 + if stream not in ['rawhide', 'branched']: + logger.info(f'Skipping stream release for stream: {stream}') + return + + # we'll start emitting a single message with `basearches` in the future + basearches = msg.get('basearches') or [msg['basearch']] + buildid = msg['build_id'] + + self.process_build_lockfiles(stream, buildid, basearches) @catch_exceptions_and_continue - def process_lockfiles(self, rev): + def process_git_lockfiles(self, rev): # Now grab lockfile data from the commit we should operate on: desiredrpms = set() @@ -341,6 +371,39 @@ def archify(f): logger.warning('Continuing...') return + self.tag_rpms(desiredrpms) + + def process_build_lockfiles(self, stream, buildid, basearches): + # Now grab lockfile data from the commit we should operate on: + desiredrpms = set() + + for basearch in basearches: + url = f'https://builds.coreos.fedoraproject.org/prod/streams/{stream}/builds/{buildid}/{basearch}/manifest-lock.generated.{basearch}.json' + logger.info(f'Attempting to retrieve data from {url}') + r = requests.get(url) + if r.ok: + # parse the lockfile and add the set of rpm NEVRAs (strings) + desiredrpms.update(parse_lockfile_data(r.text, 'json')) + else: + # Log any errors we encounter. + logger.warning('URL request error: %s' % r.text.strip()) + if not desiredrpms: + logger.warning('No locked RPMs found!') + logger.warning(f"Is the build {buildid} missing its generated lockfile?") + logger.warning('Continuing...') + return + + self.tag_rpms(desiredrpms) + + def tag_rpms(self, desiredrpms): + + # Check the login. If it's bad then a koji.AuthError exception will get thrown. + # and we'll exit because catch_exceptions_and_continue() will raise it. + # This is the only way we've come up with to deal with the problem in + # https://github.com/coreos/fedora-coreos-releng-automation/issues/70 + if self.keytab_file: + self.koji_client.getLoggedInUser() + # NOMENCLATURE: # # In koji there is the concept of a pkg and a build. A pkg @@ -665,6 +728,7 @@ def get_releasever_from_buildroottag(buildroottag: str) -> str: afterburn-dracut: evra: 4.1.1-3.module_f30+4804+1c3d5e42.x86_64 """ + sample_lockfile_json = json.dumps(yaml.safe_load(sample_lockfile)) # Make requests.get() return the above sample lockfile # for only one of the requested lockfiles. Otherwise 404 @@ -673,16 +737,26 @@ def side_effect(*args, **kwargs): if args[0].endswith('lock.x86_64.yaml'): requests_response.ok = True requests_response.text = sample_lockfile + elif 'generated' in args[0]: + requests_response.ok = True + requests_response.text = sample_lockfile_json else: requests_response.ok = False requests_response.text = "URL request error: 404: Not Found" return requests_response requests.get = Mock(side_effect=side_effect) - # Note that the following will call process_lockfiles twice. Once + c = Consumer() + + # Note that the following will call process_git_lockfiles twice. Once # for startup and once for the fake message we're passing. m = fedora_messaging.api.Message( topic = 'org.fedoraproject.prod.github.push', - body = EXAMPLE_MESSAGE_BODY) - c = Consumer() + body = EXAMPLE_GITHUB_PUSH_MESSAGE_BODY) + c.__call__(m) + + # Now also test rawhide build lockfile scrapping + m = fedora_messaging.api.Message( + topic = 'org.fedoraproject.prod.coreos.stream.release', + body = EXAMPLE_COREOS_STREAM_RELEASE_MESSAGE_BODY) c.__call__(m)