Merge pull request coreos#39 from dustymabe/dusty

coreos-koji-tagger: add support for overrides and multi-arch lockfiles

coreos-koji-tagger/Dockerfile

@@ -10,12 +10,6 @@ RUN dnf update -y && dnf clean all
 # Install pagure/fedmsg libraries
 RUN dnf -y install dnf-plugins-core python3-libpagure fedora-messaging koji krb5-workstation && dnf clean all
 
-# Update koji src to 1.18 for multicall support (not yet packaged as an RPM)
-RUN dnf -y install git && dnf clean all
-RUN rm -rf /usr/lib/python3.7/site-packages/koji
-RUN git -C /opt/ clone -b koji-1.18.0 https://pagure.io/koji.git
-ENV PYTHONPATH=/opt/koji/
-
 # Grab the kerberos/koji configuration (i.e. /usr/bin/stg-koji) by
 # installing the fedora-packager rpm. We don't need the deps because
 # we aren't building anything.

coreos-koji-tagger/README.md

@@ -31,13 +31,27 @@ OpenShift web interface.
 
 # Testing in Fedora Stage
 
-In order to test in Fedora Stage you must change the manifest file(s)
-in the branch of the repo currently being monitored by stage. You can
-see the branch/repo info in 
+In order to test a new version of coreos-koji-tagger in Fedora Stage
+there are two inputs which you can control:
+
+- The coreos-koji-tagger source code
+- The input manifest lockfiles
+
+In order to update the source code you need to push to the repo/branch
+currently being monitored by the
 [the buildconfig](https://infrastructure.fedoraproject.org/cgit/ansible.git/tree/roles/openshift-apps/coreos-koji-tagger/templates/buildconfig.yml).
+for the staging environment. This will most likely be the
+`fedora-infra-staging` branch of this git repo.
+
+Once you have the version of coreos-koji-tagger that you want running
+in stage you need to push code to the repo/branch currently being monitored
+by the staging coreos-koji-tagger. This involves changing the manifest file(s)
+and pushing to the git repo. To see the branch/repo currently being
+monitored you can see that in the
+[deploymentconfig](https://infrastructure.fedoraproject.org/cgit/ansible.git/tree/roles/openshift-apps/coreos-koji-tagger/templates/deploymentconfig.yml).
 
 You'll need to either push to the target branch/repo or you'll need to
-update the buildconfig to point to another one that you control. The
+update the deploymentconfig to point to another one that you control. The
 repo will need to be set up publish events to fedmsg using
 [github2fedmsg](https://apps.fedoraproject.org/github2fedmsg) so that
 the script can pick up the event and process it.

coreos-koji-tagger/coreos_koji_tagger.py

@@ -262,7 +262,6 @@ def process(self, message: fedora_messaging.api.Message):
         msg = message.body
         branch = msg['ref']
         repo   = msg['repository']['full_name']
-        commit = msg['head_commit']['id']
 
         if (repo != self.github_repo_fullname):
             logger.info(f'Skipping message from unrelated repo: {repo}')
@@ -272,12 +271,28 @@ def process(self, message: fedora_messaging.api.Message):
             logger.info(f'Skipping message from unrelated branch: {branch}')
             return
 
-        # Now grab data from the commit we should operate on:
-        # XXX: should update for multi-arch
-        url = f'https://raw.githubusercontent.com/{repo}/{commit}/manifest-lock.x86_64.json'
-        logger.info(f'Attempting to retrieve data from {url}')
-        r = requests.get(url)
+        # Some messages don't have commit information
+        # For example: https://apps.fedoraproject.org/datagrepper/id?id=2019-f32c811b-658b-4ac7-a455-a7edf616a033&is_raw=true&size=extra-large
+        commit = None
+        if msg['head_commit']:
+            commit = msg['head_commit']['id']
+        if commit is None:
+            logger.error('No commit id in message!')
+            return
 
+        # Now grab lockfile data from the commit we should operate on:
+        desiredrpms = set()
+        for arch in ['x86_64', 'aarch64', 'ppc64le', 's390x']:
+            for lockfile in ['manifest-lock', 'manifest-lock.overrides']:
+                url = f'https://raw.githubusercontent.com/{repo}/{commit}/{lockfile}.{arch}.json'
+                logger.info(f'Attempting to retrieve data from {url}')
+                r = requests.get(url)
+                if r.ok:
+                    # parse the lockfile and add the set of rpm NEVRAs (strings)
+                    desiredrpms.update(parse_lockfile_data(r.text))
+                else:
+                    # Log any errors we encounter. 404s are ok, but won't hurt to log
+                    logger.warn('URL request error: %s' % r.text.strip())
 
         # NOMENCLATURE:
         # 
@@ -302,9 +317,6 @@ def process(self, message: fedora_messaging.api.Message):
         # koji builds that aren't in the tag we can add the koji pkg to the
         # tag (if needed) and then tag the koji build into the tag.
 
-        # parse the lockfile and get a set of rpm NEVRAs (strings)
-        desiredrpms = set(parse_lockfile_data(r.text))
-
         # convert the NEVRAs into a dict of build IDs -> BuildInfo objects
         buildsinfo = self.get_buildsinfo_from_rpmnevras(desiredrpms)
         desiredbuildids = buildsinfo.keys()
@@ -473,9 +485,9 @@ def get_NVRA_from_NEVRA(string: str) -> str:
     nvra = f"{rpminfo.name}-{rpminfo.version}-{rpminfo.release}.{rpminfo.arch}"
     return nvra
 
-def parse_lockfile_data(text: str) -> list:
+def parse_lockfile_data(text: str) -> set:
     """
-    Parse the rpm lockfile format and return a list of rpms in
+    Parse the rpm lockfile format and return a set of rpms in
     NEVRA form.
     Best documention on the format for now:
         https://github.com/projectatomic/rpm-ostree/commit/8ff0ee9c89ecc0540182b5b506455fc275d27a61
@@ -498,7 +510,7 @@ def parse_lockfile_data(text: str) -> list:
     logger.debug(json.dumps(data, indent=4, sort_keys=True))
 
     # We only care about the NEVRAs, so just accumulate those and return
-    return [f'{name}-{v["evra"]}' for name, v in data['packages'].items()]
+    return set([f'{name}-{v["evra"]}' for name, v in data['packages'].items()])
 
 def get_releasever_from_buildroottag(buildroottag: str) -> str:
     logger.debug(f'Checking buildroottag {buildroottag}')