diff --git a/coreos-koji-tagger/Dockerfile b/coreos-koji-tagger/Dockerfile index 977ac76..63404f7 100644 --- a/coreos-koji-tagger/Dockerfile +++ b/coreos-koji-tagger/Dockerfile @@ -10,12 +10,6 @@ RUN dnf update -y && dnf clean all # Install pagure/fedmsg libraries RUN dnf -y install dnf-plugins-core python3-libpagure fedora-messaging koji krb5-workstation && dnf clean all -# Update koji src to 1.18 for multicall support (not yet packaged as an RPM) -RUN dnf -y install git && dnf clean all -RUN rm -rf /usr/lib/python3.7/site-packages/koji -RUN git -C /opt/ clone -b koji-1.18.0 https://pagure.io/koji.git -ENV PYTHONPATH=/opt/koji/ - # Grab the kerberos/koji configuration (i.e. /usr/bin/stg-koji) by # installing the fedora-packager rpm. We don't need the deps because # we aren't building anything. diff --git a/coreos-koji-tagger/README.md b/coreos-koji-tagger/README.md index db01ad3..94ecc19 100644 --- a/coreos-koji-tagger/README.md +++ b/coreos-koji-tagger/README.md @@ -31,13 +31,27 @@ OpenShift web interface. # Testing in Fedora Stage -In order to test in Fedora Stage you must change the manifest file(s) -in the branch of the repo currently being monitored by stage. You can -see the branch/repo info in +In order to test a new version of coreos-koji-tagger in Fedora Stage +there are two inputs which you can control: + +- The coreos-koji-tagger source code +- The input manifest lockfiles + +In order to update the source code you need to push to the repo/branch +currently being monitored by the [the buildconfig](https://infrastructure.fedoraproject.org/cgit/ansible.git/tree/roles/openshift-apps/coreos-koji-tagger/templates/buildconfig.yml). +for the staging environment. This will most likely be the +`fedora-infra-staging` branch of this git repo. + +Once you have the version of coreos-koji-tagger that you want running +in stage you need to push code to the repo/branch currently being monitored +by the staging coreos-koji-tagger. This involves changing the manifest file(s) +and pushing to the git repo. To see the branch/repo currently being +monitored you can see that in the +[deploymentconfig](https://infrastructure.fedoraproject.org/cgit/ansible.git/tree/roles/openshift-apps/coreos-koji-tagger/templates/deploymentconfig.yml). You'll need to either push to the target branch/repo or you'll need to -update the buildconfig to point to another one that you control. The +update the deploymentconfig to point to another one that you control. The repo will need to be set up publish events to fedmsg using [github2fedmsg](https://apps.fedoraproject.org/github2fedmsg) so that the script can pick up the event and process it. diff --git a/coreos-koji-tagger/coreos_koji_tagger.py b/coreos-koji-tagger/coreos_koji_tagger.py index 44d9131..9720404 100755 --- a/coreos-koji-tagger/coreos_koji_tagger.py +++ b/coreos-koji-tagger/coreos_koji_tagger.py @@ -262,7 +262,6 @@ def process(self, message: fedora_messaging.api.Message): msg = message.body branch = msg['ref'] repo = msg['repository']['full_name'] - commit = msg['head_commit']['id'] if (repo != self.github_repo_fullname): logger.info(f'Skipping message from unrelated repo: {repo}') @@ -272,12 +271,28 @@ def process(self, message: fedora_messaging.api.Message): logger.info(f'Skipping message from unrelated branch: {branch}') return - # Now grab data from the commit we should operate on: - # XXX: should update for multi-arch - url = f'https://raw.githubusercontent.com/{repo}/{commit}/manifest-lock.x86_64.json' - logger.info(f'Attempting to retrieve data from {url}') - r = requests.get(url) + # Some messages don't have commit information + # For example: https://apps.fedoraproject.org/datagrepper/id?id=2019-f32c811b-658b-4ac7-a455-a7edf616a033&is_raw=true&size=extra-large + commit = None + if msg['head_commit']: + commit = msg['head_commit']['id'] + if commit is None: + logger.error('No commit id in message!') + return + # Now grab lockfile data from the commit we should operate on: + desiredrpms = set() + for arch in ['x86_64', 'aarch64', 'ppc64le', 's390x']: + for lockfile in ['manifest-lock', 'manifest-lock.overrides']: + url = f'https://raw.githubusercontent.com/{repo}/{commit}/{lockfile}.{arch}.json' + logger.info(f'Attempting to retrieve data from {url}') + r = requests.get(url) + if r.ok: + # parse the lockfile and add the set of rpm NEVRAs (strings) + desiredrpms.update(parse_lockfile_data(r.text)) + else: + # Log any errors we encounter. 404s are ok, but won't hurt to log + logger.warn('URL request error: %s' % r.text.strip()) # NOMENCLATURE: # @@ -302,9 +317,6 @@ def process(self, message: fedora_messaging.api.Message): # koji builds that aren't in the tag we can add the koji pkg to the # tag (if needed) and then tag the koji build into the tag. - # parse the lockfile and get a set of rpm NEVRAs (strings) - desiredrpms = set(parse_lockfile_data(r.text)) - # convert the NEVRAs into a dict of build IDs -> BuildInfo objects buildsinfo = self.get_buildsinfo_from_rpmnevras(desiredrpms) desiredbuildids = buildsinfo.keys() @@ -473,9 +485,9 @@ def get_NVRA_from_NEVRA(string: str) -> str: nvra = f"{rpminfo.name}-{rpminfo.version}-{rpminfo.release}.{rpminfo.arch}" return nvra -def parse_lockfile_data(text: str) -> list: +def parse_lockfile_data(text: str) -> set: """ - Parse the rpm lockfile format and return a list of rpms in + Parse the rpm lockfile format and return a set of rpms in NEVRA form. Best documention on the format for now: https://github.com/projectatomic/rpm-ostree/commit/8ff0ee9c89ecc0540182b5b506455fc275d27a61 @@ -498,7 +510,7 @@ def parse_lockfile_data(text: str) -> list: logger.debug(json.dumps(data, indent=4, sort_keys=True)) # We only care about the NEVRAs, so just accumulate those and return - return [f'{name}-{v["evra"]}' for name, v in data['packages'].items()] + return set([f'{name}-{v["evra"]}' for name, v in data['packages'].items()]) def get_releasever_from_buildroottag(buildroottag: str) -> str: logger.debug(f'Checking buildroottag {buildroottag}')