From e9b588ad3bf3e6130cf6e3f613aa4865ac6433db Mon Sep 17 00:00:00 2001 From: Philip Guyton Date: Fri, 29 Nov 2019 21:05:10 +0000 Subject: [PATCH] [openSUSE] replace legacy yum with dnf-yum. Fixes #2092 Yum has now long been superseded by dnf and given a recent suspected deprecation of legacy yum (which is python 2 based), and a missing yum package within Tumbleweed, if we move now to dnf-yum in both Leap15.x and Tumbleweed bases we further reduce our legacy dependencies. Includes: - Addresses a minor date output formatting difference to enable the proposed dependency change from yum/yum-changelog to dnf-yum/dnf-plugins-core. - Add legacy yum/dnf-yum distro specific yum changelog options. - Re-use existing distro.id() call to avoid re-calling. - Generalise rockstor package changelog parser - dnf-yum compatibility. - Include conditional to account for further variation in 'rockstor' version line parsing: dnf-yum --changelog has a different output than legacy yum --changelog so we adapt accordingly. - User faster re.match (over re.search) as our new target allows this. Previously the search target was mid line, requiring the significantly slower re.search. - Correct / update prior pkg_mgmt.py comments - Unit test rpm_build_info() - pky_mgmt to ensure dnf-yum compatibility. - Unit test pkg_latest_available() in pkg_mgmt.py. - Abstract pkg_latest_available() function used when no changelog found to improves rockstor_pkg_update_check() code testability. - Also removes non architecture aware component of abstracted function. --- src/rockstor/system/pkg_mgmt.py | 96 +++++-- src/rockstor/system/tests/test_pkg_mgmt.py | 298 ++++++++++++++++++++- 2 files changed, 374 insertions(+), 20 deletions(-) diff --git a/src/rockstor/system/pkg_mgmt.py b/src/rockstor/system/pkg_mgmt.py index 7496477fa..be3d8e605 100644 --- a/src/rockstor/system/pkg_mgmt.py +++ b/src/rockstor/system/pkg_mgmt.py @@ -104,10 +104,12 @@ def current_version(): def rpm_build_info(pkg): version = "Unknown Version" date = None + distro_id = distro.id() try: o, e, rc = run_command([YUM, "info", "installed", "-v", pkg]) except CommandException as e: - # Catch "No matching Packages to list" so we can return None, None. + # Catch "No matching Packages to list" so we can return: + # "Unknown Version", None emsg = "Error: No matching Packages to list" # By checking both the first error element and the second to last we # catch one yum waiting for another to release yum lock. @@ -118,11 +120,17 @@ def rpm_build_info(pkg): raise e for l in o: if re.match("Buildtime", l) is not None: - # eg: "Buildtime : Tue Dec 5 13:34:06 2017" - # we return 2017-Dec-06 + # Legacy Rockstor (using original yum): + # "Buildtime : Tue Dec 5 13:34:06 2017" + # openSUSE Rocsktor (using dnf-yum): + # "Buildtime : Fri 29 Nov 2019 18:34:43 GMT" + # we return 2017-Dec-06 or 2019-Nov-29 # Note the one day on from retrieved Buildtime with zero padding. dfields = l.strip().split() - dstr = dfields[6] + " " + dfields[3] + " " + dfields[4] + if distro_id == "rockstor": # CentOS based Rockstor conditional + dstr = dfields[6] + " " + dfields[3] + " " + dfields[4] + else: # Assuming we are openSUSE variant and so using dnf-yum + dstr = dfields[5] + " " + dfields[4] + " " + dfields[3] bdate = datetime.strptime(dstr, "%Y %b %d") bdate += timedelta(days=1) date = bdate.strftime("%Y-%b-%d") @@ -244,6 +252,8 @@ def repo_status(subscription): def rockstor_pkg_update_check(subscription=None): + distro_id = distro.id() + machine_arch = platform.machine() if subscription is not None: switch_repo(subscription) pkg = "rockstor" @@ -255,17 +265,26 @@ def rockstor_pkg_update_check(subscription=None): available = False new_version = None updates = [] + if distro_id == "rockstor": + changelog_cmd = [YUM, "changelog", date, pkg] + else: # We are assuming openSUSE with dnf-yum specific options + if date != "all": + changelog_cmd = [YUM, "changelog", "--since", date, pkg] + else: + # Here we list the default number of changelog entries: + # defaults to last 8 releases but states "Listing all changelogs" + changelog_cmd = [YUM, "changelog", pkg] try: - o, e, rc = run_command([YUM, "changelog", date, pkg]) + o, e, rc = run_command(changelog_cmd) except CommandException as e: - # Catch as yet unconfigured repos ie Leap 15.1: error log accordingly. + # Catch as yet unconfigured repos ie openSUSE Stable error log accordingly. # Avoids breaking current version display and update channel selection. emsg = "Error\\: Cannot retrieve repository metadata \\(repomd.xml\\)" if re.match(emsg, e.err[-2]) is not None: logger.error( "Rockstor repo for distro.id ({}) version ({}) may " "not exist: pending or deprecated.\nReceived: ({}).".format( - distro.id(), distro.version(), e.err + distro_id, distro.version(), e.err ) ) new_version = version # Explicitly set (flag) for code clarity. @@ -273,15 +292,30 @@ def rockstor_pkg_update_check(subscription=None): # otherwise we raise an exception as normal. raise e for l in o: - if re.search("Available Packages", l) is not None: + # We have possible targets of: + # "Listing changelogs since 2019-11-29" - legacy yum and dnf-yum + # "Listing all changelogs" - legacy yum and dnf-yum with no --count=# + # "Listing # latest changelogs" - dnf-yum with a --count=# options + if re.match("Listing", l) is not None: available = True if not available: continue - if new_version is None and (re.match("rockstor-", l) is not None): + if new_version is None: machine_arch = platform.machine() - new_version = ( - l.split()[0].split("rockstor-")[1].split(".{}".format(machine_arch))[0] - ) + if re.match("rockstor-", l) is not None: # legacy yum + # eg: "rockstor-3.9.2-51.2089.x86_64" + new_version = ( + l.split()[0] + .split("rockstor-")[1] + .split(".{}".format(machine_arch))[0] + ) + if re.match("Changelogs for rockstor-", l) is not None: # dnf-yum + # eg: "Changelogs for rockstor-3.9.2-51.2089.x86_64" + new_version = ( + l.split()[2] + .split("rockstor-")[1] + .split(".{}".format(machine_arch))[0] + ) if log is True: updates.append(l) if len(l.strip()) == 0: @@ -290,16 +324,40 @@ def rockstor_pkg_update_check(subscription=None): updates.append(l) log = True if new_version is None: - new_version = version - # do a second check which is valid for updates without changelog + logger.debug("No changelog found: trying yum update for info.") + # Do a second check which is valid for updates without changelog # updates. eg: same day updates, testing updates. - o, e, rc = run_command([YUM, "update", pkg, "--assumeno"], throw=False) - if rc == 1: - for l in o: + new_version = pkg_latest_available(pkg, machine_arch, distro_id) + if new_version is None: + new_version = version + return version, new_version, updates + + +def pkg_latest_available(pkg_name, arch, distro_id): + """ + Simple wrapper around "yum update pkg_name --assumeno" to retrieve + latest version available from "Version" column + :return: + """ + new_version = None + # TODO: We might use "zypper se -s --match-exact rockstor" and parse first + # line with rockstor in second column but unit test will be defunct. + # Advantage: works with no rockstor version installed, no so dnf-yum + o, e, rc = run_command([YUM, "update", pkg_name, "--assumeno"], throw=False) + if rc == 1: + for l in o: + if distro_id == "rockstor": + # Legacy Yum appropriate parsing, all info on one line. + # "Package rockstor.x86_64 0:3.9.2-51.2089 will be an update" if re.search("will be an update", l) is not None: - if re.search("rockstor.x86_64", l) is not None: + if re.search("rockstor.{}".format(arch), l) is not None: new_version = l.strip().split()[3].split(":")[1] - return version, new_version, updates + else: # We are assuming openSUSE with dnf-yum output format + # dnf-yum output line of interest; when presented: + # " rockstor x86_64 3.9.2-51.2089 localrepo 15 M" + if re.match(" rockstor", l) is not None: + new_version = l.strip().split()[2] + return new_version def update_run(subscription=None, update_all_other=False): diff --git a/src/rockstor/system/tests/test_pkg_mgmt.py b/src/rockstor/system/tests/test_pkg_mgmt.py index 246aaaaff..574dfe34a 100644 --- a/src/rockstor/system/tests/test_pkg_mgmt.py +++ b/src/rockstor/system/tests/test_pkg_mgmt.py @@ -15,7 +15,13 @@ import unittest from mock import patch -from system.pkg_mgmt import pkg_update_check, pkg_changelog, zypper_repos_list +from system.pkg_mgmt import ( + pkg_update_check, + pkg_changelog, + zypper_repos_list, + rpm_build_info, + pkg_latest_available, +) class SystemPackageTests(unittest.TestCase): @@ -331,3 +337,293 @@ def test_zypper_repos_list(self): "returned = ({}).\n " "expected = ({}).".format(returned, expected), ) + + def test_rpm_build_info(self): + """ + rpm_build_info strips out and concatenates Version and Release info for the + rockstor package. This is returned along with a standardised date format for + the Buildtime which is also parse out. N.B. the build time has 1 day added + to it for historical reasons. + """ + # legacy rockstor/CentOS YUM: + dist_id = ["rockstor"] + out = [ + [ + 'Loading "changelog" plugin', + 'Loading "fastestmirror" plugin', + "Config time: 0.010", + "Yum version: 3.4.3", + "rpmdb time: 0.000", + "Installed Packages", + "Name : rockstor", + "Arch : x86_64", + "Version : 3.9.2", + "Release : 50.2093", + "Size : 85 M", + "Repo : installed", + "From repo : localrepo", + "Committer : Philip Guyton ", + "Committime : Wed Nov 13 04:00:00 2019", + "Buildtime : Fri Nov 29 14:03:17 2019", + "Install time: Sun Dec 1 07:23:38 2019", + "Installed by: root ", + "Changed by : System ", + "Summary : Btrfs Network Attached Storage (NAS) Appliance.", + "URL : http://rockstor.com/", + "License : GPL", + "Description : Software raid, snapshot capable NAS solution with built-in file", + " : integrity protection. Allows for file sharing between network", + " : attached devices.", + "", + "", + ] + ] + err = [[""]] + rc = [0] + expected_results = [("3.9.2-50.2093", "2019-Nov-30")] + # Leap15.1 dnf-yum + dist_id.append("opensuse-leap") + out.append( + [ + "Loaded plugins: builddep, changelog, config-manager, copr, debug, debuginfo-install, download, generate_completion_cache, needs-restarting, playground, repoclosure, repodiff, repograph, repomanage, reposync", # noqa E501 + "DNF version: 4.2.6", + "cachedir: /var/cache/dnf", + "Waiting for process with pid 30565 to finish.", + "No module defaults found", + "Installed Packages", + "Name : rockstor", + "Version : 3.9.2", + "Release : 50.2093", + "Architecture : x86_64", + "Size : 82 M", + "Source : rockstor-3.9.2-50.2093.src.rpm", + "Repository : @System", + "Packager : None", + "Buildtime : Sat 30 Nov 2019 11:50:41 AM GMT", + "Install time : Sun 01 Dec 2019 03:23:03 PM GMT", + "Summary : Btrfs Network Attached Storage (NAS) Appliance.", + "URL : http://rockstor.com/", + "License : GPL", + "Description : Software raid, snapshot capable NAS solution with built-in file", + " : integrity protection. Allows for file sharing between network", + " : attached devices.", + "", + "", + ] + ) + err.append([""]) + rc.append(0) + expected_results.append(("3.9.2-50.2093", "2019-Dec-01")) + # Tumbleweed dnf-yum + dist_id.append("opensuse-tumbleweed") + out.append( + [ + "Loaded plugins: builddep, changelog, config-manager, copr, debug, debuginfo-install, download, generate_completion_cache, needs-restarting, playground, repoclosure, repodiff, repograph, repomanage, reposync", # noqa E501 + "DNF version: 4.2.6", + "cachedir: /var/cache/dnf", + "No module defaults found", + "Installed Packages", + "Name : rockstor", + "Version : 3.9.2", + "Release : 50.2093", + "Architecture : x86_64", + "Size : 84 M", + "Source : rockstor-3.9.2-50.2093.src.rpm", + "Repository : @System", + "Packager : None", + "Buildtime : Fri 29 Nov 2019 10:03:53 PM GMT", + "Install time : Sun 01 Dec 2019 03:23:33 PM GMT", + "Summary : Btrfs Network Attached Storage (NAS) Appliance.", + "URL : http://rockstor.com/", + "License : GPL", + "Description : Software raid, snapshot capable NAS solution with built-in file", + " : integrity protection. Allows for file sharing between network", + " : attached devices.", + "", + "", + ] + ) + err.append([""]) + rc.append(0) + expected_results.append(("3.9.2-50.2093", "2019-Nov-30")) + # Source install where we key from the error message: + dist_id.append("opensuse-tumbleweed") + out.append( + [ + "Loaded plugins: builddep, changelog, config-manager, copr, debug, debuginfo-install, download, generate_completion_cache, needs-restarting, playground, repoclosure, repodiff, repograph, repomanage, reposync", # noqa E501 + "DNF version: 4.2.6", + "cachedir: /var/cache/dnf", + "No module defaults found", + "", + ] + ) + err.append(["Error: No matching Packages to list", ""]) + rc.append(1) + expected_results.append(("Unknown Version", None)) + + for o, e, r, expected, distro in zip(out, err, rc, expected_results, dist_id): + self.mock_run_command.return_value = (o, e, r) + self.mock_distro.id.return_value = distro + returned = rpm_build_info("rockstor") + self.assertEqual( + returned, + expected, + msg="Un-expected rpm_build_info() result:\n " + "returned = ({}).\n " + "expected = ({}).".format(returned, expected), + ) + + def test_pkg_latest_available(self): + """ + This procedure was extracted from a fail through position at the end + of rockstor_pkg_update_check() to enable discrete testing. + Note that at time of extraction it was not believed to work as intended + with dnf-yum. + :return: + """ + # CentOS Legacy yum - rockstor rpm installed with update available. + # another process holding the rpm db (not relevant currently) + arch = "x86_64" + dist_id = ["rockstor"] + out = [ + [ + "Loaded plugins: changelog, fastestmirror", + "Loading mirror speeds from cached hostfile", + " * base: mirrors.melbourne.co.uk", + " * epel: mirrors.coreix.net", + " * extras: mozart.ee.ic.ac.uk", + " * updates: mirrors.coreix.net", + "Resolving Dependencies", + "--> Running transaction check", + "---> Package rockstor.x86_64 0:3.9.2-50.2093 will be updated", + "---> Package rockstor.x86_64 0:3.9.2-51.2089 will be an update", # This is the parsed in legacy YUM + "--> Finished Dependency Resolution", + "", + "Dependencies Resolved", + "", + "================================================================================", + " Package Arch Version Repository Size", + "================================================================================", + "Updating:", + " rockstor x86_64 3.9.2-51.2089 localrepo 17 M", + "", + "Transaction Summary", + "================================================================================", + "Upgrade 1 Package", + "", + "Total download size: 17 M", + "Exiting on user command", + "Your transaction was saved, rerun it with:", + " yum load-transaction /tmp/yum_save_tx.2019-12-02.10-37.0b42CF.yumtx", + "", + ] + ] + err = [ + [ + "Existing lock /var/run/yum.pid: another copy is running as pid 18540.", + "Another app is currently holding the yum lock; waiting for it to exit...", + " The other application is: yum", + " Memory : 35 M RSS (712 MB VSZ)", + " Started: Mon Dec 2 10:37:13 2019 - 00:01 ago", + " State : Sleeping, pid: 18540", + "Another app is currently holding the yum lock; waiting for it to exit...", + " The other application is: yum", + " Memory : 35 M RSS (712 MB VSZ)", + " Started: Mon Dec 2 10:37:13 2019 - 00:03 ago", + " State : Sleeping, pid: 18540", + "", + ] + ] + rc = [1] + expected_result = ["3.9.2-51.2089"] + # CentOS Legacy yum - rockstor rpm installed with update available. + # no rpm db lock in place + dist_id.append("rockstor") + out.append( + [ + "Loaded plugins: changelog, fastestmirror", + "Loading mirror speeds from cached hostfile", + " * base: mirrors.melbourne.co.uk", + " * epel: mirrors.coreix.net", + " * extras: mozart.ee.ic.ac.uk", + " * updates: mozart.ee.ic.ac.uk", + "Resolving Dependencies", + "--> Running transaction check", + "---> Package rockstor.x86_64 0:3.9.2-50.2093 will be updated", + "---> Package rockstor.x86_64 0:3.9.2-51.2089 will be an update", + "--> Finished Dependency Resolution", + "", + "Dependencies Resolved", + "", + "================================================================================", + " Package Arch Version Repository Size", + "================================================================================", + "Updating:", + " rockstor x86_64 3.9.2-51.2089 localrepo 17 M", + "", + "Transaction Summary", + "================================================================================", + "Upgrade 1 Package", + "", + "Total download size: 17 M", + "Exiting on user command", + "Your transaction was saved, rerun it with:", + " yum load-transaction /tmp/yum_save_tx.2019-12-02.10-47.uHJM6L.yumtx", + "", + ] + ) + err.append([""]) + rc.append(1) + expected_result.append("3.9.2-51.2089") + # Tumblweed dnf-yum - no rockstor rpm installed but one available. + dist_id.append("opensuse-tumbleweed") + out.append( + [ + "Local Repository 2.9 MB/s | 3.0 kB 00:00 ", + "No match for argument: rockstor", + "", + ] + ) + err.append( + [ + "Package rockstor available, but not installed.", + "Error: No packages marked for upgrade.", + "", + ] + ) + rc.append(1) + expected_result.append(None) + # Leap15.1 dnf-yum - rockstor package installed with update available: + dist_id.append("opensuse-leap") + out.append( + [ + "Last metadata expiration check: 0:00:10 ago on Mon 02 Dec 2019 06:22:10 PM GMT.", + "Dependencies resolved.", + "================================================================================", + " Package Architecture Version Repository Size", + "================================================================================", + "Upgrading:", + " rockstor x86_64 3.9.2-51.2089 localrepo 15 M", + "", + "Transaction Summary", + "================================================================================", + "Upgrade 1 Package", + "", + "Total size: 15 M", + "", + ] + ) + err.append(["Operation aborted.", ""]) + rc.append(1) + expected_result.append("3.9.2-51.2089") + + for o, e, r, expected, distro in zip(out, err, rc, expected_result, dist_id): + self.mock_run_command.return_value = (o, e, r) + returned = pkg_latest_available("rockstor", arch, distro) + self.assertEqual( + returned, + expected, + msg="Un-expected pkg_latest_available('rockstor') result:\n " + "returned = ({}).\n " + "expected = ({}).".format(returned, expected), + )