diff --git a/.pylintrc b/.pylintrc index 797c019..1603302 100644 --- a/.pylintrc +++ b/.pylintrc @@ -8,7 +8,7 @@ max-branches=20 [FORMAT] max-line-length = 120 -disable=locally-disabled,locally-enabled,missing-docstring,duplicate-code,fixme,cyclic-import,redefined-variable-type,stop-iteration-return, R1705, E1136, W0613 +disable=locally-disabled,locally-enabled,missing-docstring,duplicate-code,fixme,cyclic-import,redefined-variable-type,stop-iteration-return, R1705, E1136, W0613, R0902 [TYPECHECK] diff --git a/Makefile b/Makefile index eed347e..1897148 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ help: @echo 'make snapshot: Create a tar.gz of the current git revision' @echo 'make pypi_sdist: Release a new sdist to PyPI' -test: test_pylint test_flake8 test_pylint +test: test_pylint test_flake8 test_pytest @echo "All test ran..." test_pylint: @@ -54,7 +54,7 @@ test_flake8: @echo "Running flake8..." flake8 $(TEST_PATHS) -test_pylint: +test_pytest: @echo "Running pylint..." pytest diff --git a/README.md b/README.md index 68b5da4..e6eb06c 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,12 @@ -Build status: [![Build Status](https://travis-ci.org/palikar/code_manager.svg?branch=master)](https://travis-ci.org/palikar/code_manager) +Build status: + [![Build Status](https://travis-ci.org/palikar/code_manager.svg?branch=master)](https://travis-ci.org/palikar/code_manager) + [![Build Status](https://pyup.io/repos/github/palikar/code_manager/shield.svg)](https://pyup.io/repos/github/palikar/code_manager) + [![Build Status](https://pyup.io/repos/github/palikar/code_manager/python-3-shield.svg)](https://pyup.io/repos/github/palikar/code_manager/) + [![Coverage Status](https://coveralls.io/repos/github/palikar/code_manager/badge.svg?branch=master)](https://coveralls.io/github/palikar/code_manager?branch=master) # Code Manager -![img](https://travis-ci.org/palikar/code_manager.svg?branch=master) ![img](https://pyup.io/repos/github/palikar/code_manager/shield.svg) ![img](https://pyup.io/repos/github/palikar/code_manager/python-3-shield.svg) ![img](https://coveralls.io/repos/github/palikar/code_manager/badge.svg?branch=master) - ## Abstract diff --git a/README.org b/README.org index 6832de5..4b86d22 100644 --- a/README.org +++ b/README.org @@ -15,14 +15,16 @@ #+CREATOR: Emacs 26.1 (Org mode 9.1.13) - +#+BEGIN_EXAMPLE +Build status: +[![Build Status](https://travis-ci.org/palikar/code_manager.svg?branch=master)](https://travis-ci.org/palikar/code_manager) +[![Build Status](https://pyup.io/repos/github/palikar/code_manager/shield.svg)](https://pyup.io/repos/github/palikar/code_manager) +[![Build Status](https://pyup.io/repos/github/palikar/code_manager/python-3-shield.svg)](https://pyup.io/repos/github/palikar/code_manager/) +[![Coverage Status](https://coveralls.io/repos/github/palikar/code_manager/badge.svg?branch=master)](https://coveralls.io/github/palikar/code_manager?branch=master) +#+END_EXAMPLE * Code Manager -![[https://travis-ci.org/palikar/code_manager.svg?branch=master][img]] -![[https://pyup.io/repos/github/palikar/code_manager/shield.svg][img]] -![[https://pyup.io/repos/github/palikar/code_manager/python-3-shield.svg][img]] -![[https://coveralls.io/repos/github/palikar/code_manager/badge.svg?branch=master][img]] ** Abstract This is my personal tool now for managing my github repositories, some system software that I use and pretty much everything that can be downloaded, compiled locally and then installed on a Debian based Linux system. Through this utility one can quickly download and install random things from all over the internet. I've always wanted some small program that would allow me to quickly bring my github repositories on my local machine so I end it up writing this in my spare time. The program is focused on automation but also on flexibility in the installation process. A lot of software is compiled and installed in some standard way but there are also things that are a little bit trickier. The utility - named appropriately ~code_manager~ - aims to provide a unified interface for the installation process of all types of software -- the trickier kind included. diff --git a/code_manager/core/debgrapher.py b/code_manager/core/debgrapher.py index 3ab685b..972bf63 100644 --- a/code_manager/core/debgrapher.py +++ b/code_manager/core/debgrapher.py @@ -23,11 +23,10 @@ def get_packages(self, group=""): # pylint: disable=R1705 def verify_package_list(self, package_names): all_packs = set(flatten(self.packages_list.values())) - print(all_packs) for pack in package_names: if pack not in all_packs: logging.info("The package %s is not\ - in the list with packags.", pack) +in the list with packags.", pack) logging.info("Did you mean any of the following: %s", ",".join(difflib.get_close_matches(pack, all_packs, 5))) exit(1) @@ -39,32 +38,40 @@ def verify_packages_tree(self): all_packs_list = set(flatten(self.packages_list.values())) all_packs_nodes = set(self.packages.keys()) # pylint: disable=E1101 + broken = [] # Every package is in a group for pack in all_packs_nodes: if pack not in all_packs_list: - logging.critical("Incosistant packages file!\ - The package %s is not in any group", pack) - exit(1) + broken.append(pack) + + if broken: + logging.critical("Inconsistent packages file!\ +The packages [%s] are not in any group", ','.join(broken)) + exit(1) + broken = [] # Every package in a group is in the nodes for pack in all_packs_list: if pack not in all_packs_nodes: - logging.critical("Incosistant packages file!\ - The package %s does not have node", pack) - exit(1) + broken.append(pack) + + if broken: + logging.critical("Incosistant packages file!\ +The packages [%s] do not have node", ','.join(broken)) + exit(1) # Every dependency is in the packages.json for pack in all_packs_nodes: for deb in self.get_dependencies(pack): if deb not in all_packs_nodes: logging.critical("%s is dependency\ - of %s but it is not in the packages.json", deb, pack) +of %s but it is not in the packages.json", deb, pack) exit(1) def get_dependencies(self, package): if package not in self.packages.keys(): # pylint: disable=E1101 logging.critical("The package %s\ - is not in the packages.json.", package) # pylint: disable=E1136 +is not in the packages.json.", package) # pylint: disable=E1136 exit(1) if 'dependencies' in self.packages[package].keys(): return self.packages[package]['dependencies'] # pylint: disable=E1136 @@ -102,7 +109,7 @@ def get_build_order(self, packages): available = [pack for pack, deps in sub_tree.items() if not deps] if not available: logging.critical('Build order cannot be generated.\ - The packages tree is maybe broken.') +The packages tree is maybe broken.') exit(1) for pack in available: build_order.append(pack) diff --git a/code_manager/core/installation.py b/code_manager/core/installation.py index 5072dca..f3255ae 100644 --- a/code_manager/core/installation.py +++ b/code_manager/core/installation.py @@ -76,7 +76,7 @@ def _add_installer(self, installer, file): def run_installer(self, name, installer): if installer not in self.installers.keys(): - logging.critical('There is no installer with the name %s', name) + logging.critical('There is no installer with the name %s', installer) if installer not in self.installer_objects.keys(): installer_obj = self.installers[installer]() @@ -84,20 +84,41 @@ def run_installer(self, name, installer): else: installer_obj = self.installer_objects[installer] - node = self.packages()[name] + node = self.packages[name] installer_obj.node = node if hasattr(installer_obj, 'manditory_attr') and isinstance(installer_obj.manditory_attr, list): for attr in installer_obj.manditory_attr: if attr not in node.keys(): logging.critical('The attribute %s is mandatory for the installer %s\ - but it is not in the package node.', - attr, installer_obj.name) + but it is not in the package node of %s.', + attr, installer_obj.name, name) result = installer_obj.execute(name) - if result > 0: - logging.critical('The installer %s failed to execute properly', installer_obj.nam) + if result is None: + logging.critical('The installer [%s] failed to execute properly', installer_obj.name) + + def install(self, package): + assert package is not None + node = self.packages[package] + + if 'install' not in node.keys(): + return 0 + + installer = node['install'] + if isinstance(installer, str): + self.run_installer(package, installer) + return 0 + elif isinstance(installer, list): + for inst in installer: + self.run_installer(package, inst) + return 0 + else: + logging.critical('Can\'t install %s.\ + Installation node is nor a list, nor a string.', package) + exit(1) + return None # class Installer: diff --git a/code_manager/core/manager.py b/code_manager/core/manager.py index 1d3a0af..db517ef 100644 --- a/code_manager/core/manager.py +++ b/code_manager/core/manager.py @@ -7,7 +7,6 @@ from code_manager.core.debgrapher import DebGrapher from code_manager.core.cache_container import CacheContainer from code_manager.utils.utils import flatten -from code_manager.utils.logger import debug_red class Manager(ConfigurationAware): @@ -30,52 +29,82 @@ def __init__(self): def _setup_all(self): self.installation.load_installer() self.cache.load_cache() + self.depender.verify_packages_tree() def _invoke(self): - pass + logging.info('Invoking installation with: %s', + ','.join(self.install_queue)) + logging.info('Steps configuration:') + logging.info('\tInstall:%s', self.install) + logging.info('\tBuild:%s', self.build) + logging.info('\tFetching:%s', self.fetching) + + self.depender.verify_package_list(self.install_queue) + + if self.install: + self._invoke_install() + + if self.fetching: + self._invoke_fetch() + + if self.fetching: + self._invoke_build() + + def _invoke_fetch(self): + for pack in self.install_queue: + with self.cache as cache: + + if not cache.is_fetched(pack): + if self.fetcher.download(pack, pack) is None: + logging.critical("The fetching of '%s' failed.", pack) + cache.set_fetched(pack, True) + else: + logging.info("\'%s\' is already fetched", pack) def _invoke_build(self): pass def _invoke_install(self): - pass - - def fetch_package(self, package): - with self.cache as cache: - if cache.is_fetched(package): - debug_red("The package '%s' is already fetched", package) - return None - if self.fetcher.download(package, package) is None: - debug_red("The fetching of '%s' failed.", package) - return None - else: - cache.set_fetched(package, True) - cache.set_root(package, package) - return 0 - - def fetch_group(self, group): - for pack in self.packages_list[group]: - self.fetch_package(pack) - - def fetch_thing(self, thing): - if thing in self.packages_list.keys(): - logging.info("'%s' is a group. Fetching all packages in it.", thing) - self.fetch_group(thing) - elif thing in flatten(self.packages_list.values()): - logging.info("'%s' is a package. Fetching it.", thing) - self.fetch_package(thing) - else: - logging.info("There is no thing with name '%s'", thing) - - def fetch(self, thing): - if isinstance(thing, list): - for thingy in thing: - self.fetch_thing(thingy) - elif isinstance(thing, str): - self.fetch_thing(thing) - else: - logging.critical("Can't install %s. It's \ - no string nor list", thing) + extended_queue = set(self.install_queue) + for pack in self.install_queue: + extended_queue.update(self.depender.get_deep_dependencies(pack)) + logging.debug('Extended queue: %s', ','.join(extended_queue)) + + ordered_packages = self.depender.get_build_order(list(extended_queue)) + logging.debug('Build order: %s', ','.join(ordered_packages)) + + self._check_install_nodes(ordered_packages) + + for pack in ordered_packages: + with self.cache as cache: + + if not cache.is_fetched(pack): + if self.fetcher.download(pack, pack) is None: + logging.critical("The fetching of '%s' failed.", pack) + cache.set_fetched(pack, True) + else: + logging.info("\'%s\' is already fetched", pack) + + if not cache.is_installed(pack): + if self.installation.install(pack) == 0: + logging.info("\'%s\' was installed", pack) + cache.set_installed(pack, True) + else: + logging.info("\'%s\' is already installed", pack) + # TODO: Update\Build the package here + + cache.set_built(pack, True) + + def _check_install_nodes(self, packages): + for pack in packages: + pack_node = self.packages[pack] + if 'install' not in pack_node.keys(): + continue + installer = pack_node['install'] + if not isinstance(installer, str) and not isinstance(installer, list): + logging.critical('Can\'t install %s.\ +Installation node is nor a list, nor a string.', pack) + exit(1) def _install_thing(self, thing): @@ -90,16 +119,16 @@ def _install_thing(self, thing): self.install_queue.append(thing) else: - logging.critical("There is no thing with name %s", thing) + self.install_queue.append(thing) def install_thing(self, thing, install=True, fetch=False, build=False): if install: self.install = True - self.fetching = True - self.build = True + self.fetching = False + self.build = False elif build: self.install = False - self.fetching = True + self.fetching = False self.build = True elif fetch: self.install = False @@ -113,5 +142,5 @@ def install_thing(self, thing, install=True, fetch=False, build=False): self._install_thing(thing) if self.install_queue: - self._setup_all() + # self._setup_all() self._invoke() diff --git a/code_manager/installers/command.py b/code_manager/installers/command.py index ed53103..6ee3893 100644 --- a/code_manager/installers/command.py +++ b/code_manager/installers/command.py @@ -10,10 +10,10 @@ def __init__(self): BasicInstaller.__init__(self) def execute(self, name): - pass + return 0 def update(self, name): - pass + return 0 ExportedClass = CommandInstaller diff --git a/code_manager/main.py b/code_manager/main.py index e4eb5ca..45ff76d 100755 --- a/code_manager/main.py +++ b/code_manager/main.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python from shutil import copyfile, copytree import os @@ -125,7 +125,7 @@ def get_arg_parser(): parser_install.add_argument( "--group", action="store", - metavar="name", + metavar='group', default=None, help="Should the packages be reinstalled", ) @@ -143,14 +143,16 @@ def get_arg_parser(): parser_fetch.add_argument( "packages", nargs="*", default=None, help="A list of packages to fetch" ) - parser_fetch.add_argument( - "--focre--clear", - dest="force_clear", - action="store_true", - default=False, - help="Will delete any\ - folders that stay on its way", - ) + + # TODO: Add support for foce clear + # parser_fetch.add_argument( + # "--focre--clear", + # dest="force_clear", + # action="store_true", + # default=False, + # help="Will delete anyfolders that stay on its way", + # ) + parser_fetch.add_argument( "--group", action="store", @@ -167,7 +169,7 @@ def get_arg_parser(): help="Builds the project of package", ) parser_build.add_argument( - "packages", nargs="*", default=None, help="A list of packages to fetch" + "packages", nargs="*", default=None, help="A list of packages to fetch." ) parser_build.add_argument( "--group", @@ -182,14 +184,13 @@ def get_arg_parser(): dest="noinstall", action="store_true", default=False, - help="If present, packages will\ - only be build but not installed", + help="If present, packages will only be build but not installed.", ) subparsers.add_parser( "list-packages", description="Lists the installed packages", - help="Lists the installed packages", + help="Lists the installed packages.", ) subparsers.add_parser("list-cache", help="Show the entries in the cache") subparsers.add_parser("clear-cache", help="Clears the entries in the cach file") @@ -198,15 +199,24 @@ def get_arg_parser(): def install(args, core): - pass + if args.group is not None: + core.install_thing(args.group, install=True) + else: + core.install_thing(args.packages, install=True) def fetch(args, core): - core.fetch(args.packages) + if args.group is not None: + core.install_thing(args.group, fetch=True) + else: + core.install_thing(args.packages, fetch=True) -def build(_, core): - pass +def build(args, core): + if args.group is not None: + core.install_thing(args.group, fetch=True) + else: + core.install_thing(args.packages, fetch=True) def list_packages(args, core): @@ -339,12 +349,11 @@ def main(): ConfigurationAware.set_configuration(CONFIG, INSTALL_SCRIPTS_DIR, CACHE, opt) - if args.debug: - logging.info("Code dir: %s", CODE_DIR) - logging.info("Usr dir: %s", USR_DIR) - logging.info("Packages file: %s", PACKAGES_FILE) - logging.info("Install script directory: %s", INSTALL_SCRIPTS_DIR) - logging.info("Cache file: %s", CACHE) + logging.debug("Code dir: %s", CODE_DIR) + logging.debug("Usr dir: %s", USR_DIR) + logging.debug("Packages file: %s", PACKAGES_FILE) + logging.debug("Install script directory: %s", INSTALL_SCRIPTS_DIR) + logging.debug("Cache file: %s", CACHE) core_manager = Manager()