From 37ecf5e5634eb8c55686f4b0bc0971b7054f9d65 Mon Sep 17 00:00:00 2001 From: Chris Griffith Date: Fri, 12 May 2017 17:58:32 -0500 Subject: [PATCH] Version 0.9.1 (#47) --- CHANGES.rst | 5 + reusables/__init__.py | 25 +- reusables/cli.py | 11 +- reusables/dt.py | 3 +- .../file_ops.py => file_operations.py} | 355 +++++++++++++++++- reusables/file_operations/__init__.py | 10 - reusables/file_operations/archive.py | 165 -------- reusables/file_operations/common_formats.py | 116 ------ reusables/file_operations/config.py | 99 ----- reusables/log.py | 5 +- .../helpers.py => process_helpers.py} | 2 +- reusables/processes/__init__.py | 8 - reusables/shared_variables.py | 3 +- .../numbers.py => string_manipulation.py} | 48 +++ reusables/string_manipulation/__init__.py | 8 - reusables/string_manipulation/strings.py | 56 --- reusables/{processes => }/tasker.py | 2 +- reusables/web.py | 3 +- reusables/wrappers.py | 3 +- 19 files changed, 435 insertions(+), 492 deletions(-) rename reusables/{file_operations/file_ops.py => file_operations.py} (62%) delete mode 100644 reusables/file_operations/__init__.py delete mode 100644 reusables/file_operations/archive.py delete mode 100644 reusables/file_operations/common_formats.py delete mode 100644 reusables/file_operations/config.py rename reusables/{processes/helpers.py => process_helpers.py} (99%) delete mode 100644 reusables/processes/__init__.py rename reusables/{string_manipulation/numbers.py => string_manipulation.py} (79%) delete mode 100644 reusables/string_manipulation/__init__.py delete mode 100644 reusables/string_manipulation/strings.py rename reusables/{processes => }/tasker.py (99%) diff --git a/CHANGES.rst b/CHANGES.rst index 2adae64..964312e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,11 @@ Changelog ========= +Version 0.9.1 +------------- + +- Fixing local imports not working when installed + Version 0.9.0 ------------- diff --git a/reusables/__init__.py b/reusables/__init__.py index bb7a44a..8cb0bfa 100644 --- a/reusables/__init__.py +++ b/reusables/__init__.py @@ -5,16 +5,19 @@ # # Copyright (c) 2014-2017 - Chris Griffith - MIT License from __future__ import absolute_import -from .namespace import * -from .dt import * -from .log import * -from .wrappers import * -from .web import * -from .shared_variables import * -from .cli import * -from .file_operations import * -from .processes import * -from .string_manipulation import * + +from reusables.string_manipulation import * +from reusables.cli import * +from reusables.dt import * +from reusables.file_operations import * +from reusables.log import * +from reusables.namespace import * +from reusables.tasker import * +from reusables.process_helpers import * +from reusables.shared_variables import * +from reusables.tasker import * +from reusables.web import * +from reusables.wrappers import * __author__ = "Chris Griffith" -__version__ = "0.9.0" +__version__ = "0.9.1" diff --git a/reusables/cli.py b/reusables/cli.py index 87130a6..dcfcc0d 100644 --- a/reusables/cli.py +++ b/reusables/cli.py @@ -5,16 +5,17 @@ # # Copyright (c) 2014-2017 - Chris Griffith - MIT License """ Functions to only be in an interactive instances to ease life. """ +from __future__ import absolute_import import os import logging import shutil from collections import deque -from .shared_variables import * -from .processes import run -from .shared_variables import win_based -from .file_operations import find_files_list -from .log import add_stream_handler +from reusables.shared_variables import * +from reusables.process_helpers import run +from reusables.shared_variables import win_based +from reusables.file_operations import find_files_list +from reusables.log import add_stream_handler __all__ = ['cmd', 'pushd', 'popd', 'pwd', 'cd', 'ls', 'find', 'head', 'cat', 'tail', 'cp'] diff --git a/reusables/dt.py b/reusables/dt.py index 2a57ae4..7ad7805 100644 --- a/reusables/dt.py +++ b/reusables/dt.py @@ -4,10 +4,11 @@ # Part of the Reusables package. # # Copyright (c) 2014-2017 - Chris Griffith - MIT License +from __future__ import absolute_import import datetime import re -from .namespace import Namespace +from reusables.namespace import Namespace __all__ = ['dt_exps', 'datetime_regex', 'now', 'datetime_format', 'datetime_from_iso', 'dtf', 'dtiso'] diff --git a/reusables/file_operations/file_ops.py b/reusables/file_operations.py similarity index 62% rename from reusables/file_operations/file_ops.py rename to reusables/file_operations.py index 31ee6c1..6949681 100644 --- a/reusables/file_operations/file_ops.py +++ b/reusables/file_operations.py @@ -5,22 +5,362 @@ # # Copyright (c) 2014-2017 - Chris Griffith - MIT License import os +import zipfile +import tarfile +import logging +import csv +import json import hashlib import glob -import logging from collections import defaultdict +try: + import ConfigParser as ConfigParser +except ImportError: + import configparser as ConfigParser -from ..shared_variables import * +from reusables.namespace import * +from reusables.shared_variables import * - -logger = logging.getLogger('reusables') - -__all__ = ['os_tree', 'check_filename', 'count_files', +__all__ = ['load_json', 'list_to_csv', 'save_json', 'csv_to_list', + 'extract', 'archive', 'config_dict', 'config_namespace', + 'os_tree', 'check_filename', 'count_files', 'directory_duplicates', 'dup_finder', 'file_hash', 'find_files', 'find_files_list', 'join_here', 'join_paths', 'remove_empty_directories', 'remove_empty_files', 'safe_filename', 'safe_path', 'touch'] +logger = logging.getLogger('reusables') + + +def extract(archive_file, path=".", delete_on_success=False, + enable_rar=False): + """ + Automatically detect archive type and extract all files to specified path. + + .. code:: python + + import os + + os.listdir(".") + # ['test_structure.zip'] + + reusables.extract("test_structure.zip") + + os.listdir(".") + # [ 'test_structure', 'test_structure.zip'] + + + :param archive_file: path to file to extract + :param path: location to extract to + :param delete_on_success: Will delete the original archive if set to True + :param enable_rar: include the rarfile import and extract + :return: path to extracted files + """ + + if not os.path.exists(archive_file) or not os.path.getsize(archive_file): + logger.error("File {0} unextractable".format(archive_file)) + raise OSError("File does not exist or has zero size") + + arch = None + if zipfile.is_zipfile(archive_file): + logger.debug("File {0} detected as a zip file".format(archive_file)) + arch = zipfile.ZipFile(archive_file) + elif tarfile.is_tarfile(archive_file): + logger.debug("File {0} detected as a tar file".format(archive_file)) + arch = tarfile.open(archive_file) + elif enable_rar: + import rarfile + if rarfile.is_rarfile(archive_file): + logger.debug("File {0} detected as " + "a rar file".format(archive_file)) + arch = rarfile.RarFile(archive_file) + + if not arch: + raise TypeError("File is not a known archive") + + logger.debug("Extracting files to {0}".format(path)) + + try: + arch.extractall(path=path) + finally: + arch.close() + + if delete_on_success: + logger.debug("Archive {0} will now be deleted".format(archive_file)) + os.unlink(archive_file) + + return os.path.abspath(path) + + +def archive(files_to_archive, name="archive.zip", archive_type=None, + overwrite=False, store=False, depth=None, err_non_exist=True, + allow_zip_64=True, **tarfile_kwargs): + """ Archive a list of files (or files inside a folder), can chose between + + - zip + - tar + - gz (tar.gz, tgz) + - bz2 (tar.bz2) + + .. code:: python + + reusables.archive(['reusables', '.travis.yml'], + name="my_archive.bz2") + # 'C:\\Users\\Me\\Reusables\\my_archive.bz2' + + :param files_to_archive: list of files and folders to archive + :param name: path and name of archive file + :param archive_type: auto-detects unless specified + :param overwrite: overwrite if archive exists + :param store: zipfile only, True will not compress files + :param depth: specify max depth for folders + :param err_non_exist: raise error if provided file does not exist + :param allow_zip_64: must be enabled for zip files larger than 2GB + :param tarfile_kwargs: extra args to pass to tarfile.open + :return: path to created archive + """ + if not isinstance(files_to_archive, (list, tuple)): + files_to_archive = [files_to_archive] + + if not archive_type: + if name.lower().endswith("zip"): + archive_type = "zip" + elif name.lower().endswith("gz"): + archive_type = "gz" + elif name.lower().endswith("z2"): + archive_type = "bz2" + elif name.lower().endswith("tar"): + archive_type = "tar" + else: + err_msg = ("Could not determine archive " + "type based off {0}".format(name)) + logger.error(err_msg) + raise ValueError(err_msg) + logger.debug("{0} file detected for {1}".format(archive_type, name)) + elif archive_type not in ("tar", "gz", "bz2", "zip"): + err_msg = ("archive_type must be zip, gz, bz2," + " or gz, was {0}".format(archive_type)) + logger.error(err_msg) + raise ValueError(err_msg) + + if not overwrite and os.path.exists(name): + err_msg = "File {0} exists and overwrite not specified".format(name) + logger.error(err_msg) + raise OSError(err_msg) + + if archive_type == "zip": + arch = zipfile.ZipFile(name, 'w', + zipfile.ZIP_STORED if store else + zipfile.ZIP_DEFLATED, + allowZip64=allow_zip_64) + write = arch.write + elif archive_type in ("tar", "gz", "bz2"): + mode = archive_type if archive_type != "tar" else "" + arch = tarfile.open(name, 'w:{0}'.format(mode), **tarfile_kwargs) + write = arch.add + else: + raise ValueError("archive_type must be zip, gz, bz2, or gz") + + try: + for file_path in files_to_archive: + if os.path.isfile(file_path): + if err_non_exist and not os.path.exists(file_path): + raise OSError("File {0} does not exist".format(file_path)) + write(file_path) + elif os.path.isdir(file_path): + for nf in find_files(file_path, abspath=False, depth=depth): + write(nf) + except (Exception, KeyboardInterrupt) as err: + logger.exception("Could not archive {0}".format(files_to_archive)) + try: + arch.close() + finally: + os.unlink(name) + raise err + else: + arch.close() + + return os.path.abspath(name) + + +def list_to_csv(my_list, csv_file): + """ + Save a matrix (list of lists) to a file as a CSV + + .. code:: python + + my_list = [["Name", "Location"], + ["Chris", "South Pole"], + ["Harry", "Depth of Winter"], + ["Bob", "Skull"]] + + reusables.list_to_csv(my_list, "example.csv") + + example.csv + + .. code:: csv + + "Name","Location" + "Chris","South Pole" + "Harry","Depth of Winter" + "Bob","Skull" + + :param my_list: list of lists to save to CSV + :param csv_file: File to save data to + """ + if PY3: + csv_handler = open(csv_file, 'w', newline='') + else: + csv_handler = open(csv_file, 'wb') + + try: + writer = csv.writer(csv_handler, delimiter=',', quoting=csv.QUOTE_ALL) + writer.writerows(my_list) + finally: + csv_handler.close() + + +def csv_to_list(csv_file): + """ + Open and transform a CSV file into a matrix (list of lists). + + .. code:: python + + reusables.csv_to_list("example.csv") + # [['Name', 'Location'], + # ['Chris', 'South Pole'], + # ['Harry', 'Depth of Winter'], + # ['Bob', 'Skull']] + + :param csv_file: Path to CSV file as str + :return: list + """ + with open(csv_file, 'r' if PY3 else 'rb') as f: + return list(csv.reader(f)) + + +def load_json(json_file, **kwargs): + """ + Open and load data from a JSON file + + .. code:: python + + reusables.load_json("example.json") + # {u'key_1': u'val_1', u'key_for_dict': {u'sub_dict_key': 8}} + + :param json_file: Path to JSON file as string + :param kwargs: Additional arguments for the json.load command + :return: Dictionary + """ + with open(json_file) as f: + return json.load(f, **kwargs) + + +def save_json(data, json_file, indent=4, **kwargs): + """ + Takes a dictionary and saves it to a file as JSON + + .. code:: python + + my_dict = {"key_1": "val_1", + "key_for_dict": {"sub_dict_key": 8}} + + reusables.save_json(my_dict,"example.json") + + example.json + + .. code:: + + { + "key_1": "val_1", + "key_for_dict": { + "sub_dict_key": 8 + } + } + + :param data: dictionary to save as JSON + :param json_file: Path to save file location as str + :param indent: Format the JSON file with so many numbers of spaces + :param kwargs: Additional arguments for the json.dump command + """ + with open(json_file, "w") as f: + json.dump(data, f, indent=indent, **kwargs) + + +def config_dict(config_file=None, auto_find=False, verify=True, **cfg_options): + """ + Return configuration options as dictionary. Accepts either a single + config file or a list of files. Auto find will search for all .cfg, .config + and .ini in the execution directory and package root (unsafe but handy). + + .. code:: python + + reusables.config_dict(os.path.join("test", "data", "test_config.ini")) + # {'General': {'example': 'A regular string'}, + # 'Section 2': {'anint': '234', + # 'examplelist': '234,123,234,543', + # 'floatly': '4.4', + # 'my_bool': 'yes'}} + + + :param config_file: path or paths to the files location + :param auto_find: look for a config type file at this location or below + :param verify: make sure the file exists before trying to read + :param cfg_options: options to pass to the parser + :return: dictionary of the config files + """ + if not config_file: + config_file = [] + + cfg_parser = ConfigParser.ConfigParser(**cfg_options) + cfg_files = [] + + if config_file: + if not isinstance(config_file, (list, tuple)): + if isinstance(config_file, str): + cfg_files.append(config_file) + else: + raise TypeError("config_files must be a list or a string") + else: + cfg_files.extend(config_file) + + if auto_find: + cfg_files.extend(find_files_list( + current_root if isinstance(auto_find, bool) else auto_find, + ext=(".cfg", ".config", ".ini"))) + + logger.info("config files to be used: {0}".format(cfg_files)) + + if verify: + cfg_parser.read([cfg for cfg in cfg_files if os.path.exists(cfg)]) + else: + cfg_parser.read(cfg_files) + + return dict((section, dict(cfg_parser.items(section))) + for section in cfg_parser.sections()) + + +def config_namespace(config_file=None, auto_find=False, + verify=True, **cfg_options): + """ + Return configuration options as a Namespace. + + .. code:: python + + reusables.config_namespace(os.path.join("test", "data", + "test_config.ini")) + # + + + :param config_file: path or paths to the files location + :param auto_find: look for a config type file at this location or below + :param verify: make sure the file exists before trying to read + :param cfg_options: options to pass to the parser + :return: Namespace of the config files + """ + return ConfigNamespace(**config_dict(config_file, auto_find, + verify, **cfg_options)) + def _walk(directory, enable_scandir=False, **kwargs): """ @@ -516,3 +856,6 @@ def safe_path(path, replacement="_"): + + + diff --git a/reusables/file_operations/__init__.py b/reusables/file_operations/__init__.py deleted file mode 100644 index e90db55..0000000 --- a/reusables/file_operations/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -# -# Part of the Reusables package. -# -# Copyright (c) 2014-2017 - Chris Griffith - MIT License -from .archive import * -from .common_formats import * -from .file_ops import * -from .config import * diff --git a/reusables/file_operations/archive.py b/reusables/file_operations/archive.py deleted file mode 100644 index a49630e..0000000 --- a/reusables/file_operations/archive.py +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -# -# Part of the Reusables package. -# -# Copyright (c) 2014-2017 - Chris Griffith - MIT License -import os -import zipfile -import tarfile -import logging - -from .file_ops import find_files - -logger = logging.getLogger('reusables') - - -def extract(archive_file, path=".", delete_on_success=False, - enable_rar=False): - """ - Automatically detect archive type and extract all files to specified path. - - .. code:: python - - import os - - os.listdir(".") - # ['test_structure.zip'] - - reusables.extract("test_structure.zip") - - os.listdir(".") - # [ 'test_structure', 'test_structure.zip'] - - - :param archive_file: path to file to extract - :param path: location to extract to - :param delete_on_success: Will delete the original archive if set to True - :param enable_rar: include the rarfile import and extract - :return: path to extracted files - """ - - if not os.path.exists(archive_file) or not os.path.getsize(archive_file): - logger.error("File {0} unextractable".format(archive_file)) - raise OSError("File does not exist or has zero size") - - arch = None - if zipfile.is_zipfile(archive_file): - logger.debug("File {0} detected as a zip file".format(archive_file)) - arch = zipfile.ZipFile(archive_file) - elif tarfile.is_tarfile(archive_file): - logger.debug("File {0} detected as a tar file".format(archive_file)) - arch = tarfile.open(archive_file) - elif enable_rar: - import rarfile - if rarfile.is_rarfile(archive_file): - logger.debug("File {0} detected as " - "a rar file".format(archive_file)) - arch = rarfile.RarFile(archive_file) - - if not arch: - raise TypeError("File is not a known archive") - - logger.debug("Extracting files to {0}".format(path)) - - try: - arch.extractall(path=path) - finally: - arch.close() - - if delete_on_success: - logger.debug("Archive {0} will now be deleted".format(archive_file)) - os.unlink(archive_file) - - return os.path.abspath(path) - - -def archive(files_to_archive, name="archive.zip", archive_type=None, - overwrite=False, store=False, depth=None, err_non_exist=True, - allow_zip_64=True, **tarfile_kwargs): - """ Archive a list of files (or files inside a folder), can chose between - - - zip - - tar - - gz (tar.gz, tgz) - - bz2 (tar.bz2) - - .. code:: python - - reusables.archive(['reusables', '.travis.yml'], - name="my_archive.bz2") - # 'C:\\Users\\Me\\Reusables\\my_archive.bz2' - - :param files_to_archive: list of files and folders to archive - :param name: path and name of archive file - :param archive_type: auto-detects unless specified - :param overwrite: overwrite if archive exists - :param store: zipfile only, True will not compress files - :param depth: specify max depth for folders - :param err_non_exist: raise error if provided file does not exist - :param allow_zip_64: must be enabled for zip files larger than 2GB - :param tarfile_kwargs: extra args to pass to tarfile.open - :return: path to created archive - """ - if not isinstance(files_to_archive, (list, tuple)): - files_to_archive = [files_to_archive] - - if not archive_type: - if name.lower().endswith("zip"): - archive_type = "zip" - elif name.lower().endswith("gz"): - archive_type = "gz" - elif name.lower().endswith("z2"): - archive_type = "bz2" - elif name.lower().endswith("tar"): - archive_type = "tar" - else: - err_msg = ("Could not determine archive " - "type based off {0}".format(name)) - logger.error(err_msg) - raise ValueError(err_msg) - logger.debug("{0} file detected for {1}".format(archive_type, name)) - elif archive_type not in ("tar", "gz", "bz2", "zip"): - err_msg = ("archive_type must be zip, gz, bz2," - " or gz, was {0}".format(archive_type)) - logger.error(err_msg) - raise ValueError(err_msg) - - if not overwrite and os.path.exists(name): - err_msg = "File {0} exists and overwrite not specified".format(name) - logger.error(err_msg) - raise OSError(err_msg) - - if archive_type == "zip": - arch = zipfile.ZipFile(name, 'w', - zipfile.ZIP_STORED if store else - zipfile.ZIP_DEFLATED, - allowZip64=allow_zip_64) - write = arch.write - elif archive_type in ("tar", "gz", "bz2"): - mode = archive_type if archive_type != "tar" else "" - arch = tarfile.open(name, 'w:{0}'.format(mode), **tarfile_kwargs) - write = arch.add - else: - raise ValueError("archive_type must be zip, gz, bz2, or gz") - - try: - for file_path in files_to_archive: - if os.path.isfile(file_path): - if err_non_exist and not os.path.exists(file_path): - raise OSError("File {0} does not exist".format(file_path)) - write(file_path) - elif os.path.isdir(file_path): - for nf in find_files(file_path, abspath=False, depth=depth): - write(nf) - except (Exception, KeyboardInterrupt) as err: - logger.exception("Could not archive {0}".format(files_to_archive)) - try: - arch.close() - finally: - os.unlink(name) - raise err - else: - arch.close() - - return os.path.abspath(name) diff --git a/reusables/file_operations/common_formats.py b/reusables/file_operations/common_formats.py deleted file mode 100644 index 63cea52..0000000 --- a/reusables/file_operations/common_formats.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -# -# Part of the Reusables package. -# -# Copyright (c) 2014-2017 - Chris Griffith - MIT License -import csv -import json - -from ..shared_variables import * - -__all__ = ['load_json', 'list_to_csv', 'save_json', 'csv_to_list'] - - -def list_to_csv(my_list, csv_file): - """ - Save a matrix (list of lists) to a file as a CSV - - .. code:: python - - my_list = [["Name", "Location"], - ["Chris", "South Pole"], - ["Harry", "Depth of Winter"], - ["Bob", "Skull"]] - - reusables.list_to_csv(my_list, "example.csv") - - example.csv - - .. code:: csv - - "Name","Location" - "Chris","South Pole" - "Harry","Depth of Winter" - "Bob","Skull" - - :param my_list: list of lists to save to CSV - :param csv_file: File to save data to - """ - if PY3: - csv_handler = open(csv_file, 'w', newline='') - else: - csv_handler = open(csv_file, 'wb') - - try: - writer = csv.writer(csv_handler, delimiter=',', quoting=csv.QUOTE_ALL) - writer.writerows(my_list) - finally: - csv_handler.close() - - -def csv_to_list(csv_file): - """ - Open and transform a CSV file into a matrix (list of lists). - - .. code:: python - - reusables.csv_to_list("example.csv") - # [['Name', 'Location'], - # ['Chris', 'South Pole'], - # ['Harry', 'Depth of Winter'], - # ['Bob', 'Skull']] - - :param csv_file: Path to CSV file as str - :return: list - """ - with open(csv_file, 'r' if PY3 else 'rb') as f: - return list(csv.reader(f)) - - -def load_json(json_file, **kwargs): - """ - Open and load data from a JSON file - - .. code:: python - - reusables.load_json("example.json") - # {u'key_1': u'val_1', u'key_for_dict': {u'sub_dict_key': 8}} - - :param json_file: Path to JSON file as string - :param kwargs: Additional arguments for the json.load command - :return: Dictionary - """ - with open(json_file) as f: - return json.load(f, **kwargs) - - -def save_json(data, json_file, indent=4, **kwargs): - """ - Takes a dictionary and saves it to a file as JSON - - .. code:: python - - my_dict = {"key_1": "val_1", - "key_for_dict": {"sub_dict_key": 8}} - - reusables.save_json(my_dict,"example.json") - - example.json - - .. code:: - - { - "key_1": "val_1", - "key_for_dict": { - "sub_dict_key": 8 - } - } - - :param data: dictionary to save as JSON - :param json_file: Path to save file location as str - :param indent: Format the JSON file with so many numbers of spaces - :param kwargs: Additional arguments for the json.dump command - """ - with open(json_file, "w") as f: - json.dump(data, f, indent=indent, **kwargs) diff --git a/reusables/file_operations/config.py b/reusables/file_operations/config.py deleted file mode 100644 index d9eda92..0000000 --- a/reusables/file_operations/config.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -# -# Part of the Reusables package. -# -# Copyright (c) 2014-2017 - Chris Griffith - MIT License -import os -try: - import ConfigParser as ConfigParser -except ImportError: - import configparser as ConfigParser -import logging - -from .file_ops import find_files_list -from ..namespace import ConfigNamespace -from ..shared_variables import current_root - -logger = logging.getLogger('reusables') - - -def config_dict(config_file=None, auto_find=False, verify=True, **cfg_options): - """ - Return configuration options as dictionary. Accepts either a single - config file or a list of files. Auto find will search for all .cfg, .config - and .ini in the execution directory and package root (unsafe but handy). - - .. code:: python - - reusables.config_dict(os.path.join("test", "data", "test_config.ini")) - # {'General': {'example': 'A regular string'}, - # 'Section 2': {'anint': '234', - # 'examplelist': '234,123,234,543', - # 'floatly': '4.4', - # 'my_bool': 'yes'}} - - - :param config_file: path or paths to the files location - :param auto_find: look for a config type file at this location or below - :param verify: make sure the file exists before trying to read - :param cfg_options: options to pass to the parser - :return: dictionary of the config files - """ - if not config_file: - config_file = [] - - cfg_parser = ConfigParser.ConfigParser(**cfg_options) - cfg_files = [] - - if config_file: - if not isinstance(config_file, (list, tuple)): - if isinstance(config_file, str): - cfg_files.append(config_file) - else: - raise TypeError("config_files must be a list or a string") - else: - cfg_files.extend(config_file) - - if auto_find: - cfg_files.extend(find_files_list( - current_root if isinstance(auto_find, bool) else auto_find, - ext=(".cfg", ".config", ".ini"))) - - logger.info("config files to be used: {0}".format(cfg_files)) - - if verify: - cfg_parser.read([cfg for cfg in cfg_files if os.path.exists(cfg)]) - else: - cfg_parser.read(cfg_files) - - return dict((section, dict(cfg_parser.items(section))) - for section in cfg_parser.sections()) - - -def config_namespace(config_file=None, auto_find=False, - verify=True, **cfg_options): - """ - Return configuration options as a Namespace. - - .. code:: python - - reusables.config_namespace(os.path.join("test", "data", - "test_config.ini")) - # - - - :param config_file: path or paths to the files location - :param auto_find: look for a config type file at this location or below - :param verify: make sure the file exists before trying to read - :param cfg_options: options to pass to the parser - :return: Namespace of the config files - """ - return ConfigNamespace(**config_dict(config_file, auto_find, - verify, **cfg_options)) - - - - - - diff --git a/reusables/log.py b/reusables/log.py index 6d03e8c..ad83552 100644 --- a/reusables/log.py +++ b/reusables/log.py @@ -7,14 +7,15 @@ """ Logging helper functions and common log formats. """ +from __future__ import absolute_import import warnings import logging import sys from logging.handlers import (RotatingFileHandler, TimedRotatingFileHandler) -from .namespace import Namespace -from .shared_variables import sizes +from reusables.namespace import Namespace +from reusables.shared_variables import sizes __all__ = ['log_formats', 'get_logger', 'get_registered_loggers', 'get_file_handler', 'get_stream_handler', 'add_file_handler', diff --git a/reusables/processes/helpers.py b/reusables/process_helpers.py similarity index 99% rename from reusables/processes/helpers.py rename to reusables/process_helpers.py index 567f540..f2383ba 100644 --- a/reusables/processes/helpers.py +++ b/reusables/process_helpers.py @@ -10,7 +10,7 @@ from multiprocessing import pool from functools import partial -from ..shared_variables import * +from reusables.shared_variables import * __all__ = ['run', 'run_in_pool'] diff --git a/reusables/processes/__init__.py b/reusables/processes/__init__.py deleted file mode 100644 index 7ff2835..0000000 --- a/reusables/processes/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -# -# Part of the Reusables package. -# -# Copyright (c) 2014-2017 - Chris Griffith - MIT License -from .helpers import * -from .tasker import * diff --git a/reusables/shared_variables.py b/reusables/shared_variables.py index cf42c32..55f7a96 100644 --- a/reusables/shared_variables.py +++ b/reusables/shared_variables.py @@ -4,12 +4,13 @@ # Part of the Reusables package. # # Copyright (c) 2014-2017 - Chris Griffith - MIT License +from __future__ import absolute_import import re as _re import os as _os import sys as _sys import tempfile as _tempfile -from .namespace import Namespace +from reusables.namespace import Namespace python_version = _sys.version_info[0:3] version_string = ".".join([str(x) for x in python_version]) diff --git a/reusables/string_manipulation/numbers.py b/reusables/string_manipulation.py similarity index 79% rename from reusables/string_manipulation/numbers.py rename to reusables/string_manipulation.py index 991d1d7..6db2089 100644 --- a/reusables/string_manipulation/numbers.py +++ b/reusables/string_manipulation.py @@ -21,6 +21,54 @@ 9: "septillion", 10: "octillion", 11: "nonillion", 12: "decillion"} +def cut(string, characters=2, trailing="normal"): + """ + Split a string into a list of N characters each. + + .. code:: python + + reusables.cut("abcdefghi") + # ['ab', 'cd', 'ef', 'gh', 'i'] + + trailing gives you the following options: + + * normal: leaves remaining characters in their own last position + * remove: return the list without the remainder characters + * combine: add the remainder characters to the previous set + * error: raise an IndexError if there are remaining characters + + .. code:: python + + reusables.cut("abcdefghi", 2, "error") + # Traceback (most recent call last): + # ... + # IndexError: String of length 9 not divisible by 2 to splice + + reusables.cut("abcdefghi", 2, "remove") + # ['ab', 'cd', 'ef', 'gh'] + + reusables.cut("abcdefghi", 2, "combine") + # ['ab', 'cd', 'ef', 'ghi'] + + :param string: string to modify + :param characters: how many characters to split it into + :param trailing: "normal", "remove", "combine", or "error" + :return: list of the cut string + """ + split_str = [string[i:i + characters] for + i in range(0, len(string), characters)] + + if trailing != "normal" and len(split_str[-1]) != characters: + if trailing.lower() == "remove": + return split_str[:-1] + if trailing.lower() == "combine" and len(split_str) >= 2: + return split_str[:-2] + [split_str[-2] + split_str[-1]] + if trailing.lower() == "error": + raise IndexError("String of length {0} not divisible by {1} to" + " cut".format(len(string), characters)) + return split_str + + def int_to_roman(integer): """ Convert an integer into a string of roman numbers. diff --git a/reusables/string_manipulation/__init__.py b/reusables/string_manipulation/__init__.py deleted file mode 100644 index 7960cb3..0000000 --- a/reusables/string_manipulation/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -# -# Part of the Reusables package. -# -# Copyright (c) 2014-2017 - Chris Griffith - MIT License -from .numbers import * -from .strings import * diff --git a/reusables/string_manipulation/strings.py b/reusables/string_manipulation/strings.py deleted file mode 100644 index 93b1ce9..0000000 --- a/reusables/string_manipulation/strings.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -# -# Part of the Reusables package. -# -# Copyright (c) 2014-2017 - Chris Griffith - MIT License - - -def cut(string, characters=2, trailing="normal"): - """ - Split a string into a list of N characters each. - - .. code:: python - - reusables.cut("abcdefghi") - # ['ab', 'cd', 'ef', 'gh', 'i'] - - trailing gives you the following options: - - * normal: leaves remaining characters in their own last position - * remove: return the list without the remainder characters - * combine: add the remainder characters to the previous set - * error: raise an IndexError if there are remaining characters - - .. code:: python - - reusables.cut("abcdefghi", 2, "error") - # Traceback (most recent call last): - # ... - # IndexError: String of length 9 not divisible by 2 to splice - - reusables.cut("abcdefghi", 2, "remove") - # ['ab', 'cd', 'ef', 'gh'] - - reusables.cut("abcdefghi", 2, "combine") - # ['ab', 'cd', 'ef', 'ghi'] - - :param string: string to modify - :param characters: how many characters to split it into - :param trailing: "normal", "remove", "combine", or "error" - :return: list of the cut string - """ - split_str = [string[i:i + characters] for - i in range(0, len(string), characters)] - - if trailing != "normal" and len(split_str[-1]) != characters: - if trailing.lower() == "remove": - return split_str[:-1] - if trailing.lower() == "combine" and len(split_str) >= 2: - return split_str[:-2] + [split_str[-2] + split_str[-1]] - if trailing.lower() == "error": - raise IndexError("String of length {0} not divisible by {1} to" - " cut".format(len(string), characters)) - return split_str - - diff --git a/reusables/processes/tasker.py b/reusables/tasker.py similarity index 99% rename from reusables/processes/tasker.py rename to reusables/tasker.py index 8764e69..05f5f52 100644 --- a/reusables/processes/tasker.py +++ b/reusables/tasker.py @@ -14,7 +14,7 @@ import logging import datetime -from ..shared_variables import win_based +from reusables.shared_variables import win_based __all__ = ['Tasker'] diff --git a/reusables/web.py b/reusables/web.py index 04ae26c..12e3efb 100644 --- a/reusables/web.py +++ b/reusables/web.py @@ -4,6 +4,7 @@ # Part of the Reusables package. # # Copyright (c) 2014-2017 - Chris Griffith - MIT License +from __future__ import absolute_import import os import logging import time @@ -20,7 +21,7 @@ from SimpleHTTPServer import SimpleHTTPRequestHandler as _handler from SocketServer import TCPServer as _server -from .file_operations import safe_filename +from reusables.file_operations import safe_filename __all__ = ['download', 'ThreadedServer', 'url_to_ip', 'url_to_ips', 'ip_to_url'] diff --git a/reusables/wrappers.py b/reusables/wrappers.py index e4a173f..43c0248 100644 --- a/reusables/wrappers.py +++ b/reusables/wrappers.py @@ -4,6 +4,7 @@ # Part of the Reusables package. # # Copyright (c) 2014-2017 - Chris Griffith - MIT License +from __future__ import absolute_import import time from threading import Lock from functools import wraps @@ -14,7 +15,7 @@ except ImportError: import Queue as _queue -from .shared_variables import python_version, ReusablesError +from reusables.shared_variables import python_version, ReusablesError __all__ = ['unique', 'time_it', 'catch_it', 'log_exception', 'retry_it', 'lock_it', 'queue_it']