From ee4d076fc465a806abd96b1ff71cdd7fb7455f01 Mon Sep 17 00:00:00 2001 From: Nissar Chababy Date: Thu, 3 Oct 2024 14:04:52 +0200 Subject: [PATCH] Introduction of a way to provide a configuration file and folder from the CLI. (#380) * Introduction of a way to provide a configuration file from the CLI. This patch fixes #377. Contributors: * @spirillen * cleanup: make downloaders less dependent from PyFunceble.storage. * cleanup: allow overwrite of config directory. * cleanup: do not declare a var if only used once. * Introduction of a way provide a custom configuration folder. This patch fixes #377. Contributors: * @spirillen * fix issue when reloading configuration. Indeed, before this patch, any custom configuration may have been destroyed when config_file and config_dir has been changed. * Add missing unit tests. * Fix some loopholes in unit tests. --- PyFunceble/cli/entry_points/pyfunceble/cli.py | 22 +++ .../cli/filesystem/dir_structure/base.py | 3 +- PyFunceble/cli/migrators/base.py | 48 ++++- .../csv_file/inactive_source_delete.py | 2 +- .../migrators/csv_file/whois_registrar_add.py | 2 +- .../cli/migrators/file_cleanup/hashes_file.py | 2 +- .../cli/migrators/file_cleanup/mining_file.py | 2 +- .../file_cleanup/production_config_file.py | 2 +- PyFunceble/cli/migrators/json2csv/inactive.py | 2 +- PyFunceble/cli/migrators/json2csv/whois.py | 2 +- PyFunceble/cli/scripts/production.py | 93 +++++++--- PyFunceble/cli/system/integrator.py | 65 +++++-- PyFunceble/cli/system/launcher.py | 19 +- PyFunceble/config/loader.py | 168 +++++++++++++++--- PyFunceble/database/credential/base.py | 49 ++++- PyFunceble/dataset/base.py | 6 +- PyFunceble/dataset/db_base.py | 44 +++++ PyFunceble/dataset/iana.py | 9 +- PyFunceble/dataset/inactive/csv.py | 2 +- PyFunceble/dataset/ipv4_reputation.py | 14 +- PyFunceble/dataset/public_suffix.py | 12 +- PyFunceble/dataset/user_agent.py | 11 +- PyFunceble/dataset/whois/csv.py | 2 +- PyFunceble/downloader/base.py | 69 ++++++- PyFunceble/downloader/iana.py | 15 +- PyFunceble/downloader/ipv4_reputation.py | 13 +- PyFunceble/downloader/public_suffix.py | 16 +- PyFunceble/downloader/user_agents.py | 16 +- PyFunceble/query/whois/query_tool.py | 1 + PyFunceble/storage.py | 19 +- docs/use/configuration/index.md | 3 +- docs/use/configuration/location.md | 13 +- tests/config/test_compare.py | 54 ++++++ tests/config/test_loader.py | 97 ++++++++++ tests/query/test_platform.py | 12 +- tests/query/whois/test_query_tool.py | 80 ++++++++- 36 files changed, 795 insertions(+), 194 deletions(-) diff --git a/PyFunceble/cli/entry_points/pyfunceble/cli.py b/PyFunceble/cli/entry_points/pyfunceble/cli.py index 55f769aa..473c4fcc 100644 --- a/PyFunceble/cli/entry_points/pyfunceble/cli.py +++ b/PyFunceble/cli/entry_points/pyfunceble/cli.py @@ -1197,6 +1197,28 @@ def get_default_group_data() -> List[Tuple[List[str], dict]]: "version": "%(prog)s " + PyFunceble.storage.PROJECT_VERSION, }, ), + ( + [ + "--config-file", + ], + { + "dest": "config_file", + "type": str, + "help": "Sets the configuration file to use. It can be a\n" + "local or remote file. Please note that this configuration can be\n" + "overwritten by your overwrite configuration file.", + }, + ), + ( + [ + "--config-dir", + ], + { + "dest": "config_dir", + "type": os.path.realpath, + "help": "Sets the configuration directory to use.", + }, + ), ] diff --git a/PyFunceble/cli/filesystem/dir_structure/base.py b/PyFunceble/cli/filesystem/dir_structure/base.py index 5bf1bf4a..50091b56 100644 --- a/PyFunceble/cli/filesystem/dir_structure/base.py +++ b/PyFunceble/cli/filesystem/dir_structure/base.py @@ -59,7 +59,6 @@ import importlib_resources as package_resources -import PyFunceble.storage from PyFunceble.cli.filesystem.cleanup import FilesystemCleanup from PyFunceble.cli.filesystem.dir_base import FilesystemDirBase @@ -78,7 +77,7 @@ def __init__( ) -> None: with package_resources.path( "PyFunceble.data.infrastructure", - PyFunceble.storage.DISTRIBUTED_DIR_STRUCTURE_FILENAME, + "dir_structure_production.json", ) as file_path: self.std_source_file = str(file_path) diff --git a/PyFunceble/cli/migrators/base.py b/PyFunceble/cli/migrators/base.py index 969f5eed..afa60fe0 100644 --- a/PyFunceble/cli/migrators/base.py +++ b/PyFunceble/cli/migrators/base.py @@ -54,6 +54,7 @@ from sqlalchemy.orm import Session +import PyFunceble.storage from PyFunceble.cli.continuous_integration.base import ContinuousIntegrationBase @@ -66,11 +67,19 @@ class MigratorBase: continuous_integration: Optional[ContinuousIntegrationBase] = None db_session: Optional[Session] = None + _config_dir: Optional[str] = None print_action_to_stdout: bool = False - def __init__(self, print_action_to_stdout: bool = False) -> None: + def __init__( + self, print_action_to_stdout: bool = False, *, config_dir: Optional[str] = None + ) -> None: self.print_action_to_stdout = print_action_to_stdout + if config_dir is not None: + self._config_dir = config_dir + else: + self._config_dir = PyFunceble.storage.CONFIG_DIRECTORY + self.__post_init__() def __post_init__(self) -> None: @@ -78,6 +87,43 @@ def __post_init__(self) -> None: A method to be called (automatically) after the __init__ execution. """ + @property + def config_dir(self) -> Optional[str]: + """ + Provides the current state of the :code:`_config_dir` attribute. + """ + + return self._config_dir + + @config_dir.setter + def config_dir(self, value: str) -> None: + """ + Sets the configuration directory. + + :param value: + The value to set. + + :raise TypeError: + When value is not a :py:class:`str`. + """ + + if not isinstance(value, str): + raise TypeError(f" should be {str}, {type(value)} given.") + + self._config_dir = value + + def set_config_dir(self, value: str) -> "MigratorBase": + """ + Sets the configuration directory. + + :param value: + The value to set. + """ + + self.config_dir = value + + return self + def start(self) -> "MigratorBase": """ Starts the migration. diff --git a/PyFunceble/cli/migrators/csv_file/inactive_source_delete.py b/PyFunceble/cli/migrators/csv_file/inactive_source_delete.py index 1d6bf269..20c8c5b4 100644 --- a/PyFunceble/cli/migrators/csv_file/inactive_source_delete.py +++ b/PyFunceble/cli/migrators/csv_file/inactive_source_delete.py @@ -77,7 +77,7 @@ class InactiveDatasetDeleteSourceColumnMigrator(CSVFileMigratorBase): def __post_init__(self) -> None: self.source_file = os.path.join( - PyFunceble.storage.CONFIG_DIRECTORY, PyFunceble.cli.storage.INACTIVE_DB_FILE + self.config_dir, PyFunceble.cli.storage.INACTIVE_DB_FILE ) return super().__post_init__() diff --git a/PyFunceble/cli/migrators/csv_file/whois_registrar_add.py b/PyFunceble/cli/migrators/csv_file/whois_registrar_add.py index 1f3e842d..9f02be98 100644 --- a/PyFunceble/cli/migrators/csv_file/whois_registrar_add.py +++ b/PyFunceble/cli/migrators/csv_file/whois_registrar_add.py @@ -71,7 +71,7 @@ class WhoisDatasetAddRegistrarColumnMigrator(CSVFileMigratorBase): def __post_init__(self) -> None: self.source_file = os.path.join( - PyFunceble.storage.CONFIG_DIRECTORY, PyFunceble.cli.storage.WHOIS_DB_FILE + self.config_dir, PyFunceble.cli.storage.WHOIS_DB_FILE ) return super().__post_init__() diff --git a/PyFunceble/cli/migrators/file_cleanup/hashes_file.py b/PyFunceble/cli/migrators/file_cleanup/hashes_file.py index 72aed269..565ebac2 100644 --- a/PyFunceble/cli/migrators/file_cleanup/hashes_file.py +++ b/PyFunceble/cli/migrators/file_cleanup/hashes_file.py @@ -64,7 +64,7 @@ class HashesFileCleanupMigrator(FileClenupMigratorBase): def __post_init__(self) -> None: self.source_file = os.path.join( - PyFunceble.storage.CONFIG_DIRECTORY, PyFunceble.cli.storage.HASHES_FILENAME + self.config_dir, PyFunceble.cli.storage.HASHES_FILENAME ) return super().__post_init__() diff --git a/PyFunceble/cli/migrators/file_cleanup/mining_file.py b/PyFunceble/cli/migrators/file_cleanup/mining_file.py index 2786b04a..c4332c86 100644 --- a/PyFunceble/cli/migrators/file_cleanup/mining_file.py +++ b/PyFunceble/cli/migrators/file_cleanup/mining_file.py @@ -64,7 +64,7 @@ class MiningFileCleanupMigrator(FileClenupMigratorBase): def __post_init__(self) -> None: self.source_file = os.path.join( - PyFunceble.storage.CONFIG_DIRECTORY, PyFunceble.cli.storage.MINING_OLD_FILE + self.config_dir, PyFunceble.cli.storage.MINING_OLD_FILE ) return super().__post_init__() diff --git a/PyFunceble/cli/migrators/file_cleanup/production_config_file.py b/PyFunceble/cli/migrators/file_cleanup/production_config_file.py index b551b5d3..00f0f316 100644 --- a/PyFunceble/cli/migrators/file_cleanup/production_config_file.py +++ b/PyFunceble/cli/migrators/file_cleanup/production_config_file.py @@ -65,7 +65,7 @@ class ProductionConfigFileCleanupMigrator(FileClenupMigratorBase): def __post_init__(self) -> None: self.source_file = os.path.join( - PyFunceble.storage.CONFIG_DIRECTORY, + self.config_dir, PyFunceble.storage.DISTRIBUTED_CONFIGURATION_FILENAME, ) diff --git a/PyFunceble/cli/migrators/json2csv/inactive.py b/PyFunceble/cli/migrators/json2csv/inactive.py index c96268a0..122e5ad6 100644 --- a/PyFunceble/cli/migrators/json2csv/inactive.py +++ b/PyFunceble/cli/migrators/json2csv/inactive.py @@ -75,7 +75,7 @@ class InactiveJSON2CSVMigrator(JSON2CSVMigratorBase): def __post_init__(self) -> None: self.source_file = os.path.join( - PyFunceble.storage.CONFIG_DIRECTORY, + self.config_dir, PyFunceble.cli.storage.INACTIVE_DB_OLD_FILE, ) diff --git a/PyFunceble/cli/migrators/json2csv/whois.py b/PyFunceble/cli/migrators/json2csv/whois.py index 29f0433c..703200bb 100644 --- a/PyFunceble/cli/migrators/json2csv/whois.py +++ b/PyFunceble/cli/migrators/json2csv/whois.py @@ -73,7 +73,7 @@ class WhoisJSON2CSVMigrator(JSON2CSVMigratorBase): def __post_init__(self) -> None: self.source_file = os.path.join( - PyFunceble.storage.CONFIG_DIRECTORY, + self.config_dir, PyFunceble.cli.storage.WHOIS_DB_OLD_FILE, ) diff --git a/PyFunceble/cli/scripts/production.py b/PyFunceble/cli/scripts/production.py index 47b4f428..667297f0 100644 --- a/PyFunceble/cli/scripts/production.py +++ b/PyFunceble/cli/scripts/production.py @@ -84,10 +84,6 @@ class ProductionPrep: you should run this only if your are developing PyFunceble. """ - VERSION_FILE_PATH: str = os.path.join( - PyFunceble.storage.CONFIG_DIRECTORY, - PyFunceble.cli.storage.DISTRIBUTED_VERSION_FILENAME, - ) AVAILABLE_BRANCHES: List[str] = ["dev", "master"] regex_helper: RegexHelper = RegexHelper() @@ -101,15 +97,23 @@ class ProductionPrep: """ _branch: Optional[str] = None + _config_dir: Optional[str] = None previous_version: Optional[str] = None """ Provides the previous version (from :code:`version_file_content`) """ - def __init__(self, branch: Optional[str] = None) -> None: + def __init__( + self, branch: Optional[str] = None, *, config_dir: Optional[str] = None + ) -> None: + if config_dir is not None: + self.config_dir = config_dir + else: + self.config_dir = PyFunceble.storage.CONFIG_DIRECTORY + self.version_file_content = self.dict_helper.from_yaml_file( - self.VERSION_FILE_PATH + self.version_file_path ) self.previous_version = copy.deepcopy( @@ -138,6 +142,54 @@ def wrapper(self, *args, **kwargs): return wrapper + @property + def config_dir(self) -> Optional[str]: + """ + Provides the current state of the :code:`_config_dir` attribute. + """ + + return self._config_dir + + @config_dir.setter + def config_dir(self, value: str) -> None: + """ + Sets the configuration directory. + + :param value: + The value to set. + + :raise TypeError: + When value is not a :py:class:`str`. + """ + + if not isinstance(value, str): + raise TypeError(f" should be {str}, {type(value)} given.") + + self._config_dir = value + + def set_config_dir(self, value: str) -> "ProductionPrep": + """ + Sets the configuration directory. + + :param value: + The value to set. + """ + + self.config_dir = value + + return self + + @property + def version_file_path(self) -> str: + """ + Provides the path to the version file. + """ + + return os.path.join( + self.config_dir, + PyFunceble.cli.storage.DISTRIBUTED_VERSION_FILENAME, + ) + @property def branch(self) -> Optional[str]: """ @@ -270,8 +322,7 @@ def update_urls(self, file: str) -> "ProductionPrep": return self - @staticmethod - def update_code_format() -> "ProductionPrep": + def update_code_format(self) -> "ProductionPrep": """ Updates the format of the source code using black. """ @@ -304,16 +355,14 @@ def format_file(file: str, isortconfig: isort.settings.Config) -> None: isort_config = isort.settings.Config(settings_file="setup.cfg") files = [ - os.path.join(PyFunceble.storage.CONFIG_DIRECTORY, "setup.py"), + os.path.join(self.config_dir, "setup.py"), ] for file in files: format_file(file, isort_config) for root, _, files in os.walk( - os.path.join( - PyFunceble.storage.CONFIG_DIRECTORY, PyFunceble.storage.PROJECT_NAME - ) + os.path.join(self.config_dir, PyFunceble.storage.PROJECT_NAME) ): if "__pycache__" in root: continue @@ -324,9 +373,7 @@ def format_file(file: str, isortconfig: isort.settings.Config) -> None: format_file(os.path.join(root, file), isort_config) - for root, _, files in os.walk( - os.path.join(PyFunceble.storage.CONFIG_DIRECTORY, "tests") - ): + for root, _, files in os.walk(os.path.join(self.config_dir, "tests")): if "__pycache__" in root: continue @@ -346,12 +393,10 @@ def update_code_urls(self) -> "ProductionPrep": ".keep", ] - self.update_urls(os.path.join(PyFunceble.storage.CONFIG_DIRECTORY, "setup.py")) + self.update_urls(os.path.join(self.config_dir, "setup.py")) for root, _, files in os.walk( - os.path.join( - PyFunceble.storage.CONFIG_DIRECTORY, PyFunceble.storage.PROJECT_NAME - ) + os.path.join(self.config_dir, PyFunceble.storage.PROJECT_NAME) ): if "__pycache__" in root: continue @@ -362,9 +407,7 @@ def update_code_urls(self) -> "ProductionPrep": self.update_urls(os.path.join(root, file)) - for root, _, files in os.walk( - os.path.join(PyFunceble.storage.CONFIG_DIRECTORY, "tests") - ): + for root, _, files in os.walk(os.path.join(self.config_dir, "tests")): if "__pycache__" in root: continue @@ -409,9 +452,7 @@ def update_setup_py(self) -> "ProductionPrep": ), ] - self.file_helper.set_path( - os.path.join(PyFunceble.storage.CONFIG_DIRECTORY, "setup.py") - ) + self.file_helper.set_path(os.path.join(self.config_dir, "setup.py")) if not self.file_helper.exists(): raise FileNotFoundError(self.file_helper.path) @@ -453,7 +494,7 @@ def update_version_file(self) -> "ProductionPrep": ) self.dict_helper.set_subject(self.version_file_content).to_yaml_file( - self.VERSION_FILE_PATH + self.version_file_path ) PyFunceble.facility.Logger.info( diff --git a/PyFunceble/cli/system/integrator.py b/PyFunceble/cli/system/integrator.py index 4821d135..39a6fc9f 100644 --- a/PyFunceble/cli/system/integrator.py +++ b/PyFunceble/cli/system/integrator.py @@ -52,6 +52,8 @@ """ import os +import sys +import traceback import colorama @@ -59,7 +61,9 @@ import PyFunceble.cli.factory import PyFunceble.cli.storage import PyFunceble.facility +import PyFunceble.helpers.exceptions import PyFunceble.storage +from PyFunceble.cli.continuous_integration.exceptions import StopExecution from PyFunceble.cli.system.base import SystemBase from PyFunceble.helpers.dict import DictHelper @@ -250,23 +254,56 @@ def start(self) -> "SystemIntegrator": Starts a group of actions provided by this interface. """ - if hasattr(self.args, "output_location") and self.args.output_location: - PyFunceble.cli.storage.OUTPUT_DIRECTORY = os.path.realpath( - os.path.join( - self.args.output_location, - PyFunceble.cli.storage.OUTPUTS.parent_directory, - ) - ) + try: + self.init_logger() - self.init_logger() + if hasattr(self.args, "output_location") and self.args.output_location: + PyFunceble.cli.storage.OUTPUT_DIRECTORY = os.path.realpath( + os.path.join( + self.args.output_location, + PyFunceble.cli.storage.OUTPUTS.parent_directory, + ) + ) - PyFunceble.facility.Logger.debug("Given arguments:\n%r", self.args) + if hasattr(self.args, "config_dir") and self.args.config_dir: + PyFunceble.facility.ConfigLoader.set_config_dir(self.args.config_dir) + PyFunceble.storage.CONFIG_DIRECTORY = self.args.config_dir + + if hasattr(self.args, "config_file") and self.args.config_file: + PyFunceble.facility.ConfigLoader.set_remote_config_location( + self.args.config_file + ).reload() + + PyFunceble.facility.Logger.debug("Given arguments:\n%r", self.args) + + self.inject_into_config() + self.check_config() + self.check_deprecated() + + PyFunceble.cli.facility.CredentialLoader.start() + PyFunceble.cli.factory.DBSession.init_db_sessions() + except (KeyboardInterrupt, StopExecution): + pass + except Exception as exception: # pylint: disable=broad-except + PyFunceble.facility.Logger.critical( + "Fatal error.", + exc_info=True, + ) + if isinstance(exception, PyFunceble.helpers.exceptions.UnableToDownload): + message = ( + f"{colorama.Fore.RED}{colorama.Style.BRIGHT}Unable to download " + f"{exception}" + ) + else: + message = ( + f"{colorama.Fore.RED}{colorama.Style.BRIGHT}Fatal Error: " + f"{exception}" + ) + print(message) - self.inject_into_config() - self.check_config() - self.check_deprecated() + if PyFunceble.facility.Logger.authorized: + print(traceback.format_exc()) - PyFunceble.cli.facility.CredentialLoader.start() - PyFunceble.cli.factory.DBSession.init_db_sessions() + sys.exit(1) return self diff --git a/PyFunceble/cli/system/launcher.py b/PyFunceble/cli/system/launcher.py index beedcf88..b4deb955 100644 --- a/PyFunceble/cli/system/launcher.py +++ b/PyFunceble/cli/system/launcher.py @@ -74,6 +74,7 @@ import PyFunceble.cli.utils.sort import PyFunceble.cli.utils.stdout import PyFunceble.facility +import PyFunceble.helpers.exceptions import PyFunceble.storage from PyFunceble.checker.syntax.url import URLSyntaxChecker from PyFunceble.cli.continuous_integration.base import ContinuousIntegrationBase @@ -1138,12 +1139,20 @@ def start(self) -> "SystemLauncher": "Fatal error.", exc_info=True, ) - print( - f"{colorama.Fore.RED}{colorama.Style.BRIGHT}Fatal Error: " - f"{exception}" - ) + if isinstance(exception, PyFunceble.helpers.exceptions.UnableToDownload): + message = ( + f"{colorama.Fore.RED}{colorama.Style.BRIGHT}Unable to download " + f"{exception}" + ) + else: + message = ( + f"{colorama.Fore.RED}{colorama.Style.BRIGHT}Fatal Error: " + f"{exception}" + ) + print(message) - print(traceback.format_exc()) + if PyFunceble.facility.Logger.authorized: + print(traceback.format_exc()) sys.exit(1) PyFunceble.cli.utils.stdout.print_thanks() diff --git a/PyFunceble/config/loader.py b/PyFunceble/config/loader.py index 638aa333..10e18241 100644 --- a/PyFunceble/config/loader.py +++ b/PyFunceble/config/loader.py @@ -60,6 +60,7 @@ import importlib_resources as package_resources from box import Box +from dotenv import load_dotenv from yaml.error import MarkedYAMLError import PyFunceble.cli.storage @@ -69,6 +70,7 @@ from PyFunceble.downloader.public_suffix import PublicSuffixDownloader from PyFunceble.downloader.user_agents import UserAgentsDownloader from PyFunceble.helpers.dict import DictHelper +from PyFunceble.helpers.download import DownloadHelper from PyFunceble.helpers.environment_variable import EnvironmentVariableHelper from PyFunceble.helpers.file import FileHelper from PyFunceble.helpers.merge import Merge @@ -86,31 +88,42 @@ class ConfigLoader: :code:`PYFUNCEBLE_AUTO_CONFIGURATION` environment variable. """ - path_to_config: Optional[str] = None + _path_to_config: Optional[str] = None + _remote_config_location: Optional[str] = None path_to_default_config: Optional[str] = None path_to_overwrite_config: Optional[str] = None _custom_config: dict = {} _merge_upstream: bool = False + _config_dir: Optional[str] = None file_helper: FileHelper = FileHelper() dict_helper: DictHelper = DictHelper() - def __init__(self, merge_upstream: Optional[bool] = None) -> None: + def __init__( + self, merge_upstream: Optional[bool] = None, *, config_dir: Optional[str] = None + ) -> None: with package_resources.path( "PyFunceble.data.infrastructure", PyFunceble.storage.DISTRIBUTED_CONFIGURATION_FILENAME, ) as file_path: self.path_to_default_config = str(file_path) + if config_dir is not None: + self.config_dir = config_dir + else: + self.config_dir = PyFunceble.storage.CONFIG_DIRECTORY + self.path_to_config = os.path.join( - PyFunceble.storage.CONFIG_DIRECTORY, + self.config_dir, PyFunceble.storage.CONFIGURATION_FILENAME, ) + self.path_to_remote_config = None + self.path_to_overwrite_config = os.path.join( - PyFunceble.storage.CONFIG_DIRECTORY, - PyFunceble.storage.CONFIGURATION_OVERWRITE_FILENAME, + self.config_dir, + ".PyFunceble.overwrite.yaml", ) if merge_upstream is not None: @@ -132,7 +145,7 @@ def wrapper(self, *args, **kwargs): result = func(self, *args, **kwargs) # pylint: disable=not-callable if self.is_already_loaded(): - self.start() + self.reload(keep_custom=True) return result @@ -200,6 +213,44 @@ def is_already_loaded() -> bool: return bool(PyFunceble.storage.CONFIGURATION) + @property + def config_dir(self) -> Optional[str]: + """ + Provides the current state of the :code:`_config_dir` attribute. + """ + + return self._config_dir + + @config_dir.setter + @reload_config + def config_dir(self, value: str) -> None: + """ + Sets the configuration directory. + + :param value: + The value to set. + + :raise TypeError: + When value is not a :py:class:`str`. + """ + + if not isinstance(value, str): + raise TypeError(f" should be {str}, {type(value)} given.") + + self._config_dir = value + + def set_config_dir(self, value: str) -> "ConfigLoader": + """ + Sets the configuration directory. + + :param value: + The value to set. + """ + + self.config_dir = value + + return self + @property def custom_config(self) -> dict: """ @@ -274,23 +325,44 @@ def set_merge_upstream(self, value: bool) -> "ConfigLoader": return self - def config_file_exist( - self, - ) -> bool: # pragma: no cover ## Existance checker already tested. + @property + def remote_config_location(self) -> Optional[str]: """ - Checks if the config file exists. + Provides the current state of the :code:`_remote_config_location` attribute. """ - return FileHelper(self.path_to_config).exists() + return self._remote_config_location - def default_config_file_exist( - self, - ) -> bool: # pragma: no cover ## Existance checker already tested. + @remote_config_location.setter + def remote_config_location(self, value: Optional[str]) -> None: """ - Checks if the default configuration file exists. + Updates the value of :code:`_remote_config_location` attribute. + + :raise TypeError: + When :code:`value` is not a :py:class:`str`. """ - return self.file_helper.set_path(self.path_to_default_config).exists() + if value is not None and not isinstance(value, str): + raise TypeError(f" should be {str}, {type(value)} given.") + + if not value.startswith("http") and not value.startswith("https"): + self.path_to_remote_config = os.path.realpath(value) + else: + self.path_to_remote_config = os.path.join( + self.config_dir, + ".PyFunceble.remote.yaml", + ) + + self._remote_config_location = value + + def set_remote_config_location(self, value: Optional[str]) -> "ConfigLoader": + """ + Updates the value of :code:`_remote_config_location` attribute. + """ + + self.remote_config_location = value + + return self def install_missing_infrastructure_files( self, @@ -347,9 +419,34 @@ def is_3_x_version(config: dict) -> bool: return config and "days_between_inactive_db_clean" in config + def download_remote_config(src: str, dest: str = None) -> None: + """ + Downloads the remote configuration. + + :param src: + The source to download from. + :param dest: + The destination to download + """ + + if src and (src.startswith("http") or src.startswith("https")): + if dest is None: + destination = os.path.join( + self.config_dir, + os.path.basename(dest), + ) + else: + destination = dest + + DownloadHelper(src).download_text(destination=destination) + if not self.is_already_loaded(): self.install_missing_infrastructure_files() self.download_dynamic_infrastructure_files() + download_remote_config( + self.remote_config_location, self.path_to_remote_config + ) + download_remote_config(self.path_to_config) try: config = self.dict_helper.from_yaml_file(self.path_to_config) @@ -377,15 +474,22 @@ def is_3_x_version(config: dict) -> bool: self.dict_helper.set_subject(config).to_yaml_file(self.path_to_config) + if ( + self.path_to_remote_config + and self.file_helper.set_path(self.path_to_remote_config).exists() + ): + remote_data = self.dict_helper.from_yaml_file(self.path_to_remote_config) + + if isinstance(remote_data, dict): + config = Merge(remote_data).into(config) + if self.file_helper.set_path(self.path_to_overwrite_config).exists(): overwrite_data = self.dict_helper.from_yaml_file( self.path_to_overwrite_config ) if isinstance(overwrite_data, dict): - config = Merge( - self.dict_helper.from_yaml_file(self.path_to_overwrite_config) - ).into(config) + config = Merge(overwrite_data).into(config) else: # pragma: no cover ## Just make it visible to end-user. self.file_helper.write("") @@ -415,11 +519,26 @@ def get_configured_value(self, entry: str) -> Any: return PyFunceble.storage.FLATTEN_CONFIGURATION[entry] + def reload(self, keep_custom: bool = False) -> "ConfigLoader": + """ + Reloads the configuration. + + :param bool keep_custom: + If set to :code:`True`, we keep the custom configuration, otherwise + we delete it. + """ + + self.destroy(keep_custom=keep_custom) + self.start() + def start(self) -> "ConfigLoader": """ Starts the loading processIs. """ + load_dotenv(os.path.join(self.config_dir, ".env")) + load_dotenv(os.path.join(self.config_dir, PyFunceble.storage.ENV_FILENAME)) + config = self.get_config_file_content() if self.custom_config: @@ -445,9 +564,13 @@ def start(self) -> "ConfigLoader": return self - def destroy(self) -> "ConfigLoader": + def destroy(self, keep_custom: bool = False) -> "ConfigLoader": """ Destroys everything loaded. + + :param bool keep_custom: + If set to :code:`True`, we keep the custom configuration, otherwise + we delete it. """ try: @@ -462,7 +585,8 @@ def destroy(self) -> "ConfigLoader": except (AttributeError, TypeError): # pragma: no cover ## Safety. pass - # This is not a mistake. - self._custom_config = {} + if not keep_custom: + # This is not a mistake. + self._custom_config = {} return self diff --git a/PyFunceble/database/credential/base.py b/PyFunceble/database/credential/base.py index 3c639233..d0c04f3c 100644 --- a/PyFunceble/database/credential/base.py +++ b/PyFunceble/database/credential/base.py @@ -100,6 +100,8 @@ class CredentialBase: _password: Optional[str] = None _charset: Optional[str] = None + _config_dir: Optional[str] = None + def __init__( self, *, @@ -109,6 +111,7 @@ def __init__( username: Optional[str] = None, password: Optional[str] = None, charset: Optional[str] = None, + config_dir: Optional[str] = None, ) -> None: if host is not None: self.host = host @@ -140,11 +143,14 @@ def __init__( else: self.charset = self.STD_CHARSET + if config_dir is not None: + self.config_dir = config_dir + else: + self.config_dir = PyFunceble.storage.CONFIG_DIRECTORY + self.dotenv_locations = [ os.path.realpath(PyFunceble.storage.ENV_FILENAME), - os.path.join( - PyFunceble.storage.CONFIG_DIRECTORY, PyFunceble.storage.ENV_FILENAME - ), + os.path.join(self.config_dir, PyFunceble.storage.ENV_FILENAME), ] def ensure_protocol_is_given(func): # pylint: disable=no-self-argument @@ -165,6 +171,43 @@ def wrapper(self, *args, **kwargs): return wrapper + @property + def config_dir(self) -> Optional[str]: + """ + Provides the current state of the :code:`_config_dir` attribute. + """ + + return self._config_dir + + @config_dir.setter + def config_dir(self, value: str) -> None: + """ + Sets the configuration directory. + + :param value: + The value to set. + + :raise TypeError: + When value is not a :py:class:`str`. + """ + + if not isinstance(value, str): + raise TypeError(f" should be {str}, {type(value)} given.") + + self._config_dir = value + + def set_config_dir(self, value: str) -> "CredentialBase": + """ + Sets the configuration directory. + + :param value: + The value to set. + """ + + self.config_dir = value + + return self + @property def host(self) -> Optional[str]: """ diff --git a/PyFunceble/dataset/base.py b/PyFunceble/dataset/base.py index fcce23b7..6e754903 100644 --- a/PyFunceble/dataset/base.py +++ b/PyFunceble/dataset/base.py @@ -65,7 +65,7 @@ class DatasetBase: """ STORAGE_INDEX: Optional[str] = None - DOWNLOADER: Optional[DownloaderBase] = None + downloader: Optional[DownloaderBase] = None source_file: Optional[str] = None @@ -129,9 +129,9 @@ def get_content(self) -> Optional[dict]: file_helper = FileHelper(self.source_file) if not file_helper.exists() and bool( - self.DOWNLOADER + self.downloader ): # pragma: no cover ## This is just a safety endpoint. - self.DOWNLOADER.start() + self.downloader.start() if not file_helper.exists(): raise FileNotFoundError(file_helper.path) diff --git a/PyFunceble/dataset/db_base.py b/PyFunceble/dataset/db_base.py index 3a1f24a1..02288e1a 100644 --- a/PyFunceble/dataset/db_base.py +++ b/PyFunceble/dataset/db_base.py @@ -53,6 +53,7 @@ import functools from typing import Any, Generator, List, Optional +import PyFunceble.storage from PyFunceble.dataset.base import DatasetBase @@ -77,7 +78,13 @@ def __init__( *, authorized: Optional[bool] = None, remove_unneeded_fields: Optional[bool] = None, + config_dir: Optional[str] = None, ) -> None: + if config_dir is not None: + self.config_dir = config_dir + else: + self.config_dir = PyFunceble.storage.CONFIG_DIRECTORY + if authorized is not None: self.set_authorized(authorized) @@ -108,6 +115,43 @@ def wrapper(self, *args, **kwargs): return inner_metdhod + @property + def config_dir(self) -> Optional[str]: + """ + Provides the current state of the :code:`_config_dir` attribute. + """ + + return self._config_dir + + @config_dir.setter + def config_dir(self, value: str) -> None: + """ + Sets the configuration directory. + + :param value: + The value to set. + + :raise TypeError: + When value is not a :py:class:`str`. + """ + + if not isinstance(value, str): + raise TypeError(f" should be {str}, {type(value)} given.") + + self._config_dir = value + + def set_config_dir(self, value: str) -> "DBDatasetBase": + """ + Sets the configuration directory. + + :param value: + The value to set. + """ + + self.config_dir = value + + return self + @property def authorized(self) -> Optional[bool]: """ diff --git a/PyFunceble/dataset/iana.py b/PyFunceble/dataset/iana.py index b5faf136..9743999b 100644 --- a/PyFunceble/dataset/iana.py +++ b/PyFunceble/dataset/iana.py @@ -50,10 +50,8 @@ limitations under the License. """ -import os from typing import Any, Optional -import PyFunceble.storage from PyFunceble.dataset.base import DatasetBase from PyFunceble.downloader.iana import IANADownloader @@ -64,12 +62,11 @@ class IanaDataset(DatasetBase): """ STORAGE_INDEX: str = "IANA" - DOWNLOADER: IANADownloader = IANADownloader() + downloader: Optional[IANADownloader] = None def __init__(self) -> None: - self.source_file = os.path.join( - PyFunceble.storage.CONFIG_DIRECTORY, PyFunceble.storage.IANA_DUMP_FILENAME - ) + self.downloader = IANADownloader() + self.source_file = self.downloader.destination def __contains__(self, value: Any) -> bool: if value.startswith("."): diff --git a/PyFunceble/dataset/inactive/csv.py b/PyFunceble/dataset/inactive/csv.py index 7e92b3f4..3cf7327f 100644 --- a/PyFunceble/dataset/inactive/csv.py +++ b/PyFunceble/dataset/inactive/csv.py @@ -68,7 +68,7 @@ class CSVInactiveDataset(CSVDatasetBase, InactiveDatasetBase): def __post_init__(self) -> None: self.source_file = os.path.join( - PyFunceble.storage.CONFIG_DIRECTORY, PyFunceble.cli.storage.INACTIVE_DB_FILE + self.config_dir, PyFunceble.cli.storage.INACTIVE_DB_FILE ) return super().__post_init__() diff --git a/PyFunceble/dataset/ipv4_reputation.py b/PyFunceble/dataset/ipv4_reputation.py index c590cba7..41a04441 100644 --- a/PyFunceble/dataset/ipv4_reputation.py +++ b/PyFunceble/dataset/ipv4_reputation.py @@ -50,10 +50,8 @@ limitations under the License. """ -import os from typing import Any, Optional -import PyFunceble.storage from PyFunceble.dataset.base import DatasetBase from PyFunceble.downloader.ipv4_reputation import IPV4ReputationDownloader from PyFunceble.helpers.file import FileHelper @@ -65,13 +63,11 @@ class IPV4ReputationDataset(DatasetBase): """ STORAGE_INDEX: Optional[str] = None - DOWNLOADER: IPV4ReputationDownloader = IPV4ReputationDownloader() + downloader: Optional[IPV4ReputationDownloader] = None def __init__(self) -> None: - self.source_file = os.path.join( - PyFunceble.storage.CONFIG_DIRECTORY, - PyFunceble.storage.IPV4_REPUTATION_FILENAME, - ) + self.downloader = IPV4ReputationDownloader() + self.source_file = self.downloader.destination def __contains__(self, value: Any) -> bool: with self.get_content() as file_stream: @@ -95,9 +91,9 @@ def get_content(self) -> open: file_helper = FileHelper(self.source_file) - if not file_helper.exists() and bool(self.DOWNLOADER): # pragma: no cover + if not file_helper.exists() and bool(self.downloader): # pragma: no cover ## pragma reason: Safety. - self.DOWNLOADER.start() + self.downloader.start() if not file_helper.exists(): raise FileNotFoundError(file_helper.path) diff --git a/PyFunceble/dataset/public_suffix.py b/PyFunceble/dataset/public_suffix.py index 10337855..f2f7d564 100644 --- a/PyFunceble/dataset/public_suffix.py +++ b/PyFunceble/dataset/public_suffix.py @@ -50,10 +50,8 @@ limitations under the License. """ -import os -from typing import Any, List +from typing import Any, List, Optional -import PyFunceble.storage from PyFunceble.dataset.base import DatasetBase from PyFunceble.downloader.public_suffix import PublicSuffixDownloader @@ -64,13 +62,11 @@ class PublicSuffixDataset(DatasetBase): """ STORAGE_INDEX: str = "PUBLIC_SUFFIX" - DOWNLOADER: PublicSuffixDownloader = PublicSuffixDownloader() + downloader: Optional[PublicSuffixDownloader] = None def __init__(self) -> None: - self.source_file = os.path.join( - PyFunceble.storage.CONFIG_DIRECTORY, - PyFunceble.storage.PUBLIC_SUFFIX_DUMP_FILENAME, - ) + self.downloader = PublicSuffixDownloader() + self.source_file = self.downloader.destination def __contains__(self, value: Any) -> bool: if value.startswith("."): diff --git a/PyFunceble/dataset/user_agent.py b/PyFunceble/dataset/user_agent.py index 4ee0aab3..f24681fe 100644 --- a/PyFunceble/dataset/user_agent.py +++ b/PyFunceble/dataset/user_agent.py @@ -50,9 +50,8 @@ limitations under the License. """ -import os import secrets -from typing import Any +from typing import Any, Optional from warnings import warn import PyFunceble.storage @@ -66,16 +65,14 @@ class UserAgentDataset(DatasetBase): """ STORAGE_INDEX: str = "USER_AGENTS" - DOWNLOADER: UserAgentsDownloader = UserAgentsDownloader() + downloader: Optional[UserAgentsDownloader] = None preferred_browser: str = "chrome" preferred_platform: str = "linux" def __init__(self) -> None: - self.source_file = os.path.join( - PyFunceble.storage.CONFIG_DIRECTORY, - PyFunceble.storage.USER_AGENT_FILENAME, - ) + self.downloader = UserAgentsDownloader() + self.source_file = self.downloader.destination def __contains__(self, value: Any) -> bool: content = self.get_content() diff --git a/PyFunceble/dataset/whois/csv.py b/PyFunceble/dataset/whois/csv.py index ecd84ac5..3f5f11bf 100644 --- a/PyFunceble/dataset/whois/csv.py +++ b/PyFunceble/dataset/whois/csv.py @@ -67,7 +67,7 @@ class CSVWhoisDataset(CSVDatasetBase, WhoisDatasetBase): def __post_init__(self) -> None: self.source_file = os.path.join( - PyFunceble.storage.CONFIG_DIRECTORY, PyFunceble.cli.storage.WHOIS_DB_FILE + self.config_dir, PyFunceble.cli.storage.WHOIS_DB_FILE ) return super().__post_init__() diff --git a/PyFunceble/downloader/base.py b/PyFunceble/downloader/base.py index 110800f3..c44e0a6a 100644 --- a/PyFunceble/downloader/base.py +++ b/PyFunceble/downloader/base.py @@ -89,25 +89,47 @@ class DownloaderBase: every hour. """ + DEFAULT_DOWNLOAD_URL: Optional[str] = None + """ + The URL to download. + """ + + DEFAULT_FILENAME: Optional[str] = None + """ + The name of the file to download. + """ + all_downtimes: Optional[dict] = {} """ Stores the download time of all files (self managed). """ + _config_dir: Optional[str] = None _destination: Optional[str] = None _download_link: Optional[str] = None - dict_helper: DictHelper = DictHelper() + dict_helper: Optional[DictHelper] = None + + def __init__(self, *, config_dir: Optional[str] = None) -> None: + self.dict_helper = DictHelper() + + if config_dir is not None: + self.config_dir = config_dir + else: + self.config_dir = PyFunceble.storage.CONFIG_DIRECTORY - def __init__(self) -> None: self.downtimes_file = FileHelper( - os.path.join( - PyFunceble.storage.CONFIG_DIRECTORY, PyFunceble.storage.DOWN_FILENAME - ) + os.path.join(self.config_dir, ".pyfunceble_intern_downtime.json") ) self.all_downtimes.update(self.get_all_downtimes()) + if self.DEFAULT_DOWNLOAD_URL is not None: + self.download_link = self.DEFAULT_DOWNLOAD_URL + + if self.DEFAULT_FILENAME is not None: + self.destination = os.path.join(self.config_dir, self.DEFAULT_FILENAME) + @property def authorized(self) -> bool: """ @@ -116,6 +138,43 @@ def authorized(self) -> bool: raise NotImplementedError() + @property + def config_dir(self) -> Optional[str]: + """ + Provides the current state of the :code:`_config_dir` attribute. + """ + + return self._config_dir + + @config_dir.setter + def config_dir(self, value: str) -> None: + """ + Sets the configuration directory. + + :param value: + The value to set. + + :raise TypeError: + When value is not a :py:class:`str`. + """ + + if not isinstance(value, str): + raise TypeError(f" should be {str}, {type(value)} given.") + + self._config_dir = value + + def set_config_dir(self, value: str) -> "DownloaderBase": + """ + Sets the configuration directory. + + :param value: + The value to set. + """ + + self.config_dir = value + + return self + @property def destination(self) -> Optional[str]: """ diff --git a/PyFunceble/downloader/iana.py b/PyFunceble/downloader/iana.py index aab17b89..02090f60 100644 --- a/PyFunceble/downloader/iana.py +++ b/PyFunceble/downloader/iana.py @@ -50,9 +50,6 @@ limitations under the License. """ -import os - -import PyFunceble.storage from PyFunceble.downloader.base import DownloaderBase @@ -64,14 +61,10 @@ class IANADownloader(DownloaderBase): DOWNTIME_INDEX: str = "iana" DOWNLOAD_FREQUENCY: int = 1 - def __init__(self) -> None: - self.destination = os.path.join( - PyFunceble.storage.CONFIG_DIRECTORY, - PyFunceble.storage.IANA_DUMP_FILENAME, - ) - self.download_link = PyFunceble.storage.IANA_DUMP_LINK - - super().__init__() + DEFAULT_DOWNLOAD_URL: str = ( + "https://raw.githubusercontent.com/PyFunceble/iana/master/iana-domains-db.json" + ) + DEFAULT_FILENAME: str = "iana-domains-db.json" @property def authorized(self) -> bool: diff --git a/PyFunceble/downloader/ipv4_reputation.py b/PyFunceble/downloader/ipv4_reputation.py index 4e14a857..394b123a 100644 --- a/PyFunceble/downloader/ipv4_reputation.py +++ b/PyFunceble/downloader/ipv4_reputation.py @@ -50,9 +50,6 @@ limitations under the License. """ -import os - -import PyFunceble.storage from PyFunceble.downloader.base import DownloaderBase @@ -64,14 +61,8 @@ class IPV4ReputationDownloader(DownloaderBase): DOWNTIME_INDEX: str = "ipv4_reputation" DOWNLOAD_FREQUENCY: int = 1 - def __init__(self) -> None: - self.destination = os.path.join( - PyFunceble.storage.CONFIG_DIRECTORY, - PyFunceble.storage.IPV4_REPUTATION_FILENAME, - ) - self.download_link = PyFunceble.storage.IPV4_REPUTATION_DUMP_LINK - - super().__init__() + DEFAULT_DOWNLOAD_URL: str = "https://reputation.alienvault.com/reputation.data" + DEFAULT_FILENAME: str = "ipv4_reputation.data" @property def authorized(self) -> bool: # pragma: no cover ## Always True. diff --git a/PyFunceble/downloader/public_suffix.py b/PyFunceble/downloader/public_suffix.py index de0bc64e..3344555a 100644 --- a/PyFunceble/downloader/public_suffix.py +++ b/PyFunceble/downloader/public_suffix.py @@ -50,9 +50,6 @@ limitations under the License. """ -import os - -import PyFunceble.storage from PyFunceble.downloader.base import DownloaderBase @@ -64,14 +61,11 @@ class PublicSuffixDownloader(DownloaderBase): DOWNTIME_INDEX: str = "psl" DOWNLOAD_FREQUENCY: int = 1 - def __init__(self) -> None: - self.destination = os.path.join( - PyFunceble.storage.CONFIG_DIRECTORY, - PyFunceble.storage.PUBLIC_SUFFIX_DUMP_FILENAME, - ) - self.download_link = PyFunceble.storage.PUBLIC_SUFFIX_DUMP_LINK - - super().__init__() + # pylint: disable=line-too-long + DEFAULT_DOWNLOAD_URL: str = ( + "https://raw.githubusercontent.com/PyFunceble/public-suffix/master/public-suffix.json" + ) + DEFAULT_FILENAME: str = "public-suffix.json" @property def authorized(self) -> bool: diff --git a/PyFunceble/downloader/user_agents.py b/PyFunceble/downloader/user_agents.py index 0908915a..128bf9b0 100644 --- a/PyFunceble/downloader/user_agents.py +++ b/PyFunceble/downloader/user_agents.py @@ -50,9 +50,6 @@ limitations under the License. """ -import os - -import PyFunceble.storage from PyFunceble.downloader.base import DownloaderBase @@ -64,14 +61,11 @@ class UserAgentsDownloader(DownloaderBase): DOWNTIME_INDEX: str = "user_agents" DOWNLOAD_FREQUENCY: int = 1 - def __init__(self) -> None: - self.destination = os.path.join( - PyFunceble.storage.CONFIG_DIRECTORY, - PyFunceble.storage.USER_AGENT_FILENAME, - ) - self.download_link = PyFunceble.storage.USER_AGENT_DUMP_LINK - - super().__init__() + # pylint: disable=line-too-long + DEFAULT_DOWNLOAD_URL: str = ( + "https://raw.githubusercontent.com/PyFunceble/user_agents/master/user_agents.json" + ) + DEFAULT_FILENAME: str = "user_agents.json" @property def authorized(self) -> bool: diff --git a/PyFunceble/query/whois/query_tool.py b/PyFunceble/query/whois/query_tool.py index a4078678..bfa1ee90 100644 --- a/PyFunceble/query/whois/query_tool.py +++ b/PyFunceble/query/whois/query_tool.py @@ -369,6 +369,7 @@ def get_whois_server( return self.iana_dataset.get_whois_server(extension) + @update_lookup_record def get_lookup_record( self, ) -> Optional[WhoisQueryToolRecord]: diff --git a/PyFunceble/storage.py b/PyFunceble/storage.py index 0cd6cdfc..37880e9e 100644 --- a/PyFunceble/storage.py +++ b/PyFunceble/storage.py @@ -64,33 +64,16 @@ PROJECT_VERSION: str = "4.2.29.dev (Blue Duckling: Tulip)" DISTRIBUTED_CONFIGURATION_FILENAME: str = ".PyFunceble_production.yaml" -DISTRIBUTED_DIR_STRUCTURE_FILENAME: str = "dir_structure_production.json" -IANA_DUMP_FILENAME: str = "iana-domains-db.json" -PUBLIC_SUFFIX_DUMP_FILENAME: str = "public-suffix.json" CONFIGURATION_FILENAME: str = ".PyFunceble.yaml" -CONFIGURATION_OVERWRITE_FILENAME: str = ".PyFunceble.overwrite.yaml" ENV_FILENAME: str = ".pyfunceble-env" -DOWN_FILENAME: str = ".pyfunceble_intern_downtime.json" -USER_AGENT_FILENAME: str = "user_agents.json" -IPV4_REPUTATION_FILENAME: str = "ipv4_reputation.data" # pylint: disable=line-too-long -IANA_DUMP_LINK: str = ( - "https://raw.githubusercontent.com/PyFunceble/iana/master/iana-domains-db.json" -) -PUBLIC_SUFFIX_DUMP_LINK: str = ( - "https://raw.githubusercontent.com/PyFunceble/public-suffix/master/public-suffix.json" -) -USER_AGENT_DUMP_LINK: str = ( - "https://raw.githubusercontent.com/PyFunceble/user_agents/master/user_agents.json" -) -IPV4_REPUTATION_DUMP_LINK: str = "https://reputation.alienvault.com/reputation.data" SHORT_REPO_LINK: str = "https://pyfunceble.github.io" REPO_LINK: str = "https://github.com/funilrys/PyFunceble" -NOT_RESOLVED_STD_HOSTNAME: str = f"pyfunceble-{secrets.token_hex(12)}.com" +NOT_RESOLVED_STD_HOSTNAME: str = f"{secrets.token_hex(12)}.mock-resolver.pyfunceble.com" IANA: Optional[dict] = {} PUBLIC_SUFFIX: Optional[dict] = {} diff --git a/docs/use/configuration/index.md b/docs/use/configuration/index.md index 47bf56a1..24d968d4 100644 --- a/docs/use/configuration/index.md +++ b/docs/use/configuration/index.md @@ -3,7 +3,7 @@ PyFunceble provides a set of functionalities that you can influence through configuration. There are multiple way to configure PyFunceble so let's get started :smile: -PyFunceble primarely load it's configuration from a file called `.PyFunceble.yaml`. +PyFunceble primarily load it's configuration from a file called `.PyFunceble.yaml`. That's the file PyFunceble generate with its default settings. However, you can overwrite any of the configuration value through a `.PyFunceble.overwrite.yaml` file or the corresponding CLI parameter. @@ -28,6 +28,7 @@ the `PYFUNCEBLE_CONFIG_DIR` environment variable. | Travis CI | Workspace | | Jenkins CI | Workspace | +At any time, you can provide your own configuration file through the `--config-file` CLI argument. If the given argument is a URL, PyFunceble will download it and use it as the configuration file. ## Filename-s diff --git a/docs/use/configuration/location.md b/docs/use/configuration/location.md index f391bdc5..d14f0c48 100644 --- a/docs/use/configuration/location.md +++ b/docs/use/configuration/location.md @@ -3,12 +3,23 @@ Depending on how and where PyFunceble is operated, it will try to load the configuration from dedicated locations. -## Custom Location +## Custom Folder If you want to skip and define your own configuration folder, you can define the storage location of the configuration files through the `PYFUNCEBLE_CONFIG_DIR` environment variable. +## Custom File + +If you want to provide your own configuration file, you can provide it through +the `--config-file` CLI argument. If the given argument is a URL, PyFunceble +will download it and use it as the configuration file. + + +!!! note + + The given configuration file will be loaded **after** the default + configuration file _(`.PyFunceble.yaml`)_ and **before** the overwrite _(`.PyFunceble.overwrite.yaml`)_ configuration file. ## Operating Systems diff --git a/tests/config/test_compare.py b/tests/config/test_compare.py index 2cb0c921..ffd50e50 100644 --- a/tests/config/test_compare.py +++ b/tests/config/test_compare.py @@ -295,6 +295,29 @@ def test_is_local_identical_uneeded_links_key(self) -> None: self.assertEqual(expected, actual) + def test_is_local_identical_platform_url_base(self) -> None: + """ + Tests the method which let us check if the given local configuration + is identical to the upstream one for the case that the platform url base + is set. + """ + + given_upstream = copy.deepcopy(self.our_config) + + given_local = copy.deepcopy(self.our_config) + given_local["platform"].update( + {"url_base": "https://example.org/PyFunceble_config.yaml"} + ) + + config_comparison = ConfigComparison( + local_config=given_local, upstream_config=given_upstream + ) + + expected = False + actual = config_comparison.is_local_identical() + + self.assertEqual(expected, actual) + def test_get_merged_no_changed(self) -> None: """ Tests the method which let us get the (clean) merged configuration. @@ -620,6 +643,37 @@ def test_get_merged_nested_old2newnegate(self) -> None: self.assertEqual(expected, actual) + def test_get_merged_delete_flatten(self) -> None: + """ + Tests the method which let us get the (clean) merged configuration for + the case that we want to delete a nested key. + """ + + given_local = copy.deepcopy(self.our_config) + given_upstream = copy.deepcopy(self.our_config) + + given_local["cli_testing"]["file_generation"]["foobar"] = True + given_local["cli_testing"]["file_generation"]["unified_results"] = True + given_upstream["cli_testing"]["file_generation"]["barfoo"] = False + + del given_upstream["cli_testing"]["file_generation"]["unified_results"] + + config_comparison = ConfigComparison( + local_config=given_local, upstream_config=given_upstream + ) + + config_comparison.DELETE_FLATTEN = [ + "cli_testing.file_generation.unified_results", + ] + + expected = copy.deepcopy(self.our_config) + del expected["cli_testing"]["file_generation"]["unified_results"] + + actual = config_comparison.get_merged() + + self.assertIn("foobar", actual["cli_testing"]["file_generation"]) + self.assertNotIn("unified_results", actual["cli_testing"]["file_generation"]) + if __name__ == "__main__": unittest.main() diff --git a/tests/config/test_loader.py b/tests/config/test_loader.py index d390f832..599d55e5 100644 --- a/tests/config/test_loader.py +++ b/tests/config/test_loader.py @@ -136,6 +136,103 @@ def test_is_not_already_loaded(self) -> None: self.assertEqual(expected, actual) + def test_set_config_dir(self) -> None: + """ + Tests the method which let us set the directory to work with. + """ + + given = "hello/world" + expected = "hello/world" + + self.config_loader.config_dir = given + actual = self.config_loader.config_dir + + self.assertEqual(expected, actual) + + def test_set_config_dir_return(self) -> None: + """ + Tests the method which let us set the directory to work with. + + In this case, we just want to be sure that the response is correct. + """ + + given = "hello/world" + + actual = self.config_loader.set_config_dir(given) + + self.assertIsInstance(actual, ConfigLoader) + + def test_set_config_dir_not_str(self) -> None: + """ + Tests the method which let us set the directory to work with for the + case that the given directory is not a :py:class:`str`. + """ + + given = ["hello", "world"] + + self.assertRaises(TypeError, lambda: self.config_loader.set_config_dir(given)) + + def test_set_remote_config_location(self) -> None: + """ + Tests the method which let us set the remote configuration location. + """ + + given = "hello/world" + expected = "hello/world" + + self.config_loader.remote_config_location = given + actual = self.config_loader.remote_config_location + + self.assertEqual(expected, actual) + + def test_set_remote_config_location_return(self) -> None: + """ + Tests the method which let us set the remote configuration location. + + In this case, we just want to be sure that the response is correct. + """ + + given = "hello/world" + + actual = self.config_loader.set_remote_config_location(given) + + self.assertIsInstance(actual, ConfigLoader) + + def test_set_remote_config_location_not_str(self) -> None: + """ + Tests the method which let us set the remote configuration location for + the case that the given location is not a :py:class:`str`. + """ + + given = ["hello", "world"] + + self.assertRaises( + TypeError, lambda: self.config_loader.set_remote_config_location(given) + ) + + def test_set_remote_config_location_url(self) -> None: + """ + Tests the method which let us set the remote configuration location for + the case that the given location is not a URL. + """ + + given = "https://example.org/hello/world" + + self.config_loader.config_dir = "./hello/world" + self.config_loader.path_to_remote_config = None + self.config_loader.remote_config_location = given + + expected = given + actual = self.config_loader.remote_config_location + + self.assertEqual(expected, actual) + + expected = os.path.join("./hello/world", ".PyFunceble.remote.yaml") + + actual = self.config_loader.path_to_remote_config + + self.assertEqual(expected, actual) + def test_set_custom_config(self) -> None: """ Tests the method which let us set the custom configuration to work with. diff --git a/tests/query/test_platform.py b/tests/query/test_platform.py index d67c13a9..5da6980c 100644 --- a/tests/query/test_platform.py +++ b/tests/query/test_platform.py @@ -255,10 +255,10 @@ def test_set_token_through_init_environment_variable_not_given(self) -> None: In this test we test the case that nothing is given or declared. """ - if "PYFUNCEBLE_COLLECTION_API_TOKEN" in os.environ: + if "PYFUNCEBLE_COLLECTION_API_TOKEN" in os.environ: # pragma: no cover del os.environ["PYFUNCEBLE_COLLECTION_API_TOKEN"] - if "PYFUNCEBLE_PLATFORM_API_TOKEN" in os.environ: + if "PYFUNCEBLE_PLATFORM_API_TOKEN" in os.environ: # pragma: no cover del os.environ["PYFUNCEBLE_PLATFORM_API_TOKEN"] expected = "" @@ -1098,10 +1098,10 @@ def test_push_with_whois_token_not_given(self) -> None: response_dict["subject"] = "example.net" self.availability_status_dataset["expiration_date"] = "23-nov-2090" - if "PYFUNCEBLE_COLLECTION_API_TOKEN" in os.environ: + if "PYFUNCEBLE_COLLECTION_API_TOKEN" in os.environ: # pragma: no cover del os.environ["PYFUNCEBLE_COLLECTION_API_TOKEN"] - if "PYFUNCEBLE_PLATFORM_API_TOKEN" in os.environ: + if "PYFUNCEBLE_PLATFORM_API_TOKEN" in os.environ: # pragma: no cover del os.environ["PYFUNCEBLE_PLATFORM_API_TOKEN"] self.query_tool.token = "" @@ -1200,10 +1200,10 @@ def test_push_token_not_given(self) -> None: In this test, we test the case that no token is given. """ - if "PYFUNCEBLE_COLLECTION_API_TOKEN" in os.environ: + if "PYFUNCEBLE_COLLECTION_API_TOKEN" in os.environ: # pragma: no cover del os.environ["PYFUNCEBLE_COLLECTION_API_TOKEN"] - if "PYFUNCEBLE_PLATFORM_API_TOKEN" in os.environ: + if "PYFUNCEBLE_PLATFORM_API_TOKEN" in os.environ: # pragma: no cover del os.environ["PYFUNCEBLE_PLATFORM_API_TOKEN"] self.query_tool.token = "" diff --git a/tests/query/whois/test_query_tool.py b/tests/query/whois/test_query_tool.py index 63ea3f0c..52d3bd3a 100644 --- a/tests/query/whois/test_query_tool.py +++ b/tests/query/whois/test_query_tool.py @@ -50,6 +50,8 @@ limitations under the License. """ +# pylint: disable=protected-access + import unittest import unittest.mock @@ -293,15 +295,79 @@ def test_set_query_timeout_through_init(self) -> None: self.assertEqual(expected, actual) + def test_read_expiration_date(self) -> None: + """ + Tests the method which let us get the expiration date. + """ + + self.query_tool.query_timeout = 10000.0 + self.query_tool.server = "whois.example.org" + self.query_tool.subject = "example.org" + self.query_tool._expiration_date = "" + + expected = "" + self.assertEqual(expected, self.query_tool.expiration_date) + + self.query_tool._expiration_date = "2021-01-01 00:00:00" + + expected = "2021-01-01 00:00:00" + self.assertEqual(expected, self.query_tool.expiration_date) + + self.query_tool._expiration_date = None + self.query_tool.lookup_record.record = "expires: 2021-01-01 00:00:00" + + expected = "01-jan-2021" + self.assertEqual(expected, self.query_tool.expiration_date) + + def test_read_registrar(self) -> None: + """ + Tests the method which let us get the registrar. + """ + + self.query_tool.query_timeout = 10000.0 + self.query_tool.server = "whois.example.org" + self.query_tool.subject = "example.org" + self.query_tool._registrar = "" + + expected = "" + self.assertEqual(expected, self.query_tool.registrar) + + self.query_tool._registrar = "Example Registrar" + + expected = "Example Registrar" + self.assertEqual(expected, self.query_tool.registrar) + + self.query_tool._registrar = None + self.query_tool.lookup_record.record = "registrar: Example Registrar" + + expected = "Example Registrar" + self.assertEqual(expected, self.query_tool.registrar) + + def test_read_record(self) -> None: + """ + Tests the method which let us get the record. + """ + + self.query_tool.query_timeout = 10000.0 + self.query_tool.server = "whois.example.org" + self.query_tool.subject = "example.org" + self.query_tool.lookup_record.record = "Hello, World!" + + expected = "Hello, World!" + self.assertEqual(expected, self.query_tool.record) + def test_get_lookup_record(self) -> None: """ Tests the method which let us get the lookup record. """ - self.query_tool.server = "whois.example.org" self.query_tool.query_timeout = 10000.0 + self.query_tool.server = "whois.example.org" self.query_tool.subject = "example.org" + self.query_tool._expiration_date = "2021-01-01 00:00:00" + self.query_tool._registrar = "Example Registrar" + actual = self.query_tool.get_lookup_record() self.assertIsInstance(actual, WhoisQueryToolRecord) @@ -309,12 +375,18 @@ def test_get_lookup_record(self) -> None: expected = "example.org" self.assertEqual(expected, actual.subject) - expected = 10000.0 - self.assertEqual(expected, actual.query_timeout) - expected = "whois.example.org" self.assertEqual(expected, actual.server) + expected = "2021-01-01 00:00:00" + self.assertEqual(expected, actual.expiration_date) + + expected = "Example Registrar" + self.assertEqual(expected, actual.registrar) + + expected = 10000.0 + self.assertEqual(expected, actual.query_timeout) + if __name__ == "__main__": unittest.main()