Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Introduction of a way to provide a configuration file and folder from the CLI. #380

Merged
merged 9 commits into from
Oct 3, 2024
12 changes: 12 additions & 0 deletions PyFunceble/cli/entry_points/pyfunceble/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -1197,6 +1197,18 @@ def get_default_group_data() -> List[Tuple[List[str], dict]]:
"version": "%(prog)s " + PyFunceble.storage.PROJECT_VERSION,
},
),
(
[
"--config-file",
Copy link
Contributor

@spirillen spirillen Sep 23, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

--config-file => --config-dir

the equivalent to ~/.config/PyFunceble with all the config files, since the values of ~/.config/PyFunceble/.pyfunceble-env are not picked up in when running in venv

],
{
"dest": "config_file",
"type": str,
"help": "Sets the configuration file to use. It can be a\n"
"local or remote file. Please note that this configuration can be\n"
"overwritten by your overwrite configuration file.",
},
),
]


Expand Down
61 changes: 47 additions & 14 deletions PyFunceble/cli/system/integrator.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,14 +52,18 @@
"""

import os
import sys
import traceback

import colorama

import PyFunceble.cli.facility
import PyFunceble.cli.factory
import PyFunceble.cli.storage
import PyFunceble.facility
import PyFunceble.helpers.exceptions
import PyFunceble.storage
from PyFunceble.cli.continuous_integration.exceptions import StopExecution
from PyFunceble.cli.system.base import SystemBase
from PyFunceble.helpers.dict import DictHelper

Expand Down Expand Up @@ -250,23 +254,52 @@ def start(self) -> "SystemIntegrator":
Starts a group of actions provided by this interface.
"""

if hasattr(self.args, "output_location") and self.args.output_location:
PyFunceble.cli.storage.OUTPUT_DIRECTORY = os.path.realpath(
os.path.join(
self.args.output_location,
PyFunceble.cli.storage.OUTPUTS.parent_directory,
)
)
try:
self.init_logger()

self.init_logger()
if hasattr(self.args, "output_location") and self.args.output_location:
PyFunceble.cli.storage.OUTPUT_DIRECTORY = os.path.realpath(
os.path.join(
self.args.output_location,
PyFunceble.cli.storage.OUTPUTS.parent_directory,
)
)

PyFunceble.facility.Logger.debug("Given arguments:\n%r", self.args)
if hasattr(self.args, "config_file") and self.args.config_file:
PyFunceble.facility.ConfigLoader.set_remote_config_location(
self.args.config_file
).reload()

PyFunceble.facility.Logger.debug("Given arguments:\n%r", self.args)

self.inject_into_config()
self.check_config()
self.check_deprecated()

PyFunceble.cli.facility.CredentialLoader.start()
PyFunceble.cli.factory.DBSession.init_db_sessions()
except (KeyboardInterrupt, StopExecution):
pass
except Exception as exception: # pylint: disable=broad-except
PyFunceble.facility.Logger.critical(
"Fatal error.",
exc_info=True,
)
if isinstance(exception, PyFunceble.helpers.exceptions.UnableToDownload):
message = (
f"{colorama.Fore.RED}{colorama.Style.BRIGHT}Unable to download "
f"{exception}"
)
else:
message = (
f"{colorama.Fore.RED}{colorama.Style.BRIGHT}Fatal Error: "
f"{exception}"
)
print(message)

self.inject_into_config()
self.check_config()
self.check_deprecated()
if PyFunceble.facility.Logger.authorized:
print(traceback.format_exc())

PyFunceble.cli.facility.CredentialLoader.start()
PyFunceble.cli.factory.DBSession.init_db_sessions()
sys.exit(1)

return self
19 changes: 14 additions & 5 deletions PyFunceble/cli/system/launcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@
import PyFunceble.cli.utils.sort
import PyFunceble.cli.utils.stdout
import PyFunceble.facility
import PyFunceble.helpers.exceptions
import PyFunceble.storage
from PyFunceble.checker.syntax.url import URLSyntaxChecker
from PyFunceble.cli.continuous_integration.base import ContinuousIntegrationBase
Expand Down Expand Up @@ -1136,12 +1137,20 @@ def start(self) -> "SystemLauncher":
"Fatal error.",
exc_info=True,
)
print(
f"{colorama.Fore.RED}{colorama.Style.BRIGHT}Fatal Error: "
f"{exception}"
)
if isinstance(exception, PyFunceble.helpers.exceptions.UnableToDownload):
message = (
f"{colorama.Fore.RED}{colorama.Style.BRIGHT}Unable to download "
f"{exception}"
)
else:
message = (
f"{colorama.Fore.RED}{colorama.Style.BRIGHT}Fatal Error: "
f"{exception}"
)
print(message)

print(traceback.format_exc())
if PyFunceble.facility.Logger.authorized:
print(traceback.format_exc())
sys.exit(1)

PyFunceble.cli.utils.stdout.print_thanks()
Expand Down
93 changes: 79 additions & 14 deletions PyFunceble/config/loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@
from PyFunceble.downloader.public_suffix import PublicSuffixDownloader
from PyFunceble.downloader.user_agents import UserAgentsDownloader
from PyFunceble.helpers.dict import DictHelper
from PyFunceble.helpers.download import DownloadHelper
from PyFunceble.helpers.environment_variable import EnvironmentVariableHelper
from PyFunceble.helpers.file import FileHelper
from PyFunceble.helpers.merge import Merge
Expand All @@ -86,7 +87,8 @@ class ConfigLoader:
:code:`PYFUNCEBLE_AUTO_CONFIGURATION` environment variable.
"""

path_to_config: Optional[str] = None
_path_to_config: Optional[str] = None
_remote_config_location: Optional[str] = None
path_to_default_config: Optional[str] = None
path_to_overwrite_config: Optional[str] = None

Expand All @@ -108,6 +110,8 @@ def __init__(self, merge_upstream: Optional[bool] = None) -> None:
PyFunceble.storage.CONFIGURATION_FILENAME,
)

self.path_to_remote_config = None

self.path_to_overwrite_config = os.path.join(
PyFunceble.storage.CONFIG_DIRECTORY,
PyFunceble.storage.CONFIGURATION_OVERWRITE_FILENAME,
Expand Down Expand Up @@ -274,23 +278,44 @@ def set_merge_upstream(self, value: bool) -> "ConfigLoader":

return self

def config_file_exist(
self,
) -> bool: # pragma: no cover ## Existance checker already tested.
@property
def remote_config_location(self) -> Optional[str]:
"""
Checks if the config file exists.
Provides the current state of the :code:`_remote_config_location` attribute.
"""

return FileHelper(self.path_to_config).exists()
return self._remote_config_location

def default_config_file_exist(
self,
) -> bool: # pragma: no cover ## Existance checker already tested.
@remote_config_location.setter
def remote_config_location(self, value: Optional[str]) -> None:
"""
Updates the value of :code:`_remote_config_location` attribute.

:raise TypeError:
When :code:`value` is not a :py:class:`str`.
"""

if value is not None and not isinstance(value, str):
raise TypeError(f"<value> should be {str}, {type(value)} given.")

if not value.startswith("http") and not value.startswith("https"):
self.path_to_remote_config = os.path.realpath(value)
else:
self.path_to_remote_config = os.path.join(
PyFunceble.storage.CONFIG_DIRECTORY,
PyFunceble.storage.CONFIGURATION_REMOTE_FILENAME,
)

self._remote_config_location = value

def set_remote_config_location(self, value: Optional[str]) -> "ConfigLoader":
"""
Checks if the default configuration file exists.
Updates the value of :code:`_remote_config_location` attribute.
"""

return self.file_helper.set_path(self.path_to_default_config).exists()
self.remote_config_location = value

return self

def install_missing_infrastructure_files(
self,
Expand Down Expand Up @@ -347,9 +372,34 @@ def is_3_x_version(config: dict) -> bool:

return config and "days_between_inactive_db_clean" in config

def download_remote_config(src: str, dest: str = None) -> None:
"""
Downloads the remote configuration.

:param src:
The source to download from.
:param dest:
The destination to download
"""

if src and (src.startswith("http") or src.startswith("https")):
if dest is None:
destination = os.path.join(
PyFunceble.storage.CONFIG_DIRECTORY,
os.path.basename(dest),
)
else:
destination = dest

DownloadHelper(src).download_text(destination=destination)

if not self.is_already_loaded():
self.install_missing_infrastructure_files()
self.download_dynamic_infrastructure_files()
download_remote_config(
self.remote_config_location, self.path_to_remote_config
)
download_remote_config(self.path_to_config)

try:
config = self.dict_helper.from_yaml_file(self.path_to_config)
Expand Down Expand Up @@ -377,15 +427,22 @@ def is_3_x_version(config: dict) -> bool:

self.dict_helper.set_subject(config).to_yaml_file(self.path_to_config)

if (
self.path_to_remote_config
and self.file_helper.set_path(self.path_to_remote_config).exists()
):
remote_data = self.dict_helper.from_yaml_file(self.path_to_remote_config)

if isinstance(remote_data, dict):
config = Merge(remote_data).into(config)

if self.file_helper.set_path(self.path_to_overwrite_config).exists():
overwrite_data = self.dict_helper.from_yaml_file(
self.path_to_overwrite_config
)

if isinstance(overwrite_data, dict):
config = Merge(
self.dict_helper.from_yaml_file(self.path_to_overwrite_config)
).into(config)
config = Merge(overwrite_data).into(config)
else: # pragma: no cover ## Just make it visible to end-user.
self.file_helper.write("")

Expand Down Expand Up @@ -415,6 +472,14 @@ def get_configured_value(self, entry: str) -> Any:

return PyFunceble.storage.FLATTEN_CONFIGURATION[entry]

def reload(self) -> "ConfigLoader":
"""
Reloads the configuration.
"""

self.destroy()
self.start()

def start(self) -> "ConfigLoader":
"""
Starts the loading processIs.
Expand Down
1 change: 1 addition & 0 deletions PyFunceble/storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@
PUBLIC_SUFFIX_DUMP_FILENAME: str = "public-suffix.json"
CONFIGURATION_FILENAME: str = ".PyFunceble.yaml"
CONFIGURATION_OVERWRITE_FILENAME: str = ".PyFunceble.overwrite.yaml"
CONFIGURATION_REMOTE_FILENAME: str = ".PyFunceble.remote.yaml"
ENV_FILENAME: str = ".pyfunceble-env"
DOWN_FILENAME: str = ".pyfunceble_intern_downtime.json"
USER_AGENT_FILENAME: str = "user_agents.json"
Expand Down
3 changes: 2 additions & 1 deletion docs/use/configuration/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
PyFunceble provides a set of functionalities that you can influence through configuration.
There are multiple way to configure PyFunceble so let's get started :smile:

PyFunceble primarely load it's configuration from a file called `.PyFunceble.yaml`.
PyFunceble primarily load it's configuration from a file called `.PyFunceble.yaml`.
That's the file PyFunceble generate with its default settings. However, you can
overwrite any of the configuration value through a `.PyFunceble.overwrite.yaml` file
or the corresponding CLI parameter.
Expand All @@ -28,6 +28,7 @@ the `PYFUNCEBLE_CONFIG_DIR` environment variable.
| Travis CI | Workspace |
| Jenkins CI | Workspace |

At any time, you can provide your own configuration file through the `--config-file` CLI argument. If the given argument is a URL, PyFunceble will download it and use it as the configuration file.

## Filename-s

Expand Down
13 changes: 12 additions & 1 deletion docs/use/configuration/location.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,23 @@
Depending on how and where PyFunceble is operated, it will try to load the
configuration from dedicated locations.

## Custom Location
## Custom Folder

If you want to skip and define your own configuration folder, you can define
the storage location of the configuration files through
the `PYFUNCEBLE_CONFIG_DIR` environment variable.

## Custom File

If you want to provide your own configuration file, you can provide it through
the `--config-file` CLI argument. If the given argument is a URL, PyFunceble
will download it and use it as the configuration file.


!!! note

The given configuration file will be loaded **after** the default
configuration file _(`.PyFunceble.yaml`)_ and **before** the overwrite _(`.PyFunceble.overwrite.yaml`)_ configuration file.

## Operating Systems

Expand Down
Loading