Improvement of the collection API.

Squashed commit of the following: commit ae27a64 Author: funilrys <[email protected]> Date: Tue Mar 26 21:43:19 2024 +0100 Fix handling of latest status commit 3e5494e Merge: 1cb8fb3 467b8aa Author: funilrys <[email protected]> Date: Tue Mar 26 17:55:57 2024 +0100 Merge remote-tracking branch 'origin/dev' into col20 commit 1cb8fb3 Author: funilrys <[email protected]> Date: Sat Mar 16 20:48:02 2024 +0100 Add missing timeout commit d2ba8ac Author: funilrys <[email protected]> Date: Sat Feb 24 20:11:29 2024 +0100 Add missing configuration value. commit af434e7 Author: funilrys <[email protected]> Date: Thu Feb 22 20:01:51 2024 +0100 Fix linting issues commit 26e8603 Author: funilrys <[email protected]> Date: Thu Feb 22 19:32:25 2024 +0100 Col20: Let PyFunceble contracts from Col20 platform.
funilrys · Mar 27, 2024 · 067dce2 · 067dce2
1 parent 30855ae
commit 067dce2
Show file tree

Hide file tree

Showing 9 changed files with 285 additions and 15 deletions.
diff --git a/PyFunceble/checker/availability/base.py b/PyFunceble/checker/availability/base.py
@@ -1043,7 +1043,13 @@ def try_to_query_status_from_collection(self) -> "AvailabilityCheckerBase":
                 self.collection_query_tool.preferred_status_origin == "latest"
                 and data["status"]["availability"]["latest"]
             ):
-                self.status.status = data["status"]["availability"]["latest"]["status"]
+                try:
+                    # legacy
+                    self.status.status = data["status"]["availability"]["latest"][
+                        "status"
+                    ]
+                except KeyError:
+                    self.status.status = data["status"]["availability"]["latest"]
                 self.status.status_source = "COLLECTION"
             elif (
                 self.collection_query_tool.preferred_status_origin == "recommended"

diff --git a/PyFunceble/cli/entry_points/pyfunceble/cli.py b/PyFunceble/cli/entry_points/pyfunceble/cli.py
@@ -69,7 +69,9 @@
 from PyFunceble.helpers.regex import RegexHelper
 
 
-def get_configured_value(entry: str, *, negate=False) -> Any:
+def get_configured_value(
+    entry: str, *, negate: bool = False, value_only: bool = False
+) -> Any:
     """
     Provides the currently configured value.
 
@@ -81,6 +83,9 @@ def get_configured_value(entry: str, *, negate=False) -> Any:
     :param negate:
         Allows us to negate the result from the configuration.
 
+    :param value_only:
+        Whether we should only return the value or the full message.
+
     :raise ValueError:
         When the given :code:`entry` is not found.
     """
@@ -102,10 +107,14 @@ def get_configured_value(entry: str, *, negate=False) -> Any:
         result = not result
 
     return (
-        f"\n{colorama.Fore.YELLOW}{colorama.Style.BRIGHT}"
-        f"Configured value: {colorama.Fore.BLUE}"
-        f"{result!r}"
-        f"{colorama.Style.RESET_ALL}"
+        (
+            f"\n{colorama.Fore.YELLOW}{colorama.Style.BRIGHT}"
+            f"Configured value: {colorama.Fore.BLUE}"
+            f"{result!r}"
+            f"{colorama.Style.RESET_ALL}"
+        )
+        if not value_only
+        else result
     )
 
 
@@ -122,6 +131,12 @@ def add_arguments_to_parser(
         if "dest" in opt_args:
             opt_args["dest"] = opt_args["dest"].replace(".", "__")
 
+        for index, value in enumerate(pos_args):
+            if value.startswith("-") and "." not in value:
+                continue
+
+            pos_args[index] = value.replace(".", "__")
+
         parser.add_argument(*pos_args, **opt_args)
 
 
@@ -1184,6 +1199,41 @@ def get_default_group_data() -> List[Tuple[List[str], dict]]:
     ]
 
 
+def platform_parser(
+    parser: Union[argparse.ArgumentParser, argparse._SubParsersAction]
+) -> None:
+    """
+    Adds the platform group to the given parser.
+    """
+
+    platform = parser.add_parser(
+        "platform",
+        add_help=False,
+        epilog=PyFunceble.cli.storage.STD_EPILOG,
+    )
+
+    args = [
+        (
+            ["cli_testing.testing_mode.platform_contribution"],
+            {
+                "default": get_configured_value(
+                    "cli_testing.testing_mode.platform_contribution", value_only=True
+                ),
+                "action": "store_%s"
+                % str(
+                    not get_configured_value(
+                        "cli_testing.testing_mode.platform_contribution",
+                        value_only=True,
+                    )
+                ).lower(),
+                "help": argparse.SUPPRESS,
+            },
+        )
+    ]
+
+    add_arguments_to_parser(platform, args)
+    add_arguments_to_parser(platform, get_default_group_data())
+
 def ask_authorization_to_merge_config(missing_key: Optional[str] = None) -> bool:
     """
     Asks the end-user for the authorization to merge the upstream
@@ -1292,6 +1342,8 @@ def tool() -> None:
 
     # pylint:  disable=possibly-unused-variable
 
+    command_sub = parser.add_subparsers(dest="command", help=argparse.SUPPRESS)
+
     shtab.add_argument_to(
         parser,
         option_string=["--show-completion"],
@@ -1321,6 +1373,8 @@ def tool() -> None:
         get_ci_group_data,
     ]
 
+    parse_funcs = [platform_parser]
+
     for func in funcs:
         parser_name = func.__name__.replace("get_", "").replace("_data", "")
 
@@ -1348,10 +1402,21 @@ def tool() -> None:
                 )
                 sys.exit(1)
 
+    for func in parse_funcs:
+        func(command_sub)
+
     add_arguments_to_parser(parser, get_default_group_data())
 
     args = parser.parse_args()
 
-    if any(getattr(args, x) for x in ["domains", "urls", "files", "url_files"]):
+    if any(
+        getattr(args, x)
+        for x in [
+            "domains",
+            "urls",
+            "files",
+            "url_files",
+        ]
+    ) or bool(args.command):
         SystemIntegrator(args).start()
         SystemLauncher(args).start()
diff --git a/PyFunceble/cli/processes/workers/base.py b/PyFunceble/cli/processes/workers/base.py
@@ -318,8 +318,10 @@ def break_now() -> bool:
 
                 try:
                     worker_name, destination_worker, consumed = self.input_queue.get()
-                except EOFError:
-                    PyFunceble.facility.Logger.info("Got EOFError. Stopping worker.")
+                except (EOFError, KeyboardInterrupt):
+                    PyFunceble.facility.Logger.info(
+                        "Got EOFError/KeyboardInterrupt. Stopping worker."
+                    )
                     self.global_exit_event.set()
                     break
 
@@ -370,7 +372,14 @@ def break_now() -> bool:
                     self.share_waiting_message(apply_breakoff=wait_for_stop)
                     continue
 
-                result = self.target(consumed)
+                try:
+                    result = self.target(consumed)
+                except (EOFError, KeyboardInterrupt):
+                    PyFunceble.facility.Logger.info(
+                        "Got EOFError/KeyboardInterrupt. Stopping worker."
+                    )
+                    self.global_exit_event.set()
+                    break
 
                 if result is not None:
                     self.add_to_output_queue(result)

diff --git a/PyFunceble/cli/processes/workers/producer.py b/PyFunceble/cli/processes/workers/producer.py
@@ -399,6 +399,11 @@ def target(self, consumed: Any) -> Optional[Tuple[Any, ...]]:
             )
             return None
 
+        if test_dataset["type"] == "platform-contribution":
+            self.collection_query_tool.deliver_contract(
+                test_dataset["contract"], test_result
+            )
+
         if (
             PyFunceble.storage.CONFIGURATION.collection.push
             and test_result.status_source != "COLLECTION"

diff --git a/PyFunceble/cli/system/launcher.py b/PyFunceble/cli/system/launcher.py
@@ -119,6 +119,7 @@
 from PyFunceble.helpers.directory import DirectoryHelper
 from PyFunceble.helpers.download import DownloadHelper
 from PyFunceble.helpers.file import FileHelper
+from PyFunceble.query.collection import CollectionQueryTool
 
 
 class SystemLauncher(SystemBase):
@@ -222,7 +223,7 @@ def __init__(self, args: Optional[argparse.Namespace] = None) -> None:
             )
             self.producer_process_manager = ChancyProducerProcessesManager(
                 self.manager,
-                max_worker=1,
+                max_worker=PyFunceble.storage.CONFIGURATION.cli_testing.max_workers,
                 continuous_integration=self.continuous_integration,
                 input_queue=self.tester_process_manager.output_queue[0],
                 daemon=True,
@@ -406,6 +407,21 @@ def fill_protocol(self) -> "SystemLauncher":
                     "Added to the protocol:\n%r", to_append
                 )
 
+        # pylint: disable=line-too-long
+        if (
+            PyFunceble.storage.CONFIGURATION.cli_testing.testing_mode.platform_contribution
+        ):
+            self.testing_protocol.append(
+                {
+                    "type": "platform-contribution",
+                    "subject_type": "any",
+                    "destination": None,
+                    "checker_type": None,
+                    "output_dir": None,
+                    "session_id": None,
+                }
+            )
+
     def ci_stop_in_the_middle_if_time_exceeded(self) -> "SystemLauncher":
         """
         Stops our processes as soon as the time is exceeded.
@@ -648,7 +664,37 @@ def handle_file(protocol: dict) -> None:
                     )
             elif protocol["type"] == "file":
                 handle_file(protocol)
+            elif protocol["type"] == "platform-contribution":
+                query_tool = CollectionQueryTool()
 
+                while True:
+                    for next_contract in next(
+                        query_tool.pull_contract(
+                            PyFunceble.storage.CONFIGURATION.cli_testing.max_workers
+                        )
+                    ):
+                        if (
+                            "subject" not in next_contract
+                            or not next_contract["subject"]
+                        ):
+                            continue
+
+                        protocol_data = copy.deepcopy(protocol)
+
+                        protocol_data["checker_type"] = next_contract[
+                            "checker_type"
+                        ].upper()
+                        protocol_data["subject_type"] = next_contract["subject_type"]
+                        protocol_data["subject"] = protocol_data[
+                            "idna_subject"
+                        ] = next_contract["subject"]["subject"]
+                        protocol_data["contract"] = copy.deepcopy(next_contract)
+
+                        self.tester_process_manager.add_to_input_queue(
+                            protocol_data, worker_name="main"
+                        )
+
+                    self.ci_stop_in_the_middle_if_time_exceeded()
         return self
 
     def generate_waiting_files(self) -> "SystemLauncher":

diff --git a/PyFunceble/config/loader.py b/PyFunceble/config/loader.py
@@ -172,6 +172,16 @@ def conditional_switch(config: dict) -> dict:
             # If timeout is set to a negative digit, switch to the default one.
             config["lookup"]["timeout"] = 5
 
+        if (
+            "cli_testing" in config
+            and "testing_mode" in config["cli_testing"]
+            and "platform_contribution" in config["cli_testing"]["testing_mode"]
+            and config["cli_testing"]["testing_mode"]["platform_contribution"]
+        ):
+            # If we are under a special testing mode. We shouldn't generate
+            # any files
+            config["cli_testing"]["file_generation"]["no_file"] = True
+
         return config
 
     @staticmethod

diff --git a/PyFunceble/data/infrastructure/.PyFunceble_production.yaml b/PyFunceble/data/infrastructure/.PyFunceble_production.yaml
@@ -173,6 +173,9 @@ cli_testing:
     # Activates the reputation test.
     reputation: no
 
+    # BETA: Activates the platform contribution test.
+    platform_contribution: no
+
   days_between:
     # Provides everything which is x days periodic.