From cf982e2a99efa8bacf6e5a6fd1952e7b104c9e8b Mon Sep 17 00:00:00 2001 From: Madhu Kanoor Date: Fri, 15 Nov 2024 10:29:50 -0500 Subject: [PATCH 1/4] feat: optionally check if all env vars match (#350) From EDA credentials we might pass in env vars for source plugins, the generic source plugin can optionally have a dictionary of env vars that it can check e.g. ``` check_env_vars: ENV_V1: value1 ENV_V2: value2 ``` --- .../eda/plugins/event_source/generic.py | 35 ++++++++++- tests/unit/event_source/test_generic.py | 58 +++++++++++++++++++ 2 files changed, 92 insertions(+), 1 deletion(-) diff --git a/extensions/eda/plugins/event_source/generic.py b/extensions/eda/plugins/event_source/generic.py index 536e5db5..e1e3ae5b 100644 --- a/extensions/eda/plugins/event_source/generic.py +++ b/extensions/eda/plugins/event_source/generic.py @@ -32,6 +32,10 @@ final payload which can be used to trigger a shutdown of the rulebook, especially when we are using rulebooks to forward messages to other running rulebooks. +check_env_vars dict Optionally check if all the defined env vars are set + before generating the events. If any of the env_var is missing + or the value doesn't match the source plugin will end + with an exception """ @@ -53,16 +57,35 @@ from __future__ import annotations import asyncio +import os import random import time from dataclasses import dataclass, fields from datetime import datetime from pathlib import Path -from typing import Any +from typing import Any, Dict, Optional import yaml +class MissingEnvVarError(Exception): + """Exception class for missing env var.""" + + def __init__(self: "MissingEnvVarError", env_var: str) -> None: + """Class constructor with the missing env_var.""" + super().__init__(f"Env Var {env_var} is required") + + +class EnvVarMismatchError(Exception): + """Exception class for mismatch in the env var value.""" + + def __init__( + self: "EnvVarMismatchError", env_var: str, value: str, expected: str + ) -> None: + """Class constructor with mismatch in env_var value.""" + super().__init__(f"Env Var {env_var} expected: {expected} passed in: {value}") + + @dataclass class Args: """Class to store all the passed in args.""" @@ -84,6 +107,7 @@ class ControlArgs: loop_count: int = 1 repeat_count: int = 1 timestamp: bool = False + check_env_vars: Optional[Dict[str, str]] = None @dataclass @@ -135,6 +159,7 @@ async def __call__(self: Generic) -> None: msg = "time_format must be one of local, iso8601, epoch" raise ValueError(msg) + await self._check_env_vars() await self._load_payload_from_file() if not isinstance(self.my_args.payload, list): @@ -174,6 +199,14 @@ async def _post_event(self: Generic, event: dict[str, Any], index: int) -> None: print(data) # noqa: T201 await self.queue.put(data) + async def _check_env_vars(self: Generic) -> None: + if self.control_args.check_env_vars: + for key, value in self.control_args.check_env_vars.items(): + if key not in os.environ: + raise MissingEnvVarError(key) + if os.environ[key] != value: + raise EnvVarMismatchError(key, os.environ[key], value) + async def _load_payload_from_file(self: Generic) -> None: if not self.my_args.payload_file: return diff --git a/tests/unit/event_source/test_generic.py b/tests/unit/event_source/test_generic.py index 909dc6ec..76ae907f 100644 --- a/tests/unit/event_source/test_generic.py +++ b/tests/unit/event_source/test_generic.py @@ -8,6 +8,10 @@ import pytest import yaml +from extensions.eda.plugins.event_source.generic import ( + EnvVarMismatchError, + MissingEnvVarError, +) from extensions.eda.plugins.event_source.generic import main as generic_main @@ -243,3 +247,57 @@ def test_generic_parsing_payload_file() -> None: }, ) ) + + +def test_env_vars_missing() -> None: + """Test missing env vars""" + myqueue = _MockQueue() + event = {"name": "fred"} + + with pytest.raises(MissingEnvVarError): + asyncio.run( + generic_main( + myqueue, + { + "payload": event, + "check_env_vars": {"NAME_MISSING": "Fred"}, + }, + ) + ) + + +def test_env_vars_mismatch() -> None: + """Test env vars with incorrect values""" + myqueue = _MockQueue() + event = {"name": "fred"} + + os.environ["TEST_ENV1"] = "Kaboom" + with pytest.raises(EnvVarMismatchError): + asyncio.run( + generic_main( + myqueue, + { + "payload": event, + "check_env_vars": {"TEST_ENV1": "Fred"}, + }, + ) + ) + + +def test_env_vars() -> None: + """Test env vars with correct values""" + myqueue = _MockQueue() + event = {"name": "fred"} + + os.environ["TEST_ENV1"] = "Fred" + asyncio.run( + generic_main( + myqueue, + { + "payload": event, + "check_env_vars": {"TEST_ENV1": "Fred"}, + }, + ) + ) + assert len(myqueue.queue) == 1 + assert myqueue.queue[0] == {"name": "fred"} From 2cd670a690ff166d7b96a873f1dd1221c9e9177a Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 18 Nov 2024 18:26:31 +0100 Subject: [PATCH 2/4] fix: ensure mandatory params for activation present (#352) Signed-off-by: Alex --- plugins/module_utils/common.py | 2 +- plugins/modules/rulebook_activation.py | 116 ++++++++++-------- .../targets/activation/tasks/main.yml | 77 ++++++++++++ 3 files changed, 141 insertions(+), 54 deletions(-) diff --git a/plugins/module_utils/common.py b/plugins/module_utils/common.py index 56020832..3ee98e53 100644 --- a/plugins/module_utils/common.py +++ b/plugins/module_utils/common.py @@ -22,7 +22,7 @@ def lookup_resource_id( endpoint: str, name: str, params: Optional[dict[str, Any]] = None, -) -> Optional[Any]: +) -> Optional[int]: result = None try: diff --git a/plugins/modules/rulebook_activation.py b/plugins/modules/rulebook_activation.py index 66946243..9212ac8e 100644 --- a/plugins/modules/rulebook_activation.py +++ b/plugins/modules/rulebook_activation.py @@ -31,12 +31,14 @@ project_name: description: - The name of the project associated with the rulebook activation. + - Required when state is present. type: str aliases: - project rulebook_name: description: - The name of the rulebook associated with the rulebook activation. + - Required when state is present. type: str aliases: - rulebook @@ -58,6 +60,7 @@ decision_environment_name: description: - The name of the decision environment associated with the rulebook activation. + - Required when state is present. type: str aliases: - decision_environment @@ -70,7 +73,7 @@ - token organization_name: description: - - The name of the organization. + - The name of the organization. Required when state is present. - This parameter is supported in AAP 2.5 and onwards. If specified for AAP 2.4, value will be ignored. type: str @@ -316,58 +319,54 @@ def create_params( ) -> Dict[str, Any]: activation_params: Dict[str, Any] = {} - # Get the project id - project_id = None - if module.params.get("project_name"): - project_id = lookup_resource_id( - module, controller, "projects", module.params["project_name"] - ) - if project_id is not None: - activation_params["project_id"] = project_id + # Get the project id, only required to get the rulebook id + project_name = module.params["project_name"] + project_id = lookup_resource_id(module, controller, "projects", project_name) + + if project_id is None: + module.fail_json(msg=f"Project {project_name} not found.") # Get the rulebook id - rulebook_id = None - params = {} - if project_id is not None: - params = {"data": {"project_id": project_id}} - if module.params.get("rulebook_name"): - rulebook_id = lookup_resource_id( - module, - controller, - "rulebooks", - module.params["rulebook_name"], - params, + rulebook_name = module.params["rulebook_name"] + params = {"data": {"project_id": project_id}} + rulebook_id = lookup_resource_id( + module, + controller, + "rulebooks", + rulebook_name, + params, + ) + if rulebook_id is None: + module.fail_json( + msg=f"Rulebook {rulebook_name} not found for project {project_name}." ) - if rulebook_id is not None: - activation_params["rulebook_id"] = rulebook_id + + activation_params["rulebook_id"] = rulebook_id # Get the decision environment id - decision_environment_id = None - if module.params.get("decision_environment_name"): - decision_environment_id = lookup_resource_id( - module, - controller, - "decision-environments", - module.params["decision_environment_name"], + decision_environment_name = module.params["decision_environment_name"] + decision_environment_id = lookup_resource_id( + module, + controller, + "decision-environments", + decision_environment_name, + ) + if decision_environment_id is None: + module.fail_json( + msg=f"Decision Environment {decision_environment_name} not found." ) - if decision_environment_id is not None: - activation_params["decision_environment_id"] = decision_environment_id + activation_params["decision_environment_id"] = decision_environment_id # Get the organization id - organization_id = None - if not is_aap_24 and module.params.get("organization_name"): + organization_name = module.params["organization_name"] + if not is_aap_24: organization_id = lookup_resource_id( - module, controller, "organizations", module.params["organization_name"] + module, controller, "organizations", organization_name ) - if organization_id is not None: + if organization_id is None: + module.fail_json(msg=f"Organization {organization_name} not found.") activation_params["organization_id"] = organization_id - if module.params.get("description"): - activation_params["description"] = module.params["description"] - - if module.params.get("extra_vars"): - activation_params["extra_var"] = module.params["extra_vars"] - # Get the AWX token id awx_token_id = None if module.params.get("awx_token_name"): @@ -377,12 +376,6 @@ def create_params( if awx_token_id is not None: activation_params["awx_token_id"] = awx_token_id - if module.params.get("restart_policy"): - activation_params["restart_policy"] = module.params["restart_policy"] - - if module.params.get("enabled"): - activation_params["is_enabled"] = module.params["enabled"] - # Get the eda credential ids eda_credential_ids = None if not is_aap_24 and module.params.get("eda_credentials"): @@ -402,12 +395,26 @@ def create_params( # Process event streams and source mappings activation_params["source_mappings"] = yaml.dump( process_event_streams( - rulebook_id=rulebook_id, + # ignore type error, if rulebook_id is None, it will fail earlier + rulebook_id=rulebook_id, # type: ignore[arg-type] controller=controller, module=module, ) ) + # Set the remaining parameters + if module.params.get("description"): + activation_params["description"] = module.params["description"] + + if module.params.get("extra_vars"): + activation_params["extra_var"] = module.params["extra_vars"] + + if module.params.get("restart_policy"): + activation_params["restart_policy"] = module.params["restart_policy"] + + if module.params.get("enabled"): + activation_params["is_enabled"] = module.params["enabled"] + if not is_aap_24 and module.params.get("log_level"): activation_params["log_level"] = module.params["log_level"] @@ -492,9 +499,13 @@ def main() -> None: organization_name = module.params.get("organization_name") if state == "present" and not is_aap_24 and organization_name is None: module.fail_json( - msg="Parameter organization_name is required when state is present." + msg=( + "Parameter organization_name is required when state " + "is present for this version of EDA." + ), ) # Attempt to find rulebook activation based on the provided name + activation = {} try: activation = controller.get_exactly_one("activations", name=name) except EDAError as e: @@ -511,14 +522,13 @@ def main() -> None: module.exit_json( msg=f"A rulebook activation with name: {name} already exists. " "The module does not support modifying a rulebook activation.", - **{"changed": False, "id": activation["id"]}, + changed=False, + id=activation["id"], ) # Activation Data that will be sent for create/update activation_params = create_params(module, controller, is_aap_24=is_aap_24) - activation_params["name"] = ( - controller.get_item_name(activation) if activation else name - ) + activation_params["name"] = name # If the state was present and we can let the module build or update the # existing activation, this will return on its own diff --git a/tests/integration/targets/activation/tasks/main.yml b/tests/integration/targets/activation/tasks/main.yml index a65cf01d..6c94e461 100644 --- a/tests/integration/targets/activation/tasks/main.yml +++ b/tests/integration/targets/activation/tasks/main.yml @@ -172,6 +172,82 @@ that: - _result.changed + - name: Create a rulebook activation with missing project name + ansible.eda.rulebook_activation: + name: Invalid_activation + description: "Example Activation description" + rulebook_name: "{{ rulebook_name }}" + decision_environment_name: "{{ decision_env_name }}" + enabled: False + awx_token_name: "{{ awx_token_name }}" + organization_name: Default + project_name: Invalid_Project + register: _result + ignore_errors: true + + - name: Check rulebook activation creation + assert: + that: + - _result.failed + - "'Project Invalid_Project not found.' in _result.msg" + + - name: Create a rulebook activation with missing rulebook name + ansible.eda.rulebook_activation: + name: Invalid_activation + description: "Example Activation description" + project_name: "{{ project_name }}" + decision_environment_name: "{{ decision_env_name }}" + enabled: False + awx_token_name: "{{ awx_token_name }}" + organization_name: Default + rulebook_name: Invalid_Rulebook + register: _result + ignore_errors: true + + - name: Check rulebook activation creation + assert: + that: + - _result.failed + - "'Rulebook Invalid_Rulebook not found' in _result.msg" + + - name: Create a rulebook activation with missing decision environment name + ansible.eda.rulebook_activation: + name: Invalid_activation + description: "Example Activation description" + project_name: "{{ project_name }}" + rulebook_name: "{{ rulebook_name }}" + enabled: False + awx_token_name: "{{ awx_token_name }}" + organization_name: Default + decision_environment_name: Invalid_Decision_Env + register: _result + ignore_errors: true + + - name: Check rulebook activation creation + assert: + that: + - _result.failed + - "'Decision Environment Invalid_Decision_Env not found.' in _result.msg" + + - name: Create a rulebook activation with missing organization name + ansible.eda.rulebook_activation: + name: Invalid_activation + description: "Example Activation description" + project_name: "{{ project_name }}" + rulebook_name: "{{ rulebook_name }}" + decision_environment_name: "{{ decision_env_name }}" + enabled: False + awx_token_name: "{{ awx_token_name }}" + organization_name: Invalid_Organization + register: _result + ignore_errors: true + + - name: Check rulebook activation creation + assert: + that: + - _result.failed + - "'Organization Invalid_Organization not found.' in _result.msg" + - name: Create a new rulebook activation again ansible.eda.rulebook_activation: name: "{{ activation_name }}" @@ -379,6 +455,7 @@ - "{{ activation_name_source_name }}" - "{{ activation_name_source_index }}" - "{{ activation_name_wrong_source }}" + - Invalid_activation ignore_errors: true - name: Delete decision environment From 34b75de3fe7152180ec0d162a5d96ab20f7acabb Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 22 Nov 2024 13:23:47 +0100 Subject: [PATCH 3/4] ci: make publication only run with tags (#355) Signed-off-by: Alex --- .github/workflows/tox.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index fcb75c2f..3fd03924 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -246,6 +246,7 @@ jobs: needs: - check environment: release + if: github.event_name == 'push' && github.ref_type == 'tag' steps: - uses: actions/checkout@v4 @@ -266,9 +267,6 @@ jobs: - run: pip install ansible-core - name: Publish the collection on Galaxy - if: | - github.event_name == 'push' && - (github.ref_type == 'tag' || github.ref == 'refs/heads/main') run: > [[ "${{ secrets.ANSIBLE_GALAXY_API_KEY != '' }}" ]] || { echo "ANSIBLE_GALAXY_API_KEY is required to publish on galaxy" ; exit 1; } @@ -277,7 +275,6 @@ jobs: secrets.ANSIBLE_GALAXY_API_KEY }}" - name: Upload the artifact to the release - if: github.ref_type == 'tag' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | From a02bac4ee453a71f012106bec7653a4efa4c27d9 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 22 Nov 2024 14:52:44 +0100 Subject: [PATCH 4/4] ci: pin tox-extra version (#357) --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 661b359c..a8e66ec5 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ skipsdist = true # this repo is not a python package isolated_build = true requires = tox >= 4.6.3 - tox-extra >= 2.0.1 # bindep check + tox-extra == 2.0.2 # bindep check setuptools >= 65.3.0 # editable installs ignore_basepython_conflict = false