From 3a75a1664b33fcb5cc5be7c984c7929e0ce0630d Mon Sep 17 00:00:00 2001 From: Alina Buzachis Date: Mon, 12 Aug 2024 23:26:47 +0200 Subject: [PATCH] Add module_utils (#244) Signed-off-by: Alina Buzachis Co-authored-by: Abhijeet Kasurde --- .pre-commit-config.yaml | 3 - plugins/doc_fragments/__init__.py | 0 plugins/doc_fragments/eda_controller.py | 47 ++++ plugins/module_utils/__init__.py | 0 plugins/module_utils/arguments.py | 34 +++ plugins/module_utils/client.py | 176 ++++++++++++ plugins/module_utils/controller.py | 353 ++++++++++++++++++++++++ plugins/module_utils/errors.py | 19 ++ plugins/modules/__init__.py | 0 pyproject.toml | 14 +- 10 files changed, 642 insertions(+), 4 deletions(-) create mode 100644 plugins/doc_fragments/__init__.py create mode 100644 plugins/doc_fragments/eda_controller.py create mode 100644 plugins/module_utils/__init__.py create mode 100644 plugins/module_utils/arguments.py create mode 100644 plugins/module_utils/client.py create mode 100644 plugins/module_utils/controller.py create mode 100644 plugins/module_utils/errors.py create mode 100644 plugins/modules/__init__.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f025d2fc..795d7c09 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -38,7 +38,6 @@ repos: args: [ --fix, --exit-non-zero-on-fix, - --ignore, E402, ] - repo: https://github.com/PyCQA/flake8 rev: 7.1.1 @@ -55,8 +54,6 @@ repos: - id: pylint args: - --output-format=colorized - - --disable=C0103,C0114,C0115,C0116,R0913,R1735, - - --max-line-length=120 additional_dependencies: - aiobotocore - aiohttp diff --git a/plugins/doc_fragments/__init__.py b/plugins/doc_fragments/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/plugins/doc_fragments/eda_controller.py b/plugins/doc_fragments/eda_controller.py new file mode 100644 index 00000000..e7c41e23 --- /dev/null +++ b/plugins/doc_fragments/eda_controller.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# Copyright: Contributors to the Ansible project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + + +class ModuleDocFragment: + + AUTHS = """ +options: + controller_host: + description: + - The URL of the EDA controller. + - If not set, the value of the C(CONTROLLER_URL) environment variable will be used. + required: true + type: str + version_added: '2.0.0' + controller_username: + description: + - Username used for authentication. + - If not set, the value of the C(CONTROLLER_USERNAME) environment variable will be used. + type: str + version_added: '2.0.0' + controller_password: + description: + - Password used for authentication. + - If not set, the value of the C(CONTROLLER_PASSWORD) environment variable will be used. + type: str + version_added: '2.0.0' + request_timeout: + description: + - Timeout in seconds for the connection with the EDA controller. + - If not set, the value of the C(CONTROLLER_TIMEOUT) environment variable will be used. + type: float + default: 10 + version_added: '2.0.0' + validate_certs: + description: + - Whether to allow insecure connections to Ansible Automation Platform EDA + Controller instance. + - If C(no), SSL certificates will not be validated. + - This should only be used on personally controlled sites using self-signed certificates. + - If value not set, will try environment variable C(CONTROLLER_VERIFY_SSL) + default: True + type: bool + version_added: '2.0.0' +""" diff --git a/plugins/module_utils/__init__.py b/plugins/module_utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/plugins/module_utils/arguments.py b/plugins/module_utils/arguments.py new file mode 100644 index 00000000..23760346 --- /dev/null +++ b/plugins/module_utils/arguments.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +# Copyright: Contributors to the Ansible project +# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause) + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +from ansible.module_utils.basic import env_fallback + +AUTH_ARGSPEC = { + "controller_host": { + "fallback": (env_fallback, ["CONTROLLER_HOST"]), + "required": True, + }, + "controller_username": { + "fallback": (env_fallback, ["CONTROLLER_USERNAME"]), + }, + "controller_password": { + "fallback": (env_fallback, ["CONTROLLER_PASSWORD"]), + "no_log": True, + }, + "validate_certs": { + "type": "bool", + "default": True, + "fallback": (env_fallback, ["CONTROLLER_VERIFY_SSL"]), + }, + "request_timeout": { + "type": "float", + "default": 10.0, + "fallback": (env_fallback, ["CONTROLLER_REQUEST_TIMEOUT"]), + }, +} diff --git a/plugins/module_utils/client.py b/plugins/module_utils/client.py new file mode 100644 index 00000000..f5f26343 --- /dev/null +++ b/plugins/module_utils/client.py @@ -0,0 +1,176 @@ +# -*- coding: utf-8 -*- + +# Copyright: Contributors to the Ansible project +# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause) + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +import json +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode, urlparse + +from ansible.module_utils.urls import Request + +from .errors import AuthError, EDAHTTPError + + +class Response: + def __init__(self, status, data, headers=None): + self.status = status + self.data = data + # [('h1', 'v1'), ('H2', 'V2')] -> {'h1': 'v1', 'h2': 'V2'} + self.headers = ( + dict((k.lower(), v) for k, v in dict(headers).items()) if headers else {} + ) + + self._json = None + + @property + def json(self): + if self._json is None: + try: + self._json = json.loads(self.data) + except ValueError as value_exp: + raise EDAHTTPError( + f"Received invalid JSON response: {self.data}" + ) from value_exp + return self._json + + +class Client: + def __init__( + self, + host, + username=None, + password=None, + timeout=None, + validate_certs=None, + ): + + if not (host or "").startswith(("https://", "http://")): + raise EDAHTTPError( + f"Invalid instance host value: '{host}'. " + "Value must start with 'https://' or 'http://'" + ) + + self.host = host + self.username = username + self.password = password + self.timeout = timeout + self.validate_certs = validate_certs + + # Try to parse the hostname as a url + try: + self.url = urlparse(self.host) + # Store URL prefix for later use in build_url + self.url_prefix = self.url.path + except Exception as e: + raise EDAHTTPError( + f"Unable to parse eda_controller_host ({self.host}): {e}" + ) from e + + self.session = Request() + + def _request(self, method, path, data=None, headers=None): + try: + raw_resp = self.session.open( + method, + path, + data=data, + url_password=self.password, + url_username=self.username, + headers=headers, + timeout=self.timeout, + validate_certs=self.validate_certs, + force_basic_auth=True, + ) + + except HTTPError as http_exp: + # Wrong username/password, or expired access token + if http_exp.code == 401: + raise AuthError( + f"Failed to authenticate with the instance: {http_exp.code} {http_exp.reason}" + ) from http_exp + # Other HTTP error codes do not necessarily mean errors. + # This is for the caller to decide. + return Response(http_exp.code, http_exp.read(), http_exp.headers) + except URLError as url_exp: + raise EDAHTTPError(url_exp.reason) from url_exp + + return Response(raw_resp.status, raw_resp.read(), raw_resp.headers) + + def build_url(self, endpoint, query_params=None, identifier=None): + # Make sure we start with /api/vX + if not endpoint.startswith("/"): + endpoint = f"/{endpoint}" + prefix = self.url_prefix.rstrip("/") + + if not endpoint.startswith(prefix + "/api/"): + endpoint = prefix + f"/api/eda/v1{endpoint}" + if not endpoint.endswith("/") and "?" not in endpoint: + endpoint = f"{endpoint}/" + + # Update the URL path with the endpoint + url = self.url._replace(path=endpoint) + + if query_params: + url = url._replace(query=urlencode(query_params)) + if identifier: + url = url._replace(path=url.path + str(identifier) + "/") + + return url + + def request(self, method, endpoint, **kwargs): + # In case someone is calling us directly; make sure we were given a + # method, let's not just assume a GET + if not method: + raise EDAHTTPError("The HTTP method must be defined") + + if method in ["POST"]: + url = self.build_url(endpoint) + elif method in ["DELETE", "PATCH", "PUT"]: + url = self.build_url(endpoint, identifier=kwargs.get("id")) + else: + url = self.build_url(endpoint, query_params=kwargs.get("data")) + + # Extract the headers, this will be used in a couple of places + headers = kwargs.get("headers", {}) + + if method in ["POST", "PUT", "PATCH"]: + headers.setdefault("Content-Type", "application/json") + kwargs["headers"] = headers + + data = ( + None # Important, if content type is not JSON, this should not + # be dict type + ) + if headers.get("Content-Type", "") == "application/json": + data = json.dumps(kwargs.get("data", {})) + + return self._request(method, url.geturl(), data=data, headers=headers) + + def get(self, path, **kwargs): + resp = self.request("GET", path, **kwargs) + if resp.status in (200, 404): + return resp + raise EDAHTTPError(f"HTTP error {resp.json}") + + def post(self, path, **kwargs): + resp = self.request("POST", path, **kwargs) + if resp.status == 201: + return resp + raise EDAHTTPError(f"HTTP error {resp.json}") + + def patch(self, path, **kwargs): + resp = self.request("PATCH", path, **kwargs) + if resp.status == 200: + return resp + raise EDAHTTPError(f"HTTP error {resp.json}") + + def delete(self, path, **kwargs): + resp = self.request("DELETE", path, **kwargs) + if resp.status == 204: + return resp + raise EDAHTTPError(f"HTTP error {resp.json}") diff --git a/plugins/module_utils/controller.py b/plugins/module_utils/controller.py new file mode 100644 index 00000000..8733ab5b --- /dev/null +++ b/plugins/module_utils/controller.py @@ -0,0 +1,353 @@ +# -*- coding: utf-8 -*- + +# Copyright: Contributors to the Ansible project +# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause) +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +from .errors import EDAError + + +class Controller: + IDENTITY_FIELDS = {"users": "username"} + ENCRYPTED_STRING = "$encrypted$" + + def __init__(self, client, module): + self.client = client + self.module = module + self.result = {"changed": False} + + if "update_secrets" in self.module.params: + self.update_secrets = self.module.params.pop("update_secrets") + else: + self.update_secrets = True + + @staticmethod + def get_name_field_from_endpoint(endpoint): + return Controller.IDENTITY_FIELDS.get(endpoint, "name") + + def get_endpoint(self, endpoint, **kwargs): + return self.client.get(endpoint, **kwargs) + + def post_endpoint(self, endpoint, **kwargs): + # Handle check mode + if self.module.check_mode: + self.result["changed"] = True + return self.result + + return self.client.post(endpoint, **kwargs) + + def patch_endpoint(self, endpoint, **kwargs): + # Handle check mode + if self.module.check_mode: + self.result["changed"] = True + return self.result + + return self.client.patch(endpoint, **kwargs) + + def delete_endpoint(self, endpoint, **kwargs): + # Handle check mode + if self.module.check_mode: + self.result["changed"] = True + return self.result + + return self.client.delete(endpoint, **kwargs) + + def get_item_name(self, item): + if item: + if "name" in item: + return item["name"] + + for field_name in Controller.IDENTITY_FIELDS.values(): + if field_name in item: + return item[field_name] + + if item: + msg = f"Cannot determine identity field for {item.get('type', 'unknown')} object." + raise EDAError(msg) + msg = "Cant determine identity field for Undefined object." + raise EDAError(msg) + + def fail_wanted_one(self, response, endpoint, query_params): + sample = response.json.copy() + if len(sample["results"]) > 1: + sample["results"] = sample["results"][:2] + ["...more results snipped..."] + url = self.client.build_url(endpoint, query_params) + host_length = len(self.client.host) + display_endpoint = url.geturl()[ + host_length: + ] # truncate to not include the base URL + msg = f"Request to {display_endpoint} returned {response.json['count']} items, expected 1" + raise EDAError(msg) + + def get_one_or_many( + self, + endpoint, + name=None, + allow_none=True, + check_exists=False, + want_one=True, + **kwargs, + ): + new_kwargs = kwargs.copy() + response = None + + if name: + name_field = self.get_name_field_from_endpoint(endpoint) + new_data = kwargs.get("data", {}).copy() + new_data[name_field] = name + new_kwargs["data"] = new_data + + response = self.get_endpoint(endpoint, **new_kwargs) + + if response.status != 200: + fail_msg = f"Got a {response.status} when trying to get from {endpoint}" + + if "detail" in response.json: + fail_msg += f",detail: {response.json['detail']}" + raise EDAError(fail_msg) + + if "count" not in response.json or "results" not in response.json: + raise EDAError("The endpoint did not provide count, results") + + if response.json["count"] == 0: + if allow_none: + return None + self.fail_wanted_one(response, endpoint, new_kwargs.get("data")) + if response.json["count"] == 1: + return response.json["results"][0] + if response.json["count"] > 1: + if want_one: + if name: + # Since we did a name or ID search and got > 1 return + # something if the id matches + for asset in response.json["results"]: + if str(asset["id"]) == name: + return asset + # We got > 1 and either didn't find something by ID (which means + # multiple names) + # Or we weren't running with a or search and just got back too + # many to begin with. + self.fail_wanted_one(response, endpoint, new_kwargs.get("data")) + else: + return response.json["results"] + + if check_exists: + self.result["id"] = response.json["results"][0]["id"] + return self.result + + def create_if_needed( + self, + existing_item, + new_item, + endpoint, + item_type="unknown", + ): + response = None + if not endpoint: + msg = f"Unable to create new {item_type}, missing endpoint" + raise EDAError(msg) + + item_url = None + if existing_item: + try: + item_url = existing_item["url"] + except KeyError as e: + msg = f"Unable to process create for {item_url}, missing data {e}" + raise EDAError(msg) from e + else: + if self.module.check_mode: + return {"changed": True} + + # If we don't have an existing_item, we can try to create it + # We will pull the item_name out from the new_item, if it exists + item_name = self.get_item_name(new_item) + response = self.post_endpoint(endpoint, **{"data": new_item}) + if response.status in [200, 201]: + self.result["id"] = response.json["id"] + self.result["changed"] = True + return self.result + if response.json and "__all__" in response.json: + msg = f"Unable to create {item_type} {item_name}: {response.json['__all__'][0]}" + raise EDAError(msg) + if response.json: + msg = f"Unable to create {item_type} {item_name}: {response.json}" + raise EDAError(msg) + msg = f"Unable to create {item_type} {item_name}: {response.status}" + raise EDAError(msg) + + def _encrypted_changed_warning(self, field, old, warning=False): + if not warning: + return + self.module.warn( + f"The field {field} of {old.get('type', 'unknown')} {old.get('id', 'unknown')} " + "has encrypted data and may inaccurately report task is changed." + ) + + @staticmethod + def has_encrypted_values(obj): + """Returns True if JSON-like python content in obj has $encrypted$ + anywhere in the data as a value + """ + if isinstance(obj, dict): + for val in obj.values(): + if Controller.has_encrypted_values(val): + return True + elif isinstance(obj, list): + for val in obj: + if Controller.has_encrypted_values(val): + return True + elif obj == Controller.ENCRYPTED_STRING: + return True + return False + + @staticmethod + def fields_could_be_same(old_field, new_field): + """Treating $encrypted$ as a wild card, + return False if the two values are KNOWN to be different + return True if the two values are the same, or could potentially be same + depending on the unknown $encrypted$ value or sub-values + """ + if isinstance(old_field, dict) and isinstance(new_field, dict): + if set(old_field.keys()) != set(new_field.keys()): + return False + for key in new_field.keys(): + if not Controller.fields_could_be_same(old_field[key], new_field[key]): + return False + return True # all sub-fields are either equal or could be equal + if old_field == Controller.ENCRYPTED_STRING: + return True + return bool(new_field == old_field) + + def objects_could_be_different(self, old, new, field_set=None, warning=False): + if field_set is None: + field_set = set(fd for fd in new.keys()) + for field in field_set: + new_field = new.get(field, None) + old_field = old.get(field, None) + if old_field != new_field: + if self.update_secrets or ( + not self.fields_could_be_same(old_field, new_field) + ): + return True # Something doesn't match, or something + # might not match + elif self.has_encrypted_values(new_field) or field not in new: + if self.update_secrets or ( + not self.fields_could_be_same(old_field, new_field) + ): + # case of 'field not in new' - user password write-only + # field that API will not display + self._encrypted_changed_warning(field, old, warning=warning) + return True + return False + + def update_if_needed( + self, + existing_item, + new_item, + endpoint, + item_type, + ): + response = None + if existing_item is None: + raise RuntimeError( + "update_if_needed called incorrectly without existing_item" + ) + + # If we have an item, we can see if it needs an update + try: + if item_type == "user": + item_name = existing_item["username"] + else: + item_name = existing_item["name"] + item_id = existing_item["id"] + except KeyError as e: + msg = f"Unable to process update, missing data {e}" + raise EDAError(msg) from e + + # Check to see if anything within the item requires to be updated + needs_patch = self.objects_could_be_different(existing_item, new_item) + + # If we decided the item needs to be updated, update it + self.result["id"] = item_id + if needs_patch: + if self.module.check_mode: + return {"changed": True} + + response = self.patch_endpoint( + endpoint, **{"data": new_item, "id": item_id} + ) + if response.status == 200: + # compare apples-to-apples, old API data to new API data + # but do so considering the fields given in parameters + self.result["changed"] |= self.objects_could_be_different( + existing_item, + response.json, + field_set=new_item.keys(), + warning=True, + ) + return self.result + if response.json and "__all__" in response.json: + raise EDAError(response.json["__all__"]) + msg = f"Unable to update {item_type} {item_name}" + raise EDAError(msg) + return self.result + + def create_or_update_if_needed( + self, + existing_item, + new_item, + endpoint=None, + item_type="unknown", + ): + if existing_item: + return self.update_if_needed( + existing_item, + new_item, + endpoint, + item_type=item_type, + ) + return self.create_if_needed( + existing_item, + new_item, + endpoint, + item_type=item_type, + ) + + def delete_if_needed(self, existing_item, endpoint, on_delete=None): + if existing_item is None: + return self.result + + # If we have an item, we can try to delete it + try: + item_id = existing_item["id"] + item_name = self.get_item_name(existing_item) + except KeyError as e: + msg = f"Unable to process delete, missing data {e}" + raise EDAError(msg) from e + + if self.module.check_mode: + return {"changed": True} + + response = self.delete_endpoint(endpoint, **{"id": item_id}) + + if response.status in [202, 204]: + if on_delete: + on_delete(self, response.json) + self.result["changed"] = True + self.result["id"] = item_id + return self.result + if response.json and "__all__" in response.json: + msg = f"Unable to delete {item_name}: {response['json']['__all__'][0]}" + raise EDAError(msg) + if response.json: + # This is from a project delete (if there is an active + # job against it) + if "error" in response.json: + msg = f"Unable to delete {item_name}: {response['json']['error']}" + raise EDAError(msg) + msg = f"Unable to delete {item_name}: {response['json']}" + raise EDAError(msg) + msg = f"Unable to delete {item_name}: {response['status_code']}" + raise EDAError(msg) diff --git a/plugins/module_utils/errors.py b/plugins/module_utils/errors.py new file mode 100644 index 00000000..4e3c042c --- /dev/null +++ b/plugins/module_utils/errors.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + +# Copyright: Contributors to the Ansible project +# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause) +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + + +class EDAError(Exception): + pass + + +class EDAHTTPError(EDAError): + pass + + +class AuthError(EDAError): + pass diff --git a/plugins/modules/__init__.py b/plugins/modules/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pyproject.toml b/pyproject.toml index 2579c997..aaf52dae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ include_trailing_comma = true force_grid_wrap = 0 use_parentheses = true ensure_newline_before_comments = true -line_length = 88 +line_length = 120 [tool.pylint.MASTER] # Temporary ignore until we are able to address issues on these: @@ -19,4 +19,16 @@ disable = [ "duplicate-code", "pointless-string-statement", "too-few-public-methods", + "too-many-instance-attributes", + "missing-module-docstring", + "missing-class-docstring", + "missing-function-docstring", + "too-many-arguments", + "too-many-branches", + "inconsistent-return-statements", + "invalid-name", ] +max-line-length=120 + +[tool.ruff.lint] +ignore = [ "E402" ]