From 66814bebb4ab9cec45179a24caf5a73155c36bb2 Mon Sep 17 00:00:00 2001 From: Madhu Kanoor Date: Mon, 30 Jan 2023 16:55:06 -0500 Subject: [PATCH] Added support for searching an array of objects in conditions Using the guidelines from selectattr and select in Ansible and Jinja we can write conditions that would check if the select/selectattr yields one or more objects to pass the condition. The result of the selectattr and select is a boolean value. It can only answer if in the array of objects there is an object that satisfies the condition, the data can be further proecssed in the playbook if need be. This also confirms to the ansible playbook guidelines of tests having either a **is** or **is not** e.g. levels is an array of integers ``` event.levels is select('>', 25) ``` addresses is an array of strings ``` event.addresses is select('regex', 'Main St') ``` addresses is an array of strings ``` event.addresses is not select('regex', 'Wall St') ``` event.people is an array of person objects, where the attribute we are checking is person.age and checking if the age is > 30. ``` event.people is selectattr('person.age', '>', 30) ``` event.people is an array of person objects, where the attribute we are checking is person.name and checking if the name is either Barney or Fred ``` event.people is selectattr('person.name', 'regex', 'Barney|Fred') ``` --- ansible_rulebook/condition_parser.py | 80 ++++++++++++++++- ansible_rulebook/condition_types.py | 17 +++- ansible_rulebook/exception.py | 10 +++ ansible_rulebook/json_generator.py | 29 +++++++ docs/conditions.rst | 89 ++++++++++++++++++- setup.cfg | 2 +- tests/examples/61_select_1.yml | 30 +++++++ tests/examples/62_select_2.yml | 33 +++++++ tests/examples/63_selectattr_1.yml | 39 +++++++++ tests/examples/64_selectattr_2.yml | 25 ++++++ tests/examples/65_selectattr_3.yml | 17 ++++ tests/test_ast.py | 68 ++++++++++++++- tests/test_examples.py | 123 +++++++++++++++++++++++++++ 13 files changed, 556 insertions(+), 6 deletions(-) create mode 100644 tests/examples/61_select_1.yml create mode 100644 tests/examples/62_select_2.yml create mode 100644 tests/examples/63_selectattr_1.yml create mode 100644 tests/examples/64_selectattr_2.yml create mode 100644 tests/examples/65_selectattr_3.yml diff --git a/ansible_rulebook/condition_parser.py b/ansible_rulebook/condition_parser.py index 0c14fa66..0f67fa48 100644 --- a/ansible_rulebook/condition_parser.py +++ b/ansible_rulebook/condition_parser.py @@ -32,6 +32,11 @@ pyparsing_common, ) +from ansible_rulebook.exception import ( + SelectattrOperatorException, + SelectOperatorException, +) + ParserElement.enable_packrat() from ansible_rulebook.condition_types import ( # noqa: E402 @@ -44,9 +49,38 @@ NegateExpression, OperatorExpression, SearchType, + SelectattrType, + SelectType, String, ) +VALID_SELECT_ATTR_OPERATORS = [ + "==", + "!=", + ">", + ">=", + "<", + "<=", + "regex", + "search", + "match", + "in", + "not in", + "contains", + "not contains", +] + +VALID_SELECT_OPERATORS = [ + "==", + "!=", + ">", + ">=", + "<", + "<=", + "regex", + "search", + "match", +] SUPPORTED_SEARCH_KINDS = ("match", "regex", "search") logger = logging.getLogger(__name__) @@ -95,6 +129,20 @@ ) list_values = Suppress("[") + delim_value + Suppress("]") +selectattr_t = ( + Literal("selectattr") + + Suppress("(") + + Group(delimitedList(ident | allowed_values | list_values)) + + Suppress(")") +) + +select_t = ( + Literal("select") + + Suppress("(") + + Group(delimitedList(ident | allowed_values | list_values)) + + Suppress(")") +) + def as_list(var): if hasattr(var.__class__, "as_list"): @@ -102,6 +150,22 @@ def as_list(var): return var +def SelectattrTypeFactory(tokens): + if tokens[1].value not in VALID_SELECT_ATTR_OPERATORS: + raise SelectattrOperatorException( + f"Operator {tokens[1]} is not supported" + ) + + return SelectattrType(tokens[0], tokens[1], as_list(tokens[2])) + + +def SelectTypeFactory(tokens): + if tokens[0].value not in VALID_SELECT_OPERATORS: + raise SelectOperatorException(f"Operator {tokens[0]} is not supported") + + return SelectType(tokens[0], as_list(tokens[1])) + + def SearchTypeFactory(kind, tokens): options = [] if len(tokens) > 1: @@ -123,6 +187,18 @@ def OperatorExpressionFactory(tokens): tokens[0], tokens[1], search_type ) tokens = tokens[4:] + elif tokens[2] == "selectattr": + select_attr_type = SelectattrTypeFactory(tokens[3]) + return_value = OperatorExpression( + tokens[0], tokens[1], select_attr_type + ) + tokens = tokens[4:] + elif tokens[2] == "select": + select_type = SelectTypeFactory(tokens[3]) + return_value = OperatorExpression( + tokens[0], tokens[1], select_type + ) + tokens = tokens[4:] else: return_value = OperatorExpression( as_list(tokens[0]), tokens[1], as_list(tokens[2]) @@ -137,7 +213,9 @@ def OperatorExpressionFactory(tokens): all_terms = ( - string_search_t + selectattr_t + | select_t + | string_search_t | list_values | float_t | integer diff --git a/ansible_rulebook/condition_types.py b/ansible_rulebook/condition_types.py index 69afba52..87990656 100644 --- a/ansible_rulebook/condition_types.py +++ b/ansible_rulebook/condition_types.py @@ -46,10 +46,21 @@ class SearchType(NamedTuple): options: List[KeywordValue] = None +class SelectattrType(NamedTuple): + key: String + operator: String + value: Union[Float, Integer, String, Boolean, List] + + +class SelectType(NamedTuple): + operator: String + value: Union[Float, Integer, String, Boolean, List] + + class OperatorExpression(NamedTuple): left: Union[Float, Integer, String, List] operator: str - right: Union[Float, Integer, String, List, SearchType] + right: Union[Float, Integer, String, List, SearchType, SelectType] class NegateExpression(NamedTuple): @@ -67,6 +78,8 @@ class Condition(NamedTuple): NegateExpression, KeywordValue, SearchType, + SelectattrType, + SelectType, ] @@ -81,4 +94,6 @@ class Condition(NamedTuple): NegateExpression, KeywordValue, SearchType, + SelectType, + SelectattrType, ] diff --git a/ansible_rulebook/exception.py b/ansible_rulebook/exception.py index c17fcfdc..fc640b7a 100644 --- a/ansible_rulebook/exception.py +++ b/ansible_rulebook/exception.py @@ -41,3 +41,13 @@ class VarsKeyMissingException(Exception): class InvalidAssignmentException(Exception): pass + + +class SelectattrOperatorException(Exception): + + pass + + +class SelectOperatorException(Exception): + + pass diff --git a/ansible_rulebook/json_generator.py b/ansible_rulebook/json_generator.py index a657d021..ced17735 100644 --- a/ansible_rulebook/json_generator.py +++ b/ansible_rulebook/json_generator.py @@ -28,6 +28,8 @@ NegateExpression, OperatorExpression, SearchType, + SelectattrType, + SelectType, String, ) from ansible_rulebook.exception import ( @@ -116,6 +118,17 @@ def visit_condition(parsed_condition: ConditionTypes, variables: Dict): visit_condition(v, variables) for v in parsed_condition.options ] return {"SearchType": data} + elif isinstance(parsed_condition, SelectattrType): + return dict( + key=visit_condition(parsed_condition.key, variables), + operator=visit_condition(parsed_condition.operator, variables), + value=visit_condition(parsed_condition.value, variables), + ) + elif isinstance(parsed_condition, SelectType): + return dict( + operator=visit_condition(parsed_condition.operator, variables), + value=visit_condition(parsed_condition.value, variables), + ) elif isinstance(parsed_condition, KeywordValue): return dict( name=visit_condition(parsed_condition.name, variables), @@ -143,6 +156,14 @@ def visit_condition(parsed_condition: ConditionTypes, variables: Dict): return create_binary_node( "SearchMatchesExpression", parsed_condition, variables ) + elif isinstance(parsed_condition.right, SelectattrType): + return create_binary_node( + "SelectAttrExpression", parsed_condition, variables + ) + elif isinstance(parsed_condition.right, SelectType): + return create_binary_node( + "SelectExpression", parsed_condition, variables + ) elif parsed_condition.operator == "is not": if isinstance(parsed_condition.right, Identifier): if parsed_condition.right.value == "defined": @@ -155,6 +176,14 @@ def visit_condition(parsed_condition: ConditionTypes, variables: Dict): return create_binary_node( "SearchNotMatchesExpression", parsed_condition, variables ) + elif isinstance(parsed_condition.right, SelectattrType): + return create_binary_node( + "SelectAttrNotExpression", parsed_condition, variables + ) + elif isinstance(parsed_condition.right, SelectType): + return create_binary_node( + "SelectNotExpression", parsed_condition, variables + ) else: raise Exception(f"Unhandled token {parsed_condition}") elif isinstance(parsed_condition, NegateExpression): diff --git a/docs/conditions.rst b/docs/conditions.rst index afadda7d..899628f2 100644 --- a/docs/conditions.rst +++ b/docs/conditions.rst @@ -38,8 +38,8 @@ A condition can contain * Multiple conditions where any one of them has to match * Multiple conditions where not all one of them have to match -Supported datatypes -******************* +Supported data types +******************** The data type is of great importance for the rules engine. The following types are supported * integers @@ -96,6 +96,14 @@ Conditions support the following operators: - To check if the regular expression pattern exists in the string * - is not regex(pattern,ignorecase=true) - To check if the regular expression pattern does not exist in the string + * - is select(operator, value) + - To check if an item exists in the list, that satisfies the test defined by operator and value + * - is not select(operator, value) + - To check if an item does not exist in the list, that does not satisfy the test defined by operator and value + * - is selectattr(key, operator, value) + - To check if an object exists in the list, that satisfies the test defined by key, operator and value + * - is not selectattr(key, operator, value) + - To check if an object does not exist in the list, that does not satisfy the test defined by key, operator and value * - `<<` - Assignment operator, to save the matching events or facts with events or facts prefix * - not @@ -582,6 +590,83 @@ String regular expression | In the above example we check if the event.url does not have "example.com" in its value | And the option controls that this is a case insensitive search. +Check if an item exists in a list based on a test +------------------------------------------------- + + .. code-block:: yaml + + name: check if an item exist in list + condition: event.levels is select('>=', 10) + action: + echo: + message: The list has an item with the value greater than or equal to 10 + +| In the above example "levels" is a list of integers e.g. [1,2,3,20], the test says +| check if any item exists in the list with a value >= 10. This test passes because +| of the presence of 20 in the list. If the value of "levels" is [1,2,3] then the +| test would yield False. + +Check if an item does not exist in a list based on a test +--------------------------------------------------------- + + .. code-block:: yaml + + name: check if an item does not exist in list + condition: event.levels is not select('>=', 10) + action: + echo: + message: The list does not have item with the value greater than or equal to 10 + +| In the above example "levels" is a list of integers e.g. [1,2,3], the test says +| check if *no* item exists with a value >= 10. This test passes because none of the items +| in the list is greater than or equal to 10. If the value of "levels" is [1,2,3,20] then +| the test would yield False because of the presence of 20 in the list. + +| The result of the *select* condition is either True or False. It doesn't return the item or items. +| The select takes 2 arguments which are comma delimited, **operator** and **value**. +| The different operators we support are >,>=,<,<=,==,!=,match,search,regex +| The value is based on the operator used, if the operator is regex then the value is a pattern. +| If the operator is one of >,>=,<,<= then the value is either an integer or a float + +Checking if an object exists in a list based on a test +------------------------------------------------------ + + .. code-block:: yaml + + name: check if an object exist in list + condition: event.objects is selectattr('age', '>=', 20) + action: + echo: + message: An object with age greater than 20 found + +| In the above example "objects" is a list of object's, with multiple properties. One of the +| properties is age, the test says check if any object exists in the list with an age >= 20. + +Checking if an object does not exist in a list based on a test +--------------------------------------------------------------- + + .. code-block:: yaml + + name: check if an object does not exist in list + condition: event.objects is not selectattr('age', '>=', 20) + action: + echo: + message: No object with age greater than 20 found + +| In the above example "objects" is a list of object's, with multiple properties. One of the +| properties is age, the test says check if *no* object exists in the list with an age >= 20. + +| The result of the *selectattr* condition is either True or False. It doesn't return the +| matching object or objects. +| The *selectattr* takes 3 arguments which are comma delimited, **key**, **operator** and **value**. +| The key is a valid key name in the object. +| The different operators we support are >, >=, <, <=, ==, !=, match, search, regex, in, not in, +| contains, not contains. +| The value is based on the operator used, if the operator is regex then the value is a pattern. +| If the operator is one of >, >=, <, <= then the value is either an integer or a float. +| If the operator is in or not in then the value is list of integer, float or string. + + FAQ *** diff --git a/setup.cfg b/setup.cfg index e45fbd54..db19aaf2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,7 +32,7 @@ install_requires = janus ansible-runner websockets - drools_jpy == 0.1.9 + drools_jpy == 0.2.0 [options.packages.find] include = diff --git a/tests/examples/61_select_1.yml b/tests/examples/61_select_1.yml new file mode 100644 index 00000000..89c10c35 --- /dev/null +++ b/tests/examples/61_select_1.yml @@ -0,0 +1,30 @@ +--- +- name: 61 select 1 + hosts: all + sources: + - generic: + payload: + - name: Fred + age: 54 + levels: + - 10 + - 20 + - 30 + - name: Barney + age: 53 + levels: + - 11 + - 15 + - 16 + - name: Wilma + age: 53 + levels: + - 1 + - 5 + - 6 + rules: + - name: r1 + condition: event.levels is select('>', 25) + action: + echo: + message: Found a player with level greater than 25 diff --git a/tests/examples/62_select_2.yml b/tests/examples/62_select_2.yml new file mode 100644 index 00000000..1a3f5f5e --- /dev/null +++ b/tests/examples/62_select_2.yml @@ -0,0 +1,33 @@ +--- +- name: 62 select 2 + hosts: all + sources: + - generic: + payload: + - name: Fred + age: 54 + addresses: + - 123 Main St, Bedrock, MI + - 545 Spring St, Cresskill, NJ + - 435 Wall Street, New York, NY + - name: Barney + age: 53 + addresses: + - 345 Bleeker St, Bedrock, MI + - 145 Wall St, Dumont, NJ + - name: Wilma + age: 47 + addresses: + - 123 Main St, Bedrock, MI + - 432 Raymond Blvd, Newark, NJ + rules: + - name: r1 + condition: event.addresses is select('regex', 'Main St') + action: + echo: + message: Some one lives on Main Street + - name: r2 + condition: event.addresses is not select('regex', 'Major St') + action: + echo: + message: No one lives on Major St diff --git a/tests/examples/63_selectattr_1.yml b/tests/examples/63_selectattr_1.yml new file mode 100644 index 00000000..76e9d731 --- /dev/null +++ b/tests/examples/63_selectattr_1.yml @@ -0,0 +1,39 @@ +--- +- name: 63 selectattr 1 + hosts: all + sources: + - generic: + payload: + - people: + - person: + name: Fred + age: 54 + - person: + name: Barney + age: 45 + - person: + name: Wilma + age: 23 + - person: + name: Betty + age: 25 + - friends: + - person: + name: Barney + hobby: golf + - person: + name: Fred + hobby: driving + + + rules: + - name: r1 + condition: event.people is selectattr('person.age', '>', 30) + action: + echo: + message: Has a person greater than 30 + - name: r2 + condition: event.friends is selectattr('person.name', 'regex', 'Barney|Fred') + action: + echo: + message: Barney or Fred in friends list diff --git a/tests/examples/64_selectattr_2.yml b/tests/examples/64_selectattr_2.yml new file mode 100644 index 00000000..63164e4c --- /dev/null +++ b/tests/examples/64_selectattr_2.yml @@ -0,0 +1,25 @@ +--- +- name: 64 selectattr 2 + hosts: all + sources: + - generic: + payload: + - people: + - person: + name: Fred + age: 54 + - person: + name: Barney + age: 45 + - person: + name: Wilma + age: 23 + - person: + name: Betty + age: 25 + rules: + - name: r1 + condition: event.people is selectattr('person.age', 'in', [55,25]) + action: + echo: + message: Found person who is either 55 or 25 diff --git a/tests/examples/65_selectattr_3.yml b/tests/examples/65_selectattr_3.yml new file mode 100644 index 00000000..ddea952a --- /dev/null +++ b/tests/examples/65_selectattr_3.yml @@ -0,0 +1,17 @@ +--- +- name: 65 selectattr 3 + hosts: all + sources: + - generic: + payload: + - person: + name: Fred + age: 54 + rules: + - name: r1 + # event.person is not an array here its an object + # we convert it to an array of 1 + condition: event.person is selectattr('age', '>', 30) + action: + echo: + message: Has a person greater than 30 diff --git a/tests/test_ast.py b/tests/test_ast.py index a12b565a..dd826d0d 100644 --- a/tests/test_ast.py +++ b/tests/test_ast.py @@ -18,7 +18,11 @@ import yaml from ansible_rulebook.condition_parser import parse_condition -from ansible_rulebook.exception import InvalidAssignmentException +from ansible_rulebook.exception import ( + InvalidAssignmentException, + SelectattrOperatorException, + SelectOperatorException, +) from ansible_rulebook.json_generator import ( generate_dict_rulesets, visit_condition, @@ -382,6 +386,68 @@ def test_parse_condition(): {}, ) + assert { + "SelectAttrExpression": { + "lhs": {"Event": "persons"}, + "rhs": { + "key": {"String": "person.age"}, + "operator": {"String": ">="}, + "value": {"Integer": 50}, + }, + } + } == visit_condition( + parse_condition('event.persons is selectattr("person.age", ">=", 50)'), + {}, + ) + + assert { + "SelectAttrNotExpression": { + "lhs": {"Event": "persons"}, + "rhs": { + "key": {"String": "person.name"}, + "operator": {"String": "=="}, + "value": {"String": "fred"}, + }, + } + } == visit_condition( + parse_condition( + 'event.persons is not selectattr("person.name", "==", "fred")' + ), + {}, + ) + + assert { + "SelectExpression": { + "lhs": {"Event": "ids"}, + "rhs": {"operator": {"String": ">="}, "value": {"Integer": 10}}, + } + } == visit_condition(parse_condition('event.ids is select(">=", 10)'), {}) + + assert { + "SelectNotExpression": { + "lhs": {"Event": "persons"}, + "rhs": { + "operator": {"String": "regex"}, + "value": {"String": "fred|barney"}, + }, + } + } == visit_condition( + parse_condition('event.persons is not select("regex", "fred|barney")'), + {}, + ) + + +def test_invalid_select_operator(): + with pytest.raises(SelectOperatorException): + parse_condition('event.persons is not select("in", ["fred","barney"])') + + +def test_invalid_selectattr_operator(): + with pytest.raises(SelectattrOperatorException): + parse_condition( + 'event.persons is not selectattr("name", "cmp", "fred")' + ) + @pytest.mark.parametrize( "rulebook", diff --git a/tests/test_examples.py b/tests/test_examples.py index 1dba0464..e7e6d021 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1567,3 +1567,126 @@ async def test_60_json_filter(): ], } validate_events(event_log, **checks) + + +@pytest.mark.asyncio +async def test_61_select_1(): + ruleset_queues, event_log = load_rulebook("examples/61_select_1.yml") + + queue = ruleset_queues[0][1] + rs = ruleset_queues[0][0] + with SourceTask(rs.sources[0], "sources", {}, queue): + await run_rulesets( + event_log, + ruleset_queues, + dict(), + load_inventory("playbooks/inventory.yml"), + ) + + checks = { + "max_events": 2, + "shutdown_events": 1, + "actions": [ + "61 select 1::r1::echo", + ], + } + validate_events(event_log, **checks) + + +@pytest.mark.asyncio +async def test_62_select_2(): + ruleset_queues, event_log = load_rulebook("examples/62_select_2.yml") + + queue = ruleset_queues[0][1] + rs = ruleset_queues[0][0] + with SourceTask(rs.sources[0], "sources", {}, queue): + await run_rulesets( + event_log, + ruleset_queues, + dict(), + load_inventory("playbooks/inventory.yml"), + ) + + checks = { + "max_events": 4, + "shutdown_events": 1, + "actions": [ + "62 select 2::r1::echo", + "62 select 2::r2::echo", + "62 select 2::r1::echo", + ], + } + validate_events(event_log, **checks) + + +@pytest.mark.asyncio +async def test_63_selectattr_1(): + ruleset_queues, event_log = load_rulebook("examples/63_selectattr_1.yml") + + queue = ruleset_queues[0][1] + rs = ruleset_queues[0][0] + with SourceTask(rs.sources[0], "sources", {}, queue): + await run_rulesets( + event_log, + ruleset_queues, + dict(), + load_inventory("playbooks/inventory.yml"), + ) + + checks = { + "max_events": 3, + "shutdown_events": 1, + "actions": [ + "63 selectattr 1::r1::echo", + "63 selectattr 1::r2::echo", + ], + } + validate_events(event_log, **checks) + + +@pytest.mark.asyncio +async def test_64_selectattr_2(): + ruleset_queues, event_log = load_rulebook("examples/64_selectattr_2.yml") + + queue = ruleset_queues[0][1] + rs = ruleset_queues[0][0] + with SourceTask(rs.sources[0], "sources", {}, queue): + await run_rulesets( + event_log, + ruleset_queues, + dict(), + load_inventory("playbooks/inventory.yml"), + ) + + checks = { + "max_events": 2, + "shutdown_events": 1, + "actions": [ + "64 selectattr 2::r1::echo", + ], + } + validate_events(event_log, **checks) + + +@pytest.mark.asyncio +async def test_65_selectattr_3(): + ruleset_queues, event_log = load_rulebook("examples/65_selectattr_3.yml") + + queue = ruleset_queues[0][1] + rs = ruleset_queues[0][0] + with SourceTask(rs.sources[0], "sources", {}, queue): + await run_rulesets( + event_log, + ruleset_queues, + dict(), + load_inventory("playbooks/inventory.yml"), + ) + + checks = { + "max_events": 2, + "shutdown_events": 1, + "actions": [ + "65 selectattr 3::r1::echo", + ], + } + validate_events(event_log, **checks)