From 34a7d6ae7bbe7b1c3380feb481139a055d54d528 Mon Sep 17 00:00:00 2001 From: Mika Ayenson Date: Mon, 2 Oct 2023 16:43:30 -0500 Subject: [PATCH 01/10] Support negative subqueries in sequences --- eql/ast.py | 19 ++++++++++++------- eql/etc/eql.g | 3 ++- eql/parser.py | 33 ++++++++++++++++++++++++++++++++- tests/test_parser.py | 30 +++++++++++++++++++++++++++--- 4 files changed, 73 insertions(+), 12 deletions(-) diff --git a/eql/ast.py b/eql/ast.py index 56893e9..d54aaec 100644 --- a/eql/ast.py +++ b/eql/ast.py @@ -890,30 +890,35 @@ def _render(self): class SubqueryBy(EqlNode): """Node for holding the :class:`~EventQuery` and parameters to join on.""" - __slots__ = 'query', 'join_values', 'fork', + __slots__ = 'query', 'join_values', 'data', - def __init__(self, query, join_values=None, fork=None): + def __init__(self, query, join_values=None, data=None): """Init. :param EventQuery query: The event query enclosed in the term :param list[Expression] join_values: The field to join values on - :param bool fork: Toggle for copying instead of moving a sequence on match + :param bool data: Toggle for copying instead of moving a sequence on match """ self.query = query self.join_values = join_values or [] - self.fork = fork + self.data = data @property def params(self): """Keep params for backwards compatibility.""" params = {} - if self.fork is not None: - params["fork"] = Boolean(self.fork) + if self.data is not None: + if "fork" in self.data: + params["fork"] = Boolean(self.data["fork"]) + if "is_negated" in self.data: + params["is_negated"] = Boolean(self.data["is_negated"]) return NamedParams(params) def _render(self): text = "[{}]".format(self.query.render()) - params = self.params.render() + param_copy = self.params + del param_copy.kv["is_negated"] + params = param_copy.render() if len(params): text += ' ' + params diff --git a/eql/etc/eql.g b/eql/etc/eql.g index 906e4d6..549cbf9 100644 --- a/eql/etc/eql.g +++ b/eql/etc/eql.g @@ -27,7 +27,7 @@ time_range: number name? subquery_by: subquery fork_param? join_values? repeated_sequence? sequence_alias? -subquery: "[" event_query "]" +subquery: ( "[" | MISSING_EVENT_OPEN ) event_query "]" fork_param: "fork" (EQUALS boolean)? // Expressions @@ -107,6 +107,7 @@ escaped_name: ESCAPED_NAME // sequence by pid [1] [true] looks identical to: // sequence by pid[1] [true] FIELD: FIELD_IDENT (ATTR | INDEX)+ +MISSING_EVENT_OPEN: "![" OPTIONAL_FIELD: "?" FIELD_IDENT (ATTR | INDEX)* ATTR: "." WHITESPACE? FIELD_IDENT INDEX: "[" WHITESPACE? UNSIGNED_INTEGER WHITESPACE? "]" diff --git a/eql/parser.py b/eql/parser.py index 2d41a30..7c3ca11 100644 --- a/eql/parser.py +++ b/eql/parser.py @@ -55,6 +55,7 @@ nullable_fields = ParserConfig(strict_fields=False) non_nullable_fields = ParserConfig(strict_fields=True) allow_enum_fields = ParserConfig(enable_enum=True) +allow_negation = ParserConfig(allow_negation=True) allow_sample = ParserConfig(allow_sample=True) allow_runs = ParserConfig(allow_runs=True) elasticsearch_syntax = ParserConfig(elasticsearch_syntax=True) @@ -151,6 +152,7 @@ def __init__(self, text): self._allow_runs = ParserConfig.read_stack("allow_runs", False) self._in_variable = False self._allow_sample = ParserConfig.read_stack("allow_sample", False) + self._allow_negation = ParserConfig.read_stack("allow_negation", False) @property def lines(self): @@ -1088,7 +1090,9 @@ def subquery_by(self, node, num_values=None, position=None, close=None, allow_fo else: join_values = [] - node_info = NodeInfo(ast.SubqueryBy(query, [v.node for v in join_values], **kwargs), source=node) + node0 = self.visit(node["subquery"])[0] + kwargs["is_negated"] = True if hasattr(node0, "type") and node0.type == "MISSING_EVENT_OPEN" else False + node_info = NodeInfo(ast.SubqueryBy(query, [v.node for v in join_values], kwargs), source=node) alias = node["sequence_alias"] if alias is not None: @@ -1108,10 +1112,16 @@ def join_values(self, node): def join(self, node): """Callback function to walk the AST.""" queries, close = self._get_subqueries_and_close(node) + if self.negative_subquery_used: + raise self._error(node, "Negative subquery not permitted in join", + cls=EqlSemanticError) return ast.Join(queries, close) def _get_subqueries_and_close(self, node, allow_fork=False, allow_runs=False): """Helper function used by join and sequence to avoid duplicate code.""" + + self.negative_subquery_used = False + if not self._subqueries_enabled: # Raise the error earlier (instead of waiting until subquery_by) so that it's more meaningful raise self._error(node, "Subqueries not supported") @@ -1120,6 +1130,8 @@ def _get_subqueries_and_close(self, node, allow_fork=False, allow_runs=False): subquery_nodes = node.get_list("subquery_by") first, first_info, first_runs_count = self.subquery_by(subquery_nodes[0], allow_fork=allow_fork, position=0, allow_runs=allow_runs) + if first.node.data["is_negated"]: + self.negative_subquery_used = True num_values = len(first_info) subqueries = [(first, first_info)] * first_runs_count @@ -1137,6 +1149,9 @@ def _get_subqueries_and_close(self, node, allow_fork=False, allow_runs=False): for pos, subquery in enumerate(subquery_nodes[1:], 1): subquery, join_values, runs_count = self.subquery_by(subquery, num_values=num_values, allow_fork=allow_fork, position=pos, allow_runs=allow_runs) + if subquery.node.data["is_negated"]: + self.negative_subquery_used = True + multiple_subqueries = [(subquery, join_values)] * runs_count subqueries.extend(multiple_subqueries) @@ -1228,6 +1243,10 @@ def sample(self, node): if len(queries) <= 1: raise self._error(node, "Only one item in the sample", cls=EqlSemanticError) + + if self.negative_subquery_used: + raise self._error(node, "Negative subquery not permitted in sample", + cls=EqlSemanticError) return ast.Sample(queries) def sequence(self, node): @@ -1241,8 +1260,20 @@ def sequence(self, node): params = self.time_range(node['with_params']['time_range']) allow_runs = self._elasticsearch_syntax and self._allow_runs + allow_negation = self._elasticsearch_syntax and self._allow_negation queries, close = self._get_subqueries_and_close(node, allow_fork=True, allow_runs=allow_runs) + + # Fail if negative operator used without exposing in elasticsearch syntax + if self.negative_subquery_used and not allow_negation: + raise self._error(node, "Negative subquery used ", + cls=EqlSemanticError if self._elasticsearch_syntax else EqlSyntaxError) + + # Fail if any subquery uses the negative operator without maxspan + if not params and self.negative_subquery_used: + raise self._error(node, "Negative subquery used without maxspan", + cls=EqlSemanticError if self._elasticsearch_syntax else EqlSyntaxError) + if len(queries) <= 1 and not self._elasticsearch_syntax: raise self._error(node, "Only one item in the sequence", cls=EqlSemanticError if self._elasticsearch_syntax else EqlSyntaxError) diff --git a/tests/test_parser.py b/tests/test_parser.py index 1043a3e..583ecfc 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -10,9 +10,9 @@ from eql.ast import * # noqa: F403 from eql.errors import EqlSchemaError, EqlSyntaxError, EqlSemanticError, EqlTypeMismatchError, EqlParseError from eql.parser import ( - allow_sample, allow_runs, parse_query, parse_expression, parse_definitions, ignore_missing_functions, parse_field, - parse_literal, extract_query_terms, keywords, elasticsearch_syntax, elastic_endpoint_syntax, - elasticsearch_validate_optional_fields + allow_negation, allow_sample, allow_runs, parse_query, parse_expression, parse_definitions, + ignore_missing_functions, parse_field, parse_literal, extract_query_terms, keywords, elasticsearch_syntax, + elastic_endpoint_syntax, elasticsearch_validate_optional_fields ) from eql.walkers import DepthFirstWalker from eql.pipes import * # noqa: F403 @@ -324,6 +324,12 @@ def test_invalid_queries(self): # bad sequence alias, without endpoint syntax 'sequence [process where process.name == "cmd.exe"] as a0 [network where a0.process.id == process.id]' + + # sequence with negative missing events without maxspan + 'sequence [process where true] ![file where true]', + + # sequence with negative missing events without elasticsearch flag + 'sequence with maxspan [process where true] ![file where true]', ] for query in invalid: self.assertRaises(EqlParseError, parse_query, query) @@ -637,6 +643,20 @@ def test_elasticsearch_flag(self): # invalid sample base query usage self.assertRaises(EqlSemanticError, parse_query, 'sample by user [process where opcode == 1] [process where opcode == 1]') + self.assertRaises(EqlSemanticError, parse_query, + 'sample by user [process where opcode == 1] ![process where opcode == 1]') + + with elasticsearch_syntax, allow_negation: + parse_query('sequence with maxspan=2s [process where true] ![file where true]') + parse_query('sequence with maxspan=2s ![process where true] [file where true]') + parse_query('sequence with maxspan=2s [process where true] ![file where true] [file where true]') + + self.assertRaises(EqlSemanticError, parse_query, + 'sequence [process where true] [file where true] ![file where true]') + self.assertRaises(EqlSemanticError, parse_query, + 'join ![process where true] [file where true] [file where true]') + self.assertRaises(EqlSemanticError, parse_query, + 'sample ![process where true] [file where true] [file where true]') with schema: parse_query("process where process_name == 'cmd.exe'") @@ -695,6 +715,7 @@ def test_elasticsearch_flag(self): event1 = '[network where p0.process.name == process.name]' event2 = '[network where p0.pid == 0]' event3 = '[network where p0.badfield == 0]' + event4 = f'!{event0}' parse_query('sequence %s as p0 %s' % (event0, event1)) parse_query('sequence by user.name %s as p0 %s' % (event0, event1)) parse_query('sequence with maxspan=1m %s by user.name as p0 %s by user.name' % (event0, event1)) @@ -703,6 +724,9 @@ def test_elasticsearch_flag(self): self.assertRaises(EqlSchemaError, parse_query, 'sequence by user.name %s as p1 %s' % (event0, event3)) self.assertRaises(EqlSyntaxError, parse_query, "process where process_name == 'cmd.exe'") + # negative runs not supported on the endpoint + self.assertRaises(EqlSemanticError, parse_query, 'sequence %s %s' % (event0, event4)) + # as fields not emmitted by the endpoint self.assertRaises(EqlSyntaxError, parse_query, 'process where client.as.organization.name == "string"') self.assertRaises(EqlSyntaxError, parse_query, 'process where destination.as.organization.name == "string"') From 7c196cb3db10aa5ba6c99047bf2f1c7f470845f4 Mon Sep 17 00:00:00 2001 From: Mika Ayenson Date: Mon, 2 Oct 2023 16:54:52 -0500 Subject: [PATCH 02/10] lint and changelog --- CHANGELOG.md | 8 ++++++++ eql/parser.py | 1 - tests/test_parser.py | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92cf1e5..4a8cfb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ # Event Query Language - Changelog The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +# Version 0.9.19 + + _Released 2023-10-10_ + +### Added + +* Support for [missing events](https://www.elastic.co/guide/en/elasticsearch/reference/current/eql-syntax.html#eql-missing-events) feature used in Elasticsearch sequence queries + # Version 0.9.18 _Released 2023-09-01_ diff --git a/eql/parser.py b/eql/parser.py index 7c3ca11..8df9cf1 100644 --- a/eql/parser.py +++ b/eql/parser.py @@ -1119,7 +1119,6 @@ def join(self, node): def _get_subqueries_and_close(self, node, allow_fork=False, allow_runs=False): """Helper function used by join and sequence to avoid duplicate code.""" - self.negative_subquery_used = False if not self._subqueries_enabled: diff --git a/tests/test_parser.py b/tests/test_parser.py index 583ecfc..ae2b785 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -715,7 +715,7 @@ def test_elasticsearch_flag(self): event1 = '[network where p0.process.name == process.name]' event2 = '[network where p0.pid == 0]' event3 = '[network where p0.badfield == 0]' - event4 = f'!{event0}' + event4 = '!{%s}' % (event0) parse_query('sequence %s as p0 %s' % (event0, event1)) parse_query('sequence by user.name %s as p0 %s' % (event0, event1)) parse_query('sequence with maxspan=1m %s by user.name as p0 %s by user.name' % (event0, event1)) From f17a00a2cf730600eed843aa374b63f166b44a05 Mon Sep 17 00:00:00 2001 From: Mika Ayenson Date: Mon, 2 Oct 2023 17:01:52 -0500 Subject: [PATCH 03/10] unit test --- tests/test_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_parser.py b/tests/test_parser.py index ae2b785..d00f745 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -715,7 +715,7 @@ def test_elasticsearch_flag(self): event1 = '[network where p0.process.name == process.name]' event2 = '[network where p0.pid == 0]' event3 = '[network where p0.badfield == 0]' - event4 = '!{%s}' % (event0) + event4 = '!%s' % (event0) parse_query('sequence %s as p0 %s' % (event0, event1)) parse_query('sequence by user.name %s as p0 %s' % (event0, event1)) parse_query('sequence with maxspan=1m %s by user.name as p0 %s by user.name' % (event0, event1)) From 1acc788f39fddcac836c4f5ef4043a0d27e585dd Mon Sep 17 00:00:00 2001 From: Mika Ayenson Date: Wed, 4 Oct 2023 07:37:18 -0500 Subject: [PATCH 04/10] bump the version --- eql/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eql/__init__.py b/eql/__init__.py index 24d682d..e5b867e 100644 --- a/eql/__init__.py +++ b/eql/__init__.py @@ -66,7 +66,7 @@ Walker, ) -__version__ = '0.9.18' +__version__ = '0.9.19' __all__ = ( "__version__", "AnalyticOutput", From c578b0f29f76b0ad56b05046630d71e54d139aab Mon Sep 17 00:00:00 2001 From: Mika Ayenson Date: Thu, 5 Oct 2023 15:34:28 -0500 Subject: [PATCH 05/10] update docstring --- eql/ast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eql/ast.py b/eql/ast.py index d54aaec..942cafe 100644 --- a/eql/ast.py +++ b/eql/ast.py @@ -897,7 +897,7 @@ def __init__(self, query, join_values=None, data=None): :param EventQuery query: The event query enclosed in the term :param list[Expression] join_values: The field to join values on - :param bool data: Toggle for copying instead of moving a sequence on match + :param dict data: Fork (copying instead of moving a sequence on match) and is_negated params """ self.query = query self.join_values = join_values or [] From 2ad68a93fac70dd3d37636f4a7baf96456932488 Mon Sep 17 00:00:00 2001 From: Mika Ayenson Date: Mon, 23 Oct 2023 17:01:40 -0500 Subject: [PATCH 06/10] add unit tests and python engine support --- eql/engine.py | 23 ++++++++++++++-- tests/test_python_engine.py | 55 ++++++++++++++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/eql/engine.py b/eql/engine.py index 8dd880d..0e32c12 100644 --- a/eql/engine.py +++ b/eql/engine.py @@ -983,11 +983,16 @@ def _convert_sequence_term(self, subquery, position, size, lookups, next_pipe=No get_join_value = self._convert_key(subquery.join_values, scoped=True) last_position = size - 1 fork = bool(subquery.params.kv.get('fork', Boolean(False)).value) + is_negated = bool(subquery.params.kv.get('is_negated', Boolean(False)).value) if position == 0: @self.event_callback(subquery.query.event_type) def start_sequence_callback(event): # type: (Event) -> None - if check_event(event): + condition = ( + (check_event(event) and not is_negated) or + (not check_event(event) and is_negated) + ) + if condition: join_value = get_join_value(event) sequence = [event] lookups[1][join_value] = sequence @@ -997,7 +1002,12 @@ def start_sequence_callback(event): # type: (Event) -> None @self.event_callback(subquery.query.event_type) def continue_sequence_callback(event): # type: (Event) -> None - if len(lookups[position]) and check_event(event): + condition = ( + len(lookups[position]) and + ((check_event(event) and not is_negated) or + (not check_event(event) and is_negated)) + ) + if condition: join_value = get_join_value(event) if join_value in lookups[position]: if fork: @@ -1010,13 +1020,20 @@ def continue_sequence_callback(event): # type: (Event) -> None else: @self.event_callback(subquery.query.event_type) def finish_sequence(event): # type: (Event) -> None - if len(lookups[position]) and check_event(event): + condition = ( + len(lookups[position]) and + ((check_event(event) and not is_negated) or + (not check_event(event) and is_negated)) + ) + if condition: join_value = get_join_value(event) if join_value in lookups[position]: if fork: sequence = list(lookups[position].get(join_value)) else: sequence = lookups[position].pop(join_value) + if is_negated: + pass sequence.append(event) next_pipe(sequence) diff --git a/tests/test_python_engine.py b/tests/test_python_engine.py index 424f76a..750eca9 100644 --- a/tests/test_python_engine.py +++ b/tests/test_python_engine.py @@ -6,7 +6,7 @@ from eql import * # noqa: F403 from eql.ast import * # noqa: F403 from eql.engine import Scope -from eql.parser import ignore_missing_functions, allow_sample, elasticsearch_syntax +from eql.parser import ignore_missing_functions, allow_sample, elasticsearch_syntax, allow_negation from eql.schema import EVENT_TYPE_GENERIC from eql.tests.base import TestEngine @@ -526,3 +526,56 @@ def evaluate(expr, event): self.assertEqual(evaluate("`a.b`", {"a.b": 1}), 1) self.assertEqual(evaluate("a.`b.c`[0]", {"a": {"b.c": [1]}}), 1) self.assertEqual(evaluate("`!@#$%^&*().`", {"!@#$%^&*().": 1}), 1) + + def test_missing_events(self): + """Test the missing event feature.""" + config = {'flatten': True} + events = [Event.from_data(d) for d in [ + { + "event_type": "process", + "process_name": "malicious.exe", + "unique_pid": "host1-1", + "@timestamp": "2023-10-23T00:00:00" + }, + { + "event_type": "process", + "process_name": "missing.exe", + "unique_pid": "host1-1", + "@timestamp": "2023-10-23T00:00:00" + }, + { + "event_type": "file", + "file_name": "suspicious.txt", + "unique_pid": "host1-1", + "@timestamp": "2023-10-23T00:01:00" + } + ]] + + # Should return no results since the malicious2.exe event is not missing + query = ''' + sequence by unique_pid with maxspan=1m + [ process where process_name == "malicious.exe" ] + ![ process where process_name == "missing.exe" ] + [ file where file_name == "suspicious.txt" ] + ''' + with elasticsearch_syntax, allow_negation: + parsed_query = parse_query(query) + + output = self.get_output(queries=[parsed_query], config=config, events=events) + + self.assertEqual(len(output), 0, "Missing or extra results") + + # Should return results since the second malicious.exe event is missing + query = ''' + sequence by unique_pid with maxspan=1m + [ process where process_name == "malicious.exe" ] + ![ process where process_name == "malicious.exe" ] + [ file where file_name == "suspicious.txt" ] + ''' + with elasticsearch_syntax, allow_negation: + parsed_query = parse_query(query) + + output = self.get_output(queries=[parsed_query], config=config, events=events) + + self.assertEqual(len(output), 3, "Missing or extra results") + From 930cd28a000e1c2828666ff37987742e4a242f7e Mon Sep 17 00:00:00 2001 From: Mika Ayenson Date: Mon, 23 Oct 2023 17:04:30 -0500 Subject: [PATCH 07/10] lint --- tests/test_python_engine.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_python_engine.py b/tests/test_python_engine.py index 750eca9..372e10f 100644 --- a/tests/test_python_engine.py +++ b/tests/test_python_engine.py @@ -578,4 +578,3 @@ def test_missing_events(self): output = self.get_output(queries=[parsed_query], config=config, events=events) self.assertEqual(len(output), 3, "Missing or extra results") - From 600c2abe6ebe7f66acd5a88ad667039f147546bb Mon Sep 17 00:00:00 2001 From: Mika Ayenson Date: Tue, 24 Oct 2023 18:17:54 -0500 Subject: [PATCH 08/10] add more unit test and refactor engine --- eql/engine.py | 52 ++++++++++++++--------------- tests/test_python_engine.py | 65 ++++++++++++++++++++++++++++++++++--- 2 files changed, 87 insertions(+), 30 deletions(-) diff --git a/eql/engine.py b/eql/engine.py index 0e32c12..c12040f 100644 --- a/eql/engine.py +++ b/eql/engine.py @@ -988,54 +988,52 @@ def _convert_sequence_term(self, subquery, position, size, lookups, next_pipe=No if position == 0: @self.event_callback(subquery.query.event_type) def start_sequence_callback(event): # type: (Event) -> None - condition = ( - (check_event(event) and not is_negated) or - (not check_event(event) and is_negated) - ) - if condition: + event_check = check_event(event) + if event_check or is_negated: join_value = get_join_value(event) - sequence = [event] - lookups[1][join_value] = sequence + if event_check and not is_negated: + sequence = [event] + lookups[1][join_value] = sequence + elif is_negated and not event_check: + sequence = [] + lookups[1][join_value] = sequence elif position < last_position: next_position = position + 1 @self.event_callback(subquery.query.event_type) def continue_sequence_callback(event): # type: (Event) -> None - condition = ( - len(lookups[position]) and - ((check_event(event) and not is_negated) or - (not check_event(event) and is_negated)) - ) - if condition: + event_check = check_event(event) + if len(lookups[position]) and (check_event(event) or is_negated): join_value = get_join_value(event) if join_value in lookups[position]: if fork: sequence = list(lookups[position].get(join_value)) else: sequence = lookups[position].pop(join_value) - sequence.append(event) - lookups[next_position][join_value] = sequence + + if is_negated and not event_check: + lookups[next_position][join_value] = sequence + elif not is_negated: + sequence.append(event) + lookups[next_position][join_value] = sequence else: @self.event_callback(subquery.query.event_type) def finish_sequence(event): # type: (Event) -> None - condition = ( - len(lookups[position]) and - ((check_event(event) and not is_negated) or - (not check_event(event) and is_negated)) - ) - if condition: + event_check = check_event(event) + if len(lookups[position]) and (check_event(event) or is_negated): join_value = get_join_value(event) if join_value in lookups[position]: if fork: sequence = list(lookups[position].get(join_value)) else: sequence = lookups[position].pop(join_value) - if is_negated: - pass - sequence.append(event) - next_pipe(sequence) + + if not is_negated or (is_negated and not event_check): + if not is_negated: + sequence.append(event) + next_pipe(sequence) def _convert_sequence(self, node, next_pipe): # type: (Sequence, callable) -> callable # Two lookups can help avoid unnecessary calls @@ -1051,8 +1049,10 @@ def check_timeout(event): # type: (Event) -> None minimum_start = event.time - max_span for sub_lookup in lookups: for join_key, sequence in list(sub_lookup.items()): - if sequence[0].time < minimum_start: + if sequence and sequence[0].time < minimum_start: sub_lookup.pop(join_key) + else: + pass if node.close: check_close_event = self.convert(node.close.query) diff --git a/tests/test_python_engine.py b/tests/test_python_engine.py index 372e10f..ca80646 100644 --- a/tests/test_python_engine.py +++ b/tests/test_python_engine.py @@ -535,19 +535,19 @@ def test_missing_events(self): "event_type": "process", "process_name": "malicious.exe", "unique_pid": "host1-1", - "@timestamp": "2023-10-23T00:00:00" + "@timestamp": 116444736000000000 }, { "event_type": "process", "process_name": "missing.exe", "unique_pid": "host1-1", - "@timestamp": "2023-10-23T00:00:00" + "@timestamp": 116444738000000000 }, { "event_type": "file", "file_name": "suspicious.txt", "unique_pid": "host1-1", - "@timestamp": "2023-10-23T00:01:00" + "@timestamp": 116444740000000000 } ]] @@ -572,9 +572,66 @@ def test_missing_events(self): ![ process where process_name == "malicious.exe" ] [ file where file_name == "suspicious.txt" ] ''' + + with elasticsearch_syntax, allow_negation: + parsed_query = parse_query(query) + + output = self.get_output(queries=[parsed_query], config=config, events=events) + + self.assertEqual(len(output), 2, "Missing or extra results") + + # Should return results since the last suspicious_file.txt event is missing + query = ''' + sequence by unique_pid with maxspan=1m + [ process where process_name == "malicious.exe" ] + [ process where process_name == "missing.exe" ] + ![ file where file_name == "suspicious_file.txt" ] + ''' + with elasticsearch_syntax, allow_negation: + parsed_query = parse_query(query) + + output = self.get_output(queries=[parsed_query], config=config, events=events) + + self.assertEqual(len(output), 2, "Missing or extra results") + + # Should return no results since the last suspicious.txt event is not missing + query = ''' + sequence by unique_pid with maxspan=1m + [ process where process_name == "malicious.exe" ] + [ process where process_name == "missing.exe" ] + ![ file where file_name == "suspicious.txt" ] + ''' + with elasticsearch_syntax, allow_negation: + parsed_query = parse_query(query) + + output = self.get_output(queries=[parsed_query], config=config, events=events) + + self.assertEqual(len(output), 0, "Missing or extra results") + + # Should return no results since the first malicious.exe event is not missing + query = ''' + sequence by unique_pid with maxspan=1m + ![ process where process_name == "malicious.exe" ] + [ process where process_name == "missing.exe" ] + [ file where file_name == "suspicious.txt" ] + ''' + with elasticsearch_syntax, allow_negation: + parsed_query = parse_query(query) + + output = self.get_output(queries=[parsed_query], config=config, events=events) + + self.assertEqual(len(output), 0, "Missing or extra results") + + # Should return results since the first malicious2.exe event is missing + query = ''' + sequence by unique_pid with maxspan=1m + ![ process where process_name == "malicious2.exe" ] + [ process where process_name == "missing.exe" ] + [ file where file_name == "suspicious.txt" ] + ''' with elasticsearch_syntax, allow_negation: parsed_query = parse_query(query) output = self.get_output(queries=[parsed_query], config=config, events=events) - self.assertEqual(len(output), 3, "Missing or extra results") + self.assertEqual(len(output), 2, "Missing or extra results") From 187e43c85c722d2d76fe3dd0ea6b4c725c419604 Mon Sep 17 00:00:00 2001 From: Mika Ayenson Date: Tue, 24 Oct 2023 18:34:07 -0500 Subject: [PATCH 09/10] properly set test data timestamp and query maxspan --- tests/test_python_engine.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_python_engine.py b/tests/test_python_engine.py index ca80646..4710088 100644 --- a/tests/test_python_engine.py +++ b/tests/test_python_engine.py @@ -535,25 +535,25 @@ def test_missing_events(self): "event_type": "process", "process_name": "malicious.exe", "unique_pid": "host1-1", - "@timestamp": 116444736000000000 + "timestamp": 116444736000000000 }, { "event_type": "process", "process_name": "missing.exe", "unique_pid": "host1-1", - "@timestamp": 116444738000000000 + "timestamp": 116444738000000000 }, { "event_type": "file", "file_name": "suspicious.txt", "unique_pid": "host1-1", - "@timestamp": 116444740000000000 + "timestamp": 116444740000000000 } ]] # Should return no results since the malicious2.exe event is not missing query = ''' - sequence by unique_pid with maxspan=1m + sequence by unique_pid with maxspan=7m [ process where process_name == "malicious.exe" ] ![ process where process_name == "missing.exe" ] [ file where file_name == "suspicious.txt" ] @@ -567,7 +567,7 @@ def test_missing_events(self): # Should return results since the second malicious.exe event is missing query = ''' - sequence by unique_pid with maxspan=1m + sequence by unique_pid with maxspan=7m [ process where process_name == "malicious.exe" ] ![ process where process_name == "malicious.exe" ] [ file where file_name == "suspicious.txt" ] @@ -582,7 +582,7 @@ def test_missing_events(self): # Should return results since the last suspicious_file.txt event is missing query = ''' - sequence by unique_pid with maxspan=1m + sequence by unique_pid with maxspan=7m [ process where process_name == "malicious.exe" ] [ process where process_name == "missing.exe" ] ![ file where file_name == "suspicious_file.txt" ] @@ -596,7 +596,7 @@ def test_missing_events(self): # Should return no results since the last suspicious.txt event is not missing query = ''' - sequence by unique_pid with maxspan=1m + sequence by unique_pid with maxspan=7m [ process where process_name == "malicious.exe" ] [ process where process_name == "missing.exe" ] ![ file where file_name == "suspicious.txt" ] @@ -610,7 +610,7 @@ def test_missing_events(self): # Should return no results since the first malicious.exe event is not missing query = ''' - sequence by unique_pid with maxspan=1m + sequence by unique_pid with maxspan=7m ![ process where process_name == "malicious.exe" ] [ process where process_name == "missing.exe" ] [ file where file_name == "suspicious.txt" ] @@ -624,7 +624,7 @@ def test_missing_events(self): # Should return results since the first malicious2.exe event is missing query = ''' - sequence by unique_pid with maxspan=1m + sequence by unique_pid with maxspan=7m ![ process where process_name == "malicious2.exe" ] [ process where process_name == "missing.exe" ] [ file where file_name == "suspicious.txt" ] From 9f00aca10e4f6291227ede2ba1f09d2143d8c82e Mon Sep 17 00:00:00 2001 From: Mika Ayenson Date: Tue, 31 Oct 2023 10:51:07 -0500 Subject: [PATCH 10/10] small cleanup --- eql/engine.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/eql/engine.py b/eql/engine.py index c12040f..14e828d 100644 --- a/eql/engine.py +++ b/eql/engine.py @@ -1051,8 +1051,6 @@ def check_timeout(event): # type: (Event) -> None for join_key, sequence in list(sub_lookup.items()): if sequence and sequence[0].time < minimum_start: sub_lookup.pop(join_key) - else: - pass if node.close: check_close_event = self.convert(node.close.query)