[FR] Support missing events (#3153)

(cherry picked from commit d0b0216)

detection_rules/schemas/definitions.py

@@ -28,6 +28,7 @@
 MINOR_SEMVER = r'^\d+\.\d+$'
 BRANCH_PATTERN = f'{VERSION_PATTERN}|^master$'
 ELASTICSEARCH_EQL_FEATURES = {
+    "allow_negation": (Version.parse('8.9.0'), None),
     "allow_runs": (Version.parse('7.16.0'), None),
     "allow_sample": (Version.parse('8.6.0'), None),
     "elasticsearch_validate_optional_fields": (Version.parse('7.16.0'), None)

kql/evaluator.py

@@ -10,7 +10,7 @@
 from eql import Walker, EqlCompileError, utils
 from eql.functions import CidrMatch
 from .errors import KqlRuntimeError, KqlCompileError
-
+from .parser import is_ipaddress
 
 class FilterGenerator(Walker):
     __cidr_cache = {}
@@ -20,8 +20,9 @@ def _walk_default(self, node, *args, **kwargs):
 
     @classmethod
     def equals(cls, term, value):
+        """Check if a term is equal to a value."""
         if utils.is_string(term) and utils.is_string(value):
-            if CidrMatch.ip_compiled.match(term) and CidrMatch.cidr_compiled.match(value):
+            if is_ipaddress(term) and eql.utils.is_cidr_pattern(value):
                 # check for an ipv4 cidr
                 if value not in cls.__cidr_cache:
                     cls.__cidr_cache[value] = CidrMatch.get_callback(None, eql.ast.String(value))

kql/parser.py

@@ -40,6 +40,15 @@ def child_tokens(self):
 lark_parser = Lark(grammar, propagate_positions=True, tree_class=KvTree, start=['query'], parser='lalr')
 
 
+def is_ipaddress(value: str) -> bool:
+    """Check if a value is an ip address."""
+    try:
+        eql.utils.get_ipaddress(value)
+        return True
+    except ValueError:
+        return False
+
+
 def wildcard2regex(wc: str) -> re.Pattern:
     parts = wc.split("*")
     return re.compile("^{regex}$".format(regex=".*?".join(re.escape(w) for w in parts)))
@@ -85,8 +94,6 @@ def elasticsearch_type_family(mapping_type: str) -> str:
 
 class BaseKqlParser(Interpreter):
     NON_SPACE_WS = re.compile(r"[^\S ]+")
-    ip_regex = re.compile("^" + eql.functions.CidrMatch.ip_re + "(/([0-2]?[0-9]|3[0-2]))?$")
-
     unquoted_escapes = {"\\t": "\t", "\\r": "\r", "\\n": "\n"}
 
     for special in "\\():<>\"*{}]":
@@ -223,7 +230,7 @@ def convert_value(self, field_name, python_value, value_tree):
                 except ValueError:
                     pass
             elif field_type_family == "ip" and value_type == "keyword":
-                if "::" in python_value or self.ip_regex.match(python_value) is not None:
+                if "::" in python_value or is_ipaddress(python_value) or eql.utils.is_cidr_pattern(python_value):
                     return python_value
             elif field_type_family == 'date' and value_type in STRING_FIELDS:
                 # this will not validate datemath syntax

pyproject.toml

@@ -24,7 +24,7 @@ classifiers = [
 dependencies = [
   	"Click~=8.1.0",
 	"elasticsearch~=8.1",
-	"eql==0.9.18",
+	"eql==0.9.19",
 	"jsl==0.2.4",
 	"jsonschema==3.2.0",
 	"marko==2.0.1",