Skip to content

Commit

Permalink
fix: CEL filtering expression
Browse files Browse the repository at this point in the history 
Trying multiple things here:
- bump CEL version
- add test in `MessageFiltersTest` on different fields of different types
- add `CelValidationTest' and 'CelTypedValidationTest' to try to understand how CEL works
  • Loading branch information
@sixdouglas
sixdouglas committed Nov 11, 2024
1 parent 0ad8695 commit fee8b9e
Show file tree
Hide file tree
Showing 5 changed files with 222 additions and 8 deletions.
6 changes: 0 additions & 6 deletions api/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,12 +215,6 @@
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>

<dependency>
<groupId>org.antlr</groupId>
<artifactId>antlr4-runtime</artifactId>
<version>${antlr4-maven-plugin.version}</version>
</dependency>

<dependency>
<groupId>org.opendatadiscovery</groupId>
<artifactId>oddrn-generator-java</artifactId>
Expand Down
104 changes: 104 additions & 0 deletions api/src/test/java/io/kafbat/ui/emitter/CelTypedValidationTest.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
package io.kafbat.ui.emitter;

import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableSet;
import dev.cel.common.CelAbstractSyntaxTree;
import dev.cel.common.CelOptions;
import dev.cel.common.CelValidationException;
import dev.cel.common.CelValidationResult;
import dev.cel.common.types.CelType;
import dev.cel.common.types.CelTypeProvider;
import dev.cel.common.types.SimpleType;
import dev.cel.common.types.StructType;
import dev.cel.compiler.CelCompiler;
import dev.cel.compiler.CelCompilerFactory;
import dev.cel.extensions.CelExtensions;
import dev.cel.parser.CelStandardMacro;
import dev.cel.runtime.CelEvaluationException;
import dev.cel.runtime.CelRuntime;
import dev.cel.runtime.CelRuntimeFactory;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;

import static org.assertj.core.api.Assertions.assertThat;

class CelTypedValidationTest {

private final StructType recordContentType = StructType.create(
"MessageContentDTO",
ImmutableSet.of("lastname", "firstname", "age"),
fieldName -> switch (fieldName) {
case "lastname", "firstname" ->
Optional.of(SimpleType.STRING);
case "age" ->
Optional.of(SimpleType.INT);
default ->
Optional.empty();
}
);

private final StructType recordType = StructType.create(
"MessageDTO",
ImmutableSet.of("value"),
fieldName -> Optional.of(recordContentType)
);

private final CelCompiler celCompiler = CelCompilerFactory.standardCelCompilerBuilder()
.setOptions(CelOptions.DEFAULT)
.setStandardMacros(CelStandardMacro.STANDARD_MACROS)
.addLibraries(CelExtensions.strings(), CelExtensions.encoders())
.addVar("record", recordType)
.setResultType(SimpleType.BOOL)
.setTypeProvider(new CelTypeProvider() {
@Override
public ImmutableCollection<CelType> types() {
return ImmutableSet.of(recordType);
}

@Override
public Optional<CelType> findType(String typeName) {
if ("MessageDTO".equals(typeName)) {
return Optional.of(recordType);
}
if ("MessageContentDTO".equals(typeName)) {
return Optional.of(recordContentType);
}
return Optional.empty();
}
})
.build();

private final CelRuntime celRuntime = CelRuntimeFactory.standardCelRuntimeBuilder().build();

@ParameterizedTest
@CsvSource({
"record.value.age == 24, true",
"record.value.age >= 24, true",
"record.value.age > 24, true",
"has(record.value.age), true",
"has(record.value.ages), false",
"record.value.age < 50, true",
"record.value.age < 24, true"
})
void typedExpressionTest(String expression, boolean expected) throws CelValidationException, CelEvaluationException {
Map<String, Integer> objectKeys = new HashMap<>();
objectKeys.put("age", 24);
Map<String, Object> valueKeys = new HashMap<>();
valueKeys.put("value", objectKeys);
Map<String, Map<String, Object>> recordKeys = new HashMap<>();
recordKeys.put("record", valueKeys);

CelValidationResult celValidationResult = celCompiler.compile(expression);
CelAbstractSyntaxTree ast = celValidationResult.getAst();
CelRuntime.Program program = celRuntime.createProgram(ast);
var programResult = program.eval(recordKeys);
assertThat(programResult)
.describedAs("The result of the assertion was incorrect")
.isNotNull()
.isEqualTo(expected)
;
}
}
98 changes: 98 additions & 0 deletions api/src/test/java/io/kafbat/ui/emitter/CelValidationTest.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
package io.kafbat.ui.emitter;

import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableSet;
import dev.cel.common.CelAbstractSyntaxTree;
import dev.cel.common.CelOptions;
import dev.cel.common.CelValidationException;
import dev.cel.common.CelValidationResult;
import dev.cel.common.types.CelType;
import dev.cel.common.types.CelTypeProvider;
import dev.cel.common.types.SimpleType;
import dev.cel.common.types.StructType;
import dev.cel.compiler.CelCompiler;
import dev.cel.compiler.CelCompilerFactory;
import dev.cel.extensions.CelExtensions;
import dev.cel.parser.CelStandardMacro;
import dev.cel.runtime.CelEvaluationException;
import dev.cel.runtime.CelRuntime;
import dev.cel.runtime.CelRuntimeFactory;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;

import static org.assertj.core.api.Assertions.assertThat;

class CelValidationTest {
private final Map<String, CelType> fields = Map.of(
"value", SimpleType.DYN
);

private final ImmutableSet<String> names = ImmutableSet
.<String>builder()
.addAll(fields.keySet())
.build();

private final StructType recordType = StructType.create(
"MessageDTO",
names,
fieldName -> Optional.ofNullable(fields.get(fieldName))
);

private final CelCompiler celCompiler = CelCompilerFactory.standardCelCompilerBuilder()
.setOptions(CelOptions.DEFAULT)
.setStandardMacros(CelStandardMacro.STANDARD_MACROS)
.addLibraries(CelExtensions.strings(), CelExtensions.encoders())
.addVar("record", recordType)
.setResultType(SimpleType.BOOL)
.setTypeProvider(new CelTypeProvider() {
@Override
public ImmutableCollection<CelType> types() {
return ImmutableSet.of(recordType);
}

@Override
public Optional<CelType> findType(String typeName) {
return "MessageDTO".equals(typeName) ? Optional.of(recordType) : Optional.empty();
}
})
.build();

private final CelRuntime celRuntime = CelRuntimeFactory.standardCelRuntimeBuilder().build();

@ParameterizedTest
@CsvSource({
"record.value.lastname == null, true",
"record.value.firstname == 'Paul', true",
"size(record.value.firstname) == 4, true",
"record.value.age == 24, true",
"record.value.age >= 24, true",
"record.value.age > 24, true",
"has(record.value.age), true",
"has(record.value.ages), false",
"record.value.age < 50, true",
"record.value.age < 24, true"
})
void expressionTest(String expression, boolean expected) throws CelValidationException, CelEvaluationException {
Map<String, Object> objectKeys = new HashMap<>();
objectKeys.put("lastname", null);
objectKeys.put("firstname", "Paul");
objectKeys.put("age", 24);
Map<String, Object> valueKeys = new HashMap<>();
valueKeys.put("value", objectKeys);
Map<String, Map<String, Object>> recordKeys = new HashMap<>();
recordKeys.put("record", valueKeys);

CelValidationResult celValidationResult = celCompiler.compile(expression);
CelAbstractSyntaxTree ast = celValidationResult.getAst();
CelRuntime.Program program = celRuntime.createProgram(ast);
var programResult = program.eval(recordKeys);
assertThat(programResult)
.describedAs("The result of the assertion was incorrect")
.isNotNull()
.isEqualTo(expected)
;
}
}
18 changes: 18 additions & 0 deletions api/src/test/java/io/kafbat/ui/emitter/MessageFiltersTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,24 @@ void canCheckValueAsJsonObjectIfItCanBeParsedToJson() {
assertFalse(f.test(msg().content("{ \"name\" : { \"second\" : \"user2\" } }")));
}

@Test
void canFilterNullValueInJsonObjectIfItCanBeParsedToJson() {
var filter1 = celScriptFilter("record.value.age == 24");
assertTrue(filter1.test(msg().content("{\"name\": null, \"age\": 24, \"address\": { \"city\": \"Lille\"}}")), "Age filter KO");
var filter2 = celScriptFilter("record.value.name == 'Paul'");
assertTrue(filter2.test(msg().content("{\"name\": \"Paul\", \"age\": 24, \"address\": { \"city\": \"Lille\"}}")), "Name filter KO");
var filter3 = celScriptFilter("record.value.name == ''");
assertTrue(filter3.test(msg().content("{\"name\": \"\", \"age\": 24, \"address\": { \"city\": \"Lille\"}}")), "Empty Name filter KO");
var filter4 = celScriptFilter("record.value.name == null");
assertFalse(filter4.test(msg().content("{\"name\": \"Paul\", \"age\": 24, \"address\": { \"city\": \"Lille\"}}")), "Null Name in filter KO");
var filter5 = celScriptFilter("record.value.address.city == \"Lille\"");
assertTrue(filter5.test(msg().content("{\"name\": null, \"age\": 24, \"address\": { \"city\": \"Lille\"}}")), "Null Name filter KO");
var filter6 = celScriptFilter("record.value.address.city == \"\"");
assertTrue(filter6.test(msg().content("{\"name\": null, \"age\": 24, \"address\": { \"city\": \"\"}}")), "Null Name filter KO");
var filter7 = celScriptFilter("record.value.address.city == null");
assertTrue(filter7.test(msg().content("{\"name\": null, \"age\": 24, \"address\": { \"city\": null}}")), "Null Name filter KO");
}

@Test
void valueSetToContentStringIfCantBeParsedToJson() {
var f = celScriptFilter("record.value == \"not json\"");
Expand Down
4 changes: 2 additions & 2 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
<git.revision>latest</git.revision>

<!-- Dependency versions -->
<antlr4-maven-plugin.version>4.12.0</antlr4-maven-plugin.version>
<antlr4-maven-plugin.version>4.13.2</antlr4-maven-plugin.version>
<apache.commons.version>2.12.0</apache.commons.version>
<assertj.version>3.25.3</assertj.version>
<avro.version>1.11.4</avro.version>
Expand All @@ -50,7 +50,7 @@
<odd-oddrn-generator.version>0.1.17</odd-oddrn-generator.version>
<odd-oddrn-client.version>0.1.39</odd-oddrn-client.version>
<org.json.version>20240303</org.json.version>
<dev.cel.version>0.3.0</dev.cel.version>
<dev.cel.version>0.7.0</dev.cel.version>
<guava.version>33.3.1-jre</guava.version>
<!-- Test dependency versions -->
<junit.version>5.11.2</junit.version>
Expand Down

0 comments on commit fee8b9e

Please sign in to comment.