fix: exclude coherent source (#35)

* test(venom): exclude coherent source * fix: exclude coherent source * docs: update changelog
CGI-FR · Sep 28, 2023 · 248ec92 · 248ec92
1 parent c47ca37
commit 248ec92
Show file tree

Hide file tree

Showing 7 changed files with 94 additions and 5 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -17,6 +17,7 @@ Types of changes
 ## [0.6.0]
 
 - `Added` log unmasked values with the `watch` flag.
+- `Fixed` exclusion with coherent source specified.
 
 ## [0.5.0]
 

diff --git a/pkg/mimo/model.go b/pkg/mimo/model.go
@@ -94,10 +94,17 @@ func (m *Metrics) Update(
 
 	m.backend.IncreaseTotalCount()
 
-	m.Coherence.Add(toStringSlice(coherenceValue), maskedValueStr)
-	m.Identifiant.Add(maskedValueStr, realValueStr)
+	excluded := isExcluded(config.Exclude, realValue, realValueStr)
+
+	if !excluded {
+		// coherence and identifiant rates are computed over all values by default (including nil values)
+		// that's the reason why this code block is located here
+		m.Coherence.Add(toStringSlice(coherenceValue), maskedValueStr)
+		m.Identifiant.Add(maskedValueStr, realValueStr)
+	}
 
 	log.Trace().
+		Str("name", fieldname).
 		Str("masked", maskedValueStr).
 		Str("real", realValueStr).
 		Str("coherence", toStringSlice(coherenceValue)).
@@ -109,7 +116,7 @@ func (m *Metrics) Update(
 		return true
 	}
 
-	if isExcluded(config.Exclude, realValue, realValueStr) {
+	if excluded {
 		m.backend.IncreaseIgnoredCount()
 
 		return true

diff --git a/test/configs/config_exclude_coherent_source.yaml b/test/configs/config_exclude_coherent_source.yaml
@@ -0,0 +1,10 @@
+version: "1"
+metrics:
+  - name: "value"
+    exclude: [""]
+    coherentWith: ["source"]
+    constraints:
+      maskingRate:
+        shouldEqual: 1
+      coherentRate:
+        shouldEqual: 1
diff --git a/test/reports/report_2.html b/test/reports/report_2.html
@@ -29,7 +29,7 @@ <h1>MIMO Report</h1>
         <td>8</td>
         <td>0</td>
         <td style="background-color: lightgreen">100.00 %</td>
-        <td style="background-color: orange">60.00 %</td>
+        <td style="background-color: orange">50.00 %</td>
         <td style="background-color: orange">80.00 %</td>
         <td style="background-color: orange">1</td>
       </tr>

diff --git a/test/reports/report_bugfix_exclude_coherent_source.html b/test/reports/report_bugfix_exclude_coherent_source.html
@@ -0,0 +1,52 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8" />
+  <title>MIMO Report</title>
+  <meta name="viewport" content="width=device-width,initial-scale=1" />
+  <meta name="description" content="MIMO Report" />
+</head>
+<body>
+  <h1>MIMO Report</h1>
+  <table border="1" cellspacing="0" cellpadding="5">
+    <thead>
+      <th>Field</th>
+      <th>Nil</th>
+      <th>Ignored</th>
+      <th>Masked</th>
+      <th>Missed</th>
+      <th>Masking Rate</th>
+      <th>Coherent Rate</th>
+      <th>Identifiable Rate</th>
+      <th>K</th>
+    </thead>
+    <tbody>
+
+      <tr>
+        <td>source</td>
+        <td>0</td>
+        <td>0</td>
+        <td>0</td>
+        <td>4</td>
+        <td style="background-color: orange">0.00 %</td>
+        <td style="background-color: lightgreen">100.00 %</td>
+        <td style="background-color: lightgreen">100.00 %</td>
+        <td style="background-color: orange">1</td>
+      </tr>
+
+      <tr>
+        <td>value</td>
+        <td>0</td>
+        <td>1</td>
+        <td>3</td>
+        <td>0</td>
+        <td style="background-color: green">100.00 %</td>
+        <td style="background-color: green">100.00 %</td>
+        <td style="background-color: orange">50.00 %</td>
+        <td style="background-color: orange">1</td>
+      </tr>
+
+    </tbody>
+  </table>
+</body>
+</html>
diff --git a/test/suites/02-validate-metrics.yml b/test/suites/02-validate-metrics.yml
@@ -66,7 +66,7 @@ testcases:
       - script: cat working/masked.jsonl | mimo --config ../configs/config_exclude.yaml -v3 working/real.jsonl
         assertions:
           - result.code ShouldEqual 0
-          - result.systemerr ShouldContainSubstring value count-ignored=2 count-masked=8 count-missed=0 count-nil=0 field=value rate-coherence=0.6 rate-identifiable=0.8 rate-masking=1
+          - result.systemerr ShouldContainSubstring value count-ignored=2 count-masked=8 count-missed=0 count-nil=0 field=value rate-coherence=0.5 rate-identifiable=0.8 rate-masking=1
 
       - script: mv report.html ../reports/report_2.html
 

diff --git a/test/suites/99-bugfixes.yml b/test/suites/99-bugfixes.yml
@@ -38,3 +38,22 @@ testcases:
           - result.systemerr ShouldContainSubstring value count-ignored=1 count-masked=3 count-missed=0 count-nil=0 field=value rate-coherence=1 rate-identifiable=1 rate-masking=1
 
       - script: mv report.html ../reports/report_bugfix_exclude_numeric.html
+
+  - name: exclude value with different coherence source
+    steps:
+      - script: echo '{"value":"1","source":"A"}' > working/real.jsonl
+      - script: echo '{"value":"2","source":"A"}' >> working/real.jsonl
+      - script: echo '{"value":"","source":"A"}' >> working/real.jsonl
+      - script: echo '{"value":"3","source":"B"}' >> working/real.jsonl
+
+      - script: echo '{"value":"X","source":"A"}' > working/masked.jsonl
+      - script: echo '{"value":"X","source":"A"}' >> working/masked.jsonl
+      - script: echo '{"value":"","source":"A"}' >> working/masked.jsonl
+      - script: echo '{"value":"Y","source":"B"}' >> working/masked.jsonl
+
+      - script: cat working/masked.jsonl | mimo --config ../configs/config_exclude_coherent_source.yaml -v3 working/real.jsonl
+        assertions:
+          - result.code ShouldEqual 0
+          - result.systemerr ShouldContainSubstring value count-ignored=1 count-masked=3 count-missed=0 count-nil=0 field=value rate-coherence=1 rate-identifiable=0.5 rate-masking=1
+
+      - script: mv report.html ../reports/report_bugfix_exclude_coherent_source.html