From 889c93c82be9e9b115b4be0628c784468cf2d824 Mon Sep 17 00:00:00 2001 From: Adrien Aury <44274230+adrienaury@users.noreply.github.com> Date: Sat, 2 Sep 2023 16:25:55 +0200 Subject: [PATCH] fix: rename empty to ignored (#23) --- CHANGELOG.md | 1 + cmd/mimo/main.go | 4 +- internal/infra/template/default.html | 4 +- pkg/mimo/model.go | 44 +++++++++---------- test/reports/report_1.html | 2 +- test/reports/report_2.html | 2 +- test/reports/report_3.html | 2 +- test/reports/report_4.html | 2 +- test/reports/report_5.html | 2 +- test/reports/report_6.html | 2 +- test/reports/report_7.html | 2 +- test/reports/report_8.html | 2 +- test/reports/report_bugfix_1.html | 2 +- .../report_bugfix_exclude_numeric.html | 2 +- test/reports/report_deep_array.html | 2 +- test/reports/report_deep_mixed.html | 2 +- test/reports/report_deep_object.html | 2 +- test/reports/report_preprocess_simple.html | 2 +- test/reports/report_reuse_previous.html | 2 +- test/reports/report_reuse_previous_alias.html | 2 +- .../reports/report_template_source_stack.html | 2 +- test/suites/02-validate-metrics.yml | 12 ++--- test/suites/03-persist-storage.yml | 6 +-- test/suites/04-dig-objects.yml | 12 ++--- test/suites/05-template-source.yml | 4 +- test/suites/06-column-alias.yml | 2 +- test/suites/07-preprocess.yml | 14 +++--- test/suites/99-bugfixes.yml | 4 +- 28 files changed, 71 insertions(+), 70 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d008cc9..b77d81c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ Types of changes ## [0.4.0] - `Added` possibility to create temporary fields with the `preprocess` configuration. +- `Fixed` rename label `empty` to `ignored` in report. ## [0.3.0] diff --git a/cmd/mimo/main.go b/cmd/mimo/main.go index 37649d9..8f0f7d3 100644 --- a/cmd/mimo/main.go +++ b/cmd/mimo/main.go @@ -184,7 +184,7 @@ func appendColumnMetric(report mimo.Report, colname string, haserror bool) bool log.Info(). Str("field", colname). Int64("count-nil", metrics.NilCount). - Int64("count-empty", metrics.EmptyCount). + Int64("count-ignored", metrics.IgnoredCount). Int64("count-masked", metrics.MaskedCount). Int64("count-missed", metrics.NonMaskedCount()). Float64("rate-masking", metrics.MaskedRate()). @@ -195,7 +195,7 @@ func appendColumnMetric(report mimo.Report, colname string, haserror bool) bool log.Error(). Str("field", colname). Int64("count-nil", metrics.NilCount). - Int64("count-empty", metrics.EmptyCount). + Int64("count-ignored", metrics.IgnoredCount). Int64("count-masked", metrics.MaskedCount). Int64("count-missed", metrics.NonMaskedCount()). Float64("rate-masking", metrics.MaskedRate()). diff --git a/internal/infra/template/default.html b/internal/infra/template/default.html index 4cfa7bb..942bd53 100644 --- a/internal/infra/template/default.html +++ b/internal/infra/template/default.html @@ -12,7 +12,7 @@

MIMO Report

Field Nil - Empty + Ignored Masked Missed Masking Rate @@ -25,7 +25,7 @@

MIMO Report

{{ $key }} {{ $value.NilCount }} - {{ $value.EmptyCount }} + {{ $value.IgnoredCount }} {{ $value.MaskedCount }} {{ $value.NonMaskedCount }} {{ if ne (toString $value.MaskedRate) "NaN" }}{{ $value.MaskedRate | mulf 100.00 | printf "%0.02f" }} %{{ else }}NaN{{ end }} diff --git a/pkg/mimo/model.go b/pkg/mimo/model.go index 2187d27..36f91c8 100644 --- a/pkg/mimo/model.go +++ b/pkg/mimo/model.go @@ -45,28 +45,28 @@ func (subs Suscribers) PostFirstNonMaskedValue(fieldname string, value any) { } type Metrics struct { - Fieldname string // Fieldname is name of column analyzed - TotalCount int64 // TotalCount is the number of values analyzed - NilCount int64 // NilCount is the number of null values in real data - EmptyCount int64 // EmptyCount is the number of empty values in real data (empty string or numbers at 0 value) - MaskedCount int64 // MaskedCount is the number of non-blank real values masked - Coherence Multimap // Coherence is a multimap used to compute the coherence rate - Identifiant Multimap // Identifiant is a multimap used to compute the identifiable rate - Constraints []Constraint // Constraints is the set of rules to validate + Fieldname string // Fieldname is name of column analyzed + TotalCount int64 // TotalCount is the number of values analyzed + NilCount int64 // NilCount is the number of null values in real data + IgnoredCount int64 // IgnoredCount is the number of ignored values in real data + MaskedCount int64 // MaskedCount is the number of non-blank real values masked + Coherence Multimap // Coherence is a multimap used to compute the coherence rate + Identifiant Multimap // Identifiant is a multimap used to compute the identifiable rate + Constraints []Constraint // Constraints is the set of rules to validate } type MultimapFactory func(fieldname string) Multimap func NewMetrics(fieldname string, multimapFactory MultimapFactory, constraints ...Constraint) Metrics { return Metrics{ - Fieldname: fieldname, - TotalCount: 0, - NilCount: 0, - EmptyCount: 0, - MaskedCount: 0, - Coherence: multimapFactory(fieldname + "-coherence"), - Identifiant: multimapFactory(fieldname + "-identifiant"), - Constraints: constraints, + Fieldname: fieldname, + TotalCount: 0, + NilCount: 0, + IgnoredCount: 0, + MaskedCount: 0, + Coherence: multimapFactory(fieldname + "-coherence"), + Identifiant: multimapFactory(fieldname + "-identifiant"), + Constraints: constraints, } } @@ -105,7 +105,7 @@ func (m *Metrics) Update( } if isExcluded(config.Exclude, realValue, realValueStr) { - m.EmptyCount++ + m.IgnoredCount++ return true } @@ -121,17 +121,17 @@ func (m *Metrics) Update( return true } -// BlankCount is the number of blank (null or empty) values in real data. +// BlankCount is the number of blank (null or ignored) values in real data. func (m Metrics) BlankCount() int64 { - return m.NilCount + m.EmptyCount + return m.NilCount + m.IgnoredCount } -// NonBlankCount is the number of non-blank (non-null and non-empty) values in real data. +// NonBlankCount is the number of non-blank (non-null and non-ignored) values in real data. func (m Metrics) NonBlankCount() int64 { return m.TotalCount - m.BlankCount() } -// NonMaskedCount is the number of non-blank (non-null and non-empty) values in real data that were not masked. +// NonMaskedCount is the number of non-blank (non-null and non-ignored) values in real data that were not masked. func (m Metrics) NonMaskedCount() int64 { return m.NonBlankCount() - m.MaskedCount } @@ -144,7 +144,7 @@ func (m Metrics) K() int { // MaskedRate is equal to // // Number of non-blank real values masked -// / (Number of values analyzed - Number of blank (null or empty) values in real data) ). +// / (Number of values analyzed - Number of blank (null or ignored) values in real data) ). func (m Metrics) MaskedRate() float64 { return float64(m.MaskedCount) / float64(m.NonBlankCount()) } diff --git a/test/reports/report_1.html b/test/reports/report_1.html index b9df8e0..a3e35b4 100644 --- a/test/reports/report_1.html +++ b/test/reports/report_1.html @@ -12,7 +12,7 @@

MIMO Report

Field Nil - Empty + Ignored Masked Missed Masking Rate diff --git a/test/reports/report_2.html b/test/reports/report_2.html index 6c780c0..d4bc657 100644 --- a/test/reports/report_2.html +++ b/test/reports/report_2.html @@ -12,7 +12,7 @@

MIMO Report

Field Nil - Empty + Ignored Masked Missed Masking Rate diff --git a/test/reports/report_3.html b/test/reports/report_3.html index ce0fad8..1f1bf50 100644 --- a/test/reports/report_3.html +++ b/test/reports/report_3.html @@ -12,7 +12,7 @@

MIMO Report

Field Nil - Empty + Ignored Masked Missed Masking Rate diff --git a/test/reports/report_4.html b/test/reports/report_4.html index 8a7130b..555c3cc 100644 --- a/test/reports/report_4.html +++ b/test/reports/report_4.html @@ -12,7 +12,7 @@

MIMO Report

Field Nil - Empty + Ignored Masked Missed Masking Rate diff --git a/test/reports/report_5.html b/test/reports/report_5.html index 4f0df7f..ac4cf2c 100644 --- a/test/reports/report_5.html +++ b/test/reports/report_5.html @@ -12,7 +12,7 @@

MIMO Report

Field Nil - Empty + Ignored Masked Missed Masking Rate diff --git a/test/reports/report_6.html b/test/reports/report_6.html index 9ec069b..48d0d56 100644 --- a/test/reports/report_6.html +++ b/test/reports/report_6.html @@ -12,7 +12,7 @@

MIMO Report

Field Nil - Empty + Ignored Masked Missed Masking Rate diff --git a/test/reports/report_7.html b/test/reports/report_7.html index 606e42b..b0789d1 100644 --- a/test/reports/report_7.html +++ b/test/reports/report_7.html @@ -12,7 +12,7 @@

MIMO Report

Field Nil - Empty + Ignored Masked Missed Masking Rate diff --git a/test/reports/report_8.html b/test/reports/report_8.html index b9df8e0..a3e35b4 100644 --- a/test/reports/report_8.html +++ b/test/reports/report_8.html @@ -12,7 +12,7 @@

MIMO Report

Field Nil - Empty + Ignored Masked Missed Masking Rate diff --git a/test/reports/report_bugfix_1.html b/test/reports/report_bugfix_1.html index 44f9aef..5f43f10 100644 --- a/test/reports/report_bugfix_1.html +++ b/test/reports/report_bugfix_1.html @@ -12,7 +12,7 @@

MIMO Report

Field Nil - Empty + Ignored Masked Missed Masking Rate diff --git a/test/reports/report_bugfix_exclude_numeric.html b/test/reports/report_bugfix_exclude_numeric.html index bcf349c..7b0489c 100644 --- a/test/reports/report_bugfix_exclude_numeric.html +++ b/test/reports/report_bugfix_exclude_numeric.html @@ -12,7 +12,7 @@

MIMO Report

Field Nil - Empty + Ignored Masked Missed Masking Rate diff --git a/test/reports/report_deep_array.html b/test/reports/report_deep_array.html index 8721c84..b658063 100644 --- a/test/reports/report_deep_array.html +++ b/test/reports/report_deep_array.html @@ -12,7 +12,7 @@

MIMO Report

Field Nil - Empty + Ignored Masked Missed Masking Rate diff --git a/test/reports/report_deep_mixed.html b/test/reports/report_deep_mixed.html index 92bb61d..e04a420 100644 --- a/test/reports/report_deep_mixed.html +++ b/test/reports/report_deep_mixed.html @@ -12,7 +12,7 @@

MIMO Report

Field Nil - Empty + Ignored Masked Missed Masking Rate diff --git a/test/reports/report_deep_object.html b/test/reports/report_deep_object.html index 9ee6b8d..c756b11 100644 --- a/test/reports/report_deep_object.html +++ b/test/reports/report_deep_object.html @@ -12,7 +12,7 @@

MIMO Report

Field Nil - Empty + Ignored Masked Missed Masking Rate diff --git a/test/reports/report_preprocess_simple.html b/test/reports/report_preprocess_simple.html index 133d304..c95e8bf 100644 --- a/test/reports/report_preprocess_simple.html +++ b/test/reports/report_preprocess_simple.html @@ -12,7 +12,7 @@

MIMO Report

Field Nil - Empty + Ignored Masked Missed Masking Rate diff --git a/test/reports/report_reuse_previous.html b/test/reports/report_reuse_previous.html index 889b61e..6ab0195 100644 --- a/test/reports/report_reuse_previous.html +++ b/test/reports/report_reuse_previous.html @@ -12,7 +12,7 @@

MIMO Report

Field Nil - Empty + Ignored Masked Missed Masking Rate diff --git a/test/reports/report_reuse_previous_alias.html b/test/reports/report_reuse_previous_alias.html index 889b61e..6ab0195 100644 --- a/test/reports/report_reuse_previous_alias.html +++ b/test/reports/report_reuse_previous_alias.html @@ -12,7 +12,7 @@

MIMO Report

Field Nil - Empty + Ignored Masked Missed Masking Rate diff --git a/test/reports/report_template_source_stack.html b/test/reports/report_template_source_stack.html index 0e5a169..aaa6987 100644 --- a/test/reports/report_template_source_stack.html +++ b/test/reports/report_template_source_stack.html @@ -12,7 +12,7 @@

MIMO Report

Field Nil - Empty + Ignored Masked Missed Masking Rate diff --git a/test/suites/02-validate-metrics.yml b/test/suites/02-validate-metrics.yml index 8bba75c..d04349e 100644 --- a/test/suites/02-validate-metrics.yml +++ b/test/suites/02-validate-metrics.yml @@ -35,7 +35,7 @@ testcases: - script: cat working/masked.jsonl | mimo -v3 working/real.jsonl assertions: - result.code ShouldEqual 0 - - result.systemerr ShouldContainSubstring value count-empty=0 count-masked=8 count-missed=2 count-nil=1 field=value rate-coherence=0.6666666666666666 rate-identifiable=0.6 rate-masking=0.8 + - result.systemerr ShouldContainSubstring value count-ignored=0 count-masked=8 count-missed=2 count-nil=1 field=value rate-coherence=0.6666666666666666 rate-identifiable=0.6 rate-masking=0.8 - script: mv report.html ../reports/report_1.html @@ -66,7 +66,7 @@ testcases: - script: cat working/masked.jsonl | mimo --config ../configs/config_exclude.yaml -v3 working/real.jsonl assertions: - result.code ShouldEqual 0 - - result.systemerr ShouldContainSubstring value count-empty=2 count-masked=8 count-missed=0 count-nil=0 field=value rate-coherence=0.6 rate-identifiable=0.8 rate-masking=1 + - result.systemerr ShouldContainSubstring value count-ignored=2 count-masked=8 count-missed=0 count-nil=0 field=value rate-coherence=0.6 rate-identifiable=0.8 rate-masking=1 - script: mv report.html ../reports/report_2.html @@ -97,14 +97,14 @@ testcases: - script: cat working/masked.jsonl | mimo --config ../configs/config_coherent.yaml -v3 working/real.jsonl assertions: - result.code ShouldEqual 0 - - result.systemerr ShouldContainSubstring value count-empty=0 count-masked=8 count-missed=2 count-nil=0 field=value rate-coherence=0 rate-identifiable=0.8 rate-masking=0.8 + - result.systemerr ShouldContainSubstring value count-ignored=0 count-masked=8 count-missed=2 count-nil=0 field=value rate-coherence=0 rate-identifiable=0.8 rate-masking=0.8 - script: mv report.html ../reports/report_3.html - script: cat working/masked.jsonl | mimo --config ../configs/config_coherent2.yaml -v3 working/real.jsonl assertions: - result.code ShouldEqual 0 - - result.systemerr ShouldContainSubstring value count-empty=0 count-masked=8 count-missed=2 count-nil=0 field=value rate-coherence=1 rate-identifiable=0.8 rate-masking=0.8 + - result.systemerr ShouldContainSubstring value count-ignored=0 count-masked=8 count-missed=2 count-nil=0 field=value rate-coherence=1 rate-identifiable=0.8 rate-masking=0.8 - script: mv report.html ../reports/report_4.html @@ -135,13 +135,13 @@ testcases: - script: cat working/masked.jsonl | mimo --config ../configs/config_constraints_1.yaml -v3 working/real.jsonl assertions: - result.code ShouldEqual 0 - - result.systemerr ShouldContainSubstring value count-empty=0 count-masked=8 count-missed=2 count-nil=0 field=value rate-coherence=0.6 rate-identifiable=0.8 rate-masking=0.8 + - result.systemerr ShouldContainSubstring value count-ignored=0 count-masked=8 count-missed=2 count-nil=0 field=value rate-coherence=0.6 rate-identifiable=0.8 rate-masking=0.8 - script: mv report.html ../reports/report_5.html - script: cat working/masked.jsonl | mimo --config ../configs/config_constraints_2.yaml -v3 working/real.jsonl assertions: - result.code ShouldEqual 1 - - result.systemerr ShouldContainSubstring value count-empty=0 count-masked=8 count-missed=2 count-nil=0 field=value rate-coherence=0.6 rate-identifiable=0.8 rate-masking=0.8 + - result.systemerr ShouldContainSubstring value count-ignored=0 count-masked=8 count-missed=2 count-nil=0 field=value rate-coherence=0.6 rate-identifiable=0.8 rate-masking=0.8 - script: mv report.html ../reports/report_6.html diff --git a/test/suites/03-persist-storage.yml b/test/suites/03-persist-storage.yml index 38e04cc..6379fdf 100644 --- a/test/suites/03-persist-storage.yml +++ b/test/suites/03-persist-storage.yml @@ -37,7 +37,7 @@ testcases: - script: cat working/masked.jsonl | mimo -v3 --disk-storage working/real.jsonl assertions: - result.code ShouldEqual 0 - - result.systemerr ShouldContainSubstring value count-empty=0 count-masked=8 count-missed=2 count-nil=1 field=value rate-coherence=0.6666666666666666 rate-identifiable=0.6 rate-masking=0.8 + - result.systemerr ShouldContainSubstring value count-ignored=0 count-masked=8 count-missed=2 count-nil=1 field=value rate-coherence=0.6666666666666666 rate-identifiable=0.6 rate-masking=0.8 - script: ls -l /tmp/ | grep mimo-pebble |wc -l assertions: @@ -80,7 +80,7 @@ testcases: - script: cat working/masked.jsonl | mimo -v3 --disk-storage --persist working/data/ working/real.jsonl assertions: - result.code ShouldEqual 0 - - result.systemerr ShouldContainSubstring value count-empty=0 count-masked=8 count-missed=2 count-nil=1 field=value rate-coherence=0.6666666666666666 rate-identifiable=0.6 rate-masking=0.8 + - result.systemerr ShouldContainSubstring value count-ignored=0 count-masked=8 count-missed=2 count-nil=1 field=value rate-coherence=0.6666666666666666 rate-identifiable=0.6 rate-masking=0.8 - script: mv report.html ../reports/report_8.html @@ -123,6 +123,6 @@ testcases: - script: cat working/masked.jsonl | mimo -v5 --disk-storage --persist working/data/ working/real.jsonl assertions: - result.code ShouldEqual 0 - - result.systemerr ShouldContainSubstring value count-empty=0 count-masked=4 count-missed=1 count-nil=1 field=value rate-coherence=0.6666666666666666 rate-identifiable=0.6 rate-masking=0.8 + - result.systemerr ShouldContainSubstring value count-ignored=0 count-masked=4 count-missed=1 count-nil=1 field=value rate-coherence=0.6666666666666666 rate-identifiable=0.6 rate-masking=0.8 - script: mv report.html ../reports/report_reuse_previous.html diff --git a/test/suites/04-dig-objects.yml b/test/suites/04-dig-objects.yml index 1dc13b1..543aa2d 100644 --- a/test/suites/04-dig-objects.yml +++ b/test/suites/04-dig-objects.yml @@ -13,8 +13,8 @@ testcases: - script: cat working/masked.jsonl | mimo -v3 working/real.jsonl assertions: - result.code ShouldEqual 0 - - result.systemerr ShouldContainSubstring person.age count-empty=0 count-masked=2 count-missed=0 count-nil=0 field=person.age rate-coherence=1 rate-identifiable=1 rate-masking=1 - - result.systemerr ShouldContainSubstring person.address.street count-empty=0 count-masked=2 count-missed=0 count-nil=0 field=person.address.street rate-coherence=0 rate-identifiable=1 rate-masking=1 + - result.systemerr ShouldContainSubstring person.age count-ignored=0 count-masked=2 count-missed=0 count-nil=0 field=person.age rate-coherence=1 rate-identifiable=1 rate-masking=1 + - result.systemerr ShouldContainSubstring person.address.street count-ignored=0 count-masked=2 count-missed=0 count-nil=0 field=person.address.street rate-coherence=0 rate-identifiable=1 rate-masking=1 - script: mv report.html ../reports/report_deep_object.html @@ -29,8 +29,8 @@ testcases: - script: cat working/masked.jsonl | mimo -v3 working/real.jsonl assertions: - result.code ShouldEqual 0 - - result.systemerr ShouldContainSubstring groups.[] count-empty=0 count-masked=2 count-missed=2 count-nil=0 field=groups.[] rate-coherence=1 rate-identifiable=1 rate-masking=0.5 - - result.systemerr ShouldContainSubstring groups.[].[] count-empty=0 count-masked=4 count-missed=0 count-nil=0 field=groups.[].[] rate-coherence=1 rate-identifiable=0 rate-masking=1 + - result.systemerr ShouldContainSubstring groups.[] count-ignored=0 count-masked=2 count-missed=2 count-nil=0 field=groups.[] rate-coherence=1 rate-identifiable=1 rate-masking=0.5 + - result.systemerr ShouldContainSubstring groups.[].[] count-ignored=0 count-masked=4 count-missed=0 count-nil=0 field=groups.[].[] rate-coherence=1 rate-identifiable=0 rate-masking=1 - script: mv report.html ../reports/report_deep_array.html @@ -45,7 +45,7 @@ testcases: - script: cat working/masked.jsonl | mimo -v3 working/real.jsonl assertions: - result.code ShouldEqual 0 - - result.systemerr ShouldContainSubstring batchs.[].id count-empty=0 count-masked=0 count-missed=3 count-nil=0 field=batchs.[].id rate-coherence=1 rate-identifiable=1 rate-masking=0 - - result.systemerr ShouldContainSubstring batchs.[].accounts.[].number count-empty=0 count-masked=4 count-missed=0 count-nil=0 field=batchs.[].accounts.[].number rate-coherence=1 rate-identifiable=1 rate-masking=1 + - result.systemerr ShouldContainSubstring batchs.[].id count-ignored=0 count-masked=0 count-missed=3 count-nil=0 field=batchs.[].id rate-coherence=1 rate-identifiable=1 rate-masking=0 + - result.systemerr ShouldContainSubstring batchs.[].accounts.[].number count-ignored=0 count-masked=4 count-missed=0 count-nil=0 field=batchs.[].accounts.[].number rate-coherence=1 rate-identifiable=1 rate-masking=1 - script: mv report.html ../reports/report_deep_mixed.html diff --git a/test/suites/05-template-source.yml b/test/suites/05-template-source.yml index 833037f..9da0051 100644 --- a/test/suites/05-template-source.yml +++ b/test/suites/05-template-source.yml @@ -13,7 +13,7 @@ testcases: - script: cat working/masked.jsonl | mimo --config ../configs/config_template.yaml -v3 working/real.jsonl assertions: - result.code ShouldEqual 0 - - result.systemerr ShouldContainSubstring batchs.[].id count-empty=0 count-masked=0 count-missed=3 count-nil=0 field=batchs.[].id rate-coherence=1 rate-identifiable=1 rate-masking=0 - - result.systemerr ShouldContainSubstring batchs.[].accounts.[].number count-empty=0 count-masked=4 count-missed=0 count-nil=0 field=batchs.[].accounts.[].number rate-coherence=0.6666666666666666 rate-identifiable=1 rate-masking=1 + - result.systemerr ShouldContainSubstring batchs.[].id count-ignored=0 count-masked=0 count-missed=3 count-nil=0 field=batchs.[].id rate-coherence=1 rate-identifiable=1 rate-masking=0 + - result.systemerr ShouldContainSubstring batchs.[].accounts.[].number count-ignored=0 count-masked=4 count-missed=0 count-nil=0 field=batchs.[].accounts.[].number rate-coherence=0.6666666666666666 rate-identifiable=1 rate-masking=1 - script: mv report.html ../reports/report_template_source_stack.html diff --git a/test/suites/06-column-alias.yml b/test/suites/06-column-alias.yml index f79a628..cd3cf1d 100644 --- a/test/suites/06-column-alias.yml +++ b/test/suites/06-column-alias.yml @@ -41,6 +41,6 @@ testcases: - script: cat working/masked.jsonl | mimo -v5 --config ../configs/config_alias.yaml --disk-storage --persist working/data/ working/real.jsonl assertions: - result.code ShouldEqual 0 - - result.systemerr ShouldContainSubstring value count-empty=0 count-masked=4 count-missed=1 count-nil=1 field=value rate-coherence=0.6666666666666666 rate-identifiable=0.6 rate-masking=0.8 + - result.systemerr ShouldContainSubstring value count-ignored=0 count-masked=4 count-missed=1 count-nil=1 field=value rate-coherence=0.6666666666666666 rate-identifiable=0.6 rate-masking=0.8 - script: mv report.html ../reports/report_reuse_previous_alias.html diff --git a/test/suites/07-preprocess.yml b/test/suites/07-preprocess.yml index 1af7101..632de93 100644 --- a/test/suites/07-preprocess.yml +++ b/test/suites/07-preprocess.yml @@ -13,10 +13,10 @@ testcases: - script: cat working/masked.jsonl | mimo --config ../configs/config_preprocess_simple.yaml -v3 working/real.jsonl assertions: - result.code ShouldEqual 0 - - result.systemerr ShouldContainSubstring id count-empty=0 count-masked=0 count-missed=2 count-nil=0 field=id rate-coherence=1 rate-identifiable=1 rate-masking=0 - - result.systemerr ShouldContainSubstring name count-empty=0 count-masked=2 count-missed=0 count-nil=0 field=name rate-coherence=1 rate-identifiable=1 rate-masking=1 - - result.systemerr ShouldContainSubstring surname count-empty=0 count-masked=2 count-missed=0 count-nil=0 field=surname rate-coherence=1 rate-identifiable=0 rate-masking=1 - - result.systemerr ShouldContainSubstring idutil count-empty=0 count-masked=2 count-missed=0 count-nil=0 field=idutil rate-coherence=1 rate-identifiable=1 rate-masking=1 + - result.systemerr ShouldContainSubstring id count-ignored=0 count-masked=0 count-missed=2 count-nil=0 field=id rate-coherence=1 rate-identifiable=1 rate-masking=0 + - result.systemerr ShouldContainSubstring name count-ignored=0 count-masked=2 count-missed=0 count-nil=0 field=name rate-coherence=1 rate-identifiable=1 rate-masking=1 + - result.systemerr ShouldContainSubstring surname count-ignored=0 count-masked=2 count-missed=0 count-nil=0 field=surname rate-coherence=1 rate-identifiable=0 rate-masking=1 + - result.systemerr ShouldContainSubstring idutil count-ignored=0 count-masked=2 count-missed=0 count-nil=0 field=idutil rate-coherence=1 rate-identifiable=1 rate-masking=1 - script: mv report.html ../reports/report_preprocess_simple.html @@ -31,8 +31,8 @@ testcases: - script: cat working/masked.jsonl | mimo --config ../configs/config_preprocess_deep.yaml -v5 working/real.jsonl assertions: - result.code ShouldEqual 0 - - result.systemerr ShouldContainSubstring batchs.[].accounts.[].number count-empty=0 count-masked=4 count-missed=0 count-nil=0 field=batchs.[].accounts.[].number rate-coherence=0 rate-identifiable=1 rate-masking=1 - - result.systemerr ShouldContainSubstring batchs.[].accounts.[].number_clean count-empty=0 count-masked=4 count-missed=0 count-nil=0 field=batchs.[].accounts.[].number_clean rate-coherence=1 rate-identifiable=1 rate-masking=1 - - result.systemerr ShouldContainSubstring batchs.[].id count-empty=0 count-masked=0 count-missed=3 count-nil=0 field=batchs.[].id rate-coherence=1 rate-identifiable=1 rate-masking=0 + - result.systemerr ShouldContainSubstring batchs.[].accounts.[].number count-ignored=0 count-masked=4 count-missed=0 count-nil=0 field=batchs.[].accounts.[].number rate-coherence=0 rate-identifiable=1 rate-masking=1 + - result.systemerr ShouldContainSubstring batchs.[].accounts.[].number_clean count-ignored=0 count-masked=4 count-missed=0 count-nil=0 field=batchs.[].accounts.[].number_clean rate-coherence=1 rate-identifiable=1 rate-masking=1 + - result.systemerr ShouldContainSubstring batchs.[].id count-ignored=0 count-masked=0 count-missed=3 count-nil=0 field=batchs.[].id rate-coherence=1 rate-identifiable=1 rate-masking=0 - script: mv report.html ../reports/report_preprocess_simple.html diff --git a/test/suites/99-bugfixes.yml b/test/suites/99-bugfixes.yml index 2edfe92..d4c9358 100644 --- a/test/suites/99-bugfixes.yml +++ b/test/suites/99-bugfixes.yml @@ -16,7 +16,7 @@ testcases: - script: cat working/masked.jsonl | mimo -v3 working/real.jsonl assertions: - result.code ShouldEqual 0 - - result.systemerr ShouldContainSubstring value count-empty=0 count-masked=0 count-missed=0 count-nil=4 field=value rate-coherence=0 rate-identifiable=1 rate-masking=NaN + - result.systemerr ShouldContainSubstring value count-ignored=0 count-masked=0 count-missed=0 count-nil=4 field=value rate-coherence=0 rate-identifiable=1 rate-masking=NaN - script: mv report.html ../reports/report_bugfix_1.html @@ -35,6 +35,6 @@ testcases: - script: cat working/masked.jsonl | mimo --config ../configs/config_exclude_numeric.yaml -v3 working/real.jsonl assertions: - result.code ShouldEqual 0 - - result.systemerr ShouldContainSubstring value count-empty=1 count-masked=3 count-missed=0 count-nil=0 field=value rate-coherence=1 rate-identifiable=1 rate-masking=1 + - result.systemerr ShouldContainSubstring value count-ignored=1 count-masked=3 count-missed=0 count-nil=0 field=value rate-coherence=1 rate-identifiable=1 rate-masking=1 - script: mv report.html ../reports/report_bugfix_exclude_numeric.html