diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e1b382..bc631f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ Types of changes ## [0.6.0] - `Added` log unmasked values with the `watch` flag. +- `Added` new config parameter `excludeTemplate` to exclude a value with a template expression. - `Fixed` exclusion with coherent source specified. ## [0.5.0] diff --git a/internal/infra/config_loader.go b/internal/infra/config_loader.go index b21d025..e375a2f 100644 --- a/internal/infra/config_loader.go +++ b/internal/infra/config_loader.go @@ -38,12 +38,13 @@ type YAMLStructure struct { // YAMLColumn defines how to store a column config in YAML format. type YAMLColumn struct { - Name string `yaml:"name"` - Exclude []any `yaml:"exclude,omitempty"` - CoherentWith []string `yaml:"coherentWith,omitempty"` - CoherentSource string `yaml:"coherentSource,omitempty"` - Constraints map[string]YAMLConstraint `yaml:"constraints,omitempty"` - Alias string `yaml:"alias,omitempty"` + Name string `yaml:"name"` + Exclude []any `yaml:"exclude,omitempty"` + ExcludeTemplate string `yaml:"excludeTemplate,omitempty"` + CoherentWith []string `yaml:"coherentWith,omitempty"` + CoherentSource string `yaml:"coherentSource,omitempty"` + Constraints map[string]YAMLConstraint `yaml:"constraints,omitempty"` + Alias string `yaml:"alias,omitempty"` } type YAMLPreprocess struct { @@ -91,11 +92,12 @@ func CreateConfig(yamlconfig *YAMLStructure) (mimo.Config, error) { for _, yamlcolumn := range yamlconfig.Columns { column := mimo.ColumnConfig{ - Exclude: yamlcolumn.Exclude, - CoherentWith: yamlcolumn.CoherentWith, - CoherentSource: yamlcolumn.CoherentSource, - Constraints: []mimo.Constraint{}, - Alias: yamlcolumn.Alias, + Exclude: yamlcolumn.Exclude, + ExcludeTemplate: yamlcolumn.ExcludeTemplate, + CoherentWith: yamlcolumn.CoherentWith, + CoherentSource: yamlcolumn.CoherentSource, + Constraints: []mimo.Constraint{}, + Alias: yamlcolumn.Alias, } for target, yamlconstraint := range yamlcolumn.Constraints { diff --git a/pkg/mimo/driver_preprocess.go b/pkg/mimo/driver_preprocess.go index 81e76a7..65dbfca 100644 --- a/pkg/mimo/driver_preprocess.go +++ b/pkg/mimo/driver_preprocess.go @@ -36,11 +36,11 @@ func preprocessValue(value any, paths []string, stack []any, templstr string, ro if len(paths) == 1 { if obj, ok := value.(map[string]any); ok { - obj[path], err = generateCoherentSource(templstr, root, append(stack, obj)) + obj[path], err = applyTemplate(templstr, root, append(stack, obj)) } if obj, ok := value.(DataRow); ok { - obj[path], err = generateCoherentSource(templstr, root, append(stack, obj)) + obj[path], err = applyTemplate(templstr, root, append(stack, obj)) } if err != nil { diff --git a/pkg/mimo/model.go b/pkg/mimo/model.go index d123a85..f385f05 100644 --- a/pkg/mimo/model.go +++ b/pkg/mimo/model.go @@ -83,18 +83,16 @@ func (m *Metrics) Update( subs Suscribers, config ColumnConfig, ) bool { - nonBlankCount := m.NonBlankCount() - realValueStr, realValueOk := toString(realValue) maskedValueStr, maskedValueOk := toString(maskedValue) if !realValueOk || !maskedValueOk { - return false // special case (arrays, objects) are not covered right now + return false } m.backend.IncreaseTotalCount() - excluded := isExcluded(config.Exclude, realValue, realValueStr) + excluded := config.excluded || isExcluded(config.Exclude, realValue, realValueStr) if !excluded { // coherence and identifiant rates are computed over all values by default (including nil values) @@ -126,16 +124,21 @@ func (m *Metrics) Update( if realValueStr != maskedValueStr { m.backend.IncreaseMaskedCount() } else { - subs.PostNonMaskedValue(fieldname, realValue) - if m.backend.GetMaskedCount() == nonBlankCount { - subs.PostFirstNonMaskedValue(fieldname, realValue) - } + m.postNonMaskedValue(subs, fieldname, realValue) } } return true } +func (m *Metrics) postNonMaskedValue(subs Suscribers, fieldname string, realValue any) { + if m.backend.GetMaskedCount() == m.NonBlankCount()-1 { + subs.PostFirstNonMaskedValue(fieldname, realValue) + } + + subs.PostNonMaskedValue(fieldname, realValue) +} + func (m Metrics) NilCount() int64 { return m.backend.GetNilCount() } @@ -422,6 +425,16 @@ func (r Report) UpdateValue(root DataRow, realValue any, maskedValue any, stack coherenceValues = []any{realValue} } + if len(config.ExcludeTemplate) > 0 { + result, err := applyTemplate(config.ExcludeTemplate, root, stack) + + log.Err(err).Str("result", result).Msg("compute exclusion from template") + + if exclude, err := strconv.ParseBool(result); exclude && err == nil { + config.excluded = true + } + } + if !metrics.Update(key, realValue, maskedValue, coherenceValues, r.subs, config) && !exists { metrics.Coherence.Close() metrics.Identifiant.Close() @@ -442,7 +455,7 @@ func computeCoherenceValues(config ColumnConfig, root DataRow, stack []any) []an } if len(config.CoherentSource) > 0 { - source, err := generateCoherentSource(config.CoherentSource, root, stack) + source, err := applyTemplate(config.CoherentSource, root, stack) log.Err(err).Str("result", source).Msg("generating coherence source from template") diff --git a/pkg/mimo/model_config.go b/pkg/mimo/model_config.go index 4277c9c..fe5e849 100644 --- a/pkg/mimo/model_config.go +++ b/pkg/mimo/model_config.go @@ -24,11 +24,14 @@ type Config struct { } type ColumnConfig struct { - Exclude []any // exclude values from the masking rate computation (default: exclude only nil values) - CoherentWith []string // list of fields from witch the coherent rate is computed (default: the current field) - CoherentSource string // template to execute to create coherence source - Constraints []Constraint // list of constraints to validate - Alias string // alias to use in persisted data + Exclude []any // exclude values from the masking rate computation (default: exclude only nil values) + ExcludeTemplate string // exclude values if template expression evaluate to True (default: False) + CoherentWith []string // list of fields from witch the coherent rate is computed (default: the current field) + CoherentSource string // template to execute to create coherence source + Constraints []Constraint // list of constraints to validate + Alias string // alias to use in persisted data + + excluded bool } type PreprocessConfig struct { @@ -70,10 +73,12 @@ func NewConfig() Config { func NewDefaultColumnConfig() ColumnConfig { return ColumnConfig{ - Exclude: []any{}, - CoherentWith: []string{}, - CoherentSource: "", - Constraints: []Constraint{}, - Alias: "", + Exclude: []any{}, + ExcludeTemplate: "", + CoherentWith: []string{}, + CoherentSource: "", + Constraints: []Constraint{}, + Alias: "", + excluded: false, } } diff --git a/pkg/mimo/template.go b/pkg/mimo/template.go index 22e5245..07ea049 100644 --- a/pkg/mimo/template.go +++ b/pkg/mimo/template.go @@ -29,7 +29,7 @@ import ( "golang.org/x/text/unicode/norm" ) -func generateCoherentSource(tmplstring string, root DataRow, stack []any) (string, error) { +func applyTemplate(tmplstring string, root DataRow, stack []any) (string, error) { funcmap := generateFuncMap() funcmap["Stack"] = generateStackFunc(stack) diff --git a/test/configs/config_exclude_template.yaml b/test/configs/config_exclude_template.yaml new file mode 100644 index 0000000..b14556a --- /dev/null +++ b/test/configs/config_exclude_template.yaml @@ -0,0 +1,4 @@ +version: "1" +metrics: + - name: "value" + excludeTemplate: '{{.email | hasSuffix ".fr"}}' diff --git a/test/reports/report_exclude_template.html b/test/reports/report_exclude_template.html new file mode 100644 index 0000000..3790d5c --- /dev/null +++ b/test/reports/report_exclude_template.html @@ -0,0 +1,52 @@ + + + + + MIMO Report + + + + +

MIMO Report

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldNilIgnoredMaskedMissedMasking RateCoherent RateIdentifiable RateK
email000100.00 %100.00 %100.00 %1
value0280100.00 %50.00 %80.00 %1
+ + diff --git a/test/suites/10-exclusion.yml b/test/suites/10-exclusion.yml new file mode 100644 index 0000000..6845d84 --- /dev/null +++ b/test/suites/10-exclusion.yml @@ -0,0 +1,32 @@ +name: value exclusion +testcases: + - name: exclude values by template + steps: + - script: echo '{"value":"A","email":"depot@domain.com"}' > working/real.jsonl + - script: echo '{"value":"A","email":"depot@domain.com"}' >> working/real.jsonl + - script: echo '{"value":"B","email":"depot@domain.com"}' >> working/real.jsonl + - script: echo '{"value":"B","email":"depot@domain.com"}' >> working/real.jsonl + - script: echo '{"value":"C","email":"depot@domain.fr"}' >> working/real.jsonl + - script: echo '{"value":"C","email":"depot@domain.fr"}' >> working/real.jsonl + - script: echo '{"value":"D","email":"depot@domain.com"}' >> working/real.jsonl + - script: echo '{"value":"D","email":"depot@domain.com"}' >> working/real.jsonl + - script: echo '{"value":"E","email":"depot@domain.com"}' >> working/real.jsonl + - script: echo '{"value":"E","email":"depot@domain.com"}' >> working/real.jsonl + + - script: echo '{"value":"X","email":"depot@domain.com"}' > working/masked.jsonl + - script: echo '{"value":"Y","email":"depot@domain.com"}' >> working/masked.jsonl + - script: echo '{"value":"C","email":"depot@domain.com"}' >> working/masked.jsonl + - script: echo '{"value":"C","email":"depot@domain.com"}' >> working/masked.jsonl + - script: echo '{"value":"C","email":"depot@domain.fr"}' >> working/masked.jsonl + - script: echo '{"value":"C","email":"depot@domain.fr"}' >> working/masked.jsonl + - script: echo '{"value":"W","email":"depot@domain.com"}' >> working/masked.jsonl + - script: echo '{"value":"W","email":"depot@domain.com"}' >> working/masked.jsonl + - script: echo '{"value":"V","email":"depot@domain.com"}' >> working/masked.jsonl + - script: echo '{"value":"C","email":"depot@domain.com"}' >> working/masked.jsonl + + - script: cat working/masked.jsonl | mimo --config ../configs/config_exclude_template.yaml -v3 working/real.jsonl + assertions: + - result.code ShouldEqual 0 + - result.systemerr ShouldContainSubstring value count-ignored=2 count-masked=8 count-missed=0 count-nil=0 field=value rate-coherence=0.5 rate-identifiable=0.8 rate-masking=1 + + - script: mv report.html ../reports/report_exclude_template.html