From f7ed865eb0fb2198f4cada2c52fce28bbdc8f712 Mon Sep 17 00:00:00 2001 From: "paul.briand" Date: Tue, 22 Feb 2022 17:18:15 +0100 Subject: [PATCH 1/9] feat(push): add import types for push --- go.mod | 2 +- go.sum | 4 +- internal/infra/push/datadestination_sql.go | 6 ++- pkg/push/model.go | 3 +- pkg/push/model_table.go | 56 ++++++++++++++++++++++ 5 files changed, 66 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index dd3460b9..ddd95c66 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/Trendyol/overlog v0.1.0 github.com/awalterschulze/gographviz v2.0.3+incompatible - github.com/cgi-fr/jsonline v0.4.0 + github.com/cgi-fr/jsonline v0.5.0 github.com/docker/docker-credential-helpers v0.6.4 github.com/go-sql-driver/mysql v1.6.0 github.com/godror/godror v0.28.0 diff --git a/go.sum b/go.sum index 7821a84a..63906dea 100644 --- a/go.sum +++ b/go.sum @@ -59,8 +59,8 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cgi-fr/jsonline v0.4.0 h1:Y64IVccv/dmcuY782g95LsbUXptIwk380QrQm3BQNXs= -github.com/cgi-fr/jsonline v0.4.0/go.mod h1:2eo1zPtPXeGiGCEI+Y2m0GYlQgmRikVeQOwLwOZtWXQ= +github.com/cgi-fr/jsonline v0.5.0 h1:UFLDJauppXLXWlUXFgAKqlZaKt0g1gRq3tLJUj95ohY= +github.com/cgi-fr/jsonline v0.5.0/go.mod h1:2eo1zPtPXeGiGCEI+Y2m0GYlQgmRikVeQOwLwOZtWXQ= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= diff --git a/internal/infra/push/datadestination_sql.go b/internal/infra/push/datadestination_sql.go index 72058980..76e35e8d 100644 --- a/internal/infra/push/datadestination_sql.go +++ b/internal/infra/push/datadestination_sql.go @@ -314,9 +314,13 @@ func (rw *SQLRowWriter) Write(row push.Row) *push.Error { return err1 } + importedRow := rw.table.Import(row) values := []interface{}{} for _, h := range rw.headers { - values = append(values, rw.dd.dialect.ConvertValue(row[h])) + value := importedRow.GetOrNil(h) + + v := rw.dd.dialect.ConvertValue(value) + values = append(values, v) } log.Trace().Strs("headers", rw.headers).Msg(fmt.Sprint(values)) diff --git a/pkg/push/model.go b/pkg/push/model.go index 94c843bb..a94f003c 100755 --- a/pkg/push/model.go +++ b/pkg/push/model.go @@ -29,6 +29,7 @@ type Table interface { Name() string PrimaryKey() []string Columns() ColumnList + Import(map[string]interface{}) ImportedRow } // ColumnList is a list of columns. @@ -62,7 +63,7 @@ type Relation interface { type Value interface{} // Row of data. -type Row map[string]Value +type Row map[string]interface{} // Error is the error type returned by the domain type Error struct { diff --git a/pkg/push/model_table.go b/pkg/push/model_table.go index 46814d0a..526ec57e 100755 --- a/pkg/push/model_table.go +++ b/pkg/push/model_table.go @@ -19,13 +19,19 @@ package push import ( "fmt" + "os" "strings" + + "github.com/cgi-fr/jsonline/pkg/jsonline" + "github.com/rs/zerolog/log" ) type table struct { name string pk []string columns ColumnList + + template jsonline.Template } // NewTable initialize a new Table object @@ -77,3 +83,53 @@ func NewColumn(name string, imp string) Column { func (c column) Name() string { return c.name } func (c column) Import() string { return c.imp } + +type ImportedRow struct { + jsonline.Row +} + +func (t *table) initTemplate() { + t.template = jsonline.NewTemplate() + + if t.columns == nil { + return + } + + if l := int(t.columns.Len()); l > 0 { + for idx := 0; idx < l; idx++ { + col := t.columns.Column(uint(idx)) + key := col.Name() + + switch col.Import() { + case "string": + t.template.WithString(key) + case "numeric": + t.template.WithNumeric(key) + case "base64": + t.template.WithBinary(key) + case "datetime": + t.template.WithDateTime(key) + case "timestamp": + t.template.WithTimestamp(key) + case "no": + t.template.WithHidden(key) + default: + t.template.WithAuto(key) + } + } + } +} + +func (t table) Import(row map[string]interface{}) ImportedRow { + if t.template == nil { + t.initTemplate() + } + + result := ImportedRow{t.template.CreateRowEmpty()} + err := result.Import(row) + if err != nil { + log.Err(err).Msg("End pimo") + os.Exit(4) + } + return result +} From a9384f3517d5b7b708128720ab3067616b9cf77d Mon Sep 17 00:00:00 2001 From: Adrien Aury Date: Tue, 8 Mar 2022 20:11:27 +0100 Subject: [PATCH 2/9] fix(import): read correct value --- internal/app/push/cli.go | 2 +- pkg/push/model.go | 1 + pkg/push/model_table.go | 8 +++++--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/internal/app/push/cli.go b/internal/app/push/cli.go index 3ffa9902..ba8e6472 100755 --- a/internal/app/push/cli.go +++ b/internal/app/push/cli.go @@ -242,7 +242,7 @@ func (c idToPushConverter) getTable(name string) push.Table { columns := []push.Column{} for _, col := range table.Columns { - columns = append(columns, push.NewColumn(col.Name, col.Import)) + columns = append(columns, push.NewColumn(col.Name, col.Export, col.Import)) } return push.NewTable(table.Name, table.Keys, push.NewColumnList(columns)) diff --git a/pkg/push/model.go b/pkg/push/model.go index a94f003c..f43fa8b7 100755 --- a/pkg/push/model.go +++ b/pkg/push/model.go @@ -41,6 +41,7 @@ type ColumnList interface { // Column of a table. type Column interface { Name() string + Export() string Import() string } diff --git a/pkg/push/model_table.go b/pkg/push/model_table.go index 526ec57e..c42217b0 100755 --- a/pkg/push/model_table.go +++ b/pkg/push/model_table.go @@ -73,15 +73,17 @@ func (l columnList) String() string { type column struct { name string + exp string imp string } // NewColumn initialize a new Column object -func NewColumn(name string, imp string) Column { - return column{name, imp} +func NewColumn(name string, exp string, imp string) Column { + return column{name, exp, imp} } func (c column) Name() string { return c.name } +func (c column) Export() string { return c.exp } func (c column) Import() string { return c.imp } type ImportedRow struct { @@ -100,7 +102,7 @@ func (t *table) initTemplate() { col := t.columns.Column(uint(idx)) key := col.Name() - switch col.Import() { + switch col.Export() { case "string": t.template.WithString(key) case "numeric": From 3784473ef4112a00a732f709869367383c9d4732 Mon Sep 17 00:00:00 2001 From: Adrien Aury Date: Tue, 8 Mar 2022 20:23:24 +0100 Subject: [PATCH 3/9] fix(import): add venom test --- tests/suites/push/import.yml | 47 ++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 tests/suites/push/import.yml diff --git a/tests/suites/push/import.yml b/tests/suites/push/import.yml new file mode 100644 index 00000000..f797f80b --- /dev/null +++ b/tests/suites/push/import.yml @@ -0,0 +1,47 @@ +# Copyright (C) 2021 CGI France +# +# This file is part of LINO. +# +# LINO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# LINO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with LINO. If not, see . + +name: import different types +testcases: + - name: prepare test + steps: + # Clean working directory + - script: rm -f * + - script: lino dataconnector add --read-only source 'postgresql://postgres:sakila@source:5432/postgres?sslmode=disable' + - script: lino dataconnector add dest 'postgresql://postgres:sakila@dest:5432/postgres?sslmode=disable' + - script: lino relation extract source + + - name: export then import binary value + steps: + - script: |- + cat > tables.yml.yml < Date: Tue, 8 Mar 2022 20:41:53 +0100 Subject: [PATCH 6/9] fix(import): fix venom test --- tests/suites/push/import.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/suites/push/import.yml b/tests/suites/push/import.yml index 2f26030c..c0845519 100644 --- a/tests/suites/push/import.yml +++ b/tests/suites/push/import.yml @@ -28,7 +28,7 @@ testcases: - name: export then import binary value steps: - script: |- - cat > tables.yml < tables.yaml < Date: Tue, 8 Mar 2022 22:13:50 +0100 Subject: [PATCH 7/9] fix(import): lint and error checking --- internal/infra/push/datadestination_sql.go | 11 ++++++----- pkg/push/model.go | 2 +- pkg/push/model_table.go | 9 ++++++--- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/internal/infra/push/datadestination_sql.go b/internal/infra/push/datadestination_sql.go index 76e35e8d..c498af3f 100644 --- a/internal/infra/push/datadestination_sql.go +++ b/internal/infra/push/datadestination_sql.go @@ -314,13 +314,14 @@ func (rw *SQLRowWriter) Write(row push.Row) *push.Error { return err1 } - importedRow := rw.table.Import(row) + importedRow, err15 := rw.table.Import(row) + if err15 != nil { + return err15 + } + values := []interface{}{} for _, h := range rw.headers { - value := importedRow.GetOrNil(h) - - v := rw.dd.dialect.ConvertValue(value) - values = append(values, v) + values = append(values, rw.dd.dialect.ConvertValue(importedRow.GetOrNil(h))) } log.Trace().Strs("headers", rw.headers).Msg(fmt.Sprint(values)) diff --git a/pkg/push/model.go b/pkg/push/model.go index f43fa8b7..6434336f 100755 --- a/pkg/push/model.go +++ b/pkg/push/model.go @@ -29,7 +29,7 @@ type Table interface { Name() string PrimaryKey() []string Columns() ColumnList - Import(map[string]interface{}) ImportedRow + Import(map[string]interface{}) (ImportedRow, *Error) } // ColumnList is a list of columns. diff --git a/pkg/push/model_table.go b/pkg/push/model_table.go index 41176933..27691af1 100755 --- a/pkg/push/model_table.go +++ b/pkg/push/model_table.go @@ -122,14 +122,17 @@ func (t *table) initTemplate() { } } -func (t table) Import(row map[string]interface{}) ImportedRow { +func (t table) Import(row map[string]interface{}) (ImportedRow, *Error) { if t.template == nil { t.initTemplate() } result := ImportedRow{t.template.CreateRowEmpty()} - _ = result.Import(row) - return result + if err := result.Import(row); err != nil { + return ImportedRow{}, &Error{Description: err.Error()} + } + + return result, nil } func parseExportType(exp string) jsonline.RawType { From 16163c58bd87eae0e1729a04b76d830437d87728 Mon Sep 17 00:00:00 2001 From: Adrien Aury Date: Tue, 8 Mar 2022 22:27:35 +0100 Subject: [PATCH 8/9] feat(import): use correct value for import --- pkg/push/model_table.go | 14 +++++++------- tests/suites/push/import.yml | 22 ++++++++++++++++++++++ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/pkg/push/model_table.go b/pkg/push/model_table.go index 27691af1..76b1bd4d 100755 --- a/pkg/push/model_table.go +++ b/pkg/push/model_table.go @@ -104,19 +104,19 @@ func (t *table) initTemplate() { switch col.Export() { case "string": - t.template.WithMappedString(key, parseExportType(col.Export())) + t.template.WithMappedString(key, parseImportType(col.Import())) case "numeric": - t.template.WithMappedNumeric(key, parseExportType(col.Export())) + t.template.WithMappedNumeric(key, parseImportType(col.Import())) case "base64", "binary": - t.template.WithMappedBinary(key, parseExportType(col.Export())) + t.template.WithMappedBinary(key, parseImportType(col.Import())) case "datetime": - t.template.WithMappedDateTime(key, parseExportType(col.Export())) + t.template.WithMappedDateTime(key, parseImportType(col.Import())) case "timestamp": - t.template.WithMappedTimestamp(key, parseExportType(col.Export())) + t.template.WithMappedTimestamp(key, parseImportType(col.Import())) case "no": t.template.WithHidden(key) default: - t.template.WithMappedAuto(key, parseExportType(col.Export())) + t.template.WithMappedAuto(key, parseImportType(col.Import())) } } } @@ -135,7 +135,7 @@ func (t table) Import(row map[string]interface{}) (ImportedRow, *Error) { return result, nil } -func parseExportType(exp string) jsonline.RawType { +func parseImportType(exp string) jsonline.RawType { switch exp { case "int": return int(0) diff --git a/tests/suites/push/import.yml b/tests/suites/push/import.yml index c0845519..d2872b86 100644 --- a/tests/suites/push/import.yml +++ b/tests/suites/push/import.yml @@ -45,3 +45,25 @@ testcases: - result.systemerr ShouldBeEmpty - result.code ShouldEqual 0 - result.systemout ShouldEqual {"staff_id":1,"picture":"iVBORw0KWgo="} + + - name: export to different format than stored in datasource + steps: + - script: |- + cat > tables.yaml < Date: Tue, 8 Mar 2022 22:47:10 +0100 Subject: [PATCH 9/9] test(import): fix venom test + changelog --- CHANGELOG.md | 2 +- tests/suites/push/import.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4294fbd0..1447aeb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ Types of changes ## [2.0.0] (Ureleased) - `Changed` order of keys in output JSON lines will be alphabetical when pulling (without configuration in tables.yaml) -- `Added` configuration of exported columns in tables.yaml, see issue #33 for more information +- `Added` configuration of export format / import type for columns in tables.yaml, see issue #33 for more information - `Added` MariaDB/MySQL support (thanks to @joaking85) - `Added` auto-select columns required by a relation but not exported in tables.yaml - `Added` new commands to configure tables : add-column and remove-column diff --git a/tests/suites/push/import.yml b/tests/suites/push/import.yml index d2872b86..139399b2 100644 --- a/tests/suites/push/import.yml +++ b/tests/suites/push/import.yml @@ -48,6 +48,8 @@ testcases: - name: export to different format than stored in datasource steps: + - script: lino table extract source + - script: lino pull source --table customer --limit 0 | lino push truncate dest --table customer - script: |- cat > tables.yaml <