diff --git a/CHANGELOG.md b/CHANGELOG.md index 4294fbd0..1447aeb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ Types of changes ## [2.0.0] (Ureleased) - `Changed` order of keys in output JSON lines will be alphabetical when pulling (without configuration in tables.yaml) -- `Added` configuration of exported columns in tables.yaml, see issue #33 for more information +- `Added` configuration of export format / import type for columns in tables.yaml, see issue #33 for more information - `Added` MariaDB/MySQL support (thanks to @joaking85) - `Added` auto-select columns required by a relation but not exported in tables.yaml - `Added` new commands to configure tables : add-column and remove-column diff --git a/internal/app/push/cli.go b/internal/app/push/cli.go index 25f4c22b..638480e1 100755 --- a/internal/app/push/cli.go +++ b/internal/app/push/cli.go @@ -242,7 +242,7 @@ func (c idToPushConverter) getTable(name string) push.Table { columns := []push.Column{} for _, col := range table.Columns { - columns = append(columns, push.NewColumn(col.Name, col.Import)) + columns = append(columns, push.NewColumn(col.Name, col.Export, col.Import)) } return push.NewTable(table.Name, table.Keys, push.NewColumnList(columns)) diff --git a/internal/infra/push/datadestination_sql.go b/internal/infra/push/datadestination_sql.go index 72058980..c498af3f 100644 --- a/internal/infra/push/datadestination_sql.go +++ b/internal/infra/push/datadestination_sql.go @@ -314,9 +314,14 @@ func (rw *SQLRowWriter) Write(row push.Row) *push.Error { return err1 } + importedRow, err15 := rw.table.Import(row) + if err15 != nil { + return err15 + } + values := []interface{}{} for _, h := range rw.headers { - values = append(values, rw.dd.dialect.ConvertValue(row[h])) + values = append(values, rw.dd.dialect.ConvertValue(importedRow.GetOrNil(h))) } log.Trace().Strs("headers", rw.headers).Msg(fmt.Sprint(values)) diff --git a/pkg/push/model.go b/pkg/push/model.go index 94c843bb..6434336f 100755 --- a/pkg/push/model.go +++ b/pkg/push/model.go @@ -29,6 +29,7 @@ type Table interface { Name() string PrimaryKey() []string Columns() ColumnList + Import(map[string]interface{}) (ImportedRow, *Error) } // ColumnList is a list of columns. @@ -40,6 +41,7 @@ type ColumnList interface { // Column of a table. type Column interface { Name() string + Export() string Import() string } @@ -62,7 +64,7 @@ type Relation interface { type Value interface{} // Row of data. -type Row map[string]Value +type Row map[string]interface{} // Error is the error type returned by the domain type Error struct { diff --git a/pkg/push/model_table.go b/pkg/push/model_table.go index 46814d0a..76b1bd4d 100755 --- a/pkg/push/model_table.go +++ b/pkg/push/model_table.go @@ -18,14 +18,20 @@ package push import ( + "encoding/json" "fmt" "strings" + "time" + + "github.com/cgi-fr/jsonline/pkg/jsonline" ) type table struct { name string pk []string columns ColumnList + + template jsonline.Template } // NewTable initialize a new Table object @@ -67,13 +73,109 @@ func (l columnList) String() string { type column struct { name string + exp string imp string } // NewColumn initialize a new Column object -func NewColumn(name string, imp string) Column { - return column{name, imp} +func NewColumn(name string, exp string, imp string) Column { + return column{name, exp, imp} } func (c column) Name() string { return c.name } +func (c column) Export() string { return c.exp } func (c column) Import() string { return c.imp } + +type ImportedRow struct { + jsonline.Row +} + +func (t *table) initTemplate() { + t.template = jsonline.NewTemplate() + + if t.columns == nil { + return + } + + if l := int(t.columns.Len()); l > 0 { + for idx := 0; idx < l; idx++ { + col := t.columns.Column(uint(idx)) + key := col.Name() + + switch col.Export() { + case "string": + t.template.WithMappedString(key, parseImportType(col.Import())) + case "numeric": + t.template.WithMappedNumeric(key, parseImportType(col.Import())) + case "base64", "binary": + t.template.WithMappedBinary(key, parseImportType(col.Import())) + case "datetime": + t.template.WithMappedDateTime(key, parseImportType(col.Import())) + case "timestamp": + t.template.WithMappedTimestamp(key, parseImportType(col.Import())) + case "no": + t.template.WithHidden(key) + default: + t.template.WithMappedAuto(key, parseImportType(col.Import())) + } + } + } +} + +func (t table) Import(row map[string]interface{}) (ImportedRow, *Error) { + if t.template == nil { + t.initTemplate() + } + + result := ImportedRow{t.template.CreateRowEmpty()} + if err := result.Import(row); err != nil { + return ImportedRow{}, &Error{Description: err.Error()} + } + + return result, nil +} + +func parseImportType(exp string) jsonline.RawType { + switch exp { + case "int": + return int(0) + case "int64": + return int64(0) + case "int32": + return int32(0) + case "int16": + return int16(0) + case "int8": + return int8(0) + case "uint": + return uint(0) + case "uint64": + return uint64(0) + case "uint32": + return uint32(0) + case "uint16": + return uint16(0) + case "uint8": + return uint8(0) + case "float64": + return float64(0) + case "float32": + return float32(0) + case "bool": + return false + case "byte": + return byte(0) + case "rune": + return rune(' ') + case "string": + return "" + case "[]byte": + return []byte{} + case "time.Time": + return time.Time{} + case "json.Number": + return json.Number("") + default: + return nil + } +} diff --git a/tests/suites/push/import.yml b/tests/suites/push/import.yml new file mode 100644 index 00000000..139399b2 --- /dev/null +++ b/tests/suites/push/import.yml @@ -0,0 +1,71 @@ +# Copyright (C) 2021 CGI France +# +# This file is part of LINO. +# +# LINO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# LINO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with LINO. If not, see . + +name: import different types +testcases: + - name: prepare test + steps: + # Clean working directory + - script: rm -f * + - script: lino dataconnector add --read-only source 'postgresql://postgres:sakila@source:5432/postgres?sslmode=disable' + - script: lino dataconnector add dest 'postgresql://postgres:sakila@dest:5432/postgres?sslmode=disable' + - script: lino relation extract source + + - name: export then import binary value + steps: + - script: |- + cat > tables.yaml <