feat(push): add import types for push (#66)

* feat(push): add import types for push

* fix(import): read correct value

* fix(import): add venom test

* fix(import): fix venom test

* feat(import): switch raw type

* fix(import): fix venom test

* fix(import): lint and error checking

* feat(import): use correct value for import

* test(import): fix venom test + changelog

Co-authored-by: Adrien Aury <[email protected]>
Co-authored-by: Adrien Aury <[email protected]>

CHANGELOG.md

@@ -17,7 +17,7 @@ Types of changes
 ## [2.0.0] (Ureleased)
 
 - `Changed` order of keys in output JSON lines will be alphabetical when pulling (without configuration in tables.yaml)
-- `Added` configuration of exported columns in tables.yaml, see issue #33 for more information
+- `Added` configuration of export format / import type for columns in tables.yaml, see issue #33 for more information
 - `Added` MariaDB/MySQL support (thanks to @joaking85)
 - `Added` auto-select columns required by a relation but not exported in tables.yaml
 - `Added` new commands to configure tables : add-column and remove-column

internal/app/push/cli.go

@@ -242,7 +242,7 @@ func (c idToPushConverter) getTable(name string) push.Table {
 
 	columns := []push.Column{}
 	for _, col := range table.Columns {
-		columns = append(columns, push.NewColumn(col.Name, col.Import))
+		columns = append(columns, push.NewColumn(col.Name, col.Export, col.Import))
 	}
 
 	return push.NewTable(table.Name, table.Keys, push.NewColumnList(columns))

internal/infra/push/datadestination_sql.go

@@ -314,9 +314,14 @@ func (rw *SQLRowWriter) Write(row push.Row) *push.Error {
 		return err1
 	}
 
+	importedRow, err15 := rw.table.Import(row)
+	if err15 != nil {
+		return err15
+	}
+
 	values := []interface{}{}
 	for _, h := range rw.headers {
-		values = append(values, rw.dd.dialect.ConvertValue(row[h]))
+		values = append(values, rw.dd.dialect.ConvertValue(importedRow.GetOrNil(h)))
 	}
 	log.Trace().Strs("headers", rw.headers).Msg(fmt.Sprint(values))
 

pkg/push/model.go

@@ -29,6 +29,7 @@ type Table interface {
 	Name() string
 	PrimaryKey() []string
 	Columns() ColumnList
+	Import(map[string]interface{}) (ImportedRow, *Error)
 }
 
 // ColumnList is a list of columns.
@@ -40,6 +41,7 @@ type ColumnList interface {
 // Column of a table.
 type Column interface {
 	Name() string
+	Export() string
 	Import() string
 }
 
@@ -62,7 +64,7 @@ type Relation interface {
 type Value interface{}
 
 // Row of data.
-type Row map[string]Value
+type Row map[string]interface{}
 
 // Error is the error type returned by the domain
 type Error struct {

pkg/push/model_table.go

@@ -18,14 +18,20 @@
 package push
 
 import (
+	"encoding/json"
 	"fmt"
 	"strings"
+	"time"
+
+	"github.com/cgi-fr/jsonline/pkg/jsonline"
 )
 
 type table struct {
 	name    string
 	pk      []string
 	columns ColumnList
+
+	template jsonline.Template
 }
 
 // NewTable initialize a new Table object
@@ -67,13 +73,109 @@ func (l columnList) String() string {
 
 type column struct {
 	name string
+	exp  string
 	imp  string
 }
 
 // NewColumn initialize a new Column object
-func NewColumn(name string, imp string) Column {
-	return column{name, imp}
+func NewColumn(name string, exp string, imp string) Column {
+	return column{name, exp, imp}
 }
 
 func (c column) Name() string   { return c.name }
+func (c column) Export() string { return c.exp }
 func (c column) Import() string { return c.imp }
+
+type ImportedRow struct {
+	jsonline.Row
+}
+
+func (t *table) initTemplate() {
+	t.template = jsonline.NewTemplate()
+
+	if t.columns == nil {
+		return
+	}
+
+	if l := int(t.columns.Len()); l > 0 {
+		for idx := 0; idx < l; idx++ {
+			col := t.columns.Column(uint(idx))
+			key := col.Name()
+
+			switch col.Export() {
+			case "string":
+				t.template.WithMappedString(key, parseImportType(col.Import()))
+			case "numeric":
+				t.template.WithMappedNumeric(key, parseImportType(col.Import()))
+			case "base64", "binary":
+				t.template.WithMappedBinary(key, parseImportType(col.Import()))
+			case "datetime":
+				t.template.WithMappedDateTime(key, parseImportType(col.Import()))
+			case "timestamp":
+				t.template.WithMappedTimestamp(key, parseImportType(col.Import()))
+			case "no":
+				t.template.WithHidden(key)
+			default:
+				t.template.WithMappedAuto(key, parseImportType(col.Import()))
+			}
+		}
+	}
+}
+
+func (t table) Import(row map[string]interface{}) (ImportedRow, *Error) {
+	if t.template == nil {
+		t.initTemplate()
+	}
+
+	result := ImportedRow{t.template.CreateRowEmpty()}
+	if err := result.Import(row); err != nil {
+		return ImportedRow{}, &Error{Description: err.Error()}
+	}
+
+	return result, nil
+}
+
+func parseImportType(exp string) jsonline.RawType {
+	switch exp {
+	case "int":
+		return int(0)
+	case "int64":
+		return int64(0)
+	case "int32":
+		return int32(0)
+	case "int16":
+		return int16(0)
+	case "int8":
+		return int8(0)
+	case "uint":
+		return uint(0)
+	case "uint64":
+		return uint64(0)
+	case "uint32":
+		return uint32(0)
+	case "uint16":
+		return uint16(0)
+	case "uint8":
+		return uint8(0)
+	case "float64":
+		return float64(0)
+	case "float32":
+		return float32(0)
+	case "bool":
+		return false
+	case "byte":
+		return byte(0)
+	case "rune":
+		return rune(' ')
+	case "string":
+		return ""
+	case "[]byte":
+		return []byte{}
+	case "time.Time":
+		return time.Time{}
+	case "json.Number":
+		return json.Number("")
+	default:
+		return nil
+	}
+}

tests/suites/push/import.yml

@@ -0,0 +1,71 @@
+# Copyright (C) 2021 CGI France
+#
+# This file is part of LINO.
+#
+# LINO is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# LINO is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with LINO.  If not, see <http:#www.gnu.org/licenses/>.
+
+name: import different types
+testcases:
+  - name: prepare test
+    steps:
+      # Clean working directory
+      - script: rm -f *
+      - script: lino dataconnector add --read-only source 'postgresql://postgres:sakila@source:5432/postgres?sslmode=disable'
+      - script: lino dataconnector add dest 'postgresql://postgres:sakila@dest:5432/postgres?sslmode=disable'
+      - script: lino relation extract source
+
+  - name: export then import binary value
+    steps:
+      - script: |-
+          cat  > tables.yaml <<EOF
+          version: v1
+          tables:
+            - name: "staff"
+              keys:
+                - "staff_id"
+              columns:
+                - name: "staff_id"
+                - name: "picture"
+                  export: "base64"
+          EOF
+      - script: lino pull source --filter staff_id=1 --table staff | lino push update dest --table staff
+      - script: lino pull dest --filter staff_id=1 --table staff
+        assertions:
+          - result.systemerr ShouldBeEmpty
+          - result.code ShouldEqual 0
+          - result.systemout ShouldEqual {"staff_id":1,"picture":"iVBORw0KWgo="}
+
+  - name: export to different format than stored in datasource
+    steps:
+      - script: lino table extract source
+      - script: lino pull source --table customer --limit 0 | lino push truncate dest --table customer
+      - script: |-
+          cat  > tables.yaml <<EOF
+          version: v1
+          tables:
+            - name: customer
+              keys:
+                - customer_id
+              columns:
+                - name: "customer_id"
+                - name: "create_date"
+                  export: "timestamp"
+                  import: "time.Time"
+          EOF
+      - script: lino pull source --table customer --filter customer_id=1 | lino push update dest --table customer
+      - script: lino pull dest --table customer --filter customer_id=1
+        assertions:
+          - result.systemerr ShouldBeEmpty
+          - result.code ShouldEqual 0
+          - result.systemout ShouldEqual {"customer_id":1,"create_date":1139875200}