From 03d52d8eeb0dc590615d00f1bc92ab7cab88e99d Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Thu, 17 Oct 2024 10:51:09 -0700 Subject: [PATCH] WIP. --- clients/bigquery/dialect/dialect_test.go | 35 ++++++++++++++++++----- clients/databricks/dialect/typing_test.go | 10 +++++-- clients/mssql/dialect/dialect_test.go | 20 ++++++++++--- clients/snowflake/dialect/dialect_test.go | 15 ++++++++-- lib/debezium/converters/basic.go | 4 +-- lib/debezium/converters/decimal.go | 8 +++--- lib/debezium/converters/geometry.go | 4 +-- 7 files changed, 72 insertions(+), 24 deletions(-) diff --git a/clients/bigquery/dialect/dialect_test.go b/clients/bigquery/dialect/dialect_test.go index 11af8b0bb..e05483759 100644 --- a/clients/bigquery/dialect/dialect_test.go +++ b/clients/bigquery/dialect/dialect_test.go @@ -65,6 +65,18 @@ func TestBigQueryDialect_KindForDataType(t *testing.T) { } } + _datetime, err := typing.NewTimeDetailsFromTemplate(typing.ETime, ext.TimestampTzKindType, "") + assert.NoError(t, err) + + _timestamp, err := typing.NewTimeDetailsFromTemplate(typing.ETime, ext.TimestampTzKindType, "") + assert.NoError(t, err) + + _time, err := typing.NewTimeDetailsFromTemplate(typing.ETime, ext.TimeKindType, "") + assert.NoError(t, err) + + _date, err := typing.NewTimeDetailsFromTemplate(typing.ETime, ext.DateKindType, "") + assert.NoError(t, err) + bqColToExpectedKind := map[string]typing.KindDetails{ // Number "numeric": typing.EDecimal, @@ -92,10 +104,10 @@ func TestBigQueryDialect_KindForDataType(t *testing.T) { "record": typing.Struct, "json": typing.Struct, // Datetime - "datetime": typing.NewTimeDetailsFromTemplate(typing.ETime, ext.TimestampTzKindType), - "timestamp": typing.NewTimeDetailsFromTemplate(typing.ETime, ext.TimestampTzKindType), - "time": typing.NewTimeDetailsFromTemplate(typing.ETime, ext.TimeKindType), - "date": typing.NewTimeDetailsFromTemplate(typing.ETime, ext.DateKindType), + "datetime": _datetime, + "timestamp": _timestamp, + "time": _time, + "date": _date, //Invalid "foo": typing.Invalid, "foofoo": typing.Invalid, @@ -129,10 +141,19 @@ func TestBigQueryDialect_KindForDataType(t *testing.T) { } func TestBigQueryDialect_KindForDataType_NoDataLoss(t *testing.T) { + _timestampTZ, err := typing.NewTimeDetailsFromTemplate(typing.ETime, ext.TimestampTzKindType, "") + assert.NoError(t, err) + + _time, err := typing.NewTimeDetailsFromTemplate(typing.ETime, ext.TimeKindType, "") + assert.NoError(t, err) + + _date, err := typing.NewTimeDetailsFromTemplate(typing.ETime, ext.DateKindType, "") + assert.NoError(t, err) + kindDetails := []typing.KindDetails{ - typing.NewTimeDetailsFromTemplate(typing.ETime, ext.TimestampTzKindType), - typing.NewTimeDetailsFromTemplate(typing.ETime, ext.TimeKindType), - typing.NewTimeDetailsFromTemplate(typing.ETime, ext.DateKindType), + _timestampTZ, + _time, + _date, typing.String, typing.Boolean, typing.Struct, diff --git a/clients/databricks/dialect/typing_test.go b/clients/databricks/dialect/typing_test.go index b6c168ec7..8f50c26ac 100644 --- a/clients/databricks/dialect/typing_test.go +++ b/clients/databricks/dialect/typing_test.go @@ -145,13 +145,19 @@ func TestDatabricksDialect_KindForDataType(t *testing.T) { // Timestamp kd, err := DatabricksDialect{}.KindForDataType("TIMESTAMP", "") assert.NoError(t, err) - assert.Equal(t, typing.NewTimeDetailsFromTemplate(typing.ETime, ext.TimestampTzKindType), kd) + + _kd, err := typing.NewTimeDetailsFromTemplate(typing.ETime, ext.TimestampTzKindType, "") + assert.NoError(t, err) + assert.Equal(t, _kd, kd) } { // Timestamp NTZ kd, err := DatabricksDialect{}.KindForDataType("TIMESTAMP_NTZ", "") assert.NoError(t, err) - assert.Equal(t, typing.NewTimeDetailsFromTemplate(typing.ETime, ext.TimestampTzKindType), kd) + + _kd, err := typing.NewTimeDetailsFromTemplate(typing.ETime, ext.TimestampTzKindType, "") + assert.NoError(t, err) + assert.Equal(t, _kd, kd) } { // Variant diff --git a/clients/mssql/dialect/dialect_test.go b/clients/mssql/dialect/dialect_test.go index 47ab14329..1b0419edd 100644 --- a/clients/mssql/dialect/dialect_test.go +++ b/clients/mssql/dialect/dialect_test.go @@ -49,6 +49,18 @@ func TestMSSQLDialect_DataTypeForKind(t *testing.T) { func TestMSSQLDialect_KindForDataType(t *testing.T) { dialect := MSSQLDialect{} + _date, err := typing.NewTimeDetailsFromTemplate(typing.ETime, ext.DateKindType, "") + assert.NoError(t, err) + + _time, err := typing.NewTimeDetailsFromTemplate(typing.ETime, ext.TimeKindType, "") + assert.NoError(t, err) + + _datetime, err := typing.NewTimeDetailsFromTemplate(typing.ETime, ext.TimestampTzKindType, "") + assert.NoError(t, err) + + _datetime2, err := typing.NewTimeDetailsFromTemplate(typing.ETime, ext.TimestampTzKindType, "") + assert.NoError(t, err) + colToExpectedKind := map[string]typing.KindDetails{ "char": typing.String, "varchar": typing.String, @@ -62,10 +74,10 @@ func TestMSSQLDialect_KindForDataType(t *testing.T) { "float": typing.Float, "real": typing.Float, "bit": typing.Boolean, - "date": typing.NewTimeDetailsFromTemplate(typing.ETime, ext.DateKindType), - "time": typing.NewTimeDetailsFromTemplate(typing.ETime, ext.TimeKindType), - "datetime": typing.NewTimeDetailsFromTemplate(typing.ETime, ext.TimestampTzKindType), - "datetime2": typing.NewTimeDetailsFromTemplate(typing.ETime, ext.TimestampTzKindType), + "date": _date, + "time": _time, + "datetime": _datetime, + "datetime2": _datetime2, } for col, expectedKind := range colToExpectedKind { diff --git a/clients/snowflake/dialect/dialect_test.go b/clients/snowflake/dialect/dialect_test.go index c360733e7..e5ec8b9bd 100644 --- a/clients/snowflake/dialect/dialect_test.go +++ b/clients/snowflake/dialect/dialect_test.go @@ -183,10 +183,19 @@ func TestSnowflakeDialect_KindForDataType_DateTime(t *testing.T) { } func TestSnowflakeDialect_KindForDataType_NoDataLoss(t *testing.T) { + _timestampTZ, err := typing.NewTimeDetailsFromTemplate(typing.ETime, ext.TimestampTzKindType, "") + assert.NoError(t, err) + + _time, err := typing.NewTimeDetailsFromTemplate(typing.ETime, ext.TimeKindType, "") + assert.NoError(t, err) + + _date, err := typing.NewTimeDetailsFromTemplate(typing.ETime, ext.DateKindType, "") + assert.NoError(t, err) + kindDetails := []typing.KindDetails{ - typing.NewTimeDetailsFromTemplate(typing.ETime, ext.TimestampTzKindType), - typing.NewTimeDetailsFromTemplate(typing.ETime, ext.TimeKindType), - typing.NewTimeDetailsFromTemplate(typing.ETime, ext.DateKindType), + _timestampTZ, + _time, + _date, typing.String, typing.Boolean, typing.Struct, diff --git a/lib/debezium/converters/basic.go b/lib/debezium/converters/basic.go index 7304233f9..4282bfb6f 100644 --- a/lib/debezium/converters/basic.go +++ b/lib/debezium/converters/basic.go @@ -24,8 +24,8 @@ func (JSON) Convert(value any) (any, error) { return jsonutil.SanitizePayload(valueString) } -func (JSON) ToKindDetails() typing.KindDetails { - return typing.Struct +func (JSON) ToKindDetails() (typing.KindDetails, error) { + return typing.Struct, nil } type Int64Passthrough struct{} diff --git a/lib/debezium/converters/decimal.go b/lib/debezium/converters/decimal.go index b5c26739d..870f4bcc9 100644 --- a/lib/debezium/converters/decimal.go +++ b/lib/debezium/converters/decimal.go @@ -86,8 +86,8 @@ type VariableDecimal struct { details decimal.Details } -func (v VariableDecimal) ToKindDetails() typing.KindDetails { - return typing.NewDecimalDetailsFromTemplate(typing.EDecimal, v.details) +func (v VariableDecimal) ToKindDetails() (typing.KindDetails, error) { + return typing.NewDecimalDetailsFromTemplate(typing.EDecimal, v.details), nil } func (v VariableDecimal) Convert(value any) (any, error) { @@ -134,8 +134,8 @@ func NewDecimal(details decimal.Details) Decimal { return Decimal{details: details} } -func (d Decimal) ToKindDetails() typing.KindDetails { - return typing.NewDecimalDetailsFromTemplate(typing.EDecimal, d.details) +func (d Decimal) ToKindDetails() (typing.KindDetails, error) { + return typing.NewDecimalDetailsFromTemplate(typing.EDecimal, d.details), nil } func (d Decimal) Convert(value any) (any, error) { diff --git a/lib/debezium/converters/geometry.go b/lib/debezium/converters/geometry.go index 2adf6c6a7..92fe0aced 100644 --- a/lib/debezium/converters/geometry.go +++ b/lib/debezium/converters/geometry.go @@ -71,9 +71,9 @@ func (GeometryPoint) Convert(value any) (any, error) { type Geometry struct{} -func (Geometry) ToKindDetails() typing.KindDetails { +func (Geometry) ToKindDetails() (typing.KindDetails, error) { // We will return this in GeoJSON format. - return typing.Struct + return typing.Struct, nil } func (Geometry) Convert(value any) (any, error) {