From 91ffe46bd2e4d3ebe60429e56e4142dc7b90dd16 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Mon, 28 Oct 2024 08:17:27 -0700 Subject: [PATCH] Date as a separate data type. --- clients/databricks/dialect/typing_test.go | 2 +- clients/shared/default_value.go | 7 +++++++ lib/debezium/converters/date_test.go | 11 +++++------ lib/debezium/schema_test.go | 2 +- .../table_data_merge_columns_test.go | 18 ++++-------------- lib/optimization/table_data_test.go | 4 ++-- lib/parquetutil/parse_values_test.go | 7 +------ lib/typing/ext/parse_test.go | 6 +++--- lib/typing/ext/time.go | 1 - 9 files changed, 24 insertions(+), 34 deletions(-) diff --git a/clients/databricks/dialect/typing_test.go b/clients/databricks/dialect/typing_test.go index 5b964f253..60f55181d 100644 --- a/clients/databricks/dialect/typing_test.go +++ b/clients/databricks/dialect/typing_test.go @@ -115,7 +115,7 @@ func TestDatabricksDialect_KindForDataType(t *testing.T) { // Date kd, err := DatabricksDialect{}.KindForDataType("DATE", "") assert.NoError(t, err) - assert.Equal(t, typing.Date.Kind, kd) + assert.Equal(t, typing.Date, kd) } { // Double diff --git a/clients/shared/default_value.go b/clients/shared/default_value.go index 6f9e8ddb2..4e84a4b13 100644 --- a/clients/shared/default_value.go +++ b/clients/shared/default_value.go @@ -21,6 +21,13 @@ func DefaultValue(column columns.Column, dialect sql.Dialect) (any, error) { switch column.KindDetails.Kind { case typing.Struct.Kind, typing.Array.Kind: return dialect.EscapeStruct(fmt.Sprint(column.DefaultValue())), nil + case typing.Date.Kind: + _time, err := ext.ParseDateFromInterface(column.DefaultValue()) + if err != nil { + return nil, fmt.Errorf("failed to cast colVal as time.Time, colVal: '%v', err: %w", column.DefaultValue(), err) + } + + return sql.QuoteLiteral(_time.Format(ext.PostgresDateFormat)), nil case typing.ETime.Kind: if err := column.KindDetails.EnsureExtendedTimeDetails(); err != nil { return nil, err diff --git a/lib/debezium/converters/date_test.go b/lib/debezium/converters/date_test.go index 639cfd687..f65d83681 100644 --- a/lib/debezium/converters/date_test.go +++ b/lib/debezium/converters/date_test.go @@ -2,8 +2,7 @@ package converters import ( "testing" - - "github.com/artie-labs/transfer/lib/typing/ext" + "time" "github.com/stretchr/testify/assert" ) @@ -18,16 +17,16 @@ func TestDate_Convert(t *testing.T) { val, err := Date{}.Convert(int64(19401)) assert.NoError(t, err) - extTime, isOk := val.(*ext.ExtendedTime) + date, isOk := val.(time.Time) assert.True(t, isOk) - assert.Equal(t, "2023-02-13", extTime.GetTime().Format(Date{}.layout())) + assert.Equal(t, "2023-02-13", date.Format(Date{}.layout())) } { val, err := Date{}.Convert(int64(19429)) assert.NoError(t, err) - extTime, isOk := val.(*ext.ExtendedTime) + date, isOk := val.(time.Time) assert.True(t, isOk) - assert.Equal(t, "2023-03-13", extTime.GetTime().Format(Date{}.layout())) + assert.Equal(t, "2023-03-13", date.Format(Date{}.layout())) } } diff --git a/lib/debezium/schema_test.go b/lib/debezium/schema_test.go index 97a8007e6..a2b2fcab0 100644 --- a/lib/debezium/schema_test.go +++ b/lib/debezium/schema_test.go @@ -249,7 +249,7 @@ func TestField_ToKindDetails(t *testing.T) { for _, dbzType := range []SupportedDebeziumType{Date, DateKafkaConnect} { kd, err := Field{DebeziumType: dbzType}.ToKindDetails() assert.NoError(t, err) - assert.Equal(t, typing.MustNewExtendedTimeDetails(typing.ETime, ext.DateKindType, ext.PostgresDateFormat), kd) + assert.Equal(t, typing.Date, kd) } } { diff --git a/lib/optimization/table_data_merge_columns_test.go b/lib/optimization/table_data_merge_columns_test.go index ac621dc8c..7c73b989f 100644 --- a/lib/optimization/table_data_merge_columns_test.go +++ b/lib/optimization/table_data_merge_columns_test.go @@ -53,21 +53,12 @@ func TestTableData_UpdateInMemoryColumnsFromDestination(t *testing.T) { { // In-memory column is a string and the destination column is a Date tableData.AddInMemoryCol(columns.NewColumn("foo", typing.String)) - - extTime := typing.ETime - nestedKind, err := ext.NewNestedKind(ext.DateKindType, "") - assert.NoError(t, err) - - extTime.ExtendedTimeDetails = &nestedKind - tsCol := columns.NewColumn("foo", extTime) + tsCol := columns.NewColumn("foo", typing.Date) assert.NoError(t, tableData.MergeColumnsFromDestination(tsCol)) updatedColumn, isOk := tableData.inMemoryColumns.GetColumn("foo") assert.True(t, isOk) - assert.Equal(t, typing.ETime.Kind, updatedColumn.KindDetails.Kind) - assert.Equal(t, ext.DateKindType, updatedColumn.KindDetails.ExtendedTimeDetails.Type) - // Format is copied over. - assert.Equal(t, ext.PostgresDateFormat, updatedColumn.KindDetails.ExtendedTimeDetails.Format) + assert.Equal(t, typing.Date, updatedColumn.KindDetails) } { // In-memory column is NUMERIC and destination column is an INTEGER @@ -138,13 +129,12 @@ func TestTableData_UpdateInMemoryColumnsFromDestination(t *testing.T) { } assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("ext_time", typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "")))) - assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("ext_date", typing.MustNewExtendedTimeDetails(typing.ETime, ext.DateKindType, "")))) + assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("ext_date", typing.Date))) assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("ext_datetime", typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")))) dateCol, isOk := tableData.inMemoryColumns.GetColumn("ext_date") assert.True(t, isOk) - assert.NotNil(t, dateCol.KindDetails.ExtendedTimeDetails) - assert.Equal(t, ext.DateKindType, dateCol.KindDetails.ExtendedTimeDetails.Type) + assert.Equal(t, typing.Date, dateCol.KindDetails) timeCol, isOk := tableData.inMemoryColumns.GetColumn("ext_time") assert.True(t, isOk) diff --git a/lib/optimization/table_data_test.go b/lib/optimization/table_data_test.go index 056f2234f..ff0800849 100644 --- a/lib/optimization/table_data_test.go +++ b/lib/optimization/table_data_test.go @@ -134,7 +134,7 @@ func TestTableData_UpdateInMemoryColumns(t *testing.T) { "FOO": typing.String, "bar": typing.Invalid, "CHANGE_me": typing.String, - "do_not_change_format": typing.MustNewExtendedTimeDetails(typing.ETime, ext.DateKindType, ""), + "do_not_change_format": typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), } { _cols.AddColumn(columns.NewColumn(colName, colKind)) } @@ -155,7 +155,7 @@ func TestTableData_UpdateInMemoryColumns(t *testing.T) { "bar": typing.Boolean, "do_not_change_format": typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), } { - tableData.MergeColumnsFromDestination(columns.NewColumn(name, colKindDetails)) + assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn(name, colKindDetails))) } // It's saved back in the original format. diff --git a/lib/parquetutil/parse_values_test.go b/lib/parquetutil/parse_values_test.go index ddf2cce0e..fd5e20124 100644 --- a/lib/parquetutil/parse_values_test.go +++ b/lib/parquetutil/parse_values_test.go @@ -68,12 +68,7 @@ func TestParseValue(t *testing.T) { } { // Date - eDate := typing.ETime - nestedKind, err := ext.NewNestedKind(ext.DateKindType, "") - assert.NoError(t, err) - - eDate.ExtendedTimeDetails = &nestedKind - value, err := ParseValue("2022-12-25", columns.NewColumn("", eDate)) + value, err := ParseValue("2022-12-25", columns.NewColumn("", typing.Date)) assert.NoError(t, err) assert.Equal(t, "2022-12-25", value) } diff --git a/lib/typing/ext/parse_test.go b/lib/typing/ext/parse_test.go index d86f9efe1..ef03590c4 100644 --- a/lib/typing/ext/parse_test.go +++ b/lib/typing/ext/parse_test.go @@ -11,7 +11,7 @@ func TestParseFromInterface(t *testing.T) { { // Extended time var vals []*ExtendedTime - vals = append(vals, NewExtendedTime(time.Now().UTC(), DateKindType, PostgresDateFormat)) + vals = append(vals, NewExtendedTime(time.Now().UTC(), TimestampNTZKindType, RFC3339NoTZ)) vals = append(vals, NewExtendedTime(time.Now().UTC(), TimestampTZKindType, ISO8601)) vals = append(vals, NewExtendedTime(time.Now().UTC(), TimeKindType, PostgresTimeFormat)) @@ -74,10 +74,10 @@ func TestParseFromInterfaceTime(t *testing.T) { } } -func TestParseFromInterfaceDate(t *testing.T) { +func TestParseDateFromInterface(t *testing.T) { now := time.Now() for _, supportedDateFormat := range supportedDateFormats { - _time, err := ParseFromInterface(now.Format(supportedDateFormat), DateKindType) + _time, err := ParseDateFromInterface(now.Format(supportedDateFormat)) assert.NoError(t, err) assert.Equal(t, _time.Format(supportedDateFormat), now.Format(supportedDateFormat)) } diff --git a/lib/typing/ext/time.go b/lib/typing/ext/time.go index 5b350b994..d338e00e3 100644 --- a/lib/typing/ext/time.go +++ b/lib/typing/ext/time.go @@ -11,7 +11,6 @@ type ExtendedTimeKindType string const ( TimestampTZKindType ExtendedTimeKindType = "timestamp_tz" TimestampNTZKindType ExtendedTimeKindType = "timestamp_ntz" - DateKindType ExtendedTimeKindType = "date" TimeKindType ExtendedTimeKindType = "time" )