Date as a separate data type.

artie-labs · Oct 28, 2024 · 91ffe46 · 91ffe46
1 parent 31eeba7
commit 91ffe46
Show file tree

Hide file tree

Showing 9 changed files with 24 additions and 34 deletions.
diff --git a/clients/databricks/dialect/typing_test.go b/clients/databricks/dialect/typing_test.go
@@ -115,7 +115,7 @@ func TestDatabricksDialect_KindForDataType(t *testing.T) {
 		// Date
 		kd, err := DatabricksDialect{}.KindForDataType("DATE", "")
 		assert.NoError(t, err)
-		assert.Equal(t, typing.Date.Kind, kd)
+		assert.Equal(t, typing.Date, kd)
 	}
 	{
 		// Double

diff --git a/clients/shared/default_value.go b/clients/shared/default_value.go
@@ -21,6 +21,13 @@ func DefaultValue(column columns.Column, dialect sql.Dialect) (any, error) {
 	switch column.KindDetails.Kind {
 	case typing.Struct.Kind, typing.Array.Kind:
 		return dialect.EscapeStruct(fmt.Sprint(column.DefaultValue())), nil
+	case typing.Date.Kind:
+		_time, err := ext.ParseDateFromInterface(column.DefaultValue())
+		if err != nil {
+			return nil, fmt.Errorf("failed to cast colVal as time.Time, colVal: '%v', err: %w", column.DefaultValue(), err)
+		}
+
+		return sql.QuoteLiteral(_time.Format(ext.PostgresDateFormat)), nil
 	case typing.ETime.Kind:
 		if err := column.KindDetails.EnsureExtendedTimeDetails(); err != nil {
 			return nil, err

diff --git a/lib/debezium/converters/date_test.go b/lib/debezium/converters/date_test.go
@@ -2,8 +2,7 @@ package converters
 
 import (
 	"testing"
-
-	"github.com/artie-labs/transfer/lib/typing/ext"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -18,16 +17,16 @@ func TestDate_Convert(t *testing.T) {
 		val, err := Date{}.Convert(int64(19401))
 		assert.NoError(t, err)
 
-		extTime, isOk := val.(*ext.ExtendedTime)
+		date, isOk := val.(time.Time)
 		assert.True(t, isOk)
-		assert.Equal(t, "2023-02-13", extTime.GetTime().Format(Date{}.layout()))
+		assert.Equal(t, "2023-02-13", date.Format(Date{}.layout()))
 	}
 	{
 		val, err := Date{}.Convert(int64(19429))
 		assert.NoError(t, err)
 
-		extTime, isOk := val.(*ext.ExtendedTime)
+		date, isOk := val.(time.Time)
 		assert.True(t, isOk)
-		assert.Equal(t, "2023-03-13", extTime.GetTime().Format(Date{}.layout()))
+		assert.Equal(t, "2023-03-13", date.Format(Date{}.layout()))
 	}
 }
diff --git a/lib/debezium/schema_test.go b/lib/debezium/schema_test.go
@@ -249,7 +249,7 @@ func TestField_ToKindDetails(t *testing.T) {
 		for _, dbzType := range []SupportedDebeziumType{Date, DateKafkaConnect} {
 			kd, err := Field{DebeziumType: dbzType}.ToKindDetails()
 			assert.NoError(t, err)
-			assert.Equal(t, typing.MustNewExtendedTimeDetails(typing.ETime, ext.DateKindType, ext.PostgresDateFormat), kd)
+			assert.Equal(t, typing.Date, kd)
 		}
 	}
 	{

diff --git a/lib/optimization/table_data_merge_columns_test.go b/lib/optimization/table_data_merge_columns_test.go
@@ -53,21 +53,12 @@ func TestTableData_UpdateInMemoryColumnsFromDestination(t *testing.T) {
 	{
 		// In-memory column is a string and the destination column is a Date
 		tableData.AddInMemoryCol(columns.NewColumn("foo", typing.String))
-
-		extTime := typing.ETime
-		nestedKind, err := ext.NewNestedKind(ext.DateKindType, "")
-		assert.NoError(t, err)
-
-		extTime.ExtendedTimeDetails = &nestedKind
-		tsCol := columns.NewColumn("foo", extTime)
+		tsCol := columns.NewColumn("foo", typing.Date)
 		assert.NoError(t, tableData.MergeColumnsFromDestination(tsCol))
 
 		updatedColumn, isOk := tableData.inMemoryColumns.GetColumn("foo")
 		assert.True(t, isOk)
-		assert.Equal(t, typing.ETime.Kind, updatedColumn.KindDetails.Kind)
-		assert.Equal(t, ext.DateKindType, updatedColumn.KindDetails.ExtendedTimeDetails.Type)
-		// Format is copied over.
-		assert.Equal(t, ext.PostgresDateFormat, updatedColumn.KindDetails.ExtendedTimeDetails.Format)
+		assert.Equal(t, typing.Date, updatedColumn.KindDetails)
 	}
 	{
 		// In-memory column is NUMERIC and destination column is an INTEGER
@@ -138,13 +129,12 @@ func TestTableData_UpdateInMemoryColumnsFromDestination(t *testing.T) {
 		}
 
 		assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("ext_time", typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, ""))))
-		assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("ext_date", typing.MustNewExtendedTimeDetails(typing.ETime, ext.DateKindType, ""))))
+		assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("ext_date", typing.Date)))
 		assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("ext_datetime", typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""))))
 
 		dateCol, isOk := tableData.inMemoryColumns.GetColumn("ext_date")
 		assert.True(t, isOk)
-		assert.NotNil(t, dateCol.KindDetails.ExtendedTimeDetails)
-		assert.Equal(t, ext.DateKindType, dateCol.KindDetails.ExtendedTimeDetails.Type)
+		assert.Equal(t, typing.Date, dateCol.KindDetails)
 
 		timeCol, isOk := tableData.inMemoryColumns.GetColumn("ext_time")
 		assert.True(t, isOk)

diff --git a/lib/optimization/table_data_test.go b/lib/optimization/table_data_test.go
@@ -134,7 +134,7 @@ func TestTableData_UpdateInMemoryColumns(t *testing.T) {
 		"FOO":                  typing.String,
 		"bar":                  typing.Invalid,
 		"CHANGE_me":            typing.String,
-		"do_not_change_format": typing.MustNewExtendedTimeDetails(typing.ETime, ext.DateKindType, ""),
+		"do_not_change_format": typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""),
 	} {
 		_cols.AddColumn(columns.NewColumn(colName, colKind))
 	}
@@ -155,7 +155,7 @@ func TestTableData_UpdateInMemoryColumns(t *testing.T) {
 		"bar":                  typing.Boolean,
 		"do_not_change_format": typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""),
 	} {
-		tableData.MergeColumnsFromDestination(columns.NewColumn(name, colKindDetails))
+		assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn(name, colKindDetails)))
 	}
 
 	// It's saved back in the original format.

diff --git a/lib/parquetutil/parse_values_test.go b/lib/parquetutil/parse_values_test.go
@@ -68,12 +68,7 @@ func TestParseValue(t *testing.T) {
 	}
 	{
 		// Date
-		eDate := typing.ETime
-		nestedKind, err := ext.NewNestedKind(ext.DateKindType, "")
-		assert.NoError(t, err)
-
-		eDate.ExtendedTimeDetails = &nestedKind
-		value, err := ParseValue("2022-12-25", columns.NewColumn("", eDate))
+		value, err := ParseValue("2022-12-25", columns.NewColumn("", typing.Date))
 		assert.NoError(t, err)
 		assert.Equal(t, "2022-12-25", value)
 	}

diff --git a/lib/typing/ext/parse_test.go b/lib/typing/ext/parse_test.go
@@ -11,7 +11,7 @@ func TestParseFromInterface(t *testing.T) {
 	{
 		// Extended time
 		var vals []*ExtendedTime
-		vals = append(vals, NewExtendedTime(time.Now().UTC(), DateKindType, PostgresDateFormat))
+		vals = append(vals, NewExtendedTime(time.Now().UTC(), TimestampNTZKindType, RFC3339NoTZ))
 		vals = append(vals, NewExtendedTime(time.Now().UTC(), TimestampTZKindType, ISO8601))
 		vals = append(vals, NewExtendedTime(time.Now().UTC(), TimeKindType, PostgresTimeFormat))
 
@@ -74,10 +74,10 @@ func TestParseFromInterfaceTime(t *testing.T) {
 	}
 }
 
-func TestParseFromInterfaceDate(t *testing.T) {
+func TestParseDateFromInterface(t *testing.T) {
 	now := time.Now()
 	for _, supportedDateFormat := range supportedDateFormats {
-		_time, err := ParseFromInterface(now.Format(supportedDateFormat), DateKindType)
+		_time, err := ParseDateFromInterface(now.Format(supportedDateFormat))
 		assert.NoError(t, err)
 		assert.Equal(t, _time.Format(supportedDateFormat), now.Format(supportedDateFormat))
 	}

diff --git a/lib/typing/ext/time.go b/lib/typing/ext/time.go
@@ -11,7 +11,6 @@ type ExtendedTimeKindType string
 const (
 	TimestampTZKindType  ExtendedTimeKindType = "timestamp_tz"
 	TimestampNTZKindType ExtendedTimeKindType = "timestamp_ntz"
-	DateKindType         ExtendedTimeKindType = "date"
 	TimeKindType         ExtendedTimeKindType = "time"
 )