From 79210b1c85a2feb03db5ce3b47fa42a76f81bd4d Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Wed, 23 Oct 2024 11:51:43 -0700 Subject: [PATCH 1/3] Clean up. --- clients/bigquery/dialect/dialect.go | 8 +++---- clients/bigquery/dialect/dialect_test.go | 14 ++++++------ clients/bigquery/storagewrite_test.go | 19 +++++++++------- clients/databricks/dialect/typing.go | 6 ++--- clients/databricks/dialect/typing_test.go | 12 +++++++--- clients/mssql/dialect/dialect.go | 8 +++---- clients/mssql/dialect/dialect_test.go | 8 +++---- clients/redshift/dialect/typing.go | 8 +++---- clients/redshift/dialect/typing_test.go | 4 ++-- clients/snowflake/ddl_test.go | 6 ++--- clients/snowflake/dialect/dialect.go | 8 +++---- clients/snowflake/dialect/dialect_test.go | 6 ++--- clients/snowflake/snowflake_test.go | 2 +- lib/debezium/converters/date.go | 2 +- lib/debezium/converters/time.go | 10 ++++----- lib/debezium/converters/time_test.go | 4 ++-- lib/debezium/converters/timestamp.go | 6 ++--- lib/debezium/converters/timestamp_test.go | 6 ++--- lib/debezium/schema_test.go | 12 +++++----- lib/destination/ddl/ddl_sflk_test.go | 8 +++---- .../table_data_merge_columns_test.go | 14 ++++++------ lib/optimization/table_data_test.go | 6 ++--- lib/typing/columns/diff_test.go | 4 ++-- lib/typing/typing.go | 22 ++++++++++++++++++- 24 files changed, 116 insertions(+), 87 deletions(-) diff --git a/clients/bigquery/dialect/dialect.go b/clients/bigquery/dialect/dialect.go index 602f51fbc..da3506bc3 100644 --- a/clients/bigquery/dialect/dialect.go +++ b/clients/bigquery/dialect/dialect.go @@ -106,13 +106,13 @@ func (BigQueryDialect) KindForDataType(rawBqType string, _ string) (typing.KindD case "array": return typing.Array, nil case "timestamp": - return typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), nil + return typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "") case "datetime": - return typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, ""), nil + return typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, "") case "time": - return typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, ""), nil + return typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "") case "date": - return typing.NewExtendedTimeDetails(typing.ETime, ext.DateKindType, ""), nil + return typing.NewExtendedTimeDetails(typing.ETime, ext.DateKindType, "") default: return typing.Invalid, nil } diff --git a/clients/bigquery/dialect/dialect_test.go b/clients/bigquery/dialect/dialect_test.go index 60b64320d..1ab1a42e9 100644 --- a/clients/bigquery/dialect/dialect_test.go +++ b/clients/bigquery/dialect/dialect_test.go @@ -92,10 +92,10 @@ func TestBigQueryDialect_KindForDataType(t *testing.T) { "record": typing.Struct, "json": typing.Struct, // Datetime - "datetime": typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), - "timestamp": typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), - "time": typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, ""), - "date": typing.NewExtendedTimeDetails(typing.ETime, ext.DateKindType, ""), + "datetime": typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), + "timestamp": typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), + "time": typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, ""), + "date": typing.MustNewExtendedTimeDetails(typing.ETime, ext.DateKindType, ""), //Invalid "foo": typing.Invalid, "foofoo": typing.Invalid, @@ -130,9 +130,9 @@ func TestBigQueryDialect_KindForDataType(t *testing.T) { func TestBigQueryDialect_KindForDataType_NoDataLoss(t *testing.T) { kindDetails := []typing.KindDetails{ - typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), - typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, ""), - typing.NewExtendedTimeDetails(typing.ETime, ext.DateKindType, ""), + typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), + typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, ""), + typing.MustNewExtendedTimeDetails(typing.ETime, ext.DateKindType, ""), typing.String, typing.Boolean, typing.Struct, diff --git a/clients/bigquery/storagewrite_test.go b/clients/bigquery/storagewrite_test.go index 156b22d14..f0641c7b7 100644 --- a/clients/bigquery/storagewrite_test.go +++ b/clients/bigquery/storagewrite_test.go @@ -44,26 +44,29 @@ func TestColumnToTableFieldSchema(t *testing.T) { } { // ETime - Time: - fieldSchema, err := columnToTableFieldSchema(columns.NewColumn("foo", typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, ""))) + fieldSchema, err := columnToTableFieldSchema(columns.NewColumn("foo", typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, ""))) assert.NoError(t, err) assert.Equal(t, storagepb.TableFieldSchema_TIME, fieldSchema.Type) } { // ETime - Date: - fieldSchema, err := columnToTableFieldSchema(columns.NewColumn("foo", typing.NewExtendedTimeDetails(typing.ETime, ext.DateKindType, ""))) + fieldSchema, err := columnToTableFieldSchema(columns.NewColumn("foo", typing.MustNewExtendedTimeDetails(typing.ETime, ext.DateKindType, ""))) assert.NoError(t, err) assert.Equal(t, storagepb.TableFieldSchema_DATE, fieldSchema.Type) } { // ETime - TimestampTZ: - fieldSchema, err := columnToTableFieldSchema(columns.NewColumn("foo", typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""))) + fieldSchema, err := columnToTableFieldSchema(columns.NewColumn("foo", typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""))) assert.NoError(t, err) assert.Equal(t, storagepb.TableFieldSchema_TIMESTAMP, fieldSchema.Type) } { // ETime - Invalid: - _, err := columnToTableFieldSchema(columns.NewColumn("foo", typing.NewExtendedTimeDetails(typing.ETime, "", ""))) - assert.ErrorContains(t, err, "unsupported extended time details type:") + nestedKind, err := typing.NewExtendedTimeDetails(typing.ETime, "", "") + assert.ErrorContains(t, err, "unknown kind type") + + _, err = columnToTableFieldSchema(columns.NewColumn("foo", nestedKind)) + assert.ErrorContains(t, err, `unsupported column kind: "invalid"`) } { // Struct: @@ -112,9 +115,9 @@ func TestRowToMessage(t *testing.T) { columns.NewColumn("c_numeric", typing.EDecimal), columns.NewColumn("c_string", typing.String), columns.NewColumn("c_string_decimal", typing.String), - columns.NewColumn("c_time", typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "")), - columns.NewColumn("c_date", typing.NewExtendedTimeDetails(typing.ETime, ext.DateKindType, "")), - columns.NewColumn("c_datetime", typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")), + columns.NewColumn("c_time", typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "")), + columns.NewColumn("c_date", typing.MustNewExtendedTimeDetails(typing.ETime, ext.DateKindType, "")), + columns.NewColumn("c_datetime", typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")), columns.NewColumn("c_struct", typing.Struct), columns.NewColumn("c_array", typing.Array), } diff --git a/clients/databricks/dialect/typing.go b/clients/databricks/dialect/typing.go index 036f28b33..ebf2c51b8 100644 --- a/clients/databricks/dialect/typing.go +++ b/clients/databricks/dialect/typing.go @@ -66,7 +66,7 @@ func (DatabricksDialect) KindForDataType(rawType string, _ string) (typing.KindD case "boolean": return typing.Boolean, nil case "date": - return typing.NewExtendedTimeDetails(typing.ETime, ext.DateKindType, ""), nil + return typing.NewExtendedTimeDetails(typing.ETime, ext.DateKindType, "") case "double", "float": return typing.Float, nil case "int": @@ -74,9 +74,9 @@ func (DatabricksDialect) KindForDataType(rawType string, _ string) (typing.KindD case "smallint", "tinyint": return typing.KindDetails{Kind: typing.Integer.Kind, OptionalIntegerKind: typing.ToPtr(typing.SmallIntegerKind)}, nil case "timestamp": - return typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), nil + return typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "") case "timestamp_ntz": - return typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, ""), nil + return typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, "") } return typing.Invalid, fmt.Errorf("unsupported data type: %q", rawType) diff --git a/clients/databricks/dialect/typing_test.go b/clients/databricks/dialect/typing_test.go index 94506096d..14e496109 100644 --- a/clients/databricks/dialect/typing_test.go +++ b/clients/databricks/dialect/typing_test.go @@ -115,7 +115,13 @@ func TestDatabricksDialect_KindForDataType(t *testing.T) { // Date kd, err := DatabricksDialect{}.KindForDataType("DATE", "") assert.NoError(t, err) - assert.Equal(t, typing.KindDetails{Kind: typing.ETime.Kind, ExtendedTimeDetails: &ext.NestedKind{Type: ext.DateKindType}}, kd) + assert.Equal(t, typing.KindDetails{ + Kind: typing.ETime.Kind, + ExtendedTimeDetails: &ext.NestedKind{ + Type: ext.DateKindType, + Format: ext.PostgresDateFormat, + }, + }, kd) } { // Double @@ -145,13 +151,13 @@ func TestDatabricksDialect_KindForDataType(t *testing.T) { // Timestamp kd, err := DatabricksDialect{}.KindForDataType("TIMESTAMP", "") assert.NoError(t, err) - assert.Equal(t, typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), kd) + assert.Equal(t, typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), kd) } { // Timestamp NTZ kd, err := DatabricksDialect{}.KindForDataType("TIMESTAMP_NTZ", "") assert.NoError(t, err) - assert.Equal(t, typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, ""), kd) + assert.Equal(t, typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, ""), kd) } { // Variant diff --git a/clients/mssql/dialect/dialect.go b/clients/mssql/dialect/dialect.go index 8556d8e49..ba5e3a5d7 100644 --- a/clients/mssql/dialect/dialect.go +++ b/clients/mssql/dialect/dialect.go @@ -116,13 +116,13 @@ func (MSSQLDialect) KindForDataType(rawType string, stringPrecision string) (typ case "datetime", "datetime2": - return typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, ""), nil + return typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, "") case "datetimeoffset": - return typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), nil + return typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "") case "time": - return typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, ""), nil + return typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "") case "date": - return typing.NewExtendedTimeDetails(typing.ETime, ext.DateKindType, ""), nil + return typing.NewExtendedTimeDetails(typing.ETime, ext.DateKindType, "") case "bit": return typing.Boolean, nil case "text": diff --git a/clients/mssql/dialect/dialect_test.go b/clients/mssql/dialect/dialect_test.go index 343ee6190..a1c522004 100644 --- a/clients/mssql/dialect/dialect_test.go +++ b/clients/mssql/dialect/dialect_test.go @@ -62,10 +62,10 @@ func TestMSSQLDialect_KindForDataType(t *testing.T) { "float": typing.Float, "real": typing.Float, "bit": typing.Boolean, - "date": typing.NewExtendedTimeDetails(typing.ETime, ext.DateKindType, ""), - "time": typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, ""), - "datetime": typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), - "datetime2": typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), + "date": typing.MustNewExtendedTimeDetails(typing.ETime, ext.DateKindType, ""), + "time": typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, ""), + "datetime": typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), + "datetime2": typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), } for col, expectedKind := range colToExpectedKind { diff --git a/clients/redshift/dialect/typing.go b/clients/redshift/dialect/typing.go index b2d9e4567..7039c4357 100644 --- a/clients/redshift/dialect/typing.go +++ b/clients/redshift/dialect/typing.go @@ -106,13 +106,13 @@ func (RedshiftDialect) KindForDataType(rawType string, stringPrecision string) ( case "double precision": return typing.Float, nil case "timestamp", "timestamp without time zone": - return typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, ""), nil + return typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, "") case "timestamp with time zone": - return typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), nil + return typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "") case "time without time zone": - return typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, ""), nil + return typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "") case "date": - return typing.NewExtendedTimeDetails(typing.ETime, ext.DateKindType, ""), nil + return typing.NewExtendedTimeDetails(typing.ETime, ext.DateKindType, "") case "boolean": return typing.Boolean, nil } diff --git a/clients/redshift/dialect/typing_test.go b/clients/redshift/dialect/typing_test.go index caa986a8a..eaa3d3fb0 100644 --- a/clients/redshift/dialect/typing_test.go +++ b/clients/redshift/dialect/typing_test.go @@ -47,11 +47,11 @@ func TestRedshiftDialect_DataTypeForKind(t *testing.T) { // Timestamps { // With timezone - assert.Equal(t, "timestamp with time zone", RedshiftDialect{}.DataTypeForKind(typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), false)) + assert.Equal(t, "timestamp with time zone", RedshiftDialect{}.DataTypeForKind(typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), false)) } { // Without timezone - assert.Equal(t, "timestamp without time zone", RedshiftDialect{}.DataTypeForKind(typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, ""), false)) + assert.Equal(t, "timestamp without time zone", RedshiftDialect{}.DataTypeForKind(typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, ""), false)) } } } diff --git a/clients/snowflake/ddl_test.go b/clients/snowflake/ddl_test.go index 62a4a2f6b..62fff39d7 100644 --- a/clients/snowflake/ddl_test.go +++ b/clients/snowflake/ddl_test.go @@ -29,7 +29,7 @@ func (s *SnowflakeTestSuite) TestMutateColumnsWithMemoryCacheDeletions() { "customer_id": typing.Integer, "price": typing.Float, "name": typing.String, - "created_at": typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), + "created_at": typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), } { cols.AddColumn(columns.NewColumn(colName, kindDetails)) } @@ -58,7 +58,7 @@ func (s *SnowflakeTestSuite) TestShouldDeleteColumn() { "customer_id": typing.Integer, "price": typing.Float, "name": typing.String, - "created_at": typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), + "created_at": typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), } { cols.AddColumn(columns.NewColumn(colName, kindDetails)) } @@ -97,7 +97,7 @@ func (s *SnowflakeTestSuite) TestManipulateShouldDeleteColumn() { "customer_id": typing.Integer, "price": typing.Float, "name": typing.String, - "created_at": typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), + "created_at": typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), } { cols.AddColumn(columns.NewColumn(colName, kindDetails)) } diff --git a/clients/snowflake/dialect/dialect.go b/clients/snowflake/dialect/dialect.go index 25b371539..06f0a98f2 100644 --- a/clients/snowflake/dialect/dialect.go +++ b/clients/snowflake/dialect/dialect.go @@ -98,13 +98,13 @@ func (SnowflakeDialect) KindForDataType(snowflakeType string, _ string) (typing. case "array": return typing.Array, nil case "timestamp_ltz", "timestamp_tz": - return typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), nil + return typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "") case "timestamp", "datetime", "timestamp_ntz": - return typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, ""), nil + return typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, "") case "time": - return typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, ""), nil + return typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "") case "date": - return typing.NewExtendedTimeDetails(typing.ETime, ext.DateKindType, ""), nil + return typing.NewExtendedTimeDetails(typing.ETime, ext.DateKindType, "") default: return typing.Invalid, nil } diff --git a/clients/snowflake/dialect/dialect_test.go b/clients/snowflake/dialect/dialect_test.go index fbf3ce19d..13e5f52a6 100644 --- a/clients/snowflake/dialect/dialect_test.go +++ b/clients/snowflake/dialect/dialect_test.go @@ -196,9 +196,9 @@ func TestSnowflakeDialect_KindForDataType_DateTime(t *testing.T) { func TestSnowflakeDialect_KindForDataType_NoDataLoss(t *testing.T) { kindDetails := []typing.KindDetails{ - typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), - typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, ""), - typing.NewExtendedTimeDetails(typing.ETime, ext.DateKindType, ""), + typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), + typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, ""), + typing.MustNewExtendedTimeDetails(typing.ETime, ext.DateKindType, ""), typing.String, typing.Boolean, typing.Struct, diff --git a/clients/snowflake/snowflake_test.go b/clients/snowflake/snowflake_test.go index 1fa828ba7..720a404c1 100644 --- a/clients/snowflake/snowflake_test.go +++ b/clients/snowflake/snowflake_test.go @@ -240,7 +240,7 @@ func (s *SnowflakeTestSuite) TestExecuteMergeDeletionFlagRemoval() { snowflakeColToKindDetailsMap := map[string]typing.KindDetails{ "id": typing.Integer, - "created_at": typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), + "created_at": typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), "name": typing.String, constants.DeleteColumnMarker: typing.Boolean, constants.OnlySetDeleteColumnMarker: typing.Boolean, diff --git a/lib/debezium/converters/date.go b/lib/debezium/converters/date.go index 1d38255d3..34ccf3209 100644 --- a/lib/debezium/converters/date.go +++ b/lib/debezium/converters/date.go @@ -15,7 +15,7 @@ func (Date) layout() string { } func (d Date) ToKindDetails() typing.KindDetails { - return typing.NewExtendedTimeDetails(typing.ETime, ext.DateKindType, d.layout()) + return typing.OldNewExtendedTimeDetails(typing.ETime, ext.DateKindType, d.layout()) } func (d Date) Convert(value any) (any, error) { diff --git a/lib/debezium/converters/time.go b/lib/debezium/converters/time.go index b8553183b..00972067d 100644 --- a/lib/debezium/converters/time.go +++ b/lib/debezium/converters/time.go @@ -16,7 +16,7 @@ func (Time) layout() string { } func (t Time) ToKindDetails() typing.KindDetails { - return typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, t.layout()) + return typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, t.layout()) } func (t Time) Convert(val any) (any, error) { @@ -36,7 +36,7 @@ func (NanoTime) layout() string { } func (n NanoTime) ToKindDetails() typing.KindDetails { - return typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, n.layout()) + return typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, n.layout()) } func (n NanoTime) Convert(value any) (any, error) { @@ -56,7 +56,7 @@ func (MicroTime) layout() string { } func (m MicroTime) ToKindDetails() typing.KindDetails { - return typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, m.layout()) + return typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, m.layout()) } func (m MicroTime) Convert(value any) (any, error) { @@ -76,7 +76,7 @@ func (ZonedTimestamp) layout() string { } func (z ZonedTimestamp) ToKindDetails() typing.KindDetails { - return typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, z.layout()) + return typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, z.layout()) } func (z ZonedTimestamp) Convert(value any) (any, error) { @@ -112,7 +112,7 @@ func (t TimeWithTimezone) layout() string { } func (t TimeWithTimezone) ToKindDetails() typing.KindDetails { - return typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, t.layout()) + return typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, t.layout()) } func (t TimeWithTimezone) Convert(value any) (any, error) { diff --git a/lib/debezium/converters/time_test.go b/lib/debezium/converters/time_test.go index c42916e46..88b7858db 100644 --- a/lib/debezium/converters/time_test.go +++ b/lib/debezium/converters/time_test.go @@ -143,7 +143,7 @@ func TestTime_Convert(t *testing.T) { } func TestNanoTime_Converter(t *testing.T) { - assert.Equal(t, typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, NanoTime{}.layout()), NanoTime{}.ToKindDetails()) + assert.Equal(t, typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, NanoTime{}.layout()), NanoTime{}.ToKindDetails()) { // Invalid data _, err := NanoTime{}.Convert("123") @@ -158,7 +158,7 @@ func TestNanoTime_Converter(t *testing.T) { } func TestMicroTime_Converter(t *testing.T) { - assert.Equal(t, typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, MicroTime{}.layout()), MicroTime{}.ToKindDetails()) + assert.Equal(t, typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, MicroTime{}.layout()), MicroTime{}.ToKindDetails()) { // Invalid data _, err := MicroTime{}.Convert("123") diff --git a/lib/debezium/converters/timestamp.go b/lib/debezium/converters/timestamp.go index 5caf18750..54c6f8a24 100644 --- a/lib/debezium/converters/timestamp.go +++ b/lib/debezium/converters/timestamp.go @@ -14,7 +14,7 @@ func (Timestamp) layout() string { } func (t Timestamp) ToKindDetails() typing.KindDetails { - return typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, t.layout()) + return typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, t.layout()) } func (t Timestamp) Convert(value any) (any, error) { @@ -34,7 +34,7 @@ func (MicroTimestamp) layout() string { } func (mt MicroTimestamp) ToKindDetails() typing.KindDetails { - return typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, mt.layout()) + return typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, mt.layout()) } func (mt MicroTimestamp) Convert(value any) (any, error) { @@ -50,7 +50,7 @@ func (mt MicroTimestamp) Convert(value any) (any, error) { type NanoTimestamp struct{} func (nt NanoTimestamp) ToKindDetails() typing.KindDetails { - return typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, nt.layout()) + return typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, nt.layout()) } func (NanoTimestamp) layout() string { diff --git a/lib/debezium/converters/timestamp_test.go b/lib/debezium/converters/timestamp_test.go index 2745e04bf..bf6a7e551 100644 --- a/lib/debezium/converters/timestamp_test.go +++ b/lib/debezium/converters/timestamp_test.go @@ -9,7 +9,7 @@ import ( ) func TestTimestamp_Converter(t *testing.T) { - assert.Equal(t, typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, ext.RFC3339MillisecondNoTZ), Timestamp{}.ToKindDetails()) + assert.Equal(t, typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, ext.RFC3339MillisecondNoTZ), Timestamp{}.ToKindDetails()) { // Invalid conversion _, err := Timestamp{}.Convert("invalid") @@ -30,7 +30,7 @@ func TestTimestamp_Converter(t *testing.T) { } func TestMicroTimestamp_Converter(t *testing.T) { - assert.Equal(t, typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, ext.RFC3339MicrosecondNoTZ), MicroTimestamp{}.ToKindDetails()) + assert.Equal(t, typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, ext.RFC3339MicrosecondNoTZ), MicroTimestamp{}.ToKindDetails()) { // Invalid conversion _, err := MicroTimestamp{}.Convert("invalid") @@ -51,7 +51,7 @@ func TestMicroTimestamp_Converter(t *testing.T) { } func TestNanoTimestamp_Converter(t *testing.T) { - assert.Equal(t, typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, ext.RFC3339NanosecondNoTZ), NanoTimestamp{}.ToKindDetails()) + assert.Equal(t, typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, ext.RFC3339NanosecondNoTZ), NanoTimestamp{}.ToKindDetails()) { // Invalid conversion _, err := NanoTimestamp{}.Convert("invalid") diff --git a/lib/debezium/schema_test.go b/lib/debezium/schema_test.go index bcdabb16b..bb955d6ce 100644 --- a/lib/debezium/schema_test.go +++ b/lib/debezium/schema_test.go @@ -233,7 +233,7 @@ func TestField_ToKindDetails(t *testing.T) { for _, dbzType := range []SupportedDebeziumType{ZonedTimestamp} { kd, err := Field{DebeziumType: dbzType}.ToKindDetails() assert.NoError(t, err) - assert.Equal(t, typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "2006-01-02T15:04:05.999999999Z"), kd) + assert.Equal(t, typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "2006-01-02T15:04:05.999999999Z"), kd) } } { @@ -250,7 +250,7 @@ func TestField_ToKindDetails(t *testing.T) { for _, dbzType := range []SupportedDebeziumType{Date, DateKafkaConnect} { kd, err := Field{DebeziumType: dbzType}.ToKindDetails() assert.NoError(t, err) - assert.Equal(t, typing.NewExtendedTimeDetails(typing.ETime, ext.DateKindType, ext.PostgresDateFormat), kd) + assert.Equal(t, typing.OldNewExtendedTimeDetails(typing.ETime, ext.DateKindType, ext.PostgresDateFormat), kd) } } { @@ -259,7 +259,7 @@ func TestField_ToKindDetails(t *testing.T) { for _, dbzType := range []SupportedDebeziumType{TimeWithTimezone} { kd, err := Field{DebeziumType: dbzType}.ToKindDetails() assert.NoError(t, err) - assert.Equal(t, typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "15:04:05.999999Z"), kd, dbzType) + assert.Equal(t, typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "15:04:05.999999Z"), kd, dbzType) } } { @@ -267,20 +267,20 @@ func TestField_ToKindDetails(t *testing.T) { for _, dbzType := range []SupportedDebeziumType{Time, TimeKafkaConnect} { kd, err := Field{DebeziumType: dbzType}.ToKindDetails() assert.NoError(t, err) - assert.Equal(t, typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "15:04:05.000"), kd, dbzType) + assert.Equal(t, typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "15:04:05.000"), kd, dbzType) } } { // Micro time kd, err := Field{DebeziumType: MicroTime}.ToKindDetails() assert.NoError(t, err) - assert.Equal(t, typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "15:04:05.000000"), kd) + assert.Equal(t, typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "15:04:05.000000"), kd) } { // Nano time kd, err := Field{DebeziumType: NanoTime}.ToKindDetails() assert.NoError(t, err) - assert.Equal(t, typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "15:04:05.000000000"), kd) + assert.Equal(t, typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "15:04:05.000000000"), kd) } } { diff --git a/lib/destination/ddl/ddl_sflk_test.go b/lib/destination/ddl/ddl_sflk_test.go index ab6e5e6a8..2a7ce2170 100644 --- a/lib/destination/ddl/ddl_sflk_test.go +++ b/lib/destination/ddl/ddl_sflk_test.go @@ -51,7 +51,7 @@ func (d *DDLTestSuite) TestAlterComplexObjects() { func (d *DDLTestSuite) TestAlterIdempotency() { cols := []columns.Column{ - columns.NewColumn("created_at", typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")), + columns.NewColumn("created_at", typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")), columns.NewColumn("id", typing.Integer), columns.NewColumn("order_name", typing.String), columns.NewColumn("start", typing.String), @@ -81,7 +81,7 @@ func (d *DDLTestSuite) TestAlterIdempotency() { func (d *DDLTestSuite) TestAlterTableAdd() { // Test adding a bunch of columns cols := []columns.Column{ - columns.NewColumn("created_at", typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")), + columns.NewColumn("created_at", typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")), columns.NewColumn("id", typing.Integer), columns.NewColumn("order_name", typing.String), columns.NewColumn("start", typing.String), @@ -123,7 +123,7 @@ func (d *DDLTestSuite) TestAlterTableAdd() { func (d *DDLTestSuite) TestAlterTableDeleteDryRun() { // Test adding a bunch of columns cols := []columns.Column{ - columns.NewColumn("created_at", typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")), + columns.NewColumn("created_at", typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")), columns.NewColumn("id", typing.Integer), columns.NewColumn("name", typing.String), columns.NewColumn("start", typing.String), @@ -180,7 +180,7 @@ func (d *DDLTestSuite) TestAlterTableDeleteDryRun() { func (d *DDLTestSuite) TestAlterTableDelete() { // Test adding a bunch of columns cols := []columns.Column{ - columns.NewColumn("created_at", typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")), + columns.NewColumn("created_at", typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")), columns.NewColumn("id", typing.Integer), columns.NewColumn("name", typing.String), columns.NewColumn("col_to_delete", typing.String), diff --git a/lib/optimization/table_data_merge_columns_test.go b/lib/optimization/table_data_merge_columns_test.go index 390024362..403e57097 100644 --- a/lib/optimization/table_data_merge_columns_test.go +++ b/lib/optimization/table_data_merge_columns_test.go @@ -14,9 +14,9 @@ func TestTableData_UpdateInMemoryColumnsFromDestination_Tz(t *testing.T) { { // In memory and destination columns are both timestamp_tz tableData := &TableData{inMemoryColumns: &columns.Columns{}} - tableData.AddInMemoryCol(columns.NewColumn("foo", typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""))) + tableData.AddInMemoryCol(columns.NewColumn("foo", typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""))) - assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("foo", typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")))) + assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("foo", typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")))) updatedColumn, isOk := tableData.inMemoryColumns.GetColumn("foo") assert.True(t, isOk) assert.Equal(t, ext.TimestampTZKindType, updatedColumn.KindDetails.ExtendedTimeDetails.Type) @@ -27,11 +27,11 @@ func TestTableData_UpdateInMemoryColumnsFromDestination_Tz(t *testing.T) { tableData.AddInMemoryCol( columns.NewColumn( "foo", - typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, ext.RFC3339MillisecondNoTZ), + typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, ext.RFC3339MillisecondNoTZ), ), ) - assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("foo", typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")))) + assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("foo", typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")))) updatedColumn, isOk := tableData.inMemoryColumns.GetColumn("foo") assert.True(t, isOk) assert.Equal(t, ext.TimestampTZKindType, updatedColumn.KindDetails.ExtendedTimeDetails.Type) @@ -137,9 +137,9 @@ func TestTableData_UpdateInMemoryColumnsFromDestination(t *testing.T) { assert.Nil(t, col.KindDetails.ExtendedTimeDetails, extTimeDetailsCol) } - assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("ext_time", typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "")))) - assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("ext_date", typing.NewExtendedTimeDetails(typing.ETime, ext.DateKindType, "")))) - assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("ext_datetime", typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")))) + assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("ext_time", typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "")))) + assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("ext_date", typing.OldNewExtendedTimeDetails(typing.ETime, ext.DateKindType, "")))) + assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("ext_datetime", typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")))) dateCol, isOk := tableData.inMemoryColumns.GetColumn("ext_date") assert.True(t, isOk) diff --git a/lib/optimization/table_data_test.go b/lib/optimization/table_data_test.go index 7b2520e83..37f16f9cd 100644 --- a/lib/optimization/table_data_test.go +++ b/lib/optimization/table_data_test.go @@ -133,7 +133,7 @@ func TestTableData_UpdateInMemoryColumns(t *testing.T) { "FOO": typing.String, "bar": typing.Invalid, "CHANGE_me": typing.String, - "do_not_change_format": typing.NewExtendedTimeDetails(typing.ETime, ext.DateKindType, ""), + "do_not_change_format": typing.OldNewExtendedTimeDetails(typing.ETime, ext.DateKindType, ""), } { _cols.AddColumn(columns.NewColumn(colName, colKind)) } @@ -150,9 +150,9 @@ func TestTableData_UpdateInMemoryColumns(t *testing.T) { for name, colKindDetails := range map[string]typing.KindDetails{ "foo": typing.String, - "change_me": typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), + "change_me": typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), "bar": typing.Boolean, - "do_not_change_format": typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), + "do_not_change_format": typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), } { tableData.MergeColumnsFromDestination(columns.NewColumn(name, colKindDetails)) } diff --git a/lib/typing/columns/diff_test.go b/lib/typing/columns/diff_test.go index 881d09dd5..46c76024a 100644 --- a/lib/typing/columns/diff_test.go +++ b/lib/typing/columns/diff_test.go @@ -249,8 +249,8 @@ func TestDiffDeterministic(t *testing.T) { func TestCopyColMap(t *testing.T) { var cols Columns cols.AddColumn(NewColumn("hello", typing.String)) - cols.AddColumn(NewColumn("created_at", typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""))) - cols.AddColumn(NewColumn("updated_at", typing.NewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""))) + cols.AddColumn(NewColumn("created_at", typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""))) + cols.AddColumn(NewColumn("updated_at", typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""))) copiedCols := CloneColumns(&cols) assert.Equal(t, copiedCols, &cols) diff --git a/lib/typing/typing.go b/lib/typing/typing.go index 1441ddaf3..2ebe87a64 100644 --- a/lib/typing/typing.go +++ b/lib/typing/typing.go @@ -83,7 +83,27 @@ func NewDecimalDetailsFromTemplate(details KindDetails, decimalDetails decimal.D return details } -func NewExtendedTimeDetails(details KindDetails, extendedType ext.ExtendedTimeKindType, format string) KindDetails { +// MustNewExtendedTimeDetails - calls NewExtendedTimeDetails and panics if there is an error returned. This is used for tests. +func MustNewExtendedTimeDetails(details KindDetails, extendedType ext.ExtendedTimeKindType, optionalFormat string) KindDetails { + nestedKind, err := NewExtendedTimeDetails(details, extendedType, optionalFormat) + if err != nil { + panic(err) + } + + return nestedKind +} + +func NewExtendedTimeDetails(details KindDetails, extendedType ext.ExtendedTimeKindType, optionalFormat string) (KindDetails, error) { + nestedKind, err := ext.NewNestedKind(extendedType, optionalFormat) + if err != nil { + return Invalid, err + } + + details.ExtendedTimeDetails = &nestedKind + return details, nil +} + +func OldNewExtendedTimeDetails(details KindDetails, extendedType ext.ExtendedTimeKindType, format string) KindDetails { // TODO: If format is not set, we should use the default format details.ExtendedTimeDetails = &ext.NestedKind{ Type: extendedType, From da8e465cbe0946c021b5fc1912644b887b56b6f6 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Wed, 23 Oct 2024 11:53:45 -0700 Subject: [PATCH 2/3] Clean up. --- lib/debezium/converters/time_test.go | 4 ++-- lib/debezium/converters/timestamp_test.go | 6 +++--- lib/debezium/schema_test.go | 12 ++++++------ lib/destination/ddl/ddl_sflk_test.go | 8 ++++---- lib/optimization/table_data_merge_columns_test.go | 14 +++++++------- lib/optimization/table_data_test.go | 6 +++--- 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/lib/debezium/converters/time_test.go b/lib/debezium/converters/time_test.go index 88b7858db..3430023d4 100644 --- a/lib/debezium/converters/time_test.go +++ b/lib/debezium/converters/time_test.go @@ -143,7 +143,7 @@ func TestTime_Convert(t *testing.T) { } func TestNanoTime_Converter(t *testing.T) { - assert.Equal(t, typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, NanoTime{}.layout()), NanoTime{}.ToKindDetails()) + assert.Equal(t, typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, NanoTime{}.layout()), NanoTime{}.ToKindDetails()) { // Invalid data _, err := NanoTime{}.Convert("123") @@ -158,7 +158,7 @@ func TestNanoTime_Converter(t *testing.T) { } func TestMicroTime_Converter(t *testing.T) { - assert.Equal(t, typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, MicroTime{}.layout()), MicroTime{}.ToKindDetails()) + assert.Equal(t, typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, MicroTime{}.layout()), MicroTime{}.ToKindDetails()) { // Invalid data _, err := MicroTime{}.Convert("123") diff --git a/lib/debezium/converters/timestamp_test.go b/lib/debezium/converters/timestamp_test.go index bf6a7e551..8eae16177 100644 --- a/lib/debezium/converters/timestamp_test.go +++ b/lib/debezium/converters/timestamp_test.go @@ -9,7 +9,7 @@ import ( ) func TestTimestamp_Converter(t *testing.T) { - assert.Equal(t, typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, ext.RFC3339MillisecondNoTZ), Timestamp{}.ToKindDetails()) + assert.Equal(t, typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, ext.RFC3339MillisecondNoTZ), Timestamp{}.ToKindDetails()) { // Invalid conversion _, err := Timestamp{}.Convert("invalid") @@ -30,7 +30,7 @@ func TestTimestamp_Converter(t *testing.T) { } func TestMicroTimestamp_Converter(t *testing.T) { - assert.Equal(t, typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, ext.RFC3339MicrosecondNoTZ), MicroTimestamp{}.ToKindDetails()) + assert.Equal(t, typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, ext.RFC3339MicrosecondNoTZ), MicroTimestamp{}.ToKindDetails()) { // Invalid conversion _, err := MicroTimestamp{}.Convert("invalid") @@ -51,7 +51,7 @@ func TestMicroTimestamp_Converter(t *testing.T) { } func TestNanoTimestamp_Converter(t *testing.T) { - assert.Equal(t, typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, ext.RFC3339NanosecondNoTZ), NanoTimestamp{}.ToKindDetails()) + assert.Equal(t, typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, ext.RFC3339NanosecondNoTZ), NanoTimestamp{}.ToKindDetails()) { // Invalid conversion _, err := NanoTimestamp{}.Convert("invalid") diff --git a/lib/debezium/schema_test.go b/lib/debezium/schema_test.go index bb955d6ce..367bca237 100644 --- a/lib/debezium/schema_test.go +++ b/lib/debezium/schema_test.go @@ -233,7 +233,7 @@ func TestField_ToKindDetails(t *testing.T) { for _, dbzType := range []SupportedDebeziumType{ZonedTimestamp} { kd, err := Field{DebeziumType: dbzType}.ToKindDetails() assert.NoError(t, err) - assert.Equal(t, typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "2006-01-02T15:04:05.999999999Z"), kd) + assert.Equal(t, typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "2006-01-02T15:04:05.999999999Z"), kd) } } { @@ -250,7 +250,7 @@ func TestField_ToKindDetails(t *testing.T) { for _, dbzType := range []SupportedDebeziumType{Date, DateKafkaConnect} { kd, err := Field{DebeziumType: dbzType}.ToKindDetails() assert.NoError(t, err) - assert.Equal(t, typing.OldNewExtendedTimeDetails(typing.ETime, ext.DateKindType, ext.PostgresDateFormat), kd) + assert.Equal(t, typing.MustNewExtendedTimeDetails(typing.ETime, ext.DateKindType, ext.PostgresDateFormat), kd) } } { @@ -259,7 +259,7 @@ func TestField_ToKindDetails(t *testing.T) { for _, dbzType := range []SupportedDebeziumType{TimeWithTimezone} { kd, err := Field{DebeziumType: dbzType}.ToKindDetails() assert.NoError(t, err) - assert.Equal(t, typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "15:04:05.999999Z"), kd, dbzType) + assert.Equal(t, typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "15:04:05.999999Z"), kd, dbzType) } } { @@ -267,20 +267,20 @@ func TestField_ToKindDetails(t *testing.T) { for _, dbzType := range []SupportedDebeziumType{Time, TimeKafkaConnect} { kd, err := Field{DebeziumType: dbzType}.ToKindDetails() assert.NoError(t, err) - assert.Equal(t, typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "15:04:05.000"), kd, dbzType) + assert.Equal(t, typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "15:04:05.000"), kd, dbzType) } } { // Micro time kd, err := Field{DebeziumType: MicroTime}.ToKindDetails() assert.NoError(t, err) - assert.Equal(t, typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "15:04:05.000000"), kd) + assert.Equal(t, typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "15:04:05.000000"), kd) } { // Nano time kd, err := Field{DebeziumType: NanoTime}.ToKindDetails() assert.NoError(t, err) - assert.Equal(t, typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "15:04:05.000000000"), kd) + assert.Equal(t, typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "15:04:05.000000000"), kd) } } { diff --git a/lib/destination/ddl/ddl_sflk_test.go b/lib/destination/ddl/ddl_sflk_test.go index 2a7ce2170..572aedacd 100644 --- a/lib/destination/ddl/ddl_sflk_test.go +++ b/lib/destination/ddl/ddl_sflk_test.go @@ -51,7 +51,7 @@ func (d *DDLTestSuite) TestAlterComplexObjects() { func (d *DDLTestSuite) TestAlterIdempotency() { cols := []columns.Column{ - columns.NewColumn("created_at", typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")), + columns.NewColumn("created_at", typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")), columns.NewColumn("id", typing.Integer), columns.NewColumn("order_name", typing.String), columns.NewColumn("start", typing.String), @@ -81,7 +81,7 @@ func (d *DDLTestSuite) TestAlterIdempotency() { func (d *DDLTestSuite) TestAlterTableAdd() { // Test adding a bunch of columns cols := []columns.Column{ - columns.NewColumn("created_at", typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")), + columns.NewColumn("created_at", typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")), columns.NewColumn("id", typing.Integer), columns.NewColumn("order_name", typing.String), columns.NewColumn("start", typing.String), @@ -123,7 +123,7 @@ func (d *DDLTestSuite) TestAlterTableAdd() { func (d *DDLTestSuite) TestAlterTableDeleteDryRun() { // Test adding a bunch of columns cols := []columns.Column{ - columns.NewColumn("created_at", typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")), + columns.NewColumn("created_at", typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")), columns.NewColumn("id", typing.Integer), columns.NewColumn("name", typing.String), columns.NewColumn("start", typing.String), @@ -180,7 +180,7 @@ func (d *DDLTestSuite) TestAlterTableDeleteDryRun() { func (d *DDLTestSuite) TestAlterTableDelete() { // Test adding a bunch of columns cols := []columns.Column{ - columns.NewColumn("created_at", typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")), + columns.NewColumn("created_at", typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")), columns.NewColumn("id", typing.Integer), columns.NewColumn("name", typing.String), columns.NewColumn("col_to_delete", typing.String), diff --git a/lib/optimization/table_data_merge_columns_test.go b/lib/optimization/table_data_merge_columns_test.go index 403e57097..37a51c3fa 100644 --- a/lib/optimization/table_data_merge_columns_test.go +++ b/lib/optimization/table_data_merge_columns_test.go @@ -14,9 +14,9 @@ func TestTableData_UpdateInMemoryColumnsFromDestination_Tz(t *testing.T) { { // In memory and destination columns are both timestamp_tz tableData := &TableData{inMemoryColumns: &columns.Columns{}} - tableData.AddInMemoryCol(columns.NewColumn("foo", typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""))) + tableData.AddInMemoryCol(columns.NewColumn("foo", typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""))) - assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("foo", typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")))) + assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("foo", typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")))) updatedColumn, isOk := tableData.inMemoryColumns.GetColumn("foo") assert.True(t, isOk) assert.Equal(t, ext.TimestampTZKindType, updatedColumn.KindDetails.ExtendedTimeDetails.Type) @@ -27,11 +27,11 @@ func TestTableData_UpdateInMemoryColumnsFromDestination_Tz(t *testing.T) { tableData.AddInMemoryCol( columns.NewColumn( "foo", - typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, ext.RFC3339MillisecondNoTZ), + typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampNTZKindType, ext.RFC3339MillisecondNoTZ), ), ) - assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("foo", typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")))) + assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("foo", typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")))) updatedColumn, isOk := tableData.inMemoryColumns.GetColumn("foo") assert.True(t, isOk) assert.Equal(t, ext.TimestampTZKindType, updatedColumn.KindDetails.ExtendedTimeDetails.Type) @@ -137,9 +137,9 @@ func TestTableData_UpdateInMemoryColumnsFromDestination(t *testing.T) { assert.Nil(t, col.KindDetails.ExtendedTimeDetails, extTimeDetailsCol) } - assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("ext_time", typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "")))) - assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("ext_date", typing.OldNewExtendedTimeDetails(typing.ETime, ext.DateKindType, "")))) - assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("ext_datetime", typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")))) + assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("ext_time", typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "")))) + assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("ext_date", typing.MustNewExtendedTimeDetails(typing.ETime, ext.DateKindType, "")))) + assert.NoError(t, tableData.MergeColumnsFromDestination(columns.NewColumn("ext_datetime", typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, "")))) dateCol, isOk := tableData.inMemoryColumns.GetColumn("ext_date") assert.True(t, isOk) diff --git a/lib/optimization/table_data_test.go b/lib/optimization/table_data_test.go index 37f16f9cd..aec0e46a5 100644 --- a/lib/optimization/table_data_test.go +++ b/lib/optimization/table_data_test.go @@ -133,7 +133,7 @@ func TestTableData_UpdateInMemoryColumns(t *testing.T) { "FOO": typing.String, "bar": typing.Invalid, "CHANGE_me": typing.String, - "do_not_change_format": typing.OldNewExtendedTimeDetails(typing.ETime, ext.DateKindType, ""), + "do_not_change_format": typing.MustNewExtendedTimeDetails(typing.ETime, ext.DateKindType, ""), } { _cols.AddColumn(columns.NewColumn(colName, colKind)) } @@ -150,9 +150,9 @@ func TestTableData_UpdateInMemoryColumns(t *testing.T) { for name, colKindDetails := range map[string]typing.KindDetails{ "foo": typing.String, - "change_me": typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), + "change_me": typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), "bar": typing.Boolean, - "do_not_change_format": typing.OldNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), + "do_not_change_format": typing.MustNewExtendedTimeDetails(typing.ETime, ext.TimestampTZKindType, ""), } { tableData.MergeColumnsFromDestination(columns.NewColumn(name, colKindDetails)) } From b96e01fac862ed5d7431a4e3f0f7026d3e9ce8a1 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Wed, 23 Oct 2024 12:00:31 -0700 Subject: [PATCH 3/3] Code clean up. --- clients/databricks/dialect/typing_test.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/clients/databricks/dialect/typing_test.go b/clients/databricks/dialect/typing_test.go index 14e496109..3ec84773a 100644 --- a/clients/databricks/dialect/typing_test.go +++ b/clients/databricks/dialect/typing_test.go @@ -115,13 +115,7 @@ func TestDatabricksDialect_KindForDataType(t *testing.T) { // Date kd, err := DatabricksDialect{}.KindForDataType("DATE", "") assert.NoError(t, err) - assert.Equal(t, typing.KindDetails{ - Kind: typing.ETime.Kind, - ExtendedTimeDetails: &ext.NestedKind{ - Type: ext.DateKindType, - Format: ext.PostgresDateFormat, - }, - }, kd) + assert.Equal(t, typing.MustNewExtendedTimeDetails(typing.ETime, ext.DateKindType, ""), kd) } { // Double