diff --git a/clients/bigquery/dialect/dialect_test.go b/clients/bigquery/dialect/dialect_test.go index 0fe35f6f8..29409a2fb 100644 --- a/clients/bigquery/dialect/dialect_test.go +++ b/clients/bigquery/dialect/dialect_test.go @@ -199,11 +199,11 @@ func TestBigQueryDialect_BuildAlterColumnQuery(t *testing.T) { ) } -func TestBuildProcessToastColExpression(t *testing.T) { +func TestBigQueryDialect_BuildProcessToastColExpression(t *testing.T) { assert.Equal(t, `CASE WHEN COALESCE(cc.bar != '__debezium_unavailable_value', true) THEN cc.bar ELSE c.bar END`, BigQueryDialect{}.BuildProcessToastColExpression("bar")) } -func TestBuildProcessToastStructColExpression(t *testing.T) { +func TestBuildBigQueryDialect_ProcessToastStructColExpression(t *testing.T) { assert.Equal(t, `CASE WHEN COALESCE(TO_JSON_STRING(cc.foo) != '{"key":"__debezium_unavailable_value"}', true) THEN cc.foo ELSE c.foo END`, BigQueryDialect{}.BuildProcessToastStructColExpression("foo")) } @@ -279,3 +279,70 @@ func TestBuildColumnsUpdateFragment(t *testing.T) { assert.Equal(t, _testCase.expectedString, actualQuery, _testCase.name) } } + +func TestColumn_DefaultValue(t *testing.T) { + dialect := BigQueryDialect{} + + birthday := time.Date(2022, time.September, 6, 3, 19, 24, 942000000, time.UTC) + birthdayExtDateTime, err := ext.ParseExtendedDateTime(birthday.Format(ext.ISO8601), nil) + assert.NoError(t, err) + + // date + dateKind := typing.ETime + dateKind.ExtendedTimeDetails = &ext.Date + // time + timeKind := typing.ETime + timeKind.ExtendedTimeDetails = &ext.Time + // date time + dateTimeKind := typing.ETime + dateTimeKind.ExtendedTimeDetails = &ext.DateTime + + testCases := []struct { + name string + col columns.Column + expectedValue any + dialectExpectedValue any + }{ + { + name: "default value = nil", + col: columns.NewColumnWithDefaultValue("", typing.String, nil), + expectedValue: nil, + }, + { + name: "string", + col: columns.NewColumnWithDefaultValue("", typing.String, "abcdef"), + expectedValue: "'abcdef'", + }, + { + name: "json", + col: columns.NewColumnWithDefaultValue("", typing.Struct, "{}"), + expectedValue: "JSON'{}'", + }, + { + name: "json w/ some values", + col: columns.NewColumnWithDefaultValue("", typing.Struct, "{\"age\": 0, \"membership_level\": \"standard\"}"), + expectedValue: "JSON'{\"age\": 0, \"membership_level\": \"standard\"}'", + }, + { + name: "date", + col: columns.NewColumnWithDefaultValue("", dateKind, birthdayExtDateTime), + expectedValue: "'2022-09-06'", + }, + { + name: "time", + col: columns.NewColumnWithDefaultValue("", timeKind, birthdayExtDateTime), + expectedValue: "'03:19:24.942'", + }, + { + name: "datetime", + col: columns.NewColumnWithDefaultValue("", dateTimeKind, birthdayExtDateTime), + expectedValue: "'2022-09-06T03:19:24.942Z'", + }, + } + + for _, testCase := range testCases { + actualValue, actualErr := testCase.col.DefaultValue(dialect, nil) + assert.NoError(t, actualErr, testCase.name) + assert.Equal(t, testCase.expectedValue, actualValue, testCase.name) + } +} diff --git a/clients/redshift/dialect/dialect_test.go b/clients/redshift/dialect/dialect_test.go index ffd7a13a7..5c0ea0a2e 100644 --- a/clients/redshift/dialect/dialect_test.go +++ b/clients/redshift/dialect/dialect_test.go @@ -3,6 +3,7 @@ package dialect import ( "fmt" "testing" + "time" "github.com/stretchr/testify/assert" @@ -12,6 +13,7 @@ import ( "github.com/artie-labs/transfer/lib/sql" "github.com/artie-labs/transfer/lib/typing" "github.com/artie-labs/transfer/lib/typing/columns" + "github.com/artie-labs/transfer/lib/typing/ext" ) func TestRedshiftDialect_QuoteIdentifier(t *testing.T) { @@ -151,7 +153,7 @@ func TestRedshiftDialect_KindForDataType(t *testing.T) { } } -func TestRedshifgDialect_IsColumnAlreadyExistsErr(t *testing.T) { +func TestRedshiftDialect_IsColumnAlreadyExistsErr(t *testing.T) { testCases := []struct { name string err error @@ -204,11 +206,11 @@ func TestQuoteIdentifiers(t *testing.T) { assert.Equal(t, []string{`"a"`, `"b"`, `"c"`}, sql.QuoteIdentifiers([]string{"a", "b", "c"}, RedshiftDialect{})) } -func TestBuildProcessToastColExpression(t *testing.T) { +func TestRedshiftDialect_BuildProcessToastColExpression(t *testing.T) { assert.Equal(t, `CASE WHEN COALESCE(cc.bar != '__debezium_unavailable_value', true) THEN cc.bar ELSE c.bar END`, RedshiftDialect{}.BuildProcessToastColExpression("bar")) } -func TestBuildProcessToastStructColExpression(t *testing.T) { +func TestRedshiftDialect_BuildProcessToastStructColExpression(t *testing.T) { assert.Equal(t, `CASE WHEN COALESCE(cc.foo != JSON_PARSE('{"key":"__debezium_unavailable_value"}'), true) THEN cc.foo ELSE c.foo END`, RedshiftDialect{}.BuildProcessToastStructColExpression("foo")) } @@ -260,3 +262,70 @@ func TestBuildColumnsUpdateFragment(t *testing.T) { assert.Equal(t, _testCase.expectedString, actualQuery, _testCase.name) } } + +func TestColumn_DefaultValue(t *testing.T) { + dialect := RedshiftDialect{} + + birthday := time.Date(2022, time.September, 6, 3, 19, 24, 942000000, time.UTC) + birthdayExtDateTime, err := ext.ParseExtendedDateTime(birthday.Format(ext.ISO8601), nil) + assert.NoError(t, err) + + // date + dateKind := typing.ETime + dateKind.ExtendedTimeDetails = &ext.Date + // time + timeKind := typing.ETime + timeKind.ExtendedTimeDetails = &ext.Time + // date time + dateTimeKind := typing.ETime + dateTimeKind.ExtendedTimeDetails = &ext.DateTime + + testCases := []struct { + name string + col columns.Column + expectedValue any + destKindToExpectedValueMap map[sql.Dialect]any + }{ + { + name: "default value = nil", + col: columns.NewColumnWithDefaultValue("", typing.String, nil), + expectedValue: nil, + }, + { + name: "string", + col: columns.NewColumnWithDefaultValue("", typing.String, "abcdef"), + expectedValue: "'abcdef'", + }, + { + name: "json", + col: columns.NewColumnWithDefaultValue("", typing.Struct, "{}"), + expectedValue: `JSON_PARSE('{}')`, + }, + { + name: "json w/ some values", + col: columns.NewColumnWithDefaultValue("", typing.Struct, "{\"age\": 0, \"membership_level\": \"standard\"}"), + expectedValue: "JSON_PARSE('{\"age\": 0, \"membership_level\": \"standard\"}')", + }, + { + name: "date", + col: columns.NewColumnWithDefaultValue("", dateKind, birthdayExtDateTime), + expectedValue: "'2022-09-06'", + }, + { + name: "time", + col: columns.NewColumnWithDefaultValue("", timeKind, birthdayExtDateTime), + expectedValue: "'03:19:24.942'", + }, + { + name: "datetime", + col: columns.NewColumnWithDefaultValue("", dateTimeKind, birthdayExtDateTime), + expectedValue: "'2022-09-06T03:19:24.942Z'", + }, + } + + for _, testCase := range testCases { + actualValue, actualErr := testCase.col.DefaultValue(dialect, nil) + assert.NoError(t, actualErr, testCase.name) + assert.Equal(t, testCase.expectedValue, actualValue, testCase.name) + } +} diff --git a/clients/snowflake/dialect/dialect_test.go b/clients/snowflake/dialect/dialect_test.go index 31685fe5e..fef6754e7 100644 --- a/clients/snowflake/dialect/dialect_test.go +++ b/clients/snowflake/dialect/dialect_test.go @@ -3,6 +3,7 @@ package dialect import ( "fmt" "testing" + "time" "github.com/artie-labs/transfer/lib/config/constants" "github.com/artie-labs/transfer/lib/mocks" @@ -244,11 +245,11 @@ func TestSnowflakeDialect_BuildAlterColumnQuery(t *testing.T) { ) } -func TestBuildProcessToastColExpression(t *testing.T) { +func TestSnowflakeDialect_BuildProcessToastColExpression(t *testing.T) { assert.Equal(t, `CASE WHEN COALESCE(cc.bar != '__debezium_unavailable_value', true) THEN cc.bar ELSE c.bar END`, SnowflakeDialect{}.BuildProcessToastColExpression("bar")) } -func TestBuildProcessToastStructColExpression(t *testing.T) { +func TestSnowflakeDialect_BuildProcessToastStructColExpression(t *testing.T) { assert.Equal(t, `CASE WHEN COALESCE(cc.foo != {'key': '__debezium_unavailable_value'}, true) THEN cc.foo ELSE c.foo END`, SnowflakeDialect{}.BuildProcessToastStructColExpression("foo")) } @@ -274,3 +275,69 @@ func TestBuildColumnsUpdateFragment(t *testing.T) { actualQuery := columns.BuildColumnsUpdateFragment(stringAndToastCols, SnowflakeDialect{}) assert.Equal(t, `"FOO"= CASE WHEN COALESCE(cc."FOO" != '__debezium_unavailable_value', true) THEN cc."FOO" ELSE c."FOO" END,"BAR"=cc."BAR"`, actualQuery) } + +func TestColumn_DefaultValue(t *testing.T) { + dialect := SnowflakeDialect{} + + birthday := time.Date(2022, time.September, 6, 3, 19, 24, 942000000, time.UTC) + birthdayExtDateTime, err := ext.ParseExtendedDateTime(birthday.Format(ext.ISO8601), nil) + assert.NoError(t, err) + + // date + dateKind := typing.ETime + dateKind.ExtendedTimeDetails = &ext.Date + // time + timeKind := typing.ETime + timeKind.ExtendedTimeDetails = &ext.Time + // date time + dateTimeKind := typing.ETime + dateTimeKind.ExtendedTimeDetails = &ext.DateTime + + testCases := []struct { + name string + col columns.Column + expectedValue any + }{ + { + name: "default value = nil", + col: columns.NewColumnWithDefaultValue("", typing.String, nil), + expectedValue: nil, + }, + { + name: "string", + col: columns.NewColumnWithDefaultValue("", typing.String, "abcdef"), + expectedValue: "'abcdef'", + }, + { + name: "json", + col: columns.NewColumnWithDefaultValue("", typing.Struct, "{}"), + expectedValue: `'{}'`, + }, + { + name: "json w/ some values", + col: columns.NewColumnWithDefaultValue("", typing.Struct, "{\"age\": 0, \"membership_level\": \"standard\"}"), + expectedValue: "'{\"age\": 0, \"membership_level\": \"standard\"}'", + }, + { + name: "date", + col: columns.NewColumnWithDefaultValue("", dateKind, birthdayExtDateTime), + expectedValue: "'2022-09-06'", + }, + { + name: "time", + col: columns.NewColumnWithDefaultValue("", timeKind, birthdayExtDateTime), + expectedValue: "'03:19:24.942'", + }, + { + name: "datetime", + col: columns.NewColumnWithDefaultValue("", dateTimeKind, birthdayExtDateTime), + expectedValue: "'2022-09-06T03:19:24.942Z'", + }, + } + + for _, testCase := range testCases { + actualValue, actualErr := testCase.col.DefaultValue(dialect, nil) + assert.NoError(t, actualErr, testCase.name) + assert.Equal(t, testCase.expectedValue, actualValue, testCase.name) + } +} diff --git a/lib/typing/columns/columns.go b/lib/typing/columns/columns.go index ea3a22c74..4048a62a9 100644 --- a/lib/typing/columns/columns.go +++ b/lib/typing/columns/columns.go @@ -49,6 +49,12 @@ func NewColumn(name string, kd typing.KindDetails) Column { } } +func NewColumnWithDefaultValue(name string, kd typing.KindDetails, defaultValue any) Column { + column := NewColumn(name, kd) + column.defaultValue = defaultValue + return column +} + func (c *Column) SetBackfilled(backfilled bool) { c.backfilled = backfilled }