From aab8d1cf9eb769fe5b8a3329c66e708ce4e99045 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Tue, 17 Dec 2024 21:24:08 -0800 Subject: [PATCH] [Typing] Setting up scaffold for String converters (#1085) --- lib/typing/converters/string_converter.go | 96 +++++++++++++++++++ .../converters/string_converter_test.go | 24 +++++ lib/typing/values/string.go | 53 ++-------- 3 files changed, 130 insertions(+), 43 deletions(-) create mode 100644 lib/typing/converters/string_converter.go create mode 100644 lib/typing/converters/string_converter_test.go diff --git a/lib/typing/converters/string_converter.go b/lib/typing/converters/string_converter.go new file mode 100644 index 00000000..edb27ab4 --- /dev/null +++ b/lib/typing/converters/string_converter.go @@ -0,0 +1,96 @@ +package converters + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/artie-labs/transfer/lib/config/constants" + "github.com/artie-labs/transfer/lib/typing" + "github.com/artie-labs/transfer/lib/typing/ext" +) + +type StringConverter interface { + Convert(value any) (string, error) +} + +func GetStringConverter(kd typing.KindDetails) (StringConverter, error) { + switch kd.Kind { + case typing.Date.Kind: + return DateConverter{}, nil + case typing.Time.Kind: + return TimeConverter{}, nil + case typing.TimestampNTZ.Kind: + return TimestampNTZConverter{}, nil + case typing.TimestampTZ.Kind: + return TimestampTZConverter{}, nil + case typing.Array.Kind: + return ArrayConverter{}, nil + } + + // TODO: Return an error when all the types are implemented. + return nil, nil +} + +type DateConverter struct{} + +func (DateConverter) Convert(value any) (string, error) { + _time, err := ext.ParseDateFromAny(value) + if err != nil { + return "", fmt.Errorf("failed to cast colVal as date, colVal: '%v', err: %w", value, err) + } + + return _time.Format(ext.PostgresDateFormat), nil +} + +type TimeConverter struct{} + +func (TimeConverter) Convert(value any) (string, error) { + _time, err := ext.ParseTimeFromAny(value) + if err != nil { + return "", fmt.Errorf("failed to cast colVal as time, colVal: '%v', err: %w", value, err) + } + + return _time.Format(ext.PostgresTimeFormatNoTZ), nil +} + +type TimestampNTZConverter struct{} + +func (TimestampNTZConverter) Convert(value any) (string, error) { + _time, err := ext.ParseTimestampNTZFromAny(value) + if err != nil { + return "", fmt.Errorf("failed to cast colVal as timestampNTZ, colVal: '%v', err: %w", value, err) + } + + return _time.Format(ext.RFC3339NoTZ), nil +} + +type TimestampTZConverter struct{} + +func (TimestampTZConverter) Convert(value any) (string, error) { + _time, err := ext.ParseTimestampTZFromAny(value) + if err != nil { + return "", fmt.Errorf("failed to cast colVal as timestampTZ, colVal: '%v', err: %w", value, err) + } + + return _time.Format(time.RFC3339Nano), nil +} + +type ArrayConverter struct{} + +func (ArrayConverter) Convert(value any) (string, error) { + // If the column value is TOASTED, we should return an array with the TOASTED placeholder + // We're doing this to make sure that the value matches the schema. + if stringValue, ok := value.(string); ok { + if stringValue == constants.ToastUnavailableValuePlaceholder { + return fmt.Sprintf(`["%s"]`, constants.ToastUnavailableValuePlaceholder), nil + } + } + + colValBytes, err := json.Marshal(value) + if err != nil { + return "", err + } + + return string(colValBytes), nil +} diff --git a/lib/typing/converters/string_converter_test.go b/lib/typing/converters/string_converter_test.go new file mode 100644 index 00000000..62f046cb --- /dev/null +++ b/lib/typing/converters/string_converter_test.go @@ -0,0 +1,24 @@ +package converters + +import ( + "testing" + + "github.com/artie-labs/transfer/lib/config/constants" + "github.com/stretchr/testify/assert" +) + +func TestArrayConverter(t *testing.T) { + // Array + { + // Normal arrays + val, err := ArrayConverter{}.Convert([]string{"foo", "bar"}) + assert.NoError(t, err) + assert.Equal(t, `["foo","bar"]`, val) + } + { + // Toasted array + val, err := ArrayConverter{}.Convert(constants.ToastUnavailableValuePlaceholder) + assert.NoError(t, err) + assert.Equal(t, `["__debezium_unavailable_value"]`, val) + } +} diff --git a/lib/typing/values/string.go b/lib/typing/values/string.go index 8114ea61..30c294ee 100644 --- a/lib/typing/values/string.go +++ b/lib/typing/values/string.go @@ -6,13 +6,12 @@ import ( "reflect" "strconv" "strings" - "time" "github.com/artie-labs/transfer/lib/config/constants" "github.com/artie-labs/transfer/lib/stringutil" "github.com/artie-labs/transfer/lib/typing" + "github.com/artie-labs/transfer/lib/typing/converters" "github.com/artie-labs/transfer/lib/typing/decimal" - "github.com/artie-labs/transfer/lib/typing/ext" ) func Float64ToString(value float64) string { @@ -36,35 +35,18 @@ func ToString(colVal any, colKind typing.KindDetails) (string, error) { return "", fmt.Errorf("colVal is nil") } - switch colKind.Kind { - case typing.Date.Kind: - _time, err := ext.ParseDateFromAny(colVal) - if err != nil { - return "", fmt.Errorf("failed to cast colVal as date, colVal: '%v', err: %w", colVal, err) - } - - return _time.Format(ext.PostgresDateFormat), nil - case typing.Time.Kind: - _time, err := ext.ParseTimeFromAny(colVal) - if err != nil { - return "", fmt.Errorf("failed to cast colVal as time, colVal: '%v', err: %w", colVal, err) - } + sv, err := converters.GetStringConverter(colKind) + if err != nil { + return "", fmt.Errorf("failed to get string converter: %w", err) + } - return _time.Format(ext.PostgresTimeFormatNoTZ), nil - case typing.TimestampNTZ.Kind: - _time, err := ext.ParseTimestampNTZFromAny(colVal) - if err != nil { - return "", fmt.Errorf("failed to cast colVal as timestampNTZ, colVal: '%v', err: %w", colVal, err) - } + if sv != nil { + return sv.Convert(colVal) + } - return _time.Format(ext.RFC3339NoTZ), nil - case typing.TimestampTZ.Kind: - _time, err := ext.ParseTimestampTZFromAny(colVal) - if err != nil { - return "", fmt.Errorf("failed to cast colVal as timestampTZ, colVal: '%v', err: %w", colVal, err) - } + // TODO: Move all of this into converter function - return _time.Format(time.RFC3339Nano), nil + switch colKind.Kind { case typing.String.Kind: isArray := reflect.ValueOf(colVal).Kind() == reflect.Slice _, isMap := colVal.(map[string]any) @@ -96,21 +78,6 @@ func ToString(colVal any, colKind typing.KindDetails) (string, error) { return string(colValBytes), nil } } - case typing.Array.Kind: - // If the column value is TOASTED, we should return an array with the TOASTED placeholder - // We're doing this to make sure that the value matches the schema. - if stringValue, ok := colVal.(string); ok { - if stringValue == constants.ToastUnavailableValuePlaceholder { - return fmt.Sprintf(`["%s"]`, constants.ToastUnavailableValuePlaceholder), nil - } - } - - colValBytes, err := json.Marshal(colVal) - if err != nil { - return "", err - } - - return string(colValBytes), nil case typing.Float.Kind: switch parsedVal := colVal.(type) { case float32: