From de3270cf1a70f6114f84ce857ed3514b6f152f27 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Tue, 17 Dec 2024 21:58:02 -0800 Subject: [PATCH] Clean up. --- clients/mssql/values.go | 4 +- clients/snowflake/snowflake_test.go | 6 +- lib/typing/converters/string_converter.go | 61 +++++++++++++++++ .../converters/string_converter_test.go | 10 ++- lib/typing/converters/util.go | 19 ++++++ lib/typing/converters/util_test.go | 68 +++++++++++++++++++ lib/typing/values/string.go | 48 ------------- lib/typing/values/string_test.go | 61 ----------------- 8 files changed, 162 insertions(+), 115 deletions(-) create mode 100644 lib/typing/converters/util.go create mode 100644 lib/typing/converters/util_test.go diff --git a/clients/mssql/values.go b/clients/mssql/values.go index b15b3d64a..c4dd4b8e9 100644 --- a/clients/mssql/values.go +++ b/clients/mssql/values.go @@ -10,9 +10,9 @@ import ( "github.com/artie-labs/transfer/lib/config/constants" "github.com/artie-labs/transfer/lib/typing" "github.com/artie-labs/transfer/lib/typing/columns" + "github.com/artie-labs/transfer/lib/typing/converters" "github.com/artie-labs/transfer/lib/typing/decimal" "github.com/artie-labs/transfer/lib/typing/ext" - "github.com/artie-labs/transfer/lib/typing/values" ) func parseValue(colVal any, colKind columns.Column) (any, error) { @@ -22,7 +22,7 @@ func parseValue(colVal any, colKind columns.Column) (any, error) { boolVal, isOk := colVal.(bool) if isOk { - colVal = values.BooleanToBit(boolVal) + colVal = converters.BooleanToBit(boolVal) } colValString := fmt.Sprint(colVal) diff --git a/clients/snowflake/snowflake_test.go b/clients/snowflake/snowflake_test.go index 5c072f2a0..b71985e9f 100644 --- a/clients/snowflake/snowflake_test.go +++ b/clients/snowflake/snowflake_test.go @@ -103,7 +103,7 @@ func (s *SnowflakeTestSuite) TestExecuteMergeReestablishAuth() { for i := 0; i < 5; i++ { rowsData[fmt.Sprintf("pk-%d", i)] = map[string]any{ - "id": fmt.Sprintf("pk-%d", i), + "id": i, "created_at": time.Now().Format(time.RFC3339Nano), "name": fmt.Sprintf("Robin-%d", i), } @@ -146,7 +146,7 @@ func (s *SnowflakeTestSuite) TestExecuteMerge() { for i := 0; i < 5; i++ { rowsData[fmt.Sprintf("pk-%d", i)] = map[string]any{ - "id": fmt.Sprintf("pk-%d", i), + "id": i, "created_at": time.Now().Format(time.RFC3339Nano), "name": fmt.Sprintf("Robin-%d", i), } @@ -211,7 +211,7 @@ func (s *SnowflakeTestSuite) TestExecuteMergeDeletionFlagRemoval() { rowsData := make(map[string]map[string]any) for i := 0; i < 5; i++ { rowsData[fmt.Sprintf("pk-%d", i)] = map[string]any{ - "id": fmt.Sprintf("pk-%d", i), + "id": i, "created_at": time.Now().Format(time.RFC3339Nano), "name": fmt.Sprintf("Robin-%d", i), } diff --git a/lib/typing/converters/string_converter.go b/lib/typing/converters/string_converter.go index edb27ab47..4bc21758d 100644 --- a/lib/typing/converters/string_converter.go +++ b/lib/typing/converters/string_converter.go @@ -5,6 +5,8 @@ import ( "fmt" "time" + "github.com/artie-labs/transfer/lib/typing/decimal" + "github.com/artie-labs/transfer/lib/config/constants" "github.com/artie-labs/transfer/lib/typing" "github.com/artie-labs/transfer/lib/typing/ext" @@ -16,6 +18,7 @@ type StringConverter interface { func GetStringConverter(kd typing.KindDetails) (StringConverter, error) { switch kd.Kind { + // Time case typing.Date.Kind: return DateConverter{}, nil case typing.Time.Kind: @@ -26,6 +29,13 @@ func GetStringConverter(kd typing.KindDetails) (StringConverter, error) { return TimestampTZConverter{}, nil case typing.Array.Kind: return ArrayConverter{}, nil + // Numbers + case typing.EDecimal.Kind: + return DecimalConverter{}, nil + case typing.Integer.Kind: + return IntegerConverter{}, nil + case typing.Float.Kind: + return FloatConverter{}, nil } // TODO: Return an error when all the types are implemented. @@ -94,3 +104,54 @@ func (ArrayConverter) Convert(value any) (string, error) { return string(colValBytes), nil } + +type IntegerConverter struct{} + +func (IntegerConverter) Convert(value any) (string, error) { + switch parsedVal := value.(type) { + case float32: + return Float32ToString(parsedVal), nil + case float64: + return Float64ToString(parsedVal), nil + case bool: + return fmt.Sprint(BooleanToBit(parsedVal)), nil + case int, int8, int16, int32, int64: + return fmt.Sprint(parsedVal), nil + default: + return "", fmt.Errorf("unexpected value: '%v', type: %T", value, value) + } +} + +type FloatConverter struct{} + +func (FloatConverter) Convert(value any) (string, error) { + switch parsedVal := value.(type) { + case float32: + return Float32ToString(parsedVal), nil + case float64: + return Float64ToString(parsedVal), nil + case int, int8, int16, int32, int64: + return fmt.Sprint(parsedVal), nil + default: + return "", fmt.Errorf("unexpected value: '%v', type: %T", value, value) + } +} + +type DecimalConverter struct{} + +func (DecimalConverter) Convert(value any) (string, error) { + switch castedColVal := value.(type) { + case float32: + return Float32ToString(castedColVal), nil + case float64: + return Float64ToString(castedColVal), nil + case int64, int32: + return fmt.Sprint(castedColVal), nil + case string: + return castedColVal, nil + case *decimal.Decimal: + return castedColVal.String(), nil + default: + return "", fmt.Errorf("unexpected value: '%v' type: %T", value, value) + } +} diff --git a/lib/typing/converters/string_converter_test.go b/lib/typing/converters/string_converter_test.go index 62f046cba..731b40a91 100644 --- a/lib/typing/converters/string_converter_test.go +++ b/lib/typing/converters/string_converter_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestArrayConverter(t *testing.T) { +func TestArrayConverter_Convert(t *testing.T) { // Array { // Normal arrays @@ -22,3 +22,11 @@ func TestArrayConverter(t *testing.T) { assert.Equal(t, `["__debezium_unavailable_value"]`, val) } } + +func TestIntegerConverter_Convert(t *testing.T) { + for _, val := range []any{42, int8(42), int16(42), int32(42), int64(42)} { + parsedVal, err := IntegerConverter{}.Convert(val) + assert.NoError(t, err) + assert.Equal(t, "42", parsedVal) + } +} diff --git a/lib/typing/converters/util.go b/lib/typing/converters/util.go new file mode 100644 index 000000000..b8444889c --- /dev/null +++ b/lib/typing/converters/util.go @@ -0,0 +1,19 @@ +package converters + +import "strconv" + +func Float64ToString(value float64) string { + return strconv.FormatFloat(value, 'f', -1, 64) +} + +func Float32ToString(value float32) string { + return strconv.FormatFloat(float64(value), 'f', -1, 32) +} + +func BooleanToBit(val bool) int { + if val { + return 1 + } else { + return 0 + } +} diff --git a/lib/typing/converters/util_test.go b/lib/typing/converters/util_test.go new file mode 100644 index 000000000..47472745f --- /dev/null +++ b/lib/typing/converters/util_test.go @@ -0,0 +1,68 @@ +package converters + +import ( + "math" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBooleanToBit(t *testing.T) { + assert.Equal(t, 1, BooleanToBit(true)) + assert.Equal(t, 0, BooleanToBit(false)) +} + +func TestFloat32ToString(t *testing.T) { + type ioPair struct { + input float32 + output string + } + + ioPairs := []ioPair{ + {123.456, "123.456"}, + {0.0, "0"}, + {-1.0, "-1"}, + {1.0, "1"}, + {340282350000000000000000000000000000000, "340282350000000000000000000000000000000"}, + {math.MaxFloat32, "340282350000000000000000000000000000000"}, + {0.000000000000000000000000000000000000000000001, "0.000000000000000000000000000000000000000000001"}, + {-340282350000000000000000000000000000000, "-340282350000000000000000000000000000000"}, + {1.401298464324817070923729583289916131280e-45, "0.000000000000000000000000000000000000000000001"}, + {1.17549435e-38, "0.000000000000000000000000000000000000011754944"}, + {-1.17549435e-38, "-0.000000000000000000000000000000000000011754944"}, + {2.71828, "2.71828"}, + {-2.71828, "-2.71828"}, + {3.14159, "3.14159"}, + {-3.14159, "-3.14159"}, + } + + for _, pair := range ioPairs { + assert.Equal(t, pair.output, Float32ToString(pair.input), pair.input) + } +} + +func TestFloat64ToString(t *testing.T) { + type ioPair struct { + input float64 + output string + } + + ioPairs := []ioPair{ + {123.456, "123.456"}, + {0.0, "0"}, + {-1.0, "-1"}, + {1.0, "1"}, + {1.7976931348623157e+308, "179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}, + {math.MaxFloat64, "179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}, + {4.9406564584124654e-324, "0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005"}, + {-1.7976931348623157e+308, "-179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}, + {2.718281828459045, "2.718281828459045"}, + {-2.718281828459045, "-2.718281828459045"}, + {3.141592653589793, "3.141592653589793"}, + {-3.141592653589793, "-3.141592653589793"}, + } + + for _, pair := range ioPairs { + assert.Equal(t, pair.output, Float64ToString(pair.input), pair.input) + } +} diff --git a/lib/typing/values/string.go b/lib/typing/values/string.go index 30c294eed..f6b123773 100644 --- a/lib/typing/values/string.go +++ b/lib/typing/values/string.go @@ -4,32 +4,14 @@ import ( "encoding/json" "fmt" "reflect" - "strconv" "strings" "github.com/artie-labs/transfer/lib/config/constants" "github.com/artie-labs/transfer/lib/stringutil" "github.com/artie-labs/transfer/lib/typing" "github.com/artie-labs/transfer/lib/typing/converters" - "github.com/artie-labs/transfer/lib/typing/decimal" ) -func Float64ToString(value float64) string { - return strconv.FormatFloat(value, 'f', -1, 64) -} - -func Float32ToString(value float32) string { - return strconv.FormatFloat(float64(value), 'f', -1, 32) -} - -func BooleanToBit(val bool) int { - if val { - return 1 - } else { - return 0 - } -} - func ToString(colVal any, colKind typing.KindDetails) (string, error) { if colVal == nil { return "", fmt.Errorf("colVal is nil") @@ -45,7 +27,6 @@ func ToString(colVal any, colKind typing.KindDetails) (string, error) { } // TODO: Move all of this into converter function - switch colKind.Kind { case typing.String.Kind: isArray := reflect.ValueOf(colVal).Kind() == reflect.Slice @@ -78,35 +59,6 @@ func ToString(colVal any, colKind typing.KindDetails) (string, error) { return string(colValBytes), nil } } - case typing.Float.Kind: - switch parsedVal := colVal.(type) { - case float32: - return Float32ToString(parsedVal), nil - case float64: - return Float64ToString(parsedVal), nil - } - case typing.Integer.Kind: - switch parsedVal := colVal.(type) { - case float32: - return Float32ToString(parsedVal), nil - case float64: - return Float64ToString(parsedVal), nil - case bool: - return fmt.Sprint(BooleanToBit(parsedVal)), nil - } - case typing.EDecimal.Kind: - switch castedColVal := colVal.(type) { - // It's okay if it's not a *decimal.Decimal, so long as it's a float or string. - // By having the flexibility of handling both *decimal.Decimal and float64/float32/string values within the same batch will increase our ability for data digestion. - case int64, int32, float64, float32: - return fmt.Sprint(castedColVal), nil - case string: - return castedColVal, nil - case *decimal.Decimal: - return castedColVal.String(), nil - } - - return "", fmt.Errorf("unexpected colVal type: %T", colVal) } return fmt.Sprint(colVal), nil diff --git a/lib/typing/values/string_test.go b/lib/typing/values/string_test.go index 5ad48a2af..20c3cd5f3 100644 --- a/lib/typing/values/string_test.go +++ b/lib/typing/values/string_test.go @@ -1,7 +1,6 @@ package values import ( - "math" "testing" "time" @@ -14,66 +13,6 @@ import ( "github.com/artie-labs/transfer/lib/typing/ext" ) -func TestFloat32ToString(t *testing.T) { - type ioPair struct { - input float32 - output string - } - - ioPairs := []ioPair{ - {123.456, "123.456"}, - {0.0, "0"}, - {-1.0, "-1"}, - {1.0, "1"}, - {340282350000000000000000000000000000000, "340282350000000000000000000000000000000"}, - {math.MaxFloat32, "340282350000000000000000000000000000000"}, - {0.000000000000000000000000000000000000000000001, "0.000000000000000000000000000000000000000000001"}, - {-340282350000000000000000000000000000000, "-340282350000000000000000000000000000000"}, - {1.401298464324817070923729583289916131280e-45, "0.000000000000000000000000000000000000000000001"}, - {1.17549435e-38, "0.000000000000000000000000000000000000011754944"}, - {-1.17549435e-38, "-0.000000000000000000000000000000000000011754944"}, - {2.71828, "2.71828"}, - {-2.71828, "-2.71828"}, - {3.14159, "3.14159"}, - {-3.14159, "-3.14159"}, - } - - for _, pair := range ioPairs { - assert.Equal(t, pair.output, Float32ToString(pair.input), pair.input) - } -} - -func TestFloat64ToString(t *testing.T) { - type ioPair struct { - input float64 - output string - } - - ioPairs := []ioPair{ - {123.456, "123.456"}, - {0.0, "0"}, - {-1.0, "-1"}, - {1.0, "1"}, - {1.7976931348623157e+308, "179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}, - {math.MaxFloat64, "179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}, - {4.9406564584124654e-324, "0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005"}, - {-1.7976931348623157e+308, "-179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}, - {2.718281828459045, "2.718281828459045"}, - {-2.718281828459045, "-2.718281828459045"}, - {3.141592653589793, "3.141592653589793"}, - {-3.141592653589793, "-3.141592653589793"}, - } - - for _, pair := range ioPairs { - assert.Equal(t, pair.output, Float64ToString(pair.input), pair.input) - } -} - -func TestBooleanToBit(t *testing.T) { - assert.Equal(t, 1, BooleanToBit(true)) - assert.Equal(t, 0, BooleanToBit(false)) -} - func TestToString(t *testing.T) { { // Nil value