Skip to content

Commit

Permalink
Upgrade Artie Transfer. (#405)
Browse files Browse the repository at this point in the history
  • Loading branch information
Tang8330 authored Jun 17, 2024
1 parent ec696b9 commit 7cc7994
Show file tree
Hide file tree
Showing 5 changed files with 436 additions and 436 deletions.
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.22

require (
github.com/DataDog/datadog-go/v5 v5.5.0
github.com/artie-labs/transfer v1.25.18
github.com/artie-labs/transfer v1.25.23
github.com/aws/aws-sdk-go v1.44.327
github.com/aws/aws-sdk-go-v2 v1.18.1
github.com/aws/aws-sdk-go-v2/config v1.18.19
Expand Down Expand Up @@ -149,6 +149,6 @@ require (
google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect
google.golang.org/grpc v1.63.2 // indirect
google.golang.org/protobuf v1.33.0 // indirect
google.golang.org/protobuf v1.34.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)
8 changes: 4 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,8 @@ github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlE
github.com/apache/thrift v0.14.2/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/apache/thrift v0.17.0 h1:cMd2aj52n+8VoAtvSvLn4kDC3aZ6IAkBuqWQ2IDu7wo=
github.com/apache/thrift v0.17.0/go.mod h1:OLxhMRJxomX+1I/KUw03qoV3mMz16BwaKI+d4fPBx7Q=
github.com/artie-labs/transfer v1.25.18 h1:TMYAKPn1PDFQ9HieogdrKuAnE7aDmxAEKtVtcLdOEM0=
github.com/artie-labs/transfer v1.25.18/go.mod h1:lv9NtzWvCcG4haLbdUAvcvEKfChbeam5fDATr26ve88=
github.com/artie-labs/transfer v1.25.23 h1:9pL6wO/87H6vmFyN8NfOuLAP28w3CjmxeELIrvWKePk=
github.com/artie-labs/transfer v1.25.23/go.mod h1:PxZjjW1+OnZDgRRJwVXUoiGY2iPsLnY2TUMrdcY3zfg=
github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
github.com/aws/aws-sdk-go v1.44.327 h1:ZS8oO4+7MOBLhkdwIhgtVeDzCeWOlTfKJS7EgggbIEY=
github.com/aws/aws-sdk-go v1.44.327/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
Expand Down Expand Up @@ -832,8 +832,8 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
Expand Down
222 changes: 111 additions & 111 deletions lib/debezium/converters/decimal_test.go
Original file line number Diff line number Diff line change
@@ -1,133 +1,133 @@
package converters

import (
"fmt"
"testing"
"fmt"
"testing"

"github.com/artie-labs/transfer/lib/debezium"
"github.com/artie-labs/transfer/lib/ptr"
"github.com/stretchr/testify/assert"
"github.com/artie-labs/transfer/lib/debezium"
"github.com/artie-labs/transfer/lib/ptr"
"github.com/stretchr/testify/assert"
)

func TestDecimalConverter_ToField(t *testing.T) {
{
// Without precision
converter := NewDecimalConverter(2, nil)
expected := debezium.Field{
Type: "bytes",
FieldName: "col",
DebeziumType: "org.apache.kafka.connect.data.Decimal",
Parameters: map[string]any{
"scale": "2",
},
}
assert.Equal(t, expected, converter.ToField("col"))
}
{
// With precision
converter := NewDecimalConverter(2, ptr.ToInt(3))
expected := debezium.Field{
Type: "bytes",
FieldName: "col",
DebeziumType: "org.apache.kafka.connect.data.Decimal",
Parameters: map[string]any{
"connect.decimal.precision": "3",
"scale": "2",
},
}
assert.Equal(t, expected, converter.ToField("col"))
}
{
// Without precision
converter := NewDecimalConverter(2, nil)
expected := debezium.Field{
Type: "bytes",
FieldName: "col",
DebeziumType: "org.apache.kafka.connect.data.Decimal",
Parameters: map[string]any{
"scale": "2",
},
}
assert.Equal(t, expected, converter.ToField("col"))
}
{
// With precision
converter := NewDecimalConverter(2, ptr.ToInt(3))
expected := debezium.Field{
Type: "bytes",
FieldName: "col",
DebeziumType: "org.apache.kafka.connect.data.Decimal",
Parameters: map[string]any{
"connect.decimal.precision": "3",
"scale": "2",
},
}
assert.Equal(t, expected, converter.ToField("col"))
}
}

func TestDecimalConverter_Convert(t *testing.T) {
converter := NewDecimalConverter(2, nil)
{
// Malformed value - empty string.
_, err := converter.Convert("")
assert.ErrorContains(t, err, "unable to use '' as a floating-point number")
}
{
// Malformed value - not a floating-point.
_, err := converter.Convert("11qwerty00")
assert.ErrorContains(t, err, "unable to use '11qwerty00' as a floating-point number")
}
{
// Happy path.
converted, err := converter.Convert("1.23")
assert.NoError(t, err)
bytes, ok := converted.([]byte)
assert.True(t, ok)
actualValue, err := converter.ToField("").DecodeDecimal(bytes)
assert.NoError(t, err)
assert.Equal(t, "1.23", fmt.Sprint(actualValue))
}
converter := NewDecimalConverter(2, nil)
{
// Malformed value - empty string.
_, err := converter.Convert("")
assert.ErrorContains(t, err, `unable to use "" as a floating-point number`)
}
{
// Malformed value - not a floating-point.
_, err := converter.Convert("11qwerty00")
assert.ErrorContains(t, err, `unable to use "11qwerty00" as a floating-point number`)
}
{
// Happy path.
converted, err := converter.Convert("1.23")
assert.NoError(t, err)
bytes, ok := converted.([]byte)
assert.True(t, ok)
actualValue, err := converter.ToField("").DecodeDecimal(bytes)
assert.NoError(t, err)
assert.Equal(t, "1.23", fmt.Sprint(actualValue))
}
}

func TestGetScale(t *testing.T) {
type _testCase struct {
name string
value string
expectedScale int
}
type _testCase struct {
name string
value string
expectedScale int
}

testCases := []_testCase{
{
name: "0 scale",
value: "5",
expectedScale: 0,
},
{
name: "2 scale",
value: "9.99",
expectedScale: 2,
},
{
name: "5 scale",
value: "9.12345",
expectedScale: 5,
},
}
testCases := []_testCase{
{
name: "0 scale",
value: "5",
expectedScale: 0,
},
{
name: "2 scale",
value: "9.99",
expectedScale: 2,
},
{
name: "5 scale",
value: "9.12345",
expectedScale: 5,
},
}

for _, testCase := range testCases {
actualScale := getScale(testCase.value)
assert.Equal(t, testCase.expectedScale, actualScale, testCase.name)
}
for _, testCase := range testCases {
actualScale := getScale(testCase.value)
assert.Equal(t, testCase.expectedScale, actualScale, testCase.name)
}
}

func TestVariableNumericConverter_ToField(t *testing.T) {
converter := VariableNumericConverter{}
expected := debezium.Field{
FieldName: "col",
Type: "struct",
DebeziumType: "io.debezium.data.VariableScaleDecimal",
}
assert.Equal(t, expected, converter.ToField("col"))
converter := VariableNumericConverter{}
expected := debezium.Field{
FieldName: "col",
Type: "struct",
DebeziumType: "io.debezium.data.VariableScaleDecimal",
}
assert.Equal(t, expected, converter.ToField("col"))
}

func TestVariableNumericConverter_Convert(t *testing.T) {
converter := VariableNumericConverter{}
{
// Wrong type
_, err := converter.Convert(1234)
assert.ErrorContains(t, err, "expected string got int with value: 1234")
}
{
// Malformed value - emty string.
_, err := converter.Convert("")
assert.ErrorContains(t, err, "unable to use '' as a floating-point number")
}
{
// Malformed value - not a floating point.
_, err := converter.Convert("malformed")
assert.ErrorContains(t, err, "unable to use 'malformed' as a floating-point number")
}
{
// Happy path
converted, err := converter.Convert("12.34")
assert.NoError(t, err)
assert.Equal(t, map[string]any{"scale": int32(2), "value": []byte{0x4, 0xd2}}, converted)
actualValue, err := converter.ToField("").DecodeDebeziumVariableDecimal(converted)
assert.NoError(t, err)
assert.Equal(t, "12.34", actualValue.String())
}
converter := VariableNumericConverter{}
{
// Wrong type
_, err := converter.Convert(1234)
assert.ErrorContains(t, err, "expected string got int with value: 1234")
}
{
// Malformed value - empty string.
_, err := converter.Convert("")
assert.ErrorContains(t, err, `unable to use "" as a floating-point number`)
}
{
// Malformed value - not a floating point.
_, err := converter.Convert("malformed")
assert.ErrorContains(t, err, `unable to use "malformed" as a floating-point number`)
}
{
// Happy path
converted, err := converter.Convert("12.34")
assert.NoError(t, err)
assert.Equal(t, map[string]any{"scale": int32(2), "value": []byte{0x4, 0xd2}}, converted)
actualValue, err := converter.ToField("").DecodeDebeziumVariableDecimal(converted)
assert.NoError(t, err)
assert.Equal(t, "12.34", actualValue.String())
}
}
Loading

0 comments on commit 7cc7994

Please sign in to comment.