From 288ccddf55d9770350a95eadd18f6d85096132ea Mon Sep 17 00:00:00 2001 From: Nathan <148575555+nathan-artie@users.noreply.github.com> Date: Fri, 23 Aug 2024 15:37:03 -0700 Subject: [PATCH] Remove `debezium.EncodeDecimalWithScale` (#848) --- lib/debezium/decimal.go | 38 --------- lib/debezium/decimal_test.go | 157 ----------------------------------- 2 files changed, 195 deletions(-) diff --git a/lib/debezium/decimal.go b/lib/debezium/decimal.go index 7e46c997c..9fba7aac1 100644 --- a/lib/debezium/decimal.go +++ b/lib/debezium/decimal.go @@ -61,33 +61,6 @@ func decodeBigInt(data []byte) *big.Int { return bigInt } -// decimalWithNewExponent takes a [*apd.Decimal] and returns a new [*apd.Decimal] with a the given exponent. -// If the new exponent is less precise then the extra digits will be truncated. -func decimalWithNewExponent(decimal *apd.Decimal, newExponent int32) *apd.Decimal { - exponentDelta := newExponent - decimal.Exponent // Exponent is negative. - - if exponentDelta == 0 { - return new(apd.Decimal).Set(decimal) - } - - coefficient := new(apd.BigInt).Set(&decimal.Coeff) - - if exponentDelta < 0 { - multiplier := new(apd.BigInt).Exp(apd.NewBigInt(10), apd.NewBigInt(int64(-exponentDelta)), nil) - coefficient.Mul(coefficient, multiplier) - } else if exponentDelta > 0 { - divisor := new(apd.BigInt).Exp(apd.NewBigInt(10), apd.NewBigInt(int64(exponentDelta)), nil) - coefficient.Div(coefficient, divisor) - } - - return &apd.Decimal{ - Form: decimal.Form, - Negative: decimal.Negative, - Exponent: newExponent, - Coeff: *coefficient, - } -} - // EncodeDecimal is used to encode a [*apd.Decimal] to `org.apache.kafka.connect.data.Decimal`. // The scale of the value (which is the negated exponent of the decimal) is returned as the second argument. func EncodeDecimal(decimal *apd.Decimal) ([]byte, int32) { @@ -99,17 +72,6 @@ func EncodeDecimal(decimal *apd.Decimal) ([]byte, int32) { return encodeBigInt(bigIntValue), -decimal.Exponent } -// EncodeDecimalWithScale is used to encode a [*apd.Decimal] to `org.apache.kafka.connect.data.Decimal` -// using a specific scale. -func EncodeDecimalWithScale(decimal *apd.Decimal, scale int32) []byte { - targetExponent := -scale // Negate scale since [Decimal.Exponent] is negative. - if decimal.Exponent != targetExponent { - decimal = decimalWithNewExponent(decimal, targetExponent) - } - bytes, _ := EncodeDecimal(decimal) - return bytes -} - // DecodeDecimal is used to decode `org.apache.kafka.connect.data.Decimal`. func DecodeDecimal(data []byte, scale int32) *apd.Decimal { bigInt := new(apd.BigInt).SetMathBigInt(decodeBigInt(data)) diff --git a/lib/debezium/decimal_test.go b/lib/debezium/decimal_test.go index 931468710..bd0222772 100644 --- a/lib/debezium/decimal_test.go +++ b/lib/debezium/decimal_test.go @@ -1,13 +1,11 @@ package debezium import ( - "fmt" "math" "math/big" "testing" "github.com/artie-labs/transfer/lib/numbers" - "github.com/cockroachdb/apd/v3" "github.com/stretchr/testify/assert" ) @@ -42,26 +40,6 @@ func TestDecodeBigInt(t *testing.T) { } } -func TestDecimalWithNewExponent(t *testing.T) { - assert.Equal(t, "0", decimalWithNewExponent(apd.New(0, 0), 0).Text('f')) - assert.Equal(t, "00", decimalWithNewExponent(apd.New(0, 1), 1).Text('f')) - assert.Equal(t, "0", decimalWithNewExponent(apd.New(0, 100), 0).Text('f')) - assert.Equal(t, "00", decimalWithNewExponent(apd.New(0, 0), 1).Text('f')) - assert.Equal(t, "0.0", decimalWithNewExponent(apd.New(0, 0), -1).Text('f')) - - // Same exponent: - assert.Equal(t, "12.349", decimalWithNewExponent(numbers.MustParseDecimal("12.349"), -3).Text('f')) - // More precise exponent: - assert.Equal(t, "12.3490", decimalWithNewExponent(numbers.MustParseDecimal("12.349"), -4).Text('f')) - assert.Equal(t, "12.34900", decimalWithNewExponent(numbers.MustParseDecimal("12.349"), -5).Text('f')) - // Lest precise exponent: - // Extra digits should be truncated rather than rounded. - assert.Equal(t, "12.34", decimalWithNewExponent(numbers.MustParseDecimal("12.349"), -2).Text('f')) - assert.Equal(t, "12.3", decimalWithNewExponent(numbers.MustParseDecimal("12.349"), -1).Text('f')) - assert.Equal(t, "12", decimalWithNewExponent(numbers.MustParseDecimal("12.349"), 0).Text('f')) - assert.Equal(t, "10", decimalWithNewExponent(numbers.MustParseDecimal("12.349"), 1).Text('f')) -} - func TestEncodeDecimal(t *testing.T) { testEncodeDecimal := func(value string, expectedScale int32) { bytes, scale := EncodeDecimal(numbers.MustParseDecimal(value)) @@ -83,138 +61,3 @@ func TestEncodeDecimal(t *testing.T) { testEncodeDecimal("145.183000000000009", 15) testEncodeDecimal("-145.183000000000009", 15) } - -func TestEncodeDecimalWithScale(t *testing.T) { - mustEncodeAndDecodeDecimal := func(value string, scale int32) string { - bytes := EncodeDecimalWithScale(numbers.MustParseDecimal(value), scale) - return DecodeDecimal(bytes, scale).String() - } - - // Whole numbers: - for i := range 100_000 { - strValue := fmt.Sprint(i) - assert.Equal(t, strValue, mustEncodeAndDecodeDecimal(strValue, 0)) - if i != 0 { - strValue := "-" + strValue - assert.Equal(t, strValue, mustEncodeAndDecodeDecimal(strValue, 0)) - } - } - - // Scale of 15 that is equal to the amount of decimal places in the value: - assert.Equal(t, "145.183000000000000", mustEncodeAndDecodeDecimal("145.183000000000000", 15)) - assert.Equal(t, "-145.183000000000000", mustEncodeAndDecodeDecimal("-145.183000000000000", 15)) - // If scale is smaller than the amount of decimal places then the extra places should be truncated without rounding: - assert.Equal(t, "145.18300000000000", mustEncodeAndDecodeDecimal("145.183000000000000", 14)) - assert.Equal(t, "145.18300000000000", mustEncodeAndDecodeDecimal("145.183000000000005", 14)) - assert.Equal(t, "-145.18300000000000", mustEncodeAndDecodeDecimal("-145.183000000000005", 14)) - assert.Equal(t, "145.18300000000000", mustEncodeAndDecodeDecimal("145.183000000000009", 14)) - assert.Equal(t, "-145.18300000000000", mustEncodeAndDecodeDecimal("-145.183000000000009", 14)) - assert.Equal(t, "-145.18300000000000", mustEncodeAndDecodeDecimal("-145.183000000000000", 14)) - assert.Equal(t, "145.18300000000000", mustEncodeAndDecodeDecimal("145.183000000000001", 14)) - assert.Equal(t, "-145.18300000000000", mustEncodeAndDecodeDecimal("-145.183000000000001", 14)) - assert.Equal(t, "145.18300000000000", mustEncodeAndDecodeDecimal("145.183000000000004", 14)) - assert.Equal(t, "-145.18300000000000", mustEncodeAndDecodeDecimal("-145.183000000000004", 14)) - // If scale is larger than the amount of decimal places then the extra places should be padded with zeros: - assert.Equal(t, "145.1830000000000000", mustEncodeAndDecodeDecimal("145.183000000000000", 16)) - assert.Equal(t, "-145.1830000000000000", mustEncodeAndDecodeDecimal("-145.183000000000000", 16)) - assert.Equal(t, "145.1830000000000010", mustEncodeAndDecodeDecimal("145.183000000000001", 16)) - assert.Equal(t, "-145.1830000000000010", mustEncodeAndDecodeDecimal("-145.183000000000001", 16)) - assert.Equal(t, "145.1830000000000040", mustEncodeAndDecodeDecimal("145.183000000000004", 16)) - assert.Equal(t, "-145.1830000000000040", mustEncodeAndDecodeDecimal("-145.183000000000004", 16)) - assert.Equal(t, "145.1830000000000050", mustEncodeAndDecodeDecimal("145.183000000000005", 16)) - assert.Equal(t, "-145.1830000000000050", mustEncodeAndDecodeDecimal("-145.183000000000005", 16)) - assert.Equal(t, "145.1830000000000090", mustEncodeAndDecodeDecimal("145.183000000000009", 16)) - assert.Equal(t, "-145.1830000000000090", mustEncodeAndDecodeDecimal("-145.183000000000009", 16)) - - assert.Equal(t, "-9063701308.217222135", mustEncodeAndDecodeDecimal("-9063701308.217222135", 9)) - assert.Equal(t, "-74961544796695.89960242", mustEncodeAndDecodeDecimal("-74961544796695.89960242", 8)) - - testCases := []struct { - name string - value string - scale int32 - }{ - { - name: "0 scale", - value: "5", - }, - { - name: "2 scale", - value: "23131319.99", - scale: 2, - }, - { - name: "5 scale", - value: "9.12345", - scale: 5, - }, - { - name: "negative number", - value: "-105.2813669", - scale: 7, - }, - // Longitude #1 - { - name: "long 1", - value: "-75.765611", - scale: 6, - }, - // Latitude #1 - { - name: "lat", - value: "40.0335495", - scale: 7, - }, - // Long #2 - { - name: "long 2", - value: "-119.65575", - scale: 5, - }, - { - name: "lat 2", - value: "36.3303", - scale: 4, - }, - { - name: "long 3", - value: "-81.76254098", - scale: 8, - }, - { - name: "amount", - value: "6408.355", - scale: 3, - }, - { - name: "total", - value: "1.05", - scale: 2, - }, - { - name: "negative number: 2^16 - 255", - value: "-65281", - scale: 0, - }, - { - name: "negative number: 2^16 - 1", - value: "-65535", - scale: 0, - }, - { - name: "number with a scale of 15", - value: "0.000022998904125", - scale: 15, - }, - { - name: "number with a scale of 15", - value: "145.183000000000000", - scale: 15, - }, - } - - for _, testCase := range testCases { - actual := mustEncodeAndDecodeDecimal(testCase.value, testCase.scale) - assert.Equal(t, testCase.value, actual, testCase.name) - } -}