diff --git a/clients/bigquery/storagewrite_test.go b/clients/bigquery/storagewrite_test.go index 23f66115f..3527286f5 100644 --- a/clients/bigquery/storagewrite_test.go +++ b/clients/bigquery/storagewrite_test.go @@ -129,9 +129,9 @@ func TestRowToMessage(t *testing.T) { "c_float_int32": int32(1234), "c_float_int64": int64(1234), "c_float_string": "4444.55555", - "c_numeric": decimal.NewDecimal(nil, numbers.MustParseDecimal("3.14159")), + "c_numeric": decimal.NewDecimal(decimal.PrecisionNotSpecified, numbers.MustParseDecimal("3.14159")), "c_string": "foo bar", - "c_string_decimal": decimal.NewDecimal(nil, numbers.MustParseDecimal("1.61803")), + "c_string_decimal": decimal.NewDecimal(decimal.PrecisionNotSpecified, numbers.MustParseDecimal("1.61803")), "c_time": ext.NewExtendedTime(time.Date(0, 0, 0, 4, 5, 6, 7, time.UTC), ext.TimeKindType, ""), "c_date": ext.NewExtendedTime(time.Date(2001, 2, 3, 0, 0, 0, 0, time.UTC), ext.DateKindType, ""), "c_datetime": ext.NewExtendedTime(time.Date(2001, 2, 3, 4, 5, 6, 7, time.UTC), ext.DateTimeKindType, ""), diff --git a/lib/debezium/types.go b/lib/debezium/types.go index c7af11e9c..0b2f25823 100644 --- a/lib/debezium/types.go +++ b/lib/debezium/types.go @@ -8,7 +8,6 @@ import ( "github.com/artie-labs/transfer/lib/config/constants" "github.com/artie-labs/transfer/lib/jsonutil" - "github.com/artie-labs/transfer/lib/ptr" "github.com/artie-labs/transfer/lib/typing/decimal" "github.com/artie-labs/transfer/lib/maputil" @@ -232,11 +231,17 @@ func FromDebeziumTypeToTime(supportedType SupportedDebeziumType, val int64) (*ex // - `scale` (number of digits following decimal point) // - `connect.decimal.precision` which is an optional parameter. (If -1, then it's variable and .Value() will be in STRING). func (f Field) DecodeDecimal(encoded []byte) (*decimal.Decimal, error) { - scale, precision, err := f.GetScaleAndPrecision() + scale, precisionPtr, err := f.GetScaleAndPrecision() if err != nil { return nil, fmt.Errorf("failed to get scale and/or precision: %w", err) } _decimal := DecodeDecimal(encoded, int32(scale)) + + var precision int32 = decimal.PrecisionNotSpecified + if precisionPtr != nil { + precision = int32(*precisionPtr) + } + return decimal.NewDecimal(precision, _decimal), nil } @@ -261,5 +266,5 @@ func (f Field) DecodeDebeziumVariableDecimal(value any) (*decimal.Decimal, error return nil, err } _decimal := DecodeDecimal(bytes, int32(scale)) - return decimal.NewDecimal(ptr.ToInt(decimal.PrecisionNotSpecified), _decimal), nil + return decimal.NewDecimal(decimal.PrecisionNotSpecified, _decimal), nil } diff --git a/lib/typing/decimal/decimal.go b/lib/typing/decimal/decimal.go index 00ba25949..5cb429cd0 100644 --- a/lib/typing/decimal/decimal.go +++ b/lib/typing/decimal/decimal.go @@ -22,20 +22,18 @@ const ( MaxPrecisionBeforeString = 38 ) -func NewDecimal(precision *int, value *apd.Decimal) *Decimal { - if precision != nil { - scale := int(-value.Exponent) - if scale > *precision && *precision != -1 { - // Note: -1 precision means it's not specified. +func NewDecimal(precision int32, value *apd.Decimal) *Decimal { + scale := -value.Exponent + if scale > precision && precision != -1 { + // Note: -1 precision means it's not specified. - // This is typically not possible, but Postgres has a design flaw that allows you to do things like: NUMERIC(5, 6) which actually equates to NUMERIC(7, 6) - // We are setting precision to be scale + 1 to account for the leading zero for decimal numbers. - precision = ptr.ToInt(scale + 1) - } + // This is typically not possible, but Postgres has a design flaw that allows you to do things like: NUMERIC(5, 6) which actually equates to NUMERIC(7, 6) + // We are setting precision to be scale + 1 to account for the leading zero for decimal numbers. + precision = scale + 1 } return &Decimal{ - precision: precision, + precision: ptr.ToInt(int(precision)), value: value, } } diff --git a/lib/typing/decimal/decimal_test.go b/lib/typing/decimal/decimal_test.go index 666761b20..e652b4b3e 100644 --- a/lib/typing/decimal/decimal_test.go +++ b/lib/typing/decimal/decimal_test.go @@ -9,47 +9,38 @@ import ( ) func TestNewDecimal(t *testing.T) { - // Nil precision: - assert.Equal(t, "0", NewDecimal(nil, numbers.MustParseDecimal("0")).String()) - // Precision = -1 (PrecisionNotSpecified): - assert.Equal(t, DecimalDetails{scale: 2, precision: ptr.ToInt(-1)}, NewDecimal(ptr.ToInt(-1), numbers.MustParseDecimal("12.34")).Details()) + // PrecisionNotSpecified: + assert.Equal(t, DecimalDetails{scale: 2, precision: ptr.ToInt(-1)}, NewDecimal(PrecisionNotSpecified, numbers.MustParseDecimal("12.34")).Details()) // Precision = scale: - assert.Equal(t, DecimalDetails{scale: 2, precision: ptr.ToInt(2)}, NewDecimal(ptr.ToInt(2), numbers.MustParseDecimal("12.34")).Details()) + assert.Equal(t, DecimalDetails{scale: 2, precision: ptr.ToInt(2)}, NewDecimal(2, numbers.MustParseDecimal("12.34")).Details()) // Precision < scale: - assert.Equal(t, DecimalDetails{scale: 2, precision: ptr.ToInt(3)}, NewDecimal(ptr.ToInt(1), numbers.MustParseDecimal("12.34")).Details()) + assert.Equal(t, DecimalDetails{scale: 2, precision: ptr.ToInt(3)}, NewDecimal(1, numbers.MustParseDecimal("12.34")).Details()) // Precision > scale: - assert.Equal(t, DecimalDetails{scale: 2, precision: ptr.ToInt(4)}, NewDecimal(ptr.ToInt(4), numbers.MustParseDecimal("12.34")).Details()) + assert.Equal(t, DecimalDetails{scale: 2, precision: ptr.ToInt(4)}, NewDecimal(4, numbers.MustParseDecimal("12.34")).Details()) } func TestDecimal_Scale(t *testing.T) { - assert.Equal(t, 0, NewDecimal(nil, numbers.MustParseDecimal("0")).Scale()) - assert.Equal(t, 0, NewDecimal(nil, numbers.MustParseDecimal("12345")).Scale()) - assert.Equal(t, 0, NewDecimal(nil, numbers.MustParseDecimal("12300")).Scale()) - assert.Equal(t, 1, NewDecimal(nil, numbers.MustParseDecimal("12300.0")).Scale()) - assert.Equal(t, 2, NewDecimal(nil, numbers.MustParseDecimal("12300.00")).Scale()) - assert.Equal(t, 2, NewDecimal(nil, numbers.MustParseDecimal("12345.12")).Scale()) - assert.Equal(t, 3, NewDecimal(nil, numbers.MustParseDecimal("-12345.123")).Scale()) + assert.Equal(t, 0, NewDecimal(PrecisionNotSpecified, numbers.MustParseDecimal("0")).Scale()) + assert.Equal(t, 0, NewDecimal(PrecisionNotSpecified, numbers.MustParseDecimal("12345")).Scale()) + assert.Equal(t, 0, NewDecimal(PrecisionNotSpecified, numbers.MustParseDecimal("12300")).Scale()) + assert.Equal(t, 1, NewDecimal(PrecisionNotSpecified, numbers.MustParseDecimal("12300.0")).Scale()) + assert.Equal(t, 2, NewDecimal(PrecisionNotSpecified, numbers.MustParseDecimal("12300.00")).Scale()) + assert.Equal(t, 2, NewDecimal(PrecisionNotSpecified, numbers.MustParseDecimal("12345.12")).Scale()) + assert.Equal(t, 3, NewDecimal(PrecisionNotSpecified, numbers.MustParseDecimal("-12345.123")).Scale()) } func TestDecimal_Details(t *testing.T) { - // Nil precision: - assert.Equal(t, DecimalDetails{scale: 0}, NewDecimal(nil, numbers.MustParseDecimal("0")).Details()) - assert.Equal(t, DecimalDetails{scale: 0}, NewDecimal(nil, numbers.MustParseDecimal("12345")).Details()) - assert.Equal(t, DecimalDetails{scale: 0}, NewDecimal(nil, numbers.MustParseDecimal("-12")).Details()) - assert.Equal(t, DecimalDetails{scale: 2}, NewDecimal(nil, numbers.MustParseDecimal("12345.12")).Details()) - assert.Equal(t, DecimalDetails{scale: 3}, NewDecimal(nil, numbers.MustParseDecimal("-12345.123")).Details()) - // -1 precision (PrecisionNotSpecified): - assert.Equal(t, DecimalDetails{scale: 0, precision: ptr.ToInt(-1)}, NewDecimal(ptr.ToInt(-1), numbers.MustParseDecimal("0")).Details()) - assert.Equal(t, DecimalDetails{scale: 0, precision: ptr.ToInt(-1)}, NewDecimal(ptr.ToInt(-1), numbers.MustParseDecimal("12345")).Details()) - assert.Equal(t, DecimalDetails{scale: 0, precision: ptr.ToInt(-1)}, NewDecimal(ptr.ToInt(-1), numbers.MustParseDecimal("-12")).Details()) - assert.Equal(t, DecimalDetails{scale: 2, precision: ptr.ToInt(-1)}, NewDecimal(ptr.ToInt(-1), numbers.MustParseDecimal("12345.12")).Details()) - assert.Equal(t, DecimalDetails{scale: 3, precision: ptr.ToInt(-1)}, NewDecimal(ptr.ToInt(-1), numbers.MustParseDecimal("-12345.123")).Details()) + assert.Equal(t, DecimalDetails{scale: 0, precision: ptr.ToInt(-1)}, NewDecimal(PrecisionNotSpecified, numbers.MustParseDecimal("0")).Details()) + assert.Equal(t, DecimalDetails{scale: 0, precision: ptr.ToInt(-1)}, NewDecimal(PrecisionNotSpecified, numbers.MustParseDecimal("12345")).Details()) + assert.Equal(t, DecimalDetails{scale: 0, precision: ptr.ToInt(-1)}, NewDecimal(PrecisionNotSpecified, numbers.MustParseDecimal("-12")).Details()) + assert.Equal(t, DecimalDetails{scale: 2, precision: ptr.ToInt(-1)}, NewDecimal(PrecisionNotSpecified, numbers.MustParseDecimal("12345.12")).Details()) + assert.Equal(t, DecimalDetails{scale: 3, precision: ptr.ToInt(-1)}, NewDecimal(PrecisionNotSpecified, numbers.MustParseDecimal("-12345.123")).Details()) // 10 precision: - assert.Equal(t, DecimalDetails{scale: 0, precision: ptr.ToInt(10)}, NewDecimal(ptr.ToInt(10), numbers.MustParseDecimal("0")).Details()) - assert.Equal(t, DecimalDetails{scale: 0, precision: ptr.ToInt(10)}, NewDecimal(ptr.ToInt(10), numbers.MustParseDecimal("12345")).Details()) - assert.Equal(t, DecimalDetails{scale: 0, precision: ptr.ToInt(10)}, NewDecimal(ptr.ToInt(10), numbers.MustParseDecimal("-12")).Details()) - assert.Equal(t, DecimalDetails{scale: 2, precision: ptr.ToInt(10)}, NewDecimal(ptr.ToInt(10), numbers.MustParseDecimal("12345.12")).Details()) - assert.Equal(t, DecimalDetails{scale: 3, precision: ptr.ToInt(10)}, NewDecimal(ptr.ToInt(10), numbers.MustParseDecimal("-12345.123")).Details()) + assert.Equal(t, DecimalDetails{scale: 0, precision: ptr.ToInt(10)}, NewDecimal(10, numbers.MustParseDecimal("0")).Details()) + assert.Equal(t, DecimalDetails{scale: 0, precision: ptr.ToInt(10)}, NewDecimal(10, numbers.MustParseDecimal("12345")).Details()) + assert.Equal(t, DecimalDetails{scale: 0, precision: ptr.ToInt(10)}, NewDecimal(10, numbers.MustParseDecimal("-12")).Details()) + assert.Equal(t, DecimalDetails{scale: 2, precision: ptr.ToInt(10)}, NewDecimal(10, numbers.MustParseDecimal("12345.12")).Details()) + assert.Equal(t, DecimalDetails{scale: 3, precision: ptr.ToInt(10)}, NewDecimal(10, numbers.MustParseDecimal("-12345.123")).Details()) }