Skip to content

Commit

Permalink
WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
nathan-artie committed Jun 24, 2024
1 parent aa2902e commit 75f9c09
Show file tree
Hide file tree
Showing 2 changed files with 129 additions and 18 deletions.
39 changes: 24 additions & 15 deletions lib/debezium/decimal.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@ package debezium

import (
"fmt"
"log/slog"
"math/big"
"slices"

"github.com/artie-labs/transfer/lib/typing/decimal"
)
Expand All @@ -17,17 +19,19 @@ func EncodeDecimal(value string, scale uint16) ([]byte, error) {
scaledValue := new(big.Int).Exp(big.NewInt(10), big.NewInt(int64(scale)), nil)
bigFloatValue.Mul(bigFloatValue, new(big.Float).SetInt(scaledValue))

// Extract the scaled integer value.
bigIntValue := new(big.Int)
if _, success := bigIntValue.SetString(bigFloatValue.String(), 10); !success {
return nil, fmt.Errorf("unable to use %q as a floating-point number", value)
if !bigFloatValue.IsInt() {
// Add 0.5 before calling [Int] so that the value is always rounded up.
bigFloatValue.Add(bigFloatValue, new(big.Float).SetFloat64(0.5*float64(bigFloatValue.Sign())))
}
bigIntValue, _ := bigFloatValue.Int(nil)

data := bigIntValue.Bytes() // [Bytes] returns the absolute value.

data := bigIntValue.Bytes()
if bigIntValue.Sign() < 0 {
// Convert to two's complement if the number is negative
bigIntValue = bigIntValue.Neg(bigIntValue)
data = bigIntValue.Bytes()
if data[0] > 127 {
// If the first bit is set then prepend an empty byte
data = slices.Concat([]byte{0}, data)
}

// Inverting bits for two's complement.
for i := range data {
Expand Down Expand Up @@ -72,16 +76,21 @@ func DecodeDecimal(data []byte, precision *int, scale int) *decimal.Decimal {
bigInt.SetBytes(data)
}

slog.Info("decoded value", "x", bigInt.String())

// Convert the big integer to a big float
bigFloat := new(big.Float).SetInt(bigInt)

// Compute divisor as 10^scale with big.Int's Exp, then convert to big.Float
scaleInt := big.NewInt(int64(scale))
ten := big.NewInt(10)
divisorInt := new(big.Int).Exp(ten, scaleInt, nil)
divisorFloat := new(big.Float).SetInt(divisorInt)
if scale > 0 {
// Compute divisor as 10^scale with big.Int's Exp, then convert to big.Float
scaleInt := big.NewInt(int64(scale))
ten := big.NewInt(10)
divisorInt := new(big.Int).Exp(ten, scaleInt, nil)
divisorFloat := new(big.Float).SetInt(divisorInt)

// Perform the division
bigFloat.Quo(bigFloat, divisorFloat)
}

// Perform the division
bigFloat.Quo(bigFloat, divisorFloat)
return decimal.NewDecimal(precision, scale, bigFloat)
}
108 changes: 105 additions & 3 deletions lib/debezium/decimal_test.go
Original file line number Diff line number Diff line change
@@ -1,12 +1,110 @@
package debezium

import (
"fmt"
"log/slog"
"math/big"
"math/rand"
"strings"
"testing"

"github.com/stretchr/testify/assert"
)

func encodeDecode(value string, scale uint16) (string, error) {
bytes, err := EncodeDecimal(value, scale)
if err != nil {
return "", err
}
out := DecodeDecimal(bytes, nil, int(scale)).String()
slog.Info("encoded bytes", slog.String("in", value), slog.Any("bytes", bytes), slog.String("out", out))
return out, nil
}

func mustEncodeDecode(value string, scale uint16) string {
out, err := encodeDecode(value, scale)
if err != nil {
panic(err)
}

return out
}

func TestBtyes(t *testing.T) {
val1 := big.NewInt(65500)
val2 := big.NewInt(-65500)
assert.Equal(t, val1.Bytes(), val2.Bytes())
}

func TestEncodeDecimal(t *testing.T) {
// assert.Equal(t, "0", mustEncodeDecode("0", 0))
// assert.Equal(t, "0.0", mustEncodeDecode("0", 1))
// assert.Equal(t, "0.00", mustEncodeDecode("0", 2))
// assert.Equal(t, "0.00000000000000000000", mustEncodeDecode("0", 20))

// // Large scales:
// assert.Len(t, mustEncodeDecode("0", 1000), 1002)
// assert.Len(t, mustEncodeDecode("0", math.MaxUint16), math.MaxUint16+2)
// assert.Equal(t, ".", strings.Trim(mustEncodeDecode("0", math.MaxUint16), "0"), math.MaxUint16)

// // Tiny numbers:
// assert.Equal(t, "0.0000000000000000000", mustEncodeDecode("0.00000000000000000001", 19))
// assert.Equal(t, "0.0000000000000000001", mustEncodeDecode("0.00000000000000000005", 19))
// assert.Equal(t, "-0.0000000000000000001", mustEncodeDecode("-0.00000000000000000005", 19))
// assert.Equal(t, "0.00000000000000000001", mustEncodeDecode("0.00000000000000000001", 20))
// assert.Equal(t, "0.000000000000000000010", mustEncodeDecode("0.00000000000000000001", 21))

// assert.Equal(t, "100", mustEncodeDecode("100", 0))
// assert.Equal(t, "100.0", mustEncodeDecode("100", 1))
// assert.Equal(t, "100.00", mustEncodeDecode("100", 2))

// assert.Equal(t, "101", mustEncodeDecode("100.5", 0))
// assert.Equal(t, "100.5", mustEncodeDecode("100.5", 1))
// assert.Equal(t, "100.50", mustEncodeDecode("100.5", 2))

// assert.Equal(t, "-65500", mustEncodeDecode("-220", 0))
assert.Equal(t, "-65500", mustEncodeDecode("-65500", 0))
// assert.Equal(t, "-65500", mustEncodeDecode("-65501", 0))

for range 0 {
// scale := rand.Intn(100)
negative := rand.Intn(2) == 1
beforeDecimal := 1 + rand.Intn(10)
afterDecimal := 1

builder := strings.Builder{}
if negative {
builder.WriteRune('-')
}
var wroteNonZero bool
for range beforeDecimal {
digit := rand.Intn(10)
if digit == 0 {
if !wroteNonZero {
continue
}
} else {
wroteNonZero = true
}
builder.WriteString(fmt.Sprint(digit))
}
if !wroteNonZero {
continue
}

if afterDecimal > 0 {
builder.WriteRune('.')
}
for range afterDecimal {
builder.WriteString(fmt.Sprint(rand.Intn(10)))
}
number := builder.String()

assert.Equal(t, number, mustEncodeDecode(number, uint16(afterDecimal)), fmt.Sprintf("%s//%d", number, afterDecimal))
}
}

func TestEncodeDecimal_Symmetry(t *testing.T) {
testCases := []struct {
name string
value string
Expand Down Expand Up @@ -71,6 +169,11 @@ func TestEncodeDecimal(t *testing.T) {
value: "1.05",
scale: 2,
},
{
name: "negative number with a 255 initial byte",
value: "-65500",
scale: 0,
},
{
name: "malformed - empty string",
value: "",
Expand All @@ -84,10 +187,9 @@ func TestEncodeDecimal(t *testing.T) {
}

for _, testCase := range testCases {
encodedValue, err := EncodeDecimal(testCase.value, testCase.scale)
result, err := encodeDecode(testCase.value, testCase.scale)
if testCase.expectedErr == "" {
decodedValue := DecodeDecimal(encodedValue, nil, int(testCase.scale))
assert.Equal(t, testCase.value, decodedValue.String(), testCase.name)
assert.Equal(t, testCase.value, result, testCase.name)
} else {
assert.ErrorContains(t, err, testCase.expectedErr, testCase.name)
}
Expand Down

0 comments on commit 75f9c09

Please sign in to comment.