Skip to content

Commit

Permalink
Minor refactor.
Browse files Browse the repository at this point in the history
  • Loading branch information
Tang8330 committed Jul 24, 2024
1 parent d0af414 commit b2af6d6
Show file tree
Hide file tree
Showing 4 changed files with 88 additions and 61 deletions.
35 changes: 35 additions & 0 deletions lib/debezium/converters/time.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package converters

import (
"fmt"
"strings"

"github.com/artie-labs/transfer/lib/typing/ext"
)

func ConvertDateTimeWithTimezone(value any) (*ext.ExtendedTime, error) {
dtString, isOk := value.(string)
if !isOk {
return nil, fmt.Errorf("expected string got '%v' with type %T", value, value)
}

// We don't need to pass `additionalDateFormats` because this data type layout is standardized by Debezium
extTime, err := ext.ParseExtendedDateTime(dtString, nil)
if err == nil {
return extTime, nil
}

// Check for negative years
if strings.HasPrefix(dtString, "-") {
return nil, nil
}

if parts := strings.Split(dtString, "-"); len(parts) == 3 {
// Check if year exceeds 9999
if len(parts[0]) > 4 {
return nil, nil
}
}

return nil, fmt.Errorf("failed to parse %q, err: %w", dtString, err)
}
45 changes: 45 additions & 0 deletions lib/debezium/converters/time_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package converters

import (
"testing"
"time"

"github.com/artie-labs/transfer/lib/typing/ext"

"github.com/stretchr/testify/assert"
)

func TestConvertDateTimeWithTimezone(t *testing.T) {
{
// Invalid data
_, err := ConvertDateTimeWithTimezone(123)
assert.ErrorContains(t, err, "expected string got '123' with type int")
}
{
// Edge case (Year exceeds 9999)
val, err := ConvertDateTimeWithTimezone("+275760-09-13T00:00:00.000000Z")
assert.NoError(t, err)
assert.Nil(t, val)
}
{
// Edge case (Negative year)
val, err := ConvertDateTimeWithTimezone("-0999-10-10T10:10:10.000000Z")
assert.NoError(t, err)
assert.Nil(t, val)
}
{
// Valid
val, err := ConvertDateTimeWithTimezone("2025-09-13T00:00:00.000000Z")
assert.NoError(t, err)

expectedExtTime := &ext.ExtendedTime{
Time: time.Date(2025, time.September, 13, 0, 0, 0, 0, time.UTC),
NestedKind: ext.NestedKind{
Type: ext.DateTimeKindType,
Format: "2006-01-02T15:04:05Z07:00",
},
}

assert.Equal(t, expectedExtTime, val)
}
}
36 changes: 8 additions & 28 deletions lib/debezium/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@ package debezium
import (
"encoding/base64"
"fmt"
"strings"
"time"

"github.com/artie-labs/transfer/lib/debezium/converters"

"github.com/artie-labs/transfer/lib/config/constants"
"github.com/artie-labs/transfer/lib/jsonutil"
"github.com/artie-labs/transfer/lib/typing/decimal"
Expand Down Expand Up @@ -136,30 +137,7 @@ func (f Field) ParseValue(value any) (any, error) {
case KafkaVariableNumericType:
return f.DecodeDebeziumVariableDecimal(value)
case DateTimeWithTimezone:
dtString, isOk := value.(string)
if !isOk {
return nil, fmt.Errorf("expected string got '%v' with type %T", value, value)
}

// We don't need to pass `additionalDateFormats` because this data type layout is standardized by Debezium
extTime, err := ext.ParseExtendedDateTime(dtString, nil)
if err == nil {
return extTime, nil
}

// Check for negative years
if strings.HasPrefix(dtString, "-") {
return nil, nil
}

if parts := strings.Split(dtString, "-"); len(parts) == 3 {
// Check if year exceeds 9999
if len(parts[0]) > 4 {
return nil, nil
}
}

return nil, fmt.Errorf("failed to parse %q, err: %w", dtString, err)
return converters.ConvertDateTimeWithTimezone(value)
case
Timestamp,
MicroTimestamp,
Expand Down Expand Up @@ -247,15 +225,16 @@ func (f Field) DecodeDecimal(encoded []byte) (*decimal.Decimal, error) {
if err != nil {
return nil, fmt.Errorf("failed to get scale and/or precision: %w", err)
}

_decimal := DecodeDecimal(encoded, scale)
if precision == nil {
return decimal.NewDecimal(_decimal), nil
} else {
return decimal.NewDecimalWithPrecision(_decimal, *precision), nil
}

return decimal.NewDecimalWithPrecision(_decimal, *precision), nil
}

func (f Field) DecodeDebeziumVariableDecimal(value any) (*decimal.Decimal, error) {
func (Field) DecodeDebeziumVariableDecimal(value any) (*decimal.Decimal, error) {
valueStruct, isOk := value.(map[string]any)
if !isOk {
return nil, fmt.Errorf("value is not map[string]any type")
Expand All @@ -275,5 +254,6 @@ func (f Field) DecodeDebeziumVariableDecimal(value any) (*decimal.Decimal, error
if err != nil {
return nil, err
}

return decimal.NewDecimal(DecodeDecimal(bytes, scale)), nil
}
33 changes: 0 additions & 33 deletions lib/debezium/types_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -330,39 +330,6 @@ func TestField_ParseValue(t *testing.T) {
value: "1712609795827000",
expectedErr: "failed to cast value '1712609795827000' with type 'string' to int64",
},
{
name: "string - datetime with timezone",
field: Field{
Type: String,
DebeziumType: DateTimeWithTimezone,
},
value: "2025-09-13T00:00:00.000000Z",
expectedValue: &ext.ExtendedTime{
Time: time.Date(2025, time.September, 13, 0, 0, 0, 0, time.UTC),
NestedKind: ext.NestedKind{
Type: ext.DateTimeKindType,
Format: "2006-01-02T15:04:05Z07:00",
},
},
},
{
name: "string - datetime with timezone (edge case, year exceeds 9999)",
field: Field{
Type: String,
DebeziumType: DateTimeWithTimezone,
},
value: "+275760-09-13T00:00:00.000000Z",
expectedValue: nil,
},
{
name: "string - datetime with timezone (edge case, negative years)",
field: Field{
Type: String,
DebeziumType: DateTimeWithTimezone,
},
value: "-0999-10-10T10:10:10.000000Z",
expectedValue: nil,
},
{
name: "[]byte",
field: Field{
Expand Down

0 comments on commit b2af6d6

Please sign in to comment.