diff --git a/go.mod b/go.mod index 5919e221..705adfab 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.23.0 require ( github.com/DataDog/datadog-go/v5 v5.5.0 - github.com/artie-labs/transfer v1.27.8 + github.com/artie-labs/transfer v1.27.9 github.com/aws/aws-sdk-go-v2 v1.30.3 github.com/aws/aws-sdk-go-v2/config v1.27.27 github.com/aws/aws-sdk-go-v2/credentials v1.17.27 diff --git a/go.sum b/go.sum index 3de86f3e..344a2f54 100644 --- a/go.sum +++ b/go.sum @@ -93,8 +93,8 @@ github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlE github.com/apache/thrift v0.14.2/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.17.0 h1:cMd2aj52n+8VoAtvSvLn4kDC3aZ6IAkBuqWQ2IDu7wo= github.com/apache/thrift v0.17.0/go.mod h1:OLxhMRJxomX+1I/KUw03qoV3mMz16BwaKI+d4fPBx7Q= -github.com/artie-labs/transfer v1.27.8 h1:mUr9OV/z8HUFMqtxlCQDv57FHjMtVF3WEvb/G81vFGU= -github.com/artie-labs/transfer v1.27.8/go.mod h1:+a/UhlQVRIpdz3muS1yhSvyX42RQL0LHOdovGZfEsDE= +github.com/artie-labs/transfer v1.27.9 h1:/X8+sJYTtOB6Jf6JZr9fXiDaowpYAUTzg6KWJte9ELU= +github.com/artie-labs/transfer v1.27.9/go.mod h1:+a/UhlQVRIpdz3muS1yhSvyX42RQL0LHOdovGZfEsDE= github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go-v2 v1.16.12/go.mod h1:C+Ym0ag2LIghJbXhfXZ0YEEp49rBWowxKzJLUoob0ts= github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY= diff --git a/integration_tests/mongo/main.go b/integration_tests/mongo/main.go index a0a4e811..ed78eac0 100644 --- a/integration_tests/mongo/main.go +++ b/integration_tests/mongo/main.go @@ -183,11 +183,11 @@ func testTypes(ctx context.Context, db *mongo.Database, mongoCfg config.MongoDB) expectedPayload := map[string]any{ "objectId": "66a95fae3776c2f21f0ff568", "array": []any{"item1", int32(2), true, 3.14}, - "datetime": ext.NewExtendedTime(ts, ext.DateTimeKindType, "2006-01-02T15:04:05.999-07:00"), + "datetime": ext.NewExtendedTime(ts, ext.TimestampTzKindType, "2006-01-02T15:04:05.999-07:00"), "int64": int64(64), "__artie_delete": false, "__artie_only_set_delete": false, - "timestamp": ext.NewExtendedTime(ts, ext.DateTimeKindType, "2006-01-02T15:04:05.999-07:00"), + "timestamp": ext.NewExtendedTime(ts, ext.TimestampTzKindType, "2006-01-02T15:04:05.999-07:00"), "embeddedDocument": `{"field1":"value1","field2":"value2"}`, "embeddedMap": `{"foo":"bar","hello":"world","pi":3.14159}`, "binary": `{"$binary":{"base64":"YmluYXJ5IGRhdGE=","subType":"00"}}`, diff --git a/lib/debezium/converters/time.go b/lib/debezium/converters/time.go index b8fbe41c..a83902a6 100644 --- a/lib/debezium/converters/time.go +++ b/lib/debezium/converters/time.go @@ -188,7 +188,10 @@ func (ZonedTimestampConverter) Convert(value any) (any, error) { return nil, nil } - return timeValue.Format(time.RFC3339Nano), nil + // A string representation of a timestamp with timezone information, where the timezone is GMT. + // This layout supports upto microsecond precision. + layout := "2006-01-02T15:04:05.999999Z" + return timeValue.UTC().Format(layout), nil } type YearConverter struct{} diff --git a/lib/debezium/converters/time_test.go b/lib/debezium/converters/time_test.go index 237ddc7d..eadadeef 100644 --- a/lib/debezium/converters/time_test.go +++ b/lib/debezium/converters/time_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/artie-labs/transfer/lib/debezium/converters" "github.com/artie-labs/transfer/lib/typing" "github.com/artie-labs/transfer/lib/typing/ext" "github.com/stretchr/testify/assert" @@ -245,7 +246,7 @@ func TestMicroTimestampConverter_Convert(t *testing.T) { transferValue, err := parseUsingTransfer(converter, value.(int64)) assert.NoError(t, err) assert.Equal(t, timeValue, transferValue.GetTime()) - assert.Equal(t, ext.DateTimeKindType, transferValue.GetNestedKind().Type) + assert.Equal(t, ext.TimestampTzKindType, transferValue.GetNestedKind().Type) } } @@ -285,9 +286,51 @@ func TestZonedTimestampConverter_Convert(t *testing.T) { } { // time.Time - value, err := converter.Convert(time.Date(2001, 2, 3, 4, 5, 0, 0, time.UTC)) + _ts := time.Date(2001, 2, 3, 4, 5, 0, 0, time.UTC) + value, err := converter.Convert(_ts) assert.NoError(t, err) assert.Equal(t, "2001-02-03T04:05:00Z", value) + + // Check Transfer to ensure no precision loss + ts, err := converters.DateTimeWithTimezone{}.Convert(value) + assert.NoError(t, err) + assert.Equal(t, _ts, ts.(*ext.ExtendedTime).GetTime()) + } + { + // time.Time (ms) + _ts := time.Date(2001, 2, 3, 4, 5, 1, 900000, time.UTC) + value, err := converter.Convert(_ts) + assert.NoError(t, err) + assert.Equal(t, "2001-02-03T04:05:01.0009Z", value) + + // Check Transfer to ensure no precision loss + ts, err := converters.DateTimeWithTimezone{}.Convert(value) + assert.NoError(t, err) + assert.Equal(t, _ts, ts.(*ext.ExtendedTime).GetTime()) + } + { + // time.Time (microseconds) + _ts := time.Date(2001, 2, 3, 4, 5, 1, 909000, time.UTC) + value, err := converter.Convert(_ts) + assert.NoError(t, err) + assert.Equal(t, "2001-02-03T04:05:01.000909Z", value) + + // Check Transfer to ensure no precision loss + ts, err := converters.DateTimeWithTimezone{}.Convert(value) + assert.NoError(t, err) + assert.Equal(t, _ts, ts.(*ext.ExtendedTime).GetTime()) + } + { + // Different timezone + _ts := time.Date(2001, 2, 3, 4, 5, 0, 0, time.FixedZone("CET", 1*60*60)) + value, err := converter.Convert(_ts) + assert.NoError(t, err) + assert.Equal(t, "2001-02-03T03:05:00Z", value) + + // Check Transfer to ensure no precision loss + ts, err := converters.DateTimeWithTimezone{}.Convert(value) + assert.NoError(t, err) + assert.Equal(t, _ts.UTC(), ts.(*ext.ExtendedTime).GetTime()) } } diff --git a/lib/mongo/message_test.go b/lib/mongo/message_test.go index 509bcae4..70af4a74 100644 --- a/lib/mongo/message_test.go +++ b/lib/mongo/message_test.go @@ -106,7 +106,7 @@ func TestParseMessage(t *testing.T) { "decimal": "1234.5", "subDocument": `{"nestedString":"Nested value"}`, "array": []any{"apple", "banana", "cherry"}, - "datetime": ext.NewExtendedTime(time.Date(2024, time.February, 13, 20, 37, 48, 0, time.UTC), ext.DateTimeKindType, "2006-01-02T15:04:05.999-07:00"), + "datetime": ext.NewExtendedTime(time.Date(2024, time.February, 13, 20, 37, 48, 0, time.UTC), ext.TimestampTzKindType, "2006-01-02T15:04:05.999-07:00"), "trueValue": true, "falseValue": false, "nullValue": nil,