Upgrade deps + fix ZonedTimestampConverter (#496)

go.mod

@@ -4,7 +4,7 @@ go 1.23.0
 
 require (
 	github.com/DataDog/datadog-go/v5 v5.5.0
-	github.com/artie-labs/transfer v1.27.8
+	github.com/artie-labs/transfer v1.27.9
 	github.com/aws/aws-sdk-go-v2 v1.30.3
 	github.com/aws/aws-sdk-go-v2/config v1.27.27
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.27

go.sum

@@ -93,8 +93,8 @@ github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlE
 github.com/apache/thrift v0.14.2/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/apache/thrift v0.17.0 h1:cMd2aj52n+8VoAtvSvLn4kDC3aZ6IAkBuqWQ2IDu7wo=
 github.com/apache/thrift v0.17.0/go.mod h1:OLxhMRJxomX+1I/KUw03qoV3mMz16BwaKI+d4fPBx7Q=
-github.com/artie-labs/transfer v1.27.8 h1:mUr9OV/z8HUFMqtxlCQDv57FHjMtVF3WEvb/G81vFGU=
-github.com/artie-labs/transfer v1.27.8/go.mod h1:+a/UhlQVRIpdz3muS1yhSvyX42RQL0LHOdovGZfEsDE=
+github.com/artie-labs/transfer v1.27.9 h1:/X8+sJYTtOB6Jf6JZr9fXiDaowpYAUTzg6KWJte9ELU=
+github.com/artie-labs/transfer v1.27.9/go.mod h1:+a/UhlQVRIpdz3muS1yhSvyX42RQL0LHOdovGZfEsDE=
 github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/aws/aws-sdk-go-v2 v1.16.12/go.mod h1:C+Ym0ag2LIghJbXhfXZ0YEEp49rBWowxKzJLUoob0ts=
 github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY=

integration_tests/mongo/main.go

@@ -183,11 +183,11 @@ func testTypes(ctx context.Context, db *mongo.Database, mongoCfg config.MongoDB)
 	expectedPayload := map[string]any{
 		"objectId":                "66a95fae3776c2f21f0ff568",
 		"array":                   []any{"item1", int32(2), true, 3.14},
-		"datetime":                ext.NewExtendedTime(ts, ext.DateTimeKindType, "2006-01-02T15:04:05.999-07:00"),
+		"datetime":                ext.NewExtendedTime(ts, ext.TimestampTzKindType, "2006-01-02T15:04:05.999-07:00"),
 		"int64":                   int64(64),
 		"__artie_delete":          false,
 		"__artie_only_set_delete": false,
-		"timestamp":               ext.NewExtendedTime(ts, ext.DateTimeKindType, "2006-01-02T15:04:05.999-07:00"),
+		"timestamp":               ext.NewExtendedTime(ts, ext.TimestampTzKindType, "2006-01-02T15:04:05.999-07:00"),
 		"embeddedDocument":        `{"field1":"value1","field2":"value2"}`,
 		"embeddedMap":             `{"foo":"bar","hello":"world","pi":3.14159}`,
 		"binary":                  `{"$binary":{"base64":"YmluYXJ5IGRhdGE=","subType":"00"}}`,

lib/debezium/converters/time.go

@@ -188,7 +188,10 @@ func (ZonedTimestampConverter) Convert(value any) (any, error) {
 		return nil, nil
 	}
 
-	return timeValue.Format(time.RFC3339Nano), nil
+	// A string representation of a timestamp with timezone information, where the timezone is GMT.
+	// This layout supports upto microsecond precision.
+	layout := "2006-01-02T15:04:05.999999Z"
+	return timeValue.UTC().Format(layout), nil
 }
 
 type YearConverter struct{}

lib/debezium/converters/time_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/artie-labs/transfer/lib/debezium/converters"
 	"github.com/artie-labs/transfer/lib/typing"
 	"github.com/artie-labs/transfer/lib/typing/ext"
 	"github.com/stretchr/testify/assert"
@@ -245,7 +246,7 @@ func TestMicroTimestampConverter_Convert(t *testing.T) {
 		transferValue, err := parseUsingTransfer(converter, value.(int64))
 		assert.NoError(t, err)
 		assert.Equal(t, timeValue, transferValue.GetTime())
-		assert.Equal(t, ext.DateTimeKindType, transferValue.GetNestedKind().Type)
+		assert.Equal(t, ext.TimestampTzKindType, transferValue.GetNestedKind().Type)
 	}
 }
 
@@ -285,9 +286,51 @@ func TestZonedTimestampConverter_Convert(t *testing.T) {
 	}
 	{
 		// time.Time
-		value, err := converter.Convert(time.Date(2001, 2, 3, 4, 5, 0, 0, time.UTC))
+		_ts := time.Date(2001, 2, 3, 4, 5, 0, 0, time.UTC)
+		value, err := converter.Convert(_ts)
 		assert.NoError(t, err)
 		assert.Equal(t, "2001-02-03T04:05:00Z", value)
+
+		// Check Transfer to ensure no precision loss
+		ts, err := converters.DateTimeWithTimezone{}.Convert(value)
+		assert.NoError(t, err)
+		assert.Equal(t, _ts, ts.(*ext.ExtendedTime).GetTime())
+	}
+	{
+		// time.Time (ms)
+		_ts := time.Date(2001, 2, 3, 4, 5, 1, 900000, time.UTC)
+		value, err := converter.Convert(_ts)
+		assert.NoError(t, err)
+		assert.Equal(t, "2001-02-03T04:05:01.0009Z", value)
+
+		// Check Transfer to ensure no precision loss
+		ts, err := converters.DateTimeWithTimezone{}.Convert(value)
+		assert.NoError(t, err)
+		assert.Equal(t, _ts, ts.(*ext.ExtendedTime).GetTime())
+	}
+	{
+		// time.Time (microseconds)
+		_ts := time.Date(2001, 2, 3, 4, 5, 1, 909000, time.UTC)
+		value, err := converter.Convert(_ts)
+		assert.NoError(t, err)
+		assert.Equal(t, "2001-02-03T04:05:01.000909Z", value)
+
+		// Check Transfer to ensure no precision loss
+		ts, err := converters.DateTimeWithTimezone{}.Convert(value)
+		assert.NoError(t, err)
+		assert.Equal(t, _ts, ts.(*ext.ExtendedTime).GetTime())
+	}
+	{
+		// Different timezone
+		_ts := time.Date(2001, 2, 3, 4, 5, 0, 0, time.FixedZone("CET", 1*60*60))
+		value, err := converter.Convert(_ts)
+		assert.NoError(t, err)
+		assert.Equal(t, "2001-02-03T03:05:00Z", value)
+
+		// Check Transfer to ensure no precision loss
+		ts, err := converters.DateTimeWithTimezone{}.Convert(value)
+		assert.NoError(t, err)
+		assert.Equal(t, _ts.UTC(), ts.(*ext.ExtendedTime).GetTime())
 	}
 }
 

lib/mongo/message_test.go

@@ -106,7 +106,7 @@ func TestParseMessage(t *testing.T) {
 		"decimal":     "1234.5",
 		"subDocument": `{"nestedString":"Nested value"}`,
 		"array":       []any{"apple", "banana", "cherry"},
-		"datetime":    ext.NewExtendedTime(time.Date(2024, time.February, 13, 20, 37, 48, 0, time.UTC), ext.DateTimeKindType, "2006-01-02T15:04:05.999-07:00"),
+		"datetime":    ext.NewExtendedTime(time.Date(2024, time.February, 13, 20, 37, 48, 0, time.UTC), ext.TimestampTzKindType, "2006-01-02T15:04:05.999-07:00"),
 		"trueValue":   true,
 		"falseValue":  false,
 		"nullValue":   nil,