[DateTimeWithTimezone] Native parsing (#831)

artie-labs · Aug 12, 2024 · c300864 · c300864
1 parent 502122d
commit c300864
Show file tree

Hide file tree

Showing 2 changed files with 146 additions and 21 deletions.
diff --git a/lib/debezium/converters/time.go b/lib/debezium/converters/time.go
@@ -25,37 +25,51 @@ func (Time) Convert(val any) (any, error) {
 	return ext.NewExtendedTime(time.UnixMilli(valInt64).In(time.UTC), ext.TimeKindType, ""), nil
 }
 
+var SupportedDateTimeWithTimezoneFormats = []string{
+	"2006-01-02T15:04:05Z",         // w/o fractional seconds
+	"2006-01-02T15:04:05.0Z",       // 1 digit
+	"2006-01-02T15:04:05.00Z",      // 2 digits
+	"2006-01-02T15:04:05.000Z",     // 3 digits
+	"2006-01-02T15:04:05.0000Z",    // 4 digits
+	"2006-01-02T15:04:05.00000Z",   // 5 digits
+	"2006-01-02T15:04:05.000000Z",  // 6 digits
+	"2006-01-02T15:04:05.0000000Z", // 7 digits
+}
+
 type DateTimeWithTimezone struct{}
 
 func (DateTimeWithTimezone) ToKindDetails() typing.KindDetails {
 	return typing.NewKindDetailsFromTemplate(typing.ETime, ext.DateTimeKindType)
 }
 
 func (DateTimeWithTimezone) Convert(value any) (any, error) {
-	dtString, isOk := value.(string)
+	valString, isOk := value.(string)
 	if !isOk {
 		return nil, fmt.Errorf("expected string got '%v' with type %T", value, value)
 	}
 
-	// We don't need to pass `additionalDateFormats` because this data type layout is standardized by Debezium
-	extTime, err := ext.ParseExtendedDateTime(dtString, nil)
-	if err == nil {
-		return extTime, nil
-	}
-
 	// Check for negative years
-	if strings.HasPrefix(dtString, "-") {
+	if strings.HasPrefix(valString, "-") {
 		return nil, nil
 	}
 
-	if parts := strings.Split(dtString, "-"); len(parts) == 3 {
+	if parts := strings.Split(valString, "-"); len(parts) == 3 {
 		// Check if year exceeds 9999
 		if len(parts[0]) > 4 {
 			return nil, nil
 		}
 	}
 
-	return nil, fmt.Errorf("failed to parse %q, err: %w", dtString, err)
+	var err error
+	var ts time.Time
+	for _, supportedFormat := range SupportedDateTimeWithTimezoneFormats {
+		ts, err = ext.ParseTimeExactMatch(supportedFormat, valString)
+		if err == nil {
+			return ext.NewExtendedTime(ts, ext.DateTimeKindType, supportedFormat), nil
+		}
+	}
+
+	return nil, fmt.Errorf("failed to parse %q, err: %w", valString, err)
 }
 
 var SupportedTimeWithTimezoneFormats = []string{

diff --git a/lib/debezium/converters/time_test.go b/lib/debezium/converters/time_test.go
@@ -29,21 +29,132 @@ func TestConvertDateTimeWithTimezone(t *testing.T) {
 	}
 	{
 		// Valid
-		val, err := DateTimeWithTimezone{}.Convert("2025-09-13T00:00:00.000000Z")
-		assert.NoError(t, err)
+		{
+			// No fractional seconds
+			val, err := DateTimeWithTimezone{}.Convert("2025-09-13T00:00:00Z")
+			assert.NoError(t, err)
 
-		ts, isOk := val.(*ext.ExtendedTime)
-		assert.True(t, isOk)
+			ts, isOk := val.(*ext.ExtendedTime)
+			assert.True(t, isOk)
 
-		expectedExtTime := &ext.ExtendedTime{
-			Time: time.Date(2025, time.September, 13, 0, 0, 0, 0, time.UTC),
-			NestedKind: ext.NestedKind{
-				Type:   ext.DateTimeKindType,
-				Format: "2006-01-02T15:04:05Z07:00",
-			},
+			expectedExtTime := &ext.ExtendedTime{
+				Time: time.Date(2025, time.September, 13, 0, 0, 0, 000000000, time.UTC),
+				NestedKind: ext.NestedKind{
+					Type:   ext.DateTimeKindType,
+					Format: "2006-01-02T15:04:05Z",
+				},
+			}
+
+			assert.Equal(t, expectedExtTime, ts)
 		}
+		{
+			// 1 digits
+			val, err := DateTimeWithTimezone{}.Convert("2025-09-13T00:00:00.1Z")
+			assert.NoError(t, err)
+
+			ts, isOk := val.(*ext.ExtendedTime)
+			assert.True(t, isOk)
+
+			expectedExtTime := &ext.ExtendedTime{
+				Time: time.Date(2025, time.September, 13, 0, 0, 0, 100000000, time.UTC),
+				NestedKind: ext.NestedKind{
+					Type:   ext.DateTimeKindType,
+					Format: "2006-01-02T15:04:05.0Z",
+				},
+			}
+
+			assert.Equal(t, expectedExtTime, ts)
+		}
+		{
+			// 2 digits
+			val, err := DateTimeWithTimezone{}.Convert("2025-09-13T00:00:00.12Z")
+			assert.NoError(t, err)
+
+			ts, isOk := val.(*ext.ExtendedTime)
+			assert.True(t, isOk)
+
+			expectedExtTime := &ext.ExtendedTime{
+				Time: time.Date(2025, time.September, 13, 0, 0, 0, 120000000, time.UTC),
+				NestedKind: ext.NestedKind{
+					Type:   ext.DateTimeKindType,
+					Format: "2006-01-02T15:04:05.00Z",
+				},
+			}
+
+			assert.Equal(t, expectedExtTime, ts)
+		}
+		{
+			// 3 digits
+			val, err := DateTimeWithTimezone{}.Convert("2025-09-13T00:00:00.123Z")
+			assert.NoError(t, err)
+
+			ts, isOk := val.(*ext.ExtendedTime)
+			assert.True(t, isOk)
 
-		assert.Equal(t, expectedExtTime, ts)
+			expectedExtTime := &ext.ExtendedTime{
+				Time: time.Date(2025, time.September, 13, 0, 0, 0, 123000000, time.UTC),
+				NestedKind: ext.NestedKind{
+					Type:   ext.DateTimeKindType,
+					Format: "2006-01-02T15:04:05.000Z",
+				},
+			}
+
+			assert.Equal(t, expectedExtTime, ts)
+		}
+		{
+			// 4 digits
+			val, err := DateTimeWithTimezone{}.Convert("2025-09-13T00:00:00.1234Z")
+			assert.NoError(t, err)
+
+			ts, isOk := val.(*ext.ExtendedTime)
+			assert.True(t, isOk)
+
+			expectedExtTime := &ext.ExtendedTime{
+				Time: time.Date(2025, time.September, 13, 0, 0, 0, 123400000, time.UTC),
+				NestedKind: ext.NestedKind{
+					Type:   ext.DateTimeKindType,
+					Format: "2006-01-02T15:04:05.0000Z",
+				},
+			}
+
+			assert.Equal(t, expectedExtTime, ts)
+		}
+		{
+			// 5 digits
+			val, err := DateTimeWithTimezone{}.Convert("2025-09-13T00:00:00.12345Z")
+			assert.NoError(t, err)
+
+			ts, isOk := val.(*ext.ExtendedTime)
+			assert.True(t, isOk)
+
+			expectedExtTime := &ext.ExtendedTime{
+				Time: time.Date(2025, time.September, 13, 0, 0, 0, 123450000, time.UTC),
+				NestedKind: ext.NestedKind{
+					Type:   ext.DateTimeKindType,
+					Format: "2006-01-02T15:04:05.00000Z",
+				},
+			}
+
+			assert.Equal(t, expectedExtTime, ts)
+		}
+		{
+			// 6 digits (microseconds)
+			val, err := DateTimeWithTimezone{}.Convert("2025-09-13T00:00:00.123456Z")
+			assert.NoError(t, err)
+
+			ts, isOk := val.(*ext.ExtendedTime)
+			assert.True(t, isOk)
+
+			expectedExtTime := &ext.ExtendedTime{
+				Time: time.Date(2025, time.September, 13, 0, 0, 0, 123456000, time.UTC),
+				NestedKind: ext.NestedKind{
+					Type:   ext.DateTimeKindType,
+					Format: "2006-01-02T15:04:05.000000Z",
+				},
+			}
+
+			assert.Equal(t, expectedExtTime, ts)
+		}
 	}
 }