Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ExtendedTime] More clean up #969

Merged
merged 12 commits into from
Oct 22, 2024
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions lib/debezium/converters/time.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,22 @@ import (

type Time struct{}

func (Time) ToKindDetails() typing.KindDetails {
return typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "")
func (Time) layout() string {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Addresses #967 (comment)

return "15:04:05.000"
}

func (t Time) ToKindDetails() typing.KindDetails {
return typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, t.layout())
}

func (Time) Convert(val any) (any, error) {
func (t Time) Convert(val any) (any, error) {
valInt64, isOk := val.(int64)
if !isOk {
return nil, fmt.Errorf("expected int64 got '%v' with type %T", val, val)
}

// Represents the number of milliseconds past midnight, and does not include timezone information.
return ext.NewExtendedTime(time.UnixMilli(valInt64).In(time.UTC), ext.TimeKindType, ""), nil
return ext.NewExtendedTime(time.UnixMilli(valInt64).In(time.UTC), ext.TimeKindType, t.layout()), nil
}

type NanoTime struct{}
Expand Down
10 changes: 9 additions & 1 deletion lib/debezium/converters/time_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -114,13 +114,21 @@ func TestZonedTimestamp_Convert(t *testing.T) {
}

func TestTime_Convert(t *testing.T) {
{
val, err := Time{}.Convert(int64(54720321))
assert.NoError(t, err)

extTime, isOk := val.(*ext.ExtendedTime)
assert.True(t, isOk)
assert.Equal(t, "15:12:00.321", extTime.String(""))
}
{
val, err := Time{}.Convert(int64(54720000))
assert.NoError(t, err)

extTime, isOk := val.(*ext.ExtendedTime)
assert.True(t, isOk)
assert.Equal(t, "15:12:00+00", extTime.String(""))
assert.Equal(t, "15:12:00.000", extTime.String(""))
}
}

Expand Down
17 changes: 12 additions & 5 deletions lib/debezium/schema_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -254,13 +254,20 @@ func TestField_ToKindDetails(t *testing.T) {
}
}
{
// Time
for _, dbzType := range []SupportedDebeziumType{Time, TimeKafkaConnect, TimeWithTimezone} {
kd, err := Field{DebeziumType: dbzType}.ToKindDetails()
{
// Time with timezone
kd, err := Field{DebeziumType: TimeWithTimezone}.ToKindDetails()
assert.NoError(t, err)
assert.Equal(t, typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, ""), kd, dbzType)
assert.Equal(t, typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, ""), kd)
}
{
// Time
for _, dbzType := range []SupportedDebeziumType{Time, TimeKafkaConnect} {
kd, err := Field{DebeziumType: dbzType}.ToKindDetails()
assert.NoError(t, err)
assert.Equal(t, typing.NewExtendedTimeDetails(typing.ETime, ext.TimeKindType, "15:04:05.000"), kd, dbzType)
}
}

{
// Micro time
kd, err := Field{DebeziumType: MicroTime}.ToKindDetails()
Expand Down
38 changes: 23 additions & 15 deletions lib/typing/ext/time.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,10 @@ package ext
import (
"cmp"
"encoding/json"
"fmt"
"time"
)

// TODO: This package should have a concept of default formats for each type.

type ExtendedTimeKindType string

const (
Expand All @@ -17,6 +16,21 @@ const (
TimeKindType ExtendedTimeKindType = "time"
)

func (e ExtendedTimeKindType) defaultLayout() (string, error) {
switch e {
case TimestampTZKindType:
return time.RFC3339Nano, nil
case TimestampNTZKindType:
return RFC3339NanosecondNoTZ, nil
case DateKindType:
return PostgresDateFormat, nil
case TimeKindType:
return PostgresTimeFormat, nil
default:
return "", fmt.Errorf("unknown kind type: %q", e)
}
}

type NestedKind struct {
Type ExtendedTimeKindType
Format string
Expand Down Expand Up @@ -51,29 +65,23 @@ type ExtendedTime struct {
nestedKind NestedKind
}

// MarshalJSON is a custom JSON marshaller for ExtendedTime. This is only by MongoDB where we are recursively parsing a nested object.
func (e ExtendedTime) MarshalJSON() ([]byte, error) {
return json.Marshal(e.String(""))
return json.Marshal(e.ts)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this not taking the NestedKind's type and format into account?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah yeah. I'm purposefully moving away from using the ExtendedTime's nested type and format into account.

My goal is to eventually make ExtendedTime just an alias for time.Time.

MarshalJSON was created specifically for nested MongoDB objects and this code is consistent with how Mongo marshals Datetime https://github.com/mongodb/mongo-go-driver/blob/c91b204cc72025ea0c8d0845b84816b8c9e88903/bson/primitive.go#L59-L61

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually, let me get it to look identical to MongoDB's

}

// TODO: Have this return an error instead of nil
func NewExtendedTime(t time.Time, kindType ExtendedTimeKindType, originalFormat string) *ExtendedTime {
if originalFormat == "" {
switch kindType {
case TimestampTZKindType:
originalFormat = TimestampTZ.Format
case TimestampNTZKindType:
originalFormat = TimestampNTZ.Format
case DateKindType:
originalFormat = Date.Format
case TimeKindType:
originalFormat = Time.Format
}
format, err := kindType.defaultLayout()
if err != nil {
return nil
}

return &ExtendedTime{
ts: t,
nestedKind: NestedKind{
Type: kindType,
Format: originalFormat,
Format: cmp.Or(originalFormat, format),
},
}
}
Expand Down
4 changes: 2 additions & 2 deletions lib/typing/ext/time_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ func TestExtendedTime_MarshalJSON(t *testing.T) {
// Single value
bytes, err := json.Marshal(extTime)
assert.NoError(t, err)
assert.Equal(t, `"2025-09-13T00:00:00.123Z"`, string(bytes))
assert.Equal(t, `"2025-09-13T00:00:00.123456Z"`, string(bytes))
danafallon marked this conversation as resolved.
Show resolved Hide resolved
}
{
// As a nested object
Expand All @@ -30,6 +30,6 @@ func TestExtendedTime_MarshalJSON(t *testing.T) {

bytes, err := json.Marshal(obj)
assert.NoError(t, err)
assert.Equal(t, `{"extendedTime":"2025-09-13T00:00:00.123Z","foo":"bar"}`, string(bytes))
assert.Equal(t, `{"extendedTime":"2025-09-13T00:00:00.123456Z","foo":"bar"}`, string(bytes))
}
}
4 changes: 3 additions & 1 deletion lib/typing/mongo/bson_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ func TestJSONEToMap(t *testing.T) {
extendedTime, isOk := result["test_timestamp"]
assert.True(t, isOk)
assert.Equal(t, ext.NewExtendedTime(time.Date(2023, time.March, 16, 1, 18, 37, 0, time.UTC), ext.TimestampTZKindType, ext.ISO8601), extendedTime)
assert.Equal(t, "2023-03-16T01:18:37+00:00", extendedTime.(*ext.ExtendedTime).String(""))
}
{
// Boolean
Expand All @@ -143,7 +144,7 @@ func TestJSONEToMap(t *testing.T) {
// Nested object
value, err := json.Marshal(result["test_nested_object"])
assert.NoError(t, err)
assert.Equal(t, `{"a":{"b":{"c":"hello"}},"super_nested":{"foo":"bar","test_timestamp":"2023-03-16T01:18:37+00:00"},"test_timestamp":"2023-03-16T01:18:37+00:00"}`, string(value))
assert.Equal(t, `{"a":{"b":{"c":"hello"}},"super_nested":{"foo":"bar","test_timestamp":"2023-03-16T01:18:37Z"},"test_timestamp":"2023-03-16T01:18:37Z"}`, string(value))
}
{
// NaN
Expand Down Expand Up @@ -222,6 +223,7 @@ func TestBsonValueToGoValue(t *testing.T) {
extendedTime, isOk := result.(*ext.ExtendedTime)
assert.True(t, isOk)
assert.Equal(t, ext.NewExtendedTime(time.Date(2021, time.January, 1, 0, 0, 0, 0, time.UTC), ext.TimestampTZKindType, ext.ISO8601), extendedTime)
assert.Equal(t, "2021-01-01T00:00:00+00:00", extendedTime.String(""))
}
{
// primitive.ObjectID
Expand Down