Skip to content

Commit

Permalink
[Extended Time] Marking attrs private (#886)
Browse files Browse the repository at this point in the history
  • Loading branch information
Tang8330 authored Sep 5, 2024
1 parent d156f65 commit 9783d55
Show file tree
Hide file tree
Showing 11 changed files with 48 additions and 114 deletions.
8 changes: 4 additions & 4 deletions clients/bigquery/storagewrite.go
Original file line number Diff line number Diff line change
Expand Up @@ -181,15 +181,15 @@ func rowToMessage(row map[string]any, columns []columns.Column, messageDescripto

switch column.KindDetails.ExtendedTimeDetails.Type {
case ext.TimeKindType:
message.Set(field, protoreflect.ValueOfInt64(encodePacked64TimeMicros(extTime.Time)))
message.Set(field, protoreflect.ValueOfInt64(encodePacked64TimeMicros(extTime.GetTime())))
case ext.DateKindType:
daysSinceEpoch := extTime.Unix() / (60 * 60 * 24)
daysSinceEpoch := extTime.GetTime().Unix() / (60 * 60 * 24)
message.Set(field, protoreflect.ValueOfInt32(int32(daysSinceEpoch)))
case ext.DateTimeKindType:
if err := timestamppb.New(extTime.Time).CheckValid(); err != nil {
if err := timestamppb.New(extTime.GetTime()).CheckValid(); err != nil {
return nil, err
}
message.Set(field, protoreflect.ValueOfInt64(extTime.UnixMicro()))
message.Set(field, protoreflect.ValueOfInt64(extTime.GetTime().UnixMicro()))
default:
return nil, fmt.Errorf("unsupported extended time details: %q", column.KindDetails.ExtendedTimeDetails.Type)
}
Expand Down
2 changes: 1 addition & 1 deletion lib/cdc/mongo/debezium_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ func (m *MongoTestSuite) TestMongoDBEventCustomer() {

updatedExtTime, isOk := evtDataWithIncludedAt[constants.UpdateColumnMarker].(*ext.ExtendedTime)
assert.True(m.T(), isOk)
assert.False(m.T(), updatedExtTime.IsZero())
assert.False(m.T(), updatedExtTime.GetTime().IsZero())

var nestedData map[string]any
err = json.Unmarshal([]byte(evtData["nested"].(string)), &nestedData)
Expand Down
2 changes: 1 addition & 1 deletion lib/cdc/relational/debezium_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -537,7 +537,7 @@ func (r *RelationTestSuite) TestGetEventFromBytes_MySQL() {

updatedAtExtTime, isOk := evtData[constants.UpdateColumnMarker].(*ext.ExtendedTime)
assert.True(r.T(), isOk)
assert.False(r.T(), updatedAtExtTime.IsZero())
assert.False(r.T(), updatedAtExtTime.GetTime().IsZero())

assert.Equal(r.T(), evtData["id"], int64(1001))
assert.Equal(r.T(), evtData["first_name"], "Sally")
Expand Down
71 changes: 8 additions & 63 deletions lib/debezium/converters/time_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,14 +39,7 @@ func TestConvertDateTimeWithTimezone(t *testing.T) {
ts, isOk := val.(*ext.ExtendedTime)
assert.True(t, isOk)

expectedExtTime := &ext.ExtendedTime{
Time: time.Date(2025, time.September, 13, 0, 0, 0, 000000000, time.UTC),
NestedKind: ext.NestedKind{
Type: ext.DateTimeKindType,
Format: "2006-01-02T15:04:05Z",
},
}

expectedExtTime := ext.NewExtendedTime(time.Date(2025, time.September, 13, 0, 0, 0, 000000000, time.UTC), ext.DateTimeKindType, "2006-01-02T15:04:05Z")
assert.Equal(t, expectedExtTime, ts)
}
{
Expand All @@ -57,14 +50,7 @@ func TestConvertDateTimeWithTimezone(t *testing.T) {
ts, isOk := val.(*ext.ExtendedTime)
assert.True(t, isOk)

expectedExtTime := &ext.ExtendedTime{
Time: time.Date(2025, time.September, 13, 0, 0, 0, 100000000, time.UTC),
NestedKind: ext.NestedKind{
Type: ext.DateTimeKindType,
Format: "2006-01-02T15:04:05.0Z",
},
}

expectedExtTime := ext.NewExtendedTime(time.Date(2025, time.September, 13, 0, 0, 0, 100000000, time.UTC), ext.DateTimeKindType, "2006-01-02T15:04:05.0Z")
assert.Equal(t, expectedExtTime, ts)
}
{
Expand All @@ -75,14 +61,7 @@ func TestConvertDateTimeWithTimezone(t *testing.T) {
ts, isOk := val.(*ext.ExtendedTime)
assert.True(t, isOk)

expectedExtTime := &ext.ExtendedTime{
Time: time.Date(2025, time.September, 13, 0, 0, 0, 120000000, time.UTC),
NestedKind: ext.NestedKind{
Type: ext.DateTimeKindType,
Format: "2006-01-02T15:04:05.00Z",
},
}

expectedExtTime := ext.NewExtendedTime(time.Date(2025, time.September, 13, 0, 0, 0, 120000000, time.UTC), ext.DateTimeKindType, "2006-01-02T15:04:05.00Z")
assert.Equal(t, expectedExtTime, ts)
}
{
Expand All @@ -93,14 +72,7 @@ func TestConvertDateTimeWithTimezone(t *testing.T) {
ts, isOk := val.(*ext.ExtendedTime)
assert.True(t, isOk)

expectedExtTime := &ext.ExtendedTime{
Time: time.Date(2025, time.September, 13, 0, 0, 0, 123000000, time.UTC),
NestedKind: ext.NestedKind{
Type: ext.DateTimeKindType,
Format: "2006-01-02T15:04:05.000Z",
},
}

expectedExtTime := ext.NewExtendedTime(time.Date(2025, time.September, 13, 0, 0, 0, 123000000, time.UTC), ext.DateTimeKindType, "2006-01-02T15:04:05.000Z")
assert.Equal(t, expectedExtTime, ts)
}
{
Expand All @@ -111,14 +83,7 @@ func TestConvertDateTimeWithTimezone(t *testing.T) {
ts, isOk := val.(*ext.ExtendedTime)
assert.True(t, isOk)

expectedExtTime := &ext.ExtendedTime{
Time: time.Date(2025, time.September, 13, 0, 0, 0, 123400000, time.UTC),
NestedKind: ext.NestedKind{
Type: ext.DateTimeKindType,
Format: "2006-01-02T15:04:05.0000Z",
},
}

expectedExtTime := ext.NewExtendedTime(time.Date(2025, time.September, 13, 0, 0, 0, 123400000, time.UTC), ext.DateTimeKindType, "2006-01-02T15:04:05.0000Z")
assert.Equal(t, expectedExtTime, ts)
}
{
Expand All @@ -129,14 +94,7 @@ func TestConvertDateTimeWithTimezone(t *testing.T) {
ts, isOk := val.(*ext.ExtendedTime)
assert.True(t, isOk)

expectedExtTime := &ext.ExtendedTime{
Time: time.Date(2025, time.September, 13, 0, 0, 0, 123450000, time.UTC),
NestedKind: ext.NestedKind{
Type: ext.DateTimeKindType,
Format: "2006-01-02T15:04:05.00000Z",
},
}

expectedExtTime := ext.NewExtendedTime(time.Date(2025, time.September, 13, 0, 0, 0, 123450000, time.UTC), ext.DateTimeKindType, "2006-01-02T15:04:05.00000Z")
assert.Equal(t, expectedExtTime, ts)
}
{
Expand All @@ -147,14 +105,7 @@ func TestConvertDateTimeWithTimezone(t *testing.T) {
ts, isOk := val.(*ext.ExtendedTime)
assert.True(t, isOk)

expectedExtTime := &ext.ExtendedTime{
Time: time.Date(2025, time.September, 13, 0, 0, 0, 123456000, time.UTC),
NestedKind: ext.NestedKind{
Type: ext.DateTimeKindType,
Format: "2006-01-02T15:04:05.000000Z",
},
}

expectedExtTime := ext.NewExtendedTime(time.Date(2025, time.September, 13, 0, 0, 0, 123456000, time.UTC), ext.DateTimeKindType, "2006-01-02T15:04:05.000000Z")
assert.Equal(t, expectedExtTime, ts)
}
}
Expand Down Expand Up @@ -214,14 +165,8 @@ func TestConvertTimeWithTimezone(t *testing.T) {
ts, isOk := val.(*ext.ExtendedTime)
assert.True(t, isOk)
assert.NoError(t, err)
expectedTs := &ext.ExtendedTime{
Time: time.Date(0, 1, 1, 23, 2, 6, 745116000, time.UTC),
NestedKind: ext.NestedKind{
Type: ext.TimeKindType,
Format: "15:04:05.000000Z",
},
}

expectedTs := ext.NewExtendedTime(time.Date(0, 1, 1, 23, 2, 6, 745116000, time.UTC), ext.TimeKindType, "15:04:05.000000Z")
assert.Equal(t, expectedTs, ts)
}
{
Expand Down
2 changes: 1 addition & 1 deletion lib/debezium/keys_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ func TestParsePartitionKeyStruct(t *testing.T) {
`))
assert.NoError(t, err)
assert.Equal(t, "339f3f2f-f29f-4f00-869e-476122310eff", keys["id"])
assert.Equal(t, time.Date(2024, 4, 16, 1, 8, 19, 440000000, time.UTC), keys["created_at"].(*ext.ExtendedTime).Time)
assert.Equal(t, time.Date(2024, 4, 16, 1, 8, 19, 440000000, time.UTC), keys["created_at"].(*ext.ExtendedTime).GetTime())

keys, err = parsePartitionKeyStruct([]byte(`{
"schema": {
Expand Down
2 changes: 1 addition & 1 deletion lib/debezium/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ func (f Field) ShouldSetDefaultValue(defaultValue any) bool {
case nil:
return false
case *ext.ExtendedTime:
return !castedDefaultValue.Time.IsZero()
return !castedDefaultValue.GetTime().IsZero()
case string:
if f.DebeziumType == UUID && castedDefaultValue == uuid.Nil.String() {
return false
Expand Down
4 changes: 2 additions & 2 deletions lib/debezium/types_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,11 @@ func TestField_ShouldSetDefaultValue(t *testing.T) {
{
// *ext.ExtendedTime
field := Field{}
assert.True(t, field.ShouldSetDefaultValue(&ext.ExtendedTime{Time: time.Now()}))
assert.True(t, field.ShouldSetDefaultValue(ext.NewExtendedTime(time.Now(), ext.DateTimeKindType, ext.RFC3339Millisecond)))

assert.False(t, field.ShouldSetDefaultValue(&ext.ExtendedTime{}))
var ts time.Time
assert.False(t, field.ShouldSetDefaultValue(&ext.ExtendedTime{Time: ts}))
assert.False(t, field.ShouldSetDefaultValue(ext.NewExtendedTime(ts, ext.DateTimeKindType, ext.RFC3339Millisecond)))
}
}

Expand Down
4 changes: 2 additions & 2 deletions lib/parquetutil/parse_values.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,10 @@ func ParseValue(colVal any, colKind columns.Column, additionalDateFmts []string)
}

if colKind.KindDetails.ExtendedTimeDetails.Type == ext.DateKindType || colKind.KindDetails.ExtendedTimeDetails.Type == ext.TimeKindType {
return extTime.Format(colKind.KindDetails.ExtendedTimeDetails.Format), nil
return extTime.String(colKind.KindDetails.ExtendedTimeDetails.Format), nil
}

return extTime.Time.UnixMilli(), nil
return extTime.GetTime().UnixMilli(), nil
case typing.String.Kind:
return colVal, nil
case typing.Struct.Kind:
Expand Down
42 changes: 14 additions & 28 deletions lib/typing/ext/parse_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,33 +8,19 @@ import (
)

func TestParseFromInterface(t *testing.T) {
var vals []any
vals = append(vals, &ExtendedTime{
Time: time.Now().UTC(),
NestedKind: NestedKind{
Type: DateKindType,
Format: PostgresDateFormat,
},
}, &ExtendedTime{
Time: time.Now().UTC(),
NestedKind: NestedKind{
Type: DateTimeKindType,
Format: ISO8601,
},
}, &ExtendedTime{
Time: time.Now().UTC(),
NestedKind: NestedKind{
Type: TimeKindType,
Format: PostgresTimeFormat,
},
})
{
// Extended time
var vals []any
vals = append(vals, NewExtendedTime(time.Now().UTC(), DateKindType, PostgresDateFormat))
vals = append(vals, NewExtendedTime(time.Now().UTC(), DateTimeKindType, ISO8601))
vals = append(vals, NewExtendedTime(time.Now().UTC(), TimeKindType, PostgresTimeFormat))

for _, val := range vals {
extTime, err := ParseFromInterface(val, nil)
assert.NoError(t, err)
assert.Equal(t, val, extTime)
for _, val := range vals {
extTime, err := ParseFromInterface(val, nil)
assert.NoError(t, err)
assert.Equal(t, val, extTime)
}
}

{
// Nil
_, err := ParseFromInterface(nil, nil)
Expand All @@ -57,7 +43,7 @@ func TestParseFromInterfaceDateTime(t *testing.T) {
for _, supportedDateTimeLayout := range supportedDateTimeLayouts {
et, err := ParseFromInterface(now.Format(supportedDateTimeLayout), nil)
assert.NoError(t, err)
assert.Equal(t, et.NestedKind.Type, DateTimeKindType)
assert.Equal(t, DateTimeKindType, et.GetNestedKind().Type)
assert.Equal(t, et.String(""), now.Format(supportedDateTimeLayout))
}
}
Expand All @@ -67,7 +53,7 @@ func TestParseFromInterfaceTime(t *testing.T) {
for _, supportedTimeFormat := range SupportedTimeFormatsLegacy {
et, err := ParseFromInterface(now.Format(supportedTimeFormat), nil)
assert.NoError(t, err)
assert.Equal(t, et.NestedKind.Type, TimeKindType)
assert.Equal(t, TimeKindType, et.GetNestedKind().Type)
// Without passing an override format, this should return the same preserved dt format.
assert.Equal(t, et.String(""), now.Format(supportedTimeFormat))
}
Expand All @@ -78,7 +64,7 @@ func TestParseFromInterfaceDate(t *testing.T) {
for _, supportedDateFormat := range supportedDateFormats {
et, err := ParseFromInterface(now.Format(supportedDateFormat), nil)
assert.NoError(t, err)
assert.Equal(t, et.NestedKind.Type, DateKindType)
assert.Equal(t, DateKindType, et.GetNestedKind().Type)

// Without passing an override format, this should return the same preserved dt format.
assert.Equal(t, et.String(""), now.Format(supportedDateFormat))
Expand Down
22 changes: 12 additions & 10 deletions lib/typing/ext/time.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package ext

import (
"time"
)
import "time"

type ExtendedTimeKindType string

Expand Down Expand Up @@ -37,8 +35,8 @@ var (
// ExtendedTime is created because Golang's time.Time does not allow us to explicitly cast values as a date, or time
// and only allows timestamp expressions.
type ExtendedTime struct {
time.Time
NestedKind NestedKind
ts time.Time
nestedKind NestedKind
}

func NewExtendedTime(t time.Time, kindType ExtendedTimeKindType, originalFormat string) *ExtendedTime {
Expand All @@ -54,22 +52,26 @@ func NewExtendedTime(t time.Time, kindType ExtendedTimeKindType, originalFormat
}

return &ExtendedTime{
Time: t,
NestedKind: NestedKind{
ts: t,
nestedKind: NestedKind{
Type: kindType,
Format: originalFormat,
},
}
}

func (e *ExtendedTime) GetTime() time.Time {
return e.Time
return e.ts
}

func (e *ExtendedTime) GetNestedKind() NestedKind {
return e.nestedKind
}

func (e *ExtendedTime) String(overrideFormat string) string {
if overrideFormat != "" {
return e.Time.Format(overrideFormat)
return e.ts.Format(overrideFormat)
}

return e.Time.Format(e.NestedKind.Format)
return e.ts.Format(e.nestedKind.Format)
}
3 changes: 2 additions & 1 deletion lib/typing/parse.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,10 @@ func ParseValue(key string, optionalSchema map[string]KindDetails, val any) Kind
ExtendedDecimalDetails: &extendedDetails,
}
case *ext.ExtendedTime:
nestedKind := convertedVal.GetNestedKind()
return KindDetails{
Kind: ETime.Kind,
ExtendedTimeDetails: &convertedVal.NestedKind,
ExtendedTimeDetails: &nestedKind,
}
default:
// Check if the val is one of our custom-types
Expand Down

0 comments on commit 9783d55

Please sign in to comment.