From 784cbf036c33954b0e53fc8d0e0db3d610ec2fa7 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Thu, 24 Oct 2024 10:30:27 -0700 Subject: [PATCH 01/16] Support Timestamp NTZ in BQ. --- clients/bigquery/storagewrite.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/clients/bigquery/storagewrite.go b/clients/bigquery/storagewrite.go index d6933eb98..925012723 100644 --- a/clients/bigquery/storagewrite.go +++ b/clients/bigquery/storagewrite.go @@ -46,6 +46,8 @@ func columnToTableFieldSchema(column columns.Column) (*storagepb.TableFieldSchem fieldType = storagepb.TableFieldSchema_DATE case ext.TimestampTZKindType: fieldType = storagepb.TableFieldSchema_TIMESTAMP + case ext.TimestampNTZKindType: + fieldType = storagepb.TableFieldSchema_DATETIME default: return nil, fmt.Errorf("unsupported extended time details type: %q", column.KindDetails.ExtendedTimeDetails.Type) } From 0b164f9dbb444a18b69e9eacd2f7510838e89b30 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Thu, 24 Oct 2024 10:43:24 -0700 Subject: [PATCH 02/16] Support. --- clients/bigquery/storagewrite.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/bigquery/storagewrite.go b/clients/bigquery/storagewrite.go index 925012723..adb4b07e6 100644 --- a/clients/bigquery/storagewrite.go +++ b/clients/bigquery/storagewrite.go @@ -187,8 +187,8 @@ func rowToMessage(row map[string]any, columns []columns.Column, messageDescripto case ext.DateKindType: daysSinceEpoch := _time.Unix() / (60 * 60 * 24) message.Set(field, protoreflect.ValueOfInt32(int32(daysSinceEpoch))) - case ext.TimestampTZKindType: - if err := timestamppb.New(_time).CheckValid(); err != nil { + case ext.TimestampTZKindType, ext.TimestampNTZKindType: + if err = timestamppb.New(_time).CheckValid(); err != nil { return nil, err } message.Set(field, protoreflect.ValueOfInt64(_time.UnixMicro())) From e8f7335c1c2af563b65cb3aa001ad46170022787 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Thu, 24 Oct 2024 11:07:57 -0700 Subject: [PATCH 03/16] Needs to expressed in Civil time. --- clients/bigquery/storagewrite.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/clients/bigquery/storagewrite.go b/clients/bigquery/storagewrite.go index adb4b07e6..637b054c7 100644 --- a/clients/bigquery/storagewrite.go +++ b/clients/bigquery/storagewrite.go @@ -187,11 +187,14 @@ func rowToMessage(row map[string]any, columns []columns.Column, messageDescripto case ext.DateKindType: daysSinceEpoch := _time.Unix() / (60 * 60 * 24) message.Set(field, protoreflect.ValueOfInt32(int32(daysSinceEpoch))) - case ext.TimestampTZKindType, ext.TimestampNTZKindType: + case ext.TimestampTZKindType: if err = timestamppb.New(_time).CheckValid(); err != nil { return nil, err } message.Set(field, protoreflect.ValueOfInt64(_time.UnixMicro())) + case ext.TimestampNTZKindType: + civilLayout := "2006-01-02 15:04:05.999999" + message.Set(field, protoreflect.ValueOfString(_time.Format(civilLayout))) default: return nil, fmt.Errorf("unsupported extended time details: %q", column.KindDetails.ExtendedTimeDetails.Type) } From 6a75684d5da2170a372d5df6e5fadabab706de55 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Thu, 24 Oct 2024 11:34:36 -0700 Subject: [PATCH 04/16] WIP. --- clients/bigquery/storagewrite.go | 27 +++++++++++++++++++++++++-- clients/bigquery/storagewrite_test.go | 2 ++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/clients/bigquery/storagewrite.go b/clients/bigquery/storagewrite.go index 637b054c7..046bf05ac 100644 --- a/clients/bigquery/storagewrite.go +++ b/clients/bigquery/storagewrite.go @@ -102,6 +102,30 @@ func encodePacked64TimeMicros(value time.Time) int64 { return result } +// This is a reimplementation of https://github.com/googleapis/java-bigquerystorage/blob/f79acb5cfdd12253bca1c41551c478400120d2f9/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/CivilTimeEncoder.java#L92 +func encodePacked32TimeSeconds(t time.Time) int { + bitFieldTimeSeconds := 0x0 + bitFieldTimeSeconds |= t.Hour() << 12 + bitFieldTimeSeconds |= t.Minute() << 6 + bitFieldTimeSeconds |= t.Second() << 0 + return bitFieldTimeSeconds +} + +// This is a reimplementation of https://github.com/googleapis/java-bigquerystorage/blob/f79acb5cfdd12253bca1c41551c478400120d2f9/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/CivilTimeEncoder.java#L187 +func encodePacked64DatetimeSeconds(dateTime time.Time) int64 { + var bitFieldDatetimeSeconds int64 + bitFieldDatetimeSeconds |= int64(dateTime.Year()) << 26 + bitFieldDatetimeSeconds |= int64(dateTime.Month()) << 22 + bitFieldDatetimeSeconds |= int64(dateTime.Day()) << 17 + bitFieldDatetimeSeconds |= int64(encodePacked32TimeSeconds(dateTime)) + return bitFieldDatetimeSeconds +} + +// This is a reimplementation of https://github.com/googleapis/java-bigquerystorage/blob/f79acb5cfdd12253bca1c41551c478400120d2f9/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/CivilTimeEncoder.java#L248 +func encodePacked64DatetimeMicros(dateTime time.Time) int64 { + return (encodePacked64DatetimeSeconds(dateTime) << 20) | (dateTime.UnixMicro()) +} + func rowToMessage(row map[string]any, columns []columns.Column, messageDescriptor protoreflect.MessageDescriptor) (*dynamicpb.Message, error) { message := dynamicpb.NewMessage(messageDescriptor) for _, column := range columns { @@ -193,8 +217,7 @@ func rowToMessage(row map[string]any, columns []columns.Column, messageDescripto } message.Set(field, protoreflect.ValueOfInt64(_time.UnixMicro())) case ext.TimestampNTZKindType: - civilLayout := "2006-01-02 15:04:05.999999" - message.Set(field, protoreflect.ValueOfString(_time.Format(civilLayout))) + message.Set(field, protoreflect.ValueOfInt64(encodePacked64DatetimeMicros(_time))) default: return nil, fmt.Errorf("unsupported extended time details: %q", column.KindDetails.ExtendedTimeDetails.Type) } diff --git a/clients/bigquery/storagewrite_test.go b/clients/bigquery/storagewrite_test.go index f0641c7b7..3cfa88327 100644 --- a/clients/bigquery/storagewrite_test.go +++ b/clients/bigquery/storagewrite_test.go @@ -101,6 +101,8 @@ func TestEncodePacked64TimeMicros(t *testing.T) { assert.Equal(t, int64(1<<32+1000), encodePacked64TimeMicros(epoch.Add(time.Duration(1)*time.Hour+time.Duration(1)*time.Millisecond))) } +func TestEncode + func TestRowToMessage(t *testing.T) { columns := []columns.Column{ columns.NewColumn("c_bool", typing.Boolean), From 6859819ded0e1b9e6212a8bf5c1340f313ca0d3b Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Thu, 24 Oct 2024 11:34:47 -0700 Subject: [PATCH 05/16] Clean up. --- clients/bigquery/storagewrite_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/clients/bigquery/storagewrite_test.go b/clients/bigquery/storagewrite_test.go index 3cfa88327..f0641c7b7 100644 --- a/clients/bigquery/storagewrite_test.go +++ b/clients/bigquery/storagewrite_test.go @@ -101,8 +101,6 @@ func TestEncodePacked64TimeMicros(t *testing.T) { assert.Equal(t, int64(1<<32+1000), encodePacked64TimeMicros(epoch.Add(time.Duration(1)*time.Hour+time.Duration(1)*time.Millisecond))) } -func TestEncode - func TestRowToMessage(t *testing.T) { columns := []columns.Column{ columns.NewColumn("c_bool", typing.Boolean), From f6a49aa1d9240896559890ee9356f8cde7f4df6c Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Thu, 24 Oct 2024 11:52:42 -0700 Subject: [PATCH 06/16] Adding logging --- clients/bigquery/storagewrite.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/clients/bigquery/storagewrite.go b/clients/bigquery/storagewrite.go index 046bf05ac..6d2c87fbc 100644 --- a/clients/bigquery/storagewrite.go +++ b/clients/bigquery/storagewrite.go @@ -3,6 +3,7 @@ package bigquery import ( "encoding/json" "fmt" + "log/slog" "strings" "time" @@ -217,7 +218,9 @@ func rowToMessage(row map[string]any, columns []columns.Column, messageDescripto } message.Set(field, protoreflect.ValueOfInt64(_time.UnixMicro())) case ext.TimestampNTZKindType: - message.Set(field, protoreflect.ValueOfInt64(encodePacked64DatetimeMicros(_time))) + _value := encodePacked64DatetimeMicros(_time) + slog.Info("### Encoded time ###", slog.Int64("value", _value), slog.Any("time", _time)) + message.Set(field, protoreflect.ValueOfInt64(_value)) default: return nil, fmt.Errorf("unsupported extended time details: %q", column.KindDetails.ExtendedTimeDetails.Type) } From 7723e3b13e9096c993c3f950534c151f8cb30090 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Thu, 24 Oct 2024 12:43:53 -0700 Subject: [PATCH 07/16] Clean up. --- clients/bigquery/storagewrite.go | 41 +++++++++++++++++++------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/clients/bigquery/storagewrite.go b/clients/bigquery/storagewrite.go index 6d2c87fbc..3d812a930 100644 --- a/clients/bigquery/storagewrite.go +++ b/clients/bigquery/storagewrite.go @@ -92,39 +92,48 @@ func columnsToMessageDescriptor(cols []columns.Column) (*protoreflect.MessageDes return &messageDescriptor, nil } +const ( + NANO_SHIFT = 0 + MICRO_SHIFT = 0 + SECOND_SHIFT = 0 + MINUTE_SHIFT = 6 + HOUR_SHIFT = 12 + DAY_SHIFT = 17 + MONTH_SHIFT = 22 + YEAR_SHIFT = 26 + NANO_LENGTH = 30 + MICRO_LENGTH = 20 +) + // This is a reimplementation of https://github.com/googleapis/java-bigquerystorage/blob/f79acb5cfdd12253bca1c41551c478400120d2f9/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/CivilTimeEncoder.java#L143 // See https://cloud.google.com/java/docs/reference/google-cloud-bigquerystorage/latest/com.google.cloud.bigquery.storage.v1.CivilTimeEncoder // And https://cloud.google.com/pubsub/docs/bigquery#date_time_int func encodePacked64TimeMicros(value time.Time) int64 { - var result = int64(value.Nanosecond() / 1000) - result |= int64(value.Second()) << 20 - result |= int64(value.Minute()) << 26 - result |= int64(value.Hour()) << 32 - return result + return int64(encodePacked32TimeSeconds(value))< Date: Thu, 24 Oct 2024 12:44:23 -0700 Subject: [PATCH 08/16] More. --- clients/bigquery/storagewrite.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/bigquery/storagewrite.go b/clients/bigquery/storagewrite.go index 3d812a930..2c295113a 100644 --- a/clients/bigquery/storagewrite.go +++ b/clients/bigquery/storagewrite.go @@ -133,7 +133,7 @@ func encodePacked64DatetimeSeconds(dateTime time.Time) int64 { // // This is a reimplementation of https://github.com/googleapis/java-bigquerystorage/blob/f79acb5cfdd12253bca1c41551c478400120d2f9/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/CivilTimeEncoder.java#L248 func encodePacked64DatetimeMicros(dateTime time.Time) int64 { - return (encodePacked64DatetimeSeconds(dateTime) << MICRO_LENGTH) | int64(dateTime.Nanosecond()/1_000) + return encodePacked64DatetimeSeconds(dateTime)< Date: Thu, 24 Oct 2024 13:07:12 -0700 Subject: [PATCH 09/16] Clean up. --- clients/bigquery/storagewrite.go | 33 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/clients/bigquery/storagewrite.go b/clients/bigquery/storagewrite.go index 2c295113a..cfad4246c 100644 --- a/clients/bigquery/storagewrite.go +++ b/clients/bigquery/storagewrite.go @@ -93,47 +93,44 @@ func columnsToMessageDescriptor(cols []columns.Column) (*protoreflect.MessageDes } const ( - NANO_SHIFT = 0 - MICRO_SHIFT = 0 - SECOND_SHIFT = 0 - MINUTE_SHIFT = 6 - HOUR_SHIFT = 12 - DAY_SHIFT = 17 - MONTH_SHIFT = 22 - YEAR_SHIFT = 26 - NANO_LENGTH = 30 - MICRO_LENGTH = 20 + microLength = 20 + secondShift = 0 + minuteShift = 6 + hourShift = 12 + dayShift = 17 + monthShift = 22 + yearShift = 26 ) // This is a reimplementation of https://github.com/googleapis/java-bigquerystorage/blob/f79acb5cfdd12253bca1c41551c478400120d2f9/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/CivilTimeEncoder.java#L143 // See https://cloud.google.com/java/docs/reference/google-cloud-bigquerystorage/latest/com.google.cloud.bigquery.storage.v1.CivilTimeEncoder // And https://cloud.google.com/pubsub/docs/bigquery#date_time_int func encodePacked64TimeMicros(value time.Time) int64 { - return int64(encodePacked32TimeSeconds(value))< Date: Thu, 24 Oct 2024 13:11:02 -0700 Subject: [PATCH 10/16] Add tests --- clients/bigquery/storagewrite.go | 10 +++++----- clients/bigquery/storagewrite_test.go | 10 ++++++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/clients/bigquery/storagewrite.go b/clients/bigquery/storagewrite.go index cfad4246c..b56cf1134 100644 --- a/clients/bigquery/storagewrite.go +++ b/clients/bigquery/storagewrite.go @@ -110,11 +110,11 @@ func encodePacked64TimeMicros(value time.Time) int64 { } // This is a reimplementation of https://github.com/googleapis/java-bigquerystorage/blob/f79acb5cfdd12253bca1c41551c478400120d2f9/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/CivilTimeEncoder.java#L92 -func encodePacked32TimeSeconds(t time.Time) int { - var bitFieldTimeSeconds int - bitFieldTimeSeconds |= t.Hour() << hourShift - bitFieldTimeSeconds |= t.Minute() << minuteShift - bitFieldTimeSeconds |= t.Second() << secondShift +func encodePacked32TimeSeconds(t time.Time) int32 { + var bitFieldTimeSeconds int32 + bitFieldTimeSeconds |= int32(t.Hour()) << hourShift + bitFieldTimeSeconds |= int32(t.Minute()) << minuteShift + bitFieldTimeSeconds |= int32(t.Second()) << secondShift return bitFieldTimeSeconds } diff --git a/clients/bigquery/storagewrite_test.go b/clients/bigquery/storagewrite_test.go index f0641c7b7..718de90ec 100644 --- a/clients/bigquery/storagewrite_test.go +++ b/clients/bigquery/storagewrite_test.go @@ -101,6 +101,16 @@ func TestEncodePacked64TimeMicros(t *testing.T) { assert.Equal(t, int64(1<<32+1000), encodePacked64TimeMicros(epoch.Add(time.Duration(1)*time.Hour+time.Duration(1)*time.Millisecond))) } +func TestEncodePacked32TimeSeconds(t *testing.T) { + epoch := time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC) + + assert.Equal(t, int32(0), encodePacked32TimeSeconds(epoch)) + assert.Equal(t, int32(1), encodePacked32TimeSeconds(epoch.Add(time.Duration(1)*time.Second))) + assert.Equal(t, int32(1<<6), encodePacked32TimeSeconds(epoch.Add(time.Duration(1)*time.Minute))) + assert.Equal(t, int32(1<<12), encodePacked32TimeSeconds(epoch.Add(time.Duration(1)*time.Hour))) + assert.Equal(t, int32(1<<12+1), encodePacked32TimeSeconds(epoch.Add(time.Duration(1)*time.Hour+time.Duration(1)*time.Second))) +} + func TestRowToMessage(t *testing.T) { columns := []columns.Column{ columns.NewColumn("c_bool", typing.Boolean), From bb713ec68fc94fd577c9025b3e2da6b0b6dd8d3c Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Thu, 24 Oct 2024 13:27:35 -0700 Subject: [PATCH 11/16] Clean up. --- clients/bigquery/storagewrite_test.go | 42 +++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/clients/bigquery/storagewrite_test.go b/clients/bigquery/storagewrite_test.go index 718de90ec..98511d10b 100644 --- a/clients/bigquery/storagewrite_test.go +++ b/clients/bigquery/storagewrite_test.go @@ -111,6 +111,48 @@ func TestEncodePacked32TimeSeconds(t *testing.T) { assert.Equal(t, int32(1<<12+1), encodePacked32TimeSeconds(epoch.Add(time.Duration(1)*time.Hour+time.Duration(1)*time.Second))) } +func TestEncodePacked64DatetimeSeconds(t *testing.T) { + _ts := time.Date(2024, 10, 24, 13, 1, 2, 3000000, time.UTC) + _expected := 2024<<26 + 10<<22 + 24<<17 + int64(encodePacked32TimeSeconds(_ts)) + + // Time + assert.Equal(t, _expected, encodePacked64DatetimeSeconds(_ts)) + assert.Equal(t, _expected+1<<0, encodePacked64DatetimeSeconds(_ts.Add(time.Duration(1)*time.Second))) + assert.Equal(t, _expected+1<<6, encodePacked64DatetimeSeconds(_ts.Add(time.Duration(1)*time.Minute))) + assert.Equal(t, _expected+1<<12, encodePacked64DatetimeSeconds(_ts.Add(time.Duration(1)*time.Hour))) + assert.Equal(t, _expected+1<<12+1<<0, encodePacked64DatetimeSeconds(_ts.Add(time.Duration(1)*time.Hour+time.Duration(1)*time.Second))) + + // Day + assert.Equal(t, _expected+1<<17, encodePacked64DatetimeSeconds(_ts.Add(time.Duration(24)*time.Hour))) + // Month + assert.Equal(t, _expected+1<<22, encodePacked64DatetimeSeconds(_ts.AddDate(0, 1, 0))) + // Year + assert.Equal(t, _expected+1<<26, encodePacked64DatetimeSeconds(_ts.AddDate(1, 0, 0))) + // Month and year + assert.Equal(t, _expected+1<<22+1<<26, encodePacked64DatetimeSeconds(_ts.AddDate(1, 1, 0))) +} + +func TestEncodePacked64DatetimeMicros(t *testing.T) { + _ts := time.Date(2024, 10, 24, 13, 1, 2, 123456789, time.UTC) + _expected := encodePacked64DatetimeSeconds(_ts)<<20 | int64(_ts.Nanosecond()/1000) + + // Time + assert.Equal(t, _expected, encodePacked64DatetimeMicros(_ts)) + assert.Equal(t, _expected+1<<(0+20), encodePacked64DatetimeMicros(_ts.Add(time.Duration(1)*time.Second))) + assert.Equal(t, _expected+1<<(6+20), encodePacked64DatetimeMicros(_ts.Add(time.Duration(1)*time.Minute))) + assert.Equal(t, _expected+1<<(12+20), encodePacked64DatetimeMicros(_ts.Add(time.Duration(1)*time.Hour))) + assert.Equal(t, _expected+1<<(12+20)+1<<(0+20), encodePacked64DatetimeMicros(_ts.Add(time.Duration(1)*time.Hour+time.Duration(1)*time.Second))) + + // Day + assert.Equal(t, _expected+1<<(17+20), encodePacked64DatetimeMicros(_ts.Add(time.Duration(24)*time.Hour))) + // Month + assert.Equal(t, _expected+1<<(22+20), encodePacked64DatetimeMicros(_ts.AddDate(0, 1, 0))) + // Year + assert.Equal(t, _expected+1<<(26+20), encodePacked64DatetimeMicros(_ts.AddDate(1, 0, 0))) + // Month and year + assert.Equal(t, _expected+1<<(26+20)+1<<(22+20), encodePacked64DatetimeMicros(_ts.AddDate(1, 1, 0))) +} + func TestRowToMessage(t *testing.T) { columns := []columns.Column{ columns.NewColumn("c_bool", typing.Boolean), From e671a1bb7e477909a8496421c6cc63fc381a8ac3 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Thu, 24 Oct 2024 13:28:15 -0700 Subject: [PATCH 12/16] Clean up. --- clients/bigquery/storagewrite.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/clients/bigquery/storagewrite.go b/clients/bigquery/storagewrite.go index b56cf1134..90404263f 100644 --- a/clients/bigquery/storagewrite.go +++ b/clients/bigquery/storagewrite.go @@ -3,7 +3,6 @@ package bigquery import ( "encoding/json" "fmt" - "log/slog" "strings" "time" @@ -225,7 +224,6 @@ func rowToMessage(row map[string]any, columns []columns.Column, messageDescripto message.Set(field, protoreflect.ValueOfInt64(_time.UnixMicro())) case ext.TimestampNTZKindType: _value := encodePacked64DatetimeMicros(_time) - slog.Info("### Encoded time ###", slog.Int64("value", _value), slog.Any("time", _time)) message.Set(field, protoreflect.ValueOfInt64(_value)) default: return nil, fmt.Errorf("unsupported extended time details: %q", column.KindDetails.ExtendedTimeDetails.Type) From 53e5c83df75ea811a99261a0142893aa01ab87a9 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Thu, 24 Oct 2024 13:28:44 -0700 Subject: [PATCH 13/16] Clean up --- clients/bigquery/storagewrite.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/clients/bigquery/storagewrite.go b/clients/bigquery/storagewrite.go index 90404263f..460723d92 100644 --- a/clients/bigquery/storagewrite.go +++ b/clients/bigquery/storagewrite.go @@ -223,8 +223,7 @@ func rowToMessage(row map[string]any, columns []columns.Column, messageDescripto } message.Set(field, protoreflect.ValueOfInt64(_time.UnixMicro())) case ext.TimestampNTZKindType: - _value := encodePacked64DatetimeMicros(_time) - message.Set(field, protoreflect.ValueOfInt64(_value)) + message.Set(field, protoreflect.ValueOfInt64(encodePacked64DatetimeMicros(_time))) default: return nil, fmt.Errorf("unsupported extended time details: %q", column.KindDetails.ExtendedTimeDetails.Type) } From 1b85ccb9fdd31fb41a388839ce0aa842b915debc Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Thu, 24 Oct 2024 14:09:50 -0700 Subject: [PATCH 14/16] Support int64. --- clients/bigquery/converters/converters.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/bigquery/converters/converters.go b/clients/bigquery/converters/converters.go index b396b2f31..48541c363 100644 --- a/clients/bigquery/converters/converters.go +++ b/clients/bigquery/converters/converters.go @@ -23,7 +23,7 @@ func (s StringConverter) Convert(value any) (any, error) { return castedValue, nil case *decimal.Decimal: return castedValue.String(), nil - case bool: + case bool, int64: return fmt.Sprint(castedValue), nil case *ext.ExtendedTime: if err := s.kd.EnsureExtendedTimeDetails(); err != nil { @@ -32,7 +32,7 @@ func (s StringConverter) Convert(value any) (any, error) { return castedValue.GetTime().Format(s.kd.ExtendedTimeDetails.Format), nil default: - return nil, fmt.Errorf("expected string/*decimal.Decimal/bool received %T with value %v", value, value) + return nil, fmt.Errorf("expected string/*decimal.Decimal/bool/int64 received %T with value %v", value, value) } } From cbf26652778e8a087ee7cc7e742e62e1d68a40b4 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Thu, 24 Oct 2024 14:10:11 -0700 Subject: [PATCH 15/16] Clean up --- clients/bigquery/converters/converters_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/clients/bigquery/converters/converters_test.go b/clients/bigquery/converters/converters_test.go index 75c1d4863..726b5e01b 100644 --- a/clients/bigquery/converters/converters_test.go +++ b/clients/bigquery/converters/converters_test.go @@ -31,6 +31,12 @@ func TestStringConverter_Convert(t *testing.T) { assert.NoError(t, err) assert.Equal(t, "true", val) } + { + // int64 + val, err := NewStringConverter(typing.Integer).Convert(int64(123)) + assert.NoError(t, err) + assert.Equal(t, "123", val) + } { // Invalid _, err := NewStringConverter(typing.Integer).Convert(123) From 6d886c88131dce5e319f6dab9f6be32a9e5a7425 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Thu, 24 Oct 2024 14:31:15 -0700 Subject: [PATCH 16/16] Fix test. --- clients/bigquery/converters/converters_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/bigquery/converters/converters_test.go b/clients/bigquery/converters/converters_test.go index 726b5e01b..b69d5633e 100644 --- a/clients/bigquery/converters/converters_test.go +++ b/clients/bigquery/converters/converters_test.go @@ -40,7 +40,7 @@ func TestStringConverter_Convert(t *testing.T) { { // Invalid _, err := NewStringConverter(typing.Integer).Convert(123) - assert.ErrorContains(t, err, "expected string/*decimal.Decimal/bool received int with value 123") + assert.ErrorContains(t, err, "expected string/*decimal.Decimal/bool/int64 received int with value 123") } { // Extended time