From 7d9afe57fcd340445ca6f3bcaf2693b90a07c581 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Tue, 2 Jul 2024 10:43:56 -1000 Subject: [PATCH 1/5] MySQL guardrail. --- lib/mysql/schema/convert.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/mysql/schema/convert.go b/lib/mysql/schema/convert.go index 18deee32..7cab147d 100644 --- a/lib/mysql/schema/convert.go +++ b/lib/mysql/schema/convert.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "fmt" "math" + "strings" "time" ) @@ -82,9 +83,8 @@ func ConvertValue(value any, colType DataType) (any, error) { return nil, fmt.Errorf("expected []byte got %T for value: %v", value, value) } - if string(bytesValue) == "0000-00-00 00:00:00" { - // MySQL supports '0000-00-00 00:00:00' for datetime columns. - // We are returning `nil` here because this will fail most Time parsers. + if strings.HasSuffix(string(bytesValue), "-00-00 00:00:00") { + // If MySQL strict mode isn't turned on, it can allow invalid dates like 2020-00-00 00:00:00 or 0000-00-00 00:00:00 return nil, nil } From 97dacfa2094e2f185ad79ca83d20907005e3214c Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Tue, 2 Jul 2024 10:44:53 -1000 Subject: [PATCH 2/5] Cast once. --- lib/mysql/schema/convert.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/mysql/schema/convert.go b/lib/mysql/schema/convert.go index 7cab147d..42f182db 100644 --- a/lib/mysql/schema/convert.go +++ b/lib/mysql/schema/convert.go @@ -83,12 +83,13 @@ func ConvertValue(value any, colType DataType) (any, error) { return nil, fmt.Errorf("expected []byte got %T for value: %v", value, value) } - if strings.HasSuffix(string(bytesValue), "-00-00 00:00:00") { + stringValue := string(bytesValue) + if strings.HasSuffix(stringValue, "-00-00 00:00:00") { // If MySQL strict mode isn't turned on, it can allow invalid dates like 2020-00-00 00:00:00 or 0000-00-00 00:00:00 return nil, nil } - timeValue, err := time.Parse(DateTimeFormat, string(bytesValue)) + timeValue, err := time.Parse(DateTimeFormat, stringValue) if err != nil { return nil, err } From 703ee4df8f49296b79e6b3118eb1ec777bea179c Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Tue, 2 Jul 2024 11:05:48 -1000 Subject: [PATCH 3/5] More comprehensive. --- lib/mysql/schema/convert.go | 28 ++++++++++++++++++++++++++-- lib/mysql/schema/convert_test.go | 10 ++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/lib/mysql/schema/convert.go b/lib/mysql/schema/convert.go index 42f182db..fd21dee3 100644 --- a/lib/mysql/schema/convert.go +++ b/lib/mysql/schema/convert.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "fmt" "math" + "strconv" "strings" "time" ) @@ -84,8 +85,7 @@ func ConvertValue(value any, colType DataType) (any, error) { } stringValue := string(bytesValue) - if strings.HasSuffix(stringValue, "-00-00 00:00:00") { - // If MySQL strict mode isn't turned on, it can allow invalid dates like 2020-00-00 00:00:00 or 0000-00-00 00:00:00 + if hasNonStrictModeDate(stringValue) { return nil, nil } @@ -185,3 +185,27 @@ func ConvertValues(values []any, cols []Column) error { } return nil } + +// hasNonStrictModeDate - if strict mode is not enabled, we can end up having invalid datetimes +func hasNonStrictModeDate(d string) bool { + if len(d) < 10 { + return false + } + + parts := strings.Split(d[:10], "-") + if len(parts) != 3 { + return false + } + + // Year, month, date cannot be non-zero + for _, part := range parts { + value, err := strconv.Atoi(part) + if err != nil { + return false + } + if value == 0 { + return true + } + } + return false +} diff --git a/lib/mysql/schema/convert_test.go b/lib/mysql/schema/convert_test.go index 6c2f3b19..342fa170 100644 --- a/lib/mysql/schema/convert_test.go +++ b/lib/mysql/schema/convert_test.go @@ -318,3 +318,13 @@ func TestConvertValues(t *testing.T) { assert.Equal(t, []any{int64(1234), "hello world", true}, values) } } + +func TestHasNonStrictModeDate(t *testing.T) { + assert.False(t, hasNonStrictModeDate("2021-01-02")) + assert.False(t, hasNonStrictModeDate("2021-01-02 03:04:05")) + + assert.True(t, hasNonStrictModeDate("2009-00-00")) + assert.True(t, hasNonStrictModeDate("0000-00-00")) + assert.True(t, hasNonStrictModeDate("0000-00-00 00:00:00")) + assert.True(t, hasNonStrictModeDate("2009-00-00 00:00:00")) +} From c58f00d43e67d478d14d455fad14acc5defb0a54 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Tue, 2 Jul 2024 11:11:20 -1000 Subject: [PATCH 4/5] More --- lib/mysql/schema/convert_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/mysql/schema/convert_test.go b/lib/mysql/schema/convert_test.go index 342fa170..e803df32 100644 --- a/lib/mysql/schema/convert_test.go +++ b/lib/mysql/schema/convert_test.go @@ -320,7 +320,10 @@ func TestConvertValues(t *testing.T) { } func TestHasNonStrictModeDate(t *testing.T) { + assert.False(t, hasNonStrictModeDate("")) + assert.False(t, hasNonStrictModeDate("hello world")) assert.False(t, hasNonStrictModeDate("2021-01-02")) + assert.False(t, hasNonStrictModeDate("2021--01-02")) assert.False(t, hasNonStrictModeDate("2021-01-02 03:04:05")) assert.True(t, hasNonStrictModeDate("2009-00-00")) From 8ef5a59774c47ec624bd5b470f049038efc6e937 Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Tue, 2 Jul 2024 14:51:03 -1000 Subject: [PATCH 5/5] Function rename. --- lib/mysql/schema/convert.go | 6 +++--- lib/mysql/schema/convert_test.go | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/mysql/schema/convert.go b/lib/mysql/schema/convert.go index fd21dee3..95201be2 100644 --- a/lib/mysql/schema/convert.go +++ b/lib/mysql/schema/convert.go @@ -85,7 +85,7 @@ func ConvertValue(value any, colType DataType) (any, error) { } stringValue := string(bytesValue) - if hasNonStrictModeDate(stringValue) { + if hasNonStrictModeInvalidDate(stringValue) { return nil, nil } @@ -186,8 +186,8 @@ func ConvertValues(values []any, cols []Column) error { return nil } -// hasNonStrictModeDate - if strict mode is not enabled, we can end up having invalid datetimes -func hasNonStrictModeDate(d string) bool { +// hasNonStrictModeInvalidDate - if strict mode is not enabled, we can end up having invalid datetimes +func hasNonStrictModeInvalidDate(d string) bool { if len(d) < 10 { return false } diff --git a/lib/mysql/schema/convert_test.go b/lib/mysql/schema/convert_test.go index e803df32..17695e77 100644 --- a/lib/mysql/schema/convert_test.go +++ b/lib/mysql/schema/convert_test.go @@ -319,15 +319,15 @@ func TestConvertValues(t *testing.T) { } } -func TestHasNonStrictModeDate(t *testing.T) { - assert.False(t, hasNonStrictModeDate("")) - assert.False(t, hasNonStrictModeDate("hello world")) - assert.False(t, hasNonStrictModeDate("2021-01-02")) - assert.False(t, hasNonStrictModeDate("2021--01-02")) - assert.False(t, hasNonStrictModeDate("2021-01-02 03:04:05")) +func TestHasNonStrictModeInvalidDate(t *testing.T) { + assert.False(t, hasNonStrictModeInvalidDate("")) + assert.False(t, hasNonStrictModeInvalidDate("hello world")) + assert.False(t, hasNonStrictModeInvalidDate("2021-01-02")) + assert.False(t, hasNonStrictModeInvalidDate("2021--01-02")) + assert.False(t, hasNonStrictModeInvalidDate("2021-01-02 03:04:05")) - assert.True(t, hasNonStrictModeDate("2009-00-00")) - assert.True(t, hasNonStrictModeDate("0000-00-00")) - assert.True(t, hasNonStrictModeDate("0000-00-00 00:00:00")) - assert.True(t, hasNonStrictModeDate("2009-00-00 00:00:00")) + assert.True(t, hasNonStrictModeInvalidDate("2009-00-00")) + assert.True(t, hasNonStrictModeInvalidDate("0000-00-00")) + assert.True(t, hasNonStrictModeInvalidDate("0000-00-00 00:00:00")) + assert.True(t, hasNonStrictModeInvalidDate("2009-00-00 00:00:00")) }