diff --git a/lib/optimization/event_update_test.go b/lib/optimization/event_update_test.go index f7e4322b1..c1ce2ea14 100644 --- a/lib/optimization/event_update_test.go +++ b/lib/optimization/event_update_test.go @@ -48,6 +48,21 @@ func TestTableData_UpdateInMemoryColumnsFromDestination(t *testing.T) { assert.Equal(t, ext.DateKindType, col.KindDetails.ExtendedTimeDetails.Type) assert.Equal(t, extTime.ExtendedTimeDetails, col.KindDetails.ExtendedTimeDetails) } + { + // If the in-memory column is a string, the destination column is JSON + tableDataCols := &columns.Columns{} + tableData := &TableData{ + inMemoryColumns: tableDataCols, + } + + tableData.AddInMemoryCol(columns.NewColumn("foo", typing.String)) + structCol := columns.NewColumn("foo", typing.Struct) + assert.NoError(t, tableData.MergeColumnsFromDestination(structCol)) + + col, isOk := tableData.inMemoryColumns.GetColumn("foo") + assert.True(t, isOk) + assert.Equal(t, typing.Struct, col.KindDetails) + } { tableDataCols := &columns.Columns{} tableData := &TableData{ diff --git a/lib/typing/parse.go b/lib/typing/parse.go index a70e2d171..9b4231570 100644 --- a/lib/typing/parse.go +++ b/lib/typing/parse.go @@ -57,12 +57,7 @@ func parseValue(settings Settings, val any) KindDetails { } } - if IsJSON(convertedVal) { - return Struct - } - return String - case *decimal.Decimal: extendedDetails := convertedVal.Details() return KindDetails{ diff --git a/lib/typing/typing.go b/lib/typing/typing.go index 99e92c240..0ee30a086 100644 --- a/lib/typing/typing.go +++ b/lib/typing/typing.go @@ -1,9 +1,6 @@ package typing import ( - "encoding/json" - "strings" - "github.com/artie-labs/transfer/lib/typing/decimal" "github.com/artie-labs/transfer/lib/typing/ext" ) @@ -77,25 +74,3 @@ func NewKindDetailsFromTemplate(details KindDetails, extendedType ext.ExtendedTi details.ExtendedTimeDetails.Type = extendedType return details } - -// IsJSON - We also need to check if the string is a JSON string or not -// If it could be one, it will start with { and end with }. -// Once there, we will then check if it's a JSON string or not. -// This is an optimization since JSON string checking is expensive. -func IsJSON(str string) bool { - str = strings.TrimSpace(str) - if len(str) < 2 { - return false - } - - valStringChars := []rune(str) - firstChar := string(valStringChars[0]) - lastChar := string(valStringChars[len(valStringChars)-1]) - - if (firstChar == "{" && lastChar == "}") || (firstChar == "[" && lastChar == "]") { - var js json.RawMessage - return json.Unmarshal([]byte(str), &js) == nil - } - - return false -} diff --git a/lib/typing/typing_test.go b/lib/typing/typing_test.go deleted file mode 100644 index 91303bcbc..000000000 --- a/lib/typing/typing_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package typing - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func Test_IsJSON(t *testing.T) { - { - invalidValues := []string{ - `{"hello": "world"`, - `{"hello": "world"}}`, - `{null}`, - "", - " ", - } - - for _, invalidValue := range invalidValues { - assert.False(t, IsJSON(invalidValue), invalidValue) - } - } - { - validValues := []string{ - "{}", - `{"hello": "world"}`, - `{ - "hello": { - "world": { - "nested_value": true - } - }, - "add_a_list_here": [1, 2, 3, 4], - "number": 7.5, - "integerNum": 7 - }`, - "[]", - "[1, 2, 3, 4]", - } - - for _, validValue := range validValues { - assert.True(t, IsJSON(validValue), validValue) - } - } -} diff --git a/lib/typing/values/string.go b/lib/typing/values/string.go index 6fffc1e0d..773e1a642 100644 --- a/lib/typing/values/string.go +++ b/lib/typing/values/string.go @@ -57,22 +57,23 @@ func ToString(colVal any, colKind typing.KindDetails, additionalDateFmts []strin return stringutil.EscapeBackslashes(fmt.Sprint(colVal)), nil case typing.Struct.Kind: - if colKind == typing.Struct { - if strings.Contains(fmt.Sprint(colVal), constants.ToastUnavailableValuePlaceholder) { - colVal = map[string]any{ - "key": constants.ToastUnavailableValuePlaceholder, - } - } - - if reflect.TypeOf(colVal).Kind() != reflect.String { - colValBytes, err := json.Marshal(colVal) - if err != nil { - return "", err - } + if strings.Contains(fmt.Sprint(colVal), constants.ToastUnavailableValuePlaceholder) { + return fmt.Sprintf(`{"key":"%s"}`, constants.ToastUnavailableValuePlaceholder), nil + } - return string(colValBytes), nil + isArray := reflect.ValueOf(colVal).Kind() == reflect.Slice + _, isMap := colVal.(map[string]any) + // If colVal is either an array or a JSON object, we should run JSON parse. + if isMap || isArray { + colValBytes, err := json.Marshal(colVal) + if err != nil { + return "", err } + + return string(colValBytes), nil } + + return fmt.Sprint(colVal), nil case typing.Array.Kind: colValBytes, err := json.Marshal(colVal) if err != nil { diff --git a/models/event/event_save_test.go b/models/event/event_save_test.go index cec4b517d..fd086afa6 100644 --- a/models/event/event_save_test.go +++ b/models/event/event_save_test.go @@ -4,6 +4,8 @@ import ( "fmt" "strconv" + "github.com/artie-labs/transfer/lib/debezium/converters" + "github.com/artie-labs/transfer/lib/typing/columns" "github.com/artie-labs/transfer/lib/artie" @@ -132,12 +134,13 @@ func (e *EventsTestSuite) TestEventSaveOptionalSchema() { "created_at_date_string": "2023-01-01", "created_at_date_no_schema": "2023-01-01", "json_object_string": `{"foo": "bar"}`, - "json_object_no_schema": `{"foo": "bar"}`, + "json_object": `{"foo": "bar"}`, }, OptionalSchema: map[string]typing.KindDetails{ // Explicitly casting this as a string. "created_at_date_string": typing.String, "json_object_string": typing.String, + "json_object": converters.JSON{}.ToKindDetails(), }, } @@ -158,7 +161,7 @@ func (e *EventsTestSuite) TestEventSaveOptionalSchema() { assert.True(e.T(), isOk) assert.Equal(e.T(), typing.String, column.KindDetails) - column, isOk = td.ReadOnlyInMemoryCols().GetColumn("json_object_no_schema") + column, isOk = td.ReadOnlyInMemoryCols().GetColumn("json_object") assert.True(e.T(), isOk) assert.Equal(e.T(), typing.Struct, column.KindDetails) }