Cleaning up.

lib/optimization/event_update_test.go

@@ -48,6 +48,21 @@ func TestTableData_UpdateInMemoryColumnsFromDestination(t *testing.T) {
 		assert.Equal(t, ext.DateKindType, col.KindDetails.ExtendedTimeDetails.Type)
 		assert.Equal(t, extTime.ExtendedTimeDetails, col.KindDetails.ExtendedTimeDetails)
 	}
+	{
+		// If the in-memory column is a string, the destination column is JSON
+		tableDataCols := &columns.Columns{}
+		tableData := &TableData{
+			inMemoryColumns: tableDataCols,
+		}
+
+		tableData.AddInMemoryCol(columns.NewColumn("foo", typing.String))
+		structCol := columns.NewColumn("foo", typing.Struct)
+		assert.NoError(t, tableData.MergeColumnsFromDestination(structCol))
+
+		col, isOk := tableData.inMemoryColumns.GetColumn("foo")
+		assert.True(t, isOk)
+		assert.Equal(t, typing.Struct, col.KindDetails)
+	}
 	{
 		tableDataCols := &columns.Columns{}
 		tableData := &TableData{

lib/typing/parse.go

@@ -57,12 +57,7 @@ func parseValue(settings Settings, val any) KindDetails {
 			}
 		}
 
-		if IsJSON(convertedVal) {
-			return Struct
-		}
-
 		return String
-
 	case *decimal.Decimal:
 		extendedDetails := convertedVal.Details()
 		return KindDetails{

lib/typing/typing.go

@@ -1,9 +1,6 @@
 package typing
 
 import (
-	"encoding/json"
-	"strings"
-
 	"github.com/artie-labs/transfer/lib/typing/decimal"
 	"github.com/artie-labs/transfer/lib/typing/ext"
 )
@@ -77,25 +74,3 @@ func NewKindDetailsFromTemplate(details KindDetails, extendedType ext.ExtendedTi
 	details.ExtendedTimeDetails.Type = extendedType
 	return details
 }
-
-// IsJSON - We also need to check if the string is a JSON string or not
-// If it could be one, it will start with { and end with }.
-// Once there, we will then check if it's a JSON string or not.
-// This is an optimization since JSON string checking is expensive.
-func IsJSON(str string) bool {
-	str = strings.TrimSpace(str)
-	if len(str) < 2 {
-		return false
-	}
-
-	valStringChars := []rune(str)
-	firstChar := string(valStringChars[0])
-	lastChar := string(valStringChars[len(valStringChars)-1])
-
-	if (firstChar == "{" && lastChar == "}") || (firstChar == "[" && lastChar == "]") {
-		var js json.RawMessage
-		return json.Unmarshal([]byte(str), &js) == nil
-	}
-
-	return false
-}

lib/typing/values/string.go

@@ -57,22 +57,23 @@ func ToString(colVal any, colKind typing.KindDetails, additionalDateFmts []strin
 
 		return stringutil.EscapeBackslashes(fmt.Sprint(colVal)), nil
 	case typing.Struct.Kind:
-		if colKind == typing.Struct {
-			if strings.Contains(fmt.Sprint(colVal), constants.ToastUnavailableValuePlaceholder) {
-				colVal = map[string]any{
-					"key": constants.ToastUnavailableValuePlaceholder,
-				}
-			}
-
-			if reflect.TypeOf(colVal).Kind() != reflect.String {
-				colValBytes, err := json.Marshal(colVal)
-				if err != nil {
-					return "", err
-				}
+		if strings.Contains(fmt.Sprint(colVal), constants.ToastUnavailableValuePlaceholder) {
+			return fmt.Sprintf(`{"key":"%s"}`, constants.ToastUnavailableValuePlaceholder), nil
+		}
 
-				return string(colValBytes), nil
+		isArray := reflect.ValueOf(colVal).Kind() == reflect.Slice
+		_, isMap := colVal.(map[string]any)
+		// If colVal is either an array or a JSON object, we should run JSON parse.
+		if isMap || isArray {
+			colValBytes, err := json.Marshal(colVal)
+			if err != nil {
+				return "", err
 			}
+
+			return string(colValBytes), nil
 		}
+
+		return fmt.Sprint(colVal), nil
 	case typing.Array.Kind:
 		colValBytes, err := json.Marshal(colVal)
 		if err != nil {

models/event/event_save_test.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"strconv"
 
+	"github.com/artie-labs/transfer/lib/debezium/converters"
+
 	"github.com/artie-labs/transfer/lib/typing/columns"
 
 	"github.com/artie-labs/transfer/lib/artie"
@@ -132,12 +134,13 @@ func (e *EventsTestSuite) TestEventSaveOptionalSchema() {
 			"created_at_date_string":            "2023-01-01",
 			"created_at_date_no_schema":         "2023-01-01",
 			"json_object_string":                `{"foo": "bar"}`,
-			"json_object_no_schema":             `{"foo": "bar"}`,
+			"json_object":                       `{"foo": "bar"}`,
 		},
 		OptionalSchema: map[string]typing.KindDetails{
 			// Explicitly casting this as a string.
 			"created_at_date_string": typing.String,
 			"json_object_string":     typing.String,
+			"json_object":            converters.JSON{}.ToKindDetails(),
 		},
 	}
 
@@ -158,7 +161,7 @@ func (e *EventsTestSuite) TestEventSaveOptionalSchema() {
 	assert.True(e.T(), isOk)
 	assert.Equal(e.T(), typing.String, column.KindDetails)
 
-	column, isOk = td.ReadOnlyInMemoryCols().GetColumn("json_object_no_schema")
+	column, isOk = td.ReadOnlyInMemoryCols().GetColumn("json_object")
 	assert.True(e.T(), isOk)
 	assert.Equal(e.T(), typing.Struct, column.KindDetails)
 }