TOASTED Arrays (#1066)

clients/mssql/dialect/dialect.go

@@ -49,6 +49,7 @@ func (md MSSQLDialect) BuildIsNotToastValueExpression(tableAlias constants.Table
 	if column.KindDetails == typing.Struct {
 		return fmt.Sprintf("COALESCE(%s, {}) != {'key': '%s'}", colName, constants.ToastUnavailableValuePlaceholder)
 	}
+
 	return fmt.Sprintf("COALESCE(%s, '') != '%s'", colName, constants.ToastUnavailableValuePlaceholder)
 }
 

lib/debezium/converters/basic.go

@@ -87,3 +87,17 @@ func (Float64) Convert(value any) (any, error) {
 		return nil, fmt.Errorf("unexpected type %T", value)
 	}
 }
+
+type Array struct{}
+
+func (Array) ToKindDetails() typing.KindDetails {
+	return typing.Array
+}
+
+func (Array) Convert(value any) (any, error) {
+	if fmt.Sprint(value) == fmt.Sprintf("[%s]", constants.ToastUnavailableValuePlaceholder) {
+		return constants.ToastUnavailableValuePlaceholder, nil
+	}
+
+	return value, nil
+}

lib/debezium/converters/basic_test.go

@@ -4,6 +4,8 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
+	"github.com/artie-labs/transfer/lib/config/constants"
 )
 
 func TestJSON_Convert(t *testing.T) {
@@ -85,3 +87,27 @@ func TestFloat64_Convert(t *testing.T) {
 		}
 	}
 }
+
+func TestArray_Convert(t *testing.T) {
+	{
+		// Irrelevant data type
+		value, err := Array{}.Convert([]int{1, 2, 3, 4})
+		assert.NoError(t, err)
+		assert.Equal(t, []int{1, 2, 3, 4}, value)
+	}
+	{
+		// TOASTED data
+		{
+			// As []any
+			value, err := Array{}.Convert([]any{constants.ToastUnavailableValuePlaceholder})
+			assert.NoError(t, err)
+			assert.Equal(t, constants.ToastUnavailableValuePlaceholder, value)
+		}
+		{
+			// As []string
+			value, err := Array{}.Convert([]string{constants.ToastUnavailableValuePlaceholder})
+			assert.NoError(t, err)
+			assert.Equal(t, constants.ToastUnavailableValuePlaceholder, value)
+		}
+	}
+}

lib/debezium/schema.go

@@ -137,6 +137,8 @@ func (f Field) ToValueConverter() (converters.ValueConverter, error) {
 		}
 
 		switch f.Type {
+		case Array:
+			return converters.Array{}, nil
 		case Double, Float:
 			return converters.Float64{}, nil
 		}
@@ -167,8 +169,6 @@ func (f Field) ToKindDetails() (typing.KindDetails, error) {
 		return typing.Struct, nil
 	case Boolean:
 		return typing.Boolean, nil
-	case Array:
-		return typing.Array, nil
 	default:
 		return typing.Invalid, fmt.Errorf("unhandled field type %q", f.Type)
 	}