Skip to content

Commit

Permalink
Escape TOAST.
Browse files Browse the repository at this point in the history
  • Loading branch information
Tang8330 committed Nov 22, 2024
1 parent c78ff93 commit 79cf4ba
Show file tree
Hide file tree
Showing 5 changed files with 42 additions and 18 deletions.
2 changes: 0 additions & 2 deletions clients/shared/merge.go
Original file line number Diff line number Diff line change
Expand Up @@ -151,8 +151,6 @@ func Merge(ctx context.Context, dwh destination.DataWarehouse, tableData *optimi
return fmt.Errorf("failed to generate merge statements: %w", err)
}

fmt.Println("mergeStatements", mergeStatements)

if err = destination.ExecStatements(dwh, mergeStatements); err != nil {
return fmt.Errorf("failed to execute merge statements: %w", err)
}
Expand Down
14 changes: 14 additions & 0 deletions lib/debezium/converters/basic.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,3 +87,17 @@ func (Float64) Convert(value any) (any, error) {
return nil, fmt.Errorf("unexpected type %T", value)
}
}

type Array struct{}

func (Array) ToKindDetails() typing.KindDetails {
return typing.Array
}

func (Array) Convert(value any) (any, error) {
if fmt.Sprint(value) == fmt.Sprintf("[%s]", constants.ToastUnavailableValuePlaceholder) {
return constants.ToastUnavailableValuePlaceholder, nil
}

return value, nil
}
26 changes: 26 additions & 0 deletions lib/debezium/converters/basic_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package converters
import (
"testing"

"github.com/artie-labs/transfer/lib/config/constants"

"github.com/stretchr/testify/assert"
)

Expand Down Expand Up @@ -85,3 +87,27 @@ func TestFloat64_Convert(t *testing.T) {
}
}
}

func TestArray_Convert(t *testing.T) {
{
// Irrelevant data type
value, err := Array{}.Convert([]int{1, 2, 3, 4})
assert.NoError(t, err)
assert.Equal(t, []int{1, 2, 3, 4}, value)
}
{
// TOASTED data
{
// As []any
value, err := Array{}.Convert([]any{constants.ToastUnavailableValuePlaceholder})
assert.NoError(t, err)
assert.Equal(t, constants.ToastUnavailableValuePlaceholder, value)
}
{
// As []string
value, err := Array{}.Convert([]string{constants.ToastUnavailableValuePlaceholder})
assert.NoError(t, err)
assert.Equal(t, constants.ToastUnavailableValuePlaceholder, value)
}
}
}
4 changes: 2 additions & 2 deletions lib/debezium/schema.go
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,8 @@ func (f Field) ToValueConverter() (converters.ValueConverter, error) {
}

switch f.Type {
case Array:
return converters.Array{}, nil
case Double, Float:
return converters.Float64{}, nil
}
Expand Down Expand Up @@ -167,8 +169,6 @@ func (f Field) ToKindDetails() (typing.KindDetails, error) {
return typing.Struct, nil
case Boolean:
return typing.Boolean, nil
case Array:
return typing.Array, nil
default:
return typing.Invalid, fmt.Errorf("unhandled field type %q", f.Type)
}
Expand Down
14 changes: 0 additions & 14 deletions models/event/event.go
Original file line number Diff line number Diff line change
Expand Up @@ -224,22 +224,8 @@ func (e *Event) Save(cfg config.Config, inMemDB *models.DatabaseData, tc kafkali
toastedCol = true
}
}

valArray, isOk := val.([]any)
if isOk {
if len(valArray) == 1 {
if _, isOk = valArray[0].(string); isOk {
if valArray[0] == constants.ToastUnavailableValuePlaceholder {
val = constants.ToastUnavailableValuePlaceholder
toastedCol = true
}
}
}
}
}

fmt.Println("toastedCol", toastedCol, "val", val)

if toastedCol {
err := inMemoryColumns.UpsertColumn(newColName, columns.UpsertColumnArg{
ToastCol: typing.ToPtr(true),
Expand Down

0 comments on commit 79cf4ba

Please sign in to comment.