Clean up more.

artie-labs · Nov 21, 2024 · b32e4d2 · b32e4d2
1 parent c132d13
commit b32e4d2
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 15 deletions.
diff --git a/clients/s3/s3.go b/clients/s3/s3.go
@@ -98,7 +98,7 @@ func (s *Store) Merge(ctx context.Context, tableData *optimization.TableData) er
 	for _, val := range tableData.Rows() {
 		var row []any
 		for _, col := range cols {
-			value, err := parquetutil.ParseValue(val[col.Name()], col)
+			value, err := parquetutil.ParseValue(val[col.Name()], col.KindDetails)
 			if err != nil {
 				return fmt.Errorf("failed to parse value, err: %w, value: %v, column: %q", err, val[col.Name()], col.Name())
 			}

diff --git a/lib/parquetutil/parse_values.go b/lib/parquetutil/parse_values.go
@@ -9,17 +9,16 @@ import (
 	"github.com/artie-labs/transfer/lib/array"
 	"github.com/artie-labs/transfer/lib/config/constants"
 	"github.com/artie-labs/transfer/lib/typing"
-	"github.com/artie-labs/transfer/lib/typing/columns"
 	"github.com/artie-labs/transfer/lib/typing/decimal"
 	"github.com/artie-labs/transfer/lib/typing/ext"
 )
 
-func ParseValue(colVal any, colKind columns.Column) (any, error) {
+func ParseValue(colVal any, colKind typing.KindDetails) (any, error) {
 	if colVal == nil {
 		return nil, nil
 	}
 
-	switch colKind.KindDetails.Kind {
+	switch colKind.Kind {
 	case typing.Date.Kind:
 		_time, err := ext.ParseDateFromAny(colVal)
 		if err != nil {
@@ -51,7 +50,7 @@ func ParseValue(colVal any, colKind columns.Column) (any, error) {
 	case typing.String.Kind:
 		return colVal, nil
 	case typing.Struct.Kind:
-		if colKind.KindDetails == typing.Struct {
+		if colKind == typing.Struct {
 			if strings.Contains(fmt.Sprint(colVal), constants.ToastUnavailableValuePlaceholder) {
 				colVal = map[string]any{
 					"key": constants.ToastUnavailableValuePlaceholder,

diff --git a/lib/parquetutil/parse_values_test.go b/lib/parquetutil/parse_values_test.go
@@ -5,41 +5,40 @@ import (
 
 	"github.com/artie-labs/transfer/lib/numbers"
 	"github.com/artie-labs/transfer/lib/typing"
-	"github.com/artie-labs/transfer/lib/typing/columns"
 	"github.com/artie-labs/transfer/lib/typing/decimal"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestParseValue(t *testing.T) {
 	{
 		// Nil
-		value, err := ParseValue(nil, columns.Column{})
+		value, err := ParseValue(nil, typing.KindDetails{})
 		assert.NoError(t, err)
 		assert.Nil(t, value)
 	}
 	{
 		// String
-		value, err := ParseValue("test", columns.NewColumn("", typing.String))
+		value, err := ParseValue("test", typing.String)
 		assert.NoError(t, err)
 		assert.Equal(t, "test", value)
 	}
 	{
 		// Struct
-		value, err := ParseValue(map[string]any{"foo": "bar"}, columns.NewColumn("", typing.Struct))
+		value, err := ParseValue(map[string]any{"foo": "bar"}, typing.Struct)
 		assert.NoError(t, err)
 		assert.Equal(t, `{"foo":"bar"}`, value)
 	}
 	{
 		// Arrays
 		{
 			// Arrays (numbers - converted to string)
-			value, err := ParseValue([]any{123, 456}, columns.NewColumn("", typing.Array))
+			value, err := ParseValue([]any{123, 456}, typing.Array)
 			assert.NoError(t, err)
 			assert.Equal(t, []string{"123", "456"}, value)
 		}
 		{
 			// Arrays (booleans - converted to string)
-			value, err := ParseValue([]any{false, true, false}, columns.NewColumn("", typing.Array))
+			value, err := ParseValue([]any{false, true, false}, typing.Array)
 			assert.NoError(t, err)
 			assert.Equal(t, []string{"false", "true", "false"}, value)
 		}
@@ -48,27 +47,27 @@ func TestParseValue(t *testing.T) {
 		// Decimal
 		value, err := ParseValue(decimal.NewDecimalWithPrecision(
 			numbers.MustParseDecimal("5000.22320"), 30),
-			columns.NewColumn("", typing.NewDecimalDetailsFromTemplate(typing.EDecimal, decimal.NewDetails(30, 5))),
+			typing.NewDecimalDetailsFromTemplate(typing.EDecimal, decimal.NewDetails(30, 5)),
 		)
 
 		assert.NoError(t, err)
 		assert.Equal(t, "5000.22320", value)
 	}
 	{
 		// Time
-		value, err := ParseValue("03:15:00", columns.NewColumn("", typing.Time))
+		value, err := ParseValue("03:15:00", typing.Time)
 		assert.NoError(t, err)
 		assert.Equal(t, "03:15:00Z", value)
 	}
 	{
 		// Date
-		value, err := ParseValue("2022-12-25", columns.NewColumn("", typing.Date))
+		value, err := ParseValue("2022-12-25", typing.Date)
 		assert.NoError(t, err)
 		assert.Equal(t, "2022-12-25", value)
 	}
 	{
 		// Timestamp TZ
-		value, err := ParseValue("2023-04-24T17:29:05.69944Z", columns.NewColumn("", typing.TimestampTZ))
+		value, err := ParseValue("2023-04-24T17:29:05.69944Z", typing.TimestampTZ)
 		assert.NoError(t, err)
 		assert.Equal(t, int64(1682357345699), value)
 	}