Fix tests.

artie-labs · Apr 29, 2024 · 2d3ee95 · 2d3ee95
1 parent 81aa497
commit 2d3ee95
Show file tree

Hide file tree

Showing 5 changed files with 151 additions and 127 deletions.
diff --git a/clients/shared/merge.go b/clients/shared/merge.go
@@ -69,7 +69,6 @@ func Merge(dwh destination.DataWarehouse, tableData *optimization.TableData, cfg
 	}
 
 	tableConfig.AuditColumnsToDelete(srcKeysMissing)
-
 	if err = tableData.MergeColumnsFromDestination(tableConfig.Columns().GetColumns()...); err != nil {
 		return fmt.Errorf("failed to merge columns from destination: %w", err)
 	}

diff --git a/lib/optimization/event_update_test.go b/lib/optimization/event_update_test.go
@@ -11,139 +11,150 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+const strCol = "string"
+
 func TestTableData_UpdateInMemoryColumnsFromDestination(t *testing.T) {
-	const strCol = "string"
+	{
+		tableDataCols := &columns.Columns{}
+		tableData := &TableData{
+			inMemoryColumns: tableDataCols,
+		}
 
-	tableDataCols := &columns.Columns{}
-	tableDataCols.AddColumn(columns.NewColumn("name", typing.String))
-	tableDataCols.AddColumn(columns.NewColumn("bool_backfill", typing.Boolean))
-	tableDataCols.AddColumn(columns.NewColumn("prev_invalid", typing.Invalid))
-	tableDataCols.AddColumn(columns.NewColumn("numeric_test", typing.EDecimal))
+		tableDataCols.AddColumn(columns.NewColumn("name", typing.String))
+		tableDataCols.AddColumn(columns.NewColumn("bool_backfill", typing.Boolean))
+		tableDataCols.AddColumn(columns.NewColumn("prev_invalid", typing.Invalid))
+		tableDataCols.AddColumn(columns.NewColumn("numeric_test", typing.EDecimal))
 
-	// Casting these as STRING so tableColumn via this f(x) will set it correctly.
-	tableDataCols.AddColumn(columns.NewColumn("ext_date", typing.String))
-	tableDataCols.AddColumn(columns.NewColumn("ext_time", typing.String))
-	tableDataCols.AddColumn(columns.NewColumn("ext_datetime", typing.String))
-	tableDataCols.AddColumn(columns.NewColumn("ext_dec", typing.String))
+		// Casting these as STRING so tableColumn via this f(x) will set it correctly.
+		tableDataCols.AddColumn(columns.NewColumn("ext_date", typing.String))
+		tableDataCols.AddColumn(columns.NewColumn("ext_time", typing.String))
+		tableDataCols.AddColumn(columns.NewColumn("ext_datetime", typing.String))
+		tableDataCols.AddColumn(columns.NewColumn("ext_dec", typing.String))
 
-	extDecimalType := typing.EDecimal
-	extDecimalType.ExtendedDecimalDetails = decimal.NewDecimal(ptr.ToInt(22), 2, nil)
-	tableDataCols.AddColumn(columns.NewColumn("ext_dec_filled", extDecimalType))
+		extDecimalType := typing.EDecimal
+		extDecimalType.ExtendedDecimalDetails = decimal.NewDecimal(ptr.ToInt(22), 2, nil)
+		tableDataCols.AddColumn(columns.NewColumn("ext_dec_filled", extDecimalType))
 
-	tableDataCols.AddColumn(columns.NewColumn(strCol, typing.String))
+		tableDataCols.AddColumn(columns.NewColumn(strCol, typing.String))
 
-	tableData := &TableData{
-		inMemoryColumns: tableDataCols,
-	}
+		nonExistentTableCols := []string{"dusty", "the", "mini", "aussie"}
+		var nonExistentCols []columns.Column
+		for _, nonExistentTableCol := range nonExistentTableCols {
+			nonExistentCols = append(nonExistentCols, columns.NewColumn(nonExistentTableCol, typing.String))
+		}
 
-	nonExistentTableCols := []string{"dusty", "the", "mini", "aussie"}
-	var nonExistentCols []columns.Column
-	for _, nonExistentTableCol := range nonExistentTableCols {
-		nonExistentCols = append(nonExistentCols, columns.NewColumn(nonExistentTableCol, typing.String))
-	}
+		// Testing to make sure we don't copy over non-existent columns
+		tableData.MergeColumnsFromDestination(nonExistentCols...)
+		for _, nonExistentTableCol := range nonExistentTableCols {
+			_, isOk := tableData.inMemoryColumns.GetColumn(nonExistentTableCol)
+			assert.False(t, isOk, nonExistentTableCol)
+		}
 
-	// Testing to make sure we don't copy over non-existent columns
-	tableData.MergeColumnsFromDestination(nonExistentCols...)
-	for _, nonExistentTableCol := range nonExistentTableCols {
-		_, isOk := tableData.inMemoryColumns.GetColumn(nonExistentTableCol)
-		assert.False(t, isOk, nonExistentTableCol)
-	}
+		// Making sure it's still numeric
+		tableData.MergeColumnsFromDestination(columns.NewColumn("numeric_test", typing.Integer))
+		numericCol, isOk := tableData.inMemoryColumns.GetColumn("numeric_test")
+		assert.True(t, isOk)
+		assert.Equal(t, typing.EDecimal.Kind, numericCol.KindDetails.Kind, "numeric_test")
 
-	// Making sure it's still numeric
-	tableData.MergeColumnsFromDestination(columns.NewColumn("numeric_test", typing.Integer))
-	numericCol, isOk := tableData.inMemoryColumns.GetColumn("numeric_test")
-	assert.True(t, isOk)
-	assert.Equal(t, typing.EDecimal.Kind, numericCol.KindDetails.Kind, "numeric_test")
-
-	// Testing to make sure we're copying the kindDetails over.
-	tableData.MergeColumnsFromDestination(columns.NewColumn("prev_invalid", typing.String))
-	prevInvalidCol, isOk := tableData.inMemoryColumns.GetColumn("prev_invalid")
-	assert.True(t, isOk)
-	assert.Equal(t, typing.String, prevInvalidCol.KindDetails)
-
-	// Testing backfill
-	for _, inMemoryCol := range tableData.inMemoryColumns.GetColumns() {
-		assert.False(t, inMemoryCol.Backfilled(), inMemoryCol.RawName())
-	}
-	backfilledCol := columns.NewColumn("bool_backfill", typing.Boolean)
-	backfilledCol.SetBackfilled(true)
-	tableData.MergeColumnsFromDestination(backfilledCol)
-	for _, inMemoryCol := range tableData.inMemoryColumns.GetColumns() {
-		if inMemoryCol.RawName() == backfilledCol.RawName() {
-			assert.True(t, inMemoryCol.Backfilled(), inMemoryCol.RawName())
-		} else {
+		// Testing to make sure we're copying the kindDetails over.
+		tableData.MergeColumnsFromDestination(columns.NewColumn("prev_invalid", typing.String))
+		prevInvalidCol, isOk := tableData.inMemoryColumns.GetColumn("prev_invalid")
+		assert.True(t, isOk)
+		assert.Equal(t, typing.String, prevInvalidCol.KindDetails)
+
+		// Testing backfill
+		for _, inMemoryCol := range tableData.inMemoryColumns.GetColumns() {
 			assert.False(t, inMemoryCol.Backfilled(), inMemoryCol.RawName())
 		}
-	}
+		backfilledCol := columns.NewColumn("bool_backfill", typing.Boolean)
+		backfilledCol.SetBackfilled(true)
+		tableData.MergeColumnsFromDestination(backfilledCol)
+		for _, inMemoryCol := range tableData.inMemoryColumns.GetColumns() {
+			if inMemoryCol.RawName() == backfilledCol.RawName() {
+				assert.True(t, inMemoryCol.Backfilled(), inMemoryCol.RawName())
+			} else {
+				assert.False(t, inMemoryCol.Backfilled(), inMemoryCol.RawName())
+			}
+		}
+
+		// Testing extTimeDetails
+		for _, extTimeDetailsCol := range []string{"ext_date", "ext_time", "ext_datetime"} {
+			col, isOk := tableData.inMemoryColumns.GetColumn(extTimeDetailsCol)
+			assert.True(t, isOk, extTimeDetailsCol)
+			assert.Equal(t, typing.String, col.KindDetails, extTimeDetailsCol)
+			assert.Nil(t, col.KindDetails.ExtendedTimeDetails, extTimeDetailsCol)
+		}
 
-	// Testing extTimeDetails
-	for _, extTimeDetailsCol := range []string{"ext_date", "ext_time", "ext_datetime"} {
-		col, isOk := tableData.inMemoryColumns.GetColumn(extTimeDetailsCol)
-		assert.True(t, isOk, extTimeDetailsCol)
-		assert.Equal(t, typing.String, col.KindDetails, extTimeDetailsCol)
-		assert.Nil(t, col.KindDetails.ExtendedTimeDetails, extTimeDetailsCol)
+		tableData.MergeColumnsFromDestination(columns.NewColumn("ext_time", typing.NewKindDetailsFromTemplate(typing.ETime, ext.TimeKindType)))
+		tableData.MergeColumnsFromDestination(columns.NewColumn("ext_date", typing.NewKindDetailsFromTemplate(typing.ETime, ext.DateKindType)))
+		tableData.MergeColumnsFromDestination(columns.NewColumn("ext_datetime", typing.NewKindDetailsFromTemplate(typing.ETime, ext.DateTimeKindType)))
+
+		dateCol, isOk := tableData.inMemoryColumns.GetColumn("ext_date")
+		assert.True(t, isOk)
+		assert.NotNil(t, dateCol.KindDetails.ExtendedTimeDetails)
+		assert.Equal(t, ext.DateKindType, dateCol.KindDetails.ExtendedTimeDetails.Type)
+
+		timeCol, isOk := tableData.inMemoryColumns.GetColumn("ext_time")
+		assert.True(t, isOk)
+		assert.NotNil(t, timeCol.KindDetails.ExtendedTimeDetails)
+		assert.Equal(t, ext.TimeKindType, timeCol.KindDetails.ExtendedTimeDetails.Type)
+
+		dateTimeCol, isOk := tableData.inMemoryColumns.GetColumn("ext_datetime")
+		assert.True(t, isOk)
+		assert.NotNil(t, dateTimeCol.KindDetails.ExtendedTimeDetails)
+		assert.Equal(t, ext.DateTimeKindType, dateTimeCol.KindDetails.ExtendedTimeDetails.Type)
+
+		// Testing extDecimalDetails
+		// Confirm that before you update, it's invalid.
+		extDecCol, isOk := tableData.inMemoryColumns.GetColumn("ext_dec")
+		assert.True(t, isOk)
+		assert.Equal(t, typing.String, extDecCol.KindDetails)
+
+		extDecimal := typing.EDecimal
+		extDecimal.ExtendedDecimalDetails = decimal.NewDecimal(ptr.ToInt(30), 2, nil)
+		tableData.MergeColumnsFromDestination(columns.NewColumn("ext_dec", extDecimal))
+		// Now it should be ext decimal type
+		extDecCol, isOk = tableData.inMemoryColumns.GetColumn("ext_dec")
+		assert.True(t, isOk)
+		assert.Equal(t, typing.EDecimal.Kind, extDecCol.KindDetails.Kind)
+		// Check precision and scale too.
+		assert.Equal(t, 30, *extDecCol.KindDetails.ExtendedDecimalDetails.Precision())
+		assert.Equal(t, 2, extDecCol.KindDetails.ExtendedDecimalDetails.Scale())
+
+		// Testing ext_dec_filled since it's already filled out
+		extDecColFilled, isOk := tableData.inMemoryColumns.GetColumn("ext_dec_filled")
+		assert.True(t, isOk)
+		assert.Equal(t, typing.EDecimal.Kind, extDecColFilled.KindDetails.Kind)
+		// Check precision and scale too.
+		assert.Equal(t, 22, *extDecColFilled.KindDetails.ExtendedDecimalDetails.Precision())
+		assert.Equal(t, 2, extDecColFilled.KindDetails.ExtendedDecimalDetails.Scale())
+
+		tableData.MergeColumnsFromDestination(columns.NewColumn("ext_dec_filled", extDecimal))
+		extDecColFilled, isOk = tableData.inMemoryColumns.GetColumn("ext_dec_filled")
+		assert.True(t, isOk)
+		assert.Equal(t, typing.EDecimal.Kind, extDecColFilled.KindDetails.Kind)
+		// Check precision and scale too.
+		assert.Equal(t, 22, *extDecColFilled.KindDetails.ExtendedDecimalDetails.Precision())
+		assert.Equal(t, 2, extDecColFilled.KindDetails.ExtendedDecimalDetails.Scale())
 	}
+	{
+		tableDataCols := &columns.Columns{}
+		tableData := &TableData{
+			inMemoryColumns: tableDataCols,
+		}
+
+		tableDataCols.AddColumn(columns.NewColumn(strCol, typing.String))
+
+		// Testing string precision
+		stringKindWithPrecision := typing.KindDetails{
+			Kind:                    typing.String.Kind,
+			OptionalStringPrecision: ptr.ToInt(123),
+		}
 
-	tableData.MergeColumnsFromDestination(columns.NewColumn("ext_time", typing.NewKindDetailsFromTemplate(typing.ETime, ext.TimeKindType)))
-	tableData.MergeColumnsFromDestination(columns.NewColumn("ext_date", typing.NewKindDetailsFromTemplate(typing.ETime, ext.DateKindType)))
-	tableData.MergeColumnsFromDestination(columns.NewColumn("ext_datetime", typing.NewKindDetailsFromTemplate(typing.ETime, ext.DateTimeKindType)))
-
-	dateCol, isOk := tableData.inMemoryColumns.GetColumn("ext_date")
-	assert.True(t, isOk)
-	assert.NotNil(t, dateCol.KindDetails.ExtendedTimeDetails)
-	assert.Equal(t, ext.DateKindType, dateCol.KindDetails.ExtendedTimeDetails.Type)
-
-	timeCol, isOk := tableData.inMemoryColumns.GetColumn("ext_time")
-	assert.True(t, isOk)
-	assert.NotNil(t, timeCol.KindDetails.ExtendedTimeDetails)
-	assert.Equal(t, ext.TimeKindType, timeCol.KindDetails.ExtendedTimeDetails.Type)
-
-	dateTimeCol, isOk := tableData.inMemoryColumns.GetColumn("ext_datetime")
-	assert.True(t, isOk)
-	assert.NotNil(t, dateTimeCol.KindDetails.ExtendedTimeDetails)
-	assert.Equal(t, ext.DateTimeKindType, dateTimeCol.KindDetails.ExtendedTimeDetails.Type)
-
-	// Testing extDecimalDetails
-	// Confirm that before you update, it's invalid.
-	extDecCol, isOk := tableData.inMemoryColumns.GetColumn("ext_dec")
-	assert.True(t, isOk)
-	assert.Equal(t, typing.String, extDecCol.KindDetails)
-
-	extDecimal := typing.EDecimal
-	extDecimal.ExtendedDecimalDetails = decimal.NewDecimal(ptr.ToInt(30), 2, nil)
-	tableData.MergeColumnsFromDestination(columns.NewColumn("ext_dec", extDecimal))
-	// Now it should be ext decimal type
-	extDecCol, isOk = tableData.inMemoryColumns.GetColumn("ext_dec")
-	assert.True(t, isOk)
-	assert.Equal(t, typing.EDecimal.Kind, extDecCol.KindDetails.Kind)
-	// Check precision and scale too.
-	assert.Equal(t, 30, *extDecCol.KindDetails.ExtendedDecimalDetails.Precision())
-	assert.Equal(t, 2, extDecCol.KindDetails.ExtendedDecimalDetails.Scale())
-
-	// Testing ext_dec_filled since it's already filled out
-	extDecColFilled, isOk := tableData.inMemoryColumns.GetColumn("ext_dec_filled")
-	assert.True(t, isOk)
-	assert.Equal(t, typing.EDecimal.Kind, extDecColFilled.KindDetails.Kind)
-	// Check precision and scale too.
-	assert.Equal(t, 22, *extDecColFilled.KindDetails.ExtendedDecimalDetails.Precision())
-	assert.Equal(t, 2, extDecColFilled.KindDetails.ExtendedDecimalDetails.Scale())
-
-	tableData.MergeColumnsFromDestination(columns.NewColumn("ext_dec_filled", extDecimal))
-	extDecColFilled, isOk = tableData.inMemoryColumns.GetColumn("ext_dec_filled")
-	assert.True(t, isOk)
-	assert.Equal(t, typing.EDecimal.Kind, extDecColFilled.KindDetails.Kind)
-	// Check precision and scale too.
-	assert.Equal(t, 22, *extDecColFilled.KindDetails.ExtendedDecimalDetails.Precision())
-	assert.Equal(t, 2, extDecColFilled.KindDetails.ExtendedDecimalDetails.Scale())
-
-	// Testing string precision
-	stringKindWithPrecision := typing.KindDetails{
-		Kind:                    typing.String.Kind,
-		OptionalStringPrecision: ptr.ToInt(123),
+		tableData.MergeColumnsFromDestination(columns.NewColumn(strCol, stringKindWithPrecision))
+		foundStrCol, isOk := tableData.inMemoryColumns.GetColumn(strCol)
+		assert.True(t, isOk)
+		assert.Equal(t, typing.String.Kind, foundStrCol.KindDetails.Kind)
+		assert.Equal(t, 123, *foundStrCol.KindDetails.OptionalStringPrecision)
 	}
-	tableData.MergeColumnsFromDestination(columns.NewColumn(strCol, stringKindWithPrecision))
-	foundStrCol, isOk := tableData.inMemoryColumns.GetColumn(strCol)
-	assert.True(t, isOk)
-	assert.Equal(t, typing.String.Kind, foundStrCol.KindDetails.Kind)
-	assert.Equal(t, 123, *foundStrCol.KindDetails.OptionalStringPrecision)
 }
diff --git a/lib/typing/bigquery_test.go b/lib/typing/bigquery_test.go
@@ -1,6 +1,7 @@
 package typing
 
 import (
+	"fmt"
 	"testing"
 	"time"
 
@@ -52,7 +53,12 @@ func TestBigQueryTypeToKind(t *testing.T) {
 
 	for bqCol, expectedKind := range bqColToExpectedKind {
 		kd, err := DwhTypeToKind(constants.BigQuery, bqCol, "")
-		assert.NoError(t, err)
+		if expectedKind.Kind == Invalid.Kind {
+			assert.ErrorContains(t, err, fmt.Sprintf("unable to map type: %q to dwh type", bqCol))
+		} else {
+			assert.NoError(t, err)
+		}
+
 		assert.Equal(t, expectedKind.Kind, kd.Kind, bqCol)
 	}
 }

diff --git a/lib/typing/snowflake_test.go b/lib/typing/snowflake_test.go
@@ -41,7 +41,7 @@ func TestSnowflakeTypeToKindFloats(t *testing.T) {
 	{
 		// Invalid because precision nor scale is included.
 		kd, err := DwhTypeToKind(constants.Snowflake, "NUMERIC", "")
-		assert.NoError(t, err)
+		assert.ErrorContains(t, err, `unable to map type: "numeric" to dwh type`)
 		assert.Equal(t, Invalid, kd)
 	}
 	{
@@ -105,12 +105,12 @@ func TestSnowflakeTypeToKindComplex(t *testing.T) {
 func TestSnowflakeTypeToKindErrors(t *testing.T) {
 	{
 		kd, err := DwhTypeToKind(constants.Snowflake, "", "")
-		assert.NoError(t, err)
+		assert.ErrorContains(t, err, `unable to map type: "" to dwh type`)
 		assert.Equal(t, Invalid, kd)
 	}
 	{
 		kd, err := DwhTypeToKind(constants.Snowflake, "abc123", "")
-		assert.NoError(t, err)
+		assert.ErrorContains(t, err, `unable to map type: "abc123" to dwh type`)
 		assert.Equal(t, Invalid, kd)
 	}
 }

diff --git a/lib/typing/typing.go b/lib/typing/typing.go
@@ -196,6 +196,8 @@ func DwhTypeToKind(dwh constants.DestinationKind, dwhType, stringPrecision strin
 	dwhType = strings.ToLower(dwhType)
 
 	var kd KindDetails
+	var unmatched bool
+
 	switch dwh {
 	case constants.Snowflake:
 		kd = snowflakeTypeToKind(dwhType)
@@ -205,10 +207,16 @@ func DwhTypeToKind(dwh constants.DestinationKind, dwhType, stringPrecision strin
 		kd = redshiftTypeToKind(dwhType, stringPrecision)
 	case constants.MSSQL:
 		kd = mssqlTypeToKind(dwhType, stringPrecision)
+	default:
+		unmatched = true
 	}
 
-	if kd.Kind == Invalid.Kind {
-		return Invalid, fmt.Errorf("unable to map type: %q to dwh type", dwhType)
+	if !unmatched {
+		if kd.Kind == Invalid.Kind {
+			return Invalid, fmt.Errorf("unable to map type: %q to dwh type", dwhType)
+		} else {
+			return kd, nil
+		}
 	}
 
 	return Invalid, fmt.Errorf("unexpected dwh kind, label: %v", dwh)
-Original file line number
+Diff line change
@@ Expand Up @@
     	}
     	tableConfig.AuditColumnsToDelete(srcKeysMissing)
     	if err = tableData.MergeColumnsFromDestination(tableConfig.Columns().GetColumns()...); err != nil {
     		return fmt.Errorf("failed to merge columns from destination: %w", err)
     	}
@@ Expand Down @@