[snowflake] Handle case where string length exceeds column precision (#…

…607)

clients/snowflake/staging.go

@@ -18,13 +18,24 @@ import (
 	"github.com/artie-labs/transfer/lib/typing/values"
 )
 
-func replaceExceededValues(colVal string, colKind columns.Column) string {
+func replaceExceededValues(colVal string, kindDetails typing.KindDetails) string {
 	// https://community.snowflake.com/s/article/Max-LOB-size-exceeded
 	const maxLobLength = 16777216
-	if colKind.KindDetails.Kind == typing.Struct.Kind {
+
+	switch kindDetails.Kind {
+	case typing.Struct.Kind:
 		if len(colVal) > maxLobLength {
 			return fmt.Sprintf(`{"key":"%s"}`, constants.ExceededValueMarker)
 		}
+	case typing.String.Kind:
+		maxLength := maxLobLength
+		if kindDetails.OptionalStringPrecision != nil {
+			maxLength = *kindDetails.OptionalStringPrecision
+		}
+
+		if len(colVal) > maxLength {
+			return constants.ExceededValueMarker
+		}
 	}
 
 	return colVal
@@ -43,7 +54,7 @@ func castColValStaging(colVal any, colKind columns.Column, additionalDateFmts []
 		return "", err
 	}
 
-	return replaceExceededValues(value, colKind), nil
+	return replaceExceededValues(value, colKind.KindDetails), nil
 }
 
 func (s *Store) PrepareTemporaryTable(tableData *optimization.TableData, tableConfig *types.DwhTableConfig, tempTableID types.TableIdentifier, additionalSettings types.AdditionalSettings, createTempTable bool) error {

clients/snowflake/staging_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/artie-labs/transfer/clients/shared"
 	"github.com/artie-labs/transfer/lib/config"
+	"github.com/artie-labs/transfer/lib/ptr"
 	"github.com/artie-labs/transfer/lib/typing/columns"
 
 	"github.com/artie-labs/transfer/lib/destination/types"
@@ -21,6 +22,23 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+func (s *SnowflakeTestSuite) TestReplaceExceededValues() {
+	// String + OptionalStringPrecision not set + equal to max LOB length:
+	assert.Equal(s.T(), strings.Repeat("a", 16777216), replaceExceededValues(strings.Repeat("a", 16777216), typing.String))
+	// String + OptionalStringPrecision not set + greater than max LOB length:
+	assert.Equal(s.T(), constants.ExceededValueMarker, replaceExceededValues(strings.Repeat("a", 16777217), typing.String))
+	// String + OptionalStringPrecision set + equal to OptionalStringPrecision:
+	assert.Equal(s.T(),
+		strings.Repeat("a", 100),
+		replaceExceededValues(strings.Repeat("a", 100), typing.KindDetails{Kind: typing.String.Kind, OptionalStringPrecision: ptr.ToInt(100)}),
+	)
+	// String + OptionalStringPrecision set + larger than OptionalStringPrecision:
+	assert.Equal(s.T(),
+		constants.ExceededValueMarker,
+		replaceExceededValues(strings.Repeat("a", 101), typing.KindDetails{Kind: typing.String.Kind, OptionalStringPrecision: ptr.ToInt(100)}),
+	)
+}
+
 func (s *SnowflakeTestSuite) TestCastColValStaging() {
 	{
 		// Null