Deprecate createAllColumnsIfAvailable setting (#786)

lib/config/config_test.go

@@ -22,12 +22,10 @@ func TestSharedTransferConfig(t *testing.T) {
 		validBody := `
 typingSettings:
  additionalDateFormats: ["yyyy-MM-dd1"]
- createAllColumnsIfAvailable: true
 `
 		err := yaml.Unmarshal([]byte(validBody), &sharedTransferCfg)
 		assert.NoError(t, err)
 
-		assert.True(t, sharedTransferCfg.TypingSettings.CreateAllColumnsIfAvailable)
 		assert.Equal(t, "yyyy-MM-dd1", sharedTransferCfg.TypingSettings.AdditionalDateFormats[0])
 	}
 }
@@ -372,12 +370,6 @@ snowflake:
  warehouse: %s
  region: %s
  application: %s
-
-sharedTransferConfig:
-  typingSettings:
-    createAllColumnsIfAvailable: true
-
-
 reporting:
  sentry:
   dsn: %s
@@ -396,7 +388,6 @@ reporting:
 	assert.Equal(t, bootstrapServer, config.Kafka.BootstrapServer)
 	assert.Equal(t, groupID, config.Kafka.GroupID)
 	assert.Equal(t, password, config.Kafka.Password)
-	assert.True(t, config.SharedTransferConfig.TypingSettings.CreateAllColumnsIfAvailable)
 
 	orderIdx := -1
 	customerIdx := -1

lib/typing/parse.go

@@ -9,11 +9,6 @@ import (
 )
 
 func ParseValue(settings Settings, key string, optionalSchema map[string]KindDetails, val any) KindDetails {
-	if val == nil && !settings.CreateAllColumnsIfAvailable {
-		// If the value is nil and `createAllColumnsIfAvailable` = false, then return `Invalid
-		return Invalid
-	}
-
 	if len(optionalSchema) > 0 {
 		// If the column exists in the schema, let's early exit.
 		if kindDetail, isOk := optionalSchema[key]; isOk {

lib/typing/parse_test.go

@@ -12,7 +12,6 @@ import (
 
 func Test_ParseValue(t *testing.T) {
 	{
-
 		// Invalid
 		assert.Equal(t, ParseValue(Settings{}, "", nil, nil), Invalid)
 		assert.Equal(t, ParseValue(Settings{}, "", nil, errors.New("hello")), Invalid)
@@ -135,7 +134,6 @@ func TestOptionalSchema(t *testing.T) {
 		// Respect the schema if the value is not null.
 		assert.Equal(t, String, ParseValue(Settings{}, "created_at", optionalSchema, "2023-01-01"))
 		// Kind is invalid because `createAllColumnsIfAvailable` is not enabled.
-		assert.Equal(t, Invalid, ParseValue(Settings{}, "created_at", optionalSchema, nil))
-		assert.Equal(t, String, ParseValue(Settings{CreateAllColumnsIfAvailable: true}, "created_at", optionalSchema, nil))
+		assert.Equal(t, String, ParseValue(Settings{}, "created_at", optionalSchema, nil))
 	}
 }

lib/typing/typing.go

@@ -10,11 +10,6 @@ import (
 
 type Settings struct {
 	AdditionalDateFormats []string `yaml:"additionalDateFormats"`
-
-	// CreateAllColumnsIfAvailable - If true, we will create all columns if the metadata is available regardless of
-	// whether we have a value from the column. This will also bypass our Typing library.
-	// This only works for data sources with a schema such as Postgres and MySQL
-	CreateAllColumnsIfAvailable bool `yaml:"createAllColumnsIfAvailable"`
 }
 
 type KindDetails struct {