Skip to content

Commit

Permalink
Use ShouldUppercaseEscapedNames method (#482)
Browse files Browse the repository at this point in the history
  • Loading branch information
nathan-artie authored Apr 22, 2024
1 parent ef9814c commit 2910bc9
Show file tree
Hide file tree
Showing 7 changed files with 12 additions and 13 deletions.
2 changes: 1 addition & 1 deletion clients/bigquery/append.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,5 @@ import (

func (s *Store) Append(tableData *optimization.TableData) error {
tableID := s.IdentifierFor(tableData.TopicConfig(), tableData.Name())
return shared.Append(s, tableData, s.config, types.AppendOpts{TempTableID: tableID})
return shared.Append(s, tableData, types.AppendOpts{TempTableID: tableID})
}
2 changes: 1 addition & 1 deletion clients/mssql/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func (s *Store) Merge(tableData *optimization.TableData) error {

func (s *Store) Append(tableData *optimization.TableData) error {
tableID := s.IdentifierFor(tableData.TopicConfig(), tableData.Name())
return shared.Append(s, tableData, s.config, types.AppendOpts{TempTableID: tableID})
return shared.Append(s, tableData, types.AppendOpts{TempTableID: tableID})
}

// specificIdentifierFor returns a MS SQL [TableIdentifier] for a [TopicConfig] + table name.
Expand Down
2 changes: 1 addition & 1 deletion clients/redshift/writes.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ func (s *Store) Append(tableData *optimization.TableData) error {
// Redshift is slightly different, we'll load and create the temporary table via shared.Append
// Then, we'll invoke `ALTER TABLE target APPEND FROM staging` to combine the diffs.
temporaryTableID := shared.TempTableID(tableID, tableData.TempTableSuffix())
if err := shared.Append(s, tableData, s.config, types.AppendOpts{TempTableID: temporaryTableID}); err != nil {
if err := shared.Append(s, tableData, types.AppendOpts{TempTableID: temporaryTableID}); err != nil {
return err
}

Expand Down
6 changes: 3 additions & 3 deletions clients/shared/append.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,16 @@ package shared
import (
"log/slog"

"github.com/artie-labs/transfer/lib/config"
"github.com/artie-labs/transfer/lib/config/constants"
"github.com/artie-labs/transfer/lib/destination"
"github.com/artie-labs/transfer/lib/destination/ddl"
"github.com/artie-labs/transfer/lib/destination/types"
"github.com/artie-labs/transfer/lib/optimization"
"github.com/artie-labs/transfer/lib/ptr"
"github.com/artie-labs/transfer/lib/typing/columns"
)

func Append(dwh destination.DataWarehouse, tableData *optimization.TableData, cfg config.Config, opts types.AppendOpts) error {
func Append(dwh destination.DataWarehouse, tableData *optimization.TableData, opts types.AppendOpts) error {
if tableData.ShouldSkipUpdate() {
return nil
}
Expand All @@ -35,7 +35,7 @@ func Append(dwh destination.DataWarehouse, tableData *optimization.TableData, cf
CreateTable: tableConfig.CreateTable(),
ColumnOp: constants.Add,
CdcTime: tableData.LatestCDCTs,
UppercaseEscNames: &cfg.SharedDestinationConfig.UppercaseEscapedNames,
UppercaseEscNames: ptr.ToBool(dwh.ShouldUppercaseEscapedNames()),
Mode: tableData.Mode(),
}

Expand Down
8 changes: 4 additions & 4 deletions clients/shared/merge.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ func Merge(dwh destination.DataWarehouse, tableData *optimization.TableData, cfg
CreateTable: tableConfig.CreateTable(),
ColumnOp: constants.Add,
CdcTime: tableData.LatestCDCTs,
UppercaseEscNames: &cfg.SharedDestinationConfig.UppercaseEscapedNames,
UppercaseEscNames: ptr.ToBool(dwh.ShouldUppercaseEscapedNames()),
Mode: tableData.Mode(),
}

Expand All @@ -61,7 +61,7 @@ func Merge(dwh destination.DataWarehouse, tableData *optimization.TableData, cfg
ColumnOp: constants.Delete,
ContainOtherOperations: tableData.ContainOtherOperations(),
CdcTime: tableData.LatestCDCTs,
UppercaseEscNames: &cfg.SharedDestinationConfig.UppercaseEscapedNames,
UppercaseEscNames: ptr.ToBool(dwh.ShouldUppercaseEscapedNames()),
Mode: tableData.Mode(),
}

Expand Down Expand Up @@ -123,11 +123,11 @@ func Merge(dwh destination.DataWarehouse, tableData *optimization.TableData, cfg
TableID: tableID,
SubQuery: subQuery,
IdempotentKey: tableData.TopicConfig().IdempotentKey,
PrimaryKeys: tableData.PrimaryKeys(cfg.SharedDestinationConfig.UppercaseEscapedNames, &sql.NameArgs{Escape: true, DestKind: dwh.Label()}),
PrimaryKeys: tableData.PrimaryKeys(dwh.ShouldUppercaseEscapedNames(), &sql.NameArgs{Escape: true, DestKind: dwh.Label()}),
Columns: tableData.ReadOnlyInMemoryCols(),
SoftDelete: tableData.TopicConfig().SoftDelete,
DestKind: dwh.Label(),
UppercaseEscNames: &cfg.SharedDestinationConfig.UppercaseEscapedNames,
UppercaseEscNames: ptr.ToBool(dwh.ShouldUppercaseEscapedNames()),
ContainsHardDeletes: ptr.ToBool(tableData.ContainsHardDeletes()),
}

Expand Down
3 changes: 1 addition & 2 deletions clients/shared/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,7 @@ func BackfillColumn(cfg config.Config, dwh destination.DataWarehouse, column col
return fmt.Errorf("failed to escape default value: %w", err)
}

uppercaseEscNames := cfg.SharedDestinationConfig.UppercaseEscapedNames
escapedCol := column.Name(uppercaseEscNames, &sql.NameArgs{Escape: true, DestKind: dwh.Label()})
escapedCol := column.Name(dwh.ShouldUppercaseEscapedNames(), &sql.NameArgs{Escape: true, DestKind: dwh.Label()})

// TODO: This is added because `default` is not technically a column that requires escaping, but it is required when it's in the where clause.
// Once we escape everything by default, we can remove this patch of code.
Expand Down
2 changes: 1 addition & 1 deletion clients/snowflake/writes.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ func (s *Store) Append(tableData *optimization.TableData) error {

tableID := s.IdentifierFor(tableData.TopicConfig(), tableData.Name())
// TODO: For history mode - in the future, we could also have a separate stage name for history mode so we can enable parallel processing.
err = shared.Append(s, tableData, s.config, types.AppendOpts{
err = shared.Append(s, tableData, types.AppendOpts{
TempTableID: tableID,
AdditionalCopyClause: `FILE_FORMAT = (TYPE = 'csv' FIELD_DELIMITER= '\t' FIELD_OPTIONALLY_ENCLOSED_BY='"' NULL_IF='\\N' EMPTY_FIELD_AS_NULL=FALSE) PURGE = TRUE`,
})
Expand Down

0 comments on commit 2910bc9

Please sign in to comment.