Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use ShouldUppercaseEscapedNames method #482

Merged
merged 2 commits into from
Apr 22, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion clients/bigquery/append.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,5 @@ import (

func (s *Store) Append(tableData *optimization.TableData) error {
tableID := s.IdentifierFor(tableData.TopicConfig(), tableData.Name())
return shared.Append(s, tableData, s.config, types.AppendOpts{TempTableID: tableID})
return shared.Append(s, tableData, types.AppendOpts{TempTableID: tableID})
}
2 changes: 1 addition & 1 deletion clients/mssql/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func (s *Store) Merge(tableData *optimization.TableData) error {

func (s *Store) Append(tableData *optimization.TableData) error {
tableID := s.IdentifierFor(tableData.TopicConfig(), tableData.Name())
return shared.Append(s, tableData, s.config, types.AppendOpts{TempTableID: tableID})
return shared.Append(s, tableData, types.AppendOpts{TempTableID: tableID})
}

// specificIdentifierFor returns a MS SQL [TableIdentifier] for a [TopicConfig] + table name.
Expand Down
2 changes: 1 addition & 1 deletion clients/redshift/writes.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ func (s *Store) Append(tableData *optimization.TableData) error {
// Redshift is slightly different, we'll load and create the temporary table via shared.Append
// Then, we'll invoke `ALTER TABLE target APPEND FROM staging` to combine the diffs.
temporaryTableID := shared.TempTableID(tableID, tableData.TempTableSuffix())
if err := shared.Append(s, tableData, s.config, types.AppendOpts{TempTableID: temporaryTableID}); err != nil {
if err := shared.Append(s, tableData, types.AppendOpts{TempTableID: temporaryTableID}); err != nil {
return err
}

Expand Down
6 changes: 3 additions & 3 deletions clients/shared/append.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,16 @@ package shared
import (
"log/slog"

"github.com/artie-labs/transfer/lib/config"
"github.com/artie-labs/transfer/lib/config/constants"
"github.com/artie-labs/transfer/lib/destination"
"github.com/artie-labs/transfer/lib/destination/ddl"
"github.com/artie-labs/transfer/lib/destination/types"
"github.com/artie-labs/transfer/lib/optimization"
"github.com/artie-labs/transfer/lib/ptr"
"github.com/artie-labs/transfer/lib/typing/columns"
)

func Append(dwh destination.DataWarehouse, tableData *optimization.TableData, cfg config.Config, opts types.AppendOpts) error {
func Append(dwh destination.DataWarehouse, tableData *optimization.TableData, opts types.AppendOpts) error {
if tableData.ShouldSkipUpdate() {
return nil
}
Expand All @@ -35,7 +35,7 @@ func Append(dwh destination.DataWarehouse, tableData *optimization.TableData, cf
CreateTable: tableConfig.CreateTable(),
ColumnOp: constants.Add,
CdcTime: tableData.LatestCDCTs,
UppercaseEscNames: &cfg.SharedDestinationConfig.UppercaseEscapedNames,
UppercaseEscNames: ptr.ToBool(dwh.ShouldUppercaseEscapedNames()),
Mode: tableData.Mode(),
}

Expand Down
8 changes: 4 additions & 4 deletions clients/shared/merge.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ func Merge(dwh destination.DataWarehouse, tableData *optimization.TableData, cfg
CreateTable: tableConfig.CreateTable(),
ColumnOp: constants.Add,
CdcTime: tableData.LatestCDCTs,
UppercaseEscNames: &cfg.SharedDestinationConfig.UppercaseEscapedNames,
UppercaseEscNames: ptr.ToBool(dwh.ShouldUppercaseEscapedNames()),
Mode: tableData.Mode(),
}

Expand All @@ -61,7 +61,7 @@ func Merge(dwh destination.DataWarehouse, tableData *optimization.TableData, cfg
ColumnOp: constants.Delete,
ContainOtherOperations: tableData.ContainOtherOperations(),
CdcTime: tableData.LatestCDCTs,
UppercaseEscNames: &cfg.SharedDestinationConfig.UppercaseEscapedNames,
UppercaseEscNames: ptr.ToBool(dwh.ShouldUppercaseEscapedNames()),
Mode: tableData.Mode(),
}

Expand Down Expand Up @@ -123,11 +123,11 @@ func Merge(dwh destination.DataWarehouse, tableData *optimization.TableData, cfg
TableID: tableID,
SubQuery: subQuery,
IdempotentKey: tableData.TopicConfig().IdempotentKey,
PrimaryKeys: tableData.PrimaryKeys(cfg.SharedDestinationConfig.UppercaseEscapedNames, &sql.NameArgs{Escape: true, DestKind: dwh.Label()}),
PrimaryKeys: tableData.PrimaryKeys(dwh.ShouldUppercaseEscapedNames(), &sql.NameArgs{Escape: true, DestKind: dwh.Label()}),
Columns: tableData.ReadOnlyInMemoryCols(),
SoftDelete: tableData.TopicConfig().SoftDelete,
DestKind: dwh.Label(),
UppercaseEscNames: &cfg.SharedDestinationConfig.UppercaseEscapedNames,
UppercaseEscNames: ptr.ToBool(dwh.ShouldUppercaseEscapedNames()),
ContainsHardDeletes: ptr.ToBool(tableData.ContainsHardDeletes()),
}

Expand Down
3 changes: 1 addition & 2 deletions clients/shared/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,7 @@ func BackfillColumn(cfg config.Config, dwh destination.DataWarehouse, column col
return fmt.Errorf("failed to escape default value: %w", err)
}

uppercaseEscNames := cfg.SharedDestinationConfig.UppercaseEscapedNames
escapedCol := column.Name(uppercaseEscNames, &sql.NameArgs{Escape: true, DestKind: dwh.Label()})
escapedCol := column.Name(dwh.ShouldUppercaseEscapedNames(), &sql.NameArgs{Escape: true, DestKind: dwh.Label()})

// TODO: This is added because `default` is not technically a column that requires escaping, but it is required when it's in the where clause.
// Once we escape everything by default, we can remove this patch of code.
Expand Down
2 changes: 1 addition & 1 deletion clients/snowflake/writes.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ func (s *Store) Append(tableData *optimization.TableData) error {

tableID := s.IdentifierFor(tableData.TopicConfig(), tableData.Name())
// TODO: For history mode - in the future, we could also have a separate stage name for history mode so we can enable parallel processing.
err = shared.Append(s, tableData, s.config, types.AppendOpts{
err = shared.Append(s, tableData, types.AppendOpts{
TempTableID: tableID,
AdditionalCopyClause: `FILE_FORMAT = (TYPE = 'csv' FIELD_DELIMITER= '\t' FIELD_OPTIONALLY_ENCLOSED_BY='"' NULL_IF='\\N' EMPTY_FIELD_AS_NULL=FALSE) PURGE = TRUE`,
})
Expand Down