Skip to content

Commit

Permalink
Remove uen from ma
Browse files Browse the repository at this point in the history
  • Loading branch information
@nathan-artie
nathan-artie committed May 1, 2024
1 parent ff3851c commit a9f4e5b
Show file tree
Hide file tree
Showing 23 changed files with 196 additions and 307 deletions.
19 changes: 7 additions & 12 deletions clients/bigquery/bigquery.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,13 @@ type Store struct {
func (s *Store) PrepareTemporaryTable(tableData *optimization.TableData, tableConfig *types.DwhTableConfig, tempTableID types.TableIdentifier, _ types.AdditionalSettings, createTempTable bool) error {
if createTempTable {
tempAlterTableArgs := ddl.AlterTableArgs{
Dwh: s,
Tc: tableConfig,
TableID: tempTableID,
CreateTable: true,
TemporaryTable: true,
ColumnOp: constants.Add,
UppercaseEscNames: ptr.ToBool(s.ShouldUppercaseEscapedNames()),
Mode: tableData.Mode(),
Dwh: s,
Tc: tableConfig,
TableID: tempTableID,
CreateTable: true,
TemporaryTable: true,
ColumnOp: constants.Add,
Mode: tableData.Mode(),
}

if err := tempAlterTableArgs.AlterTable(tableData.ReadOnlyInMemoryCols().GetColumns()...); err != nil {
Expand Down Expand Up @@ -119,10 +118,6 @@ func (s *Store) Dialect() sql.Dialect {
return sql.BigQueryDialect{}
}

func (s *Store) ShouldUppercaseEscapedNames() bool {
return false
}

func (s *Store) GetClient(ctx context.Context) *bigquery.Client {
client, err := bigquery.NewClient(ctx, s.config.BigQuery.ProjectID)
if err != nil {
Expand Down
16 changes: 7 additions & 9 deletions clients/mssql/staging.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,20 +9,18 @@ import (
"github.com/artie-labs/transfer/lib/destination/ddl"
"github.com/artie-labs/transfer/lib/destination/types"
"github.com/artie-labs/transfer/lib/optimization"
"github.com/artie-labs/transfer/lib/ptr"
)

func (s *Store) PrepareTemporaryTable(tableData *optimization.TableData, tableConfig *types.DwhTableConfig, tempTableID types.TableIdentifier, _ types.AdditionalSettings, createTempTable bool) error {
if createTempTable {
tempAlterTableArgs := ddl.AlterTableArgs{
Dwh: s,
Tc: tableConfig,
TableID: tempTableID,
CreateTable: true,
TemporaryTable: true,
ColumnOp: constants.Add,
UppercaseEscNames: ptr.ToBool(s.ShouldUppercaseEscapedNames()),
Mode: tableData.Mode(),
Dwh: s,
Tc: tableConfig,
TableID: tempTableID,
CreateTable: true,
TemporaryTable: true,
ColumnOp: constants.Add,
Mode: tableData.Mode(),
}

if err := tempAlterTableArgs.AlterTable(tableData.ReadOnlyInMemoryCols().GetColumns()...); err != nil {
Expand Down
4 changes: 0 additions & 4 deletions clients/mssql/store.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,6 @@ func (s *Store) Dialect() sql.Dialect {
return sql.DefaultDialect{}
}

func (s *Store) ShouldUppercaseEscapedNames() bool {
return false
}

func (s *Store) Merge(tableData *optimization.TableData) error {
return shared.Merge(s, tableData, s.config, types.MergeOpts{})
}
Expand Down
4 changes: 0 additions & 4 deletions clients/redshift/redshift.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,6 @@ func (s *Store) Dialect() sql.Dialect {
return sql.RedshiftDialect{}
}

func (s *Store) ShouldUppercaseEscapedNames() bool {
return false
}

func (s *Store) GetTableConfig(tableData *optimization.TableData) (*types.DwhTableConfig, error) {
const (
describeNameCol = "column_name"
Expand Down
16 changes: 7 additions & 9 deletions clients/redshift/staging.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,21 +12,19 @@ import (
"github.com/artie-labs/transfer/lib/destination/ddl"
"github.com/artie-labs/transfer/lib/destination/types"
"github.com/artie-labs/transfer/lib/optimization"
"github.com/artie-labs/transfer/lib/ptr"
"github.com/artie-labs/transfer/lib/s3lib"
)

func (s *Store) PrepareTemporaryTable(tableData *optimization.TableData, tableConfig *types.DwhTableConfig, tempTableID types.TableIdentifier, _ types.AdditionalSettings, _ bool) error {
// Redshift always creates a temporary table.
tempAlterTableArgs := ddl.AlterTableArgs{
Dwh: s,
Tc: tableConfig,
TableID: tempTableID,
CreateTable: true,
TemporaryTable: true,
ColumnOp: constants.Add,
UppercaseEscNames: ptr.ToBool(s.ShouldUppercaseEscapedNames()),
Mode: tableData.Mode(),
Dwh: s,
Tc: tableConfig,
TableID: tempTableID,
CreateTable: true,
TemporaryTable: true,
ColumnOp: constants.Add,
Mode: tableData.Mode(),
}

if err := tempAlterTableArgs.AlterTable(tableData.ReadOnlyInMemoryCols().GetColumns()...); err != nil {
Expand Down
16 changes: 7 additions & 9 deletions clients/shared/append.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"github.com/artie-labs/transfer/lib/destination/ddl"
"github.com/artie-labs/transfer/lib/destination/types"
"github.com/artie-labs/transfer/lib/optimization"
"github.com/artie-labs/transfer/lib/ptr"
"github.com/artie-labs/transfer/lib/typing/columns"
)

Expand All @@ -29,14 +28,13 @@ func Append(dwh destination.DataWarehouse, tableData *optimization.TableData, op
tableData.TopicConfig().IncludeDatabaseUpdatedAt, tableData.Mode())

createAlterTableArgs := ddl.AlterTableArgs{
Dwh: dwh,
Tc: tableConfig,
TableID: tableID,
CreateTable: tableConfig.CreateTable(),
ColumnOp: constants.Add,
CdcTime: tableData.LatestCDCTs,
UppercaseEscNames: ptr.ToBool(dwh.ShouldUppercaseEscapedNames()),
Mode: tableData.Mode(),
Dwh: dwh,
Tc: tableConfig,
TableID: tableID,
CreateTable: tableConfig.CreateTable(),
ColumnOp: constants.Add,
CdcTime: tableData.LatestCDCTs,
Mode: tableData.Mode(),
}

// Keys that exist in CDC stream, but not in DWH
Expand Down
17 changes: 7 additions & 10 deletions clients/shared/merge.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,13 @@ func Merge(dwh destination.DataWarehouse, tableData *optimization.TableData, cfg

tableID := dwh.IdentifierFor(tableData.TopicConfig(), tableData.Name())
createAlterTableArgs := ddl.AlterTableArgs{
Dwh: dwh,
Tc: tableConfig,
TableID: tableID,
CreateTable: tableConfig.CreateTable(),
ColumnOp: constants.Add,
CdcTime: tableData.LatestCDCTs,
UppercaseEscNames: ptr.ToBool(dwh.ShouldUppercaseEscapedNames()),
Mode: tableData.Mode(),
Dwh: dwh,
Tc: tableConfig,
TableID: tableID,
CreateTable: tableConfig.CreateTable(),
ColumnOp: constants.Add,
CdcTime: tableData.LatestCDCTs,
Mode: tableData.Mode(),
}

// Columns that are missing in DWH, but exist in our CDC stream.
Expand All @@ -60,7 +59,6 @@ func Merge(dwh destination.DataWarehouse, tableData *optimization.TableData, cfg
ColumnOp: constants.Delete,
ContainOtherOperations: tableData.ContainOtherOperations(),
CdcTime: tableData.LatestCDCTs,
UppercaseEscNames: ptr.ToBool(dwh.ShouldUppercaseEscapedNames()),
Mode: tableData.Mode(),
}

Expand Down Expand Up @@ -130,7 +128,6 @@ func Merge(dwh destination.DataWarehouse, tableData *optimization.TableData, cfg
SoftDelete: tableData.TopicConfig().SoftDelete,
DestKind: dwh.Label(),
Dialect: dwh.Dialect(),
UppercaseEscNames: ptr.ToBool(dwh.ShouldUppercaseEscapedNames()),
ContainsHardDeletes: ptr.ToBool(tableData.ContainsHardDeletes()),
}

Expand Down
1 change: 0 additions & 1 deletion clients/shared/table_config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@ func (MockDWH) PrepareTemporaryTable(tableData *optimization.TableData, tableCon
func (MockDWH) IdentifierFor(topicConfig kafkalib.TopicConfig, name string) types.TableIdentifier {
panic("not implemented")
}
func (MockDWH) ShouldUppercaseEscapedNames() bool { return true }

type MockTableIdentifier struct{ fqName string }

Expand Down
6 changes: 1 addition & 5 deletions clients/snowflake/snowflake.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,11 +78,7 @@ func (s *Store) Label() constants.DestinationKind {
}

func (s *Store) Dialect() sql.Dialect {
return sql.SnowflakeDialect{UppercaseEscNames: s.ShouldUppercaseEscapedNames()}
}

func (s *Store) ShouldUppercaseEscapedNames() bool {
return s.config.SharedDestinationConfig.UppercaseEscapedNames
return sql.SnowflakeDialect{UppercaseEscNames: s.config.SharedDestinationConfig.UppercaseEscapedNames}
}

func (s *Store) GetConfigMap() *types.DwhToTablesConfigMap {
Expand Down
16 changes: 7 additions & 9 deletions clients/snowflake/staging.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import (
"github.com/artie-labs/transfer/lib/destination/ddl"
"github.com/artie-labs/transfer/lib/destination/types"
"github.com/artie-labs/transfer/lib/optimization"
"github.com/artie-labs/transfer/lib/ptr"
"github.com/artie-labs/transfer/lib/typing"
"github.com/artie-labs/transfer/lib/typing/columns"
"github.com/artie-labs/transfer/lib/typing/values"
Expand Down Expand Up @@ -49,14 +48,13 @@ func castColValStaging(colVal any, colKind columns.Column, additionalDateFmts []
func (s *Store) PrepareTemporaryTable(tableData *optimization.TableData, tableConfig *types.DwhTableConfig, tempTableID types.TableIdentifier, additionalSettings types.AdditionalSettings, createTempTable bool) error {
if createTempTable {
tempAlterTableArgs := ddl.AlterTableArgs{
Dwh: s,
Tc: tableConfig,
TableID: tempTableID,
CreateTable: true,
TemporaryTable: true,
ColumnOp: constants.Add,
UppercaseEscNames: ptr.ToBool(s.ShouldUppercaseEscapedNames()),
Mode: tableData.Mode(),
Dwh: s,
Tc: tableConfig,
TableID: tempTableID,
CreateTable: true,
TemporaryTable: true,
ColumnOp: constants.Add,
Mode: tableData.Mode(),
}

if err := tempAlterTableArgs.AlterTable(tableData.ReadOnlyInMemoryCols().GetColumns()...); err != nil {
Expand Down
5 changes: 0 additions & 5 deletions lib/destination/ddl/ddl.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ type AlterTableArgs struct {
TableID types.TableIdentifier
CreateTable bool
TemporaryTable bool
UppercaseEscNames *bool

ColumnOp constants.ColumnOperation
Mode config.Mode
Expand All @@ -69,10 +68,6 @@ func (a AlterTableArgs) Validate() error {
}
}

if a.UppercaseEscNames == nil {
return fmt.Errorf("uppercaseEscNames cannot be nil")
}

return nil
}

Expand Down
14 changes: 0 additions & 14 deletions lib/destination/ddl/ddl_alter_delete_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@ import (

"github.com/artie-labs/transfer/lib/config"

"github.com/artie-labs/transfer/lib/ptr"

"github.com/artie-labs/transfer/lib/config/constants"
"github.com/artie-labs/transfer/lib/destination/ddl"
"github.com/artie-labs/transfer/lib/destination/types"
Expand Down Expand Up @@ -69,7 +67,6 @@ func (d *DDLTestSuite) TestAlterDelete_Complete() {
ContainOtherOperations: true,
ColumnOp: constants.Delete,
CdcTime: ts,
UppercaseEscNames: ptr.ToBool(false),
Mode: config.Replication,
}

Expand All @@ -90,7 +87,6 @@ func (d *DDLTestSuite) TestAlterDelete_Complete() {
ContainOtherOperations: true,
ColumnOp: constants.Delete,
CdcTime: ts,
UppercaseEscNames: ptr.ToBool(false),
Mode: config.Replication,
}

Expand All @@ -112,7 +108,6 @@ func (d *DDLTestSuite) TestAlterDelete_Complete() {
ContainOtherOperations: true,
ColumnOp: constants.Delete,
CdcTime: ts,
UppercaseEscNames: ptr.ToBool(false),
Mode: config.Replication,
}

Expand Down Expand Up @@ -147,7 +142,6 @@ func (d *DDLTestSuite) TestAlterDelete_Complete() {
ContainOtherOperations: false,
ColumnOp: constants.Delete,
CdcTime: ts,
UppercaseEscNames: ptr.ToBool(false),
Mode: config.Replication,
}

Expand All @@ -168,7 +162,6 @@ func (d *DDLTestSuite) TestAlterDelete_Complete() {
ContainOtherOperations: false,
ColumnOp: constants.Delete,
CdcTime: ts,
UppercaseEscNames: ptr.ToBool(false),
Mode: config.Replication,
}

Expand All @@ -189,7 +182,6 @@ func (d *DDLTestSuite) TestAlterDelete_Complete() {
ContainOtherOperations: false,
ColumnOp: constants.Delete,
CdcTime: ts,
UppercaseEscNames: ptr.ToBool(false),
Mode: config.Replication,
}

Expand Down Expand Up @@ -226,7 +218,6 @@ func (d *DDLTestSuite) TestAlterDelete_Complete() {
ContainOtherOperations: true,
ColumnOp: constants.Delete,
CdcTime: ts,
UppercaseEscNames: ptr.ToBool(false),
Mode: config.Replication,
}

Expand All @@ -243,7 +234,6 @@ func (d *DDLTestSuite) TestAlterDelete_Complete() {
ContainOtherOperations: true,
ColumnOp: constants.Delete,
CdcTime: ts,
UppercaseEscNames: ptr.ToBool(false),
Mode: config.Replication,
}

Expand All @@ -260,7 +250,6 @@ func (d *DDLTestSuite) TestAlterDelete_Complete() {
ContainOtherOperations: true,
ColumnOp: constants.Delete,
CdcTime: ts,
UppercaseEscNames: ptr.ToBool(false),
Mode: config.Replication,
}

Expand All @@ -285,7 +274,6 @@ func (d *DDLTestSuite) TestAlterDelete_Complete() {
ContainOtherOperations: true,
ColumnOp: constants.Delete,
CdcTime: ts.Add(2 * constants.DeletionConfidencePadding),
UppercaseEscNames: ptr.ToBool(true),
Mode: config.Replication,
}

Expand All @@ -300,7 +288,6 @@ func (d *DDLTestSuite) TestAlterDelete_Complete() {
ContainOtherOperations: true,
ColumnOp: constants.Delete,
CdcTime: ts.Add(2 * constants.DeletionConfidencePadding),
UppercaseEscNames: ptr.ToBool(false),
Mode: config.Replication,
}

Expand All @@ -315,7 +302,6 @@ func (d *DDLTestSuite) TestAlterDelete_Complete() {
ContainOtherOperations: true,
ColumnOp: constants.Delete,
CdcTime: ts.Add(2 * constants.DeletionConfidencePadding),
UppercaseEscNames: ptr.ToBool(false),
Mode: config.Replication,
}

Expand Down
Loading

0 comments on commit a9f4e5b

Please sign in to comment.