From f805c45ce8799e63fa0d925f2633dd85eb8166fe Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Wed, 2 Oct 2024 09:19:41 -0700 Subject: [PATCH] Clean up. --- clients/databricks/store.go | 10 +++++++--- lib/config/destination_types.go | 1 - 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/clients/databricks/store.go b/clients/databricks/store.go index e9afd5346..3cf89d6f2 100644 --- a/clients/databricks/store.go +++ b/clients/databricks/store.go @@ -27,7 +27,6 @@ import ( type Store struct { db.Store - volume string cfg config.Config configMap *types.DwhToTablesConfigMap } @@ -104,7 +103,13 @@ func (s Store) PrepareTemporaryTable(tableData *optimization.TableData, tableCon // Upload the local file to DBFS ctx := driverctx.NewContextWithStagingInfo(context.Background(), []string{"/var"}) - dbfsFilePath := fmt.Sprintf("dbfs:/Volumes/%s/%s.csv", s.volume, tempTableID.Table()) + + castedTempTableID, isOk := tempTableID.(TableIdentifier) + if !isOk { + return fmt.Errorf("failed to cast temp table ID to TableIdentifier") + } + + dbfsFilePath := fmt.Sprintf("dbfs:/Volumes/%s/%s.csv", castedTempTableID.Database(), castedTempTableID.Table()) putCommand := fmt.Sprintf("PUT '%s' INTO '%s' OVERWRITE", fp, dbfsFilePath) if _, err = s.ExecContext(ctx, putCommand); err != nil { return fmt.Errorf("failed to run PUT INTO for temporary table: %w", err) @@ -173,7 +178,6 @@ func LoadStore(cfg config.Config) (Store, error) { return Store{ Store: store, cfg: cfg, - volume: cfg.Databricks.Volume, configMap: &types.DwhToTablesConfigMap{}, }, nil } diff --git a/lib/config/destination_types.go b/lib/config/destination_types.go index 0ff0b3dbe..5c3003951 100644 --- a/lib/config/destination_types.go +++ b/lib/config/destination_types.go @@ -17,7 +17,6 @@ type Databricks struct { Port int `yaml:"port"` Catalog string `yaml:"catalog"` PersonalAccessToken string `yaml:"personalAccessToken"` - Volume string `yaml:"volume"` } type MSSQL struct {