Skip to content

Commit

Permalink
Databricks sweep.
Browse files Browse the repository at this point in the history
  • Loading branch information
Tang8330 committed Oct 4, 2024
1 parent c30be97 commit 33970d3
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 2 deletions.
10 changes: 10 additions & 0 deletions clients/databricks/dialect/dialect.go
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,16 @@ WHEN NOT MATCHED AND IFNULL(%s, false) = false THEN INSERT (%s) VALUES (%s);`,
)}, nil
}

func (DatabricksDialect) BuildSweepQuery(dbName, schemaName string) (string, []any) {
return fmt.Sprintf(`
SELECT
table_schema, table_name
FROM
%s.information_schema.tables
WHERE
UPPER(table_schema) = UPPER(?) AND table_name ILIKE ?`, dbName), []any{schemaName, "%" + constants.ArtiePrefix + "%"}
}

func (d DatabricksDialect) GetDefaultValueStrategy() sql.DefaultValueStrategy {
return sql.Native
}
12 changes: 10 additions & 2 deletions clients/databricks/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,10 @@ func (s Store) IdentifierFor(topicConfig kafkalib.TopicConfig, table string) sql
}

func (s Store) Dialect() sql.Dialect {
return s.dialect()
}

func (s Store) dialect() dialect.DatabricksDialect {
return dialect.DatabricksDialect{}
}

Expand Down Expand Up @@ -185,8 +189,12 @@ func (s Store) writeTemporaryTableFile(tableData *optimization.TableData, newTab
}

func (s Store) SweepTemporaryTables() error {
// TODO
return nil
tcs, err := s.cfg.TopicConfigs()
if err != nil {
return err
}

return shared.Sweep(s, tcs, s.dialect().BuildSweepQuery)
}

func LoadStore(cfg config.Config) (Store, error) {
Expand Down

0 comments on commit 33970d3

Please sign in to comment.