From a5a6e87097190b969865222873ee2c4f8c7e1458 Mon Sep 17 00:00:00 2001 From: Ryan Doherty Date: Mon, 29 Apr 2024 23:54:10 -0400 Subject: [PATCH] Add configuration to back up tables before db migrations --- .../model/fix/StepParamExpanderPlugin.java | 7 ++++ .../model/fix/table/TableRowInterfaces.java | 19 +++++++++++ .../wdk/model/fix/table/TableRowUpdater.java | 33 +++++++++++++++++++ .../edaanalysis/AnalysisRecordFactory.java | 6 ++++ .../edaanalysis/plugins/NoOpBackupPlugin.java | 23 +++++++++++++ .../model/fix/table/steps/StepDataWriter.java | 5 +++ 6 files changed, 93 insertions(+) create mode 100644 Model/src/main/java/org/gusdb/wdk/model/fix/table/edaanalysis/plugins/NoOpBackupPlugin.java diff --git a/Model/src/main/java/org/gusdb/wdk/model/fix/StepParamExpanderPlugin.java b/Model/src/main/java/org/gusdb/wdk/model/fix/StepParamExpanderPlugin.java index 967f4ce36e..ec17ff7997 100644 --- a/Model/src/main/java/org/gusdb/wdk/model/fix/StepParamExpanderPlugin.java +++ b/Model/src/main/java/org/gusdb/wdk/model/fix/StepParamExpanderPlugin.java @@ -3,6 +3,7 @@ import java.sql.Types; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -107,6 +108,12 @@ public Collection toValues(StepData obj) { } return values; } + + @Override + public List getTableNamesForBackup(String schema) { + // no backup needed; creating a new table in this writer + return Collections.emptyList(); + } } private AtomicInteger _numParams = new AtomicInteger(0); diff --git a/Model/src/main/java/org/gusdb/wdk/model/fix/table/TableRowInterfaces.java b/Model/src/main/java/org/gusdb/wdk/model/fix/table/TableRowInterfaces.java index 642406ba69..414bcd29d4 100644 --- a/Model/src/main/java/org/gusdb/wdk/model/fix/table/TableRowInterfaces.java +++ b/Model/src/main/java/org/gusdb/wdk/model/fix/table/TableRowInterfaces.java @@ -64,6 +64,15 @@ public interface TableRowUpdaterPlugin { * when all processing is complete to allow the plugin to dump those statistics. */ void dumpStatistics(); + + /** + * Whether to back up tables that may be written to before processing begins + * + * @return true if backup should be performed + */ + default boolean isPerformTableBackup() { + return false; + } } /** @@ -125,6 +134,16 @@ public interface TableRowWriter { */ void tearDown(WdkModel wdkModel) throws Exception; + /** + * Returns a list of table names (including schema prefix) whose rows may be overwritten + * by this writer. Plugin will decide whether backup is actually performed. The + * schema value passed in will be empty or will have a '.' as the last character. It + * is the user schema configured in the WDK model created for this update + * + * @return list of tables + */ + List getTableNamesForBackup(String schema); + /** * Provides parameterized SQL to write a single record. This SQL will be converted to a * PreparedStatement, then executed as a batch update operation using the SQL types and values diff --git a/Model/src/main/java/org/gusdb/wdk/model/fix/table/TableRowUpdater.java b/Model/src/main/java/org/gusdb/wdk/model/fix/table/TableRowUpdater.java index 36712cf9b7..aee4046a37 100644 --- a/Model/src/main/java/org/gusdb/wdk/model/fix/table/TableRowUpdater.java +++ b/Model/src/main/java/org/gusdb/wdk/model/fix/table/TableRowUpdater.java @@ -8,9 +8,13 @@ import java.lang.reflect.InvocationTargetException; import java.sql.ResultSet; import java.sql.SQLException; +import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; import java.util.Iterator; import java.util.List; +import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.ExecutionException; @@ -18,6 +22,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; import javax.sql.DataSource; @@ -30,6 +35,7 @@ import org.gusdb.fgputil.db.runner.SQLRunner; import org.gusdb.fgputil.db.runner.SQLRunner.ArgumentBatch; import org.gusdb.fgputil.db.runner.SQLRunner.ResultSetHandler; +import org.gusdb.fgputil.db.runner.SQLRunnerException; import org.gusdb.fgputil.runtime.GusHome; import org.gusdb.wdk.model.WdkModel; import org.gusdb.wdk.model.fix.table.TableRowInterfaces.RowResult; @@ -217,6 +223,18 @@ private ExitStatus run() { DatabaseInstance userDb = _wdkModel.getUserDb(); RecordQueue recordQueue = new RecordQueue<>(); + // back up tables if requested + if (_plugin.isPerformTableBackup()) { + // collect table names from writers and remove duplicates + String userSchema = getUserSchema(_wdkModel); + Set tables = _writers.stream() + .map(writer -> writer.getTableNamesForBackup(userSchema)) + .flatMap(Collection::stream) + .collect(Collectors.toSet()); + // create backup tables + backUpTables(userDb.getDataSource(), tables); + } + // create and start threads that will listen to queue and pull off records to process for (int i = 0; i < NUM_THREADS; i++) { RowHandler thread = new RowHandler<>(i + 1, recordQueue, _plugin, _writers, _wdkModel); @@ -275,6 +293,21 @@ private ExitStatus run() { return ExitStatus.SUCCESS; } + private void backUpTables(DataSource ds, Set tables) { + try { + String timestamp = new SimpleDateFormat("yyyyMMddHHmmssSSS").format(new Date()); + for (String table : tables) { + String bkTable = table + "_bk_" + timestamp; + String sql = "create table " + bkTable + " as (select * from " + table + ")"; + LOG.info("Creating backup table " + bkTable + " from " + table); + new SQLRunner(ds, sql, "table-backup").executeStatement(); + } + } + catch (SQLRunnerException e) { + throw new RuntimeException("Could not create backup table", e.getCause()); + } + } + private void setUpDatabase() throws Exception { _factory.setUp(_wdkModel); for (TableRowWriter writer : _writers) { diff --git a/Model/src/main/java/org/gusdb/wdk/model/fix/table/edaanalysis/AnalysisRecordFactory.java b/Model/src/main/java/org/gusdb/wdk/model/fix/table/edaanalysis/AnalysisRecordFactory.java index 8d1d847891..61b1dbc135 100644 --- a/Model/src/main/java/org/gusdb/wdk/model/fix/table/edaanalysis/AnalysisRecordFactory.java +++ b/Model/src/main/java/org/gusdb/wdk/model/fix/table/edaanalysis/AnalysisRecordFactory.java @@ -4,6 +4,7 @@ import java.sql.SQLException; import java.sql.Types; import java.util.Collection; +import java.util.List; import org.gusdb.fgputil.ListBuilder; import org.gusdb.fgputil.db.platform.DBPlatform; @@ -37,6 +38,11 @@ public AnalysisRow newTableRow(ResultSet rs, DBPlatform platform) throws SQLExce return new AnalysisRow(rs, platform); } + @Override + public List getTableNamesForBackup(String schema) { + return List.of(_schema + ".analysis"); + } + @Override public String getWriteSql(String schema) { return diff --git a/Model/src/main/java/org/gusdb/wdk/model/fix/table/edaanalysis/plugins/NoOpBackupPlugin.java b/Model/src/main/java/org/gusdb/wdk/model/fix/table/edaanalysis/plugins/NoOpBackupPlugin.java new file mode 100644 index 0000000000..fde6d296bc --- /dev/null +++ b/Model/src/main/java/org/gusdb/wdk/model/fix/table/edaanalysis/plugins/NoOpBackupPlugin.java @@ -0,0 +1,23 @@ +package org.gusdb.wdk.model.fix.table.edaanalysis.plugins; + +import org.gusdb.wdk.model.fix.table.TableRowInterfaces.RowResult; +import org.gusdb.wdk.model.fix.table.edaanalysis.AbstractAnalysisUpdater; +import org.gusdb.wdk.model.fix.table.edaanalysis.AnalysisRow; + +public class NoOpBackupPlugin extends AbstractAnalysisUpdater { + + @Override + public RowResult processRecord(AnalysisRow nextRow) throws Exception { + return new RowResult<>(nextRow); + } + + @Override + public void dumpStatistics() { + // nothing to do + } + + @Override + public boolean isPerformTableBackup() { + return true; + } +} diff --git a/Model/src/main/java/org/gusdb/wdk/model/fix/table/steps/StepDataWriter.java b/Model/src/main/java/org/gusdb/wdk/model/fix/table/steps/StepDataWriter.java index 4e0c07a307..39f77bb5a6 100644 --- a/Model/src/main/java/org/gusdb/wdk/model/fix/table/steps/StepDataWriter.java +++ b/Model/src/main/java/org/gusdb/wdk/model/fix/table/steps/StepDataWriter.java @@ -30,6 +30,11 @@ public class StepDataWriter implements TableRowWriter { mapToList(new ListBuilder().addAll(UPDATE_COLS).add(STEP_ID).toList(), key -> SQLTYPES.get(key)).toArray(new Integer[COLS.length]); + @Override + public List getTableNamesForBackup(String schema) { + return List.of(schema + "steps"); + } + @Override public String getWriteSql(String schema) { return "update " + schema + "steps set " + UPDATE_COLS_TEXT + " where " + STEP_ID + " = ?";