Merge pull request #807 from signal18/jobs

Add option to keep old backup temporary before valid
signal18 · Aug 9, 2024 · 08cd3a8 · 08cd3a8
2 parents 335039a + 31f5ded
commit 08cd3a8
Show file tree

Hide file tree

Showing 9 changed files with 192 additions and 12 deletions.
diff --git a/cluster/cluster_tgl.go b/cluster/cluster_tgl.go
@@ -619,3 +619,7 @@ func (cluster *Cluster) SwitchForceWriteConfig() {
 		cluster.LogModulePrintf(cluster.Conf.Verbose, config.ConstLogModGeneral, config.LvlInfo, "Configurator force write config files de-activated. Will create config files with suffix (.new) for conflicting files on next provision.")
 	}
 }
+
+func (cluster *Cluster) SwitchBackupKeepUntilValid() {
+	cluster.Conf.BackupKeepUntilValid = !cluster.Conf.BackupKeepUntilValid
+}
diff --git a/cluster/srv_bck.go b/cluster/srv_bck.go
@@ -92,3 +92,40 @@ func (server *ServerMonitor) ReadLastMetadata(method string) (*config.BackupMeta
 
 	return meta, nil
 }
+
+func (server *ServerMonitor) GetLatestMeta(method string) (int64, *config.BackupMetadata) {
+	cluster := server.ClusterGroup
+	var latest int64 = 0
+	var meta *config.BackupMetadata
+	cluster.BackupMetaMap.Range(func(k, v any) bool {
+		m := v.(*config.BackupMetadata)
+		valid := false
+		switch method {
+		case "logical":
+			if m.BackupMethod == config.BackupMethodLogical {
+				valid = true
+			}
+		case "physical":
+			if m.BackupMethod == config.BackupMethodPhysical {
+				valid = true
+			}
+		default:
+			if m.BackupTool == method {
+				valid = true
+			}
+		}
+
+		if m.Source != server.URL {
+			valid = false
+		}
+
+		if valid && latest < m.Id {
+			latest = m.Id
+			meta = m
+		}
+
+		return true
+	})
+
+	return latest, meta
+}
diff --git a/cluster/srv_job.go b/cluster/srv_job.go
@@ -311,6 +311,17 @@ func (server *ServerMonitor) JobBackupPhysical() (int64, error) {
 
 	now := time.Now()
 	// Reset last backup meta
+	var prevId int64
+	prev := cluster.BackupMetaMap.GetPreviousBackup(cluster.Conf.BackupPhysicalType, server.URL)
+	if prev != nil {
+		prevId = prev.Id
+	}
+
+	// Remove from backup list, since the file will be replaced
+	if !cluster.Conf.BackupKeepUntilValid {
+		cluster.BackupMetaMap.Delete(prevId)
+	}
+
 	server.LastBackupMeta.Physical = &config.BackupMetadata{
 		Id:             now.Unix(),
 		StartTime:      now,
@@ -320,6 +331,7 @@ func (server *ServerMonitor) JobBackupPhysical() (int64, error) {
 		Source:         server.URL,
 		Dest:           dest,
 		Compressed:     cluster.Conf.CompressBackups,
+		Previous:       prevId,
 	}
 
 	cluster.BackupMetaMap.Set(server.LastBackupMeta.Physical.Id, server.LastBackupMeta.Physical)
@@ -1888,15 +1900,29 @@ func (server *ServerMonitor) JobBackupLogical() error {
 
 	cluster.SetInLogicalBackupState(true)
 	start := time.Now()
+	var prevId int64
+	prev := cluster.BackupMetaMap.GetPreviousBackup(cluster.Conf.BackupLogicalType, server.URL)
+	if prev != nil {
+		prevId = prev.Id
+	}
+
+	// Remove from backup list, since the file will be replaced
+	if !cluster.Conf.BackupKeepUntilValid {
+		cluster.BackupMetaMap.Delete(prevId)
+	}
+
 	server.LastBackupMeta.Logical = &config.BackupMetadata{
 		Id:             start.Unix(),
 		StartTime:      start,
 		BackupMethod:   config.BackupMethodLogical,
 		BackupTool:     cluster.Conf.BackupLogicalType,
 		BackupStrategy: config.BackupStrategyFull,
 		Source:         server.URL,
+		Previous:       prevId,
 	}
 
+	cluster.BackupMetaMap.Set(server.LastBackupMeta.Logical.Id, server.LastBackupMeta.Logical)
+
 	// Removing previous valid backup state and start
 	server.DelBackupLogicalCookie()
 
@@ -1934,10 +1960,10 @@ func (server *ServerMonitor) JobBackupLogical() error {
 				if e2 := server.JobsUpdateState(task, "Backup completed", 3, 1); e2 != nil {
 					cluster.LogModulePrintf(cluster.Conf.Verbose, config.ConstLogModTask, config.LvlWarn, "Task only updated in runtime. Error while writing to jobs table: %s", e2.Error())
 				}
-				finfo, e3 := os.Stat(filename)
+				_, e3 := os.Stat(filename)
 				if e3 == nil {
 					server.LastBackupMeta.Logical.EndTime = time.Now()
-					server.LastBackupMeta.Logical.Size = finfo.Size()
+					server.LastBackupMeta.Logical.GetSize()
 					server.LastBackupMeta.Logical.Completed = true
 					server.SetBackupLogicalCookie(config.ConstBackupLogicalTypeMysqldump)
 				}
@@ -1959,10 +1985,10 @@ func (server *ServerMonitor) JobBackupLogical() error {
 				if e2 := server.JobsUpdateState(task, "Backup completed", 3, 1); e2 != nil {
 					cluster.LogModulePrintf(cluster.Conf.Verbose, config.ConstLogModTask, config.LvlWarn, "Task only updated in runtime. Error while writing to jobs table: %s", e2.Error())
 				}
-				finfo, e3 := os.Stat(outputdir)
+				_, e3 := os.Stat(outputdir)
 				if e3 == nil {
 					server.LastBackupMeta.Logical.EndTime = time.Now()
-					server.LastBackupMeta.Logical.Size = finfo.Size()
+					server.LastBackupMeta.Logical.GetSize()
 					server.LastBackupMeta.Logical.Completed = true
 					server.SetBackupLogicalCookie(config.ConstBackupLogicalTypeDumpling)
 				}
@@ -1985,10 +2011,10 @@ func (server *ServerMonitor) JobBackupLogical() error {
 					cluster.LogModulePrintf(cluster.Conf.Verbose, config.ConstLogModTask, config.LvlWarn, "Task only updated in runtime. Error while writing to jobs table: %s", e2.Error())
 				}
 
-				finfo, e3 := os.Stat(outputdir)
+				_, e3 := os.Stat(outputdir)
 				if e3 == nil {
 					server.LastBackupMeta.Logical.EndTime = time.Now()
-					server.LastBackupMeta.Logical.Size = finfo.Size()
+					server.LastBackupMeta.Logical.GetSize()
 					server.LastBackupMeta.Logical.Completed = true
 					server.SetBackupLogicalCookie(config.ConstBackupLogicalTypeDumpling)
 				}
@@ -2834,8 +2860,8 @@ func (server *ServerMonitor) WriteBackupMetadata(backtype config.BackupMethod) {
 		return
 	}
 
-	if finfo, err := os.Stat(lastmeta.Dest); err == nil {
-		lastmeta.Size = finfo.Size()
+	if _, err := os.Stat(lastmeta.Dest); err == nil {
+		lastmeta.GetSize()
 		lastmeta.EndTime = time.Now()
 	}
 
@@ -2875,13 +2901,24 @@ func (server *ServerMonitor) WriteBackupMetadata(backtype config.BackupMethod) {
 	//Don't change river
 	if cluster.Conf.BackupKeepUntilValid && lastmeta.BackupTool != config.ConstBackupLogicalTypeRiver {
 		if lastmeta.Completed {
+			// Delete previous meta with same type
+			cluster.BackupMetaMap.Delete(lastmeta.Previous)
 			cluster.LogModulePrintf(cluster.Conf.Verbose, config.ConstLogModTask, config.LvlInfo, "Backup valid, removing old backup.")
 			exec.Command("rm", "-r", lastmeta.Dest+".old").Run()
 		} else {
 			cluster.LogModulePrintf(cluster.Conf.Verbose, config.ConstLogModTask, config.LvlInfo, "Error occured in backup, rolling back to old backup.")
 			exec.Command("mv", lastmeta.Dest, lastmeta.Dest+".err").Run()
 			exec.Command("mv", lastmeta.Dest+".old", lastmeta.Dest).Run()
 			exec.Command("rm", "-r", lastmeta.Dest+".err").Run()
+
+			// Revert to previous meta with same type
+			cluster.BackupMetaMap.Delete(lastmeta.Id)
+			switch backtype {
+			case config.BackupMethodLogical:
+				_, server.LastBackupMeta.Logical = server.GetLatestMeta("logical")
+			case config.BackupMethodPhysical:
+				_, server.LastBackupMeta.Physical = server.GetLatestMeta("physical")
+			}
 		}
 	}
 }

diff --git a/config/backup.go b/config/backup.go
@@ -1,6 +1,10 @@
 package config
 
-import "time"
+import (
+	"os"
+	"path/filepath"
+	"time"
+)
 
 type BackupMethod int
 
@@ -37,4 +41,17 @@ type BackupMetadata struct {
 	BinLogFilePos  uint64         `json:"binLogFilePos"`
 	BinLogGtid     string         `json:"binLogUuid"`
 	Completed      bool           `json:"completed"`
+	Previous       int64          `json:"previous"`
+}
+
+func (bm *BackupMetadata) GetSize() error {
+	var size int64 = 0
+	err := filepath.Walk(bm.Dest, func(_ string, info os.FileInfo, err error) error {
+		if err == nil && !info.IsDir() {
+			size += info.Size()
+		}
+		return err
+	})
+	bm.Size = size
+	return err
 }
diff --git a/config/maps.go b/config/maps.go
@@ -880,3 +880,18 @@ func FromBackupMetaMap(m *BackupMetaMap, c *BackupMetaMap) *BackupMetaMap {
 
 	return m
 }
+
+// GetBackupsByToolAndSource retrieves backups with the same backupTool and source.
+func (b *BackupMetaMap) GetPreviousBackup(backupTool string, source string) *BackupMetadata {
+	var result *BackupMetadata
+	b.Map.Range(func(key, value interface{}) bool {
+		if backup, ok := value.(*BackupMetadata); ok {
+			if backup.BackupTool == backupTool && backup.Source == source {
+				result = backup
+				return false
+			}
+		}
+		return true
+	})
+	return result
+}
diff --git a/server/api_cluster.go b/server/api_cluster.go
@@ -1256,6 +1256,8 @@ func (repman *ReplicationManager) switchSettings(mycluster *cluster.Cluster, set
 		mycluster.SwitchReplicationNoRelay()
 	case "prov-db-force-write-config":
 		mycluster.SwitchForceWriteConfig()
+	case "backup-keep-until-valid":
+		mycluster.SwitchBackupKeepUntilValid()
 	}
 }
 

diff --git a/share/dashboard/app/dashboard.js b/share/dashboard/app/dashboard.js
@@ -375,6 +375,23 @@ app.controller('DashboardController', function (
     return t.state === 0 || (t.start < Math.floor((Date.now() - 300000) / 1000) && !t.end && t.state < 3)
   }
 
+  $scope.getBackupMethod = function(method) {
+    switch(method) {
+        case 1: return 'Logical';
+        case 2: return 'Physical';
+        default: return 'Unknown';
+    }
+};
+
+$scope.getBackupStrategy = function(strategy) {
+    switch(strategy) {
+        case 1: return 'Full';
+        case 2: return 'Incremental';
+        case 3: return 'Differential';
+        default: return 'Unknown';
+    }
+};
+
   $scope.SetApiTokenTimeout = function (val) {
     if ($scope.roApiTokenTimeout) {
       $scope.selectedApiTokenTimeout = Number(val)

diff --git a/share/dashboard/static/card-setting-backup.html b/share/dashboard/static/card-setting-backup.html
@@ -46,6 +46,20 @@
                 </span>
             </td>
         </tr>
+        <tr>
+            <th colspan=4>Backup keep previous until valid</th>
+        </tr>
+        <tr>
+            <td colspan=4>
+                <md-switch ng-disabled="selectedCluster.apiUsers[user].grants['cluster-settings']==false"
+                    ng-true-value="true" ng-false-value="false" ng-model="selectedCluster.config.backupKeepUntilValid"
+                    ng-click="switchsettings('backup-keep-until-valid')" aria-label="Keep previous backup until next backup is valid">
+                    <span ng-if="selectedCluster.config.backupKeepUntilValid" class="label label-primary">On</span><span
+                        ng-if="!selectedCluster.config.backupKeepUntilValid" class="label label-warning">Off</span>
+
+                </md-switch>
+            </td>
+        </tr>
     </table>
 </md-card>
 <md-card ng-if="selectedCluster">

diff --git a/share/dashboard/static/tab-cluster-backups.html b/share/dashboard/static/tab-cluster-backups.html
@@ -1,9 +1,45 @@
 <md-content class="md-padding">
    <md-card>
       <md-card-content>
-         <p class="sectionheader">Backups</p>
+         <p class="sectionheader">Current Backups</p>
+      </md-card-content>
+      <md-card>
+         <table ng-if="selectedCluster.backupList" class="table">
+            <thead>
+               <tr>
+                  <th>Id</th>
+                  <th>Start Time</th>
+                  <th>End Time</th>
+                  <th>Backup Method</th>
+                  <th>Backup Strategy</th>
+                  <th>Source</th>
+                  <th>Destination</th>
+                  <th>Size</th>
+                  <th>Compressed</th>
+                  <th>Encrypted</th>
+                  <th>Completed</th>
+               </tr>
+            </thead>
+            <tbody>
+               <tr ng-repeat="backup in selectedCluster.backupList">
+                  <td>{{backup.id}}</td>
+                  <td>{{backup.startTime | date:'yyyy-MM-dd HH:mm:ss'}}</td>
+                  <td>{{backup.endTime | date:'yyyy-MM-dd HH:mm:ss'}}</td>
+                  <td>{{getBackupMethod(backup.backupMethod)}}</td>
+                  <td>{{getBackupStrategy(backup.backupStrategy)}}</td>
+                  <td>{{backup.source}}</td>
+                  <td>{{backup.dest}}</td>
+                  <td>{{formatBytes(backup.size)}}</td>
+                  <td>{{backup.compressed ? 'Yes' : 'No'}}</td>
+                  <td>{{backup.encrypted ? 'Yes' : 'No'}}</td>
+                  <td>{{backup.completed ? 'Yes' : 'No'}}</td>
+               </tr>
+            </tbody>
+         </table>
+      </md-card>
+      <md-card-content>
+         <p class="sectionheader">Backup History</p>
       </md-card-content>
-
       <md-card>
          <table ng-if="backups" class="table">
             <thead>
@@ -22,7 +58,6 @@
             </tbody>
          </table>
       </md-card>
-
       <md-card>
          <table ng-if="backups" class="table">
             <thead>
@@ -45,5 +80,7 @@
             </tbody>
          </table>
       </md-card>
+
+
    </md-card>
 </md-content>