diff --git a/cluster/cluster_tgl.go b/cluster/cluster_tgl.go index ab14d1a6d..e18dc9dc5 100644 --- a/cluster/cluster_tgl.go +++ b/cluster/cluster_tgl.go @@ -619,3 +619,7 @@ func (cluster *Cluster) SwitchForceWriteConfig() { cluster.LogModulePrintf(cluster.Conf.Verbose, config.ConstLogModGeneral, config.LvlInfo, "Configurator force write config files de-activated. Will create config files with suffix (.new) for conflicting files on next provision.") } } + +func (cluster *Cluster) SwitchBackupKeepUntilValid() { + cluster.Conf.BackupKeepUntilValid = !cluster.Conf.BackupKeepUntilValid +} diff --git a/cluster/srv_bck.go b/cluster/srv_bck.go index 3107b157d..c8abf326b 100644 --- a/cluster/srv_bck.go +++ b/cluster/srv_bck.go @@ -92,3 +92,40 @@ func (server *ServerMonitor) ReadLastMetadata(method string) (*config.BackupMeta return meta, nil } + +func (server *ServerMonitor) GetLatestMeta(method string) (int64, *config.BackupMetadata) { + cluster := server.ClusterGroup + var latest int64 = 0 + var meta *config.BackupMetadata + cluster.BackupMetaMap.Range(func(k, v any) bool { + m := v.(*config.BackupMetadata) + valid := false + switch method { + case "logical": + if m.BackupMethod == config.BackupMethodLogical { + valid = true + } + case "physical": + if m.BackupMethod == config.BackupMethodPhysical { + valid = true + } + default: + if m.BackupTool == method { + valid = true + } + } + + if m.Source != server.URL { + valid = false + } + + if valid && latest < m.Id { + latest = m.Id + meta = m + } + + return true + }) + + return latest, meta +} diff --git a/cluster/srv_job.go b/cluster/srv_job.go index 62af123e6..3abb975d9 100644 --- a/cluster/srv_job.go +++ b/cluster/srv_job.go @@ -311,6 +311,17 @@ func (server *ServerMonitor) JobBackupPhysical() (int64, error) { now := time.Now() // Reset last backup meta + var prevId int64 + prev := cluster.BackupMetaMap.GetPreviousBackup(cluster.Conf.BackupPhysicalType, server.URL) + if prev != nil { + prevId = prev.Id + } + + // Remove from backup list, since the file will be replaced + if !cluster.Conf.BackupKeepUntilValid { + cluster.BackupMetaMap.Delete(prevId) + } + server.LastBackupMeta.Physical = &config.BackupMetadata{ Id: now.Unix(), StartTime: now, @@ -320,6 +331,7 @@ func (server *ServerMonitor) JobBackupPhysical() (int64, error) { Source: server.URL, Dest: dest, Compressed: cluster.Conf.CompressBackups, + Previous: prevId, } cluster.BackupMetaMap.Set(server.LastBackupMeta.Physical.Id, server.LastBackupMeta.Physical) @@ -1888,6 +1900,17 @@ func (server *ServerMonitor) JobBackupLogical() error { cluster.SetInLogicalBackupState(true) start := time.Now() + var prevId int64 + prev := cluster.BackupMetaMap.GetPreviousBackup(cluster.Conf.BackupLogicalType, server.URL) + if prev != nil { + prevId = prev.Id + } + + // Remove from backup list, since the file will be replaced + if !cluster.Conf.BackupKeepUntilValid { + cluster.BackupMetaMap.Delete(prevId) + } + server.LastBackupMeta.Logical = &config.BackupMetadata{ Id: start.Unix(), StartTime: start, @@ -1895,8 +1918,11 @@ func (server *ServerMonitor) JobBackupLogical() error { BackupTool: cluster.Conf.BackupLogicalType, BackupStrategy: config.BackupStrategyFull, Source: server.URL, + Previous: prevId, } + cluster.BackupMetaMap.Set(server.LastBackupMeta.Logical.Id, server.LastBackupMeta.Logical) + // Removing previous valid backup state and start server.DelBackupLogicalCookie() @@ -1934,10 +1960,10 @@ func (server *ServerMonitor) JobBackupLogical() error { if e2 := server.JobsUpdateState(task, "Backup completed", 3, 1); e2 != nil { cluster.LogModulePrintf(cluster.Conf.Verbose, config.ConstLogModTask, config.LvlWarn, "Task only updated in runtime. Error while writing to jobs table: %s", e2.Error()) } - finfo, e3 := os.Stat(filename) + _, e3 := os.Stat(filename) if e3 == nil { server.LastBackupMeta.Logical.EndTime = time.Now() - server.LastBackupMeta.Logical.Size = finfo.Size() + server.LastBackupMeta.Logical.GetSize() server.LastBackupMeta.Logical.Completed = true server.SetBackupLogicalCookie(config.ConstBackupLogicalTypeMysqldump) } @@ -1959,10 +1985,10 @@ func (server *ServerMonitor) JobBackupLogical() error { if e2 := server.JobsUpdateState(task, "Backup completed", 3, 1); e2 != nil { cluster.LogModulePrintf(cluster.Conf.Verbose, config.ConstLogModTask, config.LvlWarn, "Task only updated in runtime. Error while writing to jobs table: %s", e2.Error()) } - finfo, e3 := os.Stat(outputdir) + _, e3 := os.Stat(outputdir) if e3 == nil { server.LastBackupMeta.Logical.EndTime = time.Now() - server.LastBackupMeta.Logical.Size = finfo.Size() + server.LastBackupMeta.Logical.GetSize() server.LastBackupMeta.Logical.Completed = true server.SetBackupLogicalCookie(config.ConstBackupLogicalTypeDumpling) } @@ -1985,10 +2011,10 @@ func (server *ServerMonitor) JobBackupLogical() error { cluster.LogModulePrintf(cluster.Conf.Verbose, config.ConstLogModTask, config.LvlWarn, "Task only updated in runtime. Error while writing to jobs table: %s", e2.Error()) } - finfo, e3 := os.Stat(outputdir) + _, e3 := os.Stat(outputdir) if e3 == nil { server.LastBackupMeta.Logical.EndTime = time.Now() - server.LastBackupMeta.Logical.Size = finfo.Size() + server.LastBackupMeta.Logical.GetSize() server.LastBackupMeta.Logical.Completed = true server.SetBackupLogicalCookie(config.ConstBackupLogicalTypeDumpling) } @@ -2834,8 +2860,8 @@ func (server *ServerMonitor) WriteBackupMetadata(backtype config.BackupMethod) { return } - if finfo, err := os.Stat(lastmeta.Dest); err == nil { - lastmeta.Size = finfo.Size() + if _, err := os.Stat(lastmeta.Dest); err == nil { + lastmeta.GetSize() lastmeta.EndTime = time.Now() } @@ -2875,6 +2901,8 @@ func (server *ServerMonitor) WriteBackupMetadata(backtype config.BackupMethod) { //Don't change river if cluster.Conf.BackupKeepUntilValid && lastmeta.BackupTool != config.ConstBackupLogicalTypeRiver { if lastmeta.Completed { + // Delete previous meta with same type + cluster.BackupMetaMap.Delete(lastmeta.Previous) cluster.LogModulePrintf(cluster.Conf.Verbose, config.ConstLogModTask, config.LvlInfo, "Backup valid, removing old backup.") exec.Command("rm", "-r", lastmeta.Dest+".old").Run() } else { @@ -2882,6 +2910,15 @@ func (server *ServerMonitor) WriteBackupMetadata(backtype config.BackupMethod) { exec.Command("mv", lastmeta.Dest, lastmeta.Dest+".err").Run() exec.Command("mv", lastmeta.Dest+".old", lastmeta.Dest).Run() exec.Command("rm", "-r", lastmeta.Dest+".err").Run() + + // Revert to previous meta with same type + cluster.BackupMetaMap.Delete(lastmeta.Id) + switch backtype { + case config.BackupMethodLogical: + _, server.LastBackupMeta.Logical = server.GetLatestMeta("logical") + case config.BackupMethodPhysical: + _, server.LastBackupMeta.Physical = server.GetLatestMeta("physical") + } } } } diff --git a/config/backup.go b/config/backup.go index d09ecd41b..8bf66dab8 100644 --- a/config/backup.go +++ b/config/backup.go @@ -1,6 +1,10 @@ package config -import "time" +import ( + "os" + "path/filepath" + "time" +) type BackupMethod int @@ -37,4 +41,17 @@ type BackupMetadata struct { BinLogFilePos uint64 `json:"binLogFilePos"` BinLogGtid string `json:"binLogUuid"` Completed bool `json:"completed"` + Previous int64 `json:"previous"` +} + +func (bm *BackupMetadata) GetSize() error { + var size int64 = 0 + err := filepath.Walk(bm.Dest, func(_ string, info os.FileInfo, err error) error { + if err == nil && !info.IsDir() { + size += info.Size() + } + return err + }) + bm.Size = size + return err } diff --git a/config/maps.go b/config/maps.go index 4acb448fb..6b79a9fd4 100644 --- a/config/maps.go +++ b/config/maps.go @@ -880,3 +880,18 @@ func FromBackupMetaMap(m *BackupMetaMap, c *BackupMetaMap) *BackupMetaMap { return m } + +// GetBackupsByToolAndSource retrieves backups with the same backupTool and source. +func (b *BackupMetaMap) GetPreviousBackup(backupTool string, source string) *BackupMetadata { + var result *BackupMetadata + b.Map.Range(func(key, value interface{}) bool { + if backup, ok := value.(*BackupMetadata); ok { + if backup.BackupTool == backupTool && backup.Source == source { + result = backup + return false + } + } + return true + }) + return result +} diff --git a/server/api_cluster.go b/server/api_cluster.go index cdb833a8e..039f25814 100644 --- a/server/api_cluster.go +++ b/server/api_cluster.go @@ -1256,6 +1256,8 @@ func (repman *ReplicationManager) switchSettings(mycluster *cluster.Cluster, set mycluster.SwitchReplicationNoRelay() case "prov-db-force-write-config": mycluster.SwitchForceWriteConfig() + case "backup-keep-until-valid": + mycluster.SwitchBackupKeepUntilValid() } } diff --git a/share/dashboard/app/dashboard.js b/share/dashboard/app/dashboard.js index a32278dfa..22a14b1ef 100644 --- a/share/dashboard/app/dashboard.js +++ b/share/dashboard/app/dashboard.js @@ -375,6 +375,23 @@ app.controller('DashboardController', function ( return t.state === 0 || (t.start < Math.floor((Date.now() - 300000) / 1000) && !t.end && t.state < 3) } + $scope.getBackupMethod = function(method) { + switch(method) { + case 1: return 'Logical'; + case 2: return 'Physical'; + default: return 'Unknown'; + } +}; + +$scope.getBackupStrategy = function(strategy) { + switch(strategy) { + case 1: return 'Full'; + case 2: return 'Incremental'; + case 3: return 'Differential'; + default: return 'Unknown'; + } +}; + $scope.SetApiTokenTimeout = function (val) { if ($scope.roApiTokenTimeout) { $scope.selectedApiTokenTimeout = Number(val) diff --git a/share/dashboard/static/card-setting-backup.html b/share/dashboard/static/card-setting-backup.html index e5263d15e..e8795ade7 100644 --- a/share/dashboard/static/card-setting-backup.html +++ b/share/dashboard/static/card-setting-backup.html @@ -46,6 +46,20 @@ +
Backups
+Current Backups
+Id | +Start Time | +End Time | +Backup Method | +Backup Strategy | +Source | +Destination | +Size | +Compressed | +Encrypted | +Completed | +
---|---|---|---|---|---|---|---|---|---|---|
{{backup.id}} | +{{backup.startTime | date:'yyyy-MM-dd HH:mm:ss'}} | +{{backup.endTime | date:'yyyy-MM-dd HH:mm:ss'}} | +{{getBackupMethod(backup.backupMethod)}} | +{{getBackupStrategy(backup.backupStrategy)}} | +{{backup.source}} | +{{backup.dest}} | +{{formatBytes(backup.size)}} | +{{backup.compressed ? 'Yes' : 'No'}} | +{{backup.encrypted ? 'Yes' : 'No'}} | +{{backup.completed ? 'Yes' : 'No'}} | +
Backup History