Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

mirror:Add the printing of event time #4953

Merged
merged 25 commits into from
Sep 16, 2024
Merged
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
51ea546
Add the printing of event time
dormanze Jun 7, 2024
f0faa54
Update mirror-main.go
dormanze Jun 11, 2024
2ff79d7
Merge branch 'master' into mirror-event
dormanze Jun 11, 2024
f1ddbfe
Merge branch 'master' into mirror-event
dormanze Jun 13, 2024
6309fda
Merge branch 'master' into mirror-event
dormanze Jun 19, 2024
94e4b8a
Merge branch 'master' into mirror-event
dormanze Jun 21, 2024
85b3771
Merge branch 'master' into mirror-event
dormanze Jun 25, 2024
85a8389
Merge branch 'master' into mirror-event
dormanze Jun 29, 2024
bddf60f
Merge branch 'master' into mirror-event
dormanze Jun 30, 2024
16616c2
Merge branch 'master' into mirror-event
dormanze Jul 16, 2024
901bae3
Merge branch 'master' into mirror-event
dormanze Jul 19, 2024
1bba077
Merge branch 'master' into mirror-event
klauspost Jul 23, 2024
6565faa
Merge branch 'master' into mirror-event
dormanze Jul 27, 2024
f8dbb5e
Merge branch 'master' into mirror-event
dormanze Aug 1, 2024
8b2e738
Merge branch 'master' into mirror-event
dormanze Aug 5, 2024
54803d5
Merge branch 'master' into mirror-event
dormanze Aug 6, 2024
0ae166d
Merge branch 'master' into mirror-event
dormanze Aug 7, 2024
1c3fec0
Merge branch 'master' into mirror-event
dormanze Aug 9, 2024
d4e8f57
Merge branch 'master' into mirror-event
dormanze Aug 9, 2024
e1fcb5e
Merge branch 'master' into mirror-event
dormanze Aug 12, 2024
811475b
Merge branch 'master' into mirror-event
dormanze Aug 13, 2024
45f5de8
Merge branch 'master' into mirror-event
dormanze Aug 16, 2024
257fbea
Merge branch 'master' into mirror-event
dormanze Aug 19, 2024
3489439
Merge branch 'master' into mirror-event
dormanze Aug 27, 2024
2d3a47b
Merge branch 'master' into mirror-event
dormanze Sep 9, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 41 additions & 19 deletions cmd/mirror-main.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ import (
"sync"
"time"

"github.com/dustin/go-humanize"

"github.com/fatih/color"
"github.com/minio/cli"
json "github.com/minio/colorjson"
Expand Down Expand Up @@ -277,17 +279,31 @@ type mirrorJob struct {

// mirrorMessage container for file mirror messages
type mirrorMessage struct {
Status string `json:"status"`
Source string `json:"source"`
Target string `json:"target"`
Size int64 `json:"size"`
TotalCount int64 `json:"totalCount"`
TotalSize int64 `json:"totalSize"`
Status string `json:"status"`
Source string `json:"source"`
Target string `json:"target"`
Size int64 `json:"size"`
TotalCount int64 `json:"totalCount"`
TotalSize int64 `json:"totalSize"`
EventTime string `json:"eventTime"`
EventType notification.EventType `json:"eventType"`
}

// String colorized mirror message
func (m mirrorMessage) String() string {
return console.Colorize("Mirror", fmt.Sprintf("`%s` -> `%s`", m.Source, m.Target))
var msg string
if m.EventTime != "" {
msg = console.Colorize("Time", fmt.Sprintf("[%s] ", m.EventTime))
}
if m.EventType == notification.ObjectRemovedDelete {
return msg + "Removed " + console.Colorize("Removed", fmt.Sprintf("`%s`", m.Target))
dormanze marked this conversation as resolved.
Show resolved Hide resolved
}
if m.EventTime == "" {
return console.Colorize("Mirror", fmt.Sprintf("`%s` -> `%s`", m.Source, m.Target))
}
msg += console.Colorize("Size", fmt.Sprintf("%6s ", humanize.IBytes(uint64(m.Size))))
msg += console.Colorize("Mirror", fmt.Sprintf("`%s` -> `%s`", m.Source, m.Target))
return msg
}

// JSON jsonified mirror message
Expand Down Expand Up @@ -345,7 +361,7 @@ func (mj *mirrorJob) doDeleteBucket(ctx context.Context, sURLs URLs) URLs {
}

// doRemove - removes files on target.
func (mj *mirrorJob) doRemove(ctx context.Context, sURLs URLs) URLs {
func (mj *mirrorJob) doRemove(ctx context.Context, sURLs URLs, event EventInfo) URLs {
if mj.opts.isFake {
return sURLs.WithError(nil)
}
Expand Down Expand Up @@ -375,13 +391,21 @@ func (mj *mirrorJob) doRemove(ctx context.Context, sURLs URLs) URLs {
}
return sURLs.WithError(result.Err)
}
targetPath := filepath.ToSlash(filepath.Join(sURLs.TargetAlias, sURLs.TargetContent.URL.Path))
mj.status.PrintMsg(mirrorMessage{
Target: targetPath,
TotalCount: sURLs.TotalCount,
TotalSize: sURLs.TotalSize,
EventTime: event.Time,
EventType: event.Type,
})
}

return sURLs.WithError(nil)
}

// doMirror - Mirror an object to multiple destination. URLs status contains a copy of sURLs and error if any.
func (mj *mirrorJob) doMirrorWatch(ctx context.Context, targetPath string, tgtSSE encrypt.ServerSide, sURLs URLs) URLs {
func (mj *mirrorJob) doMirrorWatch(ctx context.Context, targetPath string, tgtSSE encrypt.ServerSide, sURLs URLs, event EventInfo) URLs {
shouldQueue := false
if !mj.opts.isOverwrite && !mj.opts.activeActive {
targetClient, err := newClient(targetPath)
Expand All @@ -405,7 +429,7 @@ func (mj *mirrorJob) doMirrorWatch(ctx context.Context, targetPath string, tgtSS
mj.status.AddCounts(1)
sURLs.TotalSize = mj.status.Get()
sURLs.TotalCount = mj.status.GetCounts()
return mj.doMirror(ctx, sURLs)
return mj.doMirror(ctx, sURLs, event)
}
return sURLs.WithError(probe.NewError(ObjectAlreadyExists{}))
}
Expand All @@ -428,7 +452,7 @@ func convertSizeToTag(size int64) string {
}

// doMirror - Mirror an object to multiple destination. URLs status contains a copy of sURLs and error if any.
func (mj *mirrorJob) doMirror(ctx context.Context, sURLs URLs) URLs {
func (mj *mirrorJob) doMirror(ctx context.Context, sURLs URLs, event EventInfo) URLs {
if sURLs.Error != nil { // Erroneous sURLs passed.
return sURLs.WithError(sURLs.Error.Trace())
}
Expand Down Expand Up @@ -481,6 +505,8 @@ func (mj *mirrorJob) doMirror(ctx context.Context, sURLs URLs) URLs {
Size: length,
TotalCount: sURLs.TotalCount,
TotalSize: sURLs.TotalSize,
EventTime: event.Time,
EventType: event.Type,
})
}
sURLs.MD5 = mj.opts.md5
Expand Down Expand Up @@ -589,10 +615,6 @@ func (mj *mirrorJob) monitorMirrorStatus(cancel context.CancelFunc) (errDuringMi

if sURLs.SourceContent != nil {
mirrorTotalUploadedBytes.Add(float64(sURLs.SourceContent.Size))
} else if sURLs.TargetContent != nil {
dormanze marked this conversation as resolved.
Show resolved Hide resolved
// Construct user facing message and path.
targetPath := filepath.ToSlash(filepath.Join(sURLs.TargetAlias, sURLs.TargetContent.URL.Path))
mj.status.PrintMsg(rmMessage{Key: targetPath})
}
}

Expand Down Expand Up @@ -687,7 +709,7 @@ func (mj *mirrorJob) watchMirrorEvents(ctx context.Context, events []EventInfo)
continue
}
mj.parallel.queueTask(func() URLs {
return mj.doMirrorWatch(ctx, targetPath, tgtSSE, mirrorURL)
return mj.doMirrorWatch(ctx, targetPath, tgtSSE, mirrorURL, event)
}, mirrorURL.SourceContent.Size)
} else if event.Type == notification.ObjectRemovedDelete {
if targetAlias != "" && strings.Contains(event.UserAgent, uaMirrorAppName+":"+targetAlias) {
Expand All @@ -707,7 +729,7 @@ func (mj *mirrorJob) watchMirrorEvents(ctx context.Context, events []EventInfo)
mirrorURL.TotalSize = mj.status.Get()
if mirrorURL.TargetContent != nil && (mj.opts.isRemove || mj.opts.activeActive) {
mj.parallel.queueTask(func() URLs {
return mj.doRemove(ctx, mirrorURL)
return mj.doRemove(ctx, mirrorURL, event)
}, 0)
}
} else if event.Type == notification.BucketCreatedAll {
Expand Down Expand Up @@ -807,11 +829,11 @@ func (mj *mirrorJob) startMirror(ctx context.Context) {

if sURLs.SourceContent != nil {
mj.parallel.queueTask(func() URLs {
return mj.doMirror(ctx, sURLs)
return mj.doMirror(ctx, sURLs, EventInfo{})
}, sURLs.SourceContent.Size)
} else if sURLs.TargetContent != nil && mj.opts.isRemove {
mj.parallel.queueTask(func() URLs {
return mj.doRemove(ctx, sURLs)
return mj.doRemove(ctx, sURLs, EventInfo{})
}, 0)
}
case <-ctx.Done():
Expand Down
Loading