mirror:support one mirror process to back up specified buckets

cmd/mirror-main.go

@@ -105,6 +105,10 @@ var (
 			Name:  "exclude-bucket",
 			Usage: "exclude bucket(s) that match specified bucket name pattern",
 		},
+		cli.StringSliceFlag{
+			Name:  "bucket",
+			Usage: "mirror bucket(s) that match specified bucket name pattern",
+		},
 		cli.StringSliceFlag{
 			Name:  "exclude-storageclass",
 			Usage: "exclude object(s) that match the specified storage class",
@@ -201,19 +205,22 @@ EXAMPLES:
       Exclude test* buckets and backup* buckets when mirroring.
       {{.Prompt}} {{.HelpName}} --exclude-bucket 'test*' --exclude 'backup*' s3 ~/test
 
-  11. Mirror objects newer than 10 days from bucket test to a local folder.
+  11. Mirror test* buckets from aliased Amazon S3 cloud storage to a local folder.
+      {{.Prompt}} {{.HelpName}} --bucket 'test*' s3 ~/test
+
+  12. Mirror objects newer than 10 days from bucket test to a local folder.
       {{.Prompt}} {{.HelpName}} --newer-than 10d s3/test ~/localfolder
 
-  12. Mirror objects older than 30 days from Amazon S3 bucket test to a local folder.
+  13. Mirror objects older than 30 days from Amazon S3 bucket test to a local folder.
       {{.Prompt}} {{.HelpName}} --older-than 30d s3/test ~/test
 
-  13. Mirror server encrypted objects from Amazon S3 cloud storage to a bucket on Amazon S3 cloud storage
+  14. Mirror server encrypted objects from Amazon S3 cloud storage to a bucket on Amazon S3 cloud storage
       {{.Prompt}} {{.HelpName}} --enc-c "minio/archive=MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDA" --enc-c "s3/archive=MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5BBB" s3/archive/ minio/archive/ 
 
-  14. Update 'Cache-Control' header on all existing objects recursively.
+  15. Update 'Cache-Control' header on all existing objects recursively.
       {{.Prompt}} {{.HelpName}} --attr "Cache-Control=max-age=90000,min-fresh=9000" myminio/video-files myminio/video-files
 
-  15. Mirror a local folder recursively to Amazon S3 cloud storage and preserve all local file attributes.
+  16. Mirror a local folder recursively to Amazon S3 cloud storage and preserve all local file attributes.
       {{.Prompt}} {{.HelpName}} -a backup/ s3/archive
 
   16. Cross mirror between sites in a active-active deployment.
@@ -637,7 +644,7 @@ func (mj *mirrorJob) watchMirrorEvents(ctx context.Context, events []EventInfo)
 			continue
 		}
 		// Skip the bucket, if it matches the Exclude options provided
-		if matchExcludeBucketOptions(mj.opts.excludeBuckets, sourceSuffix) {
+		if matchExcludeBucketOptions(mj.opts.excludeBuckets, mj.opts.buckets, sourceSuffix) {
 			continue
 		}
 
@@ -977,6 +984,7 @@ func runMirror(ctx context.Context, srcURL, dstURL string, cli *cli.Context, enc
 		skipErrors:            cli.Bool("skip-errors"),
 		excludeOptions:        cli.StringSlice("exclude"),
 		excludeBuckets:        cli.StringSlice("exclude-bucket"),
+		buckets:               cli.StringSlice("bucket"),
 		excludeStorageClasses: cli.StringSlice("exclude-storageclass"),
 		olderThan:             cli.String("older-than"),
 		newerThan:             cli.String("newer-than"),
@@ -1049,7 +1057,7 @@ func runMirror(ctx context.Context, srcURL, dstURL string, cli *cli.Context, enc
 				}
 
 				// Skip create bucket, if it matches the Exclude options provided
-				if matchExcludeBucketOptions(mopts.excludeBuckets, sourceSuffix) {
+				if matchExcludeBucketOptions(mopts.excludeBuckets, mj.opts.buckets, sourceSuffix) {
 					continue
 				}
 

cmd/mirror-url.go

@@ -106,7 +106,7 @@ func matchExcludeOptions(excludeOptions []string, srcSuffix string, typ ClientUR
 	return false
 }
 
-func matchExcludeBucketOptions(excludeBuckets []string, srcSuffix string) bool {
+func matchExcludeBucketOptions(excludeBuckets, includeBuckets []string, srcSuffix string) bool {
 	if strings.HasPrefix(srcSuffix, "/") {
 		srcSuffix = srcSuffix[1:]
 	} else if runtime.GOOS == "windows" && strings.HasPrefix(srcSuffix, `\`) {
@@ -118,12 +118,23 @@ func matchExcludeBucketOptions(excludeBuckets []string, srcSuffix string) bool {
 	} else {
 		bucketName = strings.Split(srcSuffix, "/")[0]
 	}
+	if bucketName == "" {
+		return false
+	}
 	for _, pattern := range excludeBuckets {
 		if wildcard.Match(pattern, bucketName) {
 			return true
 		}
 	}
-	return false
+	if len(includeBuckets) <= 0 {
+		return false
+	}
+	for _, pattern := range includeBuckets {
+		if wildcard.Match(pattern, bucketName) {
+			return false
+		}
+	}
+	return true
 }
 
 func deltaSourceTarget(ctx context.Context, sourceURL, targetURL string, opts mirrorOptions, URLsCh chan<- URLs) {
@@ -179,7 +190,7 @@ func deltaSourceTarget(ctx context.Context, sourceURL, targetURL string, opts mi
 		}
 
 		// Skip the source bucket if it matches the Exclude options provided
-		if matchExcludeBucketOptions(opts.excludeBuckets, srcSuffix) {
+		if matchExcludeBucketOptions(opts.excludeBuckets, opts.buckets, srcSuffix) {
 			continue
 		}
 
@@ -190,7 +201,7 @@ func deltaSourceTarget(ctx context.Context, sourceURL, targetURL string, opts mi
 		}
 
 		// Skip the target bucket if it matches the Exclude options provided
-		if matchExcludeBucketOptions(opts.excludeBuckets, tgtSuffix) {
+		if matchExcludeBucketOptions(opts.excludeBuckets, opts.buckets, tgtSuffix) {
 			continue
 		}
 
@@ -263,17 +274,17 @@ func deltaSourceTarget(ctx context.Context, sourceURL, targetURL string, opts mi
 }
 
 type mirrorOptions struct {
-	isFake, isOverwrite, activeActive                     bool
-	isWatch, isRemove, isMetadata                         bool
-	isRetriable                                           bool
-	isSummary                                             bool
-	skipErrors                                            bool
-	excludeOptions, excludeStorageClasses, excludeBuckets []string
-	encKeyDB                                              map[string][]prefixSSEPair
-	md5, disableMultipart                                 bool
-	olderThan, newerThan                                  string
-	storageClass                                          string
-	userMetadata                                          map[string]string
+	isFake, isOverwrite, activeActive                              bool
+	isWatch, isRemove, isMetadata                                  bool
+	isRetriable                                                    bool
+	isSummary                                                      bool
+	skipErrors                                                     bool
+	excludeOptions, excludeStorageClasses, excludeBuckets, buckets []string
+	encKeyDB                                                       map[string][]prefixSSEPair
+	md5, disableMultipart                                          bool
+	olderThan, newerThan                                           string
+	storageClass                                                   string
+	userMetadata                                                   map[string]string
 }
 
 // Prepares urls that need to be copied or removed based on requested options.