copying remote-bucket to same remote bucket; source metadata

* do not PUT (remote) when src == dst * comments inline Signed-off-by: Alex Aizman <[email protected]>
NVIDIA · Nov 12, 2024 · d9a141c · d9a141c
1 parent 06957c2
commit d9a141c
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 14 deletions.
diff --git a/ais/tgtobj.go b/ais/tgtobj.go
@@ -526,7 +526,7 @@ func (poi *putOI) _cleanup(buf []byte, slab *memsys.Slab, lmfh cos.LomWriter, er
 
 func (poi *putOI) validateCksum(c *cmn.CksumConf) (v bool) {
 	switch poi.owt {
-	case cmn.OwtRebalance, cmn.OwtCopy:
+	case cmn.OwtRebalance, cmn.OwtCopy, cmn.OwtCopySameBucket:
 		v = c.ValidateObjMove
 	case cmn.OwtPut:
 		v = true
@@ -1466,33 +1466,42 @@ func (coi *copyOI) _dryRun(lom *core.LOM, objnameTo string) (size int64, err err
 //
 // If destination bucket is remote:
 // - create a local replica of the object on one of the targets, and
-// - PUT to the relevant backend
+// - putRemote (with one exception below)
+//
 // An option for _not_ storing the object _in_ the cluster would be a _feature_ that can be
 // further debated.
 func (coi *copyOI) _reader(t *target, dm *bundle.DataMover, lom, dst *core.LOM) (size int64, _ int, _ error) {
 	reader, oah, errN := coi.DP.Reader(lom, coi.LatestVer, coi.Sync)
 	if errN != nil {
 		return 0, 0, errN
 	}
-	if lom.Bck().Equal(coi.BckTo, true, true) {
-		dst.CopyVersion(oah)
-	}
-
 	poi := allocPOI()
 	{
 		poi.t = t
 		poi.lom = dst
 		poi.config = coi.Config
 		poi.r = reader
-		poi.owt = coi.OWT
 		poi.xctn = coi.Xact // on behalf of
 		poi.workFQN = fs.CSM.Gen(dst, fs.WorkfileType, "copy-dp")
 		poi.atime = oah.AtimeUnix()
 		poi.cksumToUse = oah.Checksum()
+
+		poi.owt = coi.OWT
+		if dm != nil {
+			poi.owt = dm.OWT() // (precedence; cmn.OwtCopy, cmn.OwtTransform - what else?)
+		}
 	}
-	if dm != nil {
-		poi.owt = dm.OWT() // (precedence; cmn.OwtCopy, cmn.OwtTransform - what else?)
+	if poi.owt == cmn.OwtCopy {
+		// preserve src metadata when copying (vs. transforming)
+		dst.CopyVersion(lom)
+		dst.SetCustomMD(lom.GetCustomMD())
+
+		// [special] when src == dst (`ais cp s3://data s3://data --all`)
+		if backend := lom.Bck().RemoteBck(); backend != nil && backend.Equal(coi.BckTo.Bucket()) {
+			poi.owt = cmn.OwtCopySameBucket
+		}
 	}
+
 	ecode, err := poi.putObject()
 	freePOI(poi)
 	if err == nil {
@@ -1552,10 +1561,11 @@ func (coi *copyOI) send(t *target, dm *bundle.DataMover, lom *core.LOM, objNameT
 		sargs.objNameTo = objNameTo
 		sargs.tsi = tsi
 		sargs.dm = dm
+
 		sargs.owt = coi.OWT
-	}
-	if dm != nil {
-		sargs.owt = dm.OWT() // (precedence; cmn.OwtCopy, cmn.OwtTransform - what else?)
+		if dm != nil {
+			sargs.owt = dm.OWT() // (precedence; cmn.OwtCopy, cmn.OwtTransform - what else?)
+		}
 	}
 	size, err = coi._send(t, lom, sargs)
 	freeSnda(sargs)

diff --git a/cmn/bck.go b/cmn/bck.go
@@ -200,7 +200,7 @@ func (n Ns) contains(other Ns) bool {
 // Bck (value)
 /////////
 
-func (b Bck) Equal(other *Bck) bool {
+func (b *Bck) Equal(other *Bck) bool {
 	return b.Name == other.Name && b.Provider == other.Provider && b.Ns == other.Ns
 }
 
@@ -503,7 +503,10 @@ func (qbck *QueryBcks) Validate() (err error) {
 	return nil
 }
 
-func (qbck QueryBcks) Equal(bck *Bck) bool { return Bck(qbck).Equal(bck) }
+func (qbck *QueryBcks) Equal(bck *Bck) bool {
+	b := (*Bck)(qbck)
+	return b.Equal(bck)
+}
 
 // NOTE: a named bucket with no provider is assumed to be ais://
 func (qbck QueryBcks) Contains(other *Bck) bool {

diff --git a/cmn/owt.go b/cmn/owt.go
@@ -34,6 +34,10 @@ const (
 	OwtGet             // GET (with upgrading read-lock in the local-write path)
 	OwtGetPrefetchLock // (used for maximum parallelism when prefetching)
 	//
+	// when remote-src == dst
+	//
+	OwtCopySameBucket
+	//
 	// None of the above
 	//
 	OwtNone