Support xattrs

fs/layer/util_test.go

@@ -264,7 +264,7 @@ func testExistenceWithOpaque(t *testing.T, factory metadata.Store, opaque Overla
 			},
 			want: []check{
 				hasOpaque("foo/"),
-				hasNodeXattrs("foo/", "SCHILY.xattr.foo", "bar"),
+				hasNodeXattrs("foo/", "foo", "bar"),
 				fileNotExist("foo/.wh..wh..opq"),
 			},
 		},

fs/reader/reader.go

@@ -52,6 +52,7 @@ import (
 	spanmanager "github.com/awslabs/soci-snapshotter/fs/span-manager"
 	"github.com/awslabs/soci-snapshotter/metadata"
 	"github.com/awslabs/soci-snapshotter/util/ioutils"
+	"github.com/awslabs/soci-snapshotter/ztoc"
 	"github.com/awslabs/soci-snapshotter/ztoc/compression"
 	digest "github.com/opencontainers/go-digest"
 )
@@ -314,20 +315,13 @@ func attrMatchesTarHeader(attr metadata.Attr, tarh *tar.Header) bool {
 		return false
 	}
 
-	// TODO: This is probably not correct. PAXRecords are a generic
-	// key value pair in the tar header. Posix Xattrs can be encoded
-	// into PAX headers in a number of ways.
-	//
-	// We do it this way because the TOC builder is incorrectly
-	// dumping all PAXRecords into the file metadata's Xattrs.
-	// Fixing this is not backwards compatible, so we might end
-	// up filtering to Posix xattrs in the fuse GetXAttrs implementation.
-	if len(attr.Xattrs) != len(tarh.PAXRecords) {
+	tarXattrs := ztoc.Xattrs(tarh.PAXRecords)
+	if len(attr.Xattrs) != len(tarXattrs) {
 		return false
 	}
 	for k := range attr.Xattrs {
 		attrV := attr.Xattrs[k]
-		tarV := tarh.PAXRecords[k]
+		tarV := tarXattrs[k]
 		if len(attrV) != len(tarV) {
 			return false
 		}

integration/metrics_test.go

@@ -155,7 +155,7 @@ log_fuse_operations = true
 		for i, md := range zt.FileMetadata {
 			md.UncompressedOffset += 2
 			md.UncompressedSize = math.MaxInt64
-			md.Xattrs = map[string]string{"foo": "bar"}
+			md.PAXHeaders = map[string]string{"foo": "bar"}
 			zt.FileMetadata[i] = md
 		}
 	}

metadata/reader.go

@@ -575,7 +575,7 @@ func attrFromZtocEntry(src *ztoc.FileMetadata, dst *Attr) *Attr {
 	dst.DevMajor = int(src.Devmajor)
 	dst.DevMinor = int(src.Devminor)
 	xattrs := make(map[string][]byte)
-	for k, v := range src.Xattrs {
+	for k, v := range src.Xattrs() {
 		xattrs[k] = []byte(v)
 	}
 	dst.Xattrs = xattrs

metadata/util_test.go

@@ -92,7 +92,7 @@ func testReader(t *testing.T, factory readerFactory) {
 				hasModTime("xxx.txt", sampleTime),
 				hasFile("y.txt", 0),
 				// For details on the keys of Xattrs, see https://pkg.go.dev/archive/tar#Header
-				hasXattrs("y.txt", map[string]string{"SCHILY.xattr.testkey": "testval"}),
+				hasXattrs("y.txt", map[string]string{"testkey": "testval"}),
 			},
 		},
 		{
@@ -116,7 +116,7 @@ func testReader(t *testing.T, factory readerFactory) {
 				hasMode("foo", os.ModeDir|0600|os.ModeSticky),
 				hasOwner("foo/bar", 1000, 1000),
 				hasModTime("foo/a", sampleTime),
-				hasXattrs("foo/a/1", map[string]string{"SCHILY.xattr.testkey": "testval"}),
+				hasXattrs("foo/a/1", map[string]string{"testkey": "testval"}),
 				hasFile("foo/bar/baz.txt", 8),
 				hasFile("foo/bar/xxxx", 1),
 				hasFile("foo/bar/yyy", 3),

ztoc/fbs/ztoc.fbs

@@ -22,7 +22,7 @@ table FileMetadata {
 	devmajor : long;		// Major device number (valid for TypeChar or TypeBlock)
 	devminor : long;		// Minor device number (valid for TypeChar or TypeBlock)
 
-	xattrs : [Xattr];
+	xattrs : [Xattr];       // Raw PAXRecords from the tar file. The name is wrong, but changing it is backwards incompatible
 }
 
 enum CompressionAlgorithm : byte { Gzip = 1, Uncompressed }

ztoc/tar.go

@@ -16,10 +16,18 @@
 
 package ztoc
 
-import "github.com/awslabs/soci-snapshotter/ztoc/compression"
+import (
+	"strings"
 
-// TarBlockSize is the size of a tar block
-const TarBlockSize = 512
+	"github.com/awslabs/soci-snapshotter/ztoc/compression"
+)
+
+const (
+	// TarBlockSize is the size of a tar block
+	TarBlockSize = 512
+
+	schilyXattrPrefix string = "SCHILY.xattr."
+)
 
 // AlignToTarBlock aligns an offset to the next larger multiple of TarBlockSize
 func AlignToTarBlock(o compression.Offset) compression.Offset {
@@ -29,3 +37,20 @@ func AlignToTarBlock(o compression.Offset) compression.Offset {
 	}
 	return o
 }
+
+// Xattrs converts a set of tar PAXRecords to a set of Xattrs.
+// Specifically, it looks for PAXRecords where the key is prefixed
+// by `SCHILY.xattr.` - the prefix for Xattrs used by go and GNU tar.
+// Those keys are kept with the prefix stripped. Other keys are dropped.
+func Xattrs(paxHeaders map[string]string) map[string]string {
+	if len(paxHeaders) == 0 {
+		return nil
+	}
+	m := make(map[string]string)
+	for k, v := range paxHeaders {
+		if strings.HasPrefix(k, schilyXattrPrefix) {
+			m[k[len(schilyXattrPrefix):]] = v
+		}
+	}
+	return m
+}

ztoc/toc_builder.go

@@ -150,7 +150,7 @@ func metadataFromTarReader(r io.Reader) ([]FileMetadata, compression.Offset, err
 			ModTime:            hdr.ModTime,
 			Devmajor:           hdr.Devmajor,
 			Devminor:           hdr.Devminor,
-			Xattrs:             hdr.PAXRecords,
+			PAXHeaders:         hdr.PAXRecords,
 		}
 		md = append(md, metadataEntry)
 		// The next file's tar header can be found immediately after the current file + padding

ztoc/ztoc.go

@@ -87,7 +87,7 @@ type FileMetadata struct {
 	Devmajor int64     // Major device number (valid for TypeChar or TypeBlock)
 	Devminor int64     // Minor device number (valid for TypeChar or TypeBlock)
 
-	Xattrs map[string]string
+	PAXHeaders map[string]string
 }
 
 // FileMode gets file mode for the file metadata
@@ -127,17 +127,21 @@ func (src FileMetadata) Equal(o FileMetadata) bool {
 		src.Devminor != o.Devminor {
 		return false
 	}
-	if len(src.Xattrs) != len(o.Xattrs) {
+	if len(src.PAXHeaders) != len(o.PAXHeaders) {
 		return false
 	}
-	for k, v := range src.Xattrs {
-		if o.Xattrs[k] != v {
+	for k, v := range src.PAXHeaders {
+		if o.PAXHeaders[k] != v {
 			return false
 		}
 	}
 	return true
 }
 
+func (src FileMetadata) Xattrs() map[string]string {
+	return Xattrs(src.PAXHeaders)
+}
+
 // MetadataEntry is used to locate a file based on its metadata.
 type MetadataEntry struct {
 	UncompressedSize   compression.Offset

ztoc/ztoc_marshaler.go

@@ -126,13 +126,13 @@ func flatbufferToTOC(fbtoc *ztoc_flatbuffers.TOC) (TOC, error) {
 		me.ModTime = *modTime
 		me.Devmajor = metadataEntry.Devmajor()
 		me.Devminor = metadataEntry.Devminor()
-		me.Xattrs = make(map[string]string)
+		me.PAXHeaders = make(map[string]string)
 		for j := 0; j < metadataEntry.XattrsLength(); j++ {
 			xattrEntry := new(ztoc_flatbuffers.Xattr)
 			metadataEntry.Xattrs(xattrEntry, j)
 			key := string(xattrEntry.Key())
 			value := string(xattrEntry.Value())
-			me.Xattrs[key] = value
+			me.PAXHeaders[key] = value
 		}
 
 		toc.FileMetadata[i] = me
@@ -266,16 +266,16 @@ func prepareMetadataOffset(builder *flatbuffers.Builder, me FileMetadata) flatbu
 }
 
 func prepareXattrsOffset(me FileMetadata, builder *flatbuffers.Builder) flatbuffers.UOffsetT {
-	keys := make([]string, 0, len(me.Xattrs))
-	for k := range me.Xattrs {
+	keys := make([]string, 0, len(me.PAXHeaders))
+	for k := range me.PAXHeaders {
 		keys = append(keys, k)
 	}
 	sort.Strings(keys)
 
-	xattrOffsetList := make([]flatbuffers.UOffsetT, 0, len(me.Xattrs))
+	xattrOffsetList := make([]flatbuffers.UOffsetT, 0, len(me.PAXHeaders))
 	for _, key := range keys {
 		keyOffset := builder.CreateString(key)
-		valueOffset := builder.CreateString(me.Xattrs[key])
+		valueOffset := builder.CreateString(me.PAXHeaders[key])
 		ztoc_flatbuffers.XattrStart(builder)
 		ztoc_flatbuffers.XattrAddKey(builder, keyOffset)
 		ztoc_flatbuffers.XattrAddValue(builder, valueOffset)
@@ -286,7 +286,7 @@ func prepareXattrsOffset(me FileMetadata, builder *flatbuffers.Builder) flatbuff
 	for j := len(xattrOffsetList) - 1; j >= 0; j-- {
 		builder.PrependUOffsetT(xattrOffsetList[j])
 	}
-	xattrs := builder.EndVector(len(me.Xattrs))
+	xattrs := builder.EndVector(len(me.PAXHeaders))
 	return xattrs
 }
 

ztoc/ztoc_test.go

@@ -502,8 +502,8 @@ func testZtocSerialization(t *testing.T, compressionAlgo string, generator tarGe
 			// append xattrs
 			for i := 0; i < len(createdZtoc.FileMetadata); i++ {
 				for key := range tc.xattrs {
-					createdZtoc.FileMetadata[i].Xattrs = make(map[string]string)
-					createdZtoc.FileMetadata[i].Xattrs[key] = tc.xattrs[key]
+					createdZtoc.FileMetadata[i].PAXHeaders = make(map[string]string)
+					createdZtoc.FileMetadata[i].PAXHeaders[key] = tc.xattrs[key]
 				}
 			}