From 2ed5063b6fb7e846afd97e3b8f457278c82d925e Mon Sep 17 00:00:00 2001 From: Jonathan Poole Date: Wed, 15 Nov 2023 19:59:31 +0000 Subject: [PATCH] Implement iofs.StatFS for the CAS FS (#2962) * Implement iofs.StatFS for the CAS FS * lint --- src/remote/fs/fs.go | 54 +++++++++++++++--------------- src/remote/fs/fs_test.go | 61 +++++++++++++++++++++++++++++++++- src/remote/fs/info.go | 71 ++++++++++++++++++++++++++-------------- 3 files changed, 133 insertions(+), 53 deletions(-) diff --git a/src/remote/fs/fs.go b/src/remote/fs/fs.go index 1f6f866146..cff02a2a3a 100644 --- a/src/remote/fs/fs.go +++ b/src/remote/fs/fs.go @@ -35,6 +35,25 @@ type CASFileSystem struct { workingDir string } +// Stat implements StatFS so that iofs.Stat doesn't download the file to determine this info. +func (fs *CASFileSystem) Stat(name string) (iofs.FileInfo, error) { + file, dir, link, err := fs.FindNode(name) + if err != nil { + return nil, err + } + if file != nil { + return newFileInfo(file), nil + } + if dir != nil { + dirPB := fs.directories[digest.NewFromProtoUnvalidated(dir.Digest)] + return newDirInfo(dir.Name, dirPB), nil + } + if link != nil { + return newSymlinkInfo(link), nil + } + return nil, os.ErrNotExist +} + // New creates a new filesystem on top of the given proto, using client to download files from the CAS on demand. func New(c Client, tree *pb.Tree, workingDir string) *CASFileSystem { directories := make(map[digest.Digest]*pb.Directory, len(tree.Children)) @@ -94,25 +113,17 @@ func (fs *CASFileSystem) openFile(f *pb.FileNode) (*file, error) { return nil, err } - i := info{ - size: int64(len(bs)), - name: f.Name, - } - return &file{ ReadSeeker: bytes.NewReader(bs), - info: i.withProperties(f.NodeProperties), + info: newFileInfo(f), }, nil } func (fs *CASFileSystem) openDir(d *pb.DirectoryNode) (iofs.File, error) { dirPb := fs.directories[digest.NewFromProtoUnvalidated(d.Digest)] - i := &info{ - name: d.Name, - isDir: true, - } + return &dir{ - info: i.withProperties(dirPb.NodeProperties), + info: newDirInfo(d.Name, dirPb), pb: dirPb, children: fs.directories, }, nil @@ -199,33 +210,20 @@ func (p *dir) ReadDir(n int) ([]iofs.DirEntry, error) { return ret, nil } dir := p.children[digest.NewFromProtoUnvalidated(dirNode.Digest)] - i := &info{ - name: dirNode.Name, - isDir: true, - typeMode: os.ModeDir, - } - - ret = append(ret, i.withProperties(dir.NodeProperties)) + ret = append(ret, newDirInfo(dirNode.Name, dir)) } for _, file := range p.pb.Files { if n > 0 && len(ret) == n { return ret, nil } - i := &info{ - name: file.Name, - size: file.Digest.SizeBytes, - } - ret = append(ret, i.withProperties(file.NodeProperties)) + + ret = append(ret, newFileInfo(file)) } for _, link := range p.pb.Symlinks { if n > 0 && len(ret) == n { return ret, nil } - i := &info{ - name: link.Name, - typeMode: os.ModeSymlink, - } - ret = append(ret, i.withProperties(link.NodeProperties)) + ret = append(ret, newSymlinkInfo(link)) } return ret, nil } diff --git a/src/remote/fs/fs_test.go b/src/remote/fs/fs_test.go index 25d07593ed..d244b136cd 100644 --- a/src/remote/fs/fs_test.go +++ b/src/remote/fs/fs_test.go @@ -3,6 +3,7 @@ package fs import ( "context" iofs "io/fs" + "os" "testing" "github.com/bazelbuild/remote-apis-sdks/go/pkg/client" @@ -146,7 +147,7 @@ func TestReadDir(t *testing.T) { i, err := e.Info() require.NoError(t, err) // We set them all to 0777 above - assert.Equal(t, iofs.FileMode(0777), i.Mode(), "%v mode was wrong", e.Name()) + assert.Equal(t, iofs.FileMode(0777), i.Mode().Perm(), "%v mode was wrong", e.Name()) if e.Name() == "foo" { assert.Equal(t, len([]byte(fooContent)), int(i.Size())) } @@ -225,6 +226,18 @@ func TestReadFile(t *testing.T) { file: "bar/badlink", expectError: true, }, + { + name: "Open missing file", + wd: ".", + file: "bar/faff", + expectError: true, + }, + { + name: "Open directory passed file", + wd: ".", + file: "foo/bar", + expectError: true, + }, } for _, tc := range tests { @@ -239,3 +252,49 @@ func TestReadFile(t *testing.T) { }) } } + +func TestStat(t *testing.T) { + fc, tree := getTree(t) + + tests := []struct { + name string + file string + expectNotExist bool + expectedType os.FileMode + }{ + { + name: "Stat file", + file: "bar/example.go", + expectedType: 0, + }, + { + name: "Stat dir", + file: "bar", + expectedType: os.ModeDir, + }, + { + name: "Stat symlink", + file: "bar/link", + expectedType: os.ModeSymlink, + }, + { + name: "Stat not exist", + file: "bar/not_exist.go", + expectNotExist: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + i, err := iofs.Stat(New(fc, tree, "."), tc.file) + if tc.expectNotExist { + assert.True(t, os.IsNotExist(err)) + return + } + require.NoError(t, err) + assert.Equal(t, tc.expectedType.IsDir(), i.Mode().IsDir()) + assert.Equal(t, tc.expectedType.IsRegular(), i.Mode().IsRegular()) + assert.Equal(t, tc.expectedType&os.ModeSymlink != 0, i.Mode()&os.ModeSymlink != 0) + }) + } +} diff --git a/src/remote/fs/info.go b/src/remote/fs/info.go index ce0d9f42d3..69c279e2a5 100644 --- a/src/remote/fs/info.go +++ b/src/remote/fs/info.go @@ -10,16 +10,55 @@ import ( // info represents information about a file/directory type info struct { - name string - isDir bool - size int64 - modTime time.Time - mode os.FileMode - typeMode os.FileMode + name string + size int64 + modTime time.Time + mode os.FileMode +} + +func newFileInfo(f *pb.FileNode) *info { + i := info{ + size: f.Digest.SizeBytes, + name: f.Name, + } + return i.withProperties(f.NodeProperties) +} +func newDirInfo(name string, dir *pb.Directory) *info { + i := &info{ + name: name, + mode: os.ModeDir, + } + return i.withProperties(dir.NodeProperties) +} + +func newSymlinkInfo(node *pb.SymlinkNode) *info { + i := &info{ + name: node.Name, + mode: os.ModeSymlink, + } + return i.withProperties(node.NodeProperties) +} + +// withProperties safely sets the node info if it's available. +func (i *info) withProperties(nodeProperties *pb.NodeProperties) *info { + if nodeProperties == nil { + return i + } + + if nodeProperties.UnixMode != nil { + // This should in theory have the type mode set already but we bitwise or here to to make sure this is preserved + // from the constructors above in case the remote doesn't set this. + i.mode |= os.FileMode(nodeProperties.UnixMode.Value) + } + + if nodeProperties.Mtime != nil { + i.modTime = nodeProperties.Mtime.AsTime() + } + return i } func (i *info) Type() iofs.FileMode { - return i.typeMode + return i.mode.Type() } func (i *info) Info() (iofs.FileInfo, error) { @@ -43,25 +82,9 @@ func (i *info) ModTime() time.Time { } func (i *info) IsDir() bool { - return i.isDir + return i.mode.IsDir() } func (i *info) Sys() any { return nil } - -// withProperties safely sets the node info if it's available. -func (i *info) withProperties(nodeProperties *pb.NodeProperties) *info { - if nodeProperties == nil { - return i - } - - if nodeProperties.UnixMode != nil { - i.mode = os.FileMode(nodeProperties.UnixMode.Value) - } - - if nodeProperties.Mtime != nil { - i.modTime = nodeProperties.Mtime.AsTime() - } - return i -}