From 57927112a4475c3531e97af82c6fef143ce979ec Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 29 Dec 2023 10:29:00 +0000 Subject: [PATCH] squashfs: use the LRU cache for fragments and metadata - fixes #205 This also adds methods to set and read the size of the cache. --- filesystem/squashfs/metadatablock.go | 50 ++++++++--------- filesystem/squashfs/squashfs.go | 80 +++++++++++++++++++++------- filesystem/squashfs/squashfs_test.go | 23 ++++++++ 3 files changed, 110 insertions(+), 43 deletions(-) diff --git a/filesystem/squashfs/metadatablock.go b/filesystem/squashfs/metadatablock.go index fea3f064..8ef65098 100644 --- a/filesystem/squashfs/metadatablock.go +++ b/filesystem/squashfs/metadatablock.go @@ -73,32 +73,34 @@ func (m *metadatablock) toBytes(c Compressor) ([]byte, error) { } func (fs *FileSystem) readMetaBlock(r io.ReaderAt, c Compressor, location int64) (data []byte, size uint16, err error) { - // read bytes off the reader to determine how big it is and if compressed - b := make([]byte, 2) - _, _ = r.ReadAt(b, location) - size, compressed, err := getMetadataSize(b) - if err != nil { - return nil, 0, fmt.Errorf("error getting size and compression for metadata block at %d: %v", location, err) - } - b = make([]byte, size) - read, err := r.ReadAt(b, location+2) - if err != nil && err != io.EOF { - return nil, 0, fmt.Errorf("unable to read metadata block of size %d at location %d: %v", size, location, err) - } - if read != len(b) { - return nil, 0, fmt.Errorf("read %d instead of expected %d bytes for metadata block at location %d", read, size, location) - } - data = b - if compressed { - if c == nil { - return nil, 0, fmt.Errorf("metadata block at %d compressed, but no compressor provided", location) - } - data, err = c.decompress(b) + return fs.cache.get(location, func() (data []byte, size uint16, err error) { + // read bytes off the reader to determine how big it is and if compressed + b := make([]byte, 2) + _, _ = r.ReadAt(b, location) + size, compressed, err := getMetadataSize(b) if err != nil { - return nil, 0, fmt.Errorf("decompress error: %v", err) + return nil, 0, fmt.Errorf("error getting size and compression for metadata block at %d: %v", location, err) } - } - return data, size + 2, nil + b = make([]byte, size) + read, err := r.ReadAt(b, location+2) + if err != nil && err != io.EOF { + return nil, 0, fmt.Errorf("unable to read metadata block of size %d at location %d: %v", size, location, err) + } + if read != len(b) { + return nil, 0, fmt.Errorf("read %d instead of expected %d bytes for metadata block at location %d", read, size, location) + } + data = b + if compressed { + if c == nil { + return nil, 0, fmt.Errorf("metadata block at %d compressed, but no compressor provided", location) + } + data, err = c.decompress(b) + if err != nil { + return nil, 0, fmt.Errorf("decompress error: %v", err) + } + } + return data, size + 2, nil + }) } // readMetadata read as many bytes of metadata as required for the given size, with the byteOffset provided as a starting diff --git a/filesystem/squashfs/squashfs.go b/filesystem/squashfs/squashfs.go index 191993b8..fd9277ee 100644 --- a/filesystem/squashfs/squashfs.go +++ b/filesystem/squashfs/squashfs.go @@ -17,6 +17,7 @@ const ( metadataBlockSize = 8 * KB minBlocksize = 4 * KB maxBlocksize = 1 * MB + defaultCacheSize = 128 * MB ) // FileSystem implements the FileSystem interface @@ -32,6 +33,7 @@ type FileSystem struct { uidsGids []uint32 xattrs *xAttrTable rootDir inode + cache *lru } // Equal compare if two filesystems are equal @@ -111,7 +113,15 @@ func Create(f util.File, size, start, blocksize int64) (*FileSystem, error) { // which allow you to work directly with partitions, rather than having to calculate (and hopefully not make any errors) // where a partition starts and ends. // -// If the provided blocksize is 0, it will use the default of 2K bytes +// If the provided blocksize is 0, it will use the default of 2K bytes. +// +// This will use a cache for the decompressed blocks of 128 MB by +// default. (You can set this with the SetCacheSize method and read +// its size with the GetCacheSize method). A block cache is essential +// for performance when reading. This implements a cache for the +// fragments (tail ends of files) and the metadata (directory +// listings) which otherwise would be read, decompressed and discarded +// many times. func Read(file util.File, size, start, blocksize int64) (*FileSystem, error) { var ( read int @@ -185,6 +195,7 @@ func Read(file util.File, size, start, blocksize int64) (*FileSystem, error) { compressor: compress, fragments: fragments, uidsGids: uidsgids, + cache: newLRU(int(defaultCacheSize) / int(s.blocksize)), } // for efficiency, read in the root inode right now rootInode, err := fs.getInode(s.rootInode.block, s.rootInode.offset, inodeBasicDirectory) @@ -200,6 +211,30 @@ func (fs *FileSystem) Type() filesystem.Type { return filesystem.TypeSquashfs } +// SetCacheSize set the maximum memory used by the block cache to cacheSize bytes. +// +// The default is 128 MB. +// +// If this is <= 0 then the cache will be disabled. +func (fs *FileSystem) SetCacheSize(cacheSize int) { + if fs.cache == nil { + return + } + blocks := cacheSize / int(fs.blocksize) + if blocks <= 0 { + blocks = 0 + } + fs.cache.setMaxBlocks(blocks) +} + +// GetCacheSize get the maximum memory used by the block cache in bytes. +func (fs *FileSystem) GetCacheSize() int { + if fs.cache == nil { + return 0 + } + return fs.cache.maxBlocks * int(fs.blocksize) +} + // Mkdir make a directory at the given path. It is equivalent to `mkdir -p`, i.e. idempotent, in that: // // * It will make the entire tree path if it does not exist @@ -456,7 +491,7 @@ func (fs *FileSystem) getInode(blockOffset uint32, byteOffset uint16, iType inod size = inodeTypeToSize(iType) // Read more data if necessary (quite rare) if size > len(uncompressed) { - uncompressed, err = readMetadata(fs.file, fs.compressor, int64(fs.superblock.inodeTableStart), blockOffset, byteOffset, size) + uncompressed, err = fs.readMetadata(fs.file, fs.compressor, int64(fs.superblock.inodeTableStart), blockOffset, byteOffset, size) if err != nil { return nil, fmt.Errorf("error reading block at position %d: %v", blockOffset, err) } @@ -533,25 +568,32 @@ func (fs *FileSystem) readFragment(index, offset uint32, fragmentSize int64) ([] return nil, fmt.Errorf("cannot find fragment block with index %d", index) } fragmentInfo := fs.fragments[index] - // figure out the size of the compressed block and if it is compressed - b := make([]byte, fragmentInfo.size) - read, err := fs.file.ReadAt(b, int64(fragmentInfo.start)) - if err != nil && err != io.EOF { - return nil, fmt.Errorf("unable to read fragment block %d: %v", index, err) - } - if read != len(b) { - return nil, fmt.Errorf("read %d instead of expected %d bytes for fragment block %d", read, len(b), index) - } - - data := b - if fragmentInfo.compressed { - if fs.compressor == nil { - return nil, fmt.Errorf("fragment compressed but do not have valid compressor") + pos := int64(fragmentInfo.start) + data, _, err := fs.cache.get(pos, func() (data []byte, size uint16, err error) { + // figure out the size of the compressed block and if it is compressed + b := make([]byte, fragmentInfo.size) + read, err := fs.file.ReadAt(b, pos) + if err != nil && err != io.EOF { + return nil, 0, fmt.Errorf("unable to read fragment block %d: %v", index, err) } - data, err = fs.compressor.decompress(b) - if err != nil { - return nil, fmt.Errorf("decompress error: %v", err) + if read != len(b) { + return nil, 0, fmt.Errorf("read %d instead of expected %d bytes for fragment block %d", read, len(b), index) + } + + data = b + if fragmentInfo.compressed { + if fs.compressor == nil { + return nil, 0, fmt.Errorf("fragment compressed but do not have valid compressor") + } + data, err = fs.compressor.decompress(b) + if err != nil { + return nil, 0, fmt.Errorf("decompress error: %v", err) + } } + return data, 0, nil + }) + if err != nil { + return nil, err } // now get the data from the offset return data[offset : int64(offset)+fragmentSize], nil diff --git a/filesystem/squashfs/squashfs_test.go b/filesystem/squashfs/squashfs_test.go index a0ca729d..236d98e6 100644 --- a/filesystem/squashfs/squashfs_test.go +++ b/filesystem/squashfs/squashfs_test.go @@ -68,6 +68,29 @@ func TestSquashfsType(t *testing.T) { } } +func TestSquashfsSetCacheSize(t *testing.T) { + fs, err := getValidSquashfsFSReadOnly() + if err != nil { + t.Fatalf("Failed to get read-only squashfs filesystem: %v", err) + } + assertCacheSize := func(want int) { + got := fs.GetCacheSize() + if want != got { + t.Errorf("Want cache size %d but got %d", want, got) + } + } + // Check we can set the Cache size for a Read FileSystem + assertCacheSize(128 * 1024 * 1024) + fs.SetCacheSize(1024 * 1024) + assertCacheSize(1024 * 1024) + fs.SetCacheSize(0) + fs.SetCacheSize(-1) + assertCacheSize(0) + // Check we can set the Cache size for a Write FileSystem + fs = &squashfs.FileSystem{} + assertCacheSize(0) +} + func TestSquashfsMkdir(t *testing.T) { t.Run("read-only", func(t *testing.T) { fs, err := getValidSquashfsFSReadOnly()