squashfs: use the LRU cache for fragments and metadata - fixes #205

This also adds methods to set and read the size of the cache.

filesystem/squashfs/metadatablock.go

@@ -73,32 +73,34 @@ func (m *metadatablock) toBytes(c Compressor) ([]byte, error) {
 }
 
 func (fs *FileSystem) readMetaBlock(r io.ReaderAt, c Compressor, location int64) (data []byte, size uint16, err error) {
-	// read bytes off the reader to determine how big it is and if compressed
-	b := make([]byte, 2)
-	_, _ = r.ReadAt(b, location)
-	size, compressed, err := getMetadataSize(b)
-	if err != nil {
-		return nil, 0, fmt.Errorf("error getting size and compression for metadata block at %d: %v", location, err)
-	}
-	b = make([]byte, size)
-	read, err := r.ReadAt(b, location+2)
-	if err != nil && err != io.EOF {
-		return nil, 0, fmt.Errorf("unable to read metadata block of size %d at location %d: %v", size, location, err)
-	}
-	if read != len(b) {
-		return nil, 0, fmt.Errorf("read %d instead of expected %d bytes for metadata block at location %d", read, size, location)
-	}
-	data = b
-	if compressed {
-		if c == nil {
-			return nil, 0, fmt.Errorf("metadata block at %d compressed, but no compressor provided", location)
-		}
-		data, err = c.decompress(b)
+	return fs.cache.get(location, func() (data []byte, size uint16, err error) {
+		// read bytes off the reader to determine how big it is and if compressed
+		b := make([]byte, 2)
+		_, _ = r.ReadAt(b, location)
+		size, compressed, err := getMetadataSize(b)
 		if err != nil {
-			return nil, 0, fmt.Errorf("decompress error: %v", err)
+			return nil, 0, fmt.Errorf("error getting size and compression for metadata block at %d: %v", location, err)
 		}
-	}
-	return data, size + 2, nil
+		b = make([]byte, size)
+		read, err := r.ReadAt(b, location+2)
+		if err != nil && err != io.EOF {
+			return nil, 0, fmt.Errorf("unable to read metadata block of size %d at location %d: %v", size, location, err)
+		}
+		if read != len(b) {
+			return nil, 0, fmt.Errorf("read %d instead of expected %d bytes for metadata block at location %d", read, size, location)
+		}
+		data = b
+		if compressed {
+			if c == nil {
+				return nil, 0, fmt.Errorf("metadata block at %d compressed, but no compressor provided", location)
+			}
+			data, err = c.decompress(b)
+			if err != nil {
+				return nil, 0, fmt.Errorf("decompress error: %v", err)
+			}
+		}
+		return data, size + 2, nil
+	})
 }
 
 // readMetadata read as many bytes of metadata as required for the given size, with the byteOffset provided as a starting

filesystem/squashfs/squashfs.go

@@ -17,6 +17,7 @@ const (
 	metadataBlockSize = 8 * KB
 	minBlocksize      = 4 * KB
 	maxBlocksize      = 1 * MB
+	defaultCacheSize  = 128 * MB
 )
 
 // FileSystem implements the FileSystem interface
@@ -32,6 +33,7 @@ type FileSystem struct {
 	uidsGids   []uint32
 	xattrs     *xAttrTable
 	rootDir    inode
+	cache      *lru
 }
 
 // Equal compare if two filesystems are equal
@@ -111,7 +113,15 @@ func Create(f util.File, size, start, blocksize int64) (*FileSystem, error) {
 // which allow you to work directly with partitions, rather than having to calculate (and hopefully not make any errors)
 // where a partition starts and ends.
 //
-// If the provided blocksize is 0, it will use the default of 2K bytes
+// If the provided blocksize is 0, it will use the default of 2K bytes.
+//
+// This will use a cache for the decompressed blocks of 128 MB by
+// default. (You can set this with the SetCacheSize method and read
+// its size with the GetCacheSize method). A block cache is essential
+// for performance when reading. This implements a cache for the
+// fragments (tail ends of files) and the metadata (directory
+// listings) which otherwise would be read, decompressed and discarded
+// many times.
 func Read(file util.File, size, start, blocksize int64) (*FileSystem, error) {
 	var (
 		read int
@@ -185,6 +195,7 @@ func Read(file util.File, size, start, blocksize int64) (*FileSystem, error) {
 		compressor: compress,
 		fragments:  fragments,
 		uidsGids:   uidsgids,
+		cache:      newLRU(int(defaultCacheSize) / int(s.blocksize)),
 	}
 	// for efficiency, read in the root inode right now
 	rootInode, err := fs.getInode(s.rootInode.block, s.rootInode.offset, inodeBasicDirectory)
@@ -200,6 +211,30 @@ func (fs *FileSystem) Type() filesystem.Type {
 	return filesystem.TypeSquashfs
 }
 
+// SetCacheSize set the maximum memory used by the block cache to cacheSize bytes.
+//
+// The default is 128 MB.
+//
+// If this is <= 0 then the cache will be disabled.
+func (fs *FileSystem) SetCacheSize(cacheSize int) {
+	if fs.cache == nil {
+		return
+	}
+	blocks := cacheSize / int(fs.blocksize)
+	if blocks <= 0 {
+		blocks = 0
+	}
+	fs.cache.setMaxBlocks(blocks)
+}
+
+// GetCacheSize get the maximum memory used by the block cache in bytes.
+func (fs *FileSystem) GetCacheSize() int {
+	if fs.cache == nil {
+		return 0
+	}
+	return fs.cache.maxBlocks * int(fs.blocksize)
+}
+
 // Mkdir make a directory at the given path. It is equivalent to `mkdir -p`, i.e. idempotent, in that:
 //
 // * It will make the entire tree path if it does not exist
@@ -456,7 +491,7 @@ func (fs *FileSystem) getInode(blockOffset uint32, byteOffset uint16, iType inod
 		size = inodeTypeToSize(iType)
 		// Read more data if necessary (quite rare)
 		if size > len(uncompressed) {
-			uncompressed, err = readMetadata(fs.file, fs.compressor, int64(fs.superblock.inodeTableStart), blockOffset, byteOffset, size)
+			uncompressed, err = fs.readMetadata(fs.file, fs.compressor, int64(fs.superblock.inodeTableStart), blockOffset, byteOffset, size)
 			if err != nil {
 				return nil, fmt.Errorf("error reading block at position %d: %v", blockOffset, err)
 			}
@@ -533,25 +568,32 @@ func (fs *FileSystem) readFragment(index, offset uint32, fragmentSize int64) ([]
 		return nil, fmt.Errorf("cannot find fragment block with index %d", index)
 	}
 	fragmentInfo := fs.fragments[index]
-	// figure out the size of the compressed block and if it is compressed
-	b := make([]byte, fragmentInfo.size)
-	read, err := fs.file.ReadAt(b, int64(fragmentInfo.start))
-	if err != nil && err != io.EOF {
-		return nil, fmt.Errorf("unable to read fragment block %d: %v", index, err)
-	}
-	if read != len(b) {
-		return nil, fmt.Errorf("read %d instead of expected %d bytes for fragment block %d", read, len(b), index)
-	}
-
-	data := b
-	if fragmentInfo.compressed {
-		if fs.compressor == nil {
-			return nil, fmt.Errorf("fragment compressed but do not have valid compressor")
+	pos := int64(fragmentInfo.start)
+	data, _, err := fs.cache.get(pos, func() (data []byte, size uint16, err error) {
+		// figure out the size of the compressed block and if it is compressed
+		b := make([]byte, fragmentInfo.size)
+		read, err := fs.file.ReadAt(b, pos)
+		if err != nil && err != io.EOF {
+			return nil, 0, fmt.Errorf("unable to read fragment block %d: %v", index, err)
 		}
-		data, err = fs.compressor.decompress(b)
-		if err != nil {
-			return nil, fmt.Errorf("decompress error: %v", err)
+		if read != len(b) {
+			return nil, 0, fmt.Errorf("read %d instead of expected %d bytes for fragment block %d", read, len(b), index)
+		}
+
+		data = b
+		if fragmentInfo.compressed {
+			if fs.compressor == nil {
+				return nil, 0, fmt.Errorf("fragment compressed but do not have valid compressor")
+			}
+			data, err = fs.compressor.decompress(b)
+			if err != nil {
+				return nil, 0, fmt.Errorf("decompress error: %v", err)
+			}
 		}
+		return data, 0, nil
+	})
+	if err != nil {
+		return nil, err
 	}
 	// now get the data from the offset
 	return data[offset : int64(offset)+fragmentSize], nil

filesystem/squashfs/squashfs_test.go

@@ -68,6 +68,29 @@ func TestSquashfsType(t *testing.T) {
 	}
 }
 
+func TestSquashfsSetCacheSize(t *testing.T) {
+	fs, err := getValidSquashfsFSReadOnly()
+	if err != nil {
+		t.Fatalf("Failed to get read-only squashfs filesystem: %v", err)
+	}
+	assertCacheSize := func(want int) {
+		got := fs.GetCacheSize()
+		if want != got {
+			t.Errorf("Want cache size %d but got %d", want, got)
+		}
+	}
+	// Check we can set the Cache size for a Read FileSystem
+	assertCacheSize(128 * 1024 * 1024)
+	fs.SetCacheSize(1024 * 1024)
+	assertCacheSize(1024 * 1024)
+	fs.SetCacheSize(0)
+	fs.SetCacheSize(-1)
+	assertCacheSize(0)
+	// Check we can set the Cache size for a Write FileSystem
+	fs = &squashfs.FileSystem{}
+	assertCacheSize(0)
+}
+
 func TestSquashfsMkdir(t *testing.T) {
 	t.Run("read-only", func(t *testing.T) {
 		fs, err := getValidSquashfsFSReadOnly()