From cc31cce1b9706ff1a179512223948cd95b8a8bfb Mon Sep 17 00:00:00 2001 From: LTRData Date: Fri, 5 Apr 2024 01:41:14 +0200 Subject: [PATCH] Fixed SquashFs fragment table issue https://github.com/DiscUtils/DiscUtils/issues/294 --- Library/DiscUtils.SquashFs/FragmentWriter.cs | 2 + Library/DiscUtils.SquashFs/MetablockWriter.cs | 29 ++-- .../SquashFileSystemBuilder.cs | 139 ++++-------------- Library/DiscUtils.SquashFs/SuperBlock.cs | 41 +++++- .../VfsSquashFileSystemReader.cs | 2 +- .../SquashFs/SquashFileSystemBuilderTest.cs | 32 +++- 6 files changed, 119 insertions(+), 126 deletions(-) diff --git a/Library/DiscUtils.SquashFs/FragmentWriter.cs b/Library/DiscUtils.SquashFs/FragmentWriter.cs index 156fbf101..6c9aab9c3 100644 --- a/Library/DiscUtils.SquashFs/FragmentWriter.cs +++ b/Library/DiscUtils.SquashFs/FragmentWriter.cs @@ -48,6 +48,8 @@ public FragmentWriter(BuilderContext context) public int FragmentCount { get; private set; } + public int FragmentBlocksCount => _fragmentBlocks.Count; + public void Flush() { if (_currentOffset != 0) diff --git a/Library/DiscUtils.SquashFs/MetablockWriter.cs b/Library/DiscUtils.SquashFs/MetablockWriter.cs index 861abdf4b..aacab012e 100644 --- a/Library/DiscUtils.SquashFs/MetablockWriter.cs +++ b/Library/DiscUtils.SquashFs/MetablockWriter.cs @@ -1,5 +1,5 @@ // -// Copyright (c) 2008-2011, Kenneth Bell +// Copyright (c) 2008-2024, Kenneth Bell, Olof Lagerkvist // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), @@ -26,6 +26,7 @@ using DiscUtils.Compression; using DiscUtils.Streams; using DiscUtils.Streams.Compatibility; +using LTRData.Extensions.Buffers; namespace DiscUtils.SquashFs; @@ -58,13 +59,19 @@ public void Dispose() } public void Write(byte[] buffer, int offset, int count) + => Write(buffer.AsSpan(offset, count)); + + public void Write(ReadOnlySpan buffer) { var totalStored = 0; - while (totalStored < count) + while (totalStored < buffer.Length) { - var toCopy = Math.Min(_currentBlock.Length - _currentOffset, count - totalStored); - System.Buffer.BlockCopy(buffer, offset + totalStored, _currentBlock, _currentOffset, toCopy); + var toCopy = Math.Min(_currentBlock.Length - _currentOffset, buffer.Length - totalStored); + var sourceSlice = buffer.Slice(totalStored, toCopy); + var destinationSlice = new Span(_currentBlock, _currentOffset, toCopy); + sourceSlice.CopyTo(destinationSlice); + _currentOffset += toCopy; totalStored += toCopy; @@ -83,7 +90,7 @@ internal void Persist(Stream output) NextBlock(); } - output.Write(_buffer.ToArray(), 0, (int)_buffer.Length); + output.Write(_buffer.AsSpan()); } internal long DistanceFrom(MetadataRef startPos) @@ -94,29 +101,33 @@ internal long DistanceFrom(MetadataRef startPos) private void NextBlock() { + const int SQUASHFS_COMPRESSED_BIT = 1 << 15; + var compressed = new MemoryStream(); using (var compStream = new ZlibStream(compressed, CompressionMode.Compress, true)) { compStream.Write(_currentBlock, 0, _currentOffset); } - byte[] writeData; + Span writeData; ushort writeLen; + if (compressed.Length < _currentOffset) { - writeData = compressed.ToArray(); + var compressedData = compressed.AsSpan(); + writeData = compressedData; writeLen = (ushort)compressed.Length; } else { writeData = _currentBlock; - writeLen = (ushort)(_currentOffset | 0x8000); + writeLen = (ushort)(_currentOffset | SQUASHFS_COMPRESSED_BIT); } Span header = stackalloc byte[2]; EndianUtilities.WriteBytesLittleEndian(writeLen, header); _buffer.Write(header); - _buffer.Write(writeData, 0, writeLen & 0x7FFF); + _buffer.Write(writeData.Slice(0, writeLen & 0x7FFF)); ++_currentBlockNum; } diff --git a/Library/DiscUtils.SquashFs/SquashFileSystemBuilder.cs b/Library/DiscUtils.SquashFs/SquashFileSystemBuilder.cs index 48a572b6d..a47a947d3 100644 --- a/Library/DiscUtils.SquashFs/SquashFileSystemBuilder.cs +++ b/Library/DiscUtils.SquashFs/SquashFileSystemBuilder.cs @@ -1,5 +1,5 @@ // -// Copyright (c) 2008-2011, Kenneth Bell +// Copyright (c) 2008-2024, Kenneth Bell, Olof Lagerkvist // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), @@ -30,6 +30,7 @@ using DiscUtils.Internal; using DiscUtils.Streams; using DiscUtils.Streams.Compatibility; +using LTRData.Extensions.Buffers; namespace DiscUtils.SquashFs; @@ -395,7 +396,7 @@ public override void Build(Stream output) Magic = SuperBlock.SquashFsMagic, CreationTime = DateTime.Now, BlockSize = (uint)_context.DataBlockSize, - Compression = 1 // DEFLATE + Compression = SuperBlock.CompressionType.ZLib }; superBlock.BlockSizeLog2 = (ushort)MathUtilities.Log2(superBlock.BlockSize); superBlock.MajorVersion = 4; @@ -408,8 +409,9 @@ public override void Build(Stream output) fragWriter.Flush(); superBlock.RootInode = GetRoot().InodeRef; superBlock.InodesCount = _nextInode - 1; - superBlock.FragmentsCount = (uint)fragWriter.FragmentCount; + superBlock.FragmentsCount = (uint)fragWriter.FragmentBlocksCount; superBlock.UidGidCount = (ushort)idWriter.IdCount; + superBlock.Flags = SuperBlock.SuperBlockFlags.NoXAttrs | SuperBlock.SuperBlockFlags.FragmentsAlwaysGenerated; superBlock.InodeTableStart = output.Position; inodeWriter.Persist(output); @@ -421,115 +423,33 @@ public override void Build(Stream output) superBlock.LookupTableStart = -1; superBlock.UidGidTableStart = idWriter.Persist(); superBlock.ExtendedAttrsTableStart = -1; + superBlock.BytesUsed = output.Position; // Pad to 4KB var end = MathUtilities.RoundUp(output.Position, 4 * Sizes.OneKiB); if (end != output.Position) { - var padding = new byte[(int)(end - output.Position)]; - output.Write(padding, 0, padding.Length); + Span padding = stackalloc byte[(int)(end - output.Position)]; + padding.Clear(); + output.Write(padding); } // Go back and write the superblock output.Position = 0; - var buffer = new byte[superBlock.Size]; + Span buffer = stackalloc byte[superBlock.Size]; superBlock.WriteTo(buffer); - output.Write(buffer, 0, buffer.Length); + output.Write(buffer); output.Position = end; } - /// - /// Writes the file system to an existing stream. - /// - /// The stream to write to. - /// - /// The output stream must support seeking and writing. - public async override Task BuildAsync(Stream output, CancellationToken cancellationToken) + public override Task BuildAsync(Stream output, CancellationToken cancellationToken) { - if (output == null) - { - throw new ArgumentNullException(nameof(output)); - } - - if (!output.CanWrite) - { - throw new ArgumentException("Output stream must be writable", nameof(output)); - } - - if (!output.CanSeek) - { - throw new ArgumentException("Output stream must support seeking", nameof(output)); - } - - _context = new BuilderContext - { - RawStream = output, - DataBlockSize = DefaultBlockSize, - IoBuffer = new byte[DefaultBlockSize] - }; - - var inodeWriter = new MetablockWriter(); - var dirWriter = new MetablockWriter(); - var fragWriter = new FragmentWriter(_context); - var idWriter = new IdTableWriter(_context); - - _context.AllocateInode = AllocateInode; - _context.AllocateId = idWriter.AllocateId; - _context.WriteDataBlock = WriteDataBlock; - _context.WriteFragment = fragWriter.WriteFragment; - _context.InodeWriter = inodeWriter; - _context.DirectoryWriter = dirWriter; - - _nextInode = 1; - - var superBlock = new SuperBlock - { - Magic = SuperBlock.SquashFsMagic, - CreationTime = DateTime.Now, - BlockSize = (uint)_context.DataBlockSize, - Compression = 1 // DEFLATE - }; - superBlock.BlockSizeLog2 = (ushort)MathUtilities.Log2(superBlock.BlockSize); - superBlock.MajorVersion = 4; - superBlock.MinorVersion = 0; - - output.Position = superBlock.Size; - - GetRoot().Reset(); - GetRoot().Write(_context); - fragWriter.Flush(); - superBlock.RootInode = GetRoot().InodeRef; - superBlock.InodesCount = _nextInode - 1; - superBlock.FragmentsCount = (uint)fragWriter.FragmentCount; - superBlock.UidGidCount = (ushort)idWriter.IdCount; - - superBlock.InodeTableStart = output.Position; - inodeWriter.Persist(output); + cancellationToken.ThrowIfCancellationRequested(); - superBlock.DirectoryTableStart = output.Position; - dirWriter.Persist(output); + Build(output); - superBlock.FragmentTableStart = fragWriter.Persist(); - superBlock.LookupTableStart = -1; - superBlock.UidGidTableStart = idWriter.Persist(); - superBlock.ExtendedAttrsTableStart = -1; - superBlock.BytesUsed = output.Position; - - // Pad to 4KB - var end = MathUtilities.RoundUp(output.Position, 4 * Sizes.OneKiB); - if (end != output.Position) - { - var padding = new byte[(int)(end - output.Position)]; - await output.WriteAsync(padding, cancellationToken).ConfigureAwait(false); - } - - // Go back and write the superblock - output.Position = 0; - var buffer = new byte[superBlock.Size]; - superBlock.WriteTo(buffer); - await output.WriteAsync(buffer, cancellationToken).ConfigureAwait(false); - output.Position = end; + return Task.CompletedTask; } /// @@ -552,32 +472,37 @@ private uint AllocateInode() /// a flag indicating compression (or not). /// private uint WriteDataBlock(byte[] buffer, int offset, int count) + => WriteDataBlock(buffer.AsSpan(offset, count)); + + private uint WriteDataBlock(ReadOnlySpan buffer) { + const int SQUASHFS_COMPRESSED_BIT = 1 << 24; + var compressed = new MemoryStream(); using (var compStream = new ZlibStream(compressed, CompressionMode.Compress, true)) { - compStream.Write(buffer, offset, count); + compStream.Write(buffer); } - byte[] writeData; - int writeOffset; - int writeLen; - if (compressed.Length < count) + ReadOnlySpan writeData; + int returnValue; + + if (compressed.Length < buffer.Length) { - writeData = compressed.ToArray(); - writeOffset = 0; - writeLen = (int)compressed.Length; + var compressedData = compressed.AsSpan(); + compressedData[1] = 0xda; + writeData = compressedData; + returnValue = writeData.Length; } else { writeData = buffer; - writeOffset = offset; - writeLen = count | 0x01000000; + returnValue = writeData.Length | SQUASHFS_COMPRESSED_BIT; // Flag to indicate uncompressed buffer } - _context.RawStream.Write(writeData, writeOffset, writeLen & 0xFFFFFF); + _context.RawStream.Write(writeData); - return (uint)writeLen; + return (uint)returnValue; } /// diff --git a/Library/DiscUtils.SquashFs/SuperBlock.cs b/Library/DiscUtils.SquashFs/SuperBlock.cs index 8c7390595..16071f525 100644 --- a/Library/DiscUtils.SquashFs/SuperBlock.cs +++ b/Library/DiscUtils.SquashFs/SuperBlock.cs @@ -1,5 +1,5 @@ // -// Copyright (c) 2008-2011, Kenneth Bell +// Copyright (c) 2008-2024, Kenneth Bell, Olof Lagerkvist // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), @@ -27,15 +27,42 @@ namespace DiscUtils.SquashFs; internal class SuperBlock : IByteArraySerializable { + public enum CompressionType : ushort + { + Unknown, + ZLib, + LZMA, + Lzo, + Xz, + Lz4, + ZStd + } + + [Flags] + public enum SuperBlockFlags : ushort + { + UncompressedInodes = 0x0001, + UncompressedBlocks = 0x0002, + UncompressedFragments = 0x0008, + FragmentsNotUsed = 0x0010, + FragmentsAlwaysGenerated = 0x0020, + DeduplicatedData = 0x0040, + NFSExportTableExists = 0x0080, + UncompressedXAttrs = 0x0100, + NoXAttrs = 0x0200, + CompressorOptionsPresent = 0x0400, + UncompressedIdTable = 0x0800 + } + public const uint SquashFsMagic = 0x73717368; public uint BlockSize; public ushort BlockSizeLog2; public long BytesUsed; - public ushort Compression; + public CompressionType Compression; public DateTime CreationTime; public long DirectoryTableStart; public long ExtendedAttrsTableStart; - public ushort Flags; + public SuperBlockFlags Flags; public uint FragmentsCount; public long FragmentTableStart; public uint InodesCount; @@ -63,9 +90,9 @@ public int ReadFrom(ReadOnlySpan buffer) CreationTime = DateTimeOffset.FromUnixTimeSeconds(EndianUtilities.ToUInt32LittleEndian(buffer.Slice(8))).DateTime; BlockSize = EndianUtilities.ToUInt32LittleEndian(buffer.Slice(12)); FragmentsCount = EndianUtilities.ToUInt32LittleEndian(buffer.Slice(16)); - Compression = EndianUtilities.ToUInt16LittleEndian(buffer.Slice(20)); + Compression = (CompressionType)EndianUtilities.ToUInt16LittleEndian(buffer.Slice(20)); BlockSizeLog2 = EndianUtilities.ToUInt16LittleEndian(buffer.Slice(22)); - Flags = EndianUtilities.ToUInt16LittleEndian(buffer.Slice(24)); + Flags = (SuperBlockFlags)EndianUtilities.ToUInt16LittleEndian(buffer.Slice(24)); UidGidCount = EndianUtilities.ToUInt16LittleEndian(buffer.Slice(26)); MajorVersion = EndianUtilities.ToUInt16LittleEndian(buffer.Slice(28)); MinorVersion = EndianUtilities.ToUInt16LittleEndian(buffer.Slice(30)); @@ -88,9 +115,9 @@ public void WriteTo(Span buffer) EndianUtilities.WriteBytesLittleEndian(Convert.ToUInt32(new DateTimeOffset(CreationTime).ToUnixTimeSeconds()), buffer.Slice(8)); EndianUtilities.WriteBytesLittleEndian(BlockSize, buffer.Slice(12)); EndianUtilities.WriteBytesLittleEndian(FragmentsCount, buffer.Slice(16)); - EndianUtilities.WriteBytesLittleEndian(Compression, buffer.Slice(20)); + EndianUtilities.WriteBytesLittleEndian((ushort)Compression, buffer.Slice(20)); EndianUtilities.WriteBytesLittleEndian(BlockSizeLog2, buffer.Slice(22)); - EndianUtilities.WriteBytesLittleEndian(Flags, buffer.Slice(24)); + EndianUtilities.WriteBytesLittleEndian((ushort)Flags, buffer.Slice(24)); EndianUtilities.WriteBytesLittleEndian(UidGidCount, buffer.Slice(26)); EndianUtilities.WriteBytesLittleEndian(MajorVersion, buffer.Slice(28)); EndianUtilities.WriteBytesLittleEndian(MinorVersion, buffer.Slice(30)); diff --git a/Library/DiscUtils.SquashFs/VfsSquashFileSystemReader.cs b/Library/DiscUtils.SquashFs/VfsSquashFileSystemReader.cs index 445b78daa..c0542295d 100644 --- a/Library/DiscUtils.SquashFs/VfsSquashFileSystemReader.cs +++ b/Library/DiscUtils.SquashFs/VfsSquashFileSystemReader.cs @@ -59,7 +59,7 @@ public VfsSquashFileSystemReader(Stream stream) throw new IOException("Invalid SquashFS filesystem - magic mismatch"); } - if (_context.SuperBlock.Compression != 1) + if (_context.SuperBlock.Compression != SuperBlock.CompressionType.ZLib) { throw new IOException("Unsupported compression used"); } diff --git a/Tests/LibraryTests/SquashFs/SquashFileSystemBuilderTest.cs b/Tests/LibraryTests/SquashFs/SquashFileSystemBuilderTest.cs index 960e0b835..491a2a710 100644 --- a/Tests/LibraryTests/SquashFs/SquashFileSystemBuilderTest.cs +++ b/Tests/LibraryTests/SquashFs/SquashFileSystemBuilderTest.cs @@ -21,8 +21,10 @@ // using System.IO; +using System.Linq; using DiscUtils; using DiscUtils.SquashFs; +using LibraryTests.Helpers; using Xunit; namespace LibraryTests.SquashFs @@ -35,17 +37,43 @@ public void SingleFile() var fsImage = new MemoryStream(); var builder = new SquashFileSystemBuilder(); - builder.AddFile("file", new MemoryStream(new byte[] { 1, 2, 3, 4 })); + var fileData = "Contents of a test file."u8.ToArray(); + builder.AddFile("file", fileData); builder.Build(fsImage); var reader = new SquashFileSystemReader(fsImage); Assert.Single(reader.GetFileSystemEntries("\\")); - Assert.Equal(4, reader.GetFileLength("file")); Assert.True(reader.FileExists("file")); + Assert.Equal(fileData.LongLength, reader.GetFileLength("file")); + var fileVerifyData = reader.OpenFile("file", FileMode.Open).ReadAll(); + Assert.Equal(fileData, fileVerifyData); Assert.False(reader.DirectoryExists("file")); Assert.False(reader.FileExists("otherfile")); } + [Fact] + public void TwoFiles() + { + var fsImage = new MemoryStream(); + + var builder = new SquashFileSystemBuilder(); + var fileData1 = "This is file 1.\n"u8.ToArray(); + var fileData2 = "This is file 2.\n"u8.ToArray(); + builder.AddFile("file1.txt", fileData1); + builder.AddFile("file2.txt", fileData2); + builder.Build(fsImage); + + var reader = new SquashFileSystemReader(fsImage); + Assert.Equal(2, reader.GetFileSystemEntries("\\").Count()); + Assert.Equal(fileData1.LongLength, reader.GetFileLength("file1.txt")); + var fileVerifyData1 = reader.OpenFile("file1.txt", FileMode.Open).ReadAll(); + Assert.Equal(fileData1, fileVerifyData1); + var fileVerifyData2 = reader.OpenFile("file2.txt", FileMode.Open).ReadAll(); + Assert.Equal(fileData2, fileVerifyData2); + Assert.False(reader.DirectoryExists("file1.txt")); + Assert.False(reader.FileExists("otherfile")); + } + [Fact] public void CreateDirs() {