From 0c0aadeab3ab0a65276fd7d44bc182614dcfa471 Mon Sep 17 00:00:00 2001 From: Olof Lagerkvist Date: Mon, 2 Sep 2024 17:02:37 +0200 Subject: [PATCH 1/3] Create FUNDING.yml --- .github/FUNDING.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 000000000..2888fa336 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,15 @@ +# These are supported funding model platforms + +github: LTRData # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +patreon: # Replace with a single Patreon username +open_collective: # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry +polar: # Replace with a single Polar username +buy_me_a_coffee: # Replace with a single Buy Me a Coffee username +thanks_dev: # Replace with a single thanks.dev username +custom: 'https://revolut.me/ltrdatase' # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] From f78a04fc90c092c70e457d8d85c9996f7bc75c18 Mon Sep 17 00:00:00 2001 From: Olof Lagerkvist Date: Mon, 2 Sep 2024 17:15:05 +0200 Subject: [PATCH 2/3] Update FUNDING.yml --- .github/FUNDING.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 2888fa336..f14803e3c 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,6 +1,6 @@ # These are supported funding model platforms -github: LTRData # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] patreon: # Replace with a single Patreon username open_collective: # Replace with a single Open Collective username ko_fi: # Replace with a single Ko-fi username From 0f4119868efeef3de8ff7bd22d7ac8dc05ec7b75 Mon Sep 17 00:00:00 2001 From: Alexandre Mutel Date: Tue, 3 Sep 2024 00:15:21 +0200 Subject: [PATCH 3/3] Add support other compression methods for squashfs (#14) * Update .gitignore with standard VS/Resharper and various stuffs * Add PolySharp to support init properties * Add support for other compression methods to squashfs * Fix options to only emit them if necessary. Optimize MemoryStream usage for compressed streams. --- .gitignore | 376 +++++++++++++++++- DiscUtils.sln.DotSettings | 5 +- Library/Directory.Build.props | 9 + Library/DiscUtils.SquashFs/BuilderContext.cs | 5 + .../DiscUtils.SquashFs/CompressionOptions.cs | 326 +++++++++++++++ Library/DiscUtils.SquashFs/FragmentWriter.cs | 2 +- Library/DiscUtils.SquashFs/IdTableWriter.cs | 2 +- .../DiscUtils.SquashFs/MemoryStreamHelper.cs | 44 ++ Library/DiscUtils.SquashFs/Metablock.cs | 6 + Library/DiscUtils.SquashFs/MetablockWriter.cs | 17 +- .../SquashFileSystemBuilder.cs | 56 ++- .../SquashFileSystemBuilderOptions.cs | 78 ++++ .../SquashFileSystemCompressionKind.cs | 62 +++ .../SquashFileSystemReader.cs | 9 + .../SquashFileSystemReaderOptions.cs | 43 ++ .../StreamCompressorDelegate.cs | 30 ++ Library/DiscUtils.SquashFs/SuperBlock.cs | 17 +- .../VfsSquashFileSystemReader.cs | 38 +- Tests/LibraryTests/LibraryTests.csproj | 5 +- .../SquashFs/SquashFileSystemBuilderTest.cs | 128 ++++++ 20 files changed, 1201 insertions(+), 57 deletions(-) create mode 100644 Library/DiscUtils.SquashFs/CompressionOptions.cs create mode 100644 Library/DiscUtils.SquashFs/MemoryStreamHelper.cs create mode 100644 Library/DiscUtils.SquashFs/SquashFileSystemBuilderOptions.cs create mode 100644 Library/DiscUtils.SquashFs/SquashFileSystemCompressionKind.cs create mode 100644 Library/DiscUtils.SquashFs/SquashFileSystemReaderOptions.cs create mode 100644 Library/DiscUtils.SquashFs/StreamCompressorDelegate.cs diff --git a/.gitignore b/.gitignore index 119f5ba6b..8f100b470 100644 --- a/.gitignore +++ b/.gitignore @@ -1,19 +1,375 @@ -bin/ -obj/ -Build/ -Debug/ -Release/ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore +# User-specific files +*.rsuser *.suo *.user -*.lock.json -LaunchSettings.json +*.userosscache +*.sln.docstates +# Rider +.idea/ + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Mono auto generated files +mono_crash.* + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Ww][Ii][Nn]32/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +build/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ +[Ll]ogs/ + +# Visual Studio 2015/2017 cache/options directory .vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUnit +*.VisualState.xml +TestResult.xml +nunit-*.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ + +# ASP.NET Scaffolding +ScaffoldingReadMe.txt + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_i.c +*_p.c +*_h.h +*.ilk +*.meta +*.obj +*.iobj +*.pch +*.pdb +*.ipdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*_wpftmp.csproj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace +$tf/ -# Tests -Tests/LibraryTests/TestResults/ +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Coverlet is a free, cross platform Code Coverage Tool +coverage.*[.json, .xml, .info] + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages *.nupkg +# NuGet Symbol Packages *.snupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt +*.appx +*.appxbundle +*.appxupload + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!?*.[Cc]ache/ + +# Others +ClientBin/ +~$* *~ -/testenvironments.json +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings +*.rptproj.rsuser +*- [Bb]ackup.rdl +*- [Bb]ackup ([0-9]).rdl +*- [Bb]ackup ([0-9][0-9]).rdl + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# CodeRush personal settings +.cr/personal + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ + +# Local History for Visual Studio +.localhistory/ + +# BeatPulse healthcheck temp database +healthchecksdb + +# Backup folder for Package Reference Convert tool in Visual Studio 2017 +MigrationBackup/ + +# Ionide (cross platform F# VS Code tools) working folder +.ionide/ + +# Rust +/lib/blake3_dotnet/target +Cargo.lock + +# Tmp folders +tmp/ +[Tt]emp/ + +# Remove artifacts produced by dotnet-releaser +artifacts-dotnet-releaser/ + +# Verify +*.received.* \ No newline at end of file diff --git a/DiscUtils.sln.DotSettings b/DiscUtils.sln.DotSettings index 27416e2dc..d0ab9c465 100644 --- a/DiscUtils.sln.DotSettings +++ b/DiscUtils.sln.DotSettings @@ -214,6 +214,8 @@ UseExplicitType <Policy Inspect="True" Prefix="_" Suffix="" Style="aaBb" /> <Policy Inspect="True" Prefix="_" Suffix="" Style="aaBb" /> + <Policy><Descriptor Staticness="Static" AccessRightKinds="Private" Description="Static readonly fields (private)"><ElementKinds><Kind Name="READONLY_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="_" Suffix="" Style="aaBb" /></Policy> + <Policy><Descriptor Staticness="Any" AccessRightKinds="Private" Description="Constant fields (private)"><ElementKinds><Kind Name="CONSTANT_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="_" Suffix="" Style="aaBb" /></Policy> True True True @@ -221,4 +223,5 @@ True True True - True \ No newline at end of file + True + True \ No newline at end of file diff --git a/Library/Directory.Build.props b/Library/Directory.Build.props index 7d234b614..1c7dfde9a 100644 --- a/Library/Directory.Build.props +++ b/Library/Directory.Build.props @@ -28,4 +28,13 @@ false false + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + \ No newline at end of file diff --git a/Library/DiscUtils.SquashFs/BuilderContext.cs b/Library/DiscUtils.SquashFs/BuilderContext.cs index 7f0e829ea..e17d5602f 100644 --- a/Library/DiscUtils.SquashFs/BuilderContext.cs +++ b/Library/DiscUtils.SquashFs/BuilderContext.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. // +using System; using System.IO; namespace DiscUtils.SquashFs; @@ -42,4 +43,8 @@ internal sealed class BuilderContext public WriteDataBlock WriteDataBlock { get; set; } public WriteFragment WriteFragment { get; set; } + + public StreamCompressorDelegate Compressor { get; set; } + + public MemoryStream SharedMemoryStream { get; set; } } \ No newline at end of file diff --git a/Library/DiscUtils.SquashFs/CompressionOptions.cs b/Library/DiscUtils.SquashFs/CompressionOptions.cs new file mode 100644 index 000000000..fd6c6f061 --- /dev/null +++ b/Library/DiscUtils.SquashFs/CompressionOptions.cs @@ -0,0 +1,326 @@ +// +// Copyright (c) 2024, Olof Lagerkvist and contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using System; +using System.IO; +using DiscUtils.Streams; +using DiscUtils.Streams.Compatibility; + +namespace DiscUtils.SquashFs; + +/// +/// Base class for compression options. +/// +public abstract class CompressionOptions : IByteArraySerializable +{ + internal CompressionOptions(SquashFileSystemCompressionKind kind) + { + Kind = kind; + } + + /// + /// The kind of compression. + /// + public SquashFileSystemCompressionKind Kind { get; } + + + /// + public virtual int Size => 0; + + /// + public virtual int ReadFrom(ReadOnlySpan buffer) => Size; + + /// + public virtual void WriteTo(Span buffer) + { + } + + internal static int ConfigureCompressionAndGetTotalSize(SuperBlock block, CompressionOptions options) + { + if (block.Compression == SquashFileSystemCompressionKind.Lz4) + { + // LZ4 always has options + block.Flags |= SuperBlock.SuperBlockFlags.CompressorOptionsPresent; + return 8 + sizeof(ushort); // Account for the preamble ushort size + } + + if (block.Compression == SquashFileSystemCompressionKind.Xz) + { + if (options != null) + { + block.Flags |= SuperBlock.SuperBlockFlags.CompressorOptionsPresent; + return options.Size + sizeof(ushort); // Account for the preamble ushort size + } + } + + return 0; + } + + internal static CompressionOptions ReadFrom(Stream stream, SuperBlock block) + { + var hasOptions = (block.Flags & SuperBlock.SuperBlockFlags.CompressorOptionsPresent) != 0; + + switch (block.Compression) + { + case SquashFileSystemCompressionKind.Lz4: + { + // LZ4 always has options + Lz4CompressionOptions opts = new(); + var size = ReadCompressionSize(stream); + if (size < opts.Size) + { + throw new InvalidDataException($"Invalid LZ4 options size {size}. Expecting at least {opts.Size}"); + } + + opts.ReadFrom(stream, size); + if (opts.Version != Lz4CompressionFormatVersion.Legacy) + { + throw new NotSupportedException($"Unsupported LZ4 version {(uint)opts.Version}"); + } + + return new Lz4CompressionOptions(); + } + + case SquashFileSystemCompressionKind.Xz: + { + XzCompressionOptions opts = new(Math.Max(Metablock.SQUASHFS_METADATA_SIZE, (int)block.BlockSize)); + if (hasOptions) + { + var size = ReadCompressionSize(stream); + if (size < opts.Size) + { + throw new InvalidDataException($"Invalid Xz options size {size}. Expecting at least {opts.Size}"); + } + + opts.ReadFrom(stream, size); + } + return opts; + } + + case SquashFileSystemCompressionKind.ZLib: + { + ThrowIfHasOptions(); + return new ZLibCompressionOptions(); + } + case SquashFileSystemCompressionKind.Lzo: + { + ThrowIfHasOptions(); + return new LzoCompressionOptions(); + } + case SquashFileSystemCompressionKind.Lzma: + { + ThrowIfHasOptions(); + return new LzmaCompressionOptions(); + } + case SquashFileSystemCompressionKind.ZStd: + { + ThrowIfHasOptions(); + return new ZStdCompressionOptions(); + } + default: + { + throw new NotSupportedException($"Unsupported compression mode {(uint)block.Compression}"); + } + } + + void ThrowIfHasOptions() + { + if (hasOptions) + { + throw new NotSupportedException($"Unsupported compression options for {block.Compression}"); + } + } + } + + internal static void WriteTo(Stream stream, SuperBlock block, CompressionOptions options) + { + if (block.Compression == SquashFileSystemCompressionKind.Lz4) + { + // LZ4 always has options + options ??= new Lz4CompressionOptions(); + Span data = stackalloc byte[options.Size]; + WriteCompressionSize(stream, (ushort)options.Size); + options.WriteTo(data); + stream.Write(data); + } + else if (block.Compression == SquashFileSystemCompressionKind.Xz) + { + if (options != null) + { + Span data = stackalloc byte[options.Size]; + WriteCompressionSize(stream, (ushort)options.Size); + options.WriteTo(data); + stream.Write(data); + } + } + } + + internal static ushort ReadCompressionSize(Stream stream) + { + Span buffer = stackalloc byte[2]; + stream.ReadExactly(buffer); + var size = EndianUtilities.ToUInt16LittleEndian(buffer); + if ((size & Metablock.SQUASHFS_COMPRESSED_BIT) == 0) // Flag to indicate uncompressed buffer + { + throw new InvalidDataException("Invalid compressed size."); + } + return (ushort)(size & Metablock.SQUASHFS_COMPRESSED_BIT_SIZE_MASK); + } + + internal static void WriteCompressionSize(Stream stream, ushort size) + { + size |= Metablock.SQUASHFS_COMPRESSED_BIT; // Flag to indicate uncompressed buffer + Span buffer = stackalloc byte[2]; + EndianUtilities.WriteBytesLittleEndian(size, buffer); + stream.Write(buffer); + } +} + +/// +/// Compression options for LZ4. +/// +public class Lz4CompressionOptions : CompressionOptions +{ + private Lz4CompressionFormatVersion _version; + private bool _highCompression; + private const uint LZ4_HC = 1; + + /// + /// Creates a new instance of the Lz4 compression options. + /// + public Lz4CompressionOptions() : base(SquashFileSystemCompressionKind.Lz4) + { + Version = Lz4CompressionFormatVersion.Legacy; + } + + /// + /// Gets or sets the version of the LZ4 compression format. Only Legacy is supported + /// + public Lz4CompressionFormatVersion Version + { + get => _version; + init => _version = value; + } + + /// + /// Gets or sets a value indicating whether high compression is enabled. + /// + public bool HighCompression + { + get => _highCompression; + init => _highCompression = value; + } + + /// + public override int Size => 8; + + /// + public override int ReadFrom(ReadOnlySpan buffer) + { + _version = (Lz4CompressionFormatVersion)EndianUtilities.ToInt32LittleEndian(buffer); + // Read flags but ignore them + _highCompression = EndianUtilities.ToInt32LittleEndian(buffer.Slice(4)) == LZ4_HC; + return Size; + } + + /// + public override void WriteTo(Span buffer) + { + EndianUtilities.WriteBytesLittleEndian((uint)_version, buffer); + EndianUtilities.WriteBytesLittleEndian(_highCompression ? LZ4_HC : 0, buffer.Slice(4)); + } +} + +/// +/// Defines the version of the LZ4 compression format. +/// +public enum Lz4CompressionFormatVersion +{ + /// + /// Undefined. + /// + Undefined = 0, + + /// + /// The legacy version of LZ4. + /// + Legacy = 1, +} + +/// +/// Compression options for XZ. +/// +public class XzCompressionOptions : CompressionOptions +{ + private int _dictionarySize; + + /// + /// Creates a new instance of the XZ compression options. + /// + /// The size of the dictionary. + public XzCompressionOptions(int dictionarySize) : base(SquashFileSystemCompressionKind.Xz) + { + _dictionarySize = dictionarySize; + } + + /// + /// Gets the size of the dictionary. + /// + public int DictionarySize => _dictionarySize; + + /// + public override int Size => 8; + + /// + public override int ReadFrom(ReadOnlySpan buffer) + { + _dictionarySize = EndianUtilities.ToInt32LittleEndian(buffer); + // Read flags but ignore them + _ = EndianUtilities.ToInt32LittleEndian(buffer.Slice(4)); + return Size; + } + + /// + public override void WriteTo(Span buffer) + { + EndianUtilities.WriteBytesLittleEndian(_dictionarySize, buffer); + EndianUtilities.WriteBytesLittleEndian((uint)0, buffer.Slice(4)); + } +} + +/// +/// Compression options for ZLib. (No options are supported). +/// +public class ZLibCompressionOptions() : CompressionOptions(SquashFileSystemCompressionKind.ZLib); + +/// +/// Compression options for LZO. (No options are supported). +/// +public class LzoCompressionOptions() : CompressionOptions(SquashFileSystemCompressionKind.Lzo); + +/// +/// Compression options for LZMA. (No options are supported). +/// +public class LzmaCompressionOptions() : CompressionOptions(SquashFileSystemCompressionKind.Lzma); + +/// +/// ZStandard compression options. (No options are supported). +/// +public class ZStdCompressionOptions() : CompressionOptions(SquashFileSystemCompressionKind.ZStd); \ No newline at end of file diff --git a/Library/DiscUtils.SquashFs/FragmentWriter.cs b/Library/DiscUtils.SquashFs/FragmentWriter.cs index 7459d0137..6a286e824 100644 --- a/Library/DiscUtils.SquashFs/FragmentWriter.cs +++ b/Library/DiscUtils.SquashFs/FragmentWriter.cs @@ -98,7 +98,7 @@ internal long Persist() _fragmentBlocks[i].WriteTo(buffer, i * recordSize); } - var writer = new MetablockWriter(); + var writer = new MetablockWriter(_context); writer.Write(buffer, 0, _fragmentBlocks.Count * recordSize); writer.Persist(_context.RawStream); } diff --git a/Library/DiscUtils.SquashFs/IdTableWriter.cs b/Library/DiscUtils.SquashFs/IdTableWriter.cs index 8e5a8148e..860d958a3 100644 --- a/Library/DiscUtils.SquashFs/IdTableWriter.cs +++ b/Library/DiscUtils.SquashFs/IdTableWriter.cs @@ -79,7 +79,7 @@ internal long Persist() // Persist the actual Id's var blockPos = _context.RawStream.Position; - var writer = new MetablockWriter(); + var writer = new MetablockWriter(_context); writer.Write(_context.IoBuffer, 0, _ids.Count * 4); writer.Persist(_context.RawStream); diff --git a/Library/DiscUtils.SquashFs/MemoryStreamHelper.cs b/Library/DiscUtils.SquashFs/MemoryStreamHelper.cs new file mode 100644 index 000000000..05b36768e --- /dev/null +++ b/Library/DiscUtils.SquashFs/MemoryStreamHelper.cs @@ -0,0 +1,44 @@ +// +// Copyright (c) 2024, Olof Lagerkvist and contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using System; +using System.IO; + +namespace DiscUtils.SquashFs; + +internal static class MemoryStreamHelper +{ + public static MemoryStream CreateWithFixedCapacity(int capacity) + { +#if NET6_0_OR_GREATER + var array = GC.AllocateUninitializedArray(capacity); +#else + + var array = new byte[capacity]; +#endif + return new MemoryStream(array, 0, capacity, true, true); + } + + public static MemoryStream Initialize(MemoryStream stream) + { + stream.SetLength(0); + return stream; + } +} \ No newline at end of file diff --git a/Library/DiscUtils.SquashFs/Metablock.cs b/Library/DiscUtils.SquashFs/Metablock.cs index 71047b135..42dcd3298 100644 --- a/Library/DiscUtils.SquashFs/Metablock.cs +++ b/Library/DiscUtils.SquashFs/Metablock.cs @@ -26,5 +26,11 @@ namespace DiscUtils.SquashFs; internal sealed class Metablock : Block { + public const int SQUASHFS_COMPRESSED_BIT = 1 << 15; + + public const int SQUASHFS_COMPRESSED_BIT_SIZE_MASK = ~SQUASHFS_COMPRESSED_BIT; + + public const int SQUASHFS_METADATA_SIZE = 8192; + public long NextBlockStart { get; set; } } \ No newline at end of file diff --git a/Library/DiscUtils.SquashFs/MetablockWriter.cs b/Library/DiscUtils.SquashFs/MetablockWriter.cs index e855deb4e..a1ebd0ae1 100644 --- a/Library/DiscUtils.SquashFs/MetablockWriter.cs +++ b/Library/DiscUtils.SquashFs/MetablockWriter.cs @@ -33,13 +33,18 @@ namespace DiscUtils.SquashFs; internal sealed class MetablockWriter : IDisposable { private MemoryStream _buffer; + private readonly StreamCompressorDelegate _compressor; + private readonly MemoryStream _sharedMemoryStream; private readonly byte[] _currentBlock; private int _currentBlockNum; private int _currentOffset; - public MetablockWriter() + public MetablockWriter(BuilderContext context) { + _compressor = context.Compressor; + _sharedMemoryStream = context.SharedMemoryStream; + _currentBlock = new byte[8 * 1024]; _buffer = new MemoryStream(); } @@ -98,10 +103,8 @@ internal long DistanceFrom(MetadataRef startPos) private void NextBlock() { - const int SQUASHFS_COMPRESSED_BIT = 1 << 15; - - var compressed = new MemoryStream(); - using (var compStream = new ZlibStream(compressed, CompressionMode.Compress, true)) + var compressed = MemoryStreamHelper.Initialize(_sharedMemoryStream); + using (var compStream = _compressor(compressed)) { compStream.Write(_currentBlock, 0, _currentOffset); } @@ -118,13 +121,13 @@ private void NextBlock() else { writeData = _currentBlock; - writeLen = (ushort)(_currentOffset | SQUASHFS_COMPRESSED_BIT); + writeLen = (ushort)(_currentOffset | Metablock.SQUASHFS_COMPRESSED_BIT); } Span header = stackalloc byte[2]; EndianUtilities.WriteBytesLittleEndian(writeLen, header); _buffer.Write(header); - _buffer.Write(writeData.Slice(0, writeLen & 0x7FFF)); + _buffer.Write(writeData.Slice(0, writeLen & Metablock.SQUASHFS_COMPRESSED_BIT_SIZE_MASK)); ++_currentBlockNum; } diff --git a/Library/DiscUtils.SquashFs/SquashFileSystemBuilder.cs b/Library/DiscUtils.SquashFs/SquashFileSystemBuilder.cs index 87c4228a6..4f855b869 100644 --- a/Library/DiscUtils.SquashFs/SquashFileSystemBuilder.cs +++ b/Library/DiscUtils.SquashFs/SquashFileSystemBuilder.cs @@ -39,11 +39,13 @@ namespace DiscUtils.SquashFs; /// public sealed class SquashFileSystemBuilder : StreamBuilder, IFileSystemBuilder { - private const int DefaultBlockSize = 131072; + internal const int DefaultBlockSize = 131072; + private BuilderContext _context; private uint _nextInode; private BuilderDirectory _rootDir; + private readonly StreamCompressorDelegate _compressor; // Progress reporting event public event EventHandler ProgressChanged; @@ -65,7 +67,15 @@ private void AddProgress(int newFiles, int newItems) /// /// Initializes a new instance of the SquashFileSystemBuilder class. /// - public SquashFileSystemBuilder() + public SquashFileSystemBuilder() : this(null) + { + } + + /// + /// Initializes a new instance of the SquashFileSystemBuilder class. + /// + /// The options for this builder. + public SquashFileSystemBuilder(SquashFileSystemBuilderOptions options) { DefaultFilePermissions = UnixFilePermissions.OwnerRead | UnixFilePermissions.OwnerWrite | UnixFilePermissions.GroupRead | UnixFilePermissions.GroupWrite; @@ -74,8 +84,16 @@ public SquashFileSystemBuilder() UnixFilePermissions.OthersExecute; DefaultUser = 0; DefaultGroup = 0; + + Options = options ?? SquashFileSystemBuilderOptions.Default; + _compressor = Options.ResolveCompressor(); } + /// + /// Gets the options for this builder. + /// + public SquashFileSystemBuilderOptions Options { get; } + /// /// Gets or sets the default permissions used for new directories. /// @@ -399,11 +417,13 @@ public override void Build(Stream output) { RawStream = output, DataBlockSize = DefaultBlockSize, - IoBuffer = new byte[DefaultBlockSize] + IoBuffer = new byte[DefaultBlockSize], + SharedMemoryStream = MemoryStreamHelper.CreateWithFixedCapacity(DefaultBlockSize), + Compressor = _compressor }; - var inodeWriter = new MetablockWriter(); - var dirWriter = new MetablockWriter(); + var inodeWriter = new MetablockWriter(_context); + var dirWriter = new MetablockWriter(_context); var fragWriter = new FragmentWriter(_context); var idWriter = new IdTableWriter(_context); @@ -421,13 +441,16 @@ public override void Build(Stream output) Magic = SuperBlock.SquashFsMagic, CreationTime = DateTime.Now, BlockSize = (uint)_context.DataBlockSize, - Compression = SuperBlock.CompressionType.ZLib + Compression = Options.CompressionKind }; superBlock.BlockSizeLog2 = (ushort)MathUtilities.Log2(superBlock.BlockSize); superBlock.MajorVersion = 4; superBlock.MinorVersion = 0; - output.Position = superBlock.Size; + // Set flags before has it might be changed by CompressionOptions + superBlock.Flags = SuperBlock.SuperBlockFlags.NoXAttrs | SuperBlock.SuperBlockFlags.FragmentsAlwaysGenerated; + var compressionOptionsSize = CompressionOptions.ConfigureCompressionAndGetTotalSize(superBlock, Options.CompressionOptions); + output.Position = superBlock.Size + compressionOptionsSize; GetRoot().Reset(); GetRoot().Write(_context); @@ -436,7 +459,6 @@ public override void Build(Stream output) superBlock.InodesCount = _nextInode - 1; superBlock.FragmentsCount = (uint)fragWriter.FragmentBlocksCount; superBlock.UidGidCount = (ushort)idWriter.IdCount; - superBlock.Flags = SuperBlock.SuperBlockFlags.NoXAttrs | SuperBlock.SuperBlockFlags.FragmentsAlwaysGenerated; superBlock.InodeTableStart = output.Position; inodeWriter.Persist(output); @@ -465,6 +487,10 @@ public override void Build(Stream output) Span buffer = stackalloc byte[superBlock.Size]; superBlock.WriteTo(buffer); output.Write(buffer); + + // Add optional compression options + CompressionOptions.WriteTo(output, superBlock, Options.CompressionOptions); + output.Position = end; } @@ -501,10 +527,10 @@ private uint WriteDataBlock(byte[] buffer, int offset, int count) private uint WriteDataBlock(ReadOnlySpan buffer) { - const int SQUASHFS_COMPRESSED_BIT = 1 << 24; + const int SQUASHFS_COMPRESSED_BIT_BLOCK = 1 << 24; - var compressed = new MemoryStream(); - using (var compStream = new ZlibStream(compressed, CompressionMode.Compress, true)) + var compressed = MemoryStreamHelper.Initialize(_context.SharedMemoryStream); + using (var compStream = _compressor(compressed)) { compStream.Write(buffer); } @@ -515,14 +541,18 @@ private uint WriteDataBlock(ReadOnlySpan buffer) if (compressed.Length < buffer.Length) { var compressedData = compressed.AsSpan(); - compressedData[1] = 0xda; + + if (Options.CompressionKind == SquashFileSystemCompressionKind.ZLib) + { + compressedData[1] = 0xda; // Set Best compression level for zlib header + } writeData = compressedData; returnValue = writeData.Length; } else { writeData = buffer; - returnValue = writeData.Length | SQUASHFS_COMPRESSED_BIT; // Flag to indicate uncompressed buffer + returnValue = writeData.Length | SQUASHFS_COMPRESSED_BIT_BLOCK; // Flag to indicate uncompressed buffer } _context.RawStream.Write(writeData); diff --git a/Library/DiscUtils.SquashFs/SquashFileSystemBuilderOptions.cs b/Library/DiscUtils.SquashFs/SquashFileSystemBuilderOptions.cs new file mode 100644 index 000000000..50b1924a3 --- /dev/null +++ b/Library/DiscUtils.SquashFs/SquashFileSystemBuilderOptions.cs @@ -0,0 +1,78 @@ +// +// Copyright (c) 2024, Olof Lagerkvist and contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using System; +using System.IO.Compression; +using DiscUtils.Compression; + +namespace DiscUtils.SquashFs; + +/// +/// Options for building a SquashFs file system. +/// +public sealed class SquashFileSystemBuilderOptions +{ + /// + /// Default options for building a SquashFs file system. + /// + public static SquashFileSystemBuilderOptions Default = new(); + + + /// + /// Gets or sets the compression kind to use for the file system. + /// + public SquashFileSystemCompressionKind CompressionKind { get; init; } = SquashFileSystemCompressionKind.ZLib; + + /// + /// Gets or sets the compression options to use for the file system. The options must be compatible with the . Default is null. + /// + public CompressionOptions CompressionOptions { get; init; } + + /// + /// Gets or sets the compressor resolver for the file system. + /// + public Func GetCompressor { get; init; } + + /// + /// Resolves the compressor for the specified compression. + /// + /// The compressor. + /// If no compressor was found for the specified compression. + internal StreamCompressorDelegate ResolveCompressor() + { + StreamCompressorDelegate compressor = null; + if (CompressionKind == SquashFileSystemCompressionKind.ZLib) + { + compressor = static stream => new ZlibStream(stream, CompressionMode.Compress, true); + } + + if (GetCompressor != null) + { + compressor = GetCompressor(CompressionKind , CompressionOptions); + } + + if (compressor == null) + { + throw new InvalidOperationException($"No compressor found for the specified compression {CompressionOptions}"); + } + + return compressor; + } +} \ No newline at end of file diff --git a/Library/DiscUtils.SquashFs/SquashFileSystemCompressionKind.cs b/Library/DiscUtils.SquashFs/SquashFileSystemCompressionKind.cs new file mode 100644 index 000000000..6f5da19ed --- /dev/null +++ b/Library/DiscUtils.SquashFs/SquashFileSystemCompressionKind.cs @@ -0,0 +1,62 @@ +// +// Copyright (c) 2024, Olof Lagerkvist and contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +namespace DiscUtils.SquashFs; + +/// +/// The compression algorithm used in the SquashFs file system. +/// +public enum SquashFileSystemCompressionKind : ushort +{ + /// + /// The compression algorithm is unknown. + /// + Unknown, + + /// + /// The compression algorithm is ZLib. + /// + ZLib, + + /// + /// The compression algorithm is Lzma. + /// + Lzma, + + /// + /// The compression algorithm is Lzo. + /// + Lzo, + + /// + /// The compression algorithm is Xz. + /// + Xz, + + /// + /// The compression algorithm is Lz4. + /// + Lz4, + + /// + /// The compression algorithm is ZStd. + /// + ZStd +} diff --git a/Library/DiscUtils.SquashFs/SquashFileSystemReader.cs b/Library/DiscUtils.SquashFs/SquashFileSystemReader.cs index 85fbee2f9..1f2f045a6 100644 --- a/Library/DiscUtils.SquashFs/SquashFileSystemReader.cs +++ b/Library/DiscUtils.SquashFs/SquashFileSystemReader.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. // +using System; using System.IO; using DiscUtils.Streams; using DiscUtils.Vfs; @@ -42,6 +43,14 @@ public class SquashFileSystemReader : VfsFileSystemFacade, IUnixFileSystem public SquashFileSystemReader(Stream data) : base(new VfsSquashFileSystemReader(data)) {} + /// + /// Initializes a new instance of the SquashFileSystemReader class. + /// + /// The stream to read the file system image from. + /// The options for this reader. + public SquashFileSystemReader(Stream data, SquashFileSystemReaderOptions options) + : base(new VfsSquashFileSystemReader(data, options)) { } + /// /// Gets Unix file information about a file or directory. /// diff --git a/Library/DiscUtils.SquashFs/SquashFileSystemReaderOptions.cs b/Library/DiscUtils.SquashFs/SquashFileSystemReaderOptions.cs new file mode 100644 index 000000000..708383eb5 --- /dev/null +++ b/Library/DiscUtils.SquashFs/SquashFileSystemReaderOptions.cs @@ -0,0 +1,43 @@ +// +// Copyright (c) 2024, Olof Lagerkvist and contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using System; +using System.IO; + +namespace DiscUtils.SquashFs; + +/// +/// Options for the SquashFs file system reader. +/// +public sealed class SquashFileSystemReaderOptions +{ + /// + /// Gets or sets the decompressor resolver for the file system. + /// + public GetDecompressorDelegate GetDecompressor { get; init; } +} + +/// +/// Delegate to get a decompressor for a specific compression kind. +/// +/// The kind of compression +/// Optional option. Can be null. +/// A function to decode a stream of the specified compression. Returns null if not supported. +public delegate StreamCompressorDelegate GetDecompressorDelegate(SquashFileSystemCompressionKind compressionKind, CompressionOptions compressionOptions); \ No newline at end of file diff --git a/Library/DiscUtils.SquashFs/StreamCompressorDelegate.cs b/Library/DiscUtils.SquashFs/StreamCompressorDelegate.cs new file mode 100644 index 000000000..b2e7d4563 --- /dev/null +++ b/Library/DiscUtils.SquashFs/StreamCompressorDelegate.cs @@ -0,0 +1,30 @@ +// +// Copyright (c) 2024, Olof Lagerkvist and contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using System.IO; + +namespace DiscUtils.SquashFs; + +/// +/// Delegate for decompressing/compressing a stream. +/// +/// The stream to compress or decompress +/// The stream wrapped +public delegate Stream StreamCompressorDelegate(MemoryStream stream); \ No newline at end of file diff --git a/Library/DiscUtils.SquashFs/SuperBlock.cs b/Library/DiscUtils.SquashFs/SuperBlock.cs index 5bc80319c..232981910 100644 --- a/Library/DiscUtils.SquashFs/SuperBlock.cs +++ b/Library/DiscUtils.SquashFs/SuperBlock.cs @@ -27,16 +27,7 @@ namespace DiscUtils.SquashFs; internal class SuperBlock : IByteArraySerializable { - public enum CompressionType : ushort - { - Unknown, - ZLib, - LZMA, - Lzo, - Xz, - Lz4, - ZStd - } + [Flags] public enum SuperBlockFlags : ushort @@ -58,7 +49,7 @@ public enum SuperBlockFlags : ushort public uint BlockSize; public ushort BlockSizeLog2; public long BytesUsed; - public CompressionType Compression; + public SquashFileSystemCompressionKind Compression; public DateTime CreationTime; public long DirectoryTableStart; public long ExtendedAttrsTableStart; @@ -90,7 +81,7 @@ public int ReadFrom(ReadOnlySpan buffer) CreationTime = DateTimeOffset.FromUnixTimeSeconds(EndianUtilities.ToUInt32LittleEndian(buffer.Slice(8))).DateTime; BlockSize = EndianUtilities.ToUInt32LittleEndian(buffer.Slice(12)); FragmentsCount = EndianUtilities.ToUInt32LittleEndian(buffer.Slice(16)); - Compression = (CompressionType)EndianUtilities.ToUInt16LittleEndian(buffer.Slice(20)); + Compression = (SquashFileSystemCompressionKind)EndianUtilities.ToUInt16LittleEndian(buffer.Slice(20)); BlockSizeLog2 = EndianUtilities.ToUInt16LittleEndian(buffer.Slice(22)); Flags = (SuperBlockFlags)EndianUtilities.ToUInt16LittleEndian(buffer.Slice(24)); UidGidCount = EndianUtilities.ToUInt16LittleEndian(buffer.Slice(26)); @@ -130,4 +121,4 @@ public void WriteTo(Span buffer) EndianUtilities.WriteBytesLittleEndian(FragmentTableStart, buffer.Slice(80)); EndianUtilities.WriteBytesLittleEndian(LookupTableStart, buffer.Slice(88)); } -} \ No newline at end of file +} diff --git a/Library/DiscUtils.SquashFs/VfsSquashFileSystemReader.cs b/Library/DiscUtils.SquashFs/VfsSquashFileSystemReader.cs index 32d762fc5..bd58ceec3 100644 --- a/Library/DiscUtils.SquashFs/VfsSquashFileSystemReader.cs +++ b/Library/DiscUtils.SquashFs/VfsSquashFileSystemReader.cs @@ -41,7 +41,13 @@ internal class VfsSquashFileSystemReader : VfsReadOnlyFileSystem _metablockCache; - public VfsSquashFileSystemReader(Stream stream) + private readonly StreamCompressorDelegate _decompressor; + + public VfsSquashFileSystemReader(Stream stream) : this(stream, null) + { + } + + public VfsSquashFileSystemReader(Stream stream, SquashFileSystemReaderOptions options) : base(new DiscFileSystemOptions()) { _context = new Context @@ -54,14 +60,28 @@ public VfsSquashFileSystemReader(Stream stream) stream.Position = 0; _context.SuperBlock.ReadFrom(stream, _context.SuperBlock.Size); + var compressionOptions = CompressionOptions.ReadFrom(stream, _context.SuperBlock); + if (_context.SuperBlock.Magic != SuperBlock.SquashFsMagic) { throw new IOException("Invalid SquashFS filesystem - magic mismatch"); } - if (_context.SuperBlock.Compression != SuperBlock.CompressionType.ZLib) + if (_context.SuperBlock.Compression == SquashFileSystemCompressionKind.ZLib) + { + _decompressor = static stream => new ZlibStream(stream, CompressionMode.Decompress, true); + } + + // Let's override the decompressor if the user has provided one + var decompressor = options?.GetDecompressor?.Invoke(_context.SuperBlock.Compression, compressionOptions); + if (decompressor != null) + { + _decompressor = decompressor; + } + + if (_decompressor is null) { - throw new IOException("Unsupported compression used"); + throw new IOException($"Unsupported compression {_context.SuperBlock.Compression} used"); } if (_context.SuperBlock.ExtendedAttrsTableStart != -1) @@ -229,8 +249,7 @@ private Block ReadBlock(long pos, int diskLen) stream.ReadExactly(_ioBuffer, 0, readLen); - using var zlibStream = new ZlibStream(new MemoryStream(_ioBuffer, 0, readLen, false), - CompressionMode.Decompress, true); + using var zlibStream = _decompressor(new MemoryStream(_ioBuffer, 0, readLen, false, true)); block.Available = zlibStream.ReadMaximum(block.Data, 0, (int)_context.SuperBlock.BlockSize); } else @@ -257,11 +276,11 @@ private Metablock ReadMetaBlock(long pos) stream.ReadExactly(buffer); int readLen = EndianUtilities.ToUInt16LittleEndian(buffer); - var isCompressed = (readLen & 0x8000) == 0; - readLen &= 0x7FFF; + var isCompressed = (readLen & Metablock.SQUASHFS_COMPRESSED_BIT) == 0; + readLen &= Metablock.SQUASHFS_COMPRESSED_BIT_SIZE_MASK; if (readLen == 0) { - readLen = 0x8000; + readLen = Metablock.SQUASHFS_COMPRESSED_BIT; } block.NextBlockStart = pos + readLen + 2; @@ -275,8 +294,7 @@ private Metablock ReadMetaBlock(long pos) stream.ReadExactly(_ioBuffer, 0, readLen); - using var zlibStream = new ZlibStream(new MemoryStream(_ioBuffer, 0, readLen, false), - CompressionMode.Decompress, true); + using var zlibStream = _decompressor(new MemoryStream(_ioBuffer, 0, readLen, false, true)); block.Available = zlibStream.ReadMaximum(block.Data, 0, MetadataBufferSize); } else diff --git a/Tests/LibraryTests/LibraryTests.csproj b/Tests/LibraryTests/LibraryTests.csproj index 101ac1d3e..64d386cae 100644 --- a/Tests/LibraryTests/LibraryTests.csproj +++ b/Tests/LibraryTests/LibraryTests.csproj @@ -1,7 +1,7 @@  - net8.0;net48;net46 + net8.0;net48;net462 portable LibraryTests LibraryTests @@ -53,6 +53,9 @@ runtime; build; native; contentfiles; analyzers; buildtransitive + + + diff --git a/Tests/LibraryTests/SquashFs/SquashFileSystemBuilderTest.cs b/Tests/LibraryTests/SquashFs/SquashFileSystemBuilderTest.cs index fcc4c3067..252aa6cd2 100644 --- a/Tests/LibraryTests/SquashFs/SquashFileSystemBuilderTest.cs +++ b/Tests/LibraryTests/SquashFs/SquashFileSystemBuilderTest.cs @@ -20,10 +20,13 @@ // DEALINGS IN THE SOFTWARE. // +using System; using System.IO; using System.Linq; using DiscUtils; using DiscUtils.SquashFs; +using K4os.Compression.LZ4; + using LibraryTests.Helpers; using Xunit; @@ -178,4 +181,129 @@ public void BlockData() Assert.Equal(testData[i], buffer[i] /*, "Data differs at index " + i*/); } } + + [Fact] + public void BlockDataLz4() + { + var testData = new byte[(512 * 1024) + 21]; + for (var i = 0; i < testData.Length; ++i) + { + testData[i] = (byte)(i % 33); + } + + var fsImage = new MemoryStream(); + + var builder = new SquashFileSystemBuilder(new SquashFileSystemBuilderOptions() + { + + CompressionKind = SquashFileSystemCompressionKind.Lz4, + GetCompressor = (kind, options) => kind == SquashFileSystemCompressionKind.Lz4 ? static stream => new SimpleLz4Stream(stream) : null + }); + + builder.AddFile(@"file", new MemoryStream(testData)); + builder.Build(fsImage); + + var reader = new SquashFileSystemReader(fsImage, new SquashFileSystemReaderOptions() + { + GetDecompressor = (kind, options) => kind == SquashFileSystemCompressionKind.Lz4 ? static stream => new SimpleLz4Stream(stream) : null + }); + + using Stream fs = reader.OpenFile("file", FileMode.Open); + var buffer = new byte[(512 * 1024) + 1024]; + var numRead = fs.Read(buffer, 0, buffer.Length); + + Assert.Equal(testData.Length, numRead); + for (var i = 0; i < testData.Length; ++i) + { + Assert.Equal(testData[i], buffer[i] /*, "Data differs at index " + i*/); + } + } + + /// + /// A simple stream that uses LZ4 to compress and decompress data. + /// Read and Write methods expect the whole buffer to be read or written as we can only rely on LZ4 standard decode/encode buffer methods. + /// + private sealed class SimpleLz4Stream : Stream + { + private MemoryStream _inner; + + public SimpleLz4Stream(MemoryStream inner) + { + _inner = inner; + } + + public override void Flush() + { + } + + public override int Read(byte[] buffer, int offset, int count) + { + if (_inner.Position == _inner.Length) + { + return 0; + } + // This is supposed to be called only once + var srcBuffer = _inner.GetBuffer(); + var decompressed = LZ4Codec.Decode(srcBuffer, (int)_inner.Position, (int)(_inner.Length - _inner.Position), buffer, offset, count); + _inner.Position = _inner.Length; + return decompressed; + } + + public override void Write(byte[] buffer, int offset, int count) + { + _inner.SetLength(count); + var destBuffer = _inner.GetBuffer(); + var compressed = LZ4Codec.Encode(buffer, offset, count, destBuffer, 0, destBuffer.Length); + // If the dest buffer is too small, fake that we wrote the whole buffer + // In that case, the compressed buffer won't be use because it is at least equal to the uncompressed size. + if (compressed < 0) + { + compressed = count; + } + _inner.SetLength(compressed); + _inner.Position = compressed; + } + +#if NET8_0_OR_GREATER + public override int Read(Span buffer) + { + if (_inner.Position == _inner.Length) + { + return 0; + } + var srcBuffer = _inner.ToArray(); + var decompressed = LZ4Codec.Decode(new ReadOnlySpan(srcBuffer, (int)_inner.Position, (int)(_inner.Length - _inner.Position)), buffer); + _inner.Position = _inner.Length; + return decompressed; + } + public override void Write(ReadOnlySpan buffer) + { + _inner.SetLength(buffer.Length); + var destBuffer = _inner.GetBuffer(); + var compressed = LZ4Codec.Encode(buffer, destBuffer); + // If the dest buffer is too small, fake that we wrote the whole buffer + // In that case, the compressed buffer won't be use because it is at least equal to the uncompressed size. + if (compressed < 0) + { + compressed = buffer.Length; + } + _inner.SetLength(compressed); + _inner.Position = compressed; + } +#endif + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override bool CanRead => _inner.CanRead; + + public override bool CanSeek => false; + + public override bool CanWrite => _inner.CanWrite; + + public override long Length => _inner.Length; + + public override long Position { get; set; } + } }