From 6079a5052776301fce40a03f1616caef717cf3bc Mon Sep 17 00:00:00 2001 From: Joachim Metz Date: Mon, 4 Jan 2021 11:34:07 +0100 Subject: [PATCH] Replaced gzip back-end with pygzipf --- .github/workflows/test_docker.yml | 4 ++-- .github/workflows/test_tox.yml | 2 +- .pylintrc | 2 +- config/appveyor/install.ps1 | 2 +- config/dpkg/control | 2 +- dependencies.ini | 9 +++++++++ dfvfs/file_io/gzip_file_io.py | 20 +++++++++++++++++++- dfvfs/lib/gzipfile.py | 8 ++++---- requirements.txt | 1 + setup.cfg | 1 + 10 files changed, 40 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index 1e5545fb..d6761922 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -17,7 +17,7 @@ jobs: - name: Install dependencies run: | dnf copr -y enable @gift/dev - dnf install -y python3 libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-cffi python3-cryptography python3-dfdatetime python3-dtfabric python3-idna python3-mock python3-pbr python3-pytsk3 python3-pyyaml python3-setuptools python3-six + dnf install -y python3 libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libgzipf-python3 libluksde-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-cffi python3-cryptography python3-dfdatetime python3-dtfabric python3-idna python3-mock python3-pbr python3-pytsk3 python3-pyyaml python3-setuptools python3-six - name: Run tests env: LANG: C.utf8 @@ -57,7 +57,7 @@ jobs: run: | add-apt-repository -y ppa:gift/dev apt-get update -q - apt-get install -y python3 libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-cffi-backend python3-cryptography python3-dfdatetime python3-distutils python3-dtfabric python3-idna python3-mock python3-pbr python3-pytsk3 python3-setuptools python3-six python3-yaml + apt-get install -y python3 libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libgzipf-python3 libluksde-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-cffi-backend python3-cryptography python3-dfdatetime python3-distutils python3-dtfabric python3-idna python3-mock python3-pbr python3-pytsk3 python3-setuptools python3-six python3-yaml - name: Run tests env: LANG: en_US.UTF-8 diff --git a/.github/workflows/test_tox.yml b/.github/workflows/test_tox.yml index 647a0212..f4349449 100644 --- a/.github/workflows/test_tox.yml +++ b/.github/workflows/test_tox.yml @@ -37,7 +37,7 @@ jobs: add-apt-repository -y ppa:deadsnakes/ppa add-apt-repository -y ppa:gift/dev apt-get update -q - apt-get install -y build-essential git python${{ matrix.python-version }} python${{ matrix.python-version }}-dev tox libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-cffi-backend python3-cryptography python3-dfdatetime python3-distutils python3-dtfabric python3-idna python3-mock python3-pbr python3-pytsk3 python3-setuptools python3-six python3-yaml + apt-get install -y build-essential git python${{ matrix.python-version }} python${{ matrix.python-version }}-dev tox libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libgzipf-python3 libluksde-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-cffi-backend python3-cryptography python3-dfdatetime python3-distutils python3-dtfabric python3-idna python3-mock python3-pbr python3-pytsk3 python3-setuptools python3-six python3-yaml - name: Run tests env: LANG: en_US.UTF-8 diff --git a/.pylintrc b/.pylintrc index fc059a3b..bf9e523b 100644 --- a/.pylintrc +++ b/.pylintrc @@ -7,7 +7,7 @@ # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code. -extension-pkg-whitelist=pybde,pyewf,pyfsapfs,pyfsext,pyfshfs,pyfsntfs,pyfsxfs,pyfvde,pyfwnt,pyluksde,pyqcow,pysigscan,pysmdev,pysmraw,pytsk3,pyvhdi,pyvmdk,pyvsgpt,pyvshadow,pyvslvm +extension-pkg-whitelist=pybde,pyewf,pyfsapfs,pyfsext,pyfshfs,pyfsntfs,pyfsxfs,pyfvde,pyfwnt,pygzipf,pyluksde,pyqcow,pysigscan,pysmdev,pysmraw,pytsk3,pyvhdi,pyvmdk,pyvsgpt,pyvshadow,pyvslvm # Specify a score threshold to be exceeded before program exits with error. fail-under=10.0 diff --git a/config/appveyor/install.ps1 b/config/appveyor/install.ps1 index 175f5998..7bd001c8 100644 --- a/config/appveyor/install.ps1 +++ b/config/appveyor/install.ps1 @@ -1,6 +1,6 @@ # Script to set up tests on AppVeyor Windows. -$Dependencies = "PyYAML cffi cryptography dfdatetime dtfabric idna libbde libewf libfsapfs libfsext libfshfs libfsntfs libfsxfs libfvde libfwnt libluksde libqcow libsigscan libsmdev libsmraw libvhdi libvmdk libvsgpt libvshadow libvslvm mock pbr pytsk3 six" +$Dependencies = "PyYAML cffi cryptography dfdatetime dtfabric idna libbde libewf libfsapfs libfsext libfshfs libfsntfs libfsxfs libfvde libfwnt libgzipf libluksde libqcow libsigscan libsmdev libsmraw libvhdi libvmdk libvsgpt libvshadow libvslvm mock pbr pytsk3 six" $Dependencies = ${Dependencies} -split " " $Output = Invoke-Expression -Command "git clone https://github.com/log2timeline/l2tdevtools.git ..\l2tdevtools 2>&1" diff --git a/config/dpkg/control b/config/dpkg/control index cad6cb41..e5433dcf 100644 --- a/config/dpkg/control +++ b/config/dpkg/control @@ -9,7 +9,7 @@ Homepage: https://github.com/log2timeline/dfvfs Package: python3-dfvfs Architecture: all -Depends: libbde-python3 (>= 20140531), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20201107), libfsext-python3 (>= 20200819), libfshfs-python3 (>= 20201103), libfsntfs-python3 (>= 20200921), libfsxfs-python3 (>= 20201114), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20160418), libluksde-python3 (>= 20200101), libqcow-python3 (>= 20201213), libsigscan-python3 (>= 20191221), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20201014), libvmdk-python3 (>= 20140421), libvsgpt-python3 (>= 20210207), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-cffi-backend (>= 1.9.1), python3-cryptography (>= 2.0.2), python3-dfdatetime (>= 20200809), python3-dtfabric (>= 20170524), python3-idna (>= 2.5), python3-pytsk3 (>= 20160721), python3-yaml (>= 3.10), ${python3:Depends}, ${misc:Depends} +Depends: libbde-python3 (>= 20140531), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20201107), libfsext-python3 (>= 20200819), libfshfs-python3 (>= 20201103), libfsntfs-python3 (>= 20200921), libfsxfs-python3 (>= 20201114), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20160418), libgzipf-python3 (>= 20201228), libluksde-python3 (>= 20200101), libqcow-python3 (>= 20201213), libsigscan-python3 (>= 20191221), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20201014), libvmdk-python3 (>= 20140421), libvsgpt-python3 (>= 20210207), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-cffi-backend (>= 1.9.1), python3-cryptography (>= 2.0.2), python3-dfdatetime (>= 20200809), python3-dtfabric (>= 20170524), python3-idna (>= 2.5), python3-pytsk3 (>= 20160721), python3-yaml (>= 3.10), ${python3:Depends}, ${misc:Depends} Description: Python 3 module of dfVFS dfVFS, or Digital Forensics Virtual File System, provides read-only access to file-system objects from various storage media types and file formats. The goal diff --git a/dependencies.ini b/dependencies.ini index cda25e3e..ef08e5bf 100644 --- a/dependencies.ini +++ b/dependencies.ini @@ -101,6 +101,15 @@ pypi_name: libfwnt-python rpm_name: libfwnt-python3 version_property: get_version() +[pygzipf] +dpkg_name: libgzipf-python3 +is_optional: true +l2tbinaries_name: libgzipf +minimum_version: 20201228 +pypi_name: libgzipf-python +rpm_name: libgzipf-python3 +version_property: get_version() + [pyluksde] dpkg_name: libluksde-python3 l2tbinaries_name: libluksde diff --git a/dfvfs/file_io/gzip_file_io.py b/dfvfs/file_io/gzip_file_io.py index fdf4db36..feab0c3d 100644 --- a/dfvfs/file_io/gzip_file_io.py +++ b/dfvfs/file_io/gzip_file_io.py @@ -1,6 +1,11 @@ # -*- coding: utf-8 -*- """The gzip file-like object.""" +try: + import pygzipf +except ModuleNotFoundError: + pygzipf = None + from dfvfs.file_io import file_object_io from dfvfs.lib import errors from dfvfs.lib import gzipfile @@ -13,16 +18,23 @@ class GzipFile(file_object_io.FileObjectIO): @property def comments(self): """list(str): comments in the gzip file.""" - return [member.comment for member in self._file_object.members] + return [member.comments for member in self._file_object.members] @property def modification_times(self): """list(int): modification times stored in the gzip file.""" + if pygzipf: + return [member.get_modification_time_as_integer() + for member in self._file_object.members] + return [member.modification_time for member in self._file_object.members] @property def original_filenames(self): """list(str): original filenames stored in the gzip file.""" + if pygzipf: + return [member.name for member in self._file_object.members] + return [member.original_filename for member in self._file_object.members] @property @@ -54,6 +66,12 @@ def _OpenFileObject(self, path_spec): file_object = resolver.Resolver.OpenFileObject( path_spec.parent, resolver_context=self._resolver_context) + if pygzipf: + gzipf_file = pygzipf.file() + gzipf_file.open_file_object(file_object) + + return gzipf_file + gzip_compressed_stream = gzipfile.GzipCompressedStream() gzip_compressed_stream.Open(file_object) diff --git a/dfvfs/lib/gzipfile.py b/dfvfs/lib/gzipfile.py index b3568fb0..e3d4272a 100644 --- a/dfvfs/lib/gzipfile.py +++ b/dfvfs/lib/gzipfile.py @@ -83,7 +83,7 @@ class GzipMember(data_format.DataFormat): provides caching of gzip member data during the initial read of each member. Attributes: - comment (str): comment stored in the member. + comments (str): comments stored in the member. member_end_offset (int): offset to the end of the member in the parent file object. member_start_offset (int): offset to the start of the member in the parent @@ -153,7 +153,7 @@ def __init__( # Start offset of the cached uncompressed data of the member. self._cache_start_offset = None - self.comment = None + self.comments = None self.modification_time = None self.operating_system = None self.original_filename = None @@ -269,9 +269,9 @@ def _ReadMemberHeader(self, file_object): if member_header.flags & self._FLAG_FCOMMENT: file_offset = file_object.get_offset() string_value = self._ReadString( - file_object, file_offset, self._CSTRING, 'comment') + file_object, file_offset, self._CSTRING, 'comments') - self.comment = string_value.rstrip('\x00') + self.comments = string_value.rstrip('\x00') if member_header.flags & self._FLAG_FHCRC: file_object.read(2) diff --git a/requirements.txt b/requirements.txt index 96cb792f..81ac5071 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,7 @@ libfsntfs-python >= 20200921 libfsxfs-python >= 20201114 libfvde-python >= 20160719 libfwnt-python >= 20160418 +libgzipf-python >= 20201228 libluksde-python >= 20200101 libqcow-python >= 20201213 libsigscan-python >= 20191221 diff --git a/setup.cfg b/setup.cfg index 3f13643c..c335cfc6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,6 +26,7 @@ requires = libbde-python3 >= 20140531 libfsxfs-python3 >= 20201114 libfvde-python3 >= 20160719 libfwnt-python3 >= 20160418 + libgzipf-python3 >= 20201228 libluksde-python3 >= 20200101 libqcow-python3 >= 20201213 libsigscan-python3 >= 20191221