Replaced gzip back-end with pygzipf

log2timeline · Apr 3, 2021 · 6079a50 · 6079a50
1 parent bde3c3f
commit 6079a50
Show file tree

Hide file tree

Showing 10 changed files with 40 additions and 11 deletions.
diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml
@@ -17,7 +17,7 @@ jobs:
     - name: Install dependencies
       run: |
         dnf copr -y enable @gift/dev
-        dnf install -y python3 libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-cffi python3-cryptography python3-dfdatetime python3-dtfabric python3-idna python3-mock python3-pbr python3-pytsk3 python3-pyyaml python3-setuptools python3-six
+        dnf install -y python3 libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libgzipf-python3 libluksde-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-cffi python3-cryptography python3-dfdatetime python3-dtfabric python3-idna python3-mock python3-pbr python3-pytsk3 python3-pyyaml python3-setuptools python3-six
     - name: Run tests
       env:
         LANG: C.utf8
@@ -57,7 +57,7 @@ jobs:
       run: |
         add-apt-repository -y ppa:gift/dev
         apt-get update -q
-        apt-get install -y python3 libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-cffi-backend python3-cryptography python3-dfdatetime python3-distutils python3-dtfabric python3-idna python3-mock python3-pbr python3-pytsk3 python3-setuptools python3-six python3-yaml
+        apt-get install -y python3 libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libgzipf-python3 libluksde-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-cffi-backend python3-cryptography python3-dfdatetime python3-distutils python3-dtfabric python3-idna python3-mock python3-pbr python3-pytsk3 python3-setuptools python3-six python3-yaml
     - name: Run tests
       env:
         LANG: en_US.UTF-8

diff --git a/.github/workflows/test_tox.yml b/.github/workflows/test_tox.yml
@@ -37,7 +37,7 @@ jobs:
         add-apt-repository -y ppa:deadsnakes/ppa
         add-apt-repository -y ppa:gift/dev
         apt-get update -q
-        apt-get install -y build-essential git python${{ matrix.python-version }} python${{ matrix.python-version }}-dev tox libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-cffi-backend python3-cryptography python3-dfdatetime python3-distutils python3-dtfabric python3-idna python3-mock python3-pbr python3-pytsk3 python3-setuptools python3-six python3-yaml
+        apt-get install -y build-essential git python${{ matrix.python-version }} python${{ matrix.python-version }}-dev tox libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libgzipf-python3 libluksde-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-cffi-backend python3-cryptography python3-dfdatetime python3-distutils python3-dtfabric python3-idna python3-mock python3-pbr python3-pytsk3 python3-setuptools python3-six python3-yaml
     - name: Run tests
       env:
         LANG: en_US.UTF-8

diff --git a/.pylintrc b/.pylintrc
@@ -7,7 +7,7 @@
 # A comma-separated list of package or module names from where C extensions may
 # be loaded. Extensions are loading into the active Python interpreter and may
 # run arbitrary code.
-extension-pkg-whitelist=pybde,pyewf,pyfsapfs,pyfsext,pyfshfs,pyfsntfs,pyfsxfs,pyfvde,pyfwnt,pyluksde,pyqcow,pysigscan,pysmdev,pysmraw,pytsk3,pyvhdi,pyvmdk,pyvsgpt,pyvshadow,pyvslvm
+extension-pkg-whitelist=pybde,pyewf,pyfsapfs,pyfsext,pyfshfs,pyfsntfs,pyfsxfs,pyfvde,pyfwnt,pygzipf,pyluksde,pyqcow,pysigscan,pysmdev,pysmraw,pytsk3,pyvhdi,pyvmdk,pyvsgpt,pyvshadow,pyvslvm
 
 # Specify a score threshold to be exceeded before program exits with error.
 fail-under=10.0

diff --git a/config/appveyor/install.ps1 b/config/appveyor/install.ps1
@@ -1,6 +1,6 @@
 # Script to set up tests on AppVeyor Windows.
 
-$Dependencies = "PyYAML cffi cryptography dfdatetime dtfabric idna libbde libewf libfsapfs libfsext libfshfs libfsntfs libfsxfs libfvde libfwnt libluksde libqcow libsigscan libsmdev libsmraw libvhdi libvmdk libvsgpt libvshadow libvslvm mock pbr pytsk3 six"
+$Dependencies = "PyYAML cffi cryptography dfdatetime dtfabric idna libbde libewf libfsapfs libfsext libfshfs libfsntfs libfsxfs libfvde libfwnt libgzipf libluksde libqcow libsigscan libsmdev libsmraw libvhdi libvmdk libvsgpt libvshadow libvslvm mock pbr pytsk3 six"
 $Dependencies = ${Dependencies} -split " "
 
 $Output = Invoke-Expression -Command "git clone https://github.com/log2timeline/l2tdevtools.git ..\l2tdevtools 2>&1"

diff --git a/config/dpkg/control b/config/dpkg/control
@@ -9,7 +9,7 @@ Homepage: https://github.com/log2timeline/dfvfs
 
 Package: python3-dfvfs
 Architecture: all
-Depends: libbde-python3 (>= 20140531), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20201107), libfsext-python3 (>= 20200819), libfshfs-python3 (>= 20201103), libfsntfs-python3 (>= 20200921), libfsxfs-python3 (>= 20201114), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20160418), libluksde-python3 (>= 20200101), libqcow-python3 (>= 20201213), libsigscan-python3 (>= 20191221), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20201014), libvmdk-python3 (>= 20140421), libvsgpt-python3 (>= 20210207), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-cffi-backend (>= 1.9.1), python3-cryptography (>= 2.0.2), python3-dfdatetime (>= 20200809), python3-dtfabric (>= 20170524), python3-idna (>= 2.5), python3-pytsk3 (>= 20160721), python3-yaml (>= 3.10), ${python3:Depends}, ${misc:Depends}
+Depends: libbde-python3 (>= 20140531), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20201107), libfsext-python3 (>= 20200819), libfshfs-python3 (>= 20201103), libfsntfs-python3 (>= 20200921), libfsxfs-python3 (>= 20201114), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20160418), libgzipf-python3 (>= 20201228), libluksde-python3 (>= 20200101), libqcow-python3 (>= 20201213), libsigscan-python3 (>= 20191221), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20201014), libvmdk-python3 (>= 20140421), libvsgpt-python3 (>= 20210207), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-cffi-backend (>= 1.9.1), python3-cryptography (>= 2.0.2), python3-dfdatetime (>= 20200809), python3-dtfabric (>= 20170524), python3-idna (>= 2.5), python3-pytsk3 (>= 20160721), python3-yaml (>= 3.10), ${python3:Depends}, ${misc:Depends}
 Description: Python 3 module of dfVFS
  dfVFS, or Digital Forensics Virtual File System, provides read-only access to
  file-system objects from various storage media types and file formats. The goal

diff --git a/dependencies.ini b/dependencies.ini
@@ -101,6 +101,15 @@ pypi_name: libfwnt-python
 rpm_name: libfwnt-python3
 version_property: get_version()
 
+[pygzipf]
+dpkg_name: libgzipf-python3
+is_optional: true
+l2tbinaries_name: libgzipf
+minimum_version: 20201228
+pypi_name: libgzipf-python
+rpm_name: libgzipf-python3
+version_property: get_version()
+
 [pyluksde]
 dpkg_name: libluksde-python3
 l2tbinaries_name: libluksde

diff --git a/dfvfs/file_io/gzip_file_io.py b/dfvfs/file_io/gzip_file_io.py
@@ -1,6 +1,11 @@
 # -*- coding: utf-8 -*-
 """The gzip file-like object."""
 
+try:
+  import pygzipf
+except ModuleNotFoundError:
+  pygzipf = None
+
 from dfvfs.file_io import file_object_io
 from dfvfs.lib import errors
 from dfvfs.lib import gzipfile
@@ -13,16 +18,23 @@ class GzipFile(file_object_io.FileObjectIO):
   @property
   def comments(self):
     """list(str): comments in the gzip file."""
-    return [member.comment for member in self._file_object.members]
+    return [member.comments for member in self._file_object.members]
 
   @property
   def modification_times(self):
     """list(int): modification times stored in the gzip file."""
+    if pygzipf:
+      return [member.get_modification_time_as_integer()
+              for member in self._file_object.members]
+
     return [member.modification_time for member in self._file_object.members]
 
   @property
   def original_filenames(self):
     """list(str): original filenames stored in the gzip file."""
+    if pygzipf:
+      return [member.name for member in self._file_object.members]
+
     return [member.original_filename for member in self._file_object.members]
 
   @property
@@ -54,6 +66,12 @@ def _OpenFileObject(self, path_spec):
     file_object = resolver.Resolver.OpenFileObject(
         path_spec.parent, resolver_context=self._resolver_context)
 
+    if pygzipf:
+      gzipf_file = pygzipf.file()
+      gzipf_file.open_file_object(file_object)
+
+      return gzipf_file
+
     gzip_compressed_stream = gzipfile.GzipCompressedStream()
     gzip_compressed_stream.Open(file_object)
 

diff --git a/dfvfs/lib/gzipfile.py b/dfvfs/lib/gzipfile.py
@@ -83,7 +83,7 @@ class GzipMember(data_format.DataFormat):
   provides caching of gzip member data during the initial read of each member.
 
   Attributes:
-    comment (str): comment stored in the member.
+    comments (str): comments stored in the member.
     member_end_offset (int): offset to the end of the member in the parent file
         object.
     member_start_offset (int): offset to the start of the member in the parent
@@ -153,7 +153,7 @@ def __init__(
     # Start offset of the cached uncompressed data of the member.
     self._cache_start_offset = None
 
-    self.comment = None
+    self.comments = None
     self.modification_time = None
     self.operating_system = None
     self.original_filename = None
@@ -269,9 +269,9 @@ def _ReadMemberHeader(self, file_object):
     if member_header.flags & self._FLAG_FCOMMENT:
       file_offset = file_object.get_offset()
       string_value = self._ReadString(
-          file_object, file_offset, self._CSTRING, 'comment')
+          file_object, file_offset, self._CSTRING, 'comments')
 
-      self.comment = string_value.rstrip('\x00')
+      self.comments = string_value.rstrip('\x00')
 
     if member_header.flags & self._FLAG_FHCRC:
       file_object.read(2)

diff --git a/requirements.txt b/requirements.txt
@@ -14,6 +14,7 @@ libfsntfs-python >= 20200921
 libfsxfs-python >= 20201114
 libfvde-python >= 20160719
 libfwnt-python >= 20160418
+libgzipf-python >= 20201228
 libluksde-python >= 20200101
 libqcow-python >= 20201213
 libsigscan-python >= 20191221

diff --git a/setup.cfg b/setup.cfg
@@ -26,6 +26,7 @@ requires = libbde-python3 >= 20140531
            libfsxfs-python3 >= 20201114
            libfvde-python3 >= 20160719
            libfwnt-python3 >= 20160418
+           libgzipf-python3 >= 20201228
            libluksde-python3 >= 20200101
            libqcow-python3 >= 20201213
            libsigscan-python3 >= 20191221