Skip to content

Commit

Permalink
Replaced gzip back-end with pygzipf
Browse files Browse the repository at this point in the history
  • Loading branch information
joachimmetz committed Jan 1, 2023
1 parent bfe06d9 commit 2eed519
Show file tree
Hide file tree
Showing 10 changed files with 39 additions and 10 deletions.
6 changes: 3 additions & 3 deletions .github/workflows/test_docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
version: ['36']
version: ['37']
container:
image: registry.fedoraproject.org/fedora:${{ matrix.version }}
steps:
Expand All @@ -18,7 +18,7 @@ jobs:
- name: Install dependencies
run: |
dnf copr -y enable @gift/dev
dnf install -y @development-tools python3 python3-devel libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-cffi python3-cryptography python3-dfdatetime python3-dtfabric python3-idna python3-pytsk3 python3-pyxattr python3-pyyaml python3-setuptools
dnf install -y @development-tools python3 python3-devel libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libgzipf-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-cffi python3-cryptography python3-dfdatetime python3-dtfabric python3-idna python3-pytsk3 python3-pyxattr python3-pyyaml python3-setuptools
- name: Run tests
env:
LANG: C.utf8
Expand Down Expand Up @@ -58,7 +58,7 @@ jobs:
run: |
add-apt-repository -y ppa:gift/dev
apt-get update -q
apt-get install -y build-essential python3 python3-dev libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-cffi-backend python3-cryptography python3-dfdatetime python3-distutils python3-dtfabric python3-idna python3-pytsk3 python3-pyxattr python3-setuptools python3-yaml
apt-get install -y build-essential python3 python3-dev libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libgzipf-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-cffi-backend python3-cryptography python3-dfdatetime python3-distutils python3-dtfabric python3-idna python3-pytsk3 python3-pyxattr python3-setuptools python3-yaml
- name: Run tests
env:
LANG: en_US.UTF-8
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/test_docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ jobs:
add-apt-repository -y ppa:deadsnakes/ppa
add-apt-repository -y ppa:gift/dev
apt-get update -q
apt-get install -y build-essential git libffi-dev python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-cffi-backend python3-cryptography python3-dfdatetime python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-pyxattr python3-setuptools python3-yaml
apt-get install -y build-essential git libffi-dev python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libgzipf-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-cffi-backend python3-cryptography python3-dfdatetime python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-pyxattr python3-setuptools python3-yaml
- name: Install tox
run: |
python3 -m pip install tox
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/test_tox.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ jobs:
add-apt-repository -y ppa:deadsnakes/ppa
add-apt-repository -y ppa:gift/dev
apt-get update -q
apt-get install -y build-essential git libffi-dev python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-cffi-backend python3-cryptography python3-dfdatetime python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-pyxattr python3-setuptools python3-yaml
apt-get install -y build-essential git libffi-dev python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libgzipf-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 python3-cffi-backend python3-cryptography python3-dfdatetime python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-pyxattr python3-setuptools python3-yaml
- name: Install tox
run: |
python3 -m pip install tox
Expand Down
2 changes: 1 addition & 1 deletion config/appveyor/install.ps1
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Script to set up tests on AppVeyor Windows.

$Dependencies = "PyYAML cffi cryptography dfdatetime dtfabric idna libbde libewf libfsapfs libfsext libfsfat libfshfs libfsntfs libfsxfs libfvde libfwnt libluksde libmodi libphdi libqcow libsigscan libsmdev libsmraw libvhdi libvmdk libvsgpt libvshadow libvslvm pytsk3 xattr"
$Dependencies = "PyYAML cffi cryptography dfdatetime dtfabric idna libbde libewf libfsapfs libfsext libfsfat libfshfs libfsntfs libfsxfs libfvde libfwnt libgzipf libluksde libmodi libphdi libqcow libsigscan libsmdev libsmraw libvhdi libvmdk libvsgpt libvshadow libvslvm pytsk3 xattr"
$Dependencies = ${Dependencies} -split " "

$Output = Invoke-Expression -Command "git clone https://github.com/log2timeline/l2tdevtools.git ..\l2tdevtools 2>&1"
Expand Down
2 changes: 1 addition & 1 deletion config/dpkg/control
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ Homepage: https://github.com/log2timeline/dfvfs

Package: python3-dfvfs
Architecture: all
Depends: libbde-python3 (>= 20220121), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20220709), libfsext-python3 (>= 20220829), libfsfat-python3 (>= 20220925), libfshfs-python3 (>= 20220831), libfsntfs-python3 (>= 20211229), libfsxfs-python3 (>= 20220829), libfvde-python3 (>= 20220121), libfwnt-python3 (>= 20210717), libluksde-python3 (>= 20220121), libmodi-python3 (>= 20210405), libphdi-python3 (>= 20220228), libqcow-python3 (>= 20201213), libsigscan-python3 (>= 20191221), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20201014), libvmdk-python3 (>= 20140421), libvsgpt-python3 (>= 20211115), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-cffi-backend (>= 1.9.1), python3-cryptography (>= 2.0.2), python3-dfdatetime (>= 20221112), python3-dtfabric (>= 20220219), python3-idna (>= 2.5), python3-pytsk3 (>= 20210419), python3-pyxattr (>= 0.7.2), python3-yaml (>= 3.10), ${misc:Depends}
Depends: libbde-python3 (>= 20220121), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20220709), libfsext-python3 (>= 20220829), libfsfat-python3 (>= 20220925), libfshfs-python3 (>= 20220831), libfsntfs-python3 (>= 20211229), libfsxfs-python3 (>= 20220829), libfvde-python3 (>= 20220121), libfwnt-python3 (>= 20210717), libgzipf-python3 (>= 20221229), libluksde-python3 (>= 20220121), libmodi-python3 (>= 20210405), libphdi-python3 (>= 20220228), libqcow-python3 (>= 20201213), libsigscan-python3 (>= 20191221), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20201014), libvmdk-python3 (>= 20140421), libvsgpt-python3 (>= 20211115), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-cffi-backend (>= 1.9.1), python3-cryptography (>= 2.0.2), python3-dfdatetime (>= 20221112), python3-dtfabric (>= 20220219), python3-idna (>= 2.5), python3-pytsk3 (>= 20210419), python3-pyxattr (>= 0.7.2), python3-yaml (>= 3.10), ${misc:Depends}
Description: Python 3 module of dfVFS
dfVFS, or Digital Forensics Virtual File System, provides read-only access to
file-system objects from various storage media types and file formats. The goal
Expand Down
9 changes: 9 additions & 0 deletions dependencies.ini
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,15 @@ pypi_name: libfwnt-python
rpm_name: libfwnt-python3
version_property: get_version()

[pygzipf]
dpkg_name: libgzipf-python3
is_optional: true
l2tbinaries_name: libgzipf
minimum_version: 20221229
pypi_name: libgzipf-python
rpm_name: libgzipf-python3
version_property: get_version()

[pyluksde]
dpkg_name: libluksde-python3
l2tbinaries_name: libluksde
Expand Down
20 changes: 19 additions & 1 deletion dfvfs/file_io/gzip_file_io.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
# -*- coding: utf-8 -*-
"""The gzip file-like object."""

try:
import pygzipf
except ModuleNotFoundError:
pygzipf = None

from dfvfs.file_io import file_object_io
from dfvfs.lib import errors
from dfvfs.lib import gzipfile
Expand All @@ -13,16 +18,23 @@ class GzipFile(file_object_io.FileObjectIO):
@property
def comments(self):
"""list(str): comments in the gzip file."""
return [member.comment for member in self._file_object.members]
return [member.comments for member in self._file_object.members]

@property
def modification_times(self):
"""list(int): modification times stored in the gzip file."""
if pygzipf:
return [member.get_modification_time_as_integer()
for member in self._file_object.members]

return [member.modification_time for member in self._file_object.members]

@property
def original_filenames(self):
"""list(str): original filenames stored in the gzip file."""
if pygzipf:
return [member.name for member in self._file_object.members]

return [member.original_filename for member in self._file_object.members]

@property
Expand Down Expand Up @@ -54,6 +66,12 @@ def _OpenFileObject(self, path_spec):
file_object = resolver.Resolver.OpenFileObject(
path_spec.parent, resolver_context=self._resolver_context)

if pygzipf:
gzipf_file = pygzipf.file()
gzipf_file.open_file_object(file_object)

return gzipf_file

gzip_compressed_stream = gzipfile.GzipCompressedStream()
gzip_compressed_stream.Open(file_object)

Expand Down
4 changes: 2 additions & 2 deletions dfvfs/lib/gzipfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ class GzipMember(data_format.DataFormat):
provides caching of gzip member data during the initial read of each member.
Attributes:
comment (str): comment stored in the member.
comments (str): comments stored in the member.
member_end_offset (int): offset to the end of the member in the parent file
object.
member_start_offset (int): offset to the start of the member in the parent
Expand Down Expand Up @@ -147,7 +147,7 @@ def __init__(
# Start offset of the cached uncompressed data of the member.
self._cache_start_offset = None

self.comment = None
self.comments = None
self.modification_time = None
self.operating_system = None
self.original_filename = None
Expand Down
1 change: 1 addition & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ libfsntfs-python >= 20211229
libfsxfs-python >= 20220829
libfvde-python >= 20220121
libfwnt-python >= 20210717
libgzipf-python >= 20221229
libluksde-python >= 20220121
libmodi-python >= 20210405
libphdi-python >= 20220228
Expand Down
1 change: 1 addition & 0 deletions setup.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ requires = libbde-python3 >= 20220121
libfsxfs-python3 >= 20220829
libfvde-python3 >= 20220121
libfwnt-python3 >= 20210717
libgzipf-python3 >= 20221229
libluksde-python3 >= 20220121
libmodi-python3 >= 20210405
libphdi-python3 >= 20220228
Expand Down

0 comments on commit 2eed519

Please sign in to comment.