From 949f198adead3d048f1caf83aa49f89ace8870a2 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 17 Aug 2021 19:28:29 +0000 Subject: [PATCH 01/14] added support for license_file defined in PEP639 --- wheelfile.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/wheelfile.py b/wheelfile.py index 71703f0..7c7dac9 100644 --- a/wheelfile.py +++ b/wheelfile.py @@ -222,6 +222,10 @@ class MetaData: classifier is used, this parameter may be omitted or used to specify the particular version of the intended legal text. + license_file + The License-File is a string that is a path, relative to``.dist-info``, + to a license file. The license file content MUST be UTF-8 encoded text. + home_page URL of the home page for this distribution (project). @@ -338,7 +342,8 @@ def __init__(self, *, name: str, version: Union[str, Version], requires_externals: Optional[List[str]] = None, provides_extras: Optional[List[str]] = None, provides_dists: Optional[List[str]] = None, - obsoletes_dists: Optional[List[str]] = None + obsoletes_dists: Optional[List[str]] = None, + license_files: Optional[List[str]] = None, ): # self.metadata_version = '2.1' by property self.name = name @@ -373,6 +378,7 @@ def __init__(self, *, name: str, version: Union[str, Version], self.provides_extras = provides_extras or [] self.provides_dists = provides_dists or [] self.obsoletes_dists = obsoletes_dists or [] + self.license_files = license_files or [] __slots__ = _slots_from_params(__init__) From 546a6933afd2b525d4a5fcee2249fb7e8685ced3 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 26 Oct 2021 14:17:02 +0000 Subject: [PATCH 02/14] adding support for Requires keyword (multi-valued) --- wheelfile.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wheelfile.py b/wheelfile.py index 71703f0..dc13c3a 100644 --- a/wheelfile.py +++ b/wheelfile.py @@ -334,6 +334,7 @@ def __init__(self, *, name: str, version: Union[str, Version], platforms: Optional[List[str]] = None, supported_platforms: Optional[List[str]] = None, requires_python: Optional[str] = None, + requires: Optional[List[str]] = None, requires_dists: Optional[List[str]] = None, requires_externals: Optional[List[str]] = None, provides_extras: Optional[List[str]] = None, @@ -368,6 +369,7 @@ def __init__(self, *, name: str, version: Union[str, Version], self.supported_platforms = supported_platforms or [] self.requires_python = requires_python + self.requires = requires or [] self.requires_dists = requires_dists or [] self.requires_externals = requires_externals or [] self.provides_extras = provides_extras or [] @@ -383,7 +385,7 @@ def metadata_version(self): @classmethod def field_is_multiple_use(cls, field_name: str) -> bool: - field_name = field_name.lower().replace('-', '_').rstrip('s') + field_name = field_name.lower().replace('-', '_') if field_name in cls.__slots__ or field_name == 'keyword': return False if field_name + 's' in cls.__slots__: @@ -459,7 +461,6 @@ def __eq__(self, other): @classmethod def from_str(cls, s: str) -> 'MetaData': m = message_from_string(s) - # TODO: validate this when the rest of the versions are implemented # assert m['Metadata-Version'] == cls._metadata_version @@ -477,7 +478,6 @@ def from_str(cls, s: str) -> 'MetaData': args[attr] = m.get_all(field_name) args['description'] = m.get_payload() - return cls(**args) From 74a17510dbf95d6fb10fccdcb5f670158511a324 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 29 Nov 2021 16:54:03 +0000 Subject: [PATCH 03/14] ensure that file permissions are kept for distinfo and data directories --- wheelfile.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/wheelfile.py b/wheelfile.py index 71703f0..bf82f4d 100644 --- a/wheelfile.py +++ b/wheelfile.py @@ -1439,11 +1439,21 @@ def from_wheelfile( arcname_tail = '/'.join(arcname_tail_parts) if arcname_head == wf.distinfo_dirname: new_arcname = new_wf.distinfo_dirname + '/' + arcname_tail - new_wf.writestr(new_arcname, wf.zipfile.read(zinfo)) + + # create a new ZipInfo + new_zinfo = zipfile.ZipInfo(filename=new_arcname, date_time=zinfo.date_time) + new_zinfo.external_attr = zinfo.external_attr + + new_wf.writestr(new_zinfo, wf.zipfile.read(zinfo)) continue if arcname_head == wf.data_dirname: new_arcname = new_wf.data_dirname + '/' + arcname_tail - new_wf.writestr(new_arcname, wf.zipfile.read(zinfo)) + + # create a new ZipInfo + new_zinfo = zipfile.ZipInfo(filename=new_arcname, date_time=zinfo.date_time) + new_zinfo.external_attr = zinfo.external_attr + + new_wf.writestr(new_zinfo, wf.zipfile.read(zinfo)) continue new_wf.writestr(zinfo, wf.zipfile.read(zinfo)) From 24861ab352c83568203ea2fb78a4ae3d44928ed8 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Thu, 9 Dec 2021 14:20:52 +0000 Subject: [PATCH 04/14] appease flak8 --- wheelfile.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/wheelfile.py b/wheelfile.py index bf82f4d..8737010 100644 --- a/wheelfile.py +++ b/wheelfile.py @@ -1441,7 +1441,8 @@ def from_wheelfile( new_arcname = new_wf.distinfo_dirname + '/' + arcname_tail # create a new ZipInfo - new_zinfo = zipfile.ZipInfo(filename=new_arcname, date_time=zinfo.date_time) + new_zinfo = zipfile.ZipInfo(filename=new_arcname, + date_time=zinfo.date_time) new_zinfo.external_attr = zinfo.external_attr new_wf.writestr(new_zinfo, wf.zipfile.read(zinfo)) @@ -1450,7 +1451,8 @@ def from_wheelfile( new_arcname = new_wf.data_dirname + '/' + arcname_tail # create a new ZipInfo - new_zinfo = zipfile.ZipInfo(filename=new_arcname, date_time=zinfo.date_time) + new_zinfo = zipfile.ZipInfo(filename=new_arcname, + date_time=zinfo.date_time) new_zinfo.external_attr = zinfo.external_attr new_wf.writestr(new_zinfo, wf.zipfile.read(zinfo)) From 86b61a315537f661e5ebfcb56a53671f35b3f464 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Thu, 9 Dec 2021 14:50:50 +0000 Subject: [PATCH 05/14] ensure ZipInfo is propagated for data and distinfo directories, unmark the tests as xfail --- tests/test_wheelfile_cloning.py | 4 ---- wheelfile.py | 32 +++++++++++++++++++++++--------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/tests/test_wheelfile_cloning.py b/tests/test_wheelfile_cloning.py index 1ac6ea1..c0b8e4b 100644 --- a/tests/test_wheelfile_cloning.py +++ b/tests/test_wheelfile_cloning.py @@ -244,7 +244,6 @@ def test_zip_attributes_are_preserved_writestr(self, wf, buf, attr): assert getattr(czf, attr) == getattr(zf, attr) - @pytest.mark.xfail(reason="writestr_data does not propagate zinfo yet") @pytest.mark.parametrize("attr", PRESERVED_ZIPINFO_ATTRS) def test_zip_attributes_are_preserved_writestr_data(self, wf, buf, attr): zf = self.custom_zipinfo() @@ -255,9 +254,6 @@ def test_zip_attributes_are_preserved_writestr_data(self, wf, buf, attr): assert getattr(czf, attr) == getattr(zf, attr) - # writestr_data does not propagate zinfo yet - # skipped because it generates lots of warnings - @pytest.mark.xfail(reason="writestr_distinfo does not propagate zinfo yet") @pytest.mark.parametrize("attr", PRESERVED_ZIPINFO_ATTRS) def test_zip_attributes_are_preserved_writestr_distinfo(self, wf, buf, attr): diff --git a/wheelfile.py b/wheelfile.py index 8737010..614b80d 100644 --- a/wheelfile.py +++ b/wheelfile.py @@ -1441,9 +1441,7 @@ def from_wheelfile( new_arcname = new_wf.distinfo_dirname + '/' + arcname_tail # create a new ZipInfo - new_zinfo = zipfile.ZipInfo(filename=new_arcname, - date_time=zinfo.date_time) - new_zinfo.external_attr = zinfo.external_attr + new_zinfo = self._clone_and_rename_zipinfo(zinfo, new_arcname) new_wf.writestr(new_zinfo, wf.zipfile.read(zinfo)) continue @@ -1451,9 +1449,7 @@ def from_wheelfile( new_arcname = new_wf.data_dirname + '/' + arcname_tail # create a new ZipInfo - new_zinfo = zipfile.ZipInfo(filename=new_arcname, - date_time=zinfo.date_time) - new_zinfo.external_attr = zinfo.external_attr + new_zinfo = self._clone_and_rename_zipinfo(zinfo, new_arcname) new_wf.writestr(new_zinfo, wf.zipfile.read(zinfo)) continue @@ -1462,6 +1458,19 @@ def from_wheelfile( return new_wf + @staticmethod + def _clone_and_rename_zipinfo(self, zinfo: zipfile.ZipInfo, new_name: str): + # attributes to preserved + PRESERVED_ZIPINFO_ATTRS = ['date_time', 'compress_type', 'comment', + 'extra', 'create_system', 'create_version', + 'extract_version', 'flag_bits', 'volume', + 'internal_attr', 'external_attr'] + new_zinfo = zipfile.ZipInfo(filename=new_name) + for attr in PRESERVED_ZIPINFO_ATTRS: + setattr(new_zinfo, attr, getattr(zinfo, attr)) + + return new_zinfo + @staticmethod def _is_unnamed_or_directory(target: Union[Path, BinaryIO]) -> bool: return ( @@ -1983,7 +1992,6 @@ def _os_walk_path_to_arcpath(prefix: str, directory: str, path = os.path.join(arcname, directory[len(prefix):], stem) return path - # TODO: Make sure fields of given ZipInfo objects are propagated def writestr(self, zinfo_or_arcname: Union[zipfile.ZipInfo, str], data: Union[bytes, str], @@ -2107,7 +2115,6 @@ def write_data(self, filename: Union[str, Path], # TODO: drive letter should be stripped from the arcname the same way # ZipInfo.from_file does it - # TODO: Make sure fields of given ZipInfo objects are propagated def writestr_data(self, section: str, zinfo_or_arcname: Union[zipfile.ZipInfo, str], data: Union[bytes, str], @@ -2160,6 +2167,10 @@ def writestr_data(self, section: str, arcname = self._distinfo_path(section + '/' + arcname.lstrip('/'), kind='data') + # clone the rest of the attributes from provided zinfo + if isinstance(zinfo_or_arcname, zipfile.ZipInfo): + arcname = self._clone_and_rename_zipinfo(zinfo_or_arcname, arcname) + self.writestr(arcname, data, compress_type, compresslevel) # TODO: Lazy mode should permit writing meta here @@ -2248,7 +2259,6 @@ def write_distinfo(self, filename: Union[str, Path], self.write(filename, arcname, compress_type, compresslevel, recursive=recursive, skipdir=skipdir) - # TODO: Make sure fields of given ZipInfo objects are propagated def writestr_distinfo(self, zinfo_or_arcname: Union[zipfile.ZipInfo, str], data: Union[bytes, str], compress_type: Optional[int] = None, @@ -2306,6 +2316,10 @@ def writestr_distinfo(self, zinfo_or_arcname: Union[zipfile.ZipInfo, str], ) arcname = self._distinfo_path(arcname.lstrip('/')) + # clone the rest of the attributes from provided zinfo + if isinstance(zinfo_or_arcname, zipfile.ZipInfo): + arcname = self._clone_and_rename_zipinfo(zinfo_or_arcname, arcname) + self.writestr(arcname, data, compress_type, compresslevel) @staticmethod From 6d16e289449a16331797fb1e959b7f0fe39c3f3a Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 7 Feb 2022 17:12:54 +0000 Subject: [PATCH 06/14] supports requires and provides in metadata --- wheelfile.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/wheelfile.py b/wheelfile.py index 29afaa3..584c211 100644 --- a/wheelfile.py +++ b/wheelfile.py @@ -277,6 +277,12 @@ class MetaData: Each item may end with a semicolon followed by a PEP-496 environment markers. + requires + Package requirements + + provides + The name of the provided package + provides_extras List of names of optional features provided by a distribution. Used to specify which dependencies should be installed depending on which of @@ -338,13 +344,14 @@ def __init__(self, *, name: str, version: Union[str, Version], platforms: Optional[List[str]] = None, supported_platforms: Optional[List[str]] = None, requires_python: Optional[str] = None, - requires: Optional[List[str]] = None, requires_dists: Optional[List[str]] = None, requires_externals: Optional[List[str]] = None, provides_extras: Optional[List[str]] = None, provides_dists: Optional[List[str]] = None, obsoletes_dists: Optional[List[str]] = None, license_files: Optional[List[str]] = None, + provides: Optional[str] = None, + requires: Optional[str] = None, ): # self.metadata_version = '2.1' by property self.name = name @@ -374,13 +381,14 @@ def __init__(self, *, name: str, version: Union[str, Version], self.supported_platforms = supported_platforms or [] self.requires_python = requires_python - self.requires = requires or [] self.requires_dists = requires_dists or [] self.requires_externals = requires_externals or [] self.provides_extras = provides_extras or [] self.provides_dists = provides_dists or [] self.obsoletes_dists = obsoletes_dists or [] self.license_files = license_files or [] + self.provides = provides + self.requires = requires __slots__ = _slots_from_params(__init__) @@ -391,7 +399,9 @@ def metadata_version(self): @classmethod def field_is_multiple_use(cls, field_name: str) -> bool: - field_name = field_name.lower().replace('-', '_') + field_name = field_name.lower().replace('-', '_').rstrip('s') + if field_name in ['provide', 'require']: + return False if field_name in cls.__slots__ or field_name == 'keyword': return False if field_name + 's' in cls.__slots__: @@ -475,7 +485,7 @@ def from_str(cls, s: str) -> 'MetaData': args = {} for field_name in m.keys(): attr = cls._attr_name(field_name) - if not attr.endswith('s'): + if not attr.endswith('s') or attr in ["provides", "requires"]: args[attr] = m.get(field_name) else: if field_name == "Keywords": From bb44384d37581b65d4347c56d93d7c6236926885 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 7 Feb 2022 17:13:30 +0000 Subject: [PATCH 07/14] raise exceptions when caught --- wheelfile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wheelfile.py b/wheelfile.py index 584c211..40c40cf 100644 --- a/wheelfile.py +++ b/wheelfile.py @@ -1654,8 +1654,9 @@ def _read_distinfo(self): try: metadata = self.zipfile.read(self._distinfo_path('METADATA')) self.metadata = MetaData.from_str(metadata.decode('utf-8')) - except Exception: + except Exception as e: self.metadata = None + raise(e) try: wheeldata = self.zipfile.read(self._distinfo_path('WHEEL')) From 0b665c1ab91b4c29e03dfc13935ca7cb58819c80 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 7 Feb 2022 17:43:55 +0000 Subject: [PATCH 08/14] fixed self by cls for static method --- wheelfile.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wheelfile.py b/wheelfile.py index 614b80d..42ba740 100644 --- a/wheelfile.py +++ b/wheelfile.py @@ -1441,7 +1441,7 @@ def from_wheelfile( new_arcname = new_wf.distinfo_dirname + '/' + arcname_tail # create a new ZipInfo - new_zinfo = self._clone_and_rename_zipinfo(zinfo, new_arcname) + new_zinfo = cls._clone_and_rename_zipinfo(zinfo, new_arcname) new_wf.writestr(new_zinfo, wf.zipfile.read(zinfo)) continue @@ -1449,7 +1449,7 @@ def from_wheelfile( new_arcname = new_wf.data_dirname + '/' + arcname_tail # create a new ZipInfo - new_zinfo = self._clone_and_rename_zipinfo(zinfo, new_arcname) + new_zinfo = cls._clone_and_rename_zipinfo(zinfo, new_arcname) new_wf.writestr(new_zinfo, wf.zipfile.read(zinfo)) continue @@ -1459,7 +1459,7 @@ def from_wheelfile( return new_wf @staticmethod - def _clone_and_rename_zipinfo(self, zinfo: zipfile.ZipInfo, new_name: str): + def _clone_and_rename_zipinfo(zinfo: zipfile.ZipInfo, new_name: str): # attributes to preserved PRESERVED_ZIPINFO_ATTRS = ['date_time', 'compress_type', 'comment', 'extra', 'create_system', 'create_version', From 3fab6b6d56aeb333e34d34f8bdbef2bc4f2b0d0e Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 7 Feb 2022 17:45:36 +0000 Subject: [PATCH 09/14] Allow RECORD in subfolders, such as what is provided in 'py', which contained vendored dependencies --- wheelfile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wheelfile.py b/wheelfile.py index a75446d..53c88ba 100644 --- a/wheelfile.py +++ b/wheelfile.py @@ -715,7 +715,8 @@ def update(self, arcpath: str, buf: IO[bytes]): assert buf.tell() == 0, ( f"Stale buffer given - current position: {buf.tell()}." ) - assert not arcpath.endswith('.dist-info/RECORD'), ( + # if .dist-info/RECORD is not in a subdirectory, it is not allowed + assert "/" in arcpath.replace('.dist-info/RECORD',"") or not arcpath.endswith('.dist-info/RECORD') ( f"Attempt to add an entry for a RECORD file to the RECORD: " f"{repr(arcpath)}." ) From 2236e925c491917f29910a1770805a7afb35378e Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 7 Feb 2022 20:47:01 +0000 Subject: [PATCH 10/14] added missing comma --- wheelfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wheelfile.py b/wheelfile.py index 53c88ba..1b8a4c0 100644 --- a/wheelfile.py +++ b/wheelfile.py @@ -716,7 +716,7 @@ def update(self, arcpath: str, buf: IO[bytes]): f"Stale buffer given - current position: {buf.tell()}." ) # if .dist-info/RECORD is not in a subdirectory, it is not allowed - assert "/" in arcpath.replace('.dist-info/RECORD',"") or not arcpath.endswith('.dist-info/RECORD') ( + assert "/" in arcpath.replace('.dist-info/RECORD',"") or not arcpath.endswith('.dist-info/RECORD'), ( f"Attempt to add an entry for a RECORD file to the RECORD: " f"{repr(arcpath)}." ) From 766c4fae67245b8ceae21211feac286a86a504c4 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 7 Jun 2022 14:08:55 +0000 Subject: [PATCH 11/14] This fixes https://github.com/ComputeCanada/wheels_builder/issues/57 It avoid canonicalizing wheel names when patching existing wheels --- wheelfile.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/wheelfile.py b/wheelfile.py index 71703f0..d6784ce 100644 --- a/wheelfile.py +++ b/wheelfile.py @@ -1398,6 +1398,12 @@ def from_wheelfile( compresslevel=compresslevel, strict_timestamps=strict_timestamps, ) + # if we copy a wheel from an existing wheelfile, we should recreate the + # distinfo_prefix with the original distname and the modified version + # if we do not set _distinfo_prefix, the distname will get canonicalized through + # _distinfo_path(), and this is wrong when building a wheel from an existing one + # as it will break package discovery + new_wf._distinfo_prefix = f"{distname}-{version}." assert new_wf.wheeldata is not None # For MyPy From 6ce25bd15f10bdb34df5f385a806f9c965444936 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Thu, 11 Aug 2022 14:50:56 +0000 Subject: [PATCH 12/14] added support for License field --- wheelfile.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wheelfile.py b/wheelfile.py index d9fbfbf..b23e4d9 100644 --- a/wheelfile.py +++ b/wheelfile.py @@ -450,6 +450,8 @@ def __str__(self) -> str: m.add_header(field_name, value) elif field_name == 'Description': m.set_payload(content) + elif field_name == 'License': + m.set_payload(content) else: assert isinstance(content, str), ( f"Expected string, got {type(content)} instead: {attr_name}" From 8ea62a12d5fc3bacb3ab42647102f417cea4db13 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 12 Sep 2022 15:54:42 +0000 Subject: [PATCH 13/14] when WHEEL file is corrupted, set wheeldata to None instead of metadata --- wheelfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wheelfile.py b/wheelfile.py index 71703f0..2c87891 100644 --- a/wheelfile.py +++ b/wheelfile.py @@ -1624,7 +1624,7 @@ def _read_distinfo(self): wheeldata = self.zipfile.read(self._distinfo_path('WHEEL')) self.wheeldata = WheelData.from_str(wheeldata.decode('utf-8')) except Exception: - self.metadata = None + self.wheeldata = None try: record = self.zipfile.read(self._distinfo_path('RECORD')) From a2c6d11571a88de57840fd0834560890931873d0 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 21 Feb 2023 18:02:40 +0000 Subject: [PATCH 14/14] add support for PEP 0639's License-Expression field --- wheelfile.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/wheelfile.py b/wheelfile.py index 2325a0f..222e754 100644 --- a/wheelfile.py +++ b/wheelfile.py @@ -226,6 +226,10 @@ class MetaData: The License-File is a string that is a path, relative to``.dist-info``, to a license file. The license file content MUST be UTF-8 encoded text. + license_expression + The License-Expression optional field is specified to contain a text + string that is a valid SPDX license expression, as defined herein. + home_page URL of the home page for this distribution (project). @@ -350,6 +354,7 @@ def __init__(self, *, name: str, version: Union[str, Version], provides_dists: Optional[List[str]] = None, obsoletes_dists: Optional[List[str]] = None, license_files: Optional[List[str]] = None, + license_expression: Optional[str] = None, provides: Optional[str] = None, requires: Optional[str] = None, ): @@ -387,6 +392,7 @@ def __init__(self, *, name: str, version: Union[str, Version], self.provides_dists = provides_dists or [] self.obsoletes_dists = obsoletes_dists or [] self.license_files = license_files or [] + self.license_expression = license_expression or [] self.provides = provides self.requires = requires