From 970bd45f9c4d3842a82aec9aab3d580f3ef61a68 Mon Sep 17 00:00:00 2001 From: David Cournapeau Date: Mon, 12 Jun 2017 13:10:19 +0100 Subject: [PATCH 1/6] BUG: make sure every string entry in PackageInfo is unicode by default. --- okonomiyaki/file_formats/_package_info.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/okonomiyaki/file_formats/_package_info.py b/okonomiyaki/file_formats/_package_info.py index b4cf6afa..f70e1ad9 100644 --- a/okonomiyaki/file_formats/_package_info.py +++ b/okonomiyaki/file_formats/_package_info.py @@ -143,11 +143,11 @@ def from_string(cls, s): return cls(metadata_version, name, version, **kw) def __init__(self, metadata_version, name, version, platforms=None, - supported_platforms=None, summary="", description="", - keywords=None, home_page="", download_url="", author="", - author_email="", license="", classifiers=None, requires=None, - provides=None, obsoletes=None, maintainer="", - maintainer_email="", requires_python=None, + supported_platforms=None, summary=u"", description=u"", + keywords=None, home_page=u"", download_url=u"", author=u"", + author_email=u"", license=u"", classifiers=None, requires=None, + provides=None, obsoletes=None, maintainer=u"", + maintainer_email=u"", requires_python=None, requires_external=None, requires_dist=None, provides_dist=None, obsoletes_dist=None, projects_urls=None): _ensure_supported_version(metadata_version) @@ -175,8 +175,8 @@ def __init__(self, metadata_version, name, version, platforms=None, self.obsoletes = obsoletes or () # version 1.2 - self.maintainer = maintainer or "" - self.maintainer_email = maintainer_email or "" + self.maintainer = maintainer or u"" + self.maintainer_email = maintainer_email or u"" self.requires_python = requires_python or () self.requires_external = requires_external or () self.requires_dist = requires_dist or () From 6455d84975499f5c089747cb5fe14449d2d34dcf Mon Sep 17 00:00:00 2001 From: David Cournapeau Date: Mon, 12 Jun 2017 14:52:44 +0100 Subject: [PATCH 2/6] FEAT: add basic comp support for PythonImplementation for simpler testing. --- okonomiyaki/platforms/python_implementation.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/okonomiyaki/platforms/python_implementation.py b/okonomiyaki/platforms/python_implementation.py index d03968f6..2fea76c6 100644 --- a/okonomiyaki/platforms/python_implementation.py +++ b/okonomiyaki/platforms/python_implementation.py @@ -94,6 +94,20 @@ def pep425_tag(self): def __str__(self): return "{0.abbreviated_implementation}{0.major}{0.minor}".format(self) + def __eq__(self, other): + if isinstance(other, PythonImplementation): + return ( + (self.major, self.minor, self.kind) == (other.major, other.minor, other.kind) + ) + else: + return NotImplemented + + def __ne__(self, other): + if isinstance(other, PythonImplementation): + return not self == other + else: + return NotImplemented + def _abbreviated_implementation(): """Return abbreviated implementation name.""" From 2456276601c36fa9420f51c9192031dafcf6384c Mon Sep 17 00:00:00 2001 From: David Cournapeau Date: Mon, 12 Jun 2017 14:53:19 +0100 Subject: [PATCH 3/6] BUG: ensure we keep unicode in PKG-INFO fields. --- okonomiyaki/file_formats/_package_info.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/okonomiyaki/file_formats/_package_info.py b/okonomiyaki/file_formats/_package_info.py index f70e1ad9..0fa6887c 100644 --- a/okonomiyaki/file_formats/_package_info.py +++ b/okonomiyaki/file_formats/_package_info.py @@ -296,10 +296,10 @@ def _collapse_leading_ws(header, txt): lines = [x[8:] if x.startswith(' ' * 8) else x for x in txt.strip().splitlines()] # Append a line to be char-by-char compatible with distutils - lines.append('') - return '\n'.join(lines) + lines.append(u'') + return u'\n'.join(lines) else: - return ' '.join([x.strip() for x in txt.splitlines()]) + return u' '.join([x.strip() for x in txt.splitlines()]) def _convert_if_needed(data, sha256, strict): From a432def557a7bcef1f4146edfc75a63b986a372f Mon Sep 17 00:00:00 2001 From: David Cournapeau Date: Mon, 12 Jun 2017 14:54:49 +0100 Subject: [PATCH 4/6] FEAT: initial implementation of EggMetadataV2. This new format has the following improvements over 1.x: 1. support for arbitrary version ranges 2. add support for build, test dependencies, conflicts and provides 3. normalize some nomenclature While not implemented yet, all the 2.0 metadata will be consolidated into one json file to make parsing easier as well. --- okonomiyaki/file_formats/egg_metadata_v2.py | 306 ++++++++++++++++++ .../tests/test_egg_metadata_v2.py | 141 ++++++++ 2 files changed, 447 insertions(+) create mode 100644 okonomiyaki/file_formats/egg_metadata_v2.py create mode 100644 okonomiyaki/file_formats/tests/test_egg_metadata_v2.py diff --git a/okonomiyaki/file_formats/egg_metadata_v2.py b/okonomiyaki/file_formats/egg_metadata_v2.py new file mode 100644 index 00000000..c8b8fe05 --- /dev/null +++ b/okonomiyaki/file_formats/egg_metadata_v2.py @@ -0,0 +1,306 @@ +import six +import zipfile2 + +from attr import Factory, attributes, attr +from attr.validators import instance_of, optional + +from okonomiyaki.platforms import ( + EPDPlatform, PlatformABI, PythonABI, PythonImplementation +) +from okonomiyaki.utils import compute_sha256 +from okonomiyaki.versions import EnpkgVersion, MetadataVersion + +from ._blacklist import ( + may_be_in_platform_blacklist, may_be_in_python_tag_blacklist, + may_be_in_pkg_info_blacklist +) +from ._package_info import PackageInfo, _keep_position + +from . import _egg_info + + +def text_attr(**kw): + """ An attrs.attr-like descriptor to describe fields that must be unicode. + """ + for k in ("validator", ): + if k in kw: + raise ValueError("Cannot pass '{0}' argument".format(k)) + return attr(validator=instance_of(six.text_type), **kw) + + +def dependency_type(inst, attr, value): + if not isinstance(value, tuple): + raise TypeError("Dependency must be a tuple") + for item in value: + if len(item) != 2: + raise ValueError("Each dependency value must be a pair") + name, disjunctions = item + if not isinstance(name, six.text_type): + raise ValueError( + u"Expected a text type, got {!r}".format(name) + ) + if not isinstance(disjunctions, tuple): + raise ValueError( + u"Expected tuples for disjunctions, got {!r}".format(disjunctions) + ) + for disjunction in disjunctions: + if isinstance(disjunction, tuple): + for conjunction in disjunction: + if not isinstance(conjunction, six.text_type): + raise ValueError( + u"Expected conjunction to be a string, got {!r}" + .format(conjunction) + ) + else: + raise ValueError( + u"Expected tuple for disjunction, got {!r}".format(disjunction) + ) + + +def dependency_attr(**kw): + """ An attrs.attr-like descriptor to describe fields that must be + dependency like, that is a tuple of (name, disjunctions) pairs, where each + disjunction is a tuple of strings. + """ + for k in ("validator", "default"): + if k in kw: + raise ValueError("Cannot pass '{0}' argument".format(k)) + return attr(validator=dependency_type, default=Factory(tuple)) + + +def _convert_to_metadata_version(s): + if isinstance(s, MetadataVersion): + return s + else: + return MetadataVersion.from_string(s) + + +def _convert_to_enpkg_version(s): + if isinstance(s, EnpkgVersion): + return s + else: + return EnpkgVersion.from_string(s) + + +def _convert_to_epd_platform(s): + if isinstance(s, EPDPlatform): + return s + elif isinstance(s, six.text_type): + return EPDPlatform.from_string(s) + else: + return None + + +def _convert_to_python_implementation(s): + if isinstance(s, PythonImplementation): + return s + elif isinstance(s, six.text_type): + return PythonImplementation.from_string(s) + else: + return None + + +def _convert_to_python_abi(s): + if isinstance(s, PythonABI): + return s + elif isinstance(s, six.text_type): + return PythonABI(s) + else: + return None + + +def _convert_to_platform_abi(s): + if isinstance(s, PlatformABI): + return s + elif isinstance(s, six.text_type): + return PlatformABI(s) + else: + return None + + +def _convert_to_package_info(s): + if isinstance(s, PackageInfo): + return s + elif isinstance(s, six.text_type): + return PackageInfo.from_string(s) + else: + return None + + +@attributes +class EggMetadataV2(object): + metadata_version = attr( + validator=instance_of(MetadataVersion), + convert=_convert_to_metadata_version, + ) + + _raw_name = text_attr() + + version = attr( + validator=instance_of(EnpkgVersion), convert=_convert_to_enpkg_version + ) + + epd_platform = attr( + validator=optional(instance_of(EPDPlatform)), + convert=_convert_to_epd_platform, + ) + + python_implementation = attr( + validator=optional(instance_of(PythonImplementation)), + convert=_convert_to_python_implementation + ) + + python_abi = attr( + validator=optional(instance_of(PythonABI)), + convert=_convert_to_python_abi, + ) + + platform_abi = attr( + validator=optional(instance_of(PlatformABI)), convert=_convert_to_platform_abi, + ) + + package_info = attr( + validator=optional(instance_of(PackageInfo)), convert=_convert_to_package_info, + ) + + summary = text_attr() + license = attr(validator=optional(instance_of(six.text_type))) + + runtime_dependencies = dependency_attr() + + build_dependencies = dependency_attr() + test_dependencies = dependency_attr() + conflicts = dependency_attr() + provides = dependency_attr() + + @property + def name(self): + return self._raw_name.lower() + + @property + def platform_abi_tag(self): + if self.platform_abi is None: + return None + else: + return self.platform_abi.pep425_tag + + @property + def platform_abi_tag_string(self): + return PlatformABI.pep425_tag_string(self.platform_abi) + + @property + def platform_tag(self): + """ Platform tag following PEP425, except that no platform is + represented as None and not 'any'.""" + if self.epd_platform is None: + return None + else: + return self.epd_platform.pep425_tag + + @property + def platform_tag_string(self): + return EPDPlatform.pep425_tag_string(self.epd_platform) + + @property + def python_abi_tag(self): + if self.python_abi is None: + return None + else: + return self.python_abi.pep425_tag + + @property + def python_abi_tag_string(self): + return PythonABI.pep425_tag_string(self.python_abi) + + @property + def python_tag(self): + if self.python_implementation is None: + return None + else: + return self.python_implementation.pep425_tag + + @property + def python_tag_string(self): + return PythonImplementation.pep425_tag_string(self.python_implementation) + + @staticmethod + def _may_be_in_blacklist(path): + return ( + may_be_in_platform_blacklist(path) + or may_be_in_pkg_info_blacklist(path) + or may_be_in_python_tag_blacklist(path) + ) + + @classmethod + def from_egg(cls, path_or_file, strict=True): + """ Create a EggMetadata instance from an existing Enthought egg. + + Parameters + ---------- + path: str or file-like object. + If a string, understood as the path to the egg. Otherwise, + understood as a zipfile-like object. + strict: bool + If True, will fail if metadata cannot be decoded correctly (e.g. + unicode errors in EGG-INFO/PKG-INFO). If false, will ignore those + errors, at the risk of data loss. + """ + sha256 = None + if isinstance(path_or_file, six.string_types): + if cls._may_be_in_blacklist(path_or_file): + sha256 = compute_sha256(path_or_file) + else: + with _keep_position(path_or_file.fp): + sha256 = compute_sha256(path_or_file.fp) + return cls._from_egg(path_or_file, sha256, strict) + + @classmethod + def _from_egg(cls, path_or_file, sha256, strict=True): + v1_arcname = "EGG-INFO/spec/depend" + + def _read(zp): + try: + with _keep_position(zp.fp): + zp.read(v1_arcname) + except KeyError: + raise NotImplementedError() + else: + return cls._from_v1_egg_metadata( + _egg_info.EggMetadata._from_egg(zp, sha256, strict) + ) + + if isinstance(path_or_file, six.string_types): + with zipfile2.ZipFile(path_or_file) as zp: + return _read(zp) + else: + return _read(path_or_file) + + @classmethod + def _from_v1_egg_metadata(cls, m): + if m._pkg_info is not None: + license = m.pkg_info.license + else: + license = None + return cls( + m.metadata_version, m._raw_name, m.version, m.platform, m.python, + m.abi, m.platform_abi, m._pkg_info, m.summary, license, + runtime_dependencies=_convert_requirement_to_dependencies(m.runtime_dependencies), + ) + + +def _convert_requirement_to_dependencies(requirements): + def _transformer(entry): + if len(entry.version_string) == 0: + return (entry.name, ((u"*",),)) + elif entry.build_number == -1: + return ( + entry.name, + ((u"^= {}".format(entry.version_string),),) + ) + else: + return ( + entry.name, + ((u"== {}-{}".format(entry.version_string, entry.build_number),),) + ) + + return tuple(_transformer(entry) for entry in requirements) diff --git a/okonomiyaki/file_formats/tests/test_egg_metadata_v2.py b/okonomiyaki/file_formats/tests/test_egg_metadata_v2.py new file mode 100644 index 00000000..bef46343 --- /dev/null +++ b/okonomiyaki/file_formats/tests/test_egg_metadata_v2.py @@ -0,0 +1,141 @@ +import shutil +import tempfile +import unittest + +from okonomiyaki.platforms import ( + EPDPlatform, PlatformABI, PythonABI, PythonImplementation +) +from okonomiyaki.utils.test_data import ( + MKL_10_3_RH5_X86_64, NOSE_1_3_4_RH5_X86_64, NUMPY_1_9_2_WIN_X86_64 +) +from okonomiyaki.versions import EnpkgVersion, MetadataVersion +from ..egg_metadata_v2 import EggMetadataV2 +from .common import ENSTALLER_EGG + + +M = MetadataVersion.from_string +P = EPDPlatform.from_string +V = EnpkgVersion.from_string +CP27 = PythonImplementation.from_string(u"cp27") +CP27M = PythonABI(u"cp27m") +GNU = PlatformABI(u"gnu") + + +class TestEggMetadata(unittest.TestCase): + def setUp(self): + self.tempdir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.tempdir) + + def test_v1_egg_parsing_pure_python_no_platform(self): + # Given + egg = ENSTALLER_EGG + + # When + metadata = EggMetadataV2.from_egg(egg) + + # Then + self.assertEqual(metadata.metadata_version, M(u"1.1")) + self.assertEqual(metadata.name, "enstaller") + self.assertEqual(metadata.version, V("4.5.0-1")) + self.assertIsNone(metadata.epd_platform) + self.assertIsNone(metadata.python_implementation) + self.assertIsNone(metadata.python_abi) + self.assertIsNone(metadata.platform_abi) + self.assertIsNotNone(metadata.package_info) + self.assertEqual(metadata.summary, u"") + self.assertEqual(metadata.license, u"BSD") + self.assertEqual(metadata.runtime_dependencies, ()) + self.assertEqual(metadata.build_dependencies, ()) + self.assertEqual(metadata.test_dependencies, ()) + self.assertEqual(metadata.provides, ()) + self.assertEqual(metadata.conflicts, ()) + + def test_v1_egg_parsing_pure_python_with_platform(self): + # Given + egg = NOSE_1_3_4_RH5_X86_64 + + # When + metadata = EggMetadataV2.from_egg(egg) + + # Then + self.assertEqual(metadata.metadata_version, M(u"1.3")) + self.assertEqual(metadata.name, u"nose") + self.assertEqual(metadata.version, V("1.3.4-1")) + self.assertEqual(metadata.epd_platform, P(u"rh5-x86_64")) + self.assertEqual(metadata.python_implementation, CP27) + self.assertEqual(metadata.python_abi, CP27M) + self.assertEqual(metadata.platform_abi, GNU) + self.assertIsNotNone(metadata.package_info) + self.assertEqual( + metadata.summary, + u"Extends the Python Unittest module with additional disocvery and " + "running\noptions\n" + ) + self.assertEqual(metadata.license, u"GNU LGPL") + self.assertEqual(metadata.runtime_dependencies, ()) + self.assertEqual(metadata.build_dependencies, ()) + self.assertEqual(metadata.test_dependencies, ()) + self.assertEqual(metadata.provides, ()) + self.assertEqual(metadata.conflicts, ()) + + def test_v1_egg_parsing_no_python_platform(self): + # Given + egg = MKL_10_3_RH5_X86_64 + + # When + metadata = EggMetadataV2.from_egg(egg) + + # Then + self.assertEqual(metadata._raw_name, "MKL") + self.assertEqual(metadata.name, "mkl") + + self.assertEqual(metadata.metadata_version, M(u"1.3")) + self.assertIsNone(metadata.python_abi_tag) + self.assertEqual(metadata.python_abi_tag_string, u'none') + self.assertEqual(metadata.platform_tag, 'linux_x86_64') + self.assertEqual(metadata.platform_tag_string, 'linux_x86_64') + self.assertEqual(metadata.platform_abi, GNU) + self.assertEqual(metadata.platform_abi_tag, u'gnu') + self.assertEqual(metadata.platform_abi_tag_string, u'gnu') + self.assertIsNone(metadata.python_tag) + self.assertEqual(metadata.python_tag_string, 'none') + + self.assertEqual(metadata.runtime_dependencies, ()) + self.assertEqual(metadata.build_dependencies, ()) + self.assertEqual(metadata.test_dependencies, ()) + self.assertEqual(metadata.provides, ()) + self.assertEqual(metadata.conflicts, ()) + + def test_v1_egg_parsing_python_platform(self): + # Given + egg = NUMPY_1_9_2_WIN_X86_64 + + # When + metadata = EggMetadataV2.from_egg(egg) + + # Then + self.assertEqual(metadata._raw_name, "numpy") + self.assertEqual(metadata.name, "numpy") + + self.assertEqual(metadata.metadata_version, M(u"1.3")) + self.assertEqual(metadata.python_implementation, CP27) + self.assertEqual(metadata.python_tag, u"cp27") + self.assertEqual(metadata.python_tag_string, u'cp27') + self.assertEqual(metadata.python_abi, CP27M) + self.assertEqual(metadata.python_abi_tag, u"cp27m") + self.assertEqual(metadata.python_abi_tag_string, u'cp27m') + self.assertEqual(metadata.platform_tag, 'linux_x86_64') + self.assertEqual(metadata.platform_tag_string, 'linux_x86_64') + self.assertEqual(metadata.platform_abi, GNU) + self.assertEqual(metadata.platform_abi_tag, u'gnu') + self.assertEqual(metadata.platform_abi_tag_string, u'gnu') + + self.assertEqual( + metadata.runtime_dependencies, ((u"MKL", ((u"== 10.3-1", ),)),) + ) + self.assertEqual(metadata.build_dependencies, ()) + self.assertEqual(metadata.test_dependencies, ()) + self.assertEqual(metadata.provides, ()) + self.assertEqual(metadata.conflicts, ()) From 453e01025d10f7947bf25d230bad91b7f33027e6 Mon Sep 17 00:00:00 2001 From: David Cournapeau Date: Wed, 14 Jun 2017 13:06:16 +0100 Subject: [PATCH 5/6] FEAT: add EggMetadataV2.epd_platform_tag attribute. --- okonomiyaki/file_formats/egg_metadata_v2.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/okonomiyaki/file_formats/egg_metadata_v2.py b/okonomiyaki/file_formats/egg_metadata_v2.py index c8b8fe05..9f139c3e 100644 --- a/okonomiyaki/file_formats/egg_metadata_v2.py +++ b/okonomiyaki/file_formats/egg_metadata_v2.py @@ -173,6 +173,13 @@ class EggMetadataV2(object): conflicts = dependency_attr() provides = dependency_attr() + @property + def epd_platform_tag(self): + if self.epd_platform is None: + return None + else: + return str(self.epd_platform) + @property def name(self): return self._raw_name.lower() From e3752c60d03f0ee4dbf5d08b58ebebd55eb0d289 Mon Sep 17 00:00:00 2001 From: David Cournapeau Date: Fri, 23 Jun 2017 23:21:29 +0100 Subject: [PATCH 6/6] REF: use attr to simplify PythonImplementation. --- .../platforms/python_implementation.py | 24 ++++--------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/okonomiyaki/platforms/python_implementation.py b/okonomiyaki/platforms/python_implementation.py index 2fea76c6..6c1c68f1 100644 --- a/okonomiyaki/platforms/python_implementation.py +++ b/okonomiyaki/platforms/python_implementation.py @@ -43,7 +43,12 @@ def pep425_tag_string(abi): @six.python_2_unicode_compatible +@attributes(frozen=True) class PythonImplementation(object): + kind = attr(convert=lambda x: _ABBREVIATED_TO_KIND.get(x, x)) + major = attr() + minor = attr() + @staticmethod def pep425_tag_string(implementation): if implementation is None: @@ -77,11 +82,6 @@ def from_string(cls, s): minor = int(version[1]) return cls(kind, major, minor) - def __init__(self, kind, major, minor): - self.kind = _ABBREVIATED_TO_KIND.get(kind, kind) - self.major = major - self.minor = minor - @property def abbreviated_implementation(self): return _KIND_TO_ABBREVIATED.get(self.kind, self.kind) @@ -94,20 +94,6 @@ def pep425_tag(self): def __str__(self): return "{0.abbreviated_implementation}{0.major}{0.minor}".format(self) - def __eq__(self, other): - if isinstance(other, PythonImplementation): - return ( - (self.major, self.minor, self.kind) == (other.major, other.minor, other.kind) - ) - else: - return NotImplemented - - def __ne__(self, other): - if isinstance(other, PythonImplementation): - return not self == other - else: - return NotImplemented - def _abbreviated_implementation(): """Return abbreviated implementation name."""