diff --git a/okonomiyaki/file_formats/_package_info.py b/okonomiyaki/file_formats/_package_info.py index b4cf6afa..0fa6887c 100644 --- a/okonomiyaki/file_formats/_package_info.py +++ b/okonomiyaki/file_formats/_package_info.py @@ -143,11 +143,11 @@ def from_string(cls, s): return cls(metadata_version, name, version, **kw) def __init__(self, metadata_version, name, version, platforms=None, - supported_platforms=None, summary="", description="", - keywords=None, home_page="", download_url="", author="", - author_email="", license="", classifiers=None, requires=None, - provides=None, obsoletes=None, maintainer="", - maintainer_email="", requires_python=None, + supported_platforms=None, summary=u"", description=u"", + keywords=None, home_page=u"", download_url=u"", author=u"", + author_email=u"", license=u"", classifiers=None, requires=None, + provides=None, obsoletes=None, maintainer=u"", + maintainer_email=u"", requires_python=None, requires_external=None, requires_dist=None, provides_dist=None, obsoletes_dist=None, projects_urls=None): _ensure_supported_version(metadata_version) @@ -175,8 +175,8 @@ def __init__(self, metadata_version, name, version, platforms=None, self.obsoletes = obsoletes or () # version 1.2 - self.maintainer = maintainer or "" - self.maintainer_email = maintainer_email or "" + self.maintainer = maintainer or u"" + self.maintainer_email = maintainer_email or u"" self.requires_python = requires_python or () self.requires_external = requires_external or () self.requires_dist = requires_dist or () @@ -296,10 +296,10 @@ def _collapse_leading_ws(header, txt): lines = [x[8:] if x.startswith(' ' * 8) else x for x in txt.strip().splitlines()] # Append a line to be char-by-char compatible with distutils - lines.append('') - return '\n'.join(lines) + lines.append(u'') + return u'\n'.join(lines) else: - return ' '.join([x.strip() for x in txt.splitlines()]) + return u' '.join([x.strip() for x in txt.splitlines()]) def _convert_if_needed(data, sha256, strict): diff --git a/okonomiyaki/file_formats/egg_metadata_v2.py b/okonomiyaki/file_formats/egg_metadata_v2.py new file mode 100644 index 00000000..9f139c3e --- /dev/null +++ b/okonomiyaki/file_formats/egg_metadata_v2.py @@ -0,0 +1,313 @@ +import six +import zipfile2 + +from attr import Factory, attributes, attr +from attr.validators import instance_of, optional + +from okonomiyaki.platforms import ( + EPDPlatform, PlatformABI, PythonABI, PythonImplementation +) +from okonomiyaki.utils import compute_sha256 +from okonomiyaki.versions import EnpkgVersion, MetadataVersion + +from ._blacklist import ( + may_be_in_platform_blacklist, may_be_in_python_tag_blacklist, + may_be_in_pkg_info_blacklist +) +from ._package_info import PackageInfo, _keep_position + +from . import _egg_info + + +def text_attr(**kw): + """ An attrs.attr-like descriptor to describe fields that must be unicode. + """ + for k in ("validator", ): + if k in kw: + raise ValueError("Cannot pass '{0}' argument".format(k)) + return attr(validator=instance_of(six.text_type), **kw) + + +def dependency_type(inst, attr, value): + if not isinstance(value, tuple): + raise TypeError("Dependency must be a tuple") + for item in value: + if len(item) != 2: + raise ValueError("Each dependency value must be a pair") + name, disjunctions = item + if not isinstance(name, six.text_type): + raise ValueError( + u"Expected a text type, got {!r}".format(name) + ) + if not isinstance(disjunctions, tuple): + raise ValueError( + u"Expected tuples for disjunctions, got {!r}".format(disjunctions) + ) + for disjunction in disjunctions: + if isinstance(disjunction, tuple): + for conjunction in disjunction: + if not isinstance(conjunction, six.text_type): + raise ValueError( + u"Expected conjunction to be a string, got {!r}" + .format(conjunction) + ) + else: + raise ValueError( + u"Expected tuple for disjunction, got {!r}".format(disjunction) + ) + + +def dependency_attr(**kw): + """ An attrs.attr-like descriptor to describe fields that must be + dependency like, that is a tuple of (name, disjunctions) pairs, where each + disjunction is a tuple of strings. + """ + for k in ("validator", "default"): + if k in kw: + raise ValueError("Cannot pass '{0}' argument".format(k)) + return attr(validator=dependency_type, default=Factory(tuple)) + + +def _convert_to_metadata_version(s): + if isinstance(s, MetadataVersion): + return s + else: + return MetadataVersion.from_string(s) + + +def _convert_to_enpkg_version(s): + if isinstance(s, EnpkgVersion): + return s + else: + return EnpkgVersion.from_string(s) + + +def _convert_to_epd_platform(s): + if isinstance(s, EPDPlatform): + return s + elif isinstance(s, six.text_type): + return EPDPlatform.from_string(s) + else: + return None + + +def _convert_to_python_implementation(s): + if isinstance(s, PythonImplementation): + return s + elif isinstance(s, six.text_type): + return PythonImplementation.from_string(s) + else: + return None + + +def _convert_to_python_abi(s): + if isinstance(s, PythonABI): + return s + elif isinstance(s, six.text_type): + return PythonABI(s) + else: + return None + + +def _convert_to_platform_abi(s): + if isinstance(s, PlatformABI): + return s + elif isinstance(s, six.text_type): + return PlatformABI(s) + else: + return None + + +def _convert_to_package_info(s): + if isinstance(s, PackageInfo): + return s + elif isinstance(s, six.text_type): + return PackageInfo.from_string(s) + else: + return None + + +@attributes +class EggMetadataV2(object): + metadata_version = attr( + validator=instance_of(MetadataVersion), + convert=_convert_to_metadata_version, + ) + + _raw_name = text_attr() + + version = attr( + validator=instance_of(EnpkgVersion), convert=_convert_to_enpkg_version + ) + + epd_platform = attr( + validator=optional(instance_of(EPDPlatform)), + convert=_convert_to_epd_platform, + ) + + python_implementation = attr( + validator=optional(instance_of(PythonImplementation)), + convert=_convert_to_python_implementation + ) + + python_abi = attr( + validator=optional(instance_of(PythonABI)), + convert=_convert_to_python_abi, + ) + + platform_abi = attr( + validator=optional(instance_of(PlatformABI)), convert=_convert_to_platform_abi, + ) + + package_info = attr( + validator=optional(instance_of(PackageInfo)), convert=_convert_to_package_info, + ) + + summary = text_attr() + license = attr(validator=optional(instance_of(six.text_type))) + + runtime_dependencies = dependency_attr() + + build_dependencies = dependency_attr() + test_dependencies = dependency_attr() + conflicts = dependency_attr() + provides = dependency_attr() + + @property + def epd_platform_tag(self): + if self.epd_platform is None: + return None + else: + return str(self.epd_platform) + + @property + def name(self): + return self._raw_name.lower() + + @property + def platform_abi_tag(self): + if self.platform_abi is None: + return None + else: + return self.platform_abi.pep425_tag + + @property + def platform_abi_tag_string(self): + return PlatformABI.pep425_tag_string(self.platform_abi) + + @property + def platform_tag(self): + """ Platform tag following PEP425, except that no platform is + represented as None and not 'any'.""" + if self.epd_platform is None: + return None + else: + return self.epd_platform.pep425_tag + + @property + def platform_tag_string(self): + return EPDPlatform.pep425_tag_string(self.epd_platform) + + @property + def python_abi_tag(self): + if self.python_abi is None: + return None + else: + return self.python_abi.pep425_tag + + @property + def python_abi_tag_string(self): + return PythonABI.pep425_tag_string(self.python_abi) + + @property + def python_tag(self): + if self.python_implementation is None: + return None + else: + return self.python_implementation.pep425_tag + + @property + def python_tag_string(self): + return PythonImplementation.pep425_tag_string(self.python_implementation) + + @staticmethod + def _may_be_in_blacklist(path): + return ( + may_be_in_platform_blacklist(path) + or may_be_in_pkg_info_blacklist(path) + or may_be_in_python_tag_blacklist(path) + ) + + @classmethod + def from_egg(cls, path_or_file, strict=True): + """ Create a EggMetadata instance from an existing Enthought egg. + + Parameters + ---------- + path: str or file-like object. + If a string, understood as the path to the egg. Otherwise, + understood as a zipfile-like object. + strict: bool + If True, will fail if metadata cannot be decoded correctly (e.g. + unicode errors in EGG-INFO/PKG-INFO). If false, will ignore those + errors, at the risk of data loss. + """ + sha256 = None + if isinstance(path_or_file, six.string_types): + if cls._may_be_in_blacklist(path_or_file): + sha256 = compute_sha256(path_or_file) + else: + with _keep_position(path_or_file.fp): + sha256 = compute_sha256(path_or_file.fp) + return cls._from_egg(path_or_file, sha256, strict) + + @classmethod + def _from_egg(cls, path_or_file, sha256, strict=True): + v1_arcname = "EGG-INFO/spec/depend" + + def _read(zp): + try: + with _keep_position(zp.fp): + zp.read(v1_arcname) + except KeyError: + raise NotImplementedError() + else: + return cls._from_v1_egg_metadata( + _egg_info.EggMetadata._from_egg(zp, sha256, strict) + ) + + if isinstance(path_or_file, six.string_types): + with zipfile2.ZipFile(path_or_file) as zp: + return _read(zp) + else: + return _read(path_or_file) + + @classmethod + def _from_v1_egg_metadata(cls, m): + if m._pkg_info is not None: + license = m.pkg_info.license + else: + license = None + return cls( + m.metadata_version, m._raw_name, m.version, m.platform, m.python, + m.abi, m.platform_abi, m._pkg_info, m.summary, license, + runtime_dependencies=_convert_requirement_to_dependencies(m.runtime_dependencies), + ) + + +def _convert_requirement_to_dependencies(requirements): + def _transformer(entry): + if len(entry.version_string) == 0: + return (entry.name, ((u"*",),)) + elif entry.build_number == -1: + return ( + entry.name, + ((u"^= {}".format(entry.version_string),),) + ) + else: + return ( + entry.name, + ((u"== {}-{}".format(entry.version_string, entry.build_number),),) + ) + + return tuple(_transformer(entry) for entry in requirements) diff --git a/okonomiyaki/file_formats/tests/test_egg_metadata_v2.py b/okonomiyaki/file_formats/tests/test_egg_metadata_v2.py new file mode 100644 index 00000000..bef46343 --- /dev/null +++ b/okonomiyaki/file_formats/tests/test_egg_metadata_v2.py @@ -0,0 +1,141 @@ +import shutil +import tempfile +import unittest + +from okonomiyaki.platforms import ( + EPDPlatform, PlatformABI, PythonABI, PythonImplementation +) +from okonomiyaki.utils.test_data import ( + MKL_10_3_RH5_X86_64, NOSE_1_3_4_RH5_X86_64, NUMPY_1_9_2_WIN_X86_64 +) +from okonomiyaki.versions import EnpkgVersion, MetadataVersion +from ..egg_metadata_v2 import EggMetadataV2 +from .common import ENSTALLER_EGG + + +M = MetadataVersion.from_string +P = EPDPlatform.from_string +V = EnpkgVersion.from_string +CP27 = PythonImplementation.from_string(u"cp27") +CP27M = PythonABI(u"cp27m") +GNU = PlatformABI(u"gnu") + + +class TestEggMetadata(unittest.TestCase): + def setUp(self): + self.tempdir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.tempdir) + + def test_v1_egg_parsing_pure_python_no_platform(self): + # Given + egg = ENSTALLER_EGG + + # When + metadata = EggMetadataV2.from_egg(egg) + + # Then + self.assertEqual(metadata.metadata_version, M(u"1.1")) + self.assertEqual(metadata.name, "enstaller") + self.assertEqual(metadata.version, V("4.5.0-1")) + self.assertIsNone(metadata.epd_platform) + self.assertIsNone(metadata.python_implementation) + self.assertIsNone(metadata.python_abi) + self.assertIsNone(metadata.platform_abi) + self.assertIsNotNone(metadata.package_info) + self.assertEqual(metadata.summary, u"") + self.assertEqual(metadata.license, u"BSD") + self.assertEqual(metadata.runtime_dependencies, ()) + self.assertEqual(metadata.build_dependencies, ()) + self.assertEqual(metadata.test_dependencies, ()) + self.assertEqual(metadata.provides, ()) + self.assertEqual(metadata.conflicts, ()) + + def test_v1_egg_parsing_pure_python_with_platform(self): + # Given + egg = NOSE_1_3_4_RH5_X86_64 + + # When + metadata = EggMetadataV2.from_egg(egg) + + # Then + self.assertEqual(metadata.metadata_version, M(u"1.3")) + self.assertEqual(metadata.name, u"nose") + self.assertEqual(metadata.version, V("1.3.4-1")) + self.assertEqual(metadata.epd_platform, P(u"rh5-x86_64")) + self.assertEqual(metadata.python_implementation, CP27) + self.assertEqual(metadata.python_abi, CP27M) + self.assertEqual(metadata.platform_abi, GNU) + self.assertIsNotNone(metadata.package_info) + self.assertEqual( + metadata.summary, + u"Extends the Python Unittest module with additional disocvery and " + "running\noptions\n" + ) + self.assertEqual(metadata.license, u"GNU LGPL") + self.assertEqual(metadata.runtime_dependencies, ()) + self.assertEqual(metadata.build_dependencies, ()) + self.assertEqual(metadata.test_dependencies, ()) + self.assertEqual(metadata.provides, ()) + self.assertEqual(metadata.conflicts, ()) + + def test_v1_egg_parsing_no_python_platform(self): + # Given + egg = MKL_10_3_RH5_X86_64 + + # When + metadata = EggMetadataV2.from_egg(egg) + + # Then + self.assertEqual(metadata._raw_name, "MKL") + self.assertEqual(metadata.name, "mkl") + + self.assertEqual(metadata.metadata_version, M(u"1.3")) + self.assertIsNone(metadata.python_abi_tag) + self.assertEqual(metadata.python_abi_tag_string, u'none') + self.assertEqual(metadata.platform_tag, 'linux_x86_64') + self.assertEqual(metadata.platform_tag_string, 'linux_x86_64') + self.assertEqual(metadata.platform_abi, GNU) + self.assertEqual(metadata.platform_abi_tag, u'gnu') + self.assertEqual(metadata.platform_abi_tag_string, u'gnu') + self.assertIsNone(metadata.python_tag) + self.assertEqual(metadata.python_tag_string, 'none') + + self.assertEqual(metadata.runtime_dependencies, ()) + self.assertEqual(metadata.build_dependencies, ()) + self.assertEqual(metadata.test_dependencies, ()) + self.assertEqual(metadata.provides, ()) + self.assertEqual(metadata.conflicts, ()) + + def test_v1_egg_parsing_python_platform(self): + # Given + egg = NUMPY_1_9_2_WIN_X86_64 + + # When + metadata = EggMetadataV2.from_egg(egg) + + # Then + self.assertEqual(metadata._raw_name, "numpy") + self.assertEqual(metadata.name, "numpy") + + self.assertEqual(metadata.metadata_version, M(u"1.3")) + self.assertEqual(metadata.python_implementation, CP27) + self.assertEqual(metadata.python_tag, u"cp27") + self.assertEqual(metadata.python_tag_string, u'cp27') + self.assertEqual(metadata.python_abi, CP27M) + self.assertEqual(metadata.python_abi_tag, u"cp27m") + self.assertEqual(metadata.python_abi_tag_string, u'cp27m') + self.assertEqual(metadata.platform_tag, 'linux_x86_64') + self.assertEqual(metadata.platform_tag_string, 'linux_x86_64') + self.assertEqual(metadata.platform_abi, GNU) + self.assertEqual(metadata.platform_abi_tag, u'gnu') + self.assertEqual(metadata.platform_abi_tag_string, u'gnu') + + self.assertEqual( + metadata.runtime_dependencies, ((u"MKL", ((u"== 10.3-1", ),)),) + ) + self.assertEqual(metadata.build_dependencies, ()) + self.assertEqual(metadata.test_dependencies, ()) + self.assertEqual(metadata.provides, ()) + self.assertEqual(metadata.conflicts, ()) diff --git a/okonomiyaki/platforms/python_implementation.py b/okonomiyaki/platforms/python_implementation.py index d03968f6..6c1c68f1 100644 --- a/okonomiyaki/platforms/python_implementation.py +++ b/okonomiyaki/platforms/python_implementation.py @@ -43,7 +43,12 @@ def pep425_tag_string(abi): @six.python_2_unicode_compatible +@attributes(frozen=True) class PythonImplementation(object): + kind = attr(convert=lambda x: _ABBREVIATED_TO_KIND.get(x, x)) + major = attr() + minor = attr() + @staticmethod def pep425_tag_string(implementation): if implementation is None: @@ -77,11 +82,6 @@ def from_string(cls, s): minor = int(version[1]) return cls(kind, major, minor) - def __init__(self, kind, major, minor): - self.kind = _ABBREVIATED_TO_KIND.get(kind, kind) - self.major = major - self.minor = minor - @property def abbreviated_implementation(self): return _KIND_TO_ABBREVIATED.get(self.kind, self.kind)