From ee2e4c34e6d4ff9d923f96884fe02062a18e8f9b Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Sat, 20 Jul 2024 17:56:41 -0400 Subject: [PATCH 1/2] Move downloads to YAML --- cpython-unix/build-main.py | 7 +- cpython-unix/build.py | 116 +++++++---- cpython-windows/build.py | 69 ++++--- downloads.yml | 409 +++++++++++++++++++++++++++++++++++++ pythonbuild/buildenv.py | 33 +-- pythonbuild/downloads.py | 406 ------------------------------------ pythonbuild/utils.py | 22 +- 7 files changed, 565 insertions(+), 497 deletions(-) create mode 100644 downloads.yml delete mode 100644 pythonbuild/downloads.py diff --git a/cpython-unix/build-main.py b/cpython-unix/build-main.py index b9b3c5dc..e6460647 100755 --- a/cpython-unix/build-main.py +++ b/cpython-unix/build-main.py @@ -11,15 +11,16 @@ import subprocess import sys -from pythonbuild.downloads import DOWNLOADS from pythonbuild.utils import ( compress_python_archive, + get_downloads, get_target_settings, release_tag_from_git, supported_targets, ) ROOT = pathlib.Path(os.path.abspath(__file__)).parent.parent +DOWNLOADS_FILE = ROOT / "downloads.yml" BUILD = ROOT / "build" DIST = ROOT / "dist" SUPPORT = ROOT / "cpython-unix" @@ -114,6 +115,8 @@ def main(): settings = get_target_settings(TARGETS_CONFIG, target_triple) + downloads = get_downloads(DOWNLOADS_FILE) + supported_pythons = {"cpython-%s" % p for p in settings["pythons_supported"]} if args.python not in supported_pythons: @@ -145,7 +148,7 @@ def main(): env["PYBUILD_NO_DOCKER"] = "1" if not args.python_source: - entry = DOWNLOADS[args.python] + entry = downloads[args.python] env["PYBUILD_PYTHON_VERSION"] = cpython_version = entry["version"] else: # TODO consider parsing version from source checkout. Or defining version diff --git a/cpython-unix/build.py b/cpython-unix/build.py index ccea702c..ebfff8be 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -26,14 +26,15 @@ parse_setup_line, ) from pythonbuild.docker import build_docker_image, get_image, write_dockerfiles -from pythonbuild.downloads import DOWNLOADS from pythonbuild.logging import log, set_logger from pythonbuild.utils import ( + Downloads, add_env_common, add_licenses_to_extension_entry, clang_toolchain, create_tar_from_directory, download_entry, + get_downloads, get_target_settings, get_targets, target_needs, @@ -45,6 +46,7 @@ ) ROOT = pathlib.Path(os.path.abspath(__file__)).parent.parent +DOWNLOADS_FILE = ROOT / "downloads.yml" BUILD = ROOT / "build" DOWNLOADS_PATH = BUILD / "downloads" SUPPORT = ROOT / "cpython-unix" @@ -212,8 +214,8 @@ def add_target_env(env, build_platform, target_triple, build_env): env["EXTRA_TARGET_LDFLAGS"] = " ".join(extra_target_ldflags) -def toolchain_archive_path(package_name, host_platform): - entry = DOWNLOADS[package_name] +def toolchain_archive_path(package_name, host_platform, downloads): + entry = downloads[package_name] basename = "%s-%s-%s.tar" % (package_name, entry["version"], host_platform) @@ -232,11 +234,12 @@ def simple_build( host_platform, target_triple, optimizations, + downloads: Downloads, dest_archive, extra_archives=None, tools_path="deps", ): - archive = download_entry(entry, DOWNLOADS_PATH) + archive = download_entry(entry, DOWNLOADS_PATH, downloads) with build_environment(client, image) as build_env: if settings.get("needs_toolchain"): @@ -256,7 +259,7 @@ def simple_build( build_env.copy_file(SUPPORT / ("build-%s.sh" % entry)) env = { - "%s_VERSION" % entry.upper().replace("-", "_").replace(".", "_"): DOWNLOADS[ + "%s_VERSION" % entry.upper().replace("-", "_").replace(".", "_"): downloads[ entry ]["version"], } @@ -272,9 +275,9 @@ def simple_build( build_env.get_tools_archive(dest_archive, tools_path) -def build_binutils(client, image, host_platform): +def build_binutils(client, image, host_platform, downloads): """Build binutils in the Docker image.""" - archive = download_entry("binutils", DOWNLOADS_PATH) + archive = download_entry("binutils", DOWNLOADS_PATH, downloads) with build_environment(client, image) as build_env: install_sccache(build_env) @@ -282,7 +285,7 @@ def build_binutils(client, image, host_platform): build_env.copy_file(archive) build_env.copy_file(SUPPORT / "build-binutils.sh") - env = {"BINUTILS_VERSION": DOWNLOADS["binutils"]["version"]} + env = {"BINUTILS_VERSION": downloads["binutils"]["version"]} add_env_common(env) @@ -292,16 +295,16 @@ def build_binutils(client, image, host_platform): ) build_env.get_tools_archive( - toolchain_archive_path("binutils", host_platform), "host" + toolchain_archive_path("binutils", host_platform, downloads), "host" ) -def materialize_clang(host_platform: str, target_triple: str): +def materialize_clang(host_platform: str, target_triple: str, downloads): entry = clang_toolchain(host_platform, target_triple) - tar_zst = download_entry(entry, DOWNLOADS_PATH) + tar_zst = download_entry(entry, DOWNLOADS_PATH, downloads) local_filename = "%s-%s-%s.tar" % ( entry, - DOWNLOADS[entry]["version"], + downloads[entry]["version"], host_platform, ) @@ -312,8 +315,8 @@ def materialize_clang(host_platform: str, target_triple: str): dctx.copy_stream(ifh, ofh) -def build_musl(client, image, host_platform: str, target_triple: str): - musl_archive = download_entry("musl", DOWNLOADS_PATH) +def build_musl(client, image, host_platform: str, target_triple: str, downloads): + musl_archive = download_entry("musl", DOWNLOADS_PATH, downloads) with build_environment(client, image) as build_env: build_env.install_toolchain( @@ -323,7 +326,7 @@ def build_musl(client, image, host_platform: str, target_triple: str): build_env.copy_file(SUPPORT / "build-musl.sh") env = { - "MUSL_VERSION": DOWNLOADS["musl"]["version"], + "MUSL_VERSION": downloads["musl"]["version"], "TOOLCHAIN": "llvm", } @@ -335,9 +338,16 @@ def build_musl(client, image, host_platform: str, target_triple: str): def build_libedit( - settings, client, image, host_platform, target_triple, optimizations, dest_archive + settings, + client, + image, + host_platform, + target_triple, + optimizations, + dest_archive, + downloads: Downloads, ): - libedit_archive = download_entry("libedit", DOWNLOADS_PATH) + libedit_archive = download_entry("libedit", DOWNLOADS_PATH, downloads) with build_environment(client, image) as build_env: if settings.get("needs_toolchain"): @@ -357,7 +367,7 @@ def build_libedit( build_env.copy_file(SUPPORT / "build-libedit.sh") env = { - "LIBEDIT_VERSION": DOWNLOADS["libedit"]["version"], + "LIBEDIT_VERSION": downloads["libedit"]["version"], } add_target_env(env, host_platform, target_triple, build_env) @@ -367,11 +377,18 @@ def build_libedit( def build_tix( - settings, client, image, host_platform, target_triple, optimizations, dest_archive + settings, + client, + image, + host_platform, + target_triple, + optimizations, + dest_archive, + downloads: Downloads, ): - tcl_archive = download_entry("tcl", DOWNLOADS_PATH) - tk_archive = download_entry("tk", DOWNLOADS_PATH) - tix_archive = download_entry("tix", DOWNLOADS_PATH) + tcl_archive = download_entry("tcl", DOWNLOADS_PATH, downloads) + tk_archive = download_entry("tk", DOWNLOADS_PATH, downloads) + tix_archive = download_entry("tix", DOWNLOADS_PATH, downloads) with build_environment(client, image) as build_env: if settings.get("needs_toolchain"): @@ -396,9 +413,9 @@ def build_tix( env = { "TOOLCHAIN": "clang-%s" % host_platform, - "TCL_VERSION": DOWNLOADS["tcl"]["version"], - "TIX_VERSION": DOWNLOADS["tix"]["version"], - "TK_VERSION": DOWNLOADS["tk"]["version"], + "TCL_VERSION": downloads["tcl"]["version"], + "TIX_VERSION": downloads["tix"]["version"], + "TK_VERSION": downloads["tk"]["version"], } add_target_env(env, host_platform, target_triple, build_env) @@ -415,12 +432,13 @@ def build_cpython_host( target_triple: str, optimizations: str, dest_archive, + downloads: Downloads, ): """Build binutils in the Docker image.""" - archive = download_entry(entry, DOWNLOADS_PATH) + archive = download_entry(entry, DOWNLOADS_PATH, downloads) with build_environment(client, image) as build_env: - python_version = DOWNLOADS[entry]["version"] + python_version = downloads[entry]["version"] build_env.install_toolchain( BUILD, @@ -480,6 +498,7 @@ def python_build_info( optimizations, extensions, extra_metadata, + downloads: Downloads, ): """Obtain build metadata for the Python distribution.""" @@ -503,7 +522,7 @@ def python_build_info( ) if optimizations in ("lto", "pgo+lto"): - llvm_version = DOWNLOADS[clang_toolchain(platform, target_triple)][ + llvm_version = downloads[clang_toolchain(platform, target_triple)][ "version" ] if "+" in llvm_version: @@ -525,7 +544,7 @@ def python_build_info( if optimizations in ("lto", "pgo+lto"): object_file_format = ( - "llvm-bitcode:%s" % DOWNLOADS["llvm-aarch64-macos"]["version"] + "llvm-bitcode:%s" % downloads["llvm-aarch64-macos"]["version"] ) else: object_file_format = "mach-o" @@ -682,16 +701,17 @@ def build_cpython( host_platform, target_triple, optimizations, + downloads: Downloads, dest_archive, version=None, python_source=None, ): """Build CPython in a Docker image'""" entry_name = "cpython-%s" % version - entry = DOWNLOADS[entry_name] + entry = downloads[entry_name] if not python_source: python_version = entry["version"] - python_archive = download_entry(entry_name, DOWNLOADS_PATH) + python_archive = download_entry(entry_name, DOWNLOADS_PATH, downloads) else: python_version = os.environ["PYBUILD_PYTHON_VERSION"] python_archive = DOWNLOADS_PATH / ("Python-%s.tar.xz" % python_version) @@ -701,8 +721,8 @@ def build_cpython( fh, python_source, path_prefix="Python-%s" % python_version ) - setuptools_archive = download_entry("setuptools", DOWNLOADS_PATH) - pip_archive = download_entry("pip", DOWNLOADS_PATH) + setuptools_archive = download_entry("setuptools", DOWNLOADS_PATH, downloads) + pip_archive = download_entry("pip", DOWNLOADS_PATH, downloads) ems = extension_modules_config(EXTENSION_MODULES) @@ -773,10 +793,10 @@ def build_cpython( build_env.copy_file(fh.name, dest_name="Makefile.extra") env = { - "PIP_VERSION": DOWNLOADS["pip"]["version"], + "PIP_VERSION": downloads["pip"]["version"], "PYTHON_VERSION": python_version, "PYTHON_MAJMIN_VERSION": ".".join(python_version.split(".")[0:2]), - "SETUPTOOLS_VERSION": DOWNLOADS["setuptools"]["version"], + "SETUPTOOLS_VERSION": downloads["setuptools"]["version"], "TOOLCHAIN": "clang-%s" % host_platform, } @@ -852,6 +872,7 @@ def build_cpython( optimizations, enabled_extensions, extra_metadata, + downloads=downloads, ), "licenses": entry["licenses"], "license_path": "licenses/LICENSE.cpython.txt", @@ -958,6 +979,8 @@ def main(): settings = get_target_settings(TARGETS_CONFIG, target_triple) + downloads = get_downloads(DOWNLOADS_FILE) + if args.action == "dockerfiles": log_name = "dockerfiles" elif args.action == "makefiles": @@ -969,7 +992,7 @@ def main(): elif args.action.startswith("cpython-") and args.action.endswith("-host"): log_name = args.action else: - entry = DOWNLOADS[action] + entry = downloads[action] log_name = "%s-%s-%s-%s" % ( action, entry["version"], @@ -1004,10 +1027,15 @@ def main(): build_docker_image(client, image_data, BUILD, image_name) elif action == "binutils": - build_binutils(client, get_image(client, ROOT, BUILD, "gcc"), host_platform) + build_binutils( + client, + get_image(client, ROOT, BUILD, "gcc"), + host_platform, + downloads=downloads, + ) elif action == "clang": - materialize_clang(host_platform, target_triple) + materialize_clang(host_platform, target_triple, downloads=downloads) elif action == "musl": build_musl( @@ -1015,6 +1043,7 @@ def main(): get_image(client, ROOT, BUILD, "gcc"), host_platform, target_triple, + downloads=downloads, ) elif action == "autoconf": @@ -1027,6 +1056,7 @@ def main(): target_triple=target_triple, optimizations=optimizations, dest_archive=dest_archive, + downloads=downloads, tools_path="host", extra_archives=["m4"], ) @@ -1040,6 +1070,7 @@ def main(): target_triple=target_triple, optimizations=optimizations, dest_archive=dest_archive, + downloads=downloads, ) elif action in ( @@ -1079,6 +1110,7 @@ def main(): target_triple=target_triple, optimizations=optimizations, dest_archive=dest_archive, + downloads=downloads, tools_path=tools_path, ) @@ -1092,6 +1124,7 @@ def main(): target_triple=target_triple, optimizations=optimizations, dest_archive=dest_archive, + downloads=downloads, extra_archives={ "inputproto", "kbproto", @@ -1116,6 +1149,7 @@ def main(): target_triple=target_triple, optimizations=optimizations, dest_archive=dest_archive, + downloads=downloads, extra_archives={"x11-util-macros", "xproto"}, ) @@ -1129,6 +1163,7 @@ def main(): target_triple=target_triple, optimizations=optimizations, dest_archive=dest_archive, + downloads=downloads, ) elif action == "libxcb": @@ -1141,6 +1176,7 @@ def main(): target_triple=target_triple, optimizations=optimizations, dest_archive=dest_archive, + downloads=downloads, extra_archives={"libpthread-stubs", "libXau", "xcb-proto", "xproto"}, ) @@ -1153,6 +1189,7 @@ def main(): target_triple=target_triple, optimizations=optimizations, dest_archive=dest_archive, + downloads=downloads, ) elif action == "tk": @@ -1175,6 +1212,7 @@ def main(): target_triple=target_triple, optimizations=optimizations, dest_archive=dest_archive, + downloads=downloads, extra_archives=extra_archives, ) @@ -1187,6 +1225,7 @@ def main(): target_triple=target_triple, optimizations=optimizations, dest_archive=dest_archive, + downloads=downloads, ) elif action in ( @@ -1204,6 +1243,7 @@ def main(): target_triple=target_triple, optimizations=optimizations, dest_archive=dest_archive, + downloads=downloads, version=action.split("-")[1], python_source=python_source, ) diff --git a/cpython-windows/build.py b/cpython-windows/build.py index bc97cd57..85a2eaf0 100644 --- a/cpython-windows/build.py +++ b/cpython-windows/build.py @@ -22,19 +22,20 @@ meets_python_minimum_version, parse_config_c, ) -from pythonbuild.downloads import DOWNLOADS from pythonbuild.utils import ( compress_python_archive, create_tar_from_directory, download_entry, extract_tar_to_directory, extract_zip_to_directory, + get_downloads, normalize_tar_archive, release_tag_from_git, validate_python_json, ) ROOT = pathlib.Path(os.path.abspath(__file__)).parent.parent +DOWNLOADS_FILE = ROOT / "downloads.yml" BUILD = ROOT / "build" DIST = ROOT / "dist" SUPPORT = ROOT / "cpython-windows" @@ -341,17 +342,18 @@ def hack_props( td: pathlib.Path, pcbuild_path: pathlib.Path, arch: str, + downloads, ): # TODO can we pass props into msbuild.exe? # Our dependencies are in different directories from what CPython's # build system expects. Modify the config file appropriately. - bzip2_version = DOWNLOADS["bzip2"]["version"] - sqlite_version = DOWNLOADS["sqlite"]["version"] - xz_version = DOWNLOADS["xz"]["version"] - zlib_version = DOWNLOADS["zlib"]["version"] - tcltk_commit = DOWNLOADS["tk-windows-bin"]["git_commit"] + bzip2_version = downloads["bzip2"]["version"] + sqlite_version = downloads["sqlite"]["version"] + xz_version = downloads["xz"]["version"] + zlib_version = downloads["zlib"]["version"] + tcltk_commit = downloads["tk-windows-bin"]["git_commit"] sqlite_path = td / ("sqlite-autoconf-%s" % sqlite_version) bzip2_path = td / ("bzip2-%s" % bzip2_version) @@ -469,6 +471,7 @@ def hack_project_files( cpython_source_path: pathlib.Path, build_directory: str, python_version: str, + downloads, ): """Hacks Visual Studio project files to work with our build.""" @@ -482,7 +485,7 @@ def hack_project_files( # Our SQLite directory is named weirdly. This throws off version detection # in the project file. Replace the parsing logic with a static string. - sqlite3_version = DOWNLOADS["sqlite"]["actual_version"].encode("ascii") + sqlite3_version = downloads["sqlite"]["actual_version"].encode("ascii") sqlite3_version_parts = sqlite3_version.split(b".") sqlite3_path = pcbuild_path / "sqlite3.vcxproj" static_replace_in_file( @@ -868,11 +871,11 @@ def build_openssl_for_arch( openssl_version: str, nasm_archive, build_root: pathlib.Path, - profile: str, + downloads, *, jom_archive, ): - nasm_version = DOWNLOADS["nasm-windows-bin"]["version"] + nasm_version = downloads["nasm-windows-bin"]["version"] log("extracting %s to %s" % (openssl_archive, build_root)) extract_tar_to_directory(openssl_archive, build_root) @@ -960,17 +963,17 @@ def build_openssl( entry: str, perl_path: pathlib.Path, arch: str, - profile: str, dest_archive: pathlib.Path, + downloads, ): """Build OpenSSL from sources using the Perl executable specified.""" - openssl_version = DOWNLOADS[entry]["version"] + openssl_version = downloads[entry]["version"] # First ensure the dependencies are in place. - openssl_archive = download_entry(entry, BUILD) - nasm_archive = download_entry("nasm-windows-bin", BUILD) - jom_archive = download_entry("jom-windows-bin", BUILD) + openssl_archive = download_entry(entry, downloads, BUILD) + nasm_archive = download_entry("nasm-windows-bin", downloads, BUILD) + jom_archive = download_entry("jom-windows-bin", downloads, BUILD) with tempfile.TemporaryDirectory(prefix="openssl-build-") as td: td = pathlib.Path(td) @@ -987,7 +990,7 @@ def build_openssl( openssl_version, nasm_archive, root_32, - profile, + downloads=downloads, jom_archive=jom_archive, ) elif arch == "amd64": @@ -999,7 +1002,7 @@ def build_openssl( openssl_version, nasm_archive, root_64, - profile, + downloads=downloads, jom_archive=jom_archive, ) else: @@ -1022,6 +1025,7 @@ def build_libffi( sh_exe: pathlib.Path, msvc_version: str, dest_archive: pathlib.Path, + downloads, ): with tempfile.TemporaryDirectory(prefix="libffi-build-") as td: td = pathlib.Path(td) @@ -1059,10 +1063,10 @@ def build_libffi( ) # We build libffi by running the build script that CPython ships. - python_archive = download_entry(python, BUILD) + python_archive = download_entry(python, downloads, BUILD) extract_tar_to_directory(python_archive, td) - python_entry = DOWNLOADS[python] + python_entry = downloads[python] prepare_libffi = ( td / ("Python-%s" % python_entry["version"]) @@ -1113,6 +1117,7 @@ def collect_python_build_artifacts( arch: str, config: str, openssl_entry: str, + downloads, ): """Collect build artifacts from Python. @@ -1331,7 +1336,7 @@ def find_additional_dependencies(project: pathlib.Path): if name == "openssl": name = openssl_entry - download_entry = DOWNLOADS[name] + download_entry = downloads[name] # This will raise if no license metadata defined. This is # intentional because EXTENSION_TO_LIBRARY_DOWNLOADS_ENTRY is @@ -1384,6 +1389,7 @@ def build_cpython( openssl_archive, libffi_archive, openssl_entry: str, + downloads, ) -> pathlib.Path: pgo = profile == "pgo" @@ -1393,21 +1399,21 @@ def build_cpython( # The python.props file keys off MSBUILD, so it needs to be set. os.environ["MSBUILD"] = str(msbuild) - bzip2_archive = download_entry("bzip2", BUILD) - sqlite_archive = download_entry("sqlite", BUILD) + bzip2_archive = download_entry("bzip2", downloads, BUILD) + sqlite_archive = download_entry("sqlite", downloads, BUILD) tk_bin_archive = download_entry( "tk-windows-bin", BUILD, local_name="tk-windows-bin.tar.gz" ) - xz_archive = download_entry("xz", BUILD) - zlib_archive = download_entry("zlib", BUILD) + xz_archive = download_entry("xz", downloads, BUILD) + zlib_archive = download_entry("zlib", downloads, BUILD) - python_archive = download_entry(python_entry_name, BUILD) - entry = DOWNLOADS[python_entry_name] + python_archive = download_entry(python_entry_name, downloads, BUILD) + entry = downloads[python_entry_name] python_version = entry["version"] - setuptools_wheel = download_entry("setuptools", BUILD) - pip_wheel = download_entry("pip", BUILD) + setuptools_wheel = download_entry("setuptools", downloads, BUILD) + pip_wheel = download_entry("pip", downloads, BUILD) if arch == "amd64": build_platform = "x64" @@ -1668,6 +1674,7 @@ def build_cpython( build_directory, artifact_config, openssl_entry=openssl_entry, + downloads=downloads, ) for ext, init_fn in sorted(builtin_extensions.items()): @@ -1804,7 +1811,7 @@ def build_cpython( def fetch_strawberry_perl() -> pathlib.Path: - strawberryperl_zip = download_entry("strawberryperl", BUILD) + strawberryperl_zip = download_entry("strawberryperl", downloads, BUILD) strawberryperl = BUILD / "strawberry-perl" strawberryperl.mkdir(exist_ok=True) with zipfile.ZipFile(strawberryperl_zip) as zf: @@ -1852,6 +1859,8 @@ def main() -> None: args = parser.parse_args() + downloads = get_downloads(DOWNLOADS_FILE) + log_path = BUILD / "build.log" with log_path.open("wb") as log_fh: @@ -1884,8 +1893,8 @@ def main() -> None: openssl_entry, perl_path, arch, - profile=args.profile, dest_archive=openssl_archive, + downloads=downloads, ) libffi_archive = BUILD / ("libffi-%s-%s.tar" % (target_triple, args.profile)) @@ -1896,6 +1905,7 @@ def main() -> None: pathlib.Path(args.sh), args.vs, libffi_archive, + downloads=downloads, ) LOG_PREFIX[0] = "cpython" @@ -1909,6 +1919,7 @@ def main() -> None: openssl_archive=openssl_archive, libffi_archive=libffi_archive, openssl_entry=openssl_entry, + downloads=downloads, ) if "PYBUILD_RELEASE_TAG" in os.environ: diff --git a/downloads.yml b/downloads.yml new file mode 100644 index 00000000..179990ff --- /dev/null +++ b/downloads.yml @@ -0,0 +1,409 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +autoconf: + sha256: 431075ad0bf529ef13cb41e9042c542381103e80015686222b8a9d4abef42a1c + size: 2003781 + url: https://ftp.gnu.org/gnu/autoconf/autoconf-2.71.tar.gz + version: '2.71' +# 6.0.19 is the last version licensed under the Sleepycat license. +bdb: + library_names: + - db + license_file: LICENSE.bdb.txt + licenses: + - Sleepycat + sha256: 2917c28f60903908c2ca4587ded1363b812c4e830a5326aaa77c9879d13ae18e + size: 36541923 + url: https://ftp.osuosl.org/pub/blfs/conglomeration/db/db-6.0.19.tar.gz + version: 6.0.19 +binutils: + sha256: f6e4d41fd5fc778b06b7891457b3620da5ecea1006c6a4a41ae998109f85a800 + size: 27567160 + url: https://ftp.gnu.org/gnu/binutils/binutils-2.42.tar.xz + version: '2.42' +bzip2: + library_names: + - bz2 + license_file: LICENSE.bzip2.txt + licenses: + - bzip2-1.0.6 + sha256: ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269 + size: 810029 + url: https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz + version: 1.0.8 +cpython-3.10: + license_file: LICENSE.cpython.txt + licenses: + - Python-2.0 + - CNRI-Python + python_tag: cp310 + sha256: 9c50481faa8c2832329ba0fc8868d0a606a680fc4f60ec48d26ce8e076751fda + size: 19600188 + url: https://www.python.org/ftp/python/3.10.14/Python-3.10.14.tar.xz + version: 3.10.14 +cpython-3.11: + license_file: LICENSE.cpython.txt + licenses: + - Python-2.0 + - CNRI-Python + python_tag: cp311 + sha256: 9b1e896523fc510691126c864406d9360a3d1e986acbda59cda57b5abda45b87 + size: 20175816 + url: https://www.python.org/ftp/python/3.11.9/Python-3.11.9.tar.xz + version: 3.11.9 +cpython-3.12: + license_file: LICENSE.cpython.txt + licenses: + - Python-2.0 + - CNRI-Python + python_tag: cp312 + sha256: f6d419a6d8743ab26700801b4908d26d97e8b986e14f95de31b32de2b0e79554 + size: 20659356 + url: https://www.python.org/ftp/python/3.12.4/Python-3.12.4.tar.xz + version: 3.12.4 +cpython-3.8: + license_file: LICENSE.cpython.txt + licenses: + - Python-2.0 + - CNRI-Python + python_tag: cp38 + sha256: d2807ac69f69b84fd46a0b93bbd02a4fa48d3e70f4b2835ff0f72a2885040076 + size: 18975156 + url: https://www.python.org/ftp/python/3.8.19/Python-3.8.19.tar.xz + version: 3.8.19 +cpython-3.9: + license_file: LICENSE.cpython.txt + licenses: + - Python-2.0 + - CNRI-Python + python_tag: cp39 + sha256: d4892cd1618f6458cb851208c030df1482779609d0f3939991bd38184f8c679e + size: 19682840 + url: https://www.python.org/ftp/python/3.9.19/Python-3.9.19.tar.xz + version: 3.9.19 +expat: + library_names: + - expat + license_file: LICENSE.expat.txt + licenses: + - MIT + sha256: ef2420f0232c087801abf705e89ae65f6257df6b7931d37846a193ef2e8cdcbe + size: 460560 + url: https://github.com/libexpat/libexpat/releases/download/R_2_5_0/expat-2.5.0.tar.xz + version: 2.5.0 +inputproto: + sha256: 10eaadd531f38f7c92ab59ef0708ca195caf3164a75c4ed99f0c04f2913f6ef3 + size: 244334 + url: https://www.x.org/archive/individual/proto/inputproto-2.3.2.tar.gz + version: 2.3.2 +jom-windows-bin: + sha256: 128fdd846fe24f8594eed37d1d8929a0ea78df563537c0c1b1861a635013fff8 + size: 1213852 + url: http://download.qt.io/official_releases/jom/jom_1_1_3.zip + version: 1.1.3 +kbproto: + sha256: 828cb275b91268b1a3ea950d5c0c5eb076c678fdf005d517411f89cc8c3bb416 + size: 325858 + url: https://www.x.org/archive/individual/proto/kbproto-1.0.7.tar.gz + version: 1.0.7 +libX11: + library_names: + - X11 + - X11-xcb + license_file: LICENSE.libX11.txt + licenses: + - MIT + - X11 + sha256: 0fce5fc0a24a3dc728174eccd0cb8d6a1b37a2ec1654bd5628c84e5bc200d594 + size: 3168158 + url: https://www.x.org/archive/individual/lib/libX11-1.6.12.tar.gz + version: 1.6.12 +libXau: + library_names: + - Xau + license_file: LICENSE.libXau.txt + licenses: + - MIT + sha256: 3a321aaceb803577a4776a5efe78836eb095a9e44bbc7a465d29463e1a14f189 + size: 404973 + url: https://www.x.org/releases/individual/lib/libXau-1.0.11.tar.gz + version: 1.0.11 +# 20221009-3.1 fails to build on musl due to an includes issue. +libedit: + library_names: + - edit + license_file: LICENSE.libedit.txt + licenses: + - BSD-3-Clause + sha256: 6792a6a992050762edcca28ff3318cdb7de37dccf7bc30db59fcd7017eed13c5 + size: 524722 + url: https://thrysoee.dk/editline/libedit-20210910-3.1.tar.gz + version: 20210910-3.1 +libffi: + library_names: + - ffi + license_file: LICENSE.libffi.txt + licenses: + - MIT + sha256: b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e + size: 1391684 + url: https://github.com/libffi/libffi/releases/download/v3.4.6/libffi-3.4.6.tar.gz + version: 3.4.6 +libffi-3.3: + library_names: + - ffi + license_file: LICENSE.libffi.txt + licenses: + - MIT + sha256: 72fba7922703ddfa7a028d513ac15a85c8d54c8d67f55fa5a4802885dc652056 + size: 1305466 + url: https://github.com/libffi/libffi/releases/download/v3.3/libffi-3.3.tar.gz + version: '3.3' +libpthread-stubs: + sha256: 593196cc746173d1e25cb54a93a87fd749952df68699aab7e02c085530e87747 + size: 74938 + url: https://www.x.org/archive/individual/lib/libpthread-stubs-0.5.tar.gz + version: '0.5' +# Newer versions of libxcb require a modern Python to build. We can take this +# dependency once we feel like doing the work. +libxcb: + library_names: + - xcb + license_file: LICENSE.libxcb.txt + licenses: + - MIT + sha256: 2c7fcddd1da34d9b238c9caeda20d3bd7486456fc50b3cc6567185dbd5b0ad02 + size: 640322 + url: https://xcb.freedesktop.org/dist/libxcb-1.14.tar.gz + version: '1.14' +llvm-14-x86_64-linux: + sha256: 04cb77c660f09df017a57738ae9635ef23a506024789f2f18da1304b45af2023 + size: 158614671 + url: https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20220508/llvm-14.0.3+20220508-gnu_only-x86_64-unknown-linux-gnu.tar.zst + version: 14.0.3+20220508 +# Remember to update LLVM_URL in src/release.rs whenever upgrading. +llvm-18-x86_64-linux: + sha256: 080c233fc7d75031b187bbfef62a4f9abc01188effb0c68fbc7dc4bc7370ee5b + size: 242840506 + url: https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20240713/llvm-18.0.8+20240713-gnu_only-x86_64-unknown-linux-gnu.tar.zst + version: 18.0.8+20240713 +# Remember to update LLVM_URL in src/release.rs whenever upgrading. +llvm-aarch64-macos: + sha256: 320da8d639186e020e7d54cdc35b7a5473b36cef08fdf7b22c03b59a273ba593 + size: 136598617 + url: https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20240713/llvm-18.0.8+20240713-aarch64-apple-darwin.tar.zst + version: 18.0.8+20240713 +# Remember to update LLVM_URL in src/release.rs whenever upgrading. +llvm-x86_64-macos: + sha256: 3032161d1cadb8996b07fe5762444c956842b5a7d798b2fcfe5a04574fdf7549 + size: 136599290 + url: https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20240713/llvm-18.0.8+20240713-x86_64-apple-darwin.tar.zst + version: 18.0.8+20240713 +m4: + sha256: 63aede5c6d33b6d9b13511cd0be2cac046f2e70fd0a07aa9573a04a82783af96 + size: 1654908 + url: https://ftp.gnu.org/gnu/m4/m4-1.4.19.tar.xz + version: 1.4.19 +mpdecimal: + library_names: + - mpdec + license_file: LICENSE.mpdecimal.txt + licenses: + - BSD-2-Clause + sha256: 9f9cd4c041f99b5c49ffb7b59d9f12d95b683d88585608aa56a6307667b2b21f + size: 2584021 + url: https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-2.5.1.tar.gz + version: 2.5.1 +musl: + sha256: a9a118bbe84d8764da0ea0d28b3ab3fae8477fc7e4085d90102b8596fc7c75e4 + size: 1080786 + url: https://musl.libc.org/releases/musl-1.2.5.tar.gz + version: 1.2.5 +nasm-windows-bin: + sha256: 8af0ae5ceed63fa8a2ded611d44cc341027a91df22aaaa071efedc81437412a5 + size: 384826 + url: https://github.com/python/cpython-bin-deps/archive/nasm-2.11.06.tar.gz + version: 2.11.06 +ncurses: + library_names: + - ncurses + - ncursesw + - panel + - panelw + license_file: LICENSE.ncurses.txt + licenses: + - X11 + sha256: 136d91bc269a9a5785e5f9e980bc76ab57428f604ce3e5a5a90cebc767971cc6 + size: 3688489 + url: https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.5.tar.gz + version: '6.5' +# Remember to update OPENSSL_VERSION_INFO in verify_distribution.py whenever upgrading. +openssl-1.1: + library_names: + - crypto + - ssl + license_file: LICENSE.openssl-1.1.txt + licenses: + - OpenSSL + sha256: cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8 + size: 9893384 + url: https://www.openssl.org/source/openssl-1.1.1w.tar.gz + version: 1.1.1w +# We use OpenSSL 3.0 because it is an LTS release and has a longer support +# window. If CPython ends up gaining support for 3.1+ releases, we can consider +# using the latest available. +# Remember to update OPENSSL_VERSION_INFO in verify_distribution.py whenever upgrading. +openssl-3.0: + library_names: + - crypto + - ssl + license_file: LICENSE.openssl-3.txt + licenses: + - Apache-2.0 + sha256: eeca035d4dd4e84fc25846d952da6297484afa0650a6f84c682e39df3a4123ca + size: 15305497 + url: https://www.openssl.org/source/openssl-3.0.14.tar.gz + version: 3.0.14 +patchelf: + sha256: 39e8aeccd7495d54df094d2b4a7c08010ff7777036faaf24f28e07777d1598e2 + size: 173598 + url: https://github.com/NixOS/patchelf/releases/download/0.13.1/patchelf-0.13.1.tar.bz2 + version: 0.13.1 +pip: + sha256: 7cd207eed4c60b0f411b444cd1464198fe186671c323b6cd6d433ed80fc9d247 + size: 1824406 + url: https://files.pythonhosted.org/packages/e7/54/0c1c068542cee73d8863336e974fc881e608d0170f3af15d0c0f28644531/pip-24.1.2-py3-none-any.whl + version: 24.1.2 +readline: + library_names: + - readline + license_file: LICENSE.readline.txt + licenses: + - GPL-3.0-only + sha256: 3feb7171f16a84ee82ca18a36d7b9be109a52c04f492a053331d7d1095007c35 + size: 3043952 + url: https://ftp.gnu.org/gnu/readline/readline-8.2.tar.gz + version: '8.2' +setuptools: + sha256: fe384da74336c398e0d956d1cae0669bc02eed936cdb1d49b57de1990dc11ffc + size: 931070 + url: https://files.pythonhosted.org/packages/ef/15/88e46eb9387e905704b69849618e699dc2f54407d8953cc4ec4b8b46528d/setuptools-70.3.0-py3-none-any.whl + version: 70.3.0 +# Remember to update verify_distribution.py when version changed. +sqlite: + actual_version: 3.46.0.0 + library_names: + - sqlite3 + license_file: LICENSE.sqlite.txt + license_public_domain: true + sha256: 6f8e6a7b335273748816f9b3b62bbdc372a889de8782d7f048c653a447417a7d + size: 3265248 + url: https://www.sqlite.org/2024/sqlite-autoconf-3460000.tar.gz + version: '3460000' +strawberryperl: + sha256: 8b15c7c9574989568254a7859e473b7d5f68a1145d2e4418036600a81b13805c + size: 143242779 + url: http://strawberryperl.com/download/5.28.1.1/strawberry-perl-5.28.1.1-32bit-portable.zip + version: 5.28.1.1 +tcl: + library_names: + - tcl8.6 + license_file: LICENSE.tcl.txt + licenses: + - TCL + sha256: 26c995dd0f167e48b11961d891ee555f680c175f7173ff8cb829f4ebcde4c1a6 + size: 10353486 + url: https://prdownloads.sourceforge.net/tcl/tcl8.6.12-src.tar.gz + version: 8.6.12 +tix: + license_file: LICENSE.tix.txt + licenses: + - TCL + sha256: f7b21d115867a41ae5fd7c635a4c234d3ca25126c3661eb36028c6e25601f85e + size: 1836451 + url: https://github.com/python/cpython-source-deps/archive/tix-8.4.3.6.tar.gz + version: 8.4.3.6 +tk: + library_names: + - tk8.6 + license_file: LICENSE.tcl.txt + licenses: + - TCL + sha256: 12395c1f3fcb6bed2938689f797ea3cdf41ed5cb6c4766eec8ac949560310630 + size: 4515393 + url: https://prdownloads.sourceforge.net/tcl/tk8.6.12-src.tar.gz + version: 8.6.12 +tk-windows-bin: + git_commit: e3c3e9a2856124aa32b608632a52742d479eb7a9 + sha256: 01ad9c663659224e075d487cbc33ea2fed7a225593965b79bed92ca7f79b676f + size: 6787654 + url: https://github.com/python/cpython-bin-deps/archive/e3c3e9a2856124aa32b608632a52742d479eb7a9.tar.gz + version: 8.6.12 +uuid: + library_names: + - uuid + license_file: LICENSE.libuuid.txt + licenses: + - BSD-3-Clause + sha256: 46af3275291091009ad7f1b899de3d0cea0252737550e7919d17237997db5644 + size: 318256 + url: https://sourceforge.net/projects/libuuid/files/libuuid-1.0.3.tar.gz + version: 1.0.3 +x11-util-macros: + sha256: 8daf36913d551a90fd1013cb078401375dabae021cb4713b9b256a70f00eeb74 + size: 104931 + url: https://www.x.org/archive/individual/util/util-macros-1.20.0.tar.gz + version: 1.20.0 +xcb-proto: + sha256: 85cd21e9d9fbc341d0dbf11eace98d55d7db89fda724b0e598855fcddf0944fd + size: 194674 + url: https://www.x.org/archive/individual/proto/xcb-proto-1.14.1.tar.gz + version: 1.14.1 +xextproto: + sha256: 1b1bcdf91221e78c6c33738667a57bd9aaa63d5953174ad8ed9929296741c9f5 + size: 290814 + url: https://www.x.org/archive/individual/proto/xextproto-7.3.0.tar.gz + version: 7.3.0 +# Newer versions from at least 2023 have build failures for reasons we haven't +# fully investigated. +xorgproto: + sha256: 38ad1d8316515785d53c5162b4b7022918e03c11d72a5bd9df0a176607f42bca + size: 1119813 + url: https://www.x.org/archive/individual/proto/xorgproto-2019.1.tar.gz + version: '2019.1' +xproto: + sha256: 6d755eaae27b45c5cc75529a12855fed5de5969b367ed05003944cf901ed43c7 + size: 367979 + url: https://www.x.org/archive/individual/proto/xproto-7.0.31.tar.gz + version: 7.0.31 +xtrans: + sha256: a806f8a92f879dcd0146f3f1153fdffe845f2fc0df9b1a26c19312b7b0a29c86 + size: 230197 + url: https://www.x.org/archive/individual/lib/xtrans-1.5.0.tar.gz + version: 1.5.0 +# IMPORTANT: xz 5.6 has a backdoor. Be extremely cautious before taking any xz +# upgrade since it isn't clear which versions are safe. +xz: + library_names: + - lzma + # liblzma is in the public domain. Other parts of code have licenses. But + # we only use liblzma. + licenses: [] + license_file: LICENSE.liblzma.txt + license_public_domain: true + sha256: 61bda930767dcb170a5328a895ec74cab0f5aac4558cdda561c83559db582a13 + size: 2190541 + url: https://github.com/indygreg/python-build-standalone/releases/download/20240224/xz-5.2.12.tar.gz + version: 5.2.12 +zlib: + library_names: + - z + license_file: LICENSE.zlib.txt + licenses: + - Zlib + sha256: b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30 + size: 1497445 + url: https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.gz + version: 1.2.13 diff --git a/pythonbuild/buildenv.py b/pythonbuild/buildenv.py index 0ce3cd03..cc739c4e 100644 --- a/pythonbuild/buildenv.py +++ b/pythonbuild/buildenv.py @@ -12,7 +12,6 @@ import tempfile from .docker import container_exec, container_get_archive, copy_file_to_container -from .downloads import DOWNLOADS from .logging import log from .utils import ( clang_toolchain, @@ -39,9 +38,9 @@ def copy_file(self, source: pathlib.Path, dest_path=None, dest_name=None): copy_file_to_container(source, self.container, dest_path, dest_name) def install_toolchain_archive( - self, build_dir, package_name, host_platform, version=None + self, build_dir, package_name, host_platform, downloads, version=None ): - entry = DOWNLOADS[package_name] + entry = downloads[package_name] basename = "%s-%s-%s.tar" % ( package_name, version or entry["version"], @@ -53,9 +52,9 @@ def install_toolchain_archive( self.run(["/bin/tar", "-C", "/tools", "-xf", "/build/%s" % p.name]) def install_artifact_archive( - self, build_dir, package_name, target_triple, optimizations + self, build_dir, package_name, target_triple, optimizations, downloads ): - entry = DOWNLOADS[package_name] + entry = downloads[package_name] basename = "%s-%s-%s-%s.tar" % ( package_name, entry["version"], @@ -76,17 +75,19 @@ def install_toolchain( binutils=False, musl=False, clang=False, + *, + downloads, ): if binutils: - self.install_toolchain_archive(build_dir, "binutils", host_platform) + self.install_toolchain_archive(build_dir, "binutils", host_platform, downloads=downloads) if clang: self.install_toolchain_archive( - build_dir, clang_toolchain(host_platform, target_triple), host_platform + build_dir, clang_toolchain(host_platform, target_triple), host_platform, downloads=downloads ) if musl: - self.install_toolchain_archive(build_dir, "musl", host_platform) + self.install_toolchain_archive(build_dir, "musl", host_platform, downloads=downloads) def run(self, program, user="build", environment=None): if isinstance(program, str) and not program.startswith("/"): @@ -159,9 +160,9 @@ def copy_file(self, source: pathlib.Path, dest_path=None, dest_name=None): shutil.copy(source, dest_dir / dest_name) def install_toolchain_archive( - self, build_dir, package_name, host_platform, version=None + self, build_dir, package_name, host_platform, version=None, *, downloads ): - entry = DOWNLOADS[package_name] + entry = downloads[package_name] basename = "%s-%s-%s.tar" % ( package_name, version or entry["version"], @@ -174,9 +175,9 @@ def install_toolchain_archive( extract_tar_to_directory(p, dest_path) def install_artifact_archive( - self, build_dir, package_name, target_triple, optimizations + self, build_dir, package_name, target_triple, optimizations, *, downloads ): - entry = DOWNLOADS[package_name] + entry = downloads[package_name] basename = "%s-%s-%s-%s.tar" % ( package_name, entry["version"], @@ -197,17 +198,19 @@ def install_toolchain( binutils=False, musl=False, clang=False, + *, + downloads, ): if binutils: - self.install_toolchain_archive(build_dir, "binutils", platform) + self.install_toolchain_archive(build_dir, "binutils", platform, downloads=downloads) if clang: self.install_toolchain_archive( - build_dir, clang_toolchain(platform, target_triple), platform + build_dir, clang_toolchain(platform, target_triple), platform, downloads=downloads ) if musl: - self.install_toolchain_archive(build_dir, "musl", platform) + self.install_toolchain_archive(build_dir, "musl", platform, downloads=downloads) def run(self, program, user="build", environment=None): if user != "build": diff --git a/pythonbuild/downloads.py b/pythonbuild/downloads.py deleted file mode 100644 index 512f2d7f..00000000 --- a/pythonbuild/downloads.py +++ /dev/null @@ -1,406 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -DOWNLOADS = { - "autoconf": { - "url": "https://ftp.gnu.org/gnu/autoconf/autoconf-2.71.tar.gz", - "size": 2003781, - "sha256": "431075ad0bf529ef13cb41e9042c542381103e80015686222b8a9d4abef42a1c", - "version": "2.71", - }, - # 6.0.19 is the last version licensed under the Sleepycat license. - "bdb": { - "url": "https://ftp.osuosl.org/pub/blfs/conglomeration/db/db-6.0.19.tar.gz", - "size": 36541923, - "sha256": "2917c28f60903908c2ca4587ded1363b812c4e830a5326aaa77c9879d13ae18e", - "version": "6.0.19", - "library_names": ["db"], - "licenses": ["Sleepycat"], - "license_file": "LICENSE.bdb.txt", - }, - "binutils": { - "url": "https://ftp.gnu.org/gnu/binutils/binutils-2.42.tar.xz", - "size": 27567160, - "sha256": "f6e4d41fd5fc778b06b7891457b3620da5ecea1006c6a4a41ae998109f85a800", - "version": "2.42", - }, - "bzip2": { - "url": "https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz", - "size": 810029, - "sha256": "ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269", - "version": "1.0.8", - "library_names": ["bz2"], - "licenses": ["bzip2-1.0.6"], - "license_file": "LICENSE.bzip2.txt", - }, - "cpython-3.8": { - "url": "https://www.python.org/ftp/python/3.8.19/Python-3.8.19.tar.xz", - "size": 18975156, - "sha256": "d2807ac69f69b84fd46a0b93bbd02a4fa48d3e70f4b2835ff0f72a2885040076", - "version": "3.8.19", - "licenses": ["Python-2.0", "CNRI-Python"], - "license_file": "LICENSE.cpython.txt", - "python_tag": "cp38", - }, - "cpython-3.9": { - "url": "https://www.python.org/ftp/python/3.9.19/Python-3.9.19.tar.xz", - "size": 19682840, - "sha256": "d4892cd1618f6458cb851208c030df1482779609d0f3939991bd38184f8c679e", - "version": "3.9.19", - "licenses": ["Python-2.0", "CNRI-Python"], - "license_file": "LICENSE.cpython.txt", - "python_tag": "cp39", - }, - "cpython-3.10": { - "url": "https://www.python.org/ftp/python/3.10.14/Python-3.10.14.tar.xz", - "size": 19600188, - "sha256": "9c50481faa8c2832329ba0fc8868d0a606a680fc4f60ec48d26ce8e076751fda", - "version": "3.10.14", - "licenses": ["Python-2.0", "CNRI-Python"], - "license_file": "LICENSE.cpython.txt", - "python_tag": "cp310", - }, - "cpython-3.11": { - "url": "https://www.python.org/ftp/python/3.11.9/Python-3.11.9.tar.xz", - "size": 20175816, - "sha256": "9b1e896523fc510691126c864406d9360a3d1e986acbda59cda57b5abda45b87", - "version": "3.11.9", - "licenses": ["Python-2.0", "CNRI-Python"], - "license_file": "LICENSE.cpython.txt", - "python_tag": "cp311", - }, - "cpython-3.12": { - "url": "https://www.python.org/ftp/python/3.12.4/Python-3.12.4.tar.xz", - "size": 20659356, - "sha256": "f6d419a6d8743ab26700801b4908d26d97e8b986e14f95de31b32de2b0e79554", - "version": "3.12.4", - "licenses": ["Python-2.0", "CNRI-Python"], - "license_file": "LICENSE.cpython.txt", - "python_tag": "cp312", - }, - "expat": { - "url": "https://github.com/libexpat/libexpat/releases/download/R_2_5_0/expat-2.5.0.tar.xz", - "size": 460560, - "sha256": "ef2420f0232c087801abf705e89ae65f6257df6b7931d37846a193ef2e8cdcbe", - "version": "2.5.0", - "library_names": ["expat"], - "licenses": ["MIT"], - "license_file": "LICENSE.expat.txt", - }, - "inputproto": { - "url": "https://www.x.org/archive/individual/proto/inputproto-2.3.2.tar.gz", - "size": 244334, - "sha256": "10eaadd531f38f7c92ab59ef0708ca195caf3164a75c4ed99f0c04f2913f6ef3", - "version": "2.3.2", - }, - "jom-windows-bin": { - "url": "http://download.qt.io/official_releases/jom/jom_1_1_3.zip", - "size": 1213852, - "sha256": "128fdd846fe24f8594eed37d1d8929a0ea78df563537c0c1b1861a635013fff8", - "version": "1.1.3", - }, - "kbproto": { - "url": "https://www.x.org/archive/individual/proto/kbproto-1.0.7.tar.gz", - "size": 325858, - "sha256": "828cb275b91268b1a3ea950d5c0c5eb076c678fdf005d517411f89cc8c3bb416", - "version": "1.0.7", - }, - # 20221009-3.1 fails to build on musl due to an includes issue. - "libedit": { - "url": "https://thrysoee.dk/editline/libedit-20210910-3.1.tar.gz", - "size": 524722, - "sha256": "6792a6a992050762edcca28ff3318cdb7de37dccf7bc30db59fcd7017eed13c5", - "version": "20210910-3.1", - "library_names": ["edit"], - "licenses": ["BSD-3-Clause"], - "license_file": "LICENSE.libedit.txt", - }, - "libffi-3.3": { - "url": "https://github.com/libffi/libffi/releases/download/v3.3/libffi-3.3.tar.gz", - "size": 1305466, - "sha256": "72fba7922703ddfa7a028d513ac15a85c8d54c8d67f55fa5a4802885dc652056", - "version": "3.3", - "library_names": ["ffi"], - "licenses": ["MIT"], - "license_file": "LICENSE.libffi.txt", - }, - "libffi": { - "url": "https://github.com/libffi/libffi/releases/download/v3.4.6/libffi-3.4.6.tar.gz", - "size": 1391684, - "sha256": "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e", - "version": "3.4.6", - "library_names": ["ffi"], - "licenses": ["MIT"], - "license_file": "LICENSE.libffi.txt", - }, - "libpthread-stubs": { - "url": "https://www.x.org/archive/individual/lib/libpthread-stubs-0.5.tar.gz", - "size": 74938, - "sha256": "593196cc746173d1e25cb54a93a87fd749952df68699aab7e02c085530e87747", - "version": "0.5", - }, - "libX11": { - "url": "https://www.x.org/archive/individual/lib/libX11-1.6.12.tar.gz", - "size": 3168158, - "sha256": "0fce5fc0a24a3dc728174eccd0cb8d6a1b37a2ec1654bd5628c84e5bc200d594", - "version": "1.6.12", - "library_names": ["X11", "X11-xcb"], - "licenses": ["MIT", "X11"], - "license_file": "LICENSE.libX11.txt", - }, - "libXau": { - "url": "https://www.x.org/releases/individual/lib/libXau-1.0.11.tar.gz", - "size": 404973, - "sha256": "3a321aaceb803577a4776a5efe78836eb095a9e44bbc7a465d29463e1a14f189", - "version": "1.0.11", - "library_names": ["Xau"], - "licenses": ["MIT"], - "license_file": "LICENSE.libXau.txt", - }, - # Newer versions of libxcb require a modern Python to build. We can take this - # dependency once we feel like doing the work. - "libxcb": { - "url": "https://xcb.freedesktop.org/dist/libxcb-1.14.tar.gz", - "size": 640322, - "sha256": "2c7fcddd1da34d9b238c9caeda20d3bd7486456fc50b3cc6567185dbd5b0ad02", - "version": "1.14", - "library_names": ["xcb"], - "licenses": ["MIT"], - "license_file": "LICENSE.libxcb.txt", - }, - "llvm-14-x86_64-linux": { - "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20220508/llvm-14.0.3+20220508-gnu_only-x86_64-unknown-linux-gnu.tar.zst", - "size": 158614671, - "sha256": "04cb77c660f09df017a57738ae9635ef23a506024789f2f18da1304b45af2023", - "version": "14.0.3+20220508", - }, - # Remember to update LLVM_URL in src/release.rs whenever upgrading. - "llvm-18-x86_64-linux": { - "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20240713/llvm-18.0.8+20240713-gnu_only-x86_64-unknown-linux-gnu.tar.zst", - "size": 242840506, - "sha256": "080c233fc7d75031b187bbfef62a4f9abc01188effb0c68fbc7dc4bc7370ee5b", - "version": "18.0.8+20240713", - }, - # Remember to update LLVM_URL in src/release.rs whenever upgrading. - "llvm-aarch64-macos": { - "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20240713/llvm-18.0.8+20240713-aarch64-apple-darwin.tar.zst", - "size": 136598617, - "sha256": "320da8d639186e020e7d54cdc35b7a5473b36cef08fdf7b22c03b59a273ba593", - "version": "18.0.8+20240713", - }, - # Remember to update LLVM_URL in src/release.rs whenever upgrading. - "llvm-x86_64-macos": { - "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20240713/llvm-18.0.8+20240713-x86_64-apple-darwin.tar.zst", - "size": 136599290, - "sha256": "3032161d1cadb8996b07fe5762444c956842b5a7d798b2fcfe5a04574fdf7549", - "version": "18.0.8+20240713", - }, - "m4": { - "url": "https://ftp.gnu.org/gnu/m4/m4-1.4.19.tar.xz", - "size": 1654908, - "sha256": "63aede5c6d33b6d9b13511cd0be2cac046f2e70fd0a07aa9573a04a82783af96", - "version": "1.4.19", - }, - "mpdecimal": { - "url": "https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-2.5.1.tar.gz", - "size": 2584021, - "sha256": "9f9cd4c041f99b5c49ffb7b59d9f12d95b683d88585608aa56a6307667b2b21f", - "version": "2.5.1", - "library_names": ["mpdec"], - "licenses": ["BSD-2-Clause"], - "license_file": "LICENSE.mpdecimal.txt", - }, - "musl": { - "url": "https://musl.libc.org/releases/musl-1.2.5.tar.gz", - "size": 1080786, - "sha256": "a9a118bbe84d8764da0ea0d28b3ab3fae8477fc7e4085d90102b8596fc7c75e4", - "version": "1.2.5", - }, - "ncurses": { - "url": "https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.5.tar.gz", - "size": 3688489, - "sha256": "136d91bc269a9a5785e5f9e980bc76ab57428f604ce3e5a5a90cebc767971cc6", - "version": "6.5", - "library_names": ["ncurses", "ncursesw", "panel", "panelw"], - "licenses": ["X11"], - "license_file": "LICENSE.ncurses.txt", - }, - # Remember to update OPENSSL_VERSION_INFO in verify_distribution.py whenever upgrading. - "openssl-1.1": { - "url": "https://www.openssl.org/source/openssl-1.1.1w.tar.gz", - "size": 9893384, - "sha256": "cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8", - "version": "1.1.1w", - "library_names": ["crypto", "ssl"], - "licenses": ["OpenSSL"], - "license_file": "LICENSE.openssl-1.1.txt", - }, - # We use OpenSSL 3.0 because it is an LTS release and has a longer support - # window. If CPython ends up gaining support for 3.1+ releases, we can consider - # using the latest available. - # Remember to update OPENSSL_VERSION_INFO in verify_distribution.py whenever upgrading. - "openssl-3.0": { - "url": "https://www.openssl.org/source/openssl-3.0.14.tar.gz", - "size": 15305497, - "sha256": "eeca035d4dd4e84fc25846d952da6297484afa0650a6f84c682e39df3a4123ca", - "version": "3.0.14", - "library_names": ["crypto", "ssl"], - "licenses": ["Apache-2.0"], - "license_file": "LICENSE.openssl-3.txt", - }, - "nasm-windows-bin": { - "url": "https://github.com/python/cpython-bin-deps/archive/nasm-2.11.06.tar.gz", - "size": 384826, - "sha256": "8af0ae5ceed63fa8a2ded611d44cc341027a91df22aaaa071efedc81437412a5", - "version": "2.11.06", - }, - "patchelf": { - "url": "https://github.com/NixOS/patchelf/releases/download/0.13.1/patchelf-0.13.1.tar.bz2", - "size": 173598, - "sha256": "39e8aeccd7495d54df094d2b4a7c08010ff7777036faaf24f28e07777d1598e2", - "version": "0.13.1", - }, - "pip": { - "url": "https://files.pythonhosted.org/packages/e7/54/0c1c068542cee73d8863336e974fc881e608d0170f3af15d0c0f28644531/pip-24.1.2-py3-none-any.whl", - "size": 1824406, - "sha256": "7cd207eed4c60b0f411b444cd1464198fe186671c323b6cd6d433ed80fc9d247", - "version": "24.1.2", - }, - "readline": { - "url": "https://ftp.gnu.org/gnu/readline/readline-8.2.tar.gz", - "size": 3043952, - "sha256": "3feb7171f16a84ee82ca18a36d7b9be109a52c04f492a053331d7d1095007c35", - "version": "8.2", - "library_names": ["readline"], - "licenses": ["GPL-3.0-only"], - "license_file": "LICENSE.readline.txt", - }, - "setuptools": { - "url": "https://files.pythonhosted.org/packages/ef/15/88e46eb9387e905704b69849618e699dc2f54407d8953cc4ec4b8b46528d/setuptools-70.3.0-py3-none-any.whl", - "size": 931070, - "sha256": "fe384da74336c398e0d956d1cae0669bc02eed936cdb1d49b57de1990dc11ffc", - "version": "70.3.0", - }, - # Remember to update verify_distribution.py when version changed. - "sqlite": { - "url": "https://www.sqlite.org/2024/sqlite-autoconf-3460000.tar.gz", - "size": 3265248, - "sha256": "6f8e6a7b335273748816f9b3b62bbdc372a889de8782d7f048c653a447417a7d", - "version": "3460000", - "actual_version": "3.46.0.0", - "library_names": ["sqlite3"], - "licenses": [], - "license_file": "LICENSE.sqlite.txt", - "license_public_domain": True, - }, - "strawberryperl": { - "url": "http://strawberryperl.com/download/5.28.1.1/strawberry-perl-5.28.1.1-32bit-portable.zip", - "size": 143242779, - "sha256": "8b15c7c9574989568254a7859e473b7d5f68a1145d2e4418036600a81b13805c", - "version": "5.28.1.1", - }, - "tcl": { - "url": "https://prdownloads.sourceforge.net/tcl/tcl8.6.12-src.tar.gz", - "size": 10353486, - "sha256": "26c995dd0f167e48b11961d891ee555f680c175f7173ff8cb829f4ebcde4c1a6", - "version": "8.6.12", - "library_names": ["tcl8.6"], - "licenses": ["TCL"], - "license_file": "LICENSE.tcl.txt", - }, - "tix": { - "url": "https://github.com/python/cpython-source-deps/archive/tix-8.4.3.6.tar.gz", - "size": 1836451, - "sha256": "f7b21d115867a41ae5fd7c635a4c234d3ca25126c3661eb36028c6e25601f85e", - "version": "8.4.3.6", - "licenses": ["TCL"], - "license_file": "LICENSE.tix.txt", - }, - "tk": { - "url": "https://prdownloads.sourceforge.net/tcl/tk8.6.12-src.tar.gz", - "size": 4515393, - "sha256": "12395c1f3fcb6bed2938689f797ea3cdf41ed5cb6c4766eec8ac949560310630", - "version": "8.6.12", - "library_names": ["tk8.6"], - "licenses": ["TCL"], - "license_file": "LICENSE.tcl.txt", - }, - "tk-windows-bin": { - "url": "https://github.com/python/cpython-bin-deps/archive/e3c3e9a2856124aa32b608632a52742d479eb7a9.tar.gz", - "size": 6787654, - "sha256": "01ad9c663659224e075d487cbc33ea2fed7a225593965b79bed92ca7f79b676f", - "version": "8.6.12", - "git_commit": "e3c3e9a2856124aa32b608632a52742d479eb7a9", - }, - "uuid": { - "url": "https://sourceforge.net/projects/libuuid/files/libuuid-1.0.3.tar.gz", - "size": 318256, - "sha256": "46af3275291091009ad7f1b899de3d0cea0252737550e7919d17237997db5644", - "version": "1.0.3", - "library_names": ["uuid"], - "licenses": ["BSD-3-Clause"], - "license_file": "LICENSE.libuuid.txt", - }, - "x11-util-macros": { - "url": "https://www.x.org/archive/individual/util/util-macros-1.20.0.tar.gz", - "size": 104931, - "sha256": "8daf36913d551a90fd1013cb078401375dabae021cb4713b9b256a70f00eeb74", - "version": "1.20.0", - }, - "xcb-proto": { - "url": "https://www.x.org/archive/individual/proto/xcb-proto-1.14.1.tar.gz", - "size": 194674, - "sha256": "85cd21e9d9fbc341d0dbf11eace98d55d7db89fda724b0e598855fcddf0944fd", - "version": "1.14.1", - }, - "xextproto": { - "url": "https://www.x.org/archive/individual/proto/xextproto-7.3.0.tar.gz", - "size": 290814, - "sha256": "1b1bcdf91221e78c6c33738667a57bd9aaa63d5953174ad8ed9929296741c9f5", - "version": "7.3.0", - }, - # Newer versions from at least 2023 have build failures for reasons we haven't - # fully investigated. - "xorgproto": { - "url": "https://www.x.org/archive/individual/proto/xorgproto-2019.1.tar.gz", - "size": 1119813, - "sha256": "38ad1d8316515785d53c5162b4b7022918e03c11d72a5bd9df0a176607f42bca", - "version": "2019.1", - }, - "xproto": { - "url": "https://www.x.org/archive/individual/proto/xproto-7.0.31.tar.gz", - "size": 367979, - "sha256": "6d755eaae27b45c5cc75529a12855fed5de5969b367ed05003944cf901ed43c7", - "version": "7.0.31", - }, - "xtrans": { - "url": "https://www.x.org/archive/individual/lib/xtrans-1.5.0.tar.gz", - "size": 230197, - "sha256": "a806f8a92f879dcd0146f3f1153fdffe845f2fc0df9b1a26c19312b7b0a29c86", - "version": "1.5.0", - }, - # IMPORTANT: xz 5.6 has a backdoor. Be extremely cautious before taking any xz - # upgrade since it isn't clear which versions are safe. - "xz": { - "url": "https://github.com/indygreg/python-build-standalone/releases/download/20240224/xz-5.2.12.tar.gz", - "size": 2190541, - "sha256": "61bda930767dcb170a5328a895ec74cab0f5aac4558cdda561c83559db582a13", - "version": "5.2.12", - "library_names": ["lzma"], - # liblzma is in the public domain. Other parts of code have licenses. But - # we only use liblzma. - "licenses": [], - "license_file": "LICENSE.liblzma.txt", - "license_public_domain": True, - }, - "zlib": { - "url": "https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.gz", - "size": 1497445, - "sha256": "b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30", - "version": "1.2.13", - "library_names": ["z"], - "licenses": ["Zlib"], - "license_file": "LICENSE.zlib.txt", - }, -} diff --git a/pythonbuild/utils.py b/pythonbuild/utils.py index bf05d0d0..e805d6bc 100644 --- a/pythonbuild/utils.py +++ b/pythonbuild/utils.py @@ -17,6 +17,7 @@ import sys import tarfile import time +import typing import urllib.error import urllib.request import zipfile @@ -24,9 +25,16 @@ import yaml import zstandard -from .downloads import DOWNLOADS from .logging import log +Downloads = dict[str, dict[str, typing.Any]] + + +def get_downloads(downloads_path: pathlib.Path) -> Downloads: + """Obtain the parsed downloads YAML file.""" + with downloads_path.open("rb") as fh: + return yaml.load(fh, Loader=yaml.SafeLoader) + def get_targets(yaml_path: pathlib.Path): """Obtain the parsed targets YAML file.""" @@ -134,7 +142,7 @@ def write_if_different(p: pathlib.Path, data: bytes): def write_triples_makefiles( - targets, dest_dir: pathlib.Path, support_search_dir: pathlib.Path + targets, dest_dir: pathlib.Path, support_search_dir: pathlib.Path, downloads, ): """Write out makefiles containing make variable settings derived from config.""" dest_dir.mkdir(parents=True, exist_ok=True) @@ -161,7 +169,7 @@ def write_triples_makefiles( entry = clang_toolchain(host_platform, triple) lines.append( "CLANG_FILENAME := %s-%s-%s.tar\n" - % (entry, DOWNLOADS[entry]["version"], host_platform) + % (entry, downloads[entry]["version"], host_platform) ) lines.append( @@ -172,11 +180,11 @@ def write_triples_makefiles( write_if_different(makefile_path, "".join(lines).encode("ascii")) -def write_package_versions(dest_path: pathlib.Path): +def write_package_versions(dest_path: pathlib.Path, downloads): """Write out versions of packages to files in a directory.""" dest_path.mkdir(parents=True, exist_ok=True) - for k, v in DOWNLOADS.items(): + for k, v in downloads.items(): p = dest_path / ("VERSION.%s" % k) content = "%s_VERSION := %s\n" % (k.upper().replace("-", "_"), v["version"]) write_if_different(p, content.encode("ascii")) @@ -295,8 +303,8 @@ def download_to_path(url: str, path: pathlib.Path, size: int, sha256: str): print("successfully downloaded %s" % url) -def download_entry(key: str, dest_path: pathlib.Path, local_name=None) -> pathlib.Path: - entry = DOWNLOADS[key] +def download_entry(key: str, dest_path: pathlib.Path, downloads, local_name=None) -> pathlib.Path: + entry = downloads[key] url = entry["url"] size = entry["size"] sha256 = entry["sha256"] From 5580c25f474cda83f0ac11e7d6e81f243123c615 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Sun, 21 Jul 2024 20:44:56 -0400 Subject: [PATCH 2/2] Use keyword --- cpython-unix/build.py | 131 ++++++++++++++++++++++++++------------- cpython-windows/build.py | 51 ++++++++------- pythonbuild/buildenv.py | 54 ++++++++++++---- pythonbuild/utils.py | 18 ++++-- 4 files changed, 170 insertions(+), 84 deletions(-) diff --git a/cpython-unix/build.py b/cpython-unix/build.py index ebfff8be..1c71dae1 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -214,7 +214,7 @@ def add_target_env(env, build_platform, target_triple, build_env): env["EXTRA_TARGET_LDFLAGS"] = " ".join(extra_target_ldflags) -def toolchain_archive_path(package_name, host_platform, downloads): +def toolchain_archive_path(package_name, host_platform, *, downloads: Downloads): entry = downloads[package_name] basename = "%s-%s-%s.tar" % (package_name, entry["version"], host_platform) @@ -222,8 +222,8 @@ def toolchain_archive_path(package_name, host_platform, downloads): return BUILD / basename -def install_binutils(platform): - return platform != "macos" +def install_binutils(host_platform): + return host_platform != "macos" def simple_build( @@ -234,12 +234,13 @@ def simple_build( host_platform, target_triple, optimizations, - downloads: Downloads, dest_archive, extra_archives=None, tools_path="deps", + *, + downloads: Downloads, ): - archive = download_entry(entry, DOWNLOADS_PATH, downloads) + archive = download_entry(entry, DOWNLOADS_PATH, downloads=downloads) with build_environment(client, image) as build_env: if settings.get("needs_toolchain"): @@ -250,10 +251,13 @@ def simple_build( binutils=install_binutils(host_platform), clang=True, musl="musl" in target_triple, + downloads=downloads, ) for a in extra_archives or []: - build_env.install_artifact_archive(BUILD, a, target_triple, optimizations) + build_env.install_artifact_archive( + BUILD, a, target_triple, optimizations, downloads=downloads + ) build_env.copy_file(archive) build_env.copy_file(SUPPORT / ("build-%s.sh" % entry)) @@ -275,9 +279,9 @@ def simple_build( build_env.get_tools_archive(dest_archive, tools_path) -def build_binutils(client, image, host_platform, downloads): +def build_binutils(client, image, host_platform, *, downloads: Downloads): """Build binutils in the Docker image.""" - archive = download_entry("binutils", DOWNLOADS_PATH, downloads) + archive = download_entry("binutils", DOWNLOADS_PATH, downloads=downloads) with build_environment(client, image) as build_env: install_sccache(build_env) @@ -295,13 +299,18 @@ def build_binutils(client, image, host_platform, downloads): ) build_env.get_tools_archive( - toolchain_archive_path("binutils", host_platform, downloads), "host" + toolchain_archive_path( + "binutils", + host_platform, + downloads=downloads, + ), + "host", ) -def materialize_clang(host_platform: str, target_triple: str, downloads): +def materialize_clang(host_platform: str, target_triple: str, *, downloads: Downloads): entry = clang_toolchain(host_platform, target_triple) - tar_zst = download_entry(entry, DOWNLOADS_PATH, downloads) + tar_zst = download_entry(entry, DOWNLOADS_PATH, downloads=downloads) local_filename = "%s-%s-%s.tar" % ( entry, downloads[entry]["version"], @@ -315,12 +324,24 @@ def materialize_clang(host_platform: str, target_triple: str, downloads): dctx.copy_stream(ifh, ofh) -def build_musl(client, image, host_platform: str, target_triple: str, downloads): - musl_archive = download_entry("musl", DOWNLOADS_PATH, downloads) +def build_musl( + client, + image, + host_platform: str, + target_triple: str, + *, + downloads: Downloads, +): + musl_archive = download_entry("musl", DOWNLOADS_PATH, downloads=downloads) with build_environment(client, image) as build_env: build_env.install_toolchain( - BUILD, host_platform, target_triple, binutils=True, clang=True + BUILD, + host_platform, + target_triple, + binutils=True, + clang=True, + downloads=downloads, ) build_env.copy_file(musl_archive) build_env.copy_file(SUPPORT / "build-musl.sh") @@ -333,7 +354,12 @@ def build_musl(client, image, host_platform: str, target_triple: str, downloads) build_env.run("build-musl.sh", environment=env) build_env.get_tools_archive( - toolchain_archive_path("musl", host_platform), "host" + toolchain_archive_path( + "musl", + host_platform, + downloads=downloads, + ), + "host", ) @@ -345,9 +371,10 @@ def build_libedit( target_triple, optimizations, dest_archive, + *, downloads: Downloads, ): - libedit_archive = download_entry("libedit", DOWNLOADS_PATH, downloads) + libedit_archive = download_entry("libedit", DOWNLOADS_PATH, downloads=downloads) with build_environment(client, image) as build_env: if settings.get("needs_toolchain"): @@ -358,10 +385,11 @@ def build_libedit( binutils=install_binutils(host_platform), clang=True, musl="musl" in target_triple, + downloads=downloads, ) build_env.install_artifact_archive( - BUILD, "ncurses", target_triple, optimizations + BUILD, "ncurses", target_triple, optimizations, downloads=downloads ) build_env.copy_file(libedit_archive) build_env.copy_file(SUPPORT / "build-libedit.sh") @@ -384,11 +412,12 @@ def build_tix( target_triple, optimizations, dest_archive, + *, downloads: Downloads, ): - tcl_archive = download_entry("tcl", DOWNLOADS_PATH, downloads) - tk_archive = download_entry("tk", DOWNLOADS_PATH, downloads) - tix_archive = download_entry("tix", DOWNLOADS_PATH, downloads) + tcl_archive = download_entry("tcl", DOWNLOADS_PATH, downloads=downloads) + tk_archive = download_entry("tk", DOWNLOADS_PATH, downloads=downloads) + tix_archive = download_entry("tix", DOWNLOADS_PATH, downloads=downloads) with build_environment(client, image) as build_env: if settings.get("needs_toolchain"): @@ -399,6 +428,7 @@ def build_tix( binutils=install_binutils(host_platform), clang=True, musl="musl" in target_triple, + downloads=downloads, ) depends = {"tcl", "tk"} @@ -406,7 +436,9 @@ def build_tix( depends |= {"libX11", "xorgproto"} for p in sorted(depends): - build_env.install_artifact_archive(BUILD, p, target_triple, optimizations) + build_env.install_artifact_archive( + BUILD, p, target_triple, optimizations, downloads=downloads + ) for p in (tcl_archive, tk_archive, tix_archive, SUPPORT / "build-tix.sh"): build_env.copy_file(p) @@ -432,10 +464,11 @@ def build_cpython_host( target_triple: str, optimizations: str, dest_archive, + *, downloads: Downloads, ): """Build binutils in the Docker image.""" - archive = download_entry(entry, DOWNLOADS_PATH, downloads) + archive = download_entry(entry, DOWNLOADS_PATH, downloads=downloads) with build_environment(client, image) as build_env: python_version = downloads[entry]["version"] @@ -446,6 +479,7 @@ def build_cpython_host( target_triple, binutils=install_binutils(host_platform), clang=True, + downloads=downloads, ) build_env.copy_file(archive) @@ -463,7 +497,9 @@ def build_cpython_host( "m4", } for p in sorted(packages): - build_env.install_artifact_archive(BUILD, p, target_triple, optimizations) + build_env.install_artifact_archive( + BUILD, p, target_triple, optimizations, downloads=downloads + ) env = { "PYTHON_VERSION": python_version, @@ -492,12 +528,13 @@ def build_cpython_host( def python_build_info( build_env, version, - platform, + host_platform, target_triple, musl, optimizations, extensions, extra_metadata, + *, downloads: Downloads, ): """Obtain build metadata for the Python distribution.""" @@ -508,7 +545,7 @@ def python_build_info( binary_suffix = "" - if platform == "linux64": + if host_platform == "linux64": bi["core"]["static_lib"] = ( "install/lib/python{version}/config-{version}{binary_suffix}-x86_64-linux-gnu/libpython{version}{binary_suffix}.a".format( version=version, binary_suffix=binary_suffix @@ -522,7 +559,7 @@ def python_build_info( ) if optimizations in ("lto", "pgo+lto"): - llvm_version = downloads[clang_toolchain(platform, target_triple)][ + llvm_version = downloads[clang_toolchain(host_platform, target_triple)][ "version" ] if "+" in llvm_version: @@ -531,7 +568,7 @@ def python_build_info( object_file_format = f"llvm-bitcode:%{llvm_version}" else: object_file_format = "elf" - elif platform == "macos": + elif host_platform == "macos": bi["core"]["static_lib"] = ( "install/lib/python{version}/config-{version}{binary_suffix}-darwin/libpython{version}{binary_suffix}.a".format( version=version, binary_suffix=binary_suffix @@ -549,7 +586,7 @@ def python_build_info( else: object_file_format = "mach-o" else: - raise Exception("unsupported platform: %s" % platform) + raise Exception("unsupported platform: %s" % host_platform) bi["object_file_format"] = object_file_format @@ -564,9 +601,9 @@ def python_build_info( if lib.startswith("-l"): lib = lib[2:] - if platform == "linux64" and lib not in LINUX_ALLOW_SYSTEM_LIBRARIES: + if host_platform == "linux64" and lib not in LINUX_ALLOW_SYSTEM_LIBRARIES: raise Exception("unexpected library in LIBS (%s): %s" % (libs, lib)) - elif platform == "macos" and lib not in MACOS_ALLOW_SYSTEM_LIBRARIES: + elif host_platform == "macos" and lib not in MACOS_ALLOW_SYSTEM_LIBRARIES: raise Exception("unexpected library in LIBS (%s): %s" % (libs, lib)) log("adding core system link library: %s" % lib) @@ -681,7 +718,7 @@ def python_build_info( extension_suffix = extra_metadata["python_config_vars"]["EXT_SUFFIX"] entry["shared_lib"] = "%s/%s%s" % (shared_dir, extension, extension_suffix) - add_licenses_to_extension_entry(entry) + add_licenses_to_extension_entry(entry, downloads=downloads) bi["extensions"].setdefault(extension, []).append(entry) @@ -701,17 +738,18 @@ def build_cpython( host_platform, target_triple, optimizations, - downloads: Downloads, dest_archive, version=None, python_source=None, + *, + downloads: Downloads, ): """Build CPython in a Docker image'""" entry_name = "cpython-%s" % version entry = downloads[entry_name] if not python_source: python_version = entry["version"] - python_archive = download_entry(entry_name, DOWNLOADS_PATH, downloads) + python_archive = download_entry(entry_name, DOWNLOADS_PATH, downloads=downloads) else: python_version = os.environ["PYBUILD_PYTHON_VERSION"] python_archive = DOWNLOADS_PATH / ("Python-%s.tar.xz" % python_version) @@ -721,8 +759,10 @@ def build_cpython( fh, python_source, path_prefix="Python-%s" % python_version ) - setuptools_archive = download_entry("setuptools", DOWNLOADS_PATH, downloads) - pip_archive = download_entry("pip", DOWNLOADS_PATH, downloads) + setuptools_archive = download_entry( + "setuptools", DOWNLOADS_PATH, downloads=downloads + ) + pip_archive = download_entry("pip", DOWNLOADS_PATH, downloads=downloads) ems = extension_modules_config(EXTENSION_MODULES) @@ -746,6 +786,7 @@ def build_cpython( binutils=install_binutils(host_platform), clang=True, musl="musl" in target_triple, + downloads=downloads, ) packages = target_needs(TARGETS_CONFIG, target_triple, python_version) @@ -754,10 +795,16 @@ def build_cpython( packages.discard("musl") for p in sorted(packages): - build_env.install_artifact_archive(BUILD, p, target_triple, optimizations) + build_env.install_artifact_archive( + BUILD, p, target_triple, optimizations, downloads=downloads + ) build_env.install_toolchain_archive( - BUILD, entry_name, host_platform, version=python_version + BUILD, + entry_name, + host_platform, + version=python_version, + downloads=downloads, ) for p in ( @@ -1008,9 +1055,9 @@ def main(): write_dockerfiles(SUPPORT, BUILD) elif action == "makefiles": targets = get_targets(TARGETS_CONFIG) - write_triples_makefiles(targets, BUILD, SUPPORT) + write_triples_makefiles(targets, BUILD, SUPPORT, downloads=downloads) write_target_settings(targets, BUILD / "targets") - write_package_versions(BUILD / "versions") + write_package_versions(BUILD / "versions", downloads=downloads) # Override the DOWNLOADS package entry for CPython for the local build if python_source: @@ -1056,9 +1103,9 @@ def main(): target_triple=target_triple, optimizations=optimizations, dest_archive=dest_archive, - downloads=downloads, tools_path="host", extra_archives=["m4"], + downloads=downloads, ) elif action == "libedit": @@ -1110,8 +1157,8 @@ def main(): target_triple=target_triple, optimizations=optimizations, dest_archive=dest_archive, - downloads=downloads, tools_path=tools_path, + downloads=downloads, ) elif action == "libX11": @@ -1243,9 +1290,9 @@ def main(): target_triple=target_triple, optimizations=optimizations, dest_archive=dest_archive, - downloads=downloads, version=action.split("-")[1], python_source=python_source, + downloads=downloads, ) else: diff --git a/cpython-windows/build.py b/cpython-windows/build.py index 85a2eaf0..9a44e205 100644 --- a/cpython-windows/build.py +++ b/cpython-windows/build.py @@ -23,6 +23,7 @@ parse_config_c, ) from pythonbuild.utils import ( + Downloads, compress_python_archive, create_tar_from_directory, download_entry, @@ -471,17 +472,14 @@ def hack_project_files( cpython_source_path: pathlib.Path, build_directory: str, python_version: str, - downloads, + *, + downloads: Downloads, ): """Hacks Visual Studio project files to work with our build.""" pcbuild_path = cpython_source_path / "PCbuild" - hack_props( - td, - pcbuild_path, - build_directory, - ) + hack_props(td, pcbuild_path, build_directory, downloads=downloads) # Our SQLite directory is named weirdly. This throws off version detection # in the project file. Replace the parsing logic with a static string. @@ -964,6 +962,7 @@ def build_openssl( perl_path: pathlib.Path, arch: str, dest_archive: pathlib.Path, + *, downloads, ): """Build OpenSSL from sources using the Perl executable specified.""" @@ -971,9 +970,9 @@ def build_openssl( openssl_version = downloads[entry]["version"] # First ensure the dependencies are in place. - openssl_archive = download_entry(entry, downloads, BUILD) - nasm_archive = download_entry("nasm-windows-bin", downloads, BUILD) - jom_archive = download_entry("jom-windows-bin", downloads, BUILD) + openssl_archive = download_entry(entry, BUILD, downloads=downloads) + nasm_archive = download_entry("nasm-windows-bin", BUILD, downloads=downloads) + jom_archive = download_entry("jom-windows-bin", BUILD, downloads=downloads) with tempfile.TemporaryDirectory(prefix="openssl-build-") as td: td = pathlib.Path(td) @@ -1025,7 +1024,8 @@ def build_libffi( sh_exe: pathlib.Path, msvc_version: str, dest_archive: pathlib.Path, - downloads, + *, + downloads: Downloads, ): with tempfile.TemporaryDirectory(prefix="libffi-build-") as td: td = pathlib.Path(td) @@ -1063,7 +1063,7 @@ def build_libffi( ) # We build libffi by running the build script that CPython ships. - python_archive = download_entry(python, downloads, BUILD) + python_archive = download_entry(python, BUILD, downloads=downloads) extract_tar_to_directory(python_archive, td) python_entry = downloads[python] @@ -1117,6 +1117,7 @@ def collect_python_build_artifacts( arch: str, config: str, openssl_entry: str, + *, downloads, ): """Collect build artifacts from Python. @@ -1389,7 +1390,8 @@ def build_cpython( openssl_archive, libffi_archive, openssl_entry: str, - downloads, + *, + downloads: Downloads, ) -> pathlib.Path: pgo = profile == "pgo" @@ -1399,21 +1401,21 @@ def build_cpython( # The python.props file keys off MSBUILD, so it needs to be set. os.environ["MSBUILD"] = str(msbuild) - bzip2_archive = download_entry("bzip2", downloads, BUILD) - sqlite_archive = download_entry("sqlite", downloads, BUILD) + bzip2_archive = download_entry("bzip2", BUILD, downloads=downloads) + sqlite_archive = download_entry("sqlite", BUILD, downloads=downloads) tk_bin_archive = download_entry( - "tk-windows-bin", BUILD, local_name="tk-windows-bin.tar.gz" + "tk-windows-bin", BUILD, local_name="tk-windows-bin.tar.gz", downloads=downloads ) - xz_archive = download_entry("xz", downloads, BUILD) - zlib_archive = download_entry("zlib", downloads, BUILD) + xz_archive = download_entry("xz", BUILD, downloads=downloads) + zlib_archive = download_entry("zlib", BUILD, downloads=downloads) - python_archive = download_entry(python_entry_name, downloads, BUILD) + python_archive = download_entry(python_entry_name, BUILD, downloads=downloads) entry = downloads[python_entry_name] python_version = entry["version"] - setuptools_wheel = download_entry("setuptools", downloads, BUILD) - pip_wheel = download_entry("pip", downloads, BUILD) + setuptools_wheel = download_entry("setuptools", BUILD, downloads=downloads) + pip_wheel = download_entry("pip", BUILD, downloads=downloads) if arch == "amd64": build_platform = "x64" @@ -1483,6 +1485,7 @@ def build_cpython( cpython_source_path, build_directory, python_version=python_version, + downloads=downloads, ) if pgo: @@ -1810,8 +1813,8 @@ def build_cpython( return dest_path -def fetch_strawberry_perl() -> pathlib.Path: - strawberryperl_zip = download_entry("strawberryperl", downloads, BUILD) +def fetch_strawberry_perl(*, downloads: Downloads) -> pathlib.Path: + strawberryperl_zip = download_entry("strawberryperl", BUILD, downloads=downloads) strawberryperl = BUILD / "strawberry-perl" strawberryperl.mkdir(exist_ok=True) with zipfile.ZipFile(strawberryperl_zip) as zf: @@ -1887,7 +1890,9 @@ def main() -> None: "%s-%s-%s.tar" % (openssl_entry, target_triple, args.profile) ) if not openssl_archive.exists(): - perl_path = fetch_strawberry_perl() / "perl" / "bin" / "perl.exe" + perl_path = ( + fetch_strawberry_perl(downloads=downloads) / "perl" / "bin" / "perl.exe" + ) LOG_PREFIX[0] = "openssl" build_openssl( openssl_entry, diff --git a/pythonbuild/buildenv.py b/pythonbuild/buildenv.py index cc739c4e..89003597 100644 --- a/pythonbuild/buildenv.py +++ b/pythonbuild/buildenv.py @@ -10,10 +10,12 @@ import shutil import tarfile import tempfile +import typing from .docker import container_exec, container_get_archive, copy_file_to_container from .logging import log from .utils import ( + Downloads, clang_toolchain, create_tar_from_directory, exec_and_log, @@ -38,7 +40,13 @@ def copy_file(self, source: pathlib.Path, dest_path=None, dest_name=None): copy_file_to_container(source, self.container, dest_path, dest_name) def install_toolchain_archive( - self, build_dir, package_name, host_platform, downloads, version=None + self, + build_dir, + package_name, + host_platform, + version=None, + *, + downloads: Downloads, ): entry = downloads[package_name] basename = "%s-%s-%s.tar" % ( @@ -52,7 +60,13 @@ def install_toolchain_archive( self.run(["/bin/tar", "-C", "/tools", "-xf", "/build/%s" % p.name]) def install_artifact_archive( - self, build_dir, package_name, target_triple, optimizations, downloads + self, + build_dir, + package_name, + target_triple, + optimizations, + *, + downloads: Downloads, ): entry = downloads[package_name] basename = "%s-%s-%s-%s.tar" % ( @@ -75,19 +89,26 @@ def install_toolchain( binutils=False, musl=False, clang=False, - *, - downloads, + *, + downloads: Downloads, ): if binutils: - self.install_toolchain_archive(build_dir, "binutils", host_platform, downloads=downloads) + self.install_toolchain_archive( + build_dir, "binutils", host_platform, downloads=downloads + ) if clang: self.install_toolchain_archive( - build_dir, clang_toolchain(host_platform, target_triple), host_platform, downloads=downloads + build_dir, + clang_toolchain(host_platform, target_triple), + host_platform, + downloads=downloads, ) if musl: - self.install_toolchain_archive(build_dir, "musl", host_platform, downloads=downloads) + self.install_toolchain_archive( + build_dir, "musl", host_platform, downloads=downloads + ) def run(self, program, user="build", environment=None): if isinstance(program, str) and not program.startswith("/"): @@ -198,19 +219,26 @@ def install_toolchain( binutils=False, musl=False, clang=False, - *, - downloads, + *, + downloads, ): if binutils: - self.install_toolchain_archive(build_dir, "binutils", platform, downloads=downloads) + self.install_toolchain_archive( + build_dir, "binutils", platform, downloads=downloads + ) if clang: self.install_toolchain_archive( - build_dir, clang_toolchain(platform, target_triple), platform, downloads=downloads + build_dir, + clang_toolchain(platform, target_triple), + platform, + downloads=downloads, ) if musl: - self.install_toolchain_archive(build_dir, "musl", platform, downloads=downloads) + self.install_toolchain_archive( + build_dir, "musl", platform, downloads=downloads + ) def run(self, program, user="build", environment=None): if user != "build": @@ -261,7 +289,7 @@ def find_output_files(self, base_path, pattern): @contextlib.contextmanager -def build_environment(client, image): +def build_environment(client, image) -> typing.Generator[ContainerContext, None, None]: if client is not None: container = client.containers.run( image, command=["/bin/sleep", "86400"], detach=True diff --git a/pythonbuild/utils.py b/pythonbuild/utils.py index e805d6bc..387a6b07 100644 --- a/pythonbuild/utils.py +++ b/pythonbuild/utils.py @@ -33,7 +33,7 @@ def get_downloads(downloads_path: pathlib.Path) -> Downloads: """Obtain the parsed downloads YAML file.""" with downloads_path.open("rb") as fh: - return yaml.load(fh, Loader=yaml.SafeLoader) + return typing.cast(Downloads, yaml.load(fh, Loader=yaml.SafeLoader)) def get_targets(yaml_path: pathlib.Path): @@ -142,7 +142,11 @@ def write_if_different(p: pathlib.Path, data: bytes): def write_triples_makefiles( - targets, dest_dir: pathlib.Path, support_search_dir: pathlib.Path, downloads, + targets, + dest_dir: pathlib.Path, + support_search_dir: pathlib.Path, + *, + downloads: Downloads, ): """Write out makefiles containing make variable settings derived from config.""" dest_dir.mkdir(parents=True, exist_ok=True) @@ -180,7 +184,7 @@ def write_triples_makefiles( write_if_different(makefile_path, "".join(lines).encode("ascii")) -def write_package_versions(dest_path: pathlib.Path, downloads): +def write_package_versions(dest_path: pathlib.Path, *, downloads: Downloads): """Write out versions of packages to files in a directory.""" dest_path.mkdir(parents=True, exist_ok=True) @@ -303,7 +307,9 @@ def download_to_path(url: str, path: pathlib.Path, size: int, sha256: str): print("successfully downloaded %s" % url) -def download_entry(key: str, dest_path: pathlib.Path, downloads, local_name=None) -> pathlib.Path: +def download_entry( + key: str, dest_path: pathlib.Path, local_name=None, *, downloads: Downloads +) -> pathlib.Path: entry = downloads[key] url = entry["url"] size = entry["size"] @@ -457,7 +463,7 @@ def compress_python_archive( return dest_path -def add_licenses_to_extension_entry(entry): +def add_licenses_to_extension_entry(entry, *, downloads: Downloads): """Add licenses keys to a ``extensions`` entry for JSON distribution info.""" have_licenses = False @@ -472,7 +478,7 @@ def add_licenses_to_extension_entry(entry): if "path_static" in link or "path_dynamic" in link: have_local_link = True - for value in DOWNLOADS.values(): + for value in downloads.values(): if name not in value.get("library_names", []): continue