From e3c940681a38131a491263d721f14bd8fe528273 Mon Sep 17 00:00:00 2001 From: Ted Pudlik Date: Sat, 21 Dec 2024 19:40:09 -0800 Subject: [PATCH 1/5] fix: py_proto_library: external runfiles (#2516) Previously, the import path within the runfiles was only correct for the case --legacy_external_runfiles=True (which copied the runfiles into `$RUNFILES/
/external//` in addition to `$RUNFILES//`. This flag was flipped to False in Bazel 8.0.0. Fixes https://github.com/bazelbuild/rules_python/issues/2515. Tested locally against the minimal reproducer in that issue. --- .bazelignore | 1 + CHANGELOG.md | 1 + examples/bzlmod/.bazelignore | 1 + examples/bzlmod/MODULE.bazel | 7 ++++++ examples/bzlmod/py_proto_library/BUILD.bazel | 16 ++++++++++++++ .../py_proto_library/foo_external/BUILD.bazel | 22 +++++++++++++++++++ .../foo_external/MODULE.bazel | 8 +++++++ .../py_proto_library/foo_external/WORKSPACE | 0 .../foo_external/nested/foo/my_proto.proto | 6 +++++ .../foo_external/py_binary_with_proto.py | 5 +++++ python/private/proto/py_proto_library.bzl | 7 +++++- 11 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 examples/bzlmod/py_proto_library/foo_external/BUILD.bazel create mode 100644 examples/bzlmod/py_proto_library/foo_external/MODULE.bazel create mode 100644 examples/bzlmod/py_proto_library/foo_external/WORKSPACE create mode 100644 examples/bzlmod/py_proto_library/foo_external/nested/foo/my_proto.proto create mode 100644 examples/bzlmod/py_proto_library/foo_external/py_binary_with_proto.py diff --git a/.bazelignore b/.bazelignore index 60d680e9f0..e10af2035d 100644 --- a/.bazelignore +++ b/.bazelignore @@ -18,6 +18,7 @@ examples/bzlmod/other_module/bazel-bin examples/bzlmod/other_module/bazel-other_module examples/bzlmod/other_module/bazel-out examples/bzlmod/other_module/bazel-testlogs +examples/bzlmod/py_proto_library/foo_external examples/bzlmod_build_file_generation/bazel-bzlmod_build_file_generation examples/multi_python_versions/bazel-multi_python_versions examples/pip_parse/bazel-pip_parse diff --git a/CHANGELOG.md b/CHANGELOG.md index 5583399e96..9976d2027c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,7 @@ Unreleased changes template. * (pypi) Using {bzl:obj}`pip_parse.experimental_requirement_cycles` and {bzl:obj}`pip_parse.use_hub_alias_dependencies` together now works when using WORKSPACE files. +* (py_proto_library) Fix import paths in Bazel 8. [pep-695]: https://peps.python.org/pep-0695/ diff --git a/examples/bzlmod/.bazelignore b/examples/bzlmod/.bazelignore index ab3eb1635c..3927f8e910 100644 --- a/examples/bzlmod/.bazelignore +++ b/examples/bzlmod/.bazelignore @@ -1 +1,2 @@ other_module +py_proto_library/foo_external diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 0a31c3beb8..536e3b2b67 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -5,6 +5,7 @@ module( ) bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "platforms", version = "0.0.4") bazel_dep(name = "rules_python", version = "0.0.0") local_path_override( module_name = "rules_python", @@ -272,5 +273,11 @@ local_path_override( path = "other_module", ) +bazel_dep(name = "foo_external", version = "") +local_path_override( + module_name = "foo_external", + path = "py_proto_library/foo_external", +) + # example test dependencies bazel_dep(name = "rules_shell", version = "0.3.0", dev_dependency = True) diff --git a/examples/bzlmod/py_proto_library/BUILD.bazel b/examples/bzlmod/py_proto_library/BUILD.bazel index d0bc683021..24436b48ea 100644 --- a/examples/bzlmod/py_proto_library/BUILD.bazel +++ b/examples/bzlmod/py_proto_library/BUILD.bazel @@ -1,3 +1,4 @@ +load("@bazel_skylib//rules:native_binary.bzl", "native_test") load("@rules_python//python:py_test.bzl", "py_test") py_test( @@ -16,3 +17,18 @@ py_test( "//py_proto_library/example.com/another_proto:message_proto_py_pb2", ], ) + +# Regression test for https://github.com/bazelbuild/rules_python/issues/2515 +# +# This test failed before https://github.com/bazelbuild/rules_python/pull/2516 +# when ran with --legacy_external_runfiles=False (default in Bazel 8.0.0). +native_test( + name = "external_import_test", + src = "@foo_external//:py_binary_with_proto", + # Incompatible with Windows: native_test wrapping a py_binary doesn't work + # on Windows. + target_compatible_with = select({ + "@platforms//os:windows": ["@platforms//:incompatible"], + "//conditions:default": [], + }), +) diff --git a/examples/bzlmod/py_proto_library/foo_external/BUILD.bazel b/examples/bzlmod/py_proto_library/foo_external/BUILD.bazel new file mode 100644 index 0000000000..3fa22e06e7 --- /dev/null +++ b/examples/bzlmod/py_proto_library/foo_external/BUILD.bazel @@ -0,0 +1,22 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") +load("@rules_python//python:proto.bzl", "py_proto_library") +load("@rules_python//python:py_binary.bzl", "py_binary") + +package(default_visibility = ["//visibility:public"]) + +proto_library( + name = "proto_lib", + srcs = ["nested/foo/my_proto.proto"], + strip_import_prefix = "/nested/foo", +) + +py_proto_library( + name = "a_proto", + deps = [":proto_lib"], +) + +py_binary( + name = "py_binary_with_proto", + srcs = ["py_binary_with_proto.py"], + deps = [":a_proto"], +) diff --git a/examples/bzlmod/py_proto_library/foo_external/MODULE.bazel b/examples/bzlmod/py_proto_library/foo_external/MODULE.bazel new file mode 100644 index 0000000000..5063f9b2d1 --- /dev/null +++ b/examples/bzlmod/py_proto_library/foo_external/MODULE.bazel @@ -0,0 +1,8 @@ +module( + name = "foo_external", + version = "0.0.1", +) + +bazel_dep(name = "rules_python", version = "1.0.0") +bazel_dep(name = "protobuf", version = "28.2", repo_name = "com_google_protobuf") +bazel_dep(name = "rules_proto", version = "7.0.2") diff --git a/examples/bzlmod/py_proto_library/foo_external/WORKSPACE b/examples/bzlmod/py_proto_library/foo_external/WORKSPACE new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/bzlmod/py_proto_library/foo_external/nested/foo/my_proto.proto b/examples/bzlmod/py_proto_library/foo_external/nested/foo/my_proto.proto new file mode 100644 index 0000000000..7b8440cbed --- /dev/null +++ b/examples/bzlmod/py_proto_library/foo_external/nested/foo/my_proto.proto @@ -0,0 +1,6 @@ +syntax = "proto3"; + +package my_proto; + +message MyMessage { +} diff --git a/examples/bzlmod/py_proto_library/foo_external/py_binary_with_proto.py b/examples/bzlmod/py_proto_library/foo_external/py_binary_with_proto.py new file mode 100644 index 0000000000..be34264b5a --- /dev/null +++ b/examples/bzlmod/py_proto_library/foo_external/py_binary_with_proto.py @@ -0,0 +1,5 @@ +import sys + +if __name__ == "__main__": + import my_proto_pb2 + sys.exit(0) diff --git a/python/private/proto/py_proto_library.bzl b/python/private/proto/py_proto_library.bzl index d810e58c24..1e9df848ab 100644 --- a/python/private/proto/py_proto_library.bzl +++ b/python/private/proto/py_proto_library.bzl @@ -98,7 +98,12 @@ def _py_proto_aspect_impl(target, ctx): proto_root = proto_root[len(ctx.bin_dir.path) + 1:] plugin_output = ctx.bin_dir.path + "/" + proto_root - proto_root = ctx.workspace_name + "/" + proto_root + + # Import path within the runfiles tree + if proto_root.startswith("external/"): + proto_root = proto_root[len("external") + 1:] + else: + proto_root = ctx.workspace_name + "/" + proto_root proto_common.compile( actions = ctx.actions, From be950f9c2448f85332322fd4f7918b940bf45bfd Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 23 Dec 2024 21:44:54 +0900 Subject: [PATCH 2/5] refactor(pypi): A better error message when the wheel select hits no_match (#2519) With this change we get the current values of the python configuration values printed in addition to the message printed previously. This should help us advise users who don't have their builds configured correctly. We are adding an extra `build_setting` which we can set in order to get an error message instead of a `DEBUG` warning. This has been documented as part of our config settings and in the `no_match_error` in the `select` statement. Example output now ```console $ bazel cquery --@rules_python//python/config_settings:python_version=3.12 @dev_pip//sphinx DEBUG: /home/aignas/src/github/aignas/rules_python/python/private/config_settings.bzl:193:14: The current configuration rules_python config flags is: @@//python/config_settings:pip_whl: "auto" @@//python/config_settings:pip_whl_glibc_version: "" @@//python/config_settings:pip_whl_muslc_version: "" @@//python/config_settings:pip_whl_osx_arch: "arch" @@//python/config_settings:pip_whl_osx_version: "" @@//python/config_settings:py_freethreaded: "no" @@//python/config_settings:py_linux_libc: "glibc" @@//python/config_settings:python_version: "3.12" If the value is missing, then the default value is being used, see documentation: https://rules-python.readthedocs.io/en/latest/api/rules_python/python/config_settings ERROR: /home/aignas/.cache/bazel/_bazel_aignas/6f0de8c9128ee8d5dbf27ba6dcc48bdd/external/+pip+dev_pip/sphinx/BUILD.bazel:6:12: configurable attribute "actual" in @@+pip+dev_pip//sphinx:_no_matching_repository doesn't match this configuration: No matching wheel for current configuration's Python version. The current build configuration's Python version doesn't match any of the Python wheels available for this distribution. This distribution supports the following Python configuration settings: //_config:is_cp3.11_py3_none_any //_config:is_cp3.13_py3_none_any To determine the current configuration's Python version, run: `bazel config ` (shown further below) For the current configuration value see the debug message above that is printing the current flag values. If you can't see the message, then re-run the build to make it a failure instead by running the build with: --@@//python/config_settings:current_config=fail However, the command above will hide the `bazel config ` message. This instance of @@+pip+dev_pip//sphinx:_no_matching_repository has configuration identifier 29ffcf8. To inspect its configuration, run: bazel config 29ffcf8. For more help, see https://bazel.build/docs/configurable-attributes#faq-select-choose-condition. ERROR: Analysis of target '@@+pip+dev_pip//sphinx:sphinx' failed; build aborted: Analysis failed INFO: Elapsed time: 0.112s INFO: 0 processes. ERROR: Build did NOT complete successfully ``` Fixes #2466 --------- Co-authored-by: Richard Levasseur --- CHANGELOG.md | 3 + .../python/config_settings/index.md | 18 +++++ python/config_settings/BUILD.bazel | 9 +++ python/private/config_settings.bzl | 73 ++++++++++++++++++- python/private/pypi/pkg_aliases.bzl | 69 +++++++++--------- tests/pypi/pkg_aliases/pkg_aliases_test.bzl | 66 ++++++++++------- 6 files changed, 175 insertions(+), 63 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9976d2027c..9a3436487e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,9 @@ Unreleased changes template. * (pypi) Using {bzl:obj}`pip_parse.experimental_requirement_cycles` and {bzl:obj}`pip_parse.use_hub_alias_dependencies` together now works when using WORKSPACE files. +* (pypi) The error messages when the wheel distributions do not match anything + are now printing more details and include the currently active flag + values. Fixes [#2466](https://github.com/bazelbuild/rules_python/issues/2466). * (py_proto_library) Fix import paths in Bazel 8. [pep-695]: https://peps.python.org/pep-0695/ diff --git a/docs/api/rules_python/python/config_settings/index.md b/docs/api/rules_python/python/config_settings/index.md index ef829bab76..793f6e08fd 100644 --- a/docs/api/rules_python/python/config_settings/index.md +++ b/docs/api/rules_python/python/config_settings/index.md @@ -240,3 +240,21 @@ instead. ::: :::: + +::::{bzl:flag} current_config +Fail the build if the current build configuration does not match the +{obj}`pip.parse` defined wheels. + +Values: +* `fail`: Will fail in the build action ensuring that we get the error + message no matter the action cache. +* ``: (empty string) The default value, that will just print a warning. + +:::{seealso} +{obj}`pip.parse` +::: + +:::{versionadded} 1.1.0 +::: + +:::: diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel index 5455f5aef7..fcebcd76dc 100644 --- a/python/config_settings/BUILD.bazel +++ b/python/config_settings/BUILD.bazel @@ -29,6 +29,15 @@ filegroup( construct_config_settings( name = "construct_config_settings", default_version = DEFAULT_PYTHON_VERSION, + documented_flags = [ + ":pip_whl", + ":pip_whl_glibc_version", + ":pip_whl_muslc_version", + ":pip_whl_osx_arch", + ":pip_whl_osx_version", + ":py_freethreaded", + ":py_linux_libc", + ], minor_mapping = MINOR_MAPPING, versions = PYTHON_VERSIONS, ) diff --git a/python/private/config_settings.bzl b/python/private/config_settings.bzl index 10b4d686a7..e5f9d865d1 100644 --- a/python/private/config_settings.bzl +++ b/python/private/config_settings.bzl @@ -17,12 +17,21 @@ load("@bazel_skylib//lib:selects.bzl", "selects") load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load("//python/private:text_util.bzl", "render") load(":semver.bzl", "semver") _PYTHON_VERSION_FLAG = Label("//python/config_settings:python_version") _PYTHON_VERSION_MAJOR_MINOR_FLAG = Label("//python/config_settings:python_version_major_minor") -def construct_config_settings(*, name, default_version, versions, minor_mapping): # buildifier: disable=function-docstring +_DEBUG_ENV_MESSAGE_TEMPLATE = """\ +The current configuration rules_python config flags is: + {flags} + +If the value is missing, then the default value is being used, see documentation: +{docs_url}/python/config_settings +""" + +def construct_config_settings(*, name, default_version, versions, minor_mapping, documented_flags): # buildifier: disable=function-docstring """Create a 'python_version' config flag and construct all config settings used in rules_python. This mainly includes the targets that are used in the toolchain and pip hub @@ -33,6 +42,8 @@ def construct_config_settings(*, name, default_version, versions, minor_mapping) default_version: {type}`str` the default value for the `python_version` flag. versions: {type}`list[str]` A list of versions to build constraint settings for. minor_mapping: {type}`dict[str, str]` A mapping from `X.Y` to `X.Y.Z` python versions. + documented_flags: {type}`list[str]` The labels of the documented settings + that affect build configuration. """ _ = name # @unused _python_version_flag( @@ -101,6 +112,25 @@ def construct_config_settings(*, name, default_version, versions, minor_mapping) visibility = ["//visibility:public"], ) + _current_config( + name = "current_config", + build_setting_default = "", + settings = documented_flags + [_PYTHON_VERSION_FLAG.name], + visibility = ["//visibility:private"], + ) + native.config_setting( + name = "is_not_matching_current_config", + # We use the rule above instead of @platforms//:incompatible so that the + # printing of the current env always happens when the _current_config rule + # is executed. + # + # NOTE: This should in practise only happen if there is a missing compatible + # `whl_library` in the hub repo created by `pip.parse`. + flag_values = {"current_config": "will-never-match"}, + # Only public so that PyPI hub repo can access it + visibility = ["//visibility:public"], + ) + def _python_version_flag_impl(ctx): value = ctx.build_setting_value return [ @@ -122,7 +152,7 @@ _python_version_flag = rule( ) def _python_version_major_minor_flag_impl(ctx): - input = ctx.attr._python_version_flag[config_common.FeatureFlagInfo].value + input = _flag_value(ctx.attr._python_version_flag) if input: version = semver(input) value = "{}.{}".format(version.major, version.minor) @@ -140,3 +170,42 @@ _python_version_major_minor_flag = rule( ), }, ) + +def _flag_value(s): + if config_common.FeatureFlagInfo in s: + return s[config_common.FeatureFlagInfo].value + else: + return s[BuildSettingInfo].value + +def _print_current_config_impl(ctx): + flags = "\n".join([ + "{}: \"{}\"".format(k, v) + for k, v in sorted({ + str(setting.label): _flag_value(setting) + for setting in ctx.attr.settings + }.items()) + ]) + + msg = ctx.attr._template.format( + docs_url = "https://rules-python.readthedocs.io/en/latest/api/rules_python", + flags = render.indent(flags).lstrip(), + ) + if ctx.build_setting_value and ctx.build_setting_value != "fail": + fail("Only 'fail' and empty build setting values are allowed for {}".format( + str(ctx.label), + )) + elif ctx.build_setting_value: + fail(msg) + else: + print(msg) # buildifier: disable=print + + return [config_common.FeatureFlagInfo(value = "")] + +_current_config = rule( + implementation = _print_current_config_impl, + build_setting = config.string(flag = True), + attrs = { + "settings": attr.label_list(mandatory = True), + "_template": attr.string(default = _DEBUG_ENV_MESSAGE_TEMPLATE), + }, +) diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index a6872fdce9..980921b474 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -36,8 +36,6 @@ load(":whl_target_platforms.bzl", "whl_target_platforms") # it. It is more of an internal consistency check. _VERSION_NONE = (0, 0) -_CONFIG_SETTINGS_PKG = str(Label("//python/config_settings:BUILD.bazel")).partition(":")[0] - _NO_MATCH_ERROR_TEMPLATE = """\ No matching wheel for current configuration's Python version. @@ -49,37 +47,18 @@ configuration settings: To determine the current configuration's Python version, run: `bazel config ` (shown further below) -and look for one of: - {settings_pkg}:python_version - {settings_pkg}:pip_whl - {settings_pkg}:pip_whl_glibc_version - {settings_pkg}:pip_whl_muslc_version - {settings_pkg}:pip_whl_osx_arch - {settings_pkg}:pip_whl_osx_version - {settings_pkg}:py_freethreaded - {settings_pkg}:py_linux_libc - -If the value is missing, then the default value is being used, see documentation: -{docs_url}/python/config_settings""" - -def _no_match_error(actual): - if type(actual) != type({}): - return None - - if "//conditions:default" in actual: - return None - - return _NO_MATCH_ERROR_TEMPLATE.format( - config_settings = render.indent( - "\n".join(sorted([ - value - for key in actual - for value in (key if type(key) == "tuple" else [key]) - ])), - ).lstrip(), - settings_pkg = _CONFIG_SETTINGS_PKG, - docs_url = "https://rules-python.readthedocs.io/en/latest/api/rules_python", - ) +For the current configuration value see the debug message above that is +printing the current flag values. If you can't see the message, then re-run the +build to make it a failure instead by running the build with: + --{current_flags}=fail + +However, the command above will hide the `bazel config ` message. +""" + +_LABEL_NONE = Label("//python:none") +_LABEL_CURRENT_CONFIG = Label("//python/config_settings:current_config") +_LABEL_CURRENT_CONFIG_NO_MATCH = Label("//python/config_settings:is_not_matching_current_config") +_INCOMPATIBLE = "_no_matching_repository" def pkg_aliases( *, @@ -120,7 +99,26 @@ def pkg_aliases( } actual = multiplatform_whl_aliases(aliases = actual, **kwargs) - no_match_error = _no_match_error(actual) + if type(actual) == type({}) and "//conditions:default" not in actual: + native.alias( + name = _INCOMPATIBLE, + actual = select( + {_LABEL_CURRENT_CONFIG_NO_MATCH: _LABEL_NONE}, + no_match_error = _NO_MATCH_ERROR_TEMPLATE.format( + config_settings = render.indent( + "\n".join(sorted([ + value + for key in actual + for value in (key if type(key) == "tuple" else [key]) + ])), + ).lstrip(), + current_flags = str(_LABEL_CURRENT_CONFIG), + ), + ), + visibility = ["//visibility:private"], + tags = ["manual"], + ) + actual["//conditions:default"] = _INCOMPATIBLE for name, target_name in target_names.items(): if type(actual) == type(""): @@ -134,10 +132,9 @@ def pkg_aliases( v: "@{repo}//:{target_name}".format( repo = repo, target_name = name, - ) + ) if repo != _INCOMPATIBLE else repo for v, repo in actual.items() }, - no_match_error = no_match_error, ) else: fail("The `actual` arg must be a dictionary or a string") diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index 23a0f01db9..f13b62f13d 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -56,12 +56,8 @@ def _test_config_setting_aliases(env): actual_no_match_error = [] def mock_select(value, no_match_error = None): - actual_no_match_error.append(no_match_error) - env.expect.that_str(no_match_error).contains("""\ -configuration settings: - //:my_config_setting - -""") + if no_match_error and no_match_error not in actual_no_match_error: + actual_no_match_error.append(no_match_error) return value pkg_aliases( @@ -71,7 +67,7 @@ configuration settings: }, extra_aliases = ["my_special"], native = struct( - alias = lambda name, actual: got.update({name: actual}), + alias = lambda *, name, actual, visibility = None, tags = None: got.update({name: actual}), ), select = mock_select, ) @@ -80,9 +76,22 @@ configuration settings: want = { "pkg": { "//:my_config_setting": "@bar_baz_repo//:pkg", + "//conditions:default": "_no_matching_repository", }, + # This will be printing the current config values and will make sure we + # have an error. + "_no_matching_repository": {Label("//python/config_settings:is_not_matching_current_config"): Label("//python:none")}, } env.expect.that_dict(got).contains_at_least(want) + env.expect.that_collection(actual_no_match_error).has_size(1) + env.expect.that_str(actual_no_match_error[0]).contains("""\ +configuration settings: + //:my_config_setting + +""") + env.expect.that_str(actual_no_match_error[0]).contains( + "//python/config_settings:current_config=fail", + ) _tests.append(_test_config_setting_aliases) @@ -92,13 +101,8 @@ def _test_config_setting_aliases_many(env): actual_no_match_error = [] def mock_select(value, no_match_error = None): - actual_no_match_error.append(no_match_error) - env.expect.that_str(no_match_error).contains("""\ -configuration settings: - //:another_config_setting - //:my_config_setting - //:third_config_setting -""") + if no_match_error and no_match_error not in actual_no_match_error: + actual_no_match_error.append(no_match_error) return value pkg_aliases( @@ -112,7 +116,8 @@ configuration settings: }, extra_aliases = ["my_special"], native = struct( - alias = lambda name, actual: got.update({name: actual}), + alias = lambda *, name, actual, visibility = None, tags = None: got.update({name: actual}), + config_setting = lambda **_: None, ), select = mock_select, ) @@ -125,9 +130,17 @@ configuration settings: "//:another_config_setting", ): "@bar_baz_repo//:my_special", "//:third_config_setting": "@foo_repo//:my_special", + "//conditions:default": "_no_matching_repository", }, } env.expect.that_dict(got).contains_at_least(want) + env.expect.that_collection(actual_no_match_error).has_size(1) + env.expect.that_str(actual_no_match_error[0]).contains("""\ +configuration settings: + //:another_config_setting + //:my_config_setting + //:third_config_setting +""") _tests.append(_test_config_setting_aliases_many) @@ -137,15 +150,8 @@ def _test_multiplatform_whl_aliases(env): actual_no_match_error = [] def mock_select(value, no_match_error = None): - actual_no_match_error.append(no_match_error) - env.expect.that_str(no_match_error).contains("""\ -configuration settings: - //:my_config_setting - //_config:is_cp3.9_linux_x86_64 - //_config:is_cp3.9_py3_none_any - //_config:is_cp3.9_py3_none_any_linux_x86_64 - -""") + if no_match_error and no_match_error not in actual_no_match_error: + actual_no_match_error.append(no_match_error) return value pkg_aliases( @@ -168,7 +174,7 @@ configuration settings: }, extra_aliases = [], native = struct( - alias = lambda name, actual: got.update({name: actual}), + alias = lambda *, name, actual, visibility = None, tags = None: got.update({name: actual}), ), select = mock_select, glibc_versions = [], @@ -183,9 +189,19 @@ configuration settings: "//_config:is_cp3.9_linux_x86_64": "@bzlmod_repo_for_a_particular_platform//:pkg", "//_config:is_cp3.9_py3_none_any": "@filename_repo//:pkg", "//_config:is_cp3.9_py3_none_any_linux_x86_64": "@filename_repo_for_platform//:pkg", + "//conditions:default": "_no_matching_repository", }, } env.expect.that_dict(got).contains_at_least(want) + env.expect.that_collection(actual_no_match_error).has_size(1) + env.expect.that_str(actual_no_match_error[0]).contains("""\ +configuration settings: + //:my_config_setting + //_config:is_cp3.9_linux_x86_64 + //_config:is_cp3.9_py3_none_any + //_config:is_cp3.9_py3_none_any_linux_x86_64 + +""") _tests.append(_test_multiplatform_whl_aliases) From 026b300d918ced0f4e9f99a22ab8407656ed20ac Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 23 Dec 2024 13:27:03 -0800 Subject: [PATCH 3/5] refactor: consolidate py_executable_bazel, common_bazel (#2523) This furthers the work of removing the artificial split of code that stemmed from when the implementation was part of Bazel itself. Summary of changes: * Move most of `py_executable_bazel.bzl` into `py_executable.bzl` * Move most of `common_bazel.bzl` into `common.bzl` * Create `precompile.bzl` for the precompile helpers. This is to avoid a circular dependency between common.bzl and attributes.bzl. Work towards https://github.com/bazelbuild/rules_python/issues/2522 --- python/private/BUILD.bazel | 54 +- python/private/common.bzl | 61 +- .../{common_bazel.bzl => precompile.bzl} | 73 -- python/private/py_binary_macro.bzl | 2 +- python/private/py_binary_rule.bzl | 6 +- python/private/py_executable.bzl | 752 ++++++++++++++++- python/private/py_executable_bazel.bzl | 772 ------------------ python/private/py_library_rule.bzl | 4 +- python/private/py_test_macro.bzl | 2 +- python/private/py_test_rule.bzl | 6 +- .../venv_relative_path_tests.bzl | 2 +- 11 files changed, 843 insertions(+), 891 deletions(-) rename python/private/{common_bazel.bzl => precompile.bzl} (78%) delete mode 100644 python/private/py_executable_bazel.bzl diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 76e3a78778..706506a19c 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -104,30 +104,18 @@ bzl_library( deps = [":py_internal_bzl"], ) -bzl_library( - name = "common_bazel_bzl", - srcs = ["common_bazel.bzl"], - deps = [ - ":attributes_bzl", - ":common_bzl", - ":py_cc_link_params_info_bzl", - ":py_internal_bzl", - ":py_interpreter_program_bzl", - ":toolchain_types_bzl", - "@bazel_skylib//lib:paths", - ], -) - bzl_library( name = "common_bzl", srcs = ["common.bzl"], deps = [ ":cc_helper_bzl", + ":py_cc_link_params_info_bzl", ":py_info_bzl", ":py_internal_bzl", ":reexports_bzl", ":rules_cc_srcs_bzl", ":semantics_bzl", + "@bazel_skylib//lib:paths", ], ) @@ -199,6 +187,18 @@ bzl_library( srcs = ["normalize_name.bzl"], ) +bzl_library( + name = "precompile_bzl", + srcs = ["precompile.bzl"], + deps = [ + ":attributes_bzl", + ":py_internal_bzl", + ":py_interpreter_program_bzl", + ":toolchain_types_bzl", + "@bazel_skylib//lib:paths", + ], +) + bzl_library( name = "python_bzl", srcs = ["python.bzl"], @@ -265,8 +265,8 @@ bzl_library( name = "py_binary_macro_bzl", srcs = ["py_binary_macro.bzl"], deps = [ - ":common_bzl", ":py_binary_rule_bzl", + ":py_executable_bzl", ], ) @@ -275,7 +275,7 @@ bzl_library( srcs = ["py_binary_rule.bzl"], deps = [ ":attributes_bzl", - ":py_executable_bazel_bzl", + ":py_executable_bzl", ":semantics_bzl", "@bazel_skylib//lib:dicts", ], @@ -343,20 +343,6 @@ bzl_library( ], ) -bzl_library( - name = "py_executable_bazel_bzl", - srcs = ["py_executable_bazel.bzl"], - deps = [ - ":attributes_bzl", - ":common_bazel_bzl", - ":common_bzl", - ":py_executable_bzl", - ":py_internal_bzl", - ":py_runtime_info_bzl", - ":semantics_bzl", - ], -) - bzl_library( name = "py_executable_bzl", srcs = ["py_executable.bzl"], @@ -365,6 +351,7 @@ bzl_library( ":cc_helper_bzl", ":common_bzl", ":flags_bzl", + ":precompile_bzl", ":py_cc_link_params_info_bzl", ":py_executable_info_bzl", ":py_info_bzl", @@ -373,6 +360,7 @@ bzl_library( ":rules_cc_srcs_bzl", ":toolchain_types_bzl", "@bazel_skylib//lib:dicts", + "@bazel_skylib//lib:paths", "@bazel_skylib//lib:structs", "@bazel_skylib//rules:common_settings", ], @@ -431,8 +419,8 @@ bzl_library( name = "py_library_rule_bzl", srcs = ["py_library_rule.bzl"], deps = [ - ":common_bazel_bzl", ":common_bzl", + ":precompile_bzl", ":py_library_bzl", ], ) @@ -508,7 +496,7 @@ bzl_library( name = "py_test_macro_bzl", srcs = ["py_test_macro.bzl"], deps = [ - ":common_bazel_bzl", + ":py_executable_bzl", ":py_test_rule_bzl", ], ) @@ -519,7 +507,7 @@ bzl_library( deps = [ ":attributes_bzl", ":common_bzl", - ":py_executable_bazel_bzl", + ":py_executable_bzl", ":semantics_bzl", "@bazel_skylib//lib:dicts", ], diff --git a/python/private/common.bzl b/python/private/common.bzl index 97fabcebcb..9c285f97bc 100644 --- a/python/private/common.bzl +++ b/python/private/common.bzl @@ -11,9 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Various things common to Bazel and Google rule implementations.""" +"""Various things common to rule implementations.""" +load("@bazel_skylib//lib:paths.bzl", "paths") +load("@rules_cc//cc/common:cc_common.bzl", "cc_common") +load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") load(":cc_helper.bzl", "cc_helper") +load(":py_cc_link_params_info.bzl", "PyCcLinkParamsInfo") load(":py_info.bzl", "PyInfo", "PyInfoBuilder") load(":py_internal.bzl", "py_internal") load(":reexports.bzl", "BuiltinPyInfo") @@ -262,6 +266,30 @@ def filter_to_py_srcs(srcs): # as a valid extension. return [f for f in srcs if f.extension == "py"] +def collect_cc_info(ctx, extra_deps = []): + """Collect C++ information from dependencies for Bazel. + + Args: + ctx: Rule ctx; must have `deps` attribute. + extra_deps: list of Target to also collect C+ information from. + + Returns: + CcInfo provider of merged information. + """ + deps = ctx.attr.deps + if extra_deps: + deps = list(deps) + deps.extend(extra_deps) + cc_infos = [] + for dep in deps: + if CcInfo in dep: + cc_infos.append(dep[CcInfo]) + + if PyCcLinkParamsInfo in dep: + cc_infos.append(dep[PyCcLinkParamsInfo].cc_info) + + return cc_common.merge_cc_infos(cc_infos = cc_infos) + def collect_imports(ctx, semantics): """Collect the direct and transitive `imports` strings. @@ -280,6 +308,37 @@ def collect_imports(ctx, semantics): transitive.append(dep[BuiltinPyInfo].imports) return depset(direct = semantics.get_imports(ctx), transitive = transitive) +def get_imports(ctx): + """Gets the imports from a rule's `imports` attribute. + + See create_binary_semantics_struct for details about this function. + + Args: + ctx: Rule ctx. + + Returns: + List of strings. + """ + prefix = "{}/{}".format( + ctx.workspace_name, + py_internal.get_label_repo_runfiles_path(ctx.label), + ) + result = [] + for import_str in ctx.attr.imports: + import_str = ctx.expand_make_variables("imports", import_str, {}) + if import_str.startswith("/"): + continue + + # To prevent "escaping" out of the runfiles tree, we normalize + # the path and ensure it doesn't have up-level references. + import_path = paths.normalize("{}/{}".format(prefix, import_str)) + if import_path.startswith("../") or import_path == "..": + fail("Path '{}' references a path above the execution root".format( + import_str, + )) + result.append(import_path) + return result + def collect_runfiles(ctx, files = depset()): """Collects the necessary files from the rule's context. diff --git a/python/private/common_bazel.bzl b/python/private/precompile.bzl similarity index 78% rename from python/private/common_bazel.bzl rename to python/private/precompile.bzl index efbebd0252..23e8f81426 100644 --- a/python/private/common_bazel.bzl +++ b/python/private/precompile.bzl @@ -13,44 +13,12 @@ # limitations under the License. """Common functions that are specific to Bazel rule implementation""" -load("@bazel_skylib//lib:paths.bzl", "paths") load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") -load("@rules_cc//cc/common:cc_common.bzl", "cc_common") -load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") load(":attributes.bzl", "PrecompileAttr", "PrecompileInvalidationModeAttr", "PrecompileSourceRetentionAttr") -load(":common.bzl", "is_bool") load(":flags.bzl", "PrecompileFlag") -load(":py_cc_link_params_info.bzl", "PyCcLinkParamsInfo") -load(":py_internal.bzl", "py_internal") load(":py_interpreter_program.bzl", "PyInterpreterProgramInfo") load(":toolchain_types.bzl", "EXEC_TOOLS_TOOLCHAIN_TYPE", "TARGET_TOOLCHAIN_TYPE") -_py_builtins = py_internal - -def collect_cc_info(ctx, extra_deps = []): - """Collect C++ information from dependencies for Bazel. - - Args: - ctx: Rule ctx; must have `deps` attribute. - extra_deps: list of Target to also collect C+ information from. - - Returns: - CcInfo provider of merged information. - """ - deps = ctx.attr.deps - if extra_deps: - deps = list(deps) - deps.extend(extra_deps) - cc_infos = [] - for dep in deps: - if CcInfo in dep: - cc_infos.append(dep[CcInfo]) - - if PyCcLinkParamsInfo in dep: - cc_infos.append(dep[PyCcLinkParamsInfo].cc_info) - - return cc_common.merge_cc_infos(cc_infos = cc_infos) - def maybe_precompile(ctx, srcs): """Computes all the outputs (maybe precompiled) from the input srcs. @@ -237,44 +205,3 @@ def _precompile(ctx, src, *, use_pycache): toolchain = EXEC_TOOLS_TOOLCHAIN_TYPE, ) return pyc - -def get_imports(ctx): - """Gets the imports from a rule's `imports` attribute. - - See create_binary_semantics_struct for details about this function. - - Args: - ctx: Rule ctx. - - Returns: - List of strings. - """ - prefix = "{}/{}".format( - ctx.workspace_name, - _py_builtins.get_label_repo_runfiles_path(ctx.label), - ) - result = [] - for import_str in ctx.attr.imports: - import_str = ctx.expand_make_variables("imports", import_str, {}) - if import_str.startswith("/"): - continue - - # To prevent "escaping" out of the runfiles tree, we normalize - # the path and ensure it doesn't have up-level references. - import_path = paths.normalize("{}/{}".format(prefix, import_str)) - if import_path.startswith("../") or import_path == "..": - fail("Path '{}' references a path above the execution root".format( - import_str, - )) - result.append(import_path) - return result - -def convert_legacy_create_init_to_int(kwargs): - """Convert "legacy_create_init" key to int, in-place. - - Args: - kwargs: The kwargs to modify. The key "legacy_create_init", if present - and bool, will be converted to its integer value, in place. - """ - if is_bool(kwargs.get("legacy_create_init")): - kwargs["legacy_create_init"] = 1 if kwargs["legacy_create_init"] else 0 diff --git a/python/private/py_binary_macro.bzl b/python/private/py_binary_macro.bzl index 83b3c18677..d1269f2321 100644 --- a/python/private/py_binary_macro.bzl +++ b/python/private/py_binary_macro.bzl @@ -13,8 +13,8 @@ # limitations under the License. """Implementation of macro-half of py_binary rule.""" -load(":common_bazel.bzl", "convert_legacy_create_init_to_int") load(":py_binary_rule.bzl", py_binary_rule = "py_binary") +load(":py_executable.bzl", "convert_legacy_create_init_to_int") def py_binary(**kwargs): convert_legacy_create_init_to_int(kwargs) diff --git a/python/private/py_binary_rule.bzl b/python/private/py_binary_rule.bzl index 9ce0726c5e..f1c8eb1325 100644 --- a/python/private/py_binary_rule.bzl +++ b/python/private/py_binary_rule.bzl @@ -16,9 +16,9 @@ load("@bazel_skylib//lib:dicts.bzl", "dicts") load(":attributes.bzl", "AGNOSTIC_BINARY_ATTRS") load( - ":py_executable_bazel.bzl", + ":py_executable.bzl", "create_executable_rule", - "py_executable_bazel_impl", + "py_executable_impl", ) _PY_TEST_ATTRS = { @@ -39,7 +39,7 @@ _PY_TEST_ATTRS = { } def _py_binary_impl(ctx): - return py_executable_bazel_impl( + return py_executable_impl( ctx = ctx, is_test = False, inherited_environment = [], diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 8c0487d6a1..40c74100f2 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -14,6 +14,7 @@ """Common functionality between test/binary executables.""" load("@bazel_skylib//lib:dicts.bzl", "dicts") +load("@bazel_skylib//lib:paths.bzl", "paths") load("@bazel_skylib//lib:structs.bzl", "structs") load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") load("@rules_cc//cc/common:cc_common.bzl", "cc_common") @@ -21,6 +22,7 @@ load( ":attributes.bzl", "AGNOSTIC_EXECUTABLE_ATTRS", "COMMON_ATTRS", + "IMPORTS_ATTRS", "PY_SRCS_ATTRS", "PrecompileAttr", "PycCollectionAttr", @@ -33,21 +35,29 @@ load(":builders.bzl", "builders") load(":cc_helper.bzl", "cc_helper") load( ":common.bzl", + "collect_cc_info", "collect_imports", "collect_runfiles", + "create_binary_semantics_struct", + "create_cc_details_struct", + "create_executable_result_struct", "create_instrumented_files_info", "create_output_group_info", "create_py_info", "csv", "filter_to_py_srcs", + "get_imports", + "is_bool", "target_platform_has_any_constraint", "union_attrs", ) +load(":flags.bzl", "BootstrapImplFlag") +load(":precompile.bzl", "maybe_precompile") load(":py_cc_link_params_info.bzl", "PyCcLinkParamsInfo") load(":py_executable_info.bzl", "PyExecutableInfo") load(":py_info.bzl", "PyInfo") load(":py_internal.bzl", "py_internal") -load(":py_runtime_info.bzl", "PyRuntimeInfo") +load(":py_runtime_info.bzl", "DEFAULT_STUB_SHEBANG", "PyRuntimeInfo") load(":reexports.bzl", "BuiltinPyInfo", "BuiltinPyRuntimeInfo") load( ":semantics.bzl", @@ -59,10 +69,13 @@ load( load( ":toolchain_types.bzl", "EXEC_TOOLS_TOOLCHAIN_TYPE", + "TARGET_TOOLCHAIN_TYPE", TOOLCHAIN_TYPE = "TARGET_TOOLCHAIN_TYPE", ) _py_builtins = py_internal +_EXTERNAL_PATH_PREFIX = "external" +_ZIP_RUNFILES_DIRECTORY_NAME = "runfiles" # Bazel 5.4 doesn't have config_common.toolchain_type _CC_TOOLCHAINS = [config_common.toolchain_type( @@ -76,7 +89,21 @@ EXECUTABLE_ATTRS = union_attrs( COMMON_ATTRS, AGNOSTIC_EXECUTABLE_ATTRS, PY_SRCS_ATTRS, + IMPORTS_ATTRS, { + "legacy_create_init": attr.int( + default = -1, + values = [-1, 0, 1], + doc = """\ +Whether to implicitly create empty `__init__.py` files in the runfiles tree. +These are created in every directory containing Python source code or shared +libraries, and every parent directory of those directories, excluding the repo +root directory. The default, `-1` (auto), means true unless +`--incompatible_default_to_explicit_init_py` is used. If false, the user is +responsible for creating (possibly empty) `__init__.py` files and adding them to +the `srcs` of Python targets as required. + """, + ), # TODO(b/203567235): In the Java impl, any file is allowed. While marked # label, it is more treated as a string, and doesn't have to refer to # anything that exists because it gets treated as suffix-search string @@ -120,17 +147,732 @@ Valid values are: default = "//python/config_settings:bootstrap_impl", providers = [BuildSettingInfo], ), + "_bootstrap_template": attr.label( + allow_single_file = True, + default = "@bazel_tools//tools/python:python_bootstrap_template.txt", + ), + "_launcher": attr.label( + cfg = "target", + # NOTE: This is an executable, but is only used for Windows. It + # can't have executable=True because the backing target is an + # empty target for other platforms. + default = "//tools/launcher:launcher", + ), + "_py_interpreter": attr.label( + # The configuration_field args are validated when called; + # we use the precense of py_internal to indicate this Bazel + # build has that fragment and name. + default = configuration_field( + fragment = "bazel_py", + name = "python_top", + ) if py_internal else None, + ), + # TODO: This appears to be vestigial. It's only added because + # GraphlessQueryTest.testLabelsOperator relies on it to test for + # query behavior of implicit dependencies. + "_py_toolchain_type": attr.label( + default = TARGET_TOOLCHAIN_TYPE, + ), + "_python_version_flag": attr.label( + default = "//python/config_settings:python_version", + ), "_windows_constraints": attr.label_list( default = [ "@platforms//os:windows", ], ), + "_windows_launcher_maker": attr.label( + default = "@bazel_tools//tools/launcher:launcher_maker", + cfg = "exec", + executable = True, + ), + "_zipper": attr.label( + cfg = "exec", + executable = True, + default = "@bazel_tools//tools/zip:zipper", + ), }, create_srcs_version_attr(values = SRCS_VERSION_ALL_VALUES), create_srcs_attr(mandatory = True), allow_none = True, ) +def convert_legacy_create_init_to_int(kwargs): + """Convert "legacy_create_init" key to int, in-place. + + Args: + kwargs: The kwargs to modify. The key "legacy_create_init", if present + and bool, will be converted to its integer value, in place. + """ + if is_bool(kwargs.get("legacy_create_init")): + kwargs["legacy_create_init"] = 1 if kwargs["legacy_create_init"] else 0 + +def py_executable_impl(ctx, *, is_test, inherited_environment): + return py_executable_base_impl( + ctx = ctx, + semantics = create_binary_semantics(), + is_test = is_test, + inherited_environment = inherited_environment, + ) + +def create_binary_semantics(): + return create_binary_semantics_struct( + # keep-sorted start + create_executable = _create_executable, + get_cc_details_for_binary = _get_cc_details_for_binary, + get_central_uncachable_version_file = lambda ctx: None, + get_coverage_deps = _get_coverage_deps, + get_debugger_deps = _get_debugger_deps, + get_extra_common_runfiles_for_binary = lambda ctx: ctx.runfiles(), + get_extra_providers = _get_extra_providers, + get_extra_write_build_data_env = lambda ctx: {}, + get_imports = get_imports, + get_interpreter_path = _get_interpreter_path, + get_native_deps_dso_name = _get_native_deps_dso_name, + get_native_deps_user_link_flags = _get_native_deps_user_link_flags, + get_stamp_flag = _get_stamp_flag, + maybe_precompile = maybe_precompile, + should_build_native_deps_dso = lambda ctx: False, + should_create_init_files = _should_create_init_files, + should_include_build_data = lambda ctx: False, + # keep-sorted end + ) + +def _get_coverage_deps(ctx, runtime_details): + _ = ctx, runtime_details # @unused + return [] + +def _get_debugger_deps(ctx, runtime_details): + _ = ctx, runtime_details # @unused + return [] + +def _get_extra_providers(ctx, main_py, runtime_details): + _ = ctx, main_py, runtime_details # @unused + return [] + +def _get_stamp_flag(ctx): + # NOTE: Undocumented API; private to builtins + return ctx.configuration.stamp_binaries + +def _should_create_init_files(ctx): + if ctx.attr.legacy_create_init == -1: + return not ctx.fragments.py.default_to_explicit_init_py + else: + return bool(ctx.attr.legacy_create_init) + +def _create_executable( + ctx, + *, + executable, + main_py, + imports, + is_test, + runtime_details, + cc_details, + native_deps_details, + runfiles_details): + _ = is_test, cc_details, native_deps_details # @unused + + is_windows = target_platform_has_any_constraint(ctx, ctx.attr._windows_constraints) + + if is_windows: + if not executable.extension == "exe": + fail("Should not happen: somehow we are generating a non-.exe file on windows") + base_executable_name = executable.basename[0:-4] + else: + base_executable_name = executable.basename + + venv = None + + # The check for stage2_bootstrap_template is to support legacy + # BuiltinPyRuntimeInfo providers, which is likely to come from + # @bazel_tools//tools/python:autodetecting_toolchain, the toolchain used + # for workspace builds when no rules_python toolchain is configured. + if (BootstrapImplFlag.get_value(ctx) == BootstrapImplFlag.SCRIPT and + runtime_details.effective_runtime and + hasattr(runtime_details.effective_runtime, "stage2_bootstrap_template")): + venv = _create_venv( + ctx, + output_prefix = base_executable_name, + imports = imports, + runtime_details = runtime_details, + ) + + stage2_bootstrap = _create_stage2_bootstrap( + ctx, + output_prefix = base_executable_name, + output_sibling = executable, + main_py = main_py, + imports = imports, + runtime_details = runtime_details, + ) + extra_runfiles = ctx.runfiles([stage2_bootstrap] + venv.files_without_interpreter) + zip_main = _create_zip_main( + ctx, + stage2_bootstrap = stage2_bootstrap, + runtime_details = runtime_details, + venv = venv, + ) + else: + stage2_bootstrap = None + extra_runfiles = ctx.runfiles() + zip_main = ctx.actions.declare_file(base_executable_name + ".temp", sibling = executable) + _create_stage1_bootstrap( + ctx, + output = zip_main, + main_py = main_py, + imports = imports, + is_for_zip = True, + runtime_details = runtime_details, + ) + + zip_file = ctx.actions.declare_file(base_executable_name + ".zip", sibling = executable) + _create_zip_file( + ctx, + output = zip_file, + original_nonzip_executable = executable, + zip_main = zip_main, + runfiles = runfiles_details.default_runfiles.merge(extra_runfiles), + ) + + extra_files_to_build = [] + + # NOTE: --build_python_zip defaults to true on Windows + build_zip_enabled = ctx.fragments.py.build_python_zip + + # When --build_python_zip is enabled, then the zip file becomes + # one of the default outputs. + if build_zip_enabled: + extra_files_to_build.append(zip_file) + + # The logic here is a bit convoluted. Essentially, there are 3 types of + # executables produced: + # 1. (non-Windows) A bootstrap template based program. + # 2. (non-Windows) A self-executable zip file of a bootstrap template based program. + # 3. (Windows) A native Windows executable that finds and launches + # the actual underlying Bazel program (one of the above). Note that + # it implicitly assumes one of the above is located next to it, and + # that --build_python_zip defaults to true for Windows. + + should_create_executable_zip = False + bootstrap_output = None + if not is_windows: + if build_zip_enabled: + should_create_executable_zip = True + else: + bootstrap_output = executable + else: + _create_windows_exe_launcher( + ctx, + output = executable, + use_zip_file = build_zip_enabled, + python_binary_path = runtime_details.executable_interpreter_path, + ) + if not build_zip_enabled: + # On Windows, the main executable has an "exe" extension, so + # here we re-use the un-extensioned name for the bootstrap output. + bootstrap_output = ctx.actions.declare_file(base_executable_name) + + # The launcher looks for the non-zip executable next to + # itself, so add it to the default outputs. + extra_files_to_build.append(bootstrap_output) + + if should_create_executable_zip: + if bootstrap_output != None: + fail("Should not occur: bootstrap_output should not be used " + + "when creating an executable zip") + _create_executable_zip_file( + ctx, + output = executable, + zip_file = zip_file, + stage2_bootstrap = stage2_bootstrap, + runtime_details = runtime_details, + venv = venv, + ) + elif bootstrap_output: + _create_stage1_bootstrap( + ctx, + output = bootstrap_output, + stage2_bootstrap = stage2_bootstrap, + runtime_details = runtime_details, + is_for_zip = False, + imports = imports, + main_py = main_py, + venv = venv, + ) + else: + # Otherwise, this should be the Windows case of launcher + zip. + # Double check this just to make sure. + if not is_windows or not build_zip_enabled: + fail(("Should not occur: The non-executable-zip and " + + "non-bootstrap-template case should have windows and zip " + + "both true, but got " + + "is_windows={is_windows} " + + "build_zip_enabled={build_zip_enabled}").format( + is_windows = is_windows, + build_zip_enabled = build_zip_enabled, + )) + + # The interpreter is added this late in the process so that it isn't + # added to the zipped files. + if venv: + extra_runfiles = extra_runfiles.merge(ctx.runfiles([venv.interpreter])) + return create_executable_result_struct( + extra_files_to_build = depset(extra_files_to_build), + output_groups = {"python_zip_file": depset([zip_file])}, + extra_runfiles = extra_runfiles, + ) + +def _create_zip_main(ctx, *, stage2_bootstrap, runtime_details, venv): + python_binary = _runfiles_root_path(ctx, venv.interpreter.short_path) + python_binary_actual = venv.interpreter_actual_path + + # The location of this file doesn't really matter. It's added to + # the zip file as the top-level __main__.py file and not included + # elsewhere. + output = ctx.actions.declare_file(ctx.label.name + "_zip__main__.py") + ctx.actions.expand_template( + template = runtime_details.effective_runtime.zip_main_template, + output = output, + substitutions = { + "%python_binary%": python_binary, + "%python_binary_actual%": python_binary_actual, + "%stage2_bootstrap%": "{}/{}".format( + ctx.workspace_name, + stage2_bootstrap.short_path, + ), + "%workspace_name%": ctx.workspace_name, + }, + ) + return output + +def relative_path(from_, to): + """Compute a relative path from one path to another. + + Args: + from_: {type}`str` the starting directory. Note that it should be + a directory because relative-symlinks are relative to the + directory the symlink resides in. + to: {type}`str` the path that `from_` wants to point to + + Returns: + {type}`str` a relative path + """ + from_parts = from_.split("/") + to_parts = to.split("/") + + # Strip common leading parts from both paths + n = min(len(from_parts), len(to_parts)) + for _ in range(n): + if from_parts[0] == to_parts[0]: + from_parts.pop(0) + to_parts.pop(0) + else: + break + + # Impossible to compute a relative path without knowing what ".." is + if from_parts and from_parts[0] == "..": + fail("cannot compute relative path from '%s' to '%s'", from_, to) + + parts = ([".."] * len(from_parts)) + to_parts + return paths.join(*parts) + +# Create a venv the executable can use. +# For venv details and the venv startup process, see: +# * https://docs.python.org/3/library/venv.html +# * https://snarky.ca/how-virtual-environments-work/ +# * https://github.com/python/cpython/blob/main/Modules/getpath.py +# * https://github.com/python/cpython/blob/main/Lib/site.py +def _create_venv(ctx, output_prefix, imports, runtime_details): + venv = "_{}.venv".format(output_prefix.lstrip("_")) + + # The pyvenv.cfg file must be present to trigger the venv site hooks. + # Because it's paths are expected to be absolute paths, we can't reliably + # put much in it. See https://github.com/python/cpython/issues/83650 + pyvenv_cfg = ctx.actions.declare_file("{}/pyvenv.cfg".format(venv)) + ctx.actions.write(pyvenv_cfg, "") + + runtime = runtime_details.effective_runtime + if runtime.interpreter: + py_exe_basename = paths.basename(runtime.interpreter.short_path) + + # Even though ctx.actions.symlink() is used, using + # declare_symlink() is required to ensure that the resulting file + # in runfiles is always a symlink. An RBE implementation, for example, + # may choose to write what symlink() points to instead. + interpreter = ctx.actions.declare_symlink("{}/bin/{}".format(venv, py_exe_basename)) + + interpreter_actual_path = _runfiles_root_path(ctx, runtime.interpreter.short_path) + rel_path = relative_path( + # dirname is necessary because a relative symlink is relative to + # the directory the symlink resides within. + from_ = paths.dirname(_runfiles_root_path(ctx, interpreter.short_path)), + to = interpreter_actual_path, + ) + + ctx.actions.symlink(output = interpreter, target_path = rel_path) + else: + py_exe_basename = paths.basename(runtime.interpreter_path) + interpreter = ctx.actions.declare_symlink("{}/bin/{}".format(venv, py_exe_basename)) + ctx.actions.symlink(output = interpreter, target_path = runtime.interpreter_path) + interpreter_actual_path = runtime.interpreter_path + + if runtime.interpreter_version_info: + version = "{}.{}".format( + runtime.interpreter_version_info.major, + runtime.interpreter_version_info.minor, + ) + else: + version_flag = ctx.attr._python_version_flag[config_common.FeatureFlagInfo].value + version_flag_parts = version_flag.split(".")[0:2] + version = "{}.{}".format(*version_flag_parts) + + # See site.py logic: free-threaded builds append "t" to the venv lib dir name + if "t" in runtime.abi_flags: + version += "t" + + site_packages = "{}/lib/python{}/site-packages".format(venv, version) + pth = ctx.actions.declare_file("{}/bazel.pth".format(site_packages)) + ctx.actions.write(pth, "import _bazel_site_init\n") + + site_init = ctx.actions.declare_file("{}/_bazel_site_init.py".format(site_packages)) + computed_subs = ctx.actions.template_dict() + computed_subs.add_joined("%imports%", imports, join_with = ":", map_each = _map_each_identity) + ctx.actions.expand_template( + template = runtime.site_init_template, + output = site_init, + substitutions = { + "%import_all%": "True" if ctx.fragments.bazel_py.python_import_all_repositories else "False", + "%site_init_runfiles_path%": "{}/{}".format(ctx.workspace_name, site_init.short_path), + "%workspace_name%": ctx.workspace_name, + }, + computed_substitutions = computed_subs, + ) + + return struct( + interpreter = interpreter, + # Runfiles root relative path or absolute path + interpreter_actual_path = interpreter_actual_path, + files_without_interpreter = [pyvenv_cfg, pth, site_init], + ) + +def _map_each_identity(v): + return v + +def _create_stage2_bootstrap( + ctx, + *, + output_prefix, + output_sibling, + main_py, + imports, + runtime_details): + output = ctx.actions.declare_file( + # Prepend with underscore to prevent pytest from trying to + # process the bootstrap for files starting with `test_` + "_{}_stage2_bootstrap.py".format(output_prefix), + sibling = output_sibling, + ) + runtime = runtime_details.effective_runtime + if (ctx.configuration.coverage_enabled and + runtime and + runtime.coverage_tool): + coverage_tool_runfiles_path = "{}/{}".format( + ctx.workspace_name, + runtime.coverage_tool.short_path, + ) + else: + coverage_tool_runfiles_path = "" + + template = runtime.stage2_bootstrap_template + + ctx.actions.expand_template( + template = template, + output = output, + substitutions = { + "%coverage_tool%": coverage_tool_runfiles_path, + "%import_all%": "True" if ctx.fragments.bazel_py.python_import_all_repositories else "False", + "%imports%": ":".join(imports.to_list()), + "%main%": "{}/{}".format(ctx.workspace_name, main_py.short_path), + "%target%": str(ctx.label), + "%workspace_name%": ctx.workspace_name, + }, + is_executable = True, + ) + return output + +def _runfiles_root_path(ctx, short_path): + """Compute a runfiles-root relative path from `File.short_path` + + Args: + ctx: current target ctx + short_path: str, a main-repo relative path from `File.short_path` + + Returns: + {type}`str`, a runflies-root relative path + """ + + # The ../ comes from short_path is for files in other repos. + if short_path.startswith("../"): + return short_path[3:] + else: + return "{}/{}".format(ctx.workspace_name, short_path) + +def _create_stage1_bootstrap( + ctx, + *, + output, + main_py = None, + stage2_bootstrap = None, + imports = None, + is_for_zip, + runtime_details, + venv = None): + runtime = runtime_details.effective_runtime + + if venv: + python_binary_path = _runfiles_root_path(ctx, venv.interpreter.short_path) + else: + python_binary_path = runtime_details.executable_interpreter_path + + if is_for_zip and venv: + python_binary_actual = venv.interpreter_actual_path + else: + python_binary_actual = "" + + subs = { + "%is_zipfile%": "1" if is_for_zip else "0", + "%python_binary%": python_binary_path, + "%python_binary_actual%": python_binary_actual, + "%target%": str(ctx.label), + "%workspace_name%": ctx.workspace_name, + } + + if stage2_bootstrap: + subs["%stage2_bootstrap%"] = "{}/{}".format( + ctx.workspace_name, + stage2_bootstrap.short_path, + ) + template = runtime.bootstrap_template + subs["%shebang%"] = runtime.stub_shebang + else: + if (ctx.configuration.coverage_enabled and + runtime and + runtime.coverage_tool): + coverage_tool_runfiles_path = "{}/{}".format( + ctx.workspace_name, + runtime.coverage_tool.short_path, + ) + else: + coverage_tool_runfiles_path = "" + if runtime: + subs["%shebang%"] = runtime.stub_shebang + template = runtime.bootstrap_template + else: + subs["%shebang%"] = DEFAULT_STUB_SHEBANG + template = ctx.file._bootstrap_template + + subs["%coverage_tool%"] = coverage_tool_runfiles_path + subs["%import_all%"] = ("True" if ctx.fragments.bazel_py.python_import_all_repositories else "False") + subs["%imports%"] = ":".join(imports.to_list()) + subs["%main%"] = "{}/{}".format(ctx.workspace_name, main_py.short_path) + + ctx.actions.expand_template( + template = template, + output = output, + substitutions = subs, + ) + +def _create_windows_exe_launcher( + ctx, + *, + output, + python_binary_path, + use_zip_file): + launch_info = ctx.actions.args() + launch_info.use_param_file("%s", use_always = True) + launch_info.set_param_file_format("multiline") + launch_info.add("binary_type=Python") + launch_info.add(ctx.workspace_name, format = "workspace_name=%s") + launch_info.add( + "1" if py_internal.runfiles_enabled(ctx) else "0", + format = "symlink_runfiles_enabled=%s", + ) + launch_info.add(python_binary_path, format = "python_bin_path=%s") + launch_info.add("1" if use_zip_file else "0", format = "use_zip_file=%s") + + launcher = ctx.attr._launcher[DefaultInfo].files_to_run.executable + ctx.actions.run( + executable = ctx.executable._windows_launcher_maker, + arguments = [launcher.path, launch_info, output.path], + inputs = [launcher], + outputs = [output], + mnemonic = "PyBuildLauncher", + progress_message = "Creating launcher for %{label}", + # Needed to inherit PATH when using non-MSVC compilers like MinGW + use_default_shell_env = True, + ) + +def _create_zip_file(ctx, *, output, original_nonzip_executable, zip_main, runfiles): + """Create a Python zipapp (zip with __main__.py entry point).""" + workspace_name = ctx.workspace_name + legacy_external_runfiles = _py_builtins.get_legacy_external_runfiles(ctx) + + manifest = ctx.actions.args() + manifest.use_param_file("@%s", use_always = True) + manifest.set_param_file_format("multiline") + + manifest.add("__main__.py={}".format(zip_main.path)) + manifest.add("__init__.py=") + manifest.add( + "{}=".format( + _get_zip_runfiles_path("__init__.py", workspace_name, legacy_external_runfiles), + ), + ) + for path in runfiles.empty_filenames.to_list(): + manifest.add("{}=".format(_get_zip_runfiles_path(path, workspace_name, legacy_external_runfiles))) + + def map_zip_runfiles(file): + if file != original_nonzip_executable and file != output: + return "{}={}".format( + _get_zip_runfiles_path(file.short_path, workspace_name, legacy_external_runfiles), + file.path, + ) + else: + return None + + manifest.add_all(runfiles.files, map_each = map_zip_runfiles, allow_closure = True) + + inputs = [zip_main] + if _py_builtins.is_bzlmod_enabled(ctx): + zip_repo_mapping_manifest = ctx.actions.declare_file( + output.basename + ".repo_mapping", + sibling = output, + ) + _py_builtins.create_repo_mapping_manifest( + ctx = ctx, + runfiles = runfiles, + output = zip_repo_mapping_manifest, + ) + manifest.add("{}/_repo_mapping={}".format( + _ZIP_RUNFILES_DIRECTORY_NAME, + zip_repo_mapping_manifest.path, + )) + inputs.append(zip_repo_mapping_manifest) + + for artifact in runfiles.files.to_list(): + # Don't include the original executable because it isn't used by the + # zip file, so no need to build it for the action. + # Don't include the zipfile itself because it's an output. + if artifact != original_nonzip_executable and artifact != output: + inputs.append(artifact) + + zip_cli_args = ctx.actions.args() + zip_cli_args.add("cC") + zip_cli_args.add(output) + + ctx.actions.run( + executable = ctx.executable._zipper, + arguments = [zip_cli_args, manifest], + inputs = depset(inputs), + outputs = [output], + use_default_shell_env = True, + mnemonic = "PythonZipper", + progress_message = "Building Python zip: %{label}", + ) + +def _get_zip_runfiles_path(path, workspace_name, legacy_external_runfiles): + if legacy_external_runfiles and path.startswith(_EXTERNAL_PATH_PREFIX): + zip_runfiles_path = paths.relativize(path, _EXTERNAL_PATH_PREFIX) + else: + # NOTE: External runfiles (artifacts in other repos) will have a leading + # path component of "../" so that they refer outside the main workspace + # directory and into the runfiles root. By normalizing, we simplify e.g. + # "workspace/../foo/bar" to simply "foo/bar". + zip_runfiles_path = paths.normalize("{}/{}".format(workspace_name, path)) + return "{}/{}".format(_ZIP_RUNFILES_DIRECTORY_NAME, zip_runfiles_path) + +def _create_executable_zip_file( + ctx, + *, + output, + zip_file, + stage2_bootstrap, + runtime_details, + venv): + prelude = ctx.actions.declare_file( + "{}_zip_prelude.sh".format(output.basename), + sibling = output, + ) + if stage2_bootstrap: + _create_stage1_bootstrap( + ctx, + output = prelude, + stage2_bootstrap = stage2_bootstrap, + runtime_details = runtime_details, + is_for_zip = True, + venv = venv, + ) + else: + ctx.actions.write(prelude, "#!/usr/bin/env python3\n") + + ctx.actions.run_shell( + command = "cat {prelude} {zip} > {output}".format( + prelude = prelude.path, + zip = zip_file.path, + output = output.path, + ), + inputs = [prelude, zip_file], + outputs = [output], + use_default_shell_env = True, + mnemonic = "PyBuildExecutableZip", + progress_message = "Build Python zip executable: %{label}", + ) + +def _get_cc_details_for_binary(ctx, extra_deps): + cc_info = collect_cc_info(ctx, extra_deps = extra_deps) + return create_cc_details_struct( + cc_info_for_propagating = cc_info, + cc_info_for_self_link = cc_info, + cc_info_with_extra_link_time_libraries = None, + extra_runfiles = ctx.runfiles(), + # Though the rules require the CcToolchain, it isn't actually used. + cc_toolchain = None, + feature_config = None, + ) + +def _get_interpreter_path(ctx, *, runtime, flag_interpreter_path): + if runtime: + if runtime.interpreter_path: + interpreter_path = runtime.interpreter_path + else: + interpreter_path = "{}/{}".format( + ctx.workspace_name, + runtime.interpreter.short_path, + ) + + # NOTE: External runfiles (artifacts in other repos) will have a + # leading path component of "../" so that they refer outside the + # main workspace directory and into the runfiles root. By + # normalizing, we simplify e.g. "workspace/../foo/bar" to simply + # "foo/bar" + interpreter_path = paths.normalize(interpreter_path) + + elif flag_interpreter_path: + interpreter_path = flag_interpreter_path + else: + fail("Unable to determine interpreter path") + + return interpreter_path + +def _get_native_deps_dso_name(ctx): + _ = ctx # @unused + fail("Building native deps DSO not supported.") + +def _get_native_deps_user_link_flags(ctx): + _ = ctx # @unused + fail("Building native deps DSO not supported.") + def py_executable_base_impl(ctx, *, semantics, is_test, inherited_environment = []): """Base rule implementation for a Python executable. @@ -949,6 +1691,14 @@ def _create_run_environment_info(ctx, inherited_environment): inherited_environment = inherited_environment, ) +def create_executable_rule(*, attrs, **kwargs): + return create_base_executable_rule( + ##attrs = dicts.add(EXECUTABLE_ATTRS, attrs), + attrs = attrs, + fragments = ["py", "bazel_py"], + **kwargs + ) + def create_base_executable_rule(*, attrs, fragments = [], **kwargs): """Create a function for defining for Python binary/test targets. diff --git a/python/private/py_executable_bazel.bzl b/python/private/py_executable_bazel.bzl deleted file mode 100644 index 3778c192b4..0000000000 --- a/python/private/py_executable_bazel.bzl +++ /dev/null @@ -1,772 +0,0 @@ -# Copyright 2022 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Implementation for Bazel Python executable.""" - -load("@bazel_skylib//lib:dicts.bzl", "dicts") -load("@bazel_skylib//lib:paths.bzl", "paths") -load(":attributes.bzl", "IMPORTS_ATTRS") -load( - ":common.bzl", - "create_binary_semantics_struct", - "create_cc_details_struct", - "create_executable_result_struct", - "target_platform_has_any_constraint", - "union_attrs", -) -load(":common_bazel.bzl", "collect_cc_info", "get_imports", "maybe_precompile") -load(":flags.bzl", "BootstrapImplFlag") -load( - ":py_executable.bzl", - "create_base_executable_rule", - "py_executable_base_impl", -) -load(":py_internal.bzl", "py_internal") -load(":py_runtime_info.bzl", "DEFAULT_STUB_SHEBANG") -load(":toolchain_types.bzl", "TARGET_TOOLCHAIN_TYPE") - -_py_builtins = py_internal -_EXTERNAL_PATH_PREFIX = "external" -_ZIP_RUNFILES_DIRECTORY_NAME = "runfiles" - -BAZEL_EXECUTABLE_ATTRS = union_attrs( - IMPORTS_ATTRS, - { - "legacy_create_init": attr.int( - default = -1, - values = [-1, 0, 1], - doc = """\ -Whether to implicitly create empty `__init__.py` files in the runfiles tree. -These are created in every directory containing Python source code or shared -libraries, and every parent directory of those directories, excluding the repo -root directory. The default, `-1` (auto), means true unless -`--incompatible_default_to_explicit_init_py` is used. If false, the user is -responsible for creating (possibly empty) `__init__.py` files and adding them to -the `srcs` of Python targets as required. - """, - ), - "_bootstrap_template": attr.label( - allow_single_file = True, - default = "@bazel_tools//tools/python:python_bootstrap_template.txt", - ), - "_launcher": attr.label( - cfg = "target", - # NOTE: This is an executable, but is only used for Windows. It - # can't have executable=True because the backing target is an - # empty target for other platforms. - default = "//tools/launcher:launcher", - ), - "_py_interpreter": attr.label( - # The configuration_field args are validated when called; - # we use the precense of py_internal to indicate this Bazel - # build has that fragment and name. - default = configuration_field( - fragment = "bazel_py", - name = "python_top", - ) if py_internal else None, - ), - # TODO: This appears to be vestigial. It's only added because - # GraphlessQueryTest.testLabelsOperator relies on it to test for - # query behavior of implicit dependencies. - "_py_toolchain_type": attr.label( - default = TARGET_TOOLCHAIN_TYPE, - ), - "_python_version_flag": attr.label( - default = "//python/config_settings:python_version", - ), - "_windows_launcher_maker": attr.label( - default = "@bazel_tools//tools/launcher:launcher_maker", - cfg = "exec", - executable = True, - ), - "_zipper": attr.label( - cfg = "exec", - executable = True, - default = "@bazel_tools//tools/zip:zipper", - ), - }, -) - -def create_executable_rule(*, attrs, **kwargs): - return create_base_executable_rule( - attrs = dicts.add(BAZEL_EXECUTABLE_ATTRS, attrs), - fragments = ["py", "bazel_py"], - **kwargs - ) - -def py_executable_bazel_impl(ctx, *, is_test, inherited_environment): - """Common code for executables for Bazel.""" - return py_executable_base_impl( - ctx = ctx, - semantics = create_binary_semantics_bazel(), - is_test = is_test, - inherited_environment = inherited_environment, - ) - -def create_binary_semantics_bazel(): - return create_binary_semantics_struct( - # keep-sorted start - create_executable = _create_executable, - get_cc_details_for_binary = _get_cc_details_for_binary, - get_central_uncachable_version_file = lambda ctx: None, - get_coverage_deps = _get_coverage_deps, - get_debugger_deps = _get_debugger_deps, - get_extra_common_runfiles_for_binary = lambda ctx: ctx.runfiles(), - get_extra_providers = _get_extra_providers, - get_extra_write_build_data_env = lambda ctx: {}, - get_imports = get_imports, - get_interpreter_path = _get_interpreter_path, - get_native_deps_dso_name = _get_native_deps_dso_name, - get_native_deps_user_link_flags = _get_native_deps_user_link_flags, - get_stamp_flag = _get_stamp_flag, - maybe_precompile = maybe_precompile, - should_build_native_deps_dso = lambda ctx: False, - should_create_init_files = _should_create_init_files, - should_include_build_data = lambda ctx: False, - # keep-sorted end - ) - -def _get_coverage_deps(ctx, runtime_details): - _ = ctx, runtime_details # @unused - return [] - -def _get_debugger_deps(ctx, runtime_details): - _ = ctx, runtime_details # @unused - return [] - -def _get_extra_providers(ctx, main_py, runtime_details): - _ = ctx, main_py, runtime_details # @unused - return [] - -def _get_stamp_flag(ctx): - # NOTE: Undocumented API; private to builtins - return ctx.configuration.stamp_binaries - -def _should_create_init_files(ctx): - if ctx.attr.legacy_create_init == -1: - return not ctx.fragments.py.default_to_explicit_init_py - else: - return bool(ctx.attr.legacy_create_init) - -def _create_executable( - ctx, - *, - executable, - main_py, - imports, - is_test, - runtime_details, - cc_details, - native_deps_details, - runfiles_details): - _ = is_test, cc_details, native_deps_details # @unused - - is_windows = target_platform_has_any_constraint(ctx, ctx.attr._windows_constraints) - - if is_windows: - if not executable.extension == "exe": - fail("Should not happen: somehow we are generating a non-.exe file on windows") - base_executable_name = executable.basename[0:-4] - else: - base_executable_name = executable.basename - - venv = None - - # The check for stage2_bootstrap_template is to support legacy - # BuiltinPyRuntimeInfo providers, which is likely to come from - # @bazel_tools//tools/python:autodetecting_toolchain, the toolchain used - # for workspace builds when no rules_python toolchain is configured. - if (BootstrapImplFlag.get_value(ctx) == BootstrapImplFlag.SCRIPT and - runtime_details.effective_runtime and - hasattr(runtime_details.effective_runtime, "stage2_bootstrap_template")): - venv = _create_venv( - ctx, - output_prefix = base_executable_name, - imports = imports, - runtime_details = runtime_details, - ) - - stage2_bootstrap = _create_stage2_bootstrap( - ctx, - output_prefix = base_executable_name, - output_sibling = executable, - main_py = main_py, - imports = imports, - runtime_details = runtime_details, - ) - extra_runfiles = ctx.runfiles([stage2_bootstrap] + venv.files_without_interpreter) - zip_main = _create_zip_main( - ctx, - stage2_bootstrap = stage2_bootstrap, - runtime_details = runtime_details, - venv = venv, - ) - else: - stage2_bootstrap = None - extra_runfiles = ctx.runfiles() - zip_main = ctx.actions.declare_file(base_executable_name + ".temp", sibling = executable) - _create_stage1_bootstrap( - ctx, - output = zip_main, - main_py = main_py, - imports = imports, - is_for_zip = True, - runtime_details = runtime_details, - ) - - zip_file = ctx.actions.declare_file(base_executable_name + ".zip", sibling = executable) - _create_zip_file( - ctx, - output = zip_file, - original_nonzip_executable = executable, - zip_main = zip_main, - runfiles = runfiles_details.default_runfiles.merge(extra_runfiles), - ) - - extra_files_to_build = [] - - # NOTE: --build_python_zip defaults to true on Windows - build_zip_enabled = ctx.fragments.py.build_python_zip - - # When --build_python_zip is enabled, then the zip file becomes - # one of the default outputs. - if build_zip_enabled: - extra_files_to_build.append(zip_file) - - # The logic here is a bit convoluted. Essentially, there are 3 types of - # executables produced: - # 1. (non-Windows) A bootstrap template based program. - # 2. (non-Windows) A self-executable zip file of a bootstrap template based program. - # 3. (Windows) A native Windows executable that finds and launches - # the actual underlying Bazel program (one of the above). Note that - # it implicitly assumes one of the above is located next to it, and - # that --build_python_zip defaults to true for Windows. - - should_create_executable_zip = False - bootstrap_output = None - if not is_windows: - if build_zip_enabled: - should_create_executable_zip = True - else: - bootstrap_output = executable - else: - _create_windows_exe_launcher( - ctx, - output = executable, - use_zip_file = build_zip_enabled, - python_binary_path = runtime_details.executable_interpreter_path, - ) - if not build_zip_enabled: - # On Windows, the main executable has an "exe" extension, so - # here we re-use the un-extensioned name for the bootstrap output. - bootstrap_output = ctx.actions.declare_file(base_executable_name) - - # The launcher looks for the non-zip executable next to - # itself, so add it to the default outputs. - extra_files_to_build.append(bootstrap_output) - - if should_create_executable_zip: - if bootstrap_output != None: - fail("Should not occur: bootstrap_output should not be used " + - "when creating an executable zip") - _create_executable_zip_file( - ctx, - output = executable, - zip_file = zip_file, - stage2_bootstrap = stage2_bootstrap, - runtime_details = runtime_details, - venv = venv, - ) - elif bootstrap_output: - _create_stage1_bootstrap( - ctx, - output = bootstrap_output, - stage2_bootstrap = stage2_bootstrap, - runtime_details = runtime_details, - is_for_zip = False, - imports = imports, - main_py = main_py, - venv = venv, - ) - else: - # Otherwise, this should be the Windows case of launcher + zip. - # Double check this just to make sure. - if not is_windows or not build_zip_enabled: - fail(("Should not occur: The non-executable-zip and " + - "non-bootstrap-template case should have windows and zip " + - "both true, but got " + - "is_windows={is_windows} " + - "build_zip_enabled={build_zip_enabled}").format( - is_windows = is_windows, - build_zip_enabled = build_zip_enabled, - )) - - # The interpreter is added this late in the process so that it isn't - # added to the zipped files. - if venv: - extra_runfiles = extra_runfiles.merge(ctx.runfiles([venv.interpreter])) - return create_executable_result_struct( - extra_files_to_build = depset(extra_files_to_build), - output_groups = {"python_zip_file": depset([zip_file])}, - extra_runfiles = extra_runfiles, - ) - -def _create_zip_main(ctx, *, stage2_bootstrap, runtime_details, venv): - python_binary = _runfiles_root_path(ctx, venv.interpreter.short_path) - python_binary_actual = venv.interpreter_actual_path - - # The location of this file doesn't really matter. It's added to - # the zip file as the top-level __main__.py file and not included - # elsewhere. - output = ctx.actions.declare_file(ctx.label.name + "_zip__main__.py") - ctx.actions.expand_template( - template = runtime_details.effective_runtime.zip_main_template, - output = output, - substitutions = { - "%python_binary%": python_binary, - "%python_binary_actual%": python_binary_actual, - "%stage2_bootstrap%": "{}/{}".format( - ctx.workspace_name, - stage2_bootstrap.short_path, - ), - "%workspace_name%": ctx.workspace_name, - }, - ) - return output - -def relative_path(from_, to): - """Compute a relative path from one path to another. - - Args: - from_: {type}`str` the starting directory. Note that it should be - a directory because relative-symlinks are relative to the - directory the symlink resides in. - to: {type}`str` the path that `from_` wants to point to - - Returns: - {type}`str` a relative path - """ - from_parts = from_.split("/") - to_parts = to.split("/") - - # Strip common leading parts from both paths - n = min(len(from_parts), len(to_parts)) - for _ in range(n): - if from_parts[0] == to_parts[0]: - from_parts.pop(0) - to_parts.pop(0) - else: - break - - # Impossible to compute a relative path without knowing what ".." is - if from_parts and from_parts[0] == "..": - fail("cannot compute relative path from '%s' to '%s'", from_, to) - - parts = ([".."] * len(from_parts)) + to_parts - return paths.join(*parts) - -# Create a venv the executable can use. -# For venv details and the venv startup process, see: -# * https://docs.python.org/3/library/venv.html -# * https://snarky.ca/how-virtual-environments-work/ -# * https://github.com/python/cpython/blob/main/Modules/getpath.py -# * https://github.com/python/cpython/blob/main/Lib/site.py -def _create_venv(ctx, output_prefix, imports, runtime_details): - venv = "_{}.venv".format(output_prefix.lstrip("_")) - - # The pyvenv.cfg file must be present to trigger the venv site hooks. - # Because it's paths are expected to be absolute paths, we can't reliably - # put much in it. See https://github.com/python/cpython/issues/83650 - pyvenv_cfg = ctx.actions.declare_file("{}/pyvenv.cfg".format(venv)) - ctx.actions.write(pyvenv_cfg, "") - - runtime = runtime_details.effective_runtime - if runtime.interpreter: - py_exe_basename = paths.basename(runtime.interpreter.short_path) - - # Even though ctx.actions.symlink() is used, using - # declare_symlink() is required to ensure that the resulting file - # in runfiles is always a symlink. An RBE implementation, for example, - # may choose to write what symlink() points to instead. - interpreter = ctx.actions.declare_symlink("{}/bin/{}".format(venv, py_exe_basename)) - - interpreter_actual_path = _runfiles_root_path(ctx, runtime.interpreter.short_path) - rel_path = relative_path( - # dirname is necessary because a relative symlink is relative to - # the directory the symlink resides within. - from_ = paths.dirname(_runfiles_root_path(ctx, interpreter.short_path)), - to = interpreter_actual_path, - ) - - ctx.actions.symlink(output = interpreter, target_path = rel_path) - else: - py_exe_basename = paths.basename(runtime.interpreter_path) - interpreter = ctx.actions.declare_symlink("{}/bin/{}".format(venv, py_exe_basename)) - ctx.actions.symlink(output = interpreter, target_path = runtime.interpreter_path) - interpreter_actual_path = runtime.interpreter_path - - if runtime.interpreter_version_info: - version = "{}.{}".format( - runtime.interpreter_version_info.major, - runtime.interpreter_version_info.minor, - ) - else: - version_flag = ctx.attr._python_version_flag[config_common.FeatureFlagInfo].value - version_flag_parts = version_flag.split(".")[0:2] - version = "{}.{}".format(*version_flag_parts) - - # See site.py logic: free-threaded builds append "t" to the venv lib dir name - if "t" in runtime.abi_flags: - version += "t" - - site_packages = "{}/lib/python{}/site-packages".format(venv, version) - pth = ctx.actions.declare_file("{}/bazel.pth".format(site_packages)) - ctx.actions.write(pth, "import _bazel_site_init\n") - - site_init = ctx.actions.declare_file("{}/_bazel_site_init.py".format(site_packages)) - computed_subs = ctx.actions.template_dict() - computed_subs.add_joined("%imports%", imports, join_with = ":", map_each = _map_each_identity) - ctx.actions.expand_template( - template = runtime.site_init_template, - output = site_init, - substitutions = { - "%import_all%": "True" if ctx.fragments.bazel_py.python_import_all_repositories else "False", - "%site_init_runfiles_path%": "{}/{}".format(ctx.workspace_name, site_init.short_path), - "%workspace_name%": ctx.workspace_name, - }, - computed_substitutions = computed_subs, - ) - - return struct( - interpreter = interpreter, - # Runfiles root relative path or absolute path - interpreter_actual_path = interpreter_actual_path, - files_without_interpreter = [pyvenv_cfg, pth, site_init], - ) - -def _map_each_identity(v): - return v - -def _create_stage2_bootstrap( - ctx, - *, - output_prefix, - output_sibling, - main_py, - imports, - runtime_details): - output = ctx.actions.declare_file( - # Prepend with underscore to prevent pytest from trying to - # process the bootstrap for files starting with `test_` - "_{}_stage2_bootstrap.py".format(output_prefix), - sibling = output_sibling, - ) - runtime = runtime_details.effective_runtime - if (ctx.configuration.coverage_enabled and - runtime and - runtime.coverage_tool): - coverage_tool_runfiles_path = "{}/{}".format( - ctx.workspace_name, - runtime.coverage_tool.short_path, - ) - else: - coverage_tool_runfiles_path = "" - - template = runtime.stage2_bootstrap_template - - ctx.actions.expand_template( - template = template, - output = output, - substitutions = { - "%coverage_tool%": coverage_tool_runfiles_path, - "%import_all%": "True" if ctx.fragments.bazel_py.python_import_all_repositories else "False", - "%imports%": ":".join(imports.to_list()), - "%main%": "{}/{}".format(ctx.workspace_name, main_py.short_path), - "%target%": str(ctx.label), - "%workspace_name%": ctx.workspace_name, - }, - is_executable = True, - ) - return output - -def _runfiles_root_path(ctx, short_path): - """Compute a runfiles-root relative path from `File.short_path` - - Args: - ctx: current target ctx - short_path: str, a main-repo relative path from `File.short_path` - - Returns: - {type}`str`, a runflies-root relative path - """ - - # The ../ comes from short_path is for files in other repos. - if short_path.startswith("../"): - return short_path[3:] - else: - return "{}/{}".format(ctx.workspace_name, short_path) - -def _create_stage1_bootstrap( - ctx, - *, - output, - main_py = None, - stage2_bootstrap = None, - imports = None, - is_for_zip, - runtime_details, - venv = None): - runtime = runtime_details.effective_runtime - - if venv: - python_binary_path = _runfiles_root_path(ctx, venv.interpreter.short_path) - else: - python_binary_path = runtime_details.executable_interpreter_path - - if is_for_zip and venv: - python_binary_actual = venv.interpreter_actual_path - else: - python_binary_actual = "" - - subs = { - "%is_zipfile%": "1" if is_for_zip else "0", - "%python_binary%": python_binary_path, - "%python_binary_actual%": python_binary_actual, - "%target%": str(ctx.label), - "%workspace_name%": ctx.workspace_name, - } - - if stage2_bootstrap: - subs["%stage2_bootstrap%"] = "{}/{}".format( - ctx.workspace_name, - stage2_bootstrap.short_path, - ) - template = runtime.bootstrap_template - subs["%shebang%"] = runtime.stub_shebang - else: - if (ctx.configuration.coverage_enabled and - runtime and - runtime.coverage_tool): - coverage_tool_runfiles_path = "{}/{}".format( - ctx.workspace_name, - runtime.coverage_tool.short_path, - ) - else: - coverage_tool_runfiles_path = "" - if runtime: - subs["%shebang%"] = runtime.stub_shebang - template = runtime.bootstrap_template - else: - subs["%shebang%"] = DEFAULT_STUB_SHEBANG - template = ctx.file._bootstrap_template - - subs["%coverage_tool%"] = coverage_tool_runfiles_path - subs["%import_all%"] = ("True" if ctx.fragments.bazel_py.python_import_all_repositories else "False") - subs["%imports%"] = ":".join(imports.to_list()) - subs["%main%"] = "{}/{}".format(ctx.workspace_name, main_py.short_path) - - ctx.actions.expand_template( - template = template, - output = output, - substitutions = subs, - ) - -def _create_windows_exe_launcher( - ctx, - *, - output, - python_binary_path, - use_zip_file): - launch_info = ctx.actions.args() - launch_info.use_param_file("%s", use_always = True) - launch_info.set_param_file_format("multiline") - launch_info.add("binary_type=Python") - launch_info.add(ctx.workspace_name, format = "workspace_name=%s") - launch_info.add( - "1" if py_internal.runfiles_enabled(ctx) else "0", - format = "symlink_runfiles_enabled=%s", - ) - launch_info.add(python_binary_path, format = "python_bin_path=%s") - launch_info.add("1" if use_zip_file else "0", format = "use_zip_file=%s") - - launcher = ctx.attr._launcher[DefaultInfo].files_to_run.executable - ctx.actions.run( - executable = ctx.executable._windows_launcher_maker, - arguments = [launcher.path, launch_info, output.path], - inputs = [launcher], - outputs = [output], - mnemonic = "PyBuildLauncher", - progress_message = "Creating launcher for %{label}", - # Needed to inherit PATH when using non-MSVC compilers like MinGW - use_default_shell_env = True, - ) - -def _create_zip_file(ctx, *, output, original_nonzip_executable, zip_main, runfiles): - """Create a Python zipapp (zip with __main__.py entry point).""" - workspace_name = ctx.workspace_name - legacy_external_runfiles = _py_builtins.get_legacy_external_runfiles(ctx) - - manifest = ctx.actions.args() - manifest.use_param_file("@%s", use_always = True) - manifest.set_param_file_format("multiline") - - manifest.add("__main__.py={}".format(zip_main.path)) - manifest.add("__init__.py=") - manifest.add( - "{}=".format( - _get_zip_runfiles_path("__init__.py", workspace_name, legacy_external_runfiles), - ), - ) - for path in runfiles.empty_filenames.to_list(): - manifest.add("{}=".format(_get_zip_runfiles_path(path, workspace_name, legacy_external_runfiles))) - - def map_zip_runfiles(file): - if file != original_nonzip_executable and file != output: - return "{}={}".format( - _get_zip_runfiles_path(file.short_path, workspace_name, legacy_external_runfiles), - file.path, - ) - else: - return None - - manifest.add_all(runfiles.files, map_each = map_zip_runfiles, allow_closure = True) - - inputs = [zip_main] - if _py_builtins.is_bzlmod_enabled(ctx): - zip_repo_mapping_manifest = ctx.actions.declare_file( - output.basename + ".repo_mapping", - sibling = output, - ) - _py_builtins.create_repo_mapping_manifest( - ctx = ctx, - runfiles = runfiles, - output = zip_repo_mapping_manifest, - ) - manifest.add("{}/_repo_mapping={}".format( - _ZIP_RUNFILES_DIRECTORY_NAME, - zip_repo_mapping_manifest.path, - )) - inputs.append(zip_repo_mapping_manifest) - - for artifact in runfiles.files.to_list(): - # Don't include the original executable because it isn't used by the - # zip file, so no need to build it for the action. - # Don't include the zipfile itself because it's an output. - if artifact != original_nonzip_executable and artifact != output: - inputs.append(artifact) - - zip_cli_args = ctx.actions.args() - zip_cli_args.add("cC") - zip_cli_args.add(output) - - ctx.actions.run( - executable = ctx.executable._zipper, - arguments = [zip_cli_args, manifest], - inputs = depset(inputs), - outputs = [output], - use_default_shell_env = True, - mnemonic = "PythonZipper", - progress_message = "Building Python zip: %{label}", - ) - -def _get_zip_runfiles_path(path, workspace_name, legacy_external_runfiles): - if legacy_external_runfiles and path.startswith(_EXTERNAL_PATH_PREFIX): - zip_runfiles_path = paths.relativize(path, _EXTERNAL_PATH_PREFIX) - else: - # NOTE: External runfiles (artifacts in other repos) will have a leading - # path component of "../" so that they refer outside the main workspace - # directory and into the runfiles root. By normalizing, we simplify e.g. - # "workspace/../foo/bar" to simply "foo/bar". - zip_runfiles_path = paths.normalize("{}/{}".format(workspace_name, path)) - return "{}/{}".format(_ZIP_RUNFILES_DIRECTORY_NAME, zip_runfiles_path) - -def _create_executable_zip_file( - ctx, - *, - output, - zip_file, - stage2_bootstrap, - runtime_details, - venv): - prelude = ctx.actions.declare_file( - "{}_zip_prelude.sh".format(output.basename), - sibling = output, - ) - if stage2_bootstrap: - _create_stage1_bootstrap( - ctx, - output = prelude, - stage2_bootstrap = stage2_bootstrap, - runtime_details = runtime_details, - is_for_zip = True, - venv = venv, - ) - else: - ctx.actions.write(prelude, "#!/usr/bin/env python3\n") - - ctx.actions.run_shell( - command = "cat {prelude} {zip} > {output}".format( - prelude = prelude.path, - zip = zip_file.path, - output = output.path, - ), - inputs = [prelude, zip_file], - outputs = [output], - use_default_shell_env = True, - mnemonic = "PyBuildExecutableZip", - progress_message = "Build Python zip executable: %{label}", - ) - -def _get_cc_details_for_binary(ctx, extra_deps): - cc_info = collect_cc_info(ctx, extra_deps = extra_deps) - return create_cc_details_struct( - cc_info_for_propagating = cc_info, - cc_info_for_self_link = cc_info, - cc_info_with_extra_link_time_libraries = None, - extra_runfiles = ctx.runfiles(), - # Though the rules require the CcToolchain, it isn't actually used. - cc_toolchain = None, - feature_config = None, - ) - -def _get_interpreter_path(ctx, *, runtime, flag_interpreter_path): - if runtime: - if runtime.interpreter_path: - interpreter_path = runtime.interpreter_path - else: - interpreter_path = "{}/{}".format( - ctx.workspace_name, - runtime.interpreter.short_path, - ) - - # NOTE: External runfiles (artifacts in other repos) will have a - # leading path component of "../" so that they refer outside the - # main workspace directory and into the runfiles root. By - # normalizing, we simplify e.g. "workspace/../foo/bar" to simply - # "foo/bar" - interpreter_path = paths.normalize(interpreter_path) - - elif flag_interpreter_path: - interpreter_path = flag_interpreter_path - else: - fail("Unable to determine interpreter path") - - return interpreter_path - -def _get_native_deps_dso_name(ctx): - _ = ctx # @unused - fail("Building native deps DSO not supported.") - -def _get_native_deps_user_link_flags(ctx): - _ = ctx # @unused - fail("Building native deps DSO not supported.") diff --git a/python/private/py_library_rule.bzl b/python/private/py_library_rule.bzl index ed64716122..8a8d6cf380 100644 --- a/python/private/py_library_rule.bzl +++ b/python/private/py_library_rule.bzl @@ -13,8 +13,8 @@ # limitations under the License. """Implementation of py_library rule.""" -load(":common.bzl", "create_library_semantics_struct") -load(":common_bazel.bzl", "collect_cc_info", "get_imports", "maybe_precompile") +load(":common.bzl", "collect_cc_info", "create_library_semantics_struct", "get_imports") +load(":precompile.bzl", "maybe_precompile") load(":py_library.bzl", "create_py_library_rule", "py_library_impl") def _py_library_impl_with_semantics(ctx): diff --git a/python/private/py_test_macro.bzl b/python/private/py_test_macro.bzl index 1f9330f8e5..348e877225 100644 --- a/python/private/py_test_macro.bzl +++ b/python/private/py_test_macro.bzl @@ -13,7 +13,7 @@ # limitations under the License. """Implementation of macro-half of py_test rule.""" -load(":common_bazel.bzl", "convert_legacy_create_init_to_int") +load(":py_executable.bzl", "convert_legacy_create_init_to_int") load(":py_test_rule.bzl", py_test_rule = "py_test") def py_test(**kwargs): diff --git a/python/private/py_test_rule.bzl b/python/private/py_test_rule.bzl index 64d5f21f81..63000c7255 100644 --- a/python/private/py_test_rule.bzl +++ b/python/private/py_test_rule.bzl @@ -17,9 +17,9 @@ load("@bazel_skylib//lib:dicts.bzl", "dicts") load(":attributes.bzl", "AGNOSTIC_TEST_ATTRS") load(":common.bzl", "maybe_add_test_execution_info") load( - ":py_executable_bazel.bzl", + ":py_executable.bzl", "create_executable_rule", - "py_executable_bazel_impl", + "py_executable_impl", ) _BAZEL_PY_TEST_ATTRS = { @@ -40,7 +40,7 @@ _BAZEL_PY_TEST_ATTRS = { } def _py_test_impl(ctx): - providers = py_executable_bazel_impl( + providers = py_executable_impl( ctx = ctx, is_test = True, inherited_environment = ctx.attr.env_inherit, diff --git a/tests/bootstrap_impls/venv_relative_path_tests.bzl b/tests/bootstrap_impls/venv_relative_path_tests.bzl index b21f220205..ad4870fe08 100644 --- a/tests/bootstrap_impls/venv_relative_path_tests.bzl +++ b/tests/bootstrap_impls/venv_relative_path_tests.bzl @@ -15,7 +15,7 @@ "Unit tests for relative_path computation" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private:py_executable_bazel.bzl", "relative_path") # buildifier: disable=bzl-visibility +load("//python/private:py_executable.bzl", "relative_path") # buildifier: disable=bzl-visibility _tests = [] From b5729b41ef18393c2609aa1633607695175a7419 Mon Sep 17 00:00:00 2001 From: Niko Wenselowski Date: Tue, 24 Dec 2024 01:34:04 +0100 Subject: [PATCH 4/5] feat(toolchain): Add support for Python 3.13.1. (#2482) Adds 3.13.1 Python toolchain. Also updates 3.13 to map to 3.13.1. Tests have been slightly altered to make spotting the override easier. --------- Co-authored-by: Richard Levasseur Co-authored-by: Ignas Anikevicius <240938+aignas@users.noreply.github.com> --- CHANGELOG.md | 9 +++ python/versions.bzl | 103 ++++++++++++++++++++++++++++++++-- tests/python/python_tests.bzl | 19 ++++--- 3 files changed, 118 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a3436487e..eb4bcfa8da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,7 @@ Unreleased changes template. {#v0-0-0-changed} ### Changed +* (toolchains) 3.13 means 3.13.1 (previously 3.13.0) * Bazel 6 support is dropped and Bazel 7.4.1 is the minimum supported version, per our Bazel support matrix. Earlier versions are not tested by CI, so functionality cannot be guaranteed. @@ -88,6 +89,14 @@ Unreleased changes template. To select the free-threaded interpreter in the repo phase, please use the documented [env](/environment-variables.html) variables. Fixes [#2386](https://github.com/bazelbuild/rules_python/issues/2386). +* (toolchains) Use the latest astrahl-sh toolchain release [20241206] for Python versions: + * 3.9.21 + * 3.10.16 + * 3.11.11 + * 3.12.8 + * 3.13.1 + +[20241206]: https://github.com/astral-sh/python-build-standalone/releases/tag/20241206 {#v0-0-0-removed} ### Removed diff --git a/python/versions.bzl b/python/versions.bzl index 1fd0649f12..098362b7d3 100644 --- a/python/versions.bzl +++ b/python/versions.bzl @@ -252,6 +252,20 @@ TOOL_VERSIONS = { }, "strip_prefix": "python", }, + "3.9.21": { + "url": "20241206/cpython-{python_version}+20241206-{platform}-{build}.tar.gz", + "sha256": { + "aarch64-apple-darwin": "4bddc18228789d0316dcebc45b2242e0010fa6bc33c302b6b5a62a5ac39d2147", + "aarch64-unknown-linux-gnu": "7d3b4ab90f73fa9dab0c350ca64b1caa9b8e4655913acd098e594473c49921c8", + "ppc64le-unknown-linux-gnu": "966477345ca93f056cf18de9cff961aacda2318a8e641546e0fd7222f1362ee2", + "s390x-unknown-linux-gnu": "3ba05a408edce4e20ebd116643c8418e62f7c8066c8a35fe8d3b78371d90b46a", + "x86_64-apple-darwin": "619f5082288c771ad9b71e2daaf6df6bd39ca86e442638d150a71a6ccf62978d", + "x86_64-pc-windows-msvc": "82736b5a185c57b296188ce778ed865ff10edc5fe9ff1ec4cb33b39ac8e4819c", + "x86_64-unknown-linux-gnu": "208b2adc7c7e5d5df6d9385400dc7c4e3b4c3eed428e19a2326848978e98517e", + "x86_64-unknown-linux-musl": "67c058dbaae8fd8c4f68e13b10805a9227918afc94326f21a9a2ec2daca3ddbd", + }, + "strip_prefix": "python", + }, "3.10.2": { "url": "20220227/cpython-{python_version}+20220227-{platform}-{build}.tar.gz", "sha256": { @@ -372,6 +386,20 @@ TOOL_VERSIONS = { }, "strip_prefix": "python", }, + "3.10.16": { + "url": "20241206/cpython-{python_version}+20241206-{platform}-{build}.tar.gz", + "sha256": { + "aarch64-apple-darwin": "c2d25840756127f3583b04b0697bef79edacb15f1402cd980292c93488c3df22", + "aarch64-unknown-linux-gnu": "bbfc345615c5ed33916b4fd959fc16fa2e896a3c5eec1fb782c91b47c85c0542", + "ppc64le-unknown-linux-gnu": "cb474b392733d5ac2adaa1cfcc2b63b957611dc26697e76822706cc61ac21515", + "s390x-unknown-linux-gnu": "886a7effc8a3061d53cacc9cf54e82d6d57ac3665c258c6a2193528c16b557cd", + "x86_64-apple-darwin": "31a110b631eb79103675ed556255045deeea5ff533296d7f35b4d195a0df0315", + "x86_64-pc-windows-msvc": "fb7870717dc7e3aedcbab4a647782637da0046a4238db1d41eeaabb78566d814", + "x86_64-unknown-linux-gnu": "b15de0d63eed9871ed57285f81fd123cf6c4117251a9cac8f81f9cf0cccc0a53", + "x86_64-unknown-linux-musl": "bf956eeffcff002d2f38232faa750c279cbb76197b744761d1b253bf94d6f637", + }, + "strip_prefix": "python", + }, "3.11.1": { "url": "20230116/cpython-{python_version}+20230116-{platform}-{build}.tar.gz", "sha256": { @@ -487,6 +515,20 @@ TOOL_VERSIONS = { }, "strip_prefix": "python", }, + "3.11.11": { + "url": "20241206/cpython-{python_version}+20241206-{platform}-{build}.tar.gz", + "sha256": { + "aarch64-apple-darwin": "566c5e266f2c933d0c0b213a75496bc6a090e493097802f809dbe21c75cd5d13", + "aarch64-unknown-linux-gnu": "50ee364cfa24ee7d933eda955c9fe455bc0a8ebb9d998c9948f2909dac701dd9", + "ppc64le-unknown-linux-gnu": "e0cdc00e42a05191b9b75ba976fc0fca9205c66fdaef7571c20532346fd3db1e", + "s390x-unknown-linux-gnu": "3b106b8a3c5aa97ff76200cd0d9ba6eaed23d88ccb947e00ff6bb2d9f5422d2a", + "x86_64-apple-darwin": "8ecd267281fb5b2464ddcd2de79622cfa7aff42e929b17989da2721ba39d4a5e", + "x86_64-pc-windows-msvc": "d8986f026599074ddd206f3f62d6f2c323ca8fa7a854bf744989bfc0b12f5d0d", + "x86_64-unknown-linux-gnu": "57a171af687c926c5cabe3d1c7ce9950b98f00b932accd596eb60e14ca39c42d", + "x86_64-unknown-linux-musl": "8129a9a5c3f2654e1a9eed6093f5dc42399667b341050ff03219cb7df210c348", + }, + "strip_prefix": "python", + }, "3.12.0": { "url": "20231002/cpython-{python_version}+20231002-{platform}-{build}.tar.gz", "sha256": { @@ -566,6 +608,20 @@ TOOL_VERSIONS = { }, "strip_prefix": "python", }, + "3.12.8": { + "url": "20241206/cpython-{python_version}+20241206-{platform}-{build}.tar.gz", + "sha256": { + "aarch64-apple-darwin": "e3c4aa607717b23903ca2650d5c3ee24f89b97543e2db2b0f463bddc7a9e92f3", + "aarch64-unknown-linux-gnu": "ce674b55442b732973afb2932c281bb1ded4ad7e22bcf9b07071165770758c7e", + "ppc64le-unknown-linux-gnu": "b7214790b273de9ed0532420054b72ba1393d62d2fc844ec55ade193771bd90c", + "s390x-unknown-linux-gnu": "73102f5dbd7d1e7e9c2f2c80aedf2893d99a7fa407f6674ec8b2f57ba07daee5", + "x86_64-apple-darwin": "3ba35c706577d755e8e52a4c161a042464577c0e695e2a605362fa469e26de10", + "x86_64-pc-windows-msvc": "767b4be3ddf6b99e5ade519789c1615c191d8cf99d5aff4685cc18b48931f1e6", + "x86_64-unknown-linux-gnu": "b9d6ee5ddac1198e72d53112698773fc8bb597de095592eb849ca794306699ba", + "x86_64-unknown-linux-musl": "6f305888703691dd04cfff85284d23ea0b0146ed7c4415e472f1fb72b3f32cdf", + }, + "strip_prefix": "python", + }, "3.13.0": { "url": "20241016/cpython-{python_version}+20241016-{platform}-{build}.{ext}", "sha256": { @@ -603,16 +659,53 @@ TOOL_VERSIONS = { "x86_64-unknown-linux-gnu-freethreaded": "python/install", }, }, + "3.13.1": { + "url": "20241205/cpython-{python_version}+20241205-{platform}-{build}.{ext}", + "sha256": { + "aarch64-apple-darwin": "88b88b609129c12f4b3841845aca13230f61e97ba97bd0fb28ee64b0e442a34f", + "aarch64-unknown-linux-gnu": "fdfa86c2746d2ae700042c461846e6c37f70c249925b58de8cd02eb8d1423d4e", + "ppc64le-unknown-linux-gnu": "27b20b3237c55430ca1304e687d021f88373f906249f9cd272c5ff2803d5e5c3", + "s390x-unknown-linux-gnu": "7d0187e20cb5e36c689eec27e4d3de56d8b7f1c50dc5523550fc47377801521f", + "x86_64-apple-darwin": "47eef6efb8664e2d1d23a7cdaf56262d784f8ace48f3bfca1b183e95a49888d6", + "x86_64-pc-windows-msvc": "f51f0493a5f979ff0b8d8c598a8d74f2a4d86a190c2729c85e0af65c36a9cbbe", + "x86_64-unknown-linux-gnu": "242b2727df6c1e00de6a9f0f0dcb4562e168d27f428c785b0eb41a6aeb34d69a", + "x86_64-unknown-linux-musl": "76b30c6373b9c0aa2ba610e07da02f384aa210ac79643da38c66d3e6171c6ef5", + "aarch64-apple-darwin-freethreaded": "08f05618bdcf8064a7960b25d9ba92155447c9b08e0cf2f46a981e4c6a1bb5a5", + "aarch64-unknown-linux-gnu-freethreaded": "9f2fcb809f9ba6c7c014a8803073a88786701a98971135bce684355062e4bb35", + "ppc64le-unknown-linux-gnu-freethreaded": "15ceea78dff78ca8ccaac8d9c54b808af30daaa126f1f561e920a6896e098634", + "s390x-unknown-linux-gnu-freethreaded": "ed3c6118d1d12603309c930e93421ac7a30a69045ffd43006f63ecf71d72c317", + "x86_64-apple-darwin-freethreaded": "dc780fecd215d2cc9e573abf1e13a175fcfa8f6efd100ef888494a248a16cda8", + "x86_64-pc-windows-msvc-freethreaded": "7537b2ab361c0eabc0eabfca9ffd9862d7f5f6576eda13b97e98aceb5eea4fd3", + "x86_64-unknown-linux-gnu-freethreaded": "9ec1b81213f849d91f5ebe6a16196e85cd6ff7c05ca823ce0ab7ba5b0e9fee84", + }, + "strip_prefix": { + "aarch64-apple-darwin": "python", + "aarch64-unknown-linux-gnu": "python", + "ppc64le-unknown-linux-gnu": "python", + "s390x-unknown-linux-gnu": "python", + "x86_64-apple-darwin": "python", + "x86_64-pc-windows-msvc": "python", + "x86_64-unknown-linux-gnu": "python", + "x86_64-unknown-linux-musl": "python", + "aarch64-apple-darwin-freethreaded": "python/install", + "aarch64-unknown-linux-gnu-freethreaded": "python/install", + "ppc64le-unknown-linux-gnu-freethreaded": "python/install", + "s390x-unknown-linux-gnu-freethreaded": "python/install", + "x86_64-apple-darwin-freethreaded": "python/install", + "x86_64-pc-windows-msvc-freethreaded": "python/install", + "x86_64-unknown-linux-gnu-freethreaded": "python/install", + }, + }, } # buildifier: disable=unsorted-dict-items MINOR_MAPPING = { "3.8": "3.8.20", - "3.9": "3.9.20", - "3.10": "3.10.15", - "3.11": "3.11.10", - "3.12": "3.12.7", - "3.13": "3.13.0", + "3.9": "3.9.21", + "3.10": "3.10.16", + "3.11": "3.11.11", + "3.12": "3.12.8", + "3.13": "3.13.1", } def _generate_platforms(): diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index 40504302d1..e7828b92f5 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -413,7 +413,7 @@ def _test_add_new_version(env): strip_prefix = "python", platform = "aarch64-unknown-linux-gnu", coverage_tool = "specific_cov_tool", - python_version = "3.13.1", + python_version = "3.13.99", patch_strip = 2, patches = ["specific-patch.txt"], ), @@ -421,9 +421,9 @@ def _test_add_new_version(env): override = [ _override( base_url = "", - available_python_versions = ["3.12.4", "3.13.0", "3.13.1"], + available_python_versions = ["3.12.4", "3.13.0", "3.13.1", "3.13.99"], minor_mapping = { - "3.13": "3.13.0", + "3.13": "3.13.99", }, ), ], @@ -436,13 +436,14 @@ def _test_add_new_version(env): "3.12.4", "3.13.0", "3.13.1", + "3.13.99", ]) env.expect.that_dict(py.config.default["tool_versions"]["3.13.0"]).contains_exactly({ "sha256": {"aarch64-unknown-linux-gnu": "deadbeef"}, "strip_prefix": {"aarch64-unknown-linux-gnu": "prefix"}, "url": {"aarch64-unknown-linux-gnu": ["example.org"]}, }) - env.expect.that_dict(py.config.default["tool_versions"]["3.13.1"]).contains_exactly({ + env.expect.that_dict(py.config.default["tool_versions"]["3.13.99"]).contains_exactly({ "coverage_tool": {"aarch64-unknown-linux-gnu": "specific_cov_tool"}, "patch_strip": {"aarch64-unknown-linux-gnu": 2}, "patches": {"aarch64-unknown-linux-gnu": ["specific-patch.txt"]}, @@ -452,7 +453,7 @@ def _test_add_new_version(env): }) env.expect.that_dict(py.config.minor_mapping).contains_exactly({ "3.12": "3.12.4", # The `minor_mapping` will be overriden only for the missing keys - "3.13": "3.13.0", + "3.13": "3.13.99", }) env.expect.that_collection(py.toolchains).contains_exactly([ struct( @@ -484,13 +485,13 @@ def _test_register_all_versions(env): sha256 = "deadb00f", urls = ["something.org"], platform = "aarch64-unknown-linux-gnu", - python_version = "3.13.1", + python_version = "3.13.99", ), ], override = [ _override( base_url = "", - available_python_versions = ["3.12.4", "3.13.0", "3.13.1"], + available_python_versions = ["3.12.4", "3.13.0", "3.13.1", "3.13.99"], register_all_versions = True, ), ], @@ -503,11 +504,12 @@ def _test_register_all_versions(env): "3.12.4", "3.13.0", "3.13.1", + "3.13.99", ]) env.expect.that_dict(py.config.minor_mapping).contains_exactly({ # The mapping is calculated automatically "3.12": "3.12.4", - "3.13": "3.13.1", + "3.13": "3.13.99", }) env.expect.that_collection(py.toolchains).contains_exactly([ struct( @@ -521,6 +523,7 @@ def _test_register_all_versions(env): "python_3_13": "3.13", "python_3_13_0": "3.13.0", "python_3_13_1": "3.13.1", + "python_3_13_99": "3.13.99", }.items() ]) From 922929b6b7f8e9426e4a8d29aaebada9a4d14599 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 24 Dec 2024 16:12:25 +0900 Subject: [PATCH 5/5] refactor: stop warning if we don't find anything via SimpleAPI (#2532) The warning is somewhat non-actionable and the sources can be inspected via the MODULE.bazel.lock file if needed. This makes it easier to make this option a default at some point. At the same time cleanup the code since we are not using the `get_index_urls` to print the warning. Work towards #260 --- python/private/pypi/extension.bzl | 119 +++++++++------------ python/private/pypi/parse_requirements.bzl | 5 +- 2 files changed, 56 insertions(+), 68 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 9b150bdce0..e1904912fd 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -105,7 +105,6 @@ def _create_whl_repos( # containers to aggregate outputs from this function whl_map = {} - exposed_packages = {} extra_aliases = { whl_name: {alias: True for alias in aliases} for whl_name, aliases in pip_attr.extra_hub_aliases.items() @@ -219,8 +218,6 @@ def _create_whl_repos( ) for whl_name, requirements in requirements_by_platform.items(): - whl_name = normalize_name(whl_name) - group_name = whl_group_mapping.get(whl_name) group_deps = requirement_cycles.get(group_name, []) @@ -261,68 +258,55 @@ def _create_whl_repos( if v != default }) - is_exposed = False - if get_index_urls: - # TODO @aignas 2024-05-26: move to a separate function - found_something = False - for requirement in requirements: - is_exposed = is_exposed or requirement.is_exposed - dists = requirement.whls - if not pip_attr.download_only and requirement.sdist: - dists = dists + [requirement.sdist] - - for distribution in dists: - found_something = True - is_reproducible = False - - args = dict(whl_library_args) - if pip_attr.netrc: - args["netrc"] = pip_attr.netrc - if pip_attr.auth_patterns: - args["auth_patterns"] = pip_attr.auth_patterns - - if not distribution.filename.endswith(".whl"): - # pip is not used to download wheels and the python - # `whl_library` helpers are only extracting things, however - # for sdists, they will be built by `pip`, so we still - # need to pass the extra args there. - args["extra_pip_args"] = requirement.extra_pip_args - - # This is no-op because pip is not used to download the wheel. - args.pop("download_only", None) - - repo_name = whl_repo_name(pip_name, distribution.filename, distribution.sha256) - args["requirement"] = requirement.srcs.requirement - args["urls"] = [distribution.url] - args["sha256"] = distribution.sha256 - args["filename"] = distribution.filename - args["experimental_target_platforms"] = requirement.target_platforms - - # Pure python wheels or sdists may need to have a platform here - target_platforms = None - if distribution.filename.endswith("-any.whl") or not distribution.filename.endswith(".whl"): - if len(requirements) > 1: - target_platforms = requirement.target_platforms - - whl_libraries[repo_name] = args - - whl_map.setdefault(whl_name, {})[whl_config_setting( - version = major_minor, - filename = distribution.filename, - target_platforms = target_platforms, - )] = repo_name - - if found_something: - if is_exposed: - exposed_packages[whl_name] = None - continue - - is_exposed = False + # TODO @aignas 2024-05-26: move to a separate function for requirement in requirements: - is_exposed = is_exposed or requirement.is_exposed - if get_index_urls: - logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.srcs.requirement_line)) + dists = requirement.whls + if not pip_attr.download_only and requirement.sdist: + dists = dists + [requirement.sdist] + + for distribution in dists: + args = dict(whl_library_args) + if pip_attr.netrc: + args["netrc"] = pip_attr.netrc + if pip_attr.auth_patterns: + args["auth_patterns"] = pip_attr.auth_patterns + + if not distribution.filename.endswith(".whl"): + # pip is not used to download wheels and the python + # `whl_library` helpers are only extracting things, however + # for sdists, they will be built by `pip`, so we still + # need to pass the extra args there. + args["extra_pip_args"] = requirement.extra_pip_args + + # This is no-op because pip is not used to download the wheel. + args.pop("download_only", None) + + repo_name = whl_repo_name(pip_name, distribution.filename, distribution.sha256) + args["requirement"] = requirement.srcs.requirement + args["urls"] = [distribution.url] + args["sha256"] = distribution.sha256 + args["filename"] = distribution.filename + args["experimental_target_platforms"] = requirement.target_platforms + + # Pure python wheels or sdists may need to have a platform here + target_platforms = None + if distribution.filename.endswith("-any.whl") or not distribution.filename.endswith(".whl"): + if len(requirements) > 1: + target_platforms = requirement.target_platforms + + whl_libraries[repo_name] = args + + whl_map.setdefault(whl_name, {})[whl_config_setting( + version = major_minor, + filename = distribution.filename, + target_platforms = target_platforms, + )] = repo_name + + if dists: + is_reproducible = False + continue + # Fallback to a pip-installed wheel args = dict(whl_library_args) # make a copy args["requirement"] = requirement.srcs.requirement_line if requirement.extra_pip_args: @@ -343,13 +327,14 @@ def _create_whl_repos( target_platforms = target_platforms or None, )] = repo_name - if is_exposed: - exposed_packages[whl_name] = None - return struct( is_reproducible = is_reproducible, whl_map = whl_map, - exposed_packages = exposed_packages, + exposed_packages = { + whl_name: None + for whl_name, requirements in requirements_by_platform.items() + if len([r for r in requirements if r.is_exposed]) > 0 + }, extra_aliases = extra_aliases, whl_libraries = whl_libraries, ) diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index 821913d6de..d7ee285c0e 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -203,6 +203,9 @@ def parse_requirements( sorted(requirements), )) + # Return normalized names + ret_requirements = ret.setdefault(normalize_name(whl_name), []) + for r in sorted(reqs.values(), key = lambda r: r.requirement_line): whls, sdist = _add_dists( requirement = r, @@ -211,7 +214,7 @@ def parse_requirements( ) target_platforms = env_marker_target_platforms.get(r.requirement_line, r.target_platforms) - ret.setdefault(whl_name, []).append( + ret_requirements.append( struct( distribution = r.distribution, srcs = r.srcs,