Migrate python debugging logic to rules

.plzconfig

@@ -1,5 +1,5 @@
 [Please]
-Version = >=16.10.1
+Version = >=16.17.0
 
 [Parse]
 PreloadBuildDefs = test/build_defs/test.build_defs
@@ -31,6 +31,10 @@ DefaultValue = unittest
 ConfigKey = TestRunnerBootstrap
 Optional = true
 
+[PluginConfig "debugger"]
+ConfigKey = Debugger
+DefaultValue = pdb
+
 [PluginConfig "module_dir"]
 ConfigKey = ModuleDir
 DefaultValue = third_party.python

README.md

@@ -66,6 +66,7 @@ InterpreterOptions = -b -s
 DefaultInterpreter = python3
 PexTool = @self//tools/please_pex
 TestRunner = unittest
+Debugger = pdb
 ModuleDir = third_party.python
 WheelRepo = https://pypi.org/pypi
 WheelNameScheme = None

build_defs/python.build_defs

@@ -136,6 +136,11 @@ def python_binary(name:str, main:str, srcs:list=[], resources:list=[], out:str=N
     if site:
         cmd += ' -S'
 
+    if CONFIG.BUILD_CONFIG == 'dbg':
+        # Both `pdb` and `debugpy` require the pex to be exploded to get debugging to work.
+        cmd = (f'{cmd} -d=' + CONFIG.PYTHON.DEBUGGER).replace(' --zip_safe', '')
+
+
     assert main not in srcs, "main file should not be included in srcs"
 
     lib_rule = python_library(
@@ -170,13 +175,17 @@ def python_binary(name:str, main:str, srcs:list=[], resources:list=[], out:str=N
     cmd = '$TOOL z -i . -s .pex.zip -s .whl --preamble_from="$SRC" --include_other --add_init_py --strict'
     if strip:
         cmd += ' --strip_py'
+
+    debug_cmd = _debug_cmd("./$OUT")
+
     return build_rule(
         name=name,
         srcs=[pex_rule],
         deps=[lib_rule],
         outs=[out or (name + '.pex')],
         data=data,
         cmd=cmd,
+        debug_cmd=debug_cmd,
         needs_transitive_deps=True,
         binary=True,
         output_is_complete=True,
@@ -245,6 +254,10 @@ def python_test(name:str, srcs:list, data:list|dict=[], resources:list=[], deps:
     if site:
         cmd += ' -S'
 
+    if CONFIG.BUILD_CONFIG == 'dbg':
+        # Both `pdb` and `debugpy` require the pex to be exploded to get debugging to work.
+        cmd = (f'{cmd} -d=' + CONFIG.PYTHON.DEBUGGER).replace(' --zip_safe', '')
+
     # If the config specifies a PYTHON_TEST_RUNNER_BOOTSTRAP target, then this ought to contain
     # the test-runner library (e.g. pytest), and its transitive dependencies. In this case, we just
     # add the specified target as a dependency.
@@ -292,9 +305,17 @@ def python_test(name:str, srcs:list, data:list|dict=[], resources:list=[], deps:
     deps = [pex_rule, lib_rule]
 
     test_cmd = f'$TEST {flags}'
-    if worker:
-        test_cmd = f'$(worker {worker}) && {test_cmd} '
+    worker_cmd = f'$(worker {worker})' if worker else ""
+    if worker_cmd:
+        test_cmd = f'{worker_cmd} && {test_cmd} '
         deps += [worker]
+
+    debug_cmd = _debug_cmd(
+        bin = "./$TEST",
+        flags = flags,
+        pre_cmd = worker_cmd,
+    )
+
     # This rule concatenates the .pex with all the other precompiled zip files from dependent rules.
     return build_rule(
         name=name,
@@ -307,6 +328,7 @@ def python_test(name:str, srcs:list, data:list|dict=[], resources:list=[], deps:
         labels=labels + ['test_results_dir'],
         cmd='$TOOL z -i . -s .pex.zip -s .whl --preamble_from="$SRC" --include_other --add_init_py --strict',
         test_cmd=test_cmd,
+        debug_cmd=debug_cmd,
         needs_transitive_deps=True,
         output_is_complete=True,
         binary=True,
@@ -647,6 +669,22 @@ def _add_licences(name, output):
     if not found:
         log.warning('No licence found for %s, should add licences = [...] to the rule', name.lstrip('_').split('#')[0])
 
+
+# `pdb` and `debugpy` are the current debuggers supported by the PEX tool.
+def _debug_cmd(bin:str, flags:str='', pre_cmd:str=''):
+    if CONFIG.BUILD_CONFIG != 'dbg':
+        return ""
+
+    # `PEX_EXPLODE` will explode the PEX before executing to allow clients to access third party sources files.
+    # `DEBUG_PORT` is read by the PEX generated for debuggers that support server mode.
+    cmd = f"{bin} {flags}" if not CONFIG.DEBUG_PORT else f"PEX_EXPLODE=true DEBUG_PORT={CONFIG.DEBUG_PORT} {bin} {flags}"
+
+    if pre_cmd:
+        cmd = f"{pre_cmd} && {cmd}"
+
+    return cmd
+
+
 if CONFIG.BAZEL_COMPATIBILITY:
     py_library = python_library
     py_binary = python_binary

third_party/python/BUILD

@@ -129,7 +129,7 @@ python_wheel(
 python_wheel(
     name = "protobuf",
     outs = ["google"],
-    version = "3.18.1",
+    version = "3.19.3",
     deps = [":six"],
 )
 

tools/wheel_resolver/test/BUILD

@@ -25,7 +25,7 @@ gentest(
 python_wheel(
     name = "protobuf",
     outs = ["google"],
-    version = "3.18.1",
+    version = "3.19.3",
     deps = [
         "//third_party/python:six",
     ],