feat(tracer): add Python 3.11 support

.circleci/config.yml

@@ -398,6 +398,10 @@ jobs:
     steps:
       - run_test:
           pattern: "tracer"
+      - run_tox_scenario:
+          # Riot venvs break with Py 3.11 importlib, specifically with hypothesis (test_http.py).
+          # We skip the test_http.py tests in riot and run the test_http.py tests through tox.
+          pattern: '^py.\+-tracer_test_http'
 
   telemetry:
     <<: *machine_executor
@@ -419,7 +423,7 @@ jobs:
     parallelism: 7
     steps:
       - run_tox_scenario:
-          pattern: '^py..-opentracer'
+          pattern: '^py.\+-opentracer'
 
   # Building gevent (for which we never want wheels because they crash)
   # on Python 2.7 requires Microsoft Visual C++ 9.0 which is not installed. :(

ddtrace/debugging/_debugger.py

@@ -152,6 +152,11 @@ class Debugger(Service):
     def enable(cls, run_module=False):
         # type: (bool) -> None
         """Enable the debugger (idempotent)."""
+        if sys.version_info >= (3, 11, 0):
+            raise RuntimeError(
+                "Dynamic Instrumentation is not yet compatible with Python 3.11. "
+                "See tracking issue for more details: https://github.com/DataDog/dd-trace-py/issues/4149"
+            )
         if cls._instance is not None:
             log.debug("%s already enabled", cls.__name__)
             return

ddtrace/internal/pack_template.h

@@ -558,7 +558,12 @@ static inline int msgpack_pack_double(msgpack_packer* x, double d)
 {
     unsigned char buf[9];
     buf[0] = 0xcb;
-    _PyFloat_Pack8(d, &buf[1], 0);
+    // Python 3.11 introduced PyFloat_Pack8() to the public C API and moved _PyFloat_Pack8() to the internal C API
+    #if PY_VERSION_HEX <= 0x030B0000
+        _PyFloat_Pack8(d, &buf[1], 0);
+    #else
+        PyFloat_Pack8(d, &buf[1], 0);
+    #endif
     msgpack_pack_append_buffer(x, buf, 9);
 }
 

ddtrace/profiling/collector/stack.pyx

@@ -165,10 +165,17 @@ IF UNAME_SYSNAME != "Windows" and PY_MAJOR_VERSION >= 3 and PY_MINOR_VERSION >=
 
         _PyErr_StackItem * _PyErr_GetTopmostException(PyThreadState *tstate)
 
-        ctypedef struct _PyErr_StackItem:
-            PyObject* exc_type
-            PyObject* exc_value
-            PyObject* exc_traceback
+        IF PY_MINOR_VERSION < 11:
+            ctypedef struct _PyErr_StackItem:
+                PyObject* exc_type
+                PyObject* exc_value
+                PyObject* exc_traceback
+        ELSE:
+            ctypedef struct _PyErr_StackItem:
+                PyObject* exc_value
+
+        PyObject* PyException_GetTraceback(PyObject* exc)
+        PyObject* Py_TYPE(PyObject* ob)
 
     IF PY_MINOR_VERSION == 7:
         # Python 3.7
@@ -198,6 +205,8 @@ IF UNAME_SYSNAME != "Windows" and PY_MAJOR_VERSION >= 3 and PY_MINOR_VERSION >=
             # Needed for accessing _PyGC_FINALIZED when we build with -DPy_BUILD_CORE
             cdef extern from "<internal/pycore_gc.h>":
                 pass
+            cdef extern from "<Python.h>":
+                PyObject* PyThreadState_GetFrame(PyThreadState* tstate)
 ELSE:
     from cpython.ref cimport Py_DECREF
 
@@ -214,7 +223,8 @@ cdef collect_threads(thread_id_ignore_list, thread_time, thread_span_links) with
         cdef PyThreadState* tstate
         cdef _PyErr_StackItem* exc_info
         cdef PyThread_type_lock lmutex = _PyRuntime.interpreters.mutex
-
+        cdef PyObject* exc_type
+        cdef PyObject* exc_tb
         cdef dict running_threads = {}
 
         # This is an internal lock but we do need it.
@@ -230,13 +240,22 @@ cdef collect_threads(thread_id_ignore_list, thread_time, thread_span_links) with
                     tstate = PyInterpreterState_ThreadHead(interp)
                     while tstate:
                         # The frame can be NULL
-                        if tstate.frame:
-                            running_threads[tstate.thread_id] = <object>tstate.frame
-
+                        # Python 3.9 added helper function to public C API to access PyFrameObject from tstate,
+                        # Python 3.11 moved PyFrameObject to internal C API and cannot be directly accessed from tstate
+                        IF PY_MINOR_VERSION >= 9:
+                            frame = PyThreadState_GetFrame(tstate)
+                        ELSE:
+                            frame = tstate.frame
+                        if frame:
+                            running_threads[tstate.thread_id] = <object>frame
                         exc_info = _PyErr_GetTopmostException(tstate)
-                        if exc_info and exc_info.exc_type and exc_info.exc_traceback:
-                            current_exceptions[tstate.thread_id] = (<object>exc_info.exc_type, <object>exc_info.exc_traceback)
-
+                        if exc_info and exc_info.exc_value:
+                            # Python 3.11 removed exc_type, exc_traceback from exception representations, can instead
+                            # derive exc_type and exc_traceback from remaining exc_value field
+                            exc_type = Py_TYPE(exc_info.exc_value)
+                            exc_tb = PyException_GetTraceback(exc_info.exc_value)
+                            if exc_type and exc_tb:
+                                current_exceptions[tstate.thread_id] = (<object>exc_type, <object>exc_tb)
                         tstate = PyThreadState_Next(tstate)
 
                     interp = PyInterpreterState_Next(interp)

ddtrace/profiling/profiler.py

@@ -1,6 +1,7 @@
 # -*- encoding: utf-8 -*-
 import logging
 import os
+import sys
 import typing
 from typing import List
 from typing import Optional
@@ -57,6 +58,11 @@ def start(self, stop_on_exit=True, profile_children=True):
         :param profile_children: Whether to start a profiler in child processes.
         """
 
+        if sys.version_info >= (3, 11, 0):
+            raise RuntimeError(
+                "Profiling is not yet compatible with Python 3.11. "
+                "See tracking issue for more details: https://github.com/DataDog/dd-trace-py/issues/4149"
+            )
         if profile_children:
             try:
                 uwsgi.check_uwsgi(self._restart_on_fork, atexit=self.stop if stop_on_exit else None)

docs/versioning.rst

@@ -108,15 +108,15 @@ Supported runtimes
    * - Linux
      - x86-64, i686, AArch64
      - CPython
-     - 2.7, 3.5-3.10
+     - 2.7, 3.5-3.11
      - ``<2``
    * - MacOS
      - Intel, Apple Silicon
      - CPython
-     - 2.7, 3.5-3.10
+     - 2.7, 3.5-3.11
      - ``<2``
    * - Windows
      - 64bit, 32bit
      - CPython
-     - 2.7, 3.5-3.10
+     - 2.7, 3.5-3.11
      - ``<2``

releasenotes/notes/python311-tracer-a7077c0e461622d2.yaml

@@ -0,0 +1,10 @@
+---
+prelude: >
+    Initial library support has been added for Python 3.11. Continuous Profiler and Dynamic Instrumentation are not yet compatible and must be disabled in order to use the library with Python 3.11. Support for Continuous Profiler and Dynamic Instrumentation will be added in a future release.
+    For full Python 3.11 support status please see: https://github.com/DataDog/dd-trace-py/issues/4149
+features:
+  - |
+    tracer: added support for Python 3.11.
+upgrade:
+  - |
+    Python 3.11: Continuous Profiler and Dynamic Instrumentation must be disabled as they do not current support Python 3.11.