From b49958a170068bb83bca09b7cc94eb8471b7272d Mon Sep 17 00:00:00 2001 From: Will Childs-Klein Date: Wed, 31 Jan 2024 18:10:08 -0500 Subject: [PATCH] Add cpython integration test (#1359) --- .github/workflows/integrations.yml | 12 +- .gitignore | 1 + .../python_patch/3.10/aws-lc-cpython.patch | 548 ++++++++++++++++++ .../python_patch/3.11/aws-lc-cpython.patch | 542 +++++++++++++++++ .../python_patch/3.12/aws-lc-cpython.patch | 543 +++++++++++++++++ .../python_patch/main/aws-lc-cpython.patch | 543 +++++++++++++++++ .../ci/integration/run_python_integration.sh | 139 +++++ 7 files changed, 2327 insertions(+), 1 deletion(-) create mode 100644 tests/ci/integration/python_patch/3.10/aws-lc-cpython.patch create mode 100644 tests/ci/integration/python_patch/3.11/aws-lc-cpython.patch create mode 100644 tests/ci/integration/python_patch/3.12/aws-lc-cpython.patch create mode 100644 tests/ci/integration/python_patch/main/aws-lc-cpython.patch create mode 100755 tests/ci/integration/run_python_integration.sh diff --git a/.github/workflows/integrations.yml b/.github/workflows/integrations.yml index ceb388a25b..975fa33423 100644 --- a/.github/workflows/integrations.yml +++ b/.github/workflows/integrations.yml @@ -91,4 +91,14 @@ jobs: - name: Run integration build run: | ./tests/ci/integration/run_socat_integration.sh - + python: + runs-on: ubuntu-latest + steps: + - name: Install OS Dependencies + run: | + sudo apt-get update + sudo apt-get -y --no-install-recommends install cmake gcc ninja-build golang make + - uses: actions/checkout@v3 + - name: Build AWS-LC, build python, run tests + run: | + ./tests/ci/integration/run_python_integration.sh diff --git a/.gitignore b/.gitignore index 0e578f4c0f..941a5e2413 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ build/ build32/ build64/ build-fips/ +*_BUILD_ROOT/ ssl/test/runner/runner *.pyc *.swp diff --git a/tests/ci/integration/python_patch/3.10/aws-lc-cpython.patch b/tests/ci/integration/python_patch/3.10/aws-lc-cpython.patch new file mode 100644 index 0000000000..481c8143b8 --- /dev/null +++ b/tests/ci/integration/python_patch/3.10/aws-lc-cpython.patch @@ -0,0 +1,548 @@ +diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py +index 253a6c119c..2d0d10642d 100644 +--- a/Lib/test/test_asyncio/test_events.py ++++ b/Lib/test/test_asyncio/test_events.py +@@ -1106,12 +1106,12 @@ def test_create_server_ssl_match_failed(self): + # incorrect server_hostname + f_c = self.loop.create_connection(MyProto, host, port, + ssl=sslcontext_client) ++ regex = "IP address mismatch, certificate is not valid for '127.0.0.1'" ++ if ssl is not None and "AWS-LC" in ssl.OPENSSL_VERSION: ++ regex = "CERTIFICATE_VERIFY_FAILED" + with mock.patch.object(self.loop, 'call_exception_handler'): + with test_utils.disable_logger(): +- with self.assertRaisesRegex( +- ssl.CertificateError, +- "IP address mismatch, certificate is not valid for " +- "'127.0.0.1'"): ++ with self.assertRaisesRegex(ssl.CertificateError, regex): + self.loop.run_until_complete(f_c) + + # close connection +diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py +index 77152cf645..be3d11b993 100644 +--- a/Lib/test/test_httplib.py ++++ b/Lib/test/test_httplib.py +@@ -1863,7 +1863,7 @@ def test_host_port(self): + + def test_tls13_pha(self): + import ssl +- if not ssl.HAS_TLSv1_3: ++ if not ssl.HAS_TLSv1_3 or "AWS-LC" in ssl.OPENSSL_VERSION: + self.skipTest('TLS 1.3 support required') + # just check status of PHA flag + h = client.HTTPSConnection('localhost', 443) +diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py +index b5c78a5d49..41235c17fc 100644 +--- a/Lib/test/test_imaplib.py ++++ b/Lib/test/test_imaplib.py +@@ -555,9 +555,10 @@ def test_ssl_raises(self): + self.assertEqual(ssl_context.check_hostname, True) + ssl_context.load_verify_locations(CAFILE) + +- with self.assertRaisesRegex(ssl.CertificateError, +- "IP address mismatch, certificate is not valid for " +- "'127.0.0.1'"): ++ regex = "IP address mismatch, certificate is not valid for '127.0.0.1'" ++ if ssl is not None and "AWS-LC" in ssl.OPENSSL_VERSION: ++ regex = "CERTIFICATE_VERIFY_FAILED" ++ with self.assertRaisesRegex(ssl.CertificateError, regex): + _, server = self._setup(SimpleIMAPHandler) + client = self.imap_class(*server.server_address, + ssl_context=ssl_context) +@@ -960,10 +961,10 @@ def test_ssl_verified(self): + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ssl_context.load_verify_locations(CAFILE) + +- with self.assertRaisesRegex( +- ssl.CertificateError, +- "IP address mismatch, certificate is not valid for " +- "'127.0.0.1'"): ++ regex = "IP address mismatch, certificate is not valid for '127.0.0.1'" ++ if ssl is not None and "AWS-LC" in ssl.OPENSSL_VERSION: ++ regex = "CERTIFICATE_VERIFY_FAILED" ++ with self.assertRaisesRegex(ssl.CertificateError, regex): + with self.reaped_server(SimpleIMAPHandler) as server: + client = self.imap_class(*server.server_address, + ssl_context=ssl_context) +diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py +index a1a581a907..c69e71159a 100644 +--- a/Lib/test/test_ssl.py ++++ b/Lib/test/test_ssl.py +@@ -44,6 +44,7 @@ + + Py_DEBUG = hasattr(sys, 'gettotalrefcount') + Py_DEBUG_WIN32 = Py_DEBUG and sys.platform == 'win32' ++Py_OPENSSL_IS_AWSLC = "AWS-LC" in ssl.OPENSSL_VERSION + + PROTOCOLS = sorted(ssl._PROTOCOL_NAMES) + HOST = socket_helper.HOST +@@ -170,7 +171,7 @@ def is_ubuntu(): + except FileNotFoundError: + return False + +-if is_ubuntu(): ++if is_ubuntu() and not Py_OPENSSL_IS_AWSLC: + def seclevel_workaround(*ctxs): + """"Lower security level to '1' and allow all ciphers for TLS 1.0/1""" + for ctx in ctxs: +@@ -547,7 +548,7 @@ def test_openssl_version(self): + else: + openssl_ver = f"OpenSSL {major:d}.{minor:d}.{fix:d}" + self.assertTrue( +- s.startswith((openssl_ver, libressl_ver)), ++ s.startswith((openssl_ver, libressl_ver, "AWS-LC")), + (s, t, hex(n)) + ) + +@@ -1318,7 +1319,6 @@ def test_min_max_version(self): + + if has_tls_protocol(ssl.PROTOCOL_TLSv1_1): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_1) +- + self.assertIn( + ctx.minimum_version, minimum_range + ) +@@ -1378,24 +1378,25 @@ def test_load_cert_chain(self): + with self.assertRaises(OSError) as cm: + ctx.load_cert_chain(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) +- with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): ++ with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(BADCERT) +- with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): ++ with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(EMPTYCERT) + # Separate key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_cert_chain(ONLYCERT, ONLYKEY) + ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) + ctx.load_cert_chain(certfile=BYTES_ONLYCERT, keyfile=BYTES_ONLYKEY) +- with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): ++ with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(ONLYCERT) +- with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): ++ with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(ONLYKEY) +- with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): ++ with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(certfile=ONLYKEY, keyfile=ONLYCERT) + # Mismatching key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) +- with self.assertRaisesRegex(ssl.SSLError, "key values mismatch"): ++ with self.assertRaisesRegex(ssl.SSLError, ++ "(key values mismatch|KEY_VALUES_MISMATCH)"): + ctx.load_cert_chain(CAFILE_CACERT, ONLYKEY) + # Password protected key and cert + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD) +@@ -1463,7 +1464,7 @@ def test_load_verify_locations(self): + with self.assertRaises(OSError) as cm: + ctx.load_verify_locations(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) +- with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): ++ with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_verify_locations(BADCERT) + ctx.load_verify_locations(CERTFILE, CAPATH) + ctx.load_verify_locations(CERTFILE, capath=BYTES_CAPATH) +@@ -1861,10 +1862,11 @@ def test_lib_reason(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + with self.assertRaises(ssl.SSLError) as cm: + ctx.load_dh_params(CERTFILE) +- self.assertEqual(cm.exception.library, 'PEM') +- self.assertEqual(cm.exception.reason, 'NO_START_LINE') ++ if not Py_OPENSSL_IS_AWSLC: ++ self.assertEqual(cm.exception.library, 'PEM') ++ self.assertEqual(cm.exception.reason, 'NO_START_LINE') + s = str(cm.exception) +- self.assertTrue(s.startswith("[PEM: NO_START_LINE] no start line"), s) ++ self.assertTrue("NO_START_LINE" in s, s) + + def test_subclass(self): + # Check that the appropriate SSLError subclass is raised +@@ -2041,7 +2043,8 @@ def test_connect_fail(self): + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED) + self.addCleanup(s.close) +- self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", ++ msg_re = "(certificate verify failed|CERTIFICATE_VERIFY_FAILED)" ++ self.assertRaisesRegex(ssl.SSLError, msg_re, + s.connect, self.server_addr) + + def test_connect_ex(self): +@@ -2109,7 +2112,8 @@ def test_connect_with_context_fail(self): + server_hostname=SIGNED_CERTFILE_HOSTNAME + ) + self.addCleanup(s.close) +- self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", ++ msg_re = "(certificate verify failed|CERTIFICATE_VERIFY_FAILED)" ++ self.assertRaisesRegex(ssl.SSLError, msg_re, + s.connect, self.server_addr) + + def test_connect_capath(self): +@@ -2329,14 +2333,14 @@ def test_bio_handshake(self): + self.assertIsNone(sslobj.version()) + self.assertIsNone(sslobj.shared_ciphers()) + self.assertRaises(ValueError, sslobj.getpeercert) +- if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: ++ if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES and sslobj.version() != "TLSv1.3": + self.assertIsNone(sslobj.get_channel_binding('tls-unique')) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + self.assertTrue(sslobj.cipher()) + self.assertIsNone(sslobj.shared_ciphers()) + self.assertIsNotNone(sslobj.version()) + self.assertTrue(sslobj.getpeercert()) +- if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: ++ if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES and sslobj.version() != "TLSv1.3": + self.assertTrue(sslobj.get_channel_binding('tls-unique')) + try: + self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) +@@ -3058,11 +3062,11 @@ def test_crl_check(self): + client_context.verify_flags |= ssl.VERIFY_CRL_CHECK_LEAF + + server = ThreadedEchoServer(context=server_context, chatty=True) ++ msg_re = "(certificate verify failed|CERTIFICATE_VERIFY_FAILED)" + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: +- with self.assertRaisesRegex(ssl.SSLError, +- "certificate verify failed"): ++ with self.assertRaisesRegex(ssl.SSLError, msg_re): + s.connect((HOST, server.port)) + + # now load a CRL file. The CRL file is signed by the CA. +@@ -3093,12 +3097,12 @@ def test_check_hostname(self): + + # incorrect hostname should raise an exception + server = ThreadedEchoServer(context=server_context, chatty=True) ++ err_re = "(CERTIFICATE_VERIFY_FAILED|" ++ err_re += "Hostname mismatch, certificate is not valid for 'invalid'.)" + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname="invalid") as s: +- with self.assertRaisesRegex( +- ssl.CertificateError, +- "Hostname mismatch, certificate is not valid for 'invalid'."): ++ with self.assertRaisesRegex(ssl.CertificateError, err_re): + s.connect((HOST, server.port)) + + # missing server_hostname arg should cause an exception, too +@@ -3368,8 +3372,13 @@ def test_ssl_cert_verify_error(self): + self.assertIsInstance(e, ssl.SSLCertVerificationError) + self.assertEqual(e.verify_code, 20) + self.assertEqual(e.verify_message, msg) ++ if Py_OPENSSL_IS_AWSLC: ++ msg = "CERTIFICATE_VERIFY_FAILED" + self.assertIn(msg, repr(e)) +- self.assertIn('certificate verify failed', repr(e)) ++ expected_err = 'certificate verify failed' ++ if Py_OPENSSL_IS_AWSLC: ++ expected_err = "CERTIFICATE_VERIFY_FAILED" ++ self.assertIn(expected_err, repr(e)) + + @requires_tls_version('SSLv2') + def test_protocol_sslv2(self): +@@ -3916,7 +3925,10 @@ def test_no_shared_ciphers(self): + server_hostname=hostname) as s: + with self.assertRaises(OSError): + s.connect((HOST, server.port)) +- self.assertIn("no shared cipher", server.conn_errors[0]) ++ expected_err = "no shared cipher" ++ if Py_OPENSSL_IS_AWSLC: ++ expected_err = "NO_SHARED_CIPHER" ++ self.assertIn(expected_err, server.conn_errors[0]) + + def test_version_basic(self): + """ +@@ -4004,7 +4016,10 @@ def test_min_max_version_mismatch(self): + server_hostname=hostname) as s: + with self.assertRaises(ssl.SSLError) as e: + s.connect((HOST, server.port)) +- self.assertIn("alert", str(e.exception)) ++ self.assertTrue( ++ "alert"in str(e.exception) ++ or "ALERT"in str(e.exception) ++ ) + + @requires_tls_version('SSLv3') + def test_min_max_version_sslv3(self): +@@ -4046,6 +4061,9 @@ def test_tls_unique_channel_binding(self): + + client_context, server_context, hostname = testing_context() + ++ # tls-unique isn't defined as of TLSv1.3 ++ client_context.maximum_version = ssl.TLSVersion.TLSv1_2 ++ + server = ThreadedEchoServer(context=server_context, + chatty=True, + connectionchatty=False) +@@ -4118,6 +4136,7 @@ def test_compression_disabled(self): + self.assertIs(stats['compression'], None) + + @unittest.skipIf(Py_DEBUG_WIN32, "Avoid mixing debug/release CRT on Windows") ++ @unittest.skipIf(Py_OPENSSL_IS_AWSLC, "AWS-LC doesn't support (FF)DHE") + def test_dh_params(self): + # Check we can get a connection with ephemeral Diffie-Hellman + client_context, server_context, hostname = testing_context() +@@ -4132,7 +4151,7 @@ def test_dh_params(self): + cipher = stats["cipher"][0] + parts = cipher.split("-") + if "ADH" not in parts and "EDH" not in parts and "DHE" not in parts: +- self.fail("Non-DH cipher: " + cipher[0]) ++ self.fail("Non-DH kx: " + parts[0]) + + def test_ecdh_curve(self): + # server secp384r1, client auto +@@ -4299,8 +4318,10 @@ def cb_raising(ssl_sock, server_name, initial_context): + chatty=False, + sni_name='supermessage') + +- self.assertEqual(cm.exception.reason, +- 'SSLV3_ALERT_HANDSHAKE_FAILURE') ++ expected_reason = 'SSLV3_ALERT_HANDSHAKE_FAILURE' ++ if Py_OPENSSL_IS_AWSLC: ++ expected_reason = 'NO_PRIVATE_VALUE' ++ self.assertEqual(cm.exception.reason, expected_reason) + self.assertEqual(catch.unraisable.exc_type, ZeroDivisionError) + + def test_sni_callback_wrong_return_type(self): +@@ -4476,7 +4497,10 @@ def test_session_handling(self): + 'Session refers to a different SSLContext.') + + +-@unittest.skipUnless(has_tls_version('TLSv1_3'), "Test needs TLS 1.3") ++@unittest.skipUnless( ++ has_tls_version('TLSv1_3') and not Py_OPENSSL_IS_AWSLC, ++ "Test needs TLS 1.3; AWS-LC doesn't support PHA" ++) + class TestPostHandshakeAuth(unittest.TestCase): + def test_pha_setter(self): + protocols = [ +@@ -4752,6 +4776,31 @@ def test_internal_chain_server(self): + self.assertEqual(res, b'\x02\n') + + ++@unittest.skipUnless(Py_OPENSSL_IS_AWSLC, "Only test this against AWS-LC") ++class TestPostHandshakeAuthAwsLc(unittest.TestCase): ++ def test_pha(self): ++ protocols = [ ++ ssl.PROTOCOL_TLS_SERVER, ssl.PROTOCOL_TLS_CLIENT ++ ] ++ for protocol in protocols: ++ client_ctx, server_ctx, hostname = testing_context() ++ client_ctx.load_cert_chain(SIGNED_CERTFILE) ++ self.assertEqual(client_ctx.post_handshake_auth, None) ++ with self.assertRaises(AttributeError): ++ client_ctx.post_handshake_auth = True ++ with self.assertRaises(AttributeError): ++ server_ctx.post_handshake_auth = True ++ ++ with ThreadedEchoServer(context=server_ctx) as server: ++ with client_ctx.wrap_socket( ++ socket.socket(), ++ server_hostname=hostname ++ ) as ssock: ++ ssock.connect((HOST, server.port)) ++ with self.assertRaises(NotImplementedError): ++ ssock.verify_client_post_handshake() ++ ++ + HAS_KEYLOG = hasattr(ssl.SSLContext, 'keylog_filename') + requires_keylog = unittest.skipUnless( + HAS_KEYLOG, 'test requires OpenSSL 1.1.1 with keylog callback') +diff --git a/Modules/Setup b/Modules/Setup +index 87c6a152f8..3a9bc54bab 100644 +--- a/Modules/Setup ++++ b/Modules/Setup +@@ -208,8 +208,8 @@ _symtable symtablemodule.c + + # Socket module helper for SSL support; you must comment out the other + # socket line above, and edit the OPENSSL variable: +-# OPENSSL=/path/to/openssl/directory +-# _ssl _ssl.c \ ++OPENSSL=AWS_LC_INSTALL_PLACEHOLDER ++#_ssl _ssl.c \ + # -I$(OPENSSL)/include -L$(OPENSSL)/lib \ + # -lssl -lcrypto + #_hashlib _hashopenssl.c \ +@@ -217,13 +217,13 @@ _symtable symtablemodule.c + # -lcrypto + + # To statically link OpenSSL: +-# _ssl _ssl.c \ +-# -I$(OPENSSL)/include -L$(OPENSSL)/lib \ +-# -l:libssl.a -Wl,--exclude-libs,libssl.a \ +-# -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a +-#_hashlib _hashopenssl.c \ +-# -I$(OPENSSL)/include -L$(OPENSSL)/lib \ +-# -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a ++_ssl _ssl.c \ ++ -I$(OPENSSL)/include -L$(OPENSSL)/lib \ ++ -l:libssl.a -Wl,--exclude-libs,libssl.a \ ++ -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a ++_hashlib _hashopenssl.c \ ++ -I$(OPENSSL)/include -L$(OPENSSL)/lib \ ++ -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a + + # The crypt module is now disabled by default because it breaks builds + # on many systems (where -lcrypt is needed), e.g. Linux (I believe). +diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c +index 35addf49e9..77a12c6af5 100644 +--- a/Modules/_hashopenssl.c ++++ b/Modules/_hashopenssl.c +@@ -131,8 +131,12 @@ static const py_hashentry_t py_hashes[] = { + PY_HASH_ENTRY(Py_hash_shake_128, NULL, SN_shake128, NID_shake128), + PY_HASH_ENTRY(Py_hash_shake_256, NULL, SN_shake256, NID_shake256), + /* blake2 digest */ ++#if defined(NID_blake2s256) + PY_HASH_ENTRY(Py_hash_blake2s, "blake2s256", SN_blake2s256, NID_blake2s256), ++#endif ++#if defined(NID_blake2b512) + PY_HASH_ENTRY(Py_hash_blake2b, "blake2b512", SN_blake2b512, NID_blake2b512), ++#endif + PY_HASH_ENTRY(NULL, NULL, NULL, 0), + }; + +diff --git a/Modules/_ssl.c b/Modules/_ssl.c +index 7a28f2d37f..07740af98b 100644 +--- a/Modules/_ssl.c ++++ b/Modules/_ssl.c +@@ -181,6 +181,12 @@ extern const SSL_METHOD *TLSv1_2_method(void); + #endif + + ++ ++#if defined(OPENSSL_IS_AWSLC) || !defined(TLS1_3_VERSION) || defined(OPENSSL_NO_TLS1_3) ++ #define PY_SSL_NO_POST_HS_AUTH ++#endif ++ ++ + enum py_ssl_error { + /* these mirror ssl.h */ + PY_SSL_ERROR_NONE, +@@ -225,7 +231,7 @@ enum py_proto_version { + PY_PROTO_TLSv1 = TLS1_VERSION, + PY_PROTO_TLSv1_1 = TLS1_1_VERSION, + PY_PROTO_TLSv1_2 = TLS1_2_VERSION, +-#ifdef TLS1_3_VERSION ++#if defined(TLS1_3_VERSION) + PY_PROTO_TLSv1_3 = TLS1_3_VERSION, + #else + PY_PROTO_TLSv1_3 = 0x304, +@@ -287,7 +293,7 @@ typedef struct { + */ + unsigned int hostflags; + int protocol; +-#ifdef TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + int post_handshake_auth; + #endif + PyObject *msg_cb; +@@ -859,7 +865,7 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, + SSL_set_mode(self->ssl, + SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER | SSL_MODE_AUTO_RETRY); + +-#ifdef TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + if (sslctx->post_handshake_auth == 1) { + if (socket_type == PY_SSL_SERVER) { + /* bpo-37428: OpenSSL does not ignore SSL_VERIFY_POST_HANDSHAKE. +@@ -1003,6 +1009,7 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) + } while (err.ssl == SSL_ERROR_WANT_READ || + err.ssl == SSL_ERROR_WANT_WRITE); + Py_XDECREF(sock); ++ + if (ret < 1) + return PySSL_SetError(self, ret, __FILE__, __LINE__); + if (PySSL_ChainExceptions(self) < 0) +@@ -2042,7 +2049,12 @@ _ssl__SSLSocket_shared_ciphers_impl(PySSLSocket *self) + len = 0; + for (i = 0; i < sk_SSL_CIPHER_num(server_ciphers); i++) { + cipher = sk_SSL_CIPHER_value(server_ciphers, i); ++#if defined(OPENSSL_IS_AWSLC) ++ size_t unused_idx; ++ if (sk_SSL_CIPHER_find(client_ciphers, &unused_idx, cipher) == 0) ++#else + if (sk_SSL_CIPHER_find(client_ciphers, cipher) < 0) ++#endif + continue; + + PyObject *tup = cipher_to_tuple(cipher); +@@ -2771,7 +2783,7 @@ static PyObject * + _ssl__SSLSocket_verify_client_post_handshake_impl(PySSLSocket *self) + /*[clinic end generated code: output=532147f3b1341425 input=6bfa874810a3d889]*/ + { +-#ifdef TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + int err = SSL_verify_client_post_handshake(self->ssl); + if (err == 0) + return _setSSLError(get_state_sock(self), NULL, 0, __FILE__, __LINE__); +@@ -3186,7 +3198,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) + + /* Set SSL_MODE_RELEASE_BUFFERS. This potentially greatly reduces memory + usage for no cost at all. */ +- SSL_CTX_set_mode(self->ctx, SSL_MODE_RELEASE_BUFFERS); ++ SSL_CTX_set_mode(self->ctx, SSL_MODE_RELEASE_BUFFERS | SSL_MODE_AUTO_RETRY); + + #define SID_CTX "Python" + SSL_CTX_set_session_id_context(self->ctx, (const unsigned char *) SID_CTX, +@@ -3199,7 +3211,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) + X509_VERIFY_PARAM_set_flags(params, X509_V_FLAG_TRUSTED_FIRST); + X509_VERIFY_PARAM_set_hostflags(params, self->hostflags); + +-#ifdef TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + self->post_handshake_auth = 0; + SSL_CTX_set_post_handshake_auth(self->ctx, self->post_handshake_auth); + #endif +@@ -3573,7 +3585,7 @@ set_maximum_version(PySSLContext *self, PyObject *arg, void *c) + return set_min_max_proto_version(self, arg, 1); + } + +-#ifdef TLS1_3_VERSION ++#if defined(TLS1_3_VERSION) && !defined(OPENSSL_NO_TLS1_3) + static PyObject * + get_num_tickets(PySSLContext *self, void *c) + { +@@ -3604,7 +3616,7 @@ set_num_tickets(PySSLContext *self, PyObject *arg, void *c) + + PyDoc_STRVAR(PySSLContext_num_tickets_doc, + "Control the number of TLSv1.3 session tickets"); +-#endif /* TLS1_3_VERSION */ ++#endif /* defined(TLS1_3_VERSION) */ + + static PyObject * + get_security_level(PySSLContext *self, void *c) +@@ -3694,14 +3706,14 @@ set_check_hostname(PySSLContext *self, PyObject *arg, void *c) + + static PyObject * + get_post_handshake_auth(PySSLContext *self, void *c) { +-#if TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + return PyBool_FromLong(self->post_handshake_auth); + #else + Py_RETURN_NONE; + #endif + } + +-#if TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + static int + set_post_handshake_auth(PySSLContext *self, PyObject *arg, void *c) { + if (arg == NULL) { +@@ -4651,14 +4663,14 @@ static PyGetSetDef context_getsetlist[] = { + (setter) _PySSLContext_set_msg_callback, NULL}, + {"sni_callback", (getter) get_sni_callback, + (setter) set_sni_callback, PySSLContext_sni_callback_doc}, +-#ifdef TLS1_3_VERSION ++#if defined(TLS1_3_VERSION) && !defined(OPENSSL_NO_TLS1_3) + {"num_tickets", (getter) get_num_tickets, + (setter) set_num_tickets, PySSLContext_num_tickets_doc}, + #endif + {"options", (getter) get_options, + (setter) set_options, NULL}, + {"post_handshake_auth", (getter) get_post_handshake_auth, +-#ifdef TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + (setter) set_post_handshake_auth, + #else + NULL, diff --git a/tests/ci/integration/python_patch/3.11/aws-lc-cpython.patch b/tests/ci/integration/python_patch/3.11/aws-lc-cpython.patch new file mode 100644 index 0000000000..a6b0a0cc09 --- /dev/null +++ b/tests/ci/integration/python_patch/3.11/aws-lc-cpython.patch @@ -0,0 +1,542 @@ +diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py +index d7871d3e53..fb65ee3b8b 100644 +--- a/Lib/test/test_asyncio/test_events.py ++++ b/Lib/test/test_asyncio/test_events.py +@@ -1103,12 +1103,12 @@ def test_create_server_ssl_match_failed(self): + # incorrect server_hostname + f_c = self.loop.create_connection(MyProto, host, port, + ssl=sslcontext_client) ++ regex = "IP address mismatch, certificate is not valid for '127.0.0.1'" ++ if ssl is not None and "AWS-LC" in ssl.OPENSSL_VERSION: ++ regex = "CERTIFICATE_VERIFY_FAILED" + with mock.patch.object(self.loop, 'call_exception_handler'): + with test_utils.disable_logger(): +- with self.assertRaisesRegex( +- ssl.CertificateError, +- "IP address mismatch, certificate is not valid for " +- "'127.0.0.1'"): ++ with self.assertRaisesRegex(ssl.CertificateError, regex): + self.loop.run_until_complete(f_c) + + # close connection +diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py +index 015a3d1e87..ec565ad181 100644 +--- a/Lib/test/test_httplib.py ++++ b/Lib/test/test_httplib.py +@@ -2040,7 +2040,7 @@ def test_host_port(self): + + def test_tls13_pha(self): + import ssl +- if not ssl.HAS_TLSv1_3: ++ if not ssl.HAS_TLSv1_3 or "AWS-LC" in ssl.OPENSSL_VERSION: + self.skipTest('TLS 1.3 support required') + # just check status of PHA flag + h = client.HTTPSConnection('localhost', 443) +diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py +index bd0fc9c2da..3ab70f5c1f 100644 +--- a/Lib/test/test_imaplib.py ++++ b/Lib/test/test_imaplib.py +@@ -561,9 +561,10 @@ def test_ssl_raises(self): + self.assertEqual(ssl_context.check_hostname, True) + ssl_context.load_verify_locations(CAFILE) + +- with self.assertRaisesRegex(ssl.CertificateError, +- "IP address mismatch, certificate is not valid for " +- "'127.0.0.1'"): ++ regex = "IP address mismatch, certificate is not valid for '127.0.0.1'" ++ if ssl is not None and "AWS-LC" in ssl.OPENSSL_VERSION: ++ regex = "CERTIFICATE_VERIFY_FAILED" ++ with self.assertRaisesRegex(ssl.CertificateError, regex): + _, server = self._setup(SimpleIMAPHandler) + client = self.imap_class(*server.server_address, + ssl_context=ssl_context) +@@ -967,10 +968,10 @@ def test_ssl_verified(self): + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ssl_context.load_verify_locations(CAFILE) + +- with self.assertRaisesRegex( +- ssl.CertificateError, +- "IP address mismatch, certificate is not valid for " +- "'127.0.0.1'"): ++ regex = "IP address mismatch, certificate is not valid for '127.0.0.1'" ++ if ssl is not None and "AWS-LC" in ssl.OPENSSL_VERSION: ++ regex = "CERTIFICATE_VERIFY_FAILED" ++ with self.assertRaisesRegex(ssl.CertificateError, regex): + with self.reaped_server(SimpleIMAPHandler) as server: + client = self.imap_class(*server.server_address, + ssl_context=ssl_context) +diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py +index 1f881038c3..36fa1e32e8 100644 +--- a/Lib/test/test_ssl.py ++++ b/Lib/test/test_ssl.py +@@ -44,6 +44,7 @@ + + Py_DEBUG = hasattr(sys, 'gettotalrefcount') + Py_DEBUG_WIN32 = Py_DEBUG and sys.platform == 'win32' ++Py_OPENSSL_IS_AWSLC = "AWS-LC" in ssl.OPENSSL_VERSION + + PROTOCOLS = sorted(ssl._PROTOCOL_NAMES) + HOST = socket_helper.HOST +@@ -170,7 +171,7 @@ def is_ubuntu(): + except FileNotFoundError: + return False + +-if is_ubuntu(): ++if is_ubuntu() and not Py_OPENSSL_IS_AWSLC: + def seclevel_workaround(*ctxs): + """"Lower security level to '1' and allow all ciphers for TLS 1.0/1""" + for ctx in ctxs: +@@ -548,7 +549,7 @@ def test_openssl_version(self): + else: + openssl_ver = f"OpenSSL {major:d}.{minor:d}.{fix:d}" + self.assertTrue( +- s.startswith((openssl_ver, libressl_ver)), ++ s.startswith((openssl_ver, libressl_ver, "AWS-LC")), + (s, t, hex(n)) + ) + +@@ -1320,7 +1321,6 @@ def test_min_max_version(self): + + if has_tls_protocol(ssl.PROTOCOL_TLSv1_1): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_1) +- + self.assertIn( + ctx.minimum_version, minimum_range + ) +@@ -1380,24 +1380,25 @@ def test_load_cert_chain(self): + with self.assertRaises(OSError) as cm: + ctx.load_cert_chain(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) +- with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): ++ with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(BADCERT) +- with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): ++ with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(EMPTYCERT) + # Separate key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_cert_chain(ONLYCERT, ONLYKEY) + ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) + ctx.load_cert_chain(certfile=BYTES_ONLYCERT, keyfile=BYTES_ONLYKEY) +- with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): ++ with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(ONLYCERT) +- with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): ++ with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(ONLYKEY) +- with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): ++ with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(certfile=ONLYKEY, keyfile=ONLYCERT) + # Mismatching key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) +- with self.assertRaisesRegex(ssl.SSLError, "key values mismatch"): ++ with self.assertRaisesRegex(ssl.SSLError, ++ "(key values mismatch|KEY_VALUES_MISMATCH)"): + ctx.load_cert_chain(CAFILE_CACERT, ONLYKEY) + # Password protected key and cert + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD) +@@ -1465,7 +1466,7 @@ def test_load_verify_locations(self): + with self.assertRaises(OSError) as cm: + ctx.load_verify_locations(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) +- with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): ++ with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_verify_locations(BADCERT) + ctx.load_verify_locations(CERTFILE, CAPATH) + ctx.load_verify_locations(CERTFILE, capath=BYTES_CAPATH) +@@ -1863,10 +1864,11 @@ def test_lib_reason(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + with self.assertRaises(ssl.SSLError) as cm: + ctx.load_dh_params(CERTFILE) +- self.assertEqual(cm.exception.library, 'PEM') +- self.assertEqual(cm.exception.reason, 'NO_START_LINE') ++ if not Py_OPENSSL_IS_AWSLC: ++ self.assertEqual(cm.exception.library, 'PEM') ++ self.assertEqual(cm.exception.reason, 'NO_START_LINE') + s = str(cm.exception) +- self.assertTrue(s.startswith("[PEM: NO_START_LINE] no start line"), s) ++ self.assertTrue("NO_START_LINE" in s, s) + + def test_subclass(self): + # Check that the appropriate SSLError subclass is raised +@@ -2046,7 +2048,8 @@ def test_connect_fail(self): + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED) + self.addCleanup(s.close) +- self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", ++ msg_re = "(certificate verify failed|CERTIFICATE_VERIFY_FAILED)" ++ self.assertRaisesRegex(ssl.SSLError, msg_re, + s.connect, self.server_addr) + + def test_connect_ex(self): +@@ -2114,7 +2117,8 @@ def test_connect_with_context_fail(self): + server_hostname=SIGNED_CERTFILE_HOSTNAME + ) + self.addCleanup(s.close) +- self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", ++ msg_re = "(certificate verify failed|CERTIFICATE_VERIFY_FAILED)" ++ self.assertRaisesRegex(ssl.SSLError, msg_re, + s.connect, self.server_addr) + + def test_connect_capath(self): +@@ -2331,14 +2335,14 @@ def test_bio_handshake(self): + self.assertIsNone(sslobj.version()) + self.assertIsNone(sslobj.shared_ciphers()) + self.assertRaises(ValueError, sslobj.getpeercert) +- if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: ++ if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES and sslobj.version() != "TLSv1.3": + self.assertIsNone(sslobj.get_channel_binding('tls-unique')) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + self.assertTrue(sslobj.cipher()) + self.assertIsNone(sslobj.shared_ciphers()) + self.assertIsNotNone(sslobj.version()) + self.assertTrue(sslobj.getpeercert()) +- if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: ++ if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES and sslobj.version() != "TLSv1.3": + self.assertTrue(sslobj.get_channel_binding('tls-unique')) + try: + self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) +@@ -3062,11 +3066,11 @@ def test_crl_check(self): + client_context.verify_flags |= ssl.VERIFY_CRL_CHECK_LEAF + + server = ThreadedEchoServer(context=server_context, chatty=True) ++ msg_re = "(certificate verify failed|CERTIFICATE_VERIFY_FAILED)" + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: +- with self.assertRaisesRegex(ssl.SSLError, +- "certificate verify failed"): ++ with self.assertRaisesRegex(ssl.SSLError, msg_re): + s.connect((HOST, server.port)) + + # now load a CRL file. The CRL file is signed by the CA. +@@ -3097,12 +3101,12 @@ def test_check_hostname(self): + + # incorrect hostname should raise an exception + server = ThreadedEchoServer(context=server_context, chatty=True) ++ err_re = "(CERTIFICATE_VERIFY_FAILED|" ++ err_re += "Hostname mismatch, certificate is not valid for 'invalid'.)" + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname="invalid") as s: +- with self.assertRaisesRegex( +- ssl.CertificateError, +- "Hostname mismatch, certificate is not valid for 'invalid'."): ++ with self.assertRaisesRegex(ssl.CertificateError, err_re): + s.connect((HOST, server.port)) + + # missing server_hostname arg should cause an exception, too +@@ -3296,7 +3300,7 @@ def test_wrong_cert_tls13(self): + s.connect((HOST, server.port)) + with self.assertRaisesRegex( + ssl.SSLError, +- 'alert unknown ca|EOF occurred' ++ 'alert unknown ca|EOF occurred|TLSV1_ALERT_UNKNOWN_CA' + ): + # TLS 1.3 perform client cert exchange after handshake + s.write(b'data') +@@ -3365,8 +3369,13 @@ def test_ssl_cert_verify_error(self): + self.assertIsInstance(e, ssl.SSLCertVerificationError) + self.assertEqual(e.verify_code, 20) + self.assertEqual(e.verify_message, msg) ++ if Py_OPENSSL_IS_AWSLC: ++ msg = "CERTIFICATE_VERIFY_FAILED" + self.assertIn(msg, repr(e)) +- self.assertIn('certificate verify failed', repr(e)) ++ expected_err = 'certificate verify failed' ++ if Py_OPENSSL_IS_AWSLC: ++ expected_err = "CERTIFICATE_VERIFY_FAILED" ++ self.assertIn(expected_err, repr(e)) + + @requires_tls_version('SSLv2') + def test_protocol_sslv2(self): +@@ -3933,7 +3942,10 @@ def test_no_shared_ciphers(self): + server_hostname=hostname) as s: + with self.assertRaises(OSError): + s.connect((HOST, server.port)) +- self.assertIn("no shared cipher", server.conn_errors[0]) ++ expected_err = "no shared cipher" ++ if Py_OPENSSL_IS_AWSLC: ++ expected_err = "NO_SHARED_CIPHER" ++ self.assertIn(expected_err, server.conn_errors[0]) + + def test_version_basic(self): + """ +@@ -4021,7 +4033,10 @@ def test_min_max_version_mismatch(self): + server_hostname=hostname) as s: + with self.assertRaises(ssl.SSLError) as e: + s.connect((HOST, server.port)) +- self.assertIn("alert", str(e.exception)) ++ self.assertTrue( ++ "alert"in str(e.exception) ++ or "ALERT"in str(e.exception) ++ ) + + @requires_tls_version('SSLv3') + def test_min_max_version_sslv3(self): +@@ -4063,6 +4078,9 @@ def test_tls_unique_channel_binding(self): + + client_context, server_context, hostname = testing_context() + ++ # tls-unique isn't defined as of TLSv1.3 ++ client_context.maximum_version = ssl.TLSVersion.TLSv1_2 ++ + server = ThreadedEchoServer(context=server_context, + chatty=True, + connectionchatty=False) +@@ -4135,6 +4153,7 @@ def test_compression_disabled(self): + self.assertIs(stats['compression'], None) + + @unittest.skipIf(Py_DEBUG_WIN32, "Avoid mixing debug/release CRT on Windows") ++ @unittest.skipIf(Py_OPENSSL_IS_AWSLC, "AWS-LC doesn't support (FF)DHE") + def test_dh_params(self): + # Check we can get a connection with ephemeral Diffie-Hellman + client_context, server_context, hostname = testing_context() +@@ -4149,7 +4168,7 @@ def test_dh_params(self): + cipher = stats["cipher"][0] + parts = cipher.split("-") + if "ADH" not in parts and "EDH" not in parts and "DHE" not in parts: +- self.fail("Non-DH cipher: " + cipher[0]) ++ self.fail("Non-DH kx: " + parts[0]) + + def test_ecdh_curve(self): + # server secp384r1, client auto +@@ -4316,8 +4335,10 @@ def cb_raising(ssl_sock, server_name, initial_context): + chatty=False, + sni_name='supermessage') + +- self.assertEqual(cm.exception.reason, +- 'SSLV3_ALERT_HANDSHAKE_FAILURE') ++ expected_reason = 'SSLV3_ALERT_HANDSHAKE_FAILURE' ++ if Py_OPENSSL_IS_AWSLC: ++ expected_reason = 'NO_PRIVATE_VALUE' ++ self.assertEqual(cm.exception.reason, expected_reason) + self.assertEqual(catch.unraisable.exc_type, ZeroDivisionError) + + def test_sni_callback_wrong_return_type(self): +@@ -4493,7 +4514,10 @@ def test_session_handling(self): + 'Session refers to a different SSLContext.') + + +-@unittest.skipUnless(has_tls_version('TLSv1_3'), "Test needs TLS 1.3") ++@unittest.skipUnless( ++ has_tls_version('TLSv1_3') and not Py_OPENSSL_IS_AWSLC, ++ "Test needs TLS 1.3; AWS-LC doesn't support PHA" ++) + class TestPostHandshakeAuth(unittest.TestCase): + def test_pha_setter(self): + protocols = [ +@@ -4769,6 +4793,31 @@ def test_internal_chain_server(self): + self.assertEqual(res, b'\x02\n') + + ++@unittest.skipUnless(Py_OPENSSL_IS_AWSLC, "Only test this against AWS-LC") ++class TestPostHandshakeAuthAwsLc(unittest.TestCase): ++ def test_pha(self): ++ protocols = [ ++ ssl.PROTOCOL_TLS_SERVER, ssl.PROTOCOL_TLS_CLIENT ++ ] ++ for protocol in protocols: ++ client_ctx, server_ctx, hostname = testing_context() ++ client_ctx.load_cert_chain(SIGNED_CERTFILE) ++ self.assertEqual(client_ctx.post_handshake_auth, None) ++ with self.assertRaises(AttributeError): ++ client_ctx.post_handshake_auth = True ++ with self.assertRaises(AttributeError): ++ server_ctx.post_handshake_auth = True ++ ++ with ThreadedEchoServer(context=server_ctx) as server: ++ with client_ctx.wrap_socket( ++ socket.socket(), ++ server_hostname=hostname ++ ) as ssock: ++ ssock.connect((HOST, server.port)) ++ with self.assertRaises(NotImplementedError): ++ ssock.verify_client_post_handshake() ++ ++ + HAS_KEYLOG = hasattr(ssl.SSLContext, 'keylog_filename') + requires_keylog = unittest.skipUnless( + HAS_KEYLOG, 'test requires OpenSSL 1.1.1 with keylog callback') +diff --git a/Modules/Setup b/Modules/Setup +index d3647ecb99..a0ff874b6d 100644 +--- a/Modules/Setup ++++ b/Modules/Setup +@@ -216,11 +216,11 @@ PYTHONPATH=$(COREPYTHONPATH) + #_hashlib _hashopenssl.c $(OPENSSL_INCLUDES) $(OPENSSL_LDFLAGS) -lcrypto + + # To statically link OpenSSL: +-# _ssl _ssl.c $(OPENSSL_INCLUDES) $(OPENSSL_LDFLAGS) \ +-# -l:libssl.a -Wl,--exclude-libs,libssl.a \ +-# -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a +-# _hashlib _hashopenssl.c $(OPENSSL_INCLUDES) $(OPENSSL_LDFLAGS) \ +-# -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a ++_ssl _ssl.c $(OPENSSL_INCLUDES) $(OPENSSL_LDFLAGS) \ ++ -l:libssl.a -Wl,--exclude-libs,libssl.a \ ++ -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a ++_hashlib _hashopenssl.c $(OPENSSL_INCLUDES) $(OPENSSL_LDFLAGS) \ ++ -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a + + # The _tkinter module. + # +diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c +index 57d64bd80c..1132fa520c 100644 +--- a/Modules/_hashopenssl.c ++++ b/Modules/_hashopenssl.c +@@ -131,8 +131,12 @@ static const py_hashentry_t py_hashes[] = { + PY_HASH_ENTRY(Py_hash_shake_128, NULL, SN_shake128, NID_shake128), + PY_HASH_ENTRY(Py_hash_shake_256, NULL, SN_shake256, NID_shake256), + /* blake2 digest */ ++#if defined(NID_blake2s256) + PY_HASH_ENTRY(Py_hash_blake2s, "blake2s256", SN_blake2s256, NID_blake2s256), ++#endif ++#if defined(NID_blake2b512) + PY_HASH_ENTRY(Py_hash_blake2b, "blake2b512", SN_blake2b512, NID_blake2b512), ++#endif + PY_HASH_ENTRY(NULL, NULL, NULL, 0), + }; + +diff --git a/Modules/_ssl.c b/Modules/_ssl.c +index 67ce6e97af..1132d82dd9 100644 +--- a/Modules/_ssl.c ++++ b/Modules/_ssl.c +@@ -179,6 +179,12 @@ extern const SSL_METHOD *TLSv1_2_method(void); + #endif + + ++ ++#if defined(OPENSSL_IS_AWSLC) || !defined(TLS1_3_VERSION) || defined(OPENSSL_NO_TLS1_3) ++ #define PY_SSL_NO_POST_HS_AUTH ++#endif ++ ++ + enum py_ssl_error { + /* these mirror ssl.h */ + PY_SSL_ERROR_NONE, +@@ -223,7 +229,7 @@ enum py_proto_version { + PY_PROTO_TLSv1 = TLS1_VERSION, + PY_PROTO_TLSv1_1 = TLS1_1_VERSION, + PY_PROTO_TLSv1_2 = TLS1_2_VERSION, +-#ifdef TLS1_3_VERSION ++#if defined(TLS1_3_VERSION) + PY_PROTO_TLSv1_3 = TLS1_3_VERSION, + #else + PY_PROTO_TLSv1_3 = 0x304, +@@ -285,7 +291,7 @@ typedef struct { + */ + unsigned int hostflags; + int protocol; +-#ifdef TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + int post_handshake_auth; + #endif + PyObject *msg_cb; +@@ -857,7 +863,7 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, + SSL_set_mode(self->ssl, + SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER | SSL_MODE_AUTO_RETRY); + +-#ifdef TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + if (sslctx->post_handshake_auth == 1) { + if (socket_type == PY_SSL_SERVER) { + /* bpo-37428: OpenSSL does not ignore SSL_VERIFY_POST_HANDSHAKE. +@@ -1002,6 +1008,7 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) + } while (err.ssl == SSL_ERROR_WANT_READ || + err.ssl == SSL_ERROR_WANT_WRITE); + Py_XDECREF(sock); ++ + if (ret < 1) + return PySSL_SetError(self, ret, __FILE__, __LINE__); + if (PySSL_ChainExceptions(self) < 0) +@@ -2041,7 +2048,12 @@ _ssl__SSLSocket_shared_ciphers_impl(PySSLSocket *self) + len = 0; + for (i = 0; i < sk_SSL_CIPHER_num(server_ciphers); i++) { + cipher = sk_SSL_CIPHER_value(server_ciphers, i); ++#if defined(OPENSSL_IS_AWSLC) ++ size_t unused_idx; ++ if (sk_SSL_CIPHER_find(client_ciphers, &unused_idx, cipher) == 0) ++#else + if (sk_SSL_CIPHER_find(client_ciphers, cipher) < 0) ++#endif + continue; + + PyObject *tup = cipher_to_tuple(cipher); +@@ -2775,7 +2787,7 @@ static PyObject * + _ssl__SSLSocket_verify_client_post_handshake_impl(PySSLSocket *self) + /*[clinic end generated code: output=532147f3b1341425 input=6bfa874810a3d889]*/ + { +-#ifdef TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + int err = SSL_verify_client_post_handshake(self->ssl); + if (err == 0) + return _setSSLError(get_state_sock(self), NULL, 0, __FILE__, __LINE__); +@@ -3191,7 +3203,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) + + /* Set SSL_MODE_RELEASE_BUFFERS. This potentially greatly reduces memory + usage for no cost at all. */ +- SSL_CTX_set_mode(self->ctx, SSL_MODE_RELEASE_BUFFERS); ++ SSL_CTX_set_mode(self->ctx, SSL_MODE_RELEASE_BUFFERS | SSL_MODE_AUTO_RETRY); + + #define SID_CTX "Python" + SSL_CTX_set_session_id_context(self->ctx, (const unsigned char *) SID_CTX, +@@ -3204,7 +3216,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) + X509_VERIFY_PARAM_set_flags(params, X509_V_FLAG_TRUSTED_FIRST); + X509_VERIFY_PARAM_set_hostflags(params, self->hostflags); + +-#ifdef TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + self->post_handshake_auth = 0; + SSL_CTX_set_post_handshake_auth(self->ctx, self->post_handshake_auth); + #endif +@@ -3578,7 +3590,7 @@ set_maximum_version(PySSLContext *self, PyObject *arg, void *c) + return set_min_max_proto_version(self, arg, 1); + } + +-#ifdef TLS1_3_VERSION ++#if defined(TLS1_3_VERSION) && !defined(OPENSSL_NO_TLS1_3) + static PyObject * + get_num_tickets(PySSLContext *self, void *c) + { +@@ -3609,7 +3621,7 @@ set_num_tickets(PySSLContext *self, PyObject *arg, void *c) + + PyDoc_STRVAR(PySSLContext_num_tickets_doc, + "Control the number of TLSv1.3 session tickets"); +-#endif /* TLS1_3_VERSION */ ++#endif /* defined(TLS1_3_VERSION) */ + + static PyObject * + get_security_level(PySSLContext *self, void *c) +@@ -3699,14 +3711,14 @@ set_check_hostname(PySSLContext *self, PyObject *arg, void *c) + + static PyObject * + get_post_handshake_auth(PySSLContext *self, void *c) { +-#if TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + return PyBool_FromLong(self->post_handshake_auth); + #else + Py_RETURN_NONE; + #endif + } + +-#if TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + static int + set_post_handshake_auth(PySSLContext *self, PyObject *arg, void *c) { + if (arg == NULL) { +@@ -4661,14 +4673,14 @@ static PyGetSetDef context_getsetlist[] = { + (setter) _PySSLContext_set_msg_callback, NULL}, + {"sni_callback", (getter) get_sni_callback, + (setter) set_sni_callback, PySSLContext_sni_callback_doc}, +-#ifdef TLS1_3_VERSION ++#if defined(TLS1_3_VERSION) && !defined(OPENSSL_NO_TLS1_3) + {"num_tickets", (getter) get_num_tickets, + (setter) set_num_tickets, PySSLContext_num_tickets_doc}, + #endif + {"options", (getter) get_options, + (setter) set_options, NULL}, + {"post_handshake_auth", (getter) get_post_handshake_auth, +-#ifdef TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + (setter) set_post_handshake_auth, + #else + NULL, diff --git a/tests/ci/integration/python_patch/3.12/aws-lc-cpython.patch b/tests/ci/integration/python_patch/3.12/aws-lc-cpython.patch new file mode 100644 index 0000000000..be161489c5 --- /dev/null +++ b/tests/ci/integration/python_patch/3.12/aws-lc-cpython.patch @@ -0,0 +1,543 @@ +diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py +index b9069056c3..cffbc074a8 100644 +--- a/Lib/test/test_asyncio/test_events.py ++++ b/Lib/test/test_asyncio/test_events.py +@@ -1122,12 +1122,12 @@ def test_create_server_ssl_match_failed(self): + # incorrect server_hostname + f_c = self.loop.create_connection(MyProto, host, port, + ssl=sslcontext_client) ++ regex = "IP address mismatch, certificate is not valid for '127.0.0.1'" ++ if ssl is not None and "AWS-LC" in ssl.OPENSSL_VERSION: ++ regex = "CERTIFICATE_VERIFY_FAILED" + with mock.patch.object(self.loop, 'call_exception_handler'): + with test_utils.disable_logger(): +- with self.assertRaisesRegex( +- ssl.CertificateError, +- "IP address mismatch, certificate is not valid for " +- "'127.0.0.1'"): ++ with self.assertRaisesRegex(ssl.CertificateError, regex): + self.loop.run_until_complete(f_c) + + # close connection +diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py +index fe8105ee2b..157c243faf 100644 +--- a/Lib/test/test_httplib.py ++++ b/Lib/test/test_httplib.py +@@ -2035,7 +2035,7 @@ def test_host_port(self): + + def test_tls13_pha(self): + import ssl +- if not ssl.HAS_TLSv1_3: ++ if not ssl.HAS_TLSv1_3 or "AWS-LC" in ssl.OPENSSL_VERSION: + self.skipTest('TLS 1.3 support required') + # just check status of PHA flag + h = client.HTTPSConnection('localhost', 443) +diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py +index 60f5b671b1..56b64a6788 100644 +--- a/Lib/test/test_imaplib.py ++++ b/Lib/test/test_imaplib.py +@@ -555,9 +555,10 @@ def test_ssl_raises(self): + self.assertEqual(ssl_context.check_hostname, True) + ssl_context.load_verify_locations(CAFILE) + +- with self.assertRaisesRegex(ssl.CertificateError, +- "IP address mismatch, certificate is not valid for " +- "'127.0.0.1'"): ++ regex = "IP address mismatch, certificate is not valid for '127.0.0.1'" ++ if ssl is not None and "AWS-LC" in ssl.OPENSSL_VERSION: ++ regex = "CERTIFICATE_VERIFY_FAILED" ++ with self.assertRaisesRegex(ssl.CertificateError, regex): + _, server = self._setup(SimpleIMAPHandler) + client = self.imap_class(*server.server_address, + ssl_context=ssl_context) +@@ -950,10 +951,10 @@ def test_ssl_verified(self): + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ssl_context.load_verify_locations(CAFILE) + +- with self.assertRaisesRegex( +- ssl.CertificateError, +- "IP address mismatch, certificate is not valid for " +- "'127.0.0.1'"): ++ regex = "IP address mismatch, certificate is not valid for '127.0.0.1'" ++ if ssl is not None and "AWS-LC" in ssl.OPENSSL_VERSION: ++ regex = "CERTIFICATE_VERIFY_FAILED" ++ with self.assertRaisesRegex(ssl.CertificateError, regex): + with self.reaped_server(SimpleIMAPHandler) as server: + client = self.imap_class(*server.server_address, + ssl_context=ssl_context) +diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py +index 6117ca3fdb..fc93c6271b 100644 +--- a/Lib/test/test_ssl.py ++++ b/Lib/test/test_ssl.py +@@ -37,6 +37,7 @@ + from ssl import TLSVersion, _TLSContentType, _TLSMessageType, _TLSAlertType + + Py_DEBUG_WIN32 = support.Py_DEBUG and sys.platform == 'win32' ++Py_OPENSSL_IS_AWSLC = "AWS-LC" in ssl.OPENSSL_VERSION + + PROTOCOLS = sorted(ssl._PROTOCOL_NAMES) + HOST = socket_helper.HOST +@@ -163,7 +164,7 @@ def is_ubuntu(): + except FileNotFoundError: + return False + +-if is_ubuntu(): ++if is_ubuntu() and not Py_OPENSSL_IS_AWSLC: + def seclevel_workaround(*ctxs): + """"Lower security level to '1' and allow all ciphers for TLS 1.0/1""" + for ctx in ctxs: +@@ -540,7 +541,7 @@ def test_openssl_version(self): + else: + openssl_ver = f"OpenSSL {major:d}.{minor:d}.{fix:d}" + self.assertTrue( +- s.startswith((openssl_ver, libressl_ver)), ++ s.startswith((openssl_ver, libressl_ver, "AWS-LC")), + (s, t, hex(n)) + ) + +@@ -1098,7 +1099,6 @@ def test_min_max_version(self): + + if has_tls_protocol(ssl.PROTOCOL_TLSv1_1): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_1) +- + self.assertIn( + ctx.minimum_version, minimum_range + ) +@@ -1158,24 +1158,25 @@ def test_load_cert_chain(self): + with self.assertRaises(OSError) as cm: + ctx.load_cert_chain(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) +- with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): ++ with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(BADCERT) +- with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): ++ with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(EMPTYCERT) + # Separate key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_cert_chain(ONLYCERT, ONLYKEY) + ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) + ctx.load_cert_chain(certfile=BYTES_ONLYCERT, keyfile=BYTES_ONLYKEY) +- with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): ++ with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(ONLYCERT) +- with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): ++ with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(ONLYKEY) +- with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): ++ with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(certfile=ONLYKEY, keyfile=ONLYCERT) + # Mismatching key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) +- with self.assertRaisesRegex(ssl.SSLError, "key values mismatch"): ++ with self.assertRaisesRegex(ssl.SSLError, ++ "(key values mismatch|KEY_VALUES_MISMATCH)"): + ctx.load_cert_chain(CAFILE_CACERT, ONLYKEY) + # Password protected key and cert + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD) +@@ -1243,7 +1244,7 @@ def test_load_verify_locations(self): + with self.assertRaises(OSError) as cm: + ctx.load_verify_locations(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) +- with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): ++ with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_verify_locations(BADCERT) + ctx.load_verify_locations(CERTFILE, CAPATH) + ctx.load_verify_locations(CERTFILE, capath=BYTES_CAPATH) +@@ -1646,10 +1647,11 @@ def test_lib_reason(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + with self.assertRaises(ssl.SSLError) as cm: + ctx.load_dh_params(CERTFILE) +- self.assertEqual(cm.exception.library, 'PEM') +- self.assertEqual(cm.exception.reason, 'NO_START_LINE') ++ if not Py_OPENSSL_IS_AWSLC: ++ self.assertEqual(cm.exception.library, 'PEM') ++ self.assertEqual(cm.exception.reason, 'NO_START_LINE') + s = str(cm.exception) +- self.assertTrue(s.startswith("[PEM: NO_START_LINE] no start line"), s) ++ self.assertTrue("NO_START_LINE" in s, s) + + def test_subclass(self): + # Check that the appropriate SSLError subclass is raised +@@ -1825,7 +1827,8 @@ def test_connect_fail(self): + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED) + self.addCleanup(s.close) +- self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", ++ msg_re = "(certificate verify failed|CERTIFICATE_VERIFY_FAILED)" ++ self.assertRaisesRegex(ssl.SSLError, msg_re, + s.connect, self.server_addr) + + def test_connect_ex(self): +@@ -1893,7 +1896,8 @@ def test_connect_with_context_fail(self): + server_hostname=SIGNED_CERTFILE_HOSTNAME + ) + self.addCleanup(s.close) +- self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", ++ msg_re = "(certificate verify failed|CERTIFICATE_VERIFY_FAILED)" ++ self.assertRaisesRegex(ssl.SSLError, msg_re, + s.connect, self.server_addr) + + def test_connect_capath(self): +@@ -2110,14 +2114,14 @@ def test_bio_handshake(self): + self.assertIsNone(sslobj.version()) + self.assertIsNone(sslobj.shared_ciphers()) + self.assertRaises(ValueError, sslobj.getpeercert) +- if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: ++ if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES and sslobj.version() != "TLSv1.3": + self.assertIsNone(sslobj.get_channel_binding('tls-unique')) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + self.assertTrue(sslobj.cipher()) + self.assertIsNone(sslobj.shared_ciphers()) + self.assertIsNotNone(sslobj.version()) + self.assertTrue(sslobj.getpeercert()) +- if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: ++ if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES and sslobj.version() != "TLSv1.3": + self.assertTrue(sslobj.get_channel_binding('tls-unique')) + try: + self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) +@@ -2839,11 +2843,11 @@ def test_crl_check(self): + client_context.verify_flags |= ssl.VERIFY_CRL_CHECK_LEAF + + server = ThreadedEchoServer(context=server_context, chatty=True) ++ msg_re = "(certificate verify failed|CERTIFICATE_VERIFY_FAILED)" + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: +- with self.assertRaisesRegex(ssl.SSLError, +- "certificate verify failed"): ++ with self.assertRaisesRegex(ssl.SSLError, msg_re): + s.connect((HOST, server.port)) + + # now load a CRL file. The CRL file is signed by the CA. +@@ -2874,12 +2878,12 @@ def test_check_hostname(self): + + # incorrect hostname should raise an exception + server = ThreadedEchoServer(context=server_context, chatty=True) ++ err_re = "(CERTIFICATE_VERIFY_FAILED|" ++ err_re += "Hostname mismatch, certificate is not valid for 'invalid'.)" + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname="invalid") as s: +- with self.assertRaisesRegex( +- ssl.CertificateError, +- "Hostname mismatch, certificate is not valid for 'invalid'."): ++ with self.assertRaisesRegex(ssl.CertificateError, err_re): + s.connect((HOST, server.port)) + + # missing server_hostname arg should cause an exception, too +@@ -3073,7 +3077,7 @@ def test_wrong_cert_tls13(self): + s.connect((HOST, server.port)) + with self.assertRaisesRegex( + ssl.SSLError, +- 'alert unknown ca|EOF occurred' ++ 'alert unknown ca|EOF occurred|TLSV1_ALERT_UNKNOWN_CA' + ): + # TLS 1.3 perform client cert exchange after handshake + s.write(b'data') +@@ -3142,8 +3146,13 @@ def test_ssl_cert_verify_error(self): + self.assertIsInstance(e, ssl.SSLCertVerificationError) + self.assertEqual(e.verify_code, 20) + self.assertEqual(e.verify_message, msg) ++ if Py_OPENSSL_IS_AWSLC: ++ msg = "CERTIFICATE_VERIFY_FAILED" + self.assertIn(msg, repr(e)) +- self.assertIn('certificate verify failed', repr(e)) ++ expected_err = 'certificate verify failed' ++ if Py_OPENSSL_IS_AWSLC: ++ expected_err = "CERTIFICATE_VERIFY_FAILED" ++ self.assertIn(expected_err, repr(e)) + + def test_PROTOCOL_TLS(self): + """Connecting to an SSLv23 server with various client options""" +@@ -3654,7 +3663,10 @@ def test_no_shared_ciphers(self): + server_hostname=hostname) as s: + with self.assertRaises(OSError): + s.connect((HOST, server.port)) +- self.assertIn("no shared cipher", server.conn_errors[0]) ++ expected_err = "no shared cipher" ++ if Py_OPENSSL_IS_AWSLC: ++ expected_err = "NO_SHARED_CIPHER" ++ self.assertIn(expected_err, server.conn_errors[0]) + + def test_version_basic(self): + """ +@@ -3742,7 +3754,10 @@ def test_min_max_version_mismatch(self): + server_hostname=hostname) as s: + with self.assertRaises(ssl.SSLError) as e: + s.connect((HOST, server.port)) +- self.assertIn("alert", str(e.exception)) ++ self.assertTrue( ++ "alert"in str(e.exception) ++ or "ALERT"in str(e.exception) ++ ) + + @requires_tls_version('SSLv3') + def test_min_max_version_sslv3(self): +@@ -3784,6 +3799,9 @@ def test_tls_unique_channel_binding(self): + + client_context, server_context, hostname = testing_context() + ++ # tls-unique isn't defined as of TLSv1.3 ++ client_context.maximum_version = ssl.TLSVersion.TLSv1_2 ++ + server = ThreadedEchoServer(context=server_context, + chatty=True, + connectionchatty=False) +@@ -3870,6 +3888,7 @@ def test_no_legacy_server_connect(self): + sni_name=hostname) + + @unittest.skipIf(Py_DEBUG_WIN32, "Avoid mixing debug/release CRT on Windows") ++ @unittest.skipIf(Py_OPENSSL_IS_AWSLC, "AWS-LC doesn't support (FF)DHE") + def test_dh_params(self): + # Check we can get a connection with ephemeral Diffie-Hellman + client_context, server_context, hostname = testing_context() +@@ -3884,7 +3903,7 @@ def test_dh_params(self): + cipher = stats["cipher"][0] + parts = cipher.split("-") + if "ADH" not in parts and "EDH" not in parts and "DHE" not in parts: +- self.fail("Non-DH cipher: " + cipher[0]) ++ self.fail("Non-DH kx: " + parts[0]) + + def test_ecdh_curve(self): + # server secp384r1, client auto +@@ -4051,8 +4070,10 @@ def cb_raising(ssl_sock, server_name, initial_context): + chatty=False, + sni_name='supermessage') + +- self.assertEqual(cm.exception.reason, +- 'SSLV3_ALERT_HANDSHAKE_FAILURE') ++ expected_reason = 'SSLV3_ALERT_HANDSHAKE_FAILURE' ++ if Py_OPENSSL_IS_AWSLC: ++ expected_reason = 'NO_PRIVATE_VALUE' ++ self.assertEqual(cm.exception.reason, expected_reason) + self.assertEqual(catch.unraisable.exc_type, ZeroDivisionError) + + def test_sni_callback_wrong_return_type(self): +@@ -4228,7 +4249,10 @@ def test_session_handling(self): + 'Session refers to a different SSLContext.') + + +-@unittest.skipUnless(has_tls_version('TLSv1_3'), "Test needs TLS 1.3") ++@unittest.skipUnless( ++ has_tls_version('TLSv1_3') and not Py_OPENSSL_IS_AWSLC, ++ "Test needs TLS 1.3; AWS-LC doesn't support PHA" ++) + class TestPostHandshakeAuth(unittest.TestCase): + def test_pha_setter(self): + protocols = [ +@@ -4504,6 +4528,32 @@ def test_internal_chain_server(self): + self.assertEqual(res, b'\x02\n') + + ++@unittest.skipUnless(Py_OPENSSL_IS_AWSLC, "Only test this against AWS-LC") ++class TestPostHandshakeAuthAwsLc(unittest.TestCase): ++ def test_pha(self): ++ protocols = [ ++ ssl.PROTOCOL_TLS_SERVER, ssl.PROTOCOL_TLS_CLIENT ++ ] ++ for protocol in protocols: ++ client_ctx, server_ctx, hostname = testing_context() ++ client_ctx.load_cert_chain(SIGNED_CERTFILE) ++ self.assertEqual(client_ctx.post_handshake_auth, None) ++ with self.assertRaises(AttributeError): ++ client_ctx.post_handshake_auth = True ++ with self.assertRaises(AttributeError): ++ server_ctx.post_handshake_auth = True ++ ++ with ThreadedEchoServer(context=server_ctx) as server: ++ with client_ctx.wrap_socket( ++ socket.socket(), ++ server_hostname=hostname ++ ) as ssock: ++ ssock.connect((HOST, server.port)) ++ with self.assertRaises(NotImplementedError): ++ ssock.verify_client_post_handshake() ++ ++ ++ + HAS_KEYLOG = hasattr(ssl.SSLContext, 'keylog_filename') + requires_keylog = unittest.skipUnless( + HAS_KEYLOG, 'test requires OpenSSL 1.1.1 with keylog callback') +diff --git a/Modules/Setup b/Modules/Setup +index a8faa1d102..8abe8fd7d2 100644 +--- a/Modules/Setup ++++ b/Modules/Setup +@@ -215,11 +215,11 @@ PYTHONPATH=$(COREPYTHONPATH) + #_hashlib _hashopenssl.c $(OPENSSL_INCLUDES) $(OPENSSL_LDFLAGS) -lcrypto + + # To statically link OpenSSL: +-# _ssl _ssl.c $(OPENSSL_INCLUDES) $(OPENSSL_LDFLAGS) \ +-# -l:libssl.a -Wl,--exclude-libs,libssl.a \ +-# -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a +-# _hashlib _hashopenssl.c $(OPENSSL_INCLUDES) $(OPENSSL_LDFLAGS) \ +-# -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a ++_ssl _ssl.c $(OPENSSL_INCLUDES) $(OPENSSL_LDFLAGS) \ ++ -l:libssl.a -Wl,--exclude-libs,libssl.a \ ++ -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a ++_hashlib _hashopenssl.c $(OPENSSL_INCLUDES) $(OPENSSL_LDFLAGS) \ ++ -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a + + # The _tkinter module. + # +diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c +index af6d1b23d3..4b060b30a5 100644 +--- a/Modules/_hashopenssl.c ++++ b/Modules/_hashopenssl.c +@@ -130,8 +130,12 @@ static const py_hashentry_t py_hashes[] = { + PY_HASH_ENTRY(Py_hash_shake_128, NULL, SN_shake128, NID_shake128), + PY_HASH_ENTRY(Py_hash_shake_256, NULL, SN_shake256, NID_shake256), + /* blake2 digest */ ++#if defined(NID_blake2s256) + PY_HASH_ENTRY(Py_hash_blake2s, "blake2s256", SN_blake2s256, NID_blake2s256), ++#endif ++#if defined(NID_blake2b512) + PY_HASH_ENTRY(Py_hash_blake2b, "blake2b512", SN_blake2b512, NID_blake2b512), ++#endif + PY_HASH_ENTRY(NULL, NULL, NULL, 0), + }; + +diff --git a/Modules/_ssl.c b/Modules/_ssl.c +index e939f95048..8c99e2d8a1 100644 +--- a/Modules/_ssl.c ++++ b/Modules/_ssl.c +@@ -182,6 +182,12 @@ extern const SSL_METHOD *TLSv1_2_method(void); + #endif + + ++ ++#if defined(OPENSSL_IS_AWSLC) || !defined(TLS1_3_VERSION) || defined(OPENSSL_NO_TLS1_3) ++ #define PY_SSL_NO_POST_HS_AUTH ++#endif ++ ++ + enum py_ssl_error { + /* these mirror ssl.h */ + PY_SSL_ERROR_NONE, +@@ -226,7 +232,7 @@ enum py_proto_version { + PY_PROTO_TLSv1 = TLS1_VERSION, + PY_PROTO_TLSv1_1 = TLS1_1_VERSION, + PY_PROTO_TLSv1_2 = TLS1_2_VERSION, +-#ifdef TLS1_3_VERSION ++#if defined(TLS1_3_VERSION) + PY_PROTO_TLSv1_3 = TLS1_3_VERSION, + #else + PY_PROTO_TLSv1_3 = 0x304, +@@ -288,7 +294,7 @@ typedef struct { + */ + unsigned int hostflags; + int protocol; +-#ifdef TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + int post_handshake_auth; + #endif + PyObject *msg_cb; +@@ -851,7 +857,7 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, + SSL_set_mode(self->ssl, + SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER | SSL_MODE_AUTO_RETRY); + +-#ifdef TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + if (sslctx->post_handshake_auth == 1) { + if (socket_type == PY_SSL_SERVER) { + /* bpo-37428: OpenSSL does not ignore SSL_VERIFY_POST_HANDSHAKE. +@@ -996,6 +1002,7 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) + } while (err.ssl == SSL_ERROR_WANT_READ || + err.ssl == SSL_ERROR_WANT_WRITE); + Py_XDECREF(sock); ++ + if (ret < 1) + return PySSL_SetError(self, ret, __FILE__, __LINE__); + if (PySSL_ChainExceptions(self) < 0) +@@ -2028,7 +2035,12 @@ _ssl__SSLSocket_shared_ciphers_impl(PySSLSocket *self) + len = 0; + for (i = 0; i < sk_SSL_CIPHER_num(server_ciphers); i++) { + cipher = sk_SSL_CIPHER_value(server_ciphers, i); ++#if defined(OPENSSL_IS_AWSLC) ++ size_t unused_idx; ++ if (sk_SSL_CIPHER_find(client_ciphers, &unused_idx, cipher) == 0) ++#else + if (sk_SSL_CIPHER_find(client_ciphers, cipher) < 0) ++#endif + continue; + + PyObject *tup = cipher_to_tuple(cipher); +@@ -2754,7 +2766,7 @@ static PyObject * + _ssl__SSLSocket_verify_client_post_handshake_impl(PySSLSocket *self) + /*[clinic end generated code: output=532147f3b1341425 input=6bfa874810a3d889]*/ + { +-#ifdef TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + int err = SSL_verify_client_post_handshake(self->ssl); + if (err == 0) + return _setSSLError(get_state_sock(self), NULL, 0, __FILE__, __LINE__); +@@ -3169,7 +3181,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) + + /* Set SSL_MODE_RELEASE_BUFFERS. This potentially greatly reduces memory + usage for no cost at all. */ +- SSL_CTX_set_mode(self->ctx, SSL_MODE_RELEASE_BUFFERS); ++ SSL_CTX_set_mode(self->ctx, SSL_MODE_RELEASE_BUFFERS | SSL_MODE_AUTO_RETRY); + + #define SID_CTX "Python" + SSL_CTX_set_session_id_context(self->ctx, (const unsigned char *) SID_CTX, +@@ -3182,7 +3194,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) + X509_VERIFY_PARAM_set_flags(params, X509_V_FLAG_TRUSTED_FIRST); + X509_VERIFY_PARAM_set_hostflags(params, self->hostflags); + +-#ifdef TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + self->post_handshake_auth = 0; + SSL_CTX_set_post_handshake_auth(self->ctx, self->post_handshake_auth); + #endif +@@ -3556,7 +3568,7 @@ set_maximum_version(PySSLContext *self, PyObject *arg, void *c) + return set_min_max_proto_version(self, arg, 1); + } + +-#ifdef TLS1_3_VERSION ++#if defined(TLS1_3_VERSION) && !defined(OPENSSL_NO_TLS1_3) + static PyObject * + get_num_tickets(PySSLContext *self, void *c) + { +@@ -3587,7 +3599,7 @@ set_num_tickets(PySSLContext *self, PyObject *arg, void *c) + + PyDoc_STRVAR(PySSLContext_num_tickets_doc, + "Control the number of TLSv1.3 session tickets"); +-#endif /* TLS1_3_VERSION */ ++#endif /* defined(TLS1_3_VERSION) */ + + static PyObject * + get_security_level(PySSLContext *self, void *c) +@@ -3690,14 +3702,14 @@ set_check_hostname(PySSLContext *self, PyObject *arg, void *c) + + static PyObject * + get_post_handshake_auth(PySSLContext *self, void *c) { +-#if TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + return PyBool_FromLong(self->post_handshake_auth); + #else + Py_RETURN_NONE; + #endif + } + +-#if TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + static int + set_post_handshake_auth(PySSLContext *self, PyObject *arg, void *c) { + if (arg == NULL) { +@@ -4652,14 +4664,14 @@ static PyGetSetDef context_getsetlist[] = { + (setter) _PySSLContext_set_msg_callback, NULL}, + {"sni_callback", (getter) get_sni_callback, + (setter) set_sni_callback, PySSLContext_sni_callback_doc}, +-#ifdef TLS1_3_VERSION ++#if defined(TLS1_3_VERSION) && !defined(OPENSSL_NO_TLS1_3) + {"num_tickets", (getter) get_num_tickets, + (setter) set_num_tickets, PySSLContext_num_tickets_doc}, + #endif + {"options", (getter) get_options, + (setter) set_options, NULL}, + {"post_handshake_auth", (getter) get_post_handshake_auth, +-#ifdef TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + (setter) set_post_handshake_auth, + #else + NULL, diff --git a/tests/ci/integration/python_patch/main/aws-lc-cpython.patch b/tests/ci/integration/python_patch/main/aws-lc-cpython.patch new file mode 100644 index 0000000000..6f956d8289 --- /dev/null +++ b/tests/ci/integration/python_patch/main/aws-lc-cpython.patch @@ -0,0 +1,543 @@ +diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py +index b25c097573..8f07868b82 100644 +--- a/Lib/test/test_asyncio/test_events.py ++++ b/Lib/test/test_asyncio/test_events.py +@@ -1125,12 +1125,12 @@ def test_create_server_ssl_match_failed(self): + # incorrect server_hostname + f_c = self.loop.create_connection(MyProto, host, port, + ssl=sslcontext_client) ++ regex = "IP address mismatch, certificate is not valid for '127.0.0.1'" ++ if ssl is not None and "AWS-LC" in ssl.OPENSSL_VERSION: ++ regex = "CERTIFICATE_VERIFY_FAILED" + with mock.patch.object(self.loop, 'call_exception_handler'): + with test_utils.disable_logger(): +- with self.assertRaisesRegex( +- ssl.CertificateError, +- "IP address mismatch, certificate is not valid for " +- "'127.0.0.1'"): ++ with self.assertRaisesRegex(ssl.CertificateError, regex): + self.loop.run_until_complete(f_c) + + # close connection +diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py +index 089bf5be40..1713a5fbae 100644 +--- a/Lib/test/test_httplib.py ++++ b/Lib/test/test_httplib.py +@@ -2066,7 +2066,7 @@ def test_host_port(self): + + def test_tls13_pha(self): + import ssl +- if not ssl.HAS_TLSv1_3: ++ if not ssl.HAS_TLSv1_3 or "AWS-LC" in ssl.OPENSSL_VERSION: + self.skipTest('TLS 1.3 support required') + # just check status of PHA flag + h = client.HTTPSConnection('localhost', 443) +diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py +index b97474acca..f5980598c9 100644 +--- a/Lib/test/test_imaplib.py ++++ b/Lib/test/test_imaplib.py +@@ -558,9 +558,10 @@ def test_ssl_raises(self): + self.assertEqual(ssl_context.check_hostname, True) + ssl_context.load_verify_locations(CAFILE) + +- with self.assertRaisesRegex(ssl.CertificateError, +- "IP address mismatch, certificate is not valid for " +- "'127.0.0.1'"): ++ regex = "IP address mismatch, certificate is not valid for '127.0.0.1'" ++ if ssl is not None and "AWS-LC" in ssl.OPENSSL_VERSION: ++ regex = "CERTIFICATE_VERIFY_FAILED" ++ with self.assertRaisesRegex(ssl.CertificateError, regex): + _, server = self._setup(SimpleIMAPHandler) + client = self.imap_class(*server.server_address, + ssl_context=ssl_context) +@@ -954,10 +955,10 @@ def test_ssl_verified(self): + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ssl_context.load_verify_locations(CAFILE) + +- with self.assertRaisesRegex( +- ssl.CertificateError, +- "IP address mismatch, certificate is not valid for " +- "'127.0.0.1'"): ++ regex = "IP address mismatch, certificate is not valid for '127.0.0.1'" ++ if ssl is not None and "AWS-LC" in ssl.OPENSSL_VERSION: ++ regex = "CERTIFICATE_VERIFY_FAILED" ++ with self.assertRaisesRegex(ssl.CertificateError, regex): + with self.reaped_server(SimpleIMAPHandler) as server: + client = self.imap_class(*server.server_address, + ssl_context=ssl_context) +diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py +index 3fdfa29605..6f6fea7c36 100644 +--- a/Lib/test/test_ssl.py ++++ b/Lib/test/test_ssl.py +@@ -41,6 +41,7 @@ + from ssl import TLSVersion, _TLSContentType, _TLSMessageType, _TLSAlertType + + Py_DEBUG_WIN32 = support.Py_DEBUG and sys.platform == 'win32' ++Py_OPENSSL_IS_AWSLC = "AWS-LC" in ssl.OPENSSL_VERSION + + PROTOCOLS = sorted(ssl._PROTOCOL_NAMES) + HOST = socket_helper.HOST +@@ -167,7 +168,7 @@ def is_ubuntu(): + except FileNotFoundError: + return False + +-if is_ubuntu(): ++if is_ubuntu() and not Py_OPENSSL_IS_AWSLC: + def seclevel_workaround(*ctxs): + """"Lower security level to '1' and allow all ciphers for TLS 1.0/1""" + for ctx in ctxs: +@@ -544,7 +545,7 @@ def test_openssl_version(self): + else: + openssl_ver = f"OpenSSL {major:d}.{minor:d}.{fix:d}" + self.assertTrue( +- s.startswith((openssl_ver, libressl_ver)), ++ s.startswith((openssl_ver, libressl_ver, "AWS-LC")), + (s, t, hex(n)) + ) + +@@ -1102,7 +1103,6 @@ def test_min_max_version(self): + + if has_tls_protocol(ssl.PROTOCOL_TLSv1_1): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_1) +- + self.assertIn( + ctx.minimum_version, minimum_range + ) +@@ -1162,24 +1162,25 @@ def test_load_cert_chain(self): + with self.assertRaises(OSError) as cm: + ctx.load_cert_chain(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) +- with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): ++ with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(BADCERT) +- with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): ++ with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(EMPTYCERT) + # Separate key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_cert_chain(ONLYCERT, ONLYKEY) + ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) + ctx.load_cert_chain(certfile=BYTES_ONLYCERT, keyfile=BYTES_ONLYKEY) +- with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): ++ with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(ONLYCERT) +- with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): ++ with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(ONLYKEY) +- with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): ++ with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(certfile=ONLYKEY, keyfile=ONLYCERT) + # Mismatching key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) +- with self.assertRaisesRegex(ssl.SSLError, "key values mismatch"): ++ with self.assertRaisesRegex(ssl.SSLError, ++ "(key values mismatch|KEY_VALUES_MISMATCH)"): + ctx.load_cert_chain(CAFILE_CACERT, ONLYKEY) + # Password protected key and cert + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD) +@@ -1247,7 +1248,7 @@ def test_load_verify_locations(self): + with self.assertRaises(OSError) as cm: + ctx.load_verify_locations(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) +- with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): ++ with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_verify_locations(BADCERT) + ctx.load_verify_locations(CERTFILE, CAPATH) + ctx.load_verify_locations(CERTFILE, capath=BYTES_CAPATH) +@@ -1650,10 +1651,11 @@ def test_lib_reason(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + with self.assertRaises(ssl.SSLError) as cm: + ctx.load_dh_params(CERTFILE) +- self.assertEqual(cm.exception.library, 'PEM') +- self.assertEqual(cm.exception.reason, 'NO_START_LINE') ++ if not Py_OPENSSL_IS_AWSLC: ++ self.assertEqual(cm.exception.library, 'PEM') ++ self.assertEqual(cm.exception.reason, 'NO_START_LINE') + s = str(cm.exception) +- self.assertTrue(s.startswith("[PEM: NO_START_LINE] no start line"), s) ++ self.assertTrue("NO_START_LINE" in s, s) + + def test_subclass(self): + # Check that the appropriate SSLError subclass is raised +@@ -1833,7 +1835,8 @@ def test_connect_fail(self): + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED) + self.addCleanup(s.close) +- self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", ++ msg_re = "(certificate verify failed|CERTIFICATE_VERIFY_FAILED)" ++ self.assertRaisesRegex(ssl.SSLError, msg_re, + s.connect, self.server_addr) + + def test_connect_ex(self): +@@ -1901,7 +1904,8 @@ def test_connect_with_context_fail(self): + server_hostname=SIGNED_CERTFILE_HOSTNAME + ) + self.addCleanup(s.close) +- self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", ++ msg_re = "(certificate verify failed|CERTIFICATE_VERIFY_FAILED)" ++ self.assertRaisesRegex(ssl.SSLError, msg_re, + s.connect, self.server_addr) + + def test_connect_capath(self): +@@ -2118,14 +2122,14 @@ def test_bio_handshake(self): + self.assertIsNone(sslobj.version()) + self.assertIsNone(sslobj.shared_ciphers()) + self.assertRaises(ValueError, sslobj.getpeercert) +- if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: ++ if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES and sslobj.version() != "TLSv1.3": + self.assertIsNone(sslobj.get_channel_binding('tls-unique')) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + self.assertTrue(sslobj.cipher()) + self.assertIsNone(sslobj.shared_ciphers()) + self.assertIsNotNone(sslobj.version()) + self.assertTrue(sslobj.getpeercert()) +- if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: ++ if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES and sslobj.version() != "TLSv1.3": + self.assertTrue(sslobj.get_channel_binding('tls-unique')) + try: + self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) +@@ -2849,11 +2853,11 @@ def test_crl_check(self): + client_context.verify_flags |= ssl.VERIFY_CRL_CHECK_LEAF + + server = ThreadedEchoServer(context=server_context, chatty=True) ++ msg_re = "(certificate verify failed|CERTIFICATE_VERIFY_FAILED)" + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: +- with self.assertRaisesRegex(ssl.SSLError, +- "certificate verify failed"): ++ with self.assertRaisesRegex(ssl.SSLError, msg_re): + s.connect((HOST, server.port)) + + # now load a CRL file. The CRL file is signed by the CA. +@@ -2884,12 +2888,12 @@ def test_check_hostname(self): + + # incorrect hostname should raise an exception + server = ThreadedEchoServer(context=server_context, chatty=True) ++ err_re = "(CERTIFICATE_VERIFY_FAILED|" ++ err_re += "Hostname mismatch, certificate is not valid for 'invalid'.)" + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname="invalid") as s: +- with self.assertRaisesRegex( +- ssl.CertificateError, +- "Hostname mismatch, certificate is not valid for 'invalid'."): ++ with self.assertRaisesRegex(ssl.CertificateError, err_re): + s.connect((HOST, server.port)) + + # missing server_hostname arg should cause an exception, too +@@ -3083,7 +3087,7 @@ def test_wrong_cert_tls13(self): + s.connect((HOST, server.port)) + with self.assertRaisesRegex( + ssl.SSLError, +- 'alert unknown ca|EOF occurred' ++ 'alert unknown ca|EOF occurred|TLSV1_ALERT_UNKNOWN_CA' + ): + # TLS 1.3 perform client cert exchange after handshake + s.write(b'data') +@@ -3152,8 +3156,13 @@ def test_ssl_cert_verify_error(self): + self.assertIsInstance(e, ssl.SSLCertVerificationError) + self.assertEqual(e.verify_code, 20) + self.assertEqual(e.verify_message, msg) ++ if Py_OPENSSL_IS_AWSLC: ++ msg = "CERTIFICATE_VERIFY_FAILED" + self.assertIn(msg, repr(e)) +- self.assertIn('certificate verify failed', repr(e)) ++ expected_err = 'certificate verify failed' ++ if Py_OPENSSL_IS_AWSLC: ++ expected_err = "CERTIFICATE_VERIFY_FAILED" ++ self.assertIn(expected_err, repr(e)) + + def test_PROTOCOL_TLS(self): + """Connecting to an SSLv23 server with various client options""" +@@ -3685,7 +3694,10 @@ def test_no_shared_ciphers(self): + server_hostname=hostname) as s: + with self.assertRaises(OSError): + s.connect((HOST, server.port)) +- self.assertIn("no shared cipher", server.conn_errors[0]) ++ expected_err = "no shared cipher" ++ if Py_OPENSSL_IS_AWSLC: ++ expected_err = "NO_SHARED_CIPHER" ++ self.assertIn(expected_err, server.conn_errors[0]) + + def test_version_basic(self): + """ +@@ -3773,7 +3785,10 @@ def test_min_max_version_mismatch(self): + server_hostname=hostname) as s: + with self.assertRaises(ssl.SSLError) as e: + s.connect((HOST, server.port)) +- self.assertIn("alert", str(e.exception)) ++ self.assertTrue( ++ "alert"in str(e.exception) ++ or "ALERT"in str(e.exception) ++ ) + + @requires_tls_version('SSLv3') + def test_min_max_version_sslv3(self): +@@ -3815,6 +3830,9 @@ def test_tls_unique_channel_binding(self): + + client_context, server_context, hostname = testing_context() + ++ # tls-unique isn't defined as of TLSv1.3 ++ client_context.maximum_version = ssl.TLSVersion.TLSv1_2 ++ + server = ThreadedEchoServer(context=server_context, + chatty=True, + connectionchatty=False) +@@ -3901,6 +3919,7 @@ def test_no_legacy_server_connect(self): + sni_name=hostname) + + @unittest.skipIf(Py_DEBUG_WIN32, "Avoid mixing debug/release CRT on Windows") ++ @unittest.skipIf(Py_OPENSSL_IS_AWSLC, "AWS-LC doesn't support (FF)DHE") + def test_dh_params(self): + # Check we can get a connection with ephemeral Diffie-Hellman + client_context, server_context, hostname = testing_context() +@@ -3915,7 +3934,7 @@ def test_dh_params(self): + cipher = stats["cipher"][0] + parts = cipher.split("-") + if "ADH" not in parts and "EDH" not in parts and "DHE" not in parts: +- self.fail("Non-DH cipher: " + cipher[0]) ++ self.fail("Non-DH kx: " + parts[0]) + + def test_ecdh_curve(self): + # server secp384r1, client auto +@@ -4082,8 +4101,10 @@ def cb_raising(ssl_sock, server_name, initial_context): + chatty=False, + sni_name='supermessage') + +- self.assertEqual(cm.exception.reason, +- 'SSLV3_ALERT_HANDSHAKE_FAILURE') ++ expected_reason = 'SSLV3_ALERT_HANDSHAKE_FAILURE' ++ if Py_OPENSSL_IS_AWSLC: ++ expected_reason = 'NO_PRIVATE_VALUE' ++ self.assertEqual(cm.exception.reason, expected_reason) + self.assertEqual(catch.unraisable.exc_type, ZeroDivisionError) + + def test_sni_callback_wrong_return_type(self): +@@ -4360,7 +4381,10 @@ def server_callback(identity): + s.connect((HOST, server.port)) + + +-@unittest.skipUnless(has_tls_version('TLSv1_3'), "Test needs TLS 1.3") ++@unittest.skipUnless( ++ has_tls_version('TLSv1_3') and not Py_OPENSSL_IS_AWSLC, ++ "Test needs TLS 1.3; AWS-LC doesn't support PHA" ++) + class TestPostHandshakeAuth(unittest.TestCase): + def test_pha_setter(self): + protocols = [ +@@ -4636,6 +4660,31 @@ def test_internal_chain_server(self): + self.assertEqual(res, b'\x02\n') + + ++@unittest.skipUnless(Py_OPENSSL_IS_AWSLC, "Only test this against AWS-LC") ++class TestPostHandshakeAuthAwsLc(unittest.TestCase): ++ def test_pha(self): ++ protocols = [ ++ ssl.PROTOCOL_TLS_SERVER, ssl.PROTOCOL_TLS_CLIENT ++ ] ++ for protocol in protocols: ++ client_ctx, server_ctx, hostname = testing_context() ++ client_ctx.load_cert_chain(SIGNED_CERTFILE) ++ self.assertEqual(client_ctx.post_handshake_auth, None) ++ with self.assertRaises(AttributeError): ++ client_ctx.post_handshake_auth = True ++ with self.assertRaises(AttributeError): ++ server_ctx.post_handshake_auth = True ++ ++ with ThreadedEchoServer(context=server_ctx) as server: ++ with client_ctx.wrap_socket( ++ socket.socket(), ++ server_hostname=hostname ++ ) as ssock: ++ ssock.connect((HOST, server.port)) ++ with self.assertRaises(NotImplementedError): ++ ssock.verify_client_post_handshake() ++ ++ + HAS_KEYLOG = hasattr(ssl.SSLContext, 'keylog_filename') + requires_keylog = unittest.skipUnless( + HAS_KEYLOG, 'test requires OpenSSL 1.1.1 with keylog callback') +diff --git a/Modules/Setup b/Modules/Setup +index 8ad9a5aebb..6138085c61 100644 +--- a/Modules/Setup ++++ b/Modules/Setup +@@ -208,11 +208,11 @@ PYTHONPATH=$(COREPYTHONPATH) + #_hashlib _hashopenssl.c $(OPENSSL_INCLUDES) $(OPENSSL_LDFLAGS) -lcrypto + + # To statically link OpenSSL: +-# _ssl _ssl.c $(OPENSSL_INCLUDES) $(OPENSSL_LDFLAGS) \ +-# -l:libssl.a -Wl,--exclude-libs,libssl.a \ +-# -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a +-# _hashlib _hashopenssl.c $(OPENSSL_INCLUDES) $(OPENSSL_LDFLAGS) \ +-# -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a ++_ssl _ssl.c $(OPENSSL_INCLUDES) $(OPENSSL_LDFLAGS) \ ++ -l:libssl.a -Wl,--exclude-libs,libssl.a \ ++ -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a ++_hashlib _hashopenssl.c $(OPENSSL_INCLUDES) $(OPENSSL_LDFLAGS) \ ++ -l:libcrypto.a -Wl,--exclude-libs,libcrypto.a + + # The _tkinter module. + # +diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c +index 0e230f332f..390d6e0ed9 100644 +--- a/Modules/_hashopenssl.c ++++ b/Modules/_hashopenssl.c +@@ -130,8 +130,12 @@ static const py_hashentry_t py_hashes[] = { + PY_HASH_ENTRY(Py_hash_shake_128, NULL, SN_shake128, NID_shake128), + PY_HASH_ENTRY(Py_hash_shake_256, NULL, SN_shake256, NID_shake256), + /* blake2 digest */ ++#if defined(NID_blake2s256) + PY_HASH_ENTRY(Py_hash_blake2s, "blake2s256", SN_blake2s256, NID_blake2s256), ++#endif ++#if defined(NID_blake2b512) + PY_HASH_ENTRY(Py_hash_blake2b, "blake2b512", SN_blake2b512, NID_blake2b512), ++#endif + PY_HASH_ENTRY(NULL, NULL, NULL, 0), + }; + +diff --git a/Modules/_ssl.c b/Modules/_ssl.c +index bc30290942..e0e59b97e5 100644 +--- a/Modules/_ssl.c ++++ b/Modules/_ssl.c +@@ -187,6 +187,13 @@ extern const SSL_METHOD *TLSv1_2_method(void); + #endif + + ++ ++#if defined(OPENSSL_IS_AWSLC) || !defined(TLS1_3_VERSION) || defined(OPENSSL_NO_TLS1_3) ++ #define PY_SSL_NO_POST_HS_AUTH ++ #define OPENSSL_NO_PSK ++#endif ++ ++ + enum py_ssl_error { + /* these mirror ssl.h */ + PY_SSL_ERROR_NONE, +@@ -231,7 +238,7 @@ enum py_proto_version { + PY_PROTO_TLSv1 = TLS1_VERSION, + PY_PROTO_TLSv1_1 = TLS1_1_VERSION, + PY_PROTO_TLSv1_2 = TLS1_2_VERSION, +-#ifdef TLS1_3_VERSION ++#if defined(TLS1_3_VERSION) + PY_PROTO_TLSv1_3 = TLS1_3_VERSION, + #else + PY_PROTO_TLSv1_3 = 0x304, +@@ -293,7 +300,7 @@ typedef struct { + */ + unsigned int hostflags; + int protocol; +-#ifdef TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + int post_handshake_auth; + #endif + PyObject *msg_cb; +@@ -885,7 +892,7 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, + SSL_set_mode(self->ssl, + SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER | SSL_MODE_AUTO_RETRY); + +-#ifdef TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + if (sslctx->post_handshake_auth == 1) { + if (socket_type == PY_SSL_SERVER) { + /* bpo-37428: OpenSSL does not ignore SSL_VERIFY_POST_HANDSHAKE. +@@ -1028,6 +1035,7 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) + } while (err.ssl == SSL_ERROR_WANT_READ || + err.ssl == SSL_ERROR_WANT_WRITE); + Py_XDECREF(sock); ++ + if (ret < 1) + return PySSL_SetError(self, ret, __FILE__, __LINE__); + if (PySSL_ChainExceptions(self) < 0) +@@ -2060,7 +2068,12 @@ _ssl__SSLSocket_shared_ciphers_impl(PySSLSocket *self) + len = 0; + for (i = 0; i < sk_SSL_CIPHER_num(server_ciphers); i++) { + cipher = sk_SSL_CIPHER_value(server_ciphers, i); ++#if defined(OPENSSL_IS_AWSLC) ++ size_t unused_idx; ++ if (sk_SSL_CIPHER_find(client_ciphers, &unused_idx, cipher) == 0) ++#else + if (sk_SSL_CIPHER_find(client_ciphers, cipher) < 0) ++#endif + continue; + + PyObject *tup = cipher_to_tuple(cipher); +@@ -2787,7 +2800,7 @@ static PyObject * + _ssl__SSLSocket_verify_client_post_handshake_impl(PySSLSocket *self) + /*[clinic end generated code: output=532147f3b1341425 input=6bfa874810a3d889]*/ + { +-#ifdef TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + int err = SSL_verify_client_post_handshake(self->ssl); + if (err == 0) + return _setSSLError(get_state_sock(self), NULL, 0, __FILE__, __LINE__); +@@ -3203,7 +3216,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) + + /* Set SSL_MODE_RELEASE_BUFFERS. This potentially greatly reduces memory + usage for no cost at all. */ +- SSL_CTX_set_mode(self->ctx, SSL_MODE_RELEASE_BUFFERS); ++ SSL_CTX_set_mode(self->ctx, SSL_MODE_RELEASE_BUFFERS | SSL_MODE_AUTO_RETRY); + + params = SSL_CTX_get0_param(self->ctx); + /* Improve trust chain building when cross-signed intermediate +@@ -3211,7 +3224,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) + X509_VERIFY_PARAM_set_flags(params, X509_V_FLAG_TRUSTED_FIRST); + X509_VERIFY_PARAM_set_hostflags(params, self->hostflags); + +-#ifdef TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + self->post_handshake_auth = 0; + SSL_CTX_set_post_handshake_auth(self->ctx, self->post_handshake_auth); + #endif +@@ -3589,7 +3602,7 @@ set_maximum_version(PySSLContext *self, PyObject *arg, void *c) + return set_min_max_proto_version(self, arg, 1); + } + +-#ifdef TLS1_3_VERSION ++#if defined(TLS1_3_VERSION) && !defined(OPENSSL_NO_TLS1_3) + static PyObject * + get_num_tickets(PySSLContext *self, void *c) + { +@@ -3620,7 +3633,7 @@ set_num_tickets(PySSLContext *self, PyObject *arg, void *c) + + PyDoc_STRVAR(PySSLContext_num_tickets_doc, + "Control the number of TLSv1.3 session tickets"); +-#endif /* TLS1_3_VERSION */ ++#endif /* defined(TLS1_3_VERSION) */ + + static PyObject * + get_security_level(PySSLContext *self, void *c) +@@ -3723,14 +3736,14 @@ set_check_hostname(PySSLContext *self, PyObject *arg, void *c) + + static PyObject * + get_post_handshake_auth(PySSLContext *self, void *c) { +-#if TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + return PyBool_FromLong(self->post_handshake_auth); + #else + Py_RETURN_NONE; + #endif + } + +-#if TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + static int + set_post_handshake_auth(PySSLContext *self, PyObject *arg, void *c) { + if (arg == NULL) { +@@ -4917,14 +4930,14 @@ static PyGetSetDef context_getsetlist[] = { + (setter) _PySSLContext_set_msg_callback, NULL}, + {"sni_callback", (getter) get_sni_callback, + (setter) set_sni_callback, PySSLContext_sni_callback_doc}, +-#ifdef TLS1_3_VERSION ++#if defined(TLS1_3_VERSION) && !defined(OPENSSL_NO_TLS1_3) + {"num_tickets", (getter) get_num_tickets, + (setter) set_num_tickets, PySSLContext_num_tickets_doc}, + #endif + {"options", (getter) get_options, + (setter) set_options, NULL}, + {"post_handshake_auth", (getter) get_post_handshake_auth, +-#ifdef TLS1_3_VERSION ++#if !defined(PY_SSL_NO_POST_HS_AUTH) + (setter) set_post_handshake_auth, + #else + NULL, diff --git a/tests/ci/integration/run_python_integration.sh b/tests/ci/integration/run_python_integration.sh new file mode 100755 index 0000000000..168cd91403 --- /dev/null +++ b/tests/ci/integration/run_python_integration.sh @@ -0,0 +1,139 @@ +#!/bin/bash -exu +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 OR ISC + +source tests/ci/common_posix_setup.sh + +set -exuo pipefail + +# Set up environment. + +# SYS_ROOT +# - SRC_ROOT(aws-lc) +# - SCRATCH_FOLDER +# - PYTHON_SRC_FOLDER +# - 3.10 +# ... +# - PYTHON_PATCH_FOLDER +# - 3.10 +# ... +# - AWS_LC_BUILD_FOLDER +# - AWS_LC_INSTALL_FOLDER + +# Assumes script is executed from the root of aws-lc directory +SCRATCH_FOLDER="${SRC_ROOT}/PYTHON_BUILD_ROOT" +PYTHON_SRC_FOLDER="${SCRATCH_FOLDER}/python-src" +PYTHON_PATCH_FOLDER="${SRC_ROOT}/tests/ci/integration/python_patch" +AWS_LC_BUILD_FOLDER="${SCRATCH_FOLDER}/aws-lc-build" +AWS_LC_INSTALL_FOLDER="${SCRATCH_FOLDER}/aws-lc-install" + +function python_build() { + local branch=${1} + pushd ${branch} + ./configure \ + --with-openssl=${AWS_LC_INSTALL_FOLDER} \ + --with-builtin-hashlib-hashes=blake2 \ + --with-ssl-default-suites=openssl + make -j ${NUM_CPU_THREADS} + popd +} + +function python_run_tests() { + local branch=${1} + pushd ${branch} + # We statically link, so need to call into python itself to assert that we're + # correctly built against AWS-LC + ./python -c 'import ssl; print(ssl.OPENSSL_VERSION)' | grep AWS-LC + # see https://github.com/pypa/setuptools/issues/3007 + export SETUPTOOLS_USE_DISTUTILS=stdlib + # A number of python module tests fail on our public CI images, but they're + # all unrelated to AWS-LC and the ssl module. So, restrict the module tests + # we run to those relevant to our CPython integration. + local TEST_OPTS="\ + test_asyncio \ + test_audit \ + test_ftplib \ + test_hashlib \ + test_httplib \ + test_imaplib \ + test_logging \ + test_poplib \ + test_site \ + test_smtpnet \ + test_ssl \ + test_urllib \ + test_urllib2_localnet \ + test_xmlrpc \ + " + make -j ${NUM_CPU_THREADS} test TESTOPTS="${TEST_OPTS}" + popd +} + +# The per-branch patch files do a few things: +# +# - Modify various unit tests to account for error string differences between +# OpenSSL and AWS-LC. +# - In |test_bio|handshake|, check whether protocol is TLSv1.3 before testing +# tls-unique channel binding behavior, as channel bindings are not defined +# on that protocol +# - Skip FFDH(E)-reliant tests, as AWS-LC doesn't support FFDH(E) +# - Skip post handshake authentication tests, as AWS-LC doesn't support that +# - Add test specifically for AWS-LC to codify the ssl module's behavior when +# caller attempts to use post-handshake authentication +# - For 3.10, modify Modules/Setup to point to the AWS-LC installation dir +# - Modify the hashlib module's backing C code to only register BLAKE +# functions if the expected NID is available in linked libcrypto +# - Modify the ssl module's backing C code to separate notions of supporting +# TLSv1.3 from supporting post-handshake authentication as some libraries +# (namely AWS-LC) support TLSv1.3, but not the post-handshake +# authentication portion of that protocol. +# - Modify the ssl module's backing C code to account for AWS-LC's divergent +# function signature and return value for |sk_SSL_CIPHER_find| +# - Modify the ssl module's backing C code to set |SSL_MODE_AUTO_RETRY| in +# all calls to |SSL{_CTX}_set_mode| +# +# TODO: Remove these patches when we make an upstream contribution. +function python_patch() { + local branch=${1} + local src_dir="${PYTHON_SRC_FOLDER}/${branch}" + local patch_dir="${PYTHON_PATCH_FOLDER}/${branch}" + git clone https://github.com/python/cpython.git ${src_dir} \ + --depth 1 \ + --branch ${branch} + for patchfile in $(find -L ${patch_dir} -type f -name '*.patch'); do + echo "Apply patch ${patchfile}..." + cat ${patchfile} \ + | sed -e "s|AWS_LC_INSTALL_PLACEHOLDER|${AWS_LC_INSTALL_FOLDER}|g" \ + | patch -p1 --quiet -d ${src_dir} + done +} + +mkdir -p ${SCRATCH_FOLDER} +rm -rf ${SCRATCH_FOLDER}/* +cd ${SCRATCH_FOLDER} + +mkdir -p ${AWS_LC_BUILD_FOLDER} ${AWS_LC_INSTALL_FOLDER} + +aws_lc_build ${SRC_ROOT} ${AWS_LC_BUILD_FOLDER} ${AWS_LC_INSTALL_FOLDER} \ + -DBUILD_TESTING=OFF \ + -DBUILD_SHARED_LIBS=0 + +# Some systems install under "lib64" instead of "lib" +ln -s ${AWS_LC_INSTALL_FOLDER}/lib64 ${AWS_LC_INSTALL_FOLDER}/lib + +mkdir -p ${PYTHON_SRC_FOLDER} +pushd ${PYTHON_SRC_FOLDER} + +# Some environments disable IPv6 by default +which sysctl && ( sysctl -w net.ipv6.conf.all.disable_ipv6=0 || /bin/true ) +echo 0 >/proc/sys/net/ipv6/conf/all/disable_ipv6 || /bin/true + +# NOTE: cpython keeps a unique branch per version, add version branches here +# NOTE: As we add more versions to support, we may want to parallelize here +for branch in 3.10 3.11 3.12 main; do + python_patch ${branch} + python_build ${branch} + python_run_tests ${branch} +done + +popd