fix(pymysql): use pymysql instead of mysql-client (#966)

src/dashboard/Dockerfile

@@ -2,7 +2,7 @@ ARG IMAGE=python
 ARG TAG=3.10.12-slim-bullseye
 FROM ${IMAGE}:${TAG}
 
-RUN apt-get update && apt-get install -y libssl-dev default-libmysqlclient-dev libffi-dev \
+RUN apt-get update && apt-get install -y libssl-dev libffi-dev \
     build-essential libc6-dev libjpeg-dev zlib1g-dev vim procps default-mysql-client curl
 
 ARG PYPI="https://pypi.org/simple/"

src/dashboard/apigateway/apigateway/conf/default.py

@@ -31,6 +31,7 @@
 from typing import List
 from urllib.parse import quote
 
+import pymysql
 from celery.schedules import crontab
 from django.core.exceptions import ImproperlyConfigured
 from django.utils.encoding import force_bytes
@@ -41,6 +42,10 @@
 from apigateway.conf.log_utils import get_logging_config, makedirs_when_not_exists
 from apigateway.conf.utils import get_default_keepalive_options
 
+pymysql.install_as_MySQLdb()
+# Patch version info to forcedly pass Django client check
+pymysql.version_info = 1, 4, 2, "final", 0
+
 env = Env()
 
 

src/dashboard/apigateway/apigateway/tracing/instrumentor.py

@@ -161,11 +161,11 @@ def _instrument(self, **kwargs):
             print("otel instructment: celery")
 
         if getattr(settings, "OTEL_INSTRUMENT_DB_API", False):
-            import MySQLdb  # noqa
+            import pymysql  # noqa
 
             dbapi.wrap_connect(
                 __name__,
-                MySQLdb,
+                pymysql.connect,
                 "connect",
                 "mysql",
                 {"database": "db", "port": "port", "host": "host", "user": "user"},

src/dashboard/apigateway/requirements.txt

@@ -55,7 +55,6 @@ jsonfield==3.1.0 ; python_version >= "3.10" and python_version < "3.11"
 jsonschema==3.2.0 ; python_version >= "3.10" and python_version < "3.11"
 kombu==4.6.11 ; python_version >= "3.10" and python_version < "3.11"
 markupsafe==2.1.5 ; python_version >= "3.10" and python_version < "3.11"
-mysqlclient==2.1.1 ; python_version >= "3.10" and python_version < "3.11"
 opentelemetry-api==1.24.0 ; python_version >= "3.10" and python_version < "3.11"
 opentelemetry-exporter-jaeger-proto-grpc==1.21.0 ; python_version >= "3.10" and python_version < "3.11"
 opentelemetry-exporter-jaeger-thrift==1.21.0 ; python_version >= "3.10" and python_version < "3.11"
@@ -85,6 +84,7 @@ pycryptodomex==3.19.1 ; python_version >= "3.10" and python_version < "3.11"
 pydantic==1.9.2 ; python_version >= "3.10" and python_version < "3.11"
 pygments==2.17.2 ; python_version >= "3.10" and python_version < "3.11"
 pyjwt==1.7.1 ; python_version >= "3.10" and python_version < "3.11"
+pymysql==1.1.1 ; python_version >= "3.10" and python_version < "3.11"
 pypi-simple==0.8.0 ; python_version >= "3.10" and python_version < "3.11"
 pyrsistent==0.18.0 ; python_version >= "3.10" and python_version < "3.11"
 python-crontab==2.6.0 ; python_version >= "3.10" and python_version < "3.11"

src/dashboard/apigateway/requirements_dev.txt

@@ -82,7 +82,6 @@ matplotlib-inline==0.1.6 ; python_version >= "3.10" and python_version < "3.11"
 mdurl==0.1.2 ; python_version >= "3.10" and python_version < "3.11"
 mypy-extensions==1.0.0 ; python_version >= "3.10" and python_version < "3.11"
 mypy==1.9.0 ; python_version >= "3.10" and python_version < "3.11"
-mysqlclient==2.1.1 ; python_version >= "3.10" and python_version < "3.11"
 nodeenv==1.6.0 ; python_version >= "3.10" and python_version < "3.11"
 nose==1.3.7 ; python_version >= "3.10" and python_version < "3.11"
 opentelemetry-api==1.24.0 ; python_version >= "3.10" and python_version < "3.11"
@@ -123,6 +122,7 @@ pycryptodomex==3.19.1 ; python_version >= "3.10" and python_version < "3.11"
 pydantic==1.9.2 ; python_version >= "3.10" and python_version < "3.11"
 pygments==2.17.2 ; python_version >= "3.10" and python_version < "3.11"
 pyjwt==1.7.1 ; python_version >= "3.10" and python_version < "3.11"
+pymysql==1.1.1 ; python_version >= "3.10" and python_version < "3.11"
 pypi-simple==0.8.0 ; python_version >= "3.10" and python_version < "3.11"
 pyproject-api==1.6.1 ; python_version >= "3.10" and python_version < "3.11"
 pyrsistent==0.18.0 ; python_version >= "3.10" and python_version < "3.11"
@@ -165,6 +165,7 @@ tox==4.14.1 ; python_version >= "3.10" and python_version < "3.11"
 traitlets==5.14.2 ; python_version >= "3.10" and python_version < "3.11"
 types-cachetools==5.3.0.7 ; python_version >= "3.10" and python_version < "3.11"
 types-docutils==0.20.0.20240317 ; python_version >= "3.10" and python_version < "3.11"
+types-pymysql==1.1.0.20240524 ; python_version >= "3.10" and python_version < "3.11"
 types-python-dateutil==2.9.0.20240316 ; python_version >= "3.10" and python_version < "3.11"
 types-pytz==2024.1.0.20240203 ; python_version >= "3.10" and python_version < "3.11"
 types-pyyaml==6.0.12.20240311 ; python_version >= "3.10" and python_version < "3.11"

src/dashboard/pyproject.toml

@@ -8,7 +8,7 @@ authors = []
 python = ">=3.10,<3.11"
 django = "3.2.25"
 djangorestframework = "3.14.0"
-mysqlclient = "2.1.1"
+pymysql = "1.1.1"
 # celery = "5.1.2"
 celery = "4.4.7"
 # django-celery-beat = "2.2.1"
@@ -107,6 +107,7 @@ types-cachetools = "5.3.0.7"
 types-pytz = "2024.1.0.20240203"
 types-docutils = "0.20.0.20240317"
 types-PyYAML = "6.0.12.20240311"
+types-PyMySQL = "1.1.0.20240524"
 import-linter = "^2.0"
 
 [[tool.poetry.source]]

src/esb/Dockerfile

@@ -2,7 +2,7 @@ ARG IMAGE=python
 ARG TAG=3.6.15-slim-buster
 FROM ${IMAGE}:${TAG}
 
-RUN apt-get update && apt-get install -y libssl-dev default-libmysqlclient-dev libffi-dev \
+RUN apt-get update && apt-get install -y libssl-dev libffi-dev \
     libc6-dev build-essential vim procps
 
 ADD build /app
@@ -12,4 +12,4 @@ ARG PYPI="https://pypi.org/simple/"
 RUN pip config set global.index-url "${PYPI}"
 RUN pip install -r requirements.txt
 
-CMD ["bash", "/app/bin/start.sh"]
+CMD ["bash", "/app/bin/start.sh"]

src/esb/esb/esb/__init__.py

@@ -19,9 +19,9 @@
 
 from django.conf import settings
 
-from common.djmysql_pool import patch_mysql
-
 if getattr(settings, "MYSQL_POOL_ENABLED", True):
-    patch_mysql(pool_options=settings.DJ_POOL_OPTIONS)
+    # 2024-09-24 use pymysql instead of mysql-client, so can't patch mysql
+    pass
+    # patch_mysql(pool_options=settings.DJ_POOL_OPTIONS)
 
 default_app_config = "esb.apps.ESBAppConfig"

src/esb/esb/requirements.txt

@@ -29,7 +29,6 @@ jinja2==2.11.3 ; python_full_version >= "3.6.2" and python_version < "3.8"
 jsonfield==2.0.1 ; python_full_version >= "3.6.2" and python_version < "3.8"
 markdown==3.3.7 ; python_full_version >= "3.6.2" and python_version < "3.8"
 markupsafe==1.1.1 ; python_full_version >= "3.6.2" and python_version < "3.8"
-mysqlclient==2.0.1 ; python_full_version >= "3.6.2" and python_version < "3.8"
 packaging==21.2 ; python_full_version >= "3.6.2" and python_version < "3.8"
 portalocker==2.7.0 ; python_full_version >= "3.6.2" and python_version < "3.8"
 prometheus-client==0.12.0 ; python_full_version >= "3.6.2" and python_version < "3.8"
@@ -38,6 +37,7 @@ pycparser==2.20 ; python_full_version >= "3.6.2" and python_version < "3.8"
 pydantic==1.8.2 ; python_full_version >= "3.6.2" and python_version < "3.8"
 pygments==2.1.3 ; python_full_version >= "3.6.2" and python_version < "3.8"
 pyjwt==1.7.1 ; python_full_version >= "3.6.2" and python_version < "3.8"
+pymysql==1.0.2 ; python_full_version >= "3.6.2" and python_version < "3.8"
 pyparsing==2.4.7 ; python_full_version >= "3.6.2" and python_version < "3.8"
 pypi-simple==0.8.0 ; python_full_version >= "3.6.2" and python_version < "3.8"
 python-dateutil==2.8.1 ; python_full_version >= "3.6.2" and python_version < "3.8"

src/esb/esb/requirements_dev.txt

@@ -18,7 +18,7 @@ cffi==1.15.0 ; python_full_version >= "3.6.2" and python_version < "3.8"
 cfgv==3.3.1 ; python_full_version >= "3.6.2" and python_version < "3.8"
 charset-normalizer==2.0.12 ; python_full_version >= "3.6.2" and python_version < "3.8"
 click==8.0.3 ; python_full_version >= "3.6.2" and python_version < "3.8"
-colorama==0.4.4 ; python_full_version >= "3.6.2" and python_version < "3.8" and (sys_platform == "win32" or platform_system == "Windows")
+colorama==0.4.4 ; python_full_version >= "3.6.2" and python_version < "3.8" and sys_platform == "win32" or python_full_version >= "3.6.2" and python_version < "3.8" and platform_system == "Windows"
 concurrent-log-handler==0.9.24 ; python_full_version >= "3.6.2" and python_version < "3.8"
 coverage==4.5.4 ; python_full_version >= "3.6.2" and python_version < "3.8"
 cryptography==3.4.8 ; python_full_version >= "3.6.2" and python_version < "3.8"
@@ -54,7 +54,6 @@ mock==3.0.5 ; python_full_version >= "3.6.2" and python_version < "3.8"
 more-itertools==8.10.0 ; python_full_version >= "3.6.2" and python_version < "3.8"
 mypy-extensions==0.4.3 ; python_full_version >= "3.6.2" and python_version < "3.8"
 mypy==0.910 ; python_full_version >= "3.6.2" and python_version < "3.8"
-mysqlclient==2.0.1 ; python_full_version >= "3.6.2" and python_version < "3.8"
 nodeenv==1.6.0 ; python_full_version >= "3.6.2" and python_version < "3.8"
 packaging==21.2 ; python_full_version >= "3.6.2" and python_version < "3.8"
 parso==0.8.2 ; python_full_version >= "3.6.2" and python_version < "3.8"
@@ -77,6 +76,7 @@ pydantic==1.8.2 ; python_full_version >= "3.6.2" and python_version < "3.8"
 pyflakes==2.2.0 ; python_full_version >= "3.6.2" and python_version < "3.8"
 pygments==2.1.3 ; python_full_version >= "3.6.2" and python_version < "3.8"
 pyjwt==1.7.1 ; python_full_version >= "3.6.2" and python_version < "3.8"
+pymysql==1.0.2 ; python_full_version >= "3.6.2" and python_version < "3.8"
 pyparsing==2.4.7 ; python_full_version >= "3.6.2" and python_version < "3.8"
 pypi-simple==0.8.0 ; python_full_version >= "3.6.2" and python_version < "3.8"
 pytest-benchmark==3.2.3 ; python_full_version >= "3.6.2" and python_version < "3.8"
@@ -107,6 +107,7 @@ typed-ast==1.4.3 ; python_full_version >= "3.6.2" and python_version < "3.8"
 types-cachetools==5.3.0.5 ; python_full_version >= "3.6.2" and python_version < "3.8"
 types-chardet==5.0.4.6 ; python_full_version >= "3.6.2" and python_version < "3.8"
 types-markdown==3.4.2.9 ; python_full_version >= "3.6.2" and python_version < "3.8"
+types-pymysql==1.1.0.1 ; python_full_version >= "3.6.2" and python_version < "3.8"
 types-pyyaml==6.0.12.9 ; python_full_version >= "3.6.2" and python_version < "3.8"
 types-requests==2.30.0.0 ; python_full_version >= "3.6.2" and python_version < "3.8"
 types-urllib3==1.26.25.13 ; python_full_version >= "3.6.2" and python_version < "3.8"