feat!: upgrade to Python 3.12 (#824)

* feat!: upgrade to Python 3.12 * build: upgrade Python in Dockerfile and configs * build: update sql volume in Dockerfile * build: update Dockerfile * build: update Dockerfile * build: update Dockerfile * build: update Dockerfile * build: update Dockerfile * build: update Dockerfile * build: update Dockerfile * build: update Dockerfile * build: update Dockerfile * build: update Dockerfile * build: update Dockerfile * build: update Dockerfile * build: update Dockerfile * build: update Dockerfile * build: update Dockerfile * build: update Dockerfile * build: update Dockerfile * build: update Dockerfile * fix: fix quality checks * chore: remove edx-lint and pylint constraints * chore: remove edx-lint and pylint constraints * fix: update pylintrc * fix: fix quality checks * fix: fix dict literal syntax * fix: fix quality checks
openedx · May 21, 2024 · 0133c8f · 0133c8f
1 parent 718fa10
commit 0133c8f
Show file tree

Hide file tree

Showing 24 changed files with 115 additions and 1,887 deletions.
diff --git a/.github/docker-compose-github.yml b/.github/docker-compose-github.yml
@@ -1,14 +1,14 @@
 version: "2.1"
 services:
   mysql:
-    image: mysql:5.7
+    image: mysql:8.0.28-oracle
     container_name: enterprise.catalog.mysql
     environment:
       MYSQL_ROOT_PASSWORD: ""
       MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
       MYSQL_DATABASE: "enterprise_catalog"
     volumes:
-      - enterprise_catalog_mysql:/var/lib/mysql
+      - enterprise_catalog_mysql8:/var/lib/mysql
 
   app:
     image: openedx/enterprise-catalog
@@ -32,5 +32,5 @@ services:
       DJANGO_SETTINGS_MODULE: enterprise_catalog.settings.test
 
 volumes:
-  enterprise_catalog_mysql:
+  enterprise_catalog_mysql8:
     driver: local
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.8', '3.12']
+        python-version: ['3.12']
         django-version: [django42]
 
     steps:
@@ -25,11 +25,7 @@ jobs:
         docker-compose -f .github/docker-compose-github.yml up -d
     - name: Install test dependencies and run validation
       run: |
-        if [[ "${{ matrix.python-version }}" == "3.8" ]]; then
-        docker exec -e TOXENV=py38-${{ matrix.django-version }} -u root enterprise.catalog.app /edx/app/enterprise_catalog/enterprise_catalog/validate.sh
-        elif [[ "${{ matrix.python-version }}" == "3.12" ]]; then
         docker exec -e TOXENV=py312-${{ matrix.django-version }} -u root enterprise.catalog.app /edx/app/enterprise_catalog/enterprise_catalog/validate.sh
-        fi
     - name: Code Coverage
-      if: matrix.python-version == '3.8' && matrix.django-version=='django42'
+      if: matrix.python-version == '3.12' && matrix.django-version=='django42'
       uses: codecov/codecov-action@v1
diff --git a/.github/workflows/mysql-migrations-check.yml b/.github/workflows/mysql-migrations-check.yml
@@ -15,7 +15,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-20.04 ]
-        python-version: [ 3.8 ]
+        python-version: [ 3.12 ]
 
     steps:
     - name: Checkout repo

diff --git a/.github/workflows/upgrade-python-requirements.yml b/.github/workflows/upgrade-python-requirements.yml
@@ -18,6 +18,7 @@ jobs:
       branch: ${{ github.event.inputs.branch || 'master' }}
       team_reviewers: "enterprise-quokkas"
       email_address: "[email protected]"
+      python_version: 3.12
     secrets:
       requirements_bot_github_token: ${{ secrets.REQUIREMENTS_BOT_GITHUB_TOKEN }}
       requirements_bot_github_email: ${{ secrets.REQUIREMENTS_BOT_GITHUB_EMAIL }}

diff --git a/Dockerfile b/Dockerfile
@@ -8,9 +8,6 @@ MAINTAINER [email protected]
 # language-pack-en locales
 #     ubuntu locale support so that system utilities have a consistent
 #     language and time zone.
-# python3.5
-#     ubuntu doesnt ship with python, so this is the python we will use to run the
-#     application
 # python3-pip
 #     install pip to install application requirements.txt files
 # pkg-config
@@ -21,36 +18,42 @@ MAINTAINER [email protected]
 #     to install header files needed to use native C implementation for
 #     MySQL-python for performance gains.
 
+ARG PYTHON_VERSION=3.12
+ENV TZ=UTC
+ENV TERM=xterm-256color
+ENV DEBIAN_FRONTEND=noninteractive
+
 # If you add a package here please include a comment above describing what it is used for
+RUN apt-get update && \
+  apt-get install -y software-properties-common && \
+  apt-add-repository -y ppa:deadsnakes/ppa
+
 RUN apt-get update && apt-get -qy install --no-install-recommends \
+ build-essential \
  language-pack-en \
  locales \
- python3.8 \
- python3-pip \
- python3.8-venv \
- python3.8-dev \
  pkg-config \
  libmysqlclient-dev \
  libssl-dev \
- build-essential \
+ libffi-dev \
+ libsqlite3-dev \
  git \
- wget
+ wget \
+ python3.12 \
+ python3.12-dev \
+ python3.12-distutils \
+ python3-pip
 
-ENV TZ=UTC
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
-RUN apt-get update && apt-get -qy install  libffi-dev libsqlite3-dev python3-distutils
-
-RUN apt-get update && apt-get install -y build-essential wget && \
-wget https://www.python.org/ftp/python/3.12.0/Python-3.12.0.tgz && \
-tar -xzvf Python-3.12.0.tgz && cd Python-3.12.0 && ./configure --enable-optimizations && \
-make -j$(nproc) && make altinstall && python3.12 --version
+RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION}
+RUN pip install virtualenv
 
 ENV VIRTUAL_ENV=/venv
-RUN python3.8 -m venv $VIRTUAL_ENV
+RUN virtualenv -p python$PYTHON_VERSION $VIRTUAL_ENV
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
-RUN pip install pip==20.2.3 setuptools==50.3.0
+RUN pip install pip==24.0 setuptools==69.5.1
 
 RUN locale-gen en_US.UTF-8
 ENV LANG en_US.UTF-8

diff --git a/Makefile b/Makefile
@@ -47,10 +47,6 @@ piptools: ## install pinned version of pip-compile and pip-sync
 
 requirements: piptools dev_requirements ## sync to default requirements
 
-requirements312:  ## sync to requirements for local development with Python 3.12
-	pip3.12 install -qr requirements/pip-tools.txt
-	pip3.12 install -qr requirements/dev312.txt
-
 dev_requirements: ## sync to requirements for local development
 	pip-sync -q requirements/dev.txt
 

diff --git a/enterprise_catalog/apps/academy/tests/factories.py b/enterprise_catalog/apps/academy/tests/factories.py
@@ -24,7 +24,7 @@ class Meta:
     def content_metadata(self, create, extracted, **kwargs):  # pylint: disable=unused-argument
         content_metadata1 = ContentMetadataFactory.create(content_type=COURSE)
         content_metadata2 = ContentMetadataFactory.create(content_type=COURSE)
-        self.content_metadata.set([content_metadata1, content_metadata2])  # pylint: disable=no-member
+        self.content_metadata.set([content_metadata1, content_metadata2])
 
 
 class AcademyFactory(factory.django.DjangoModelFactory):
@@ -43,18 +43,18 @@ class Meta:
     def enterprise_catalogs(self, create, extracted, **kwargs):  # pylint: disable=unused-argument
         if extracted:
             for enterprise_catalog in extracted:
-                self.enterprise_catalogs.add(enterprise_catalog)  # pylint: disable=no-member
+                self.enterprise_catalogs.add(enterprise_catalog)
         else:
             enterprise_catalog1 = EnterpriseCatalogFactory()
             enterprise_catalog2 = EnterpriseCatalogFactory()
-            self.enterprise_catalogs.set([enterprise_catalog1, enterprise_catalog2])  # pylint: disable=no-member
+            self.enterprise_catalogs.set([enterprise_catalog1, enterprise_catalog2])
 
     @factory.post_generation
     def tags(self, create, extracted, **kwargs):  # pylint: disable=unused-argument
         if extracted:
             for tag in extracted:
-                self.tags.add(tag)  # pylint: disable=no-member
+                self.tags.add(tag)
         else:
             tag1 = TagFactory()
             tag2 = TagFactory()
-            self.tags.set([tag1, tag2])  # pylint: disable=no-member
+            self.tags.set([tag1, tag2])
diff --git a/enterprise_catalog/apps/ai_curation/tests/test_utils.py b/enterprise_catalog/apps/ai_curation/tests/test_utils.py
@@ -23,9 +23,7 @@
 )
 
 
-CHAT_COMPLETIONS_API_KEYWARGS = dict(
-    model='gpt-4', temperature=0.3, max_tokens=500
-)
+CHAT_COMPLETIONS_API_KEYWARGS = {"model": 'gpt-4', "temperature": 0.3, "max_tokens": 500}
 
 
 class TestUtils(TestCase):

diff --git a/enterprise_catalog/apps/api/tasks.py b/enterprise_catalog/apps/api/tasks.py
@@ -1127,14 +1127,14 @@ def update_catalog_metadata_task(self, catalog_query_id, force=False, dry_run=Fa
     except Exception as e:
         logger.exception(
             f'Something went wrong while updating content metadata from discovery using catalog: {catalog_query_id} '
-            f'after update_catalog_metadata_task_seconds={time.perf_counter()-start_time} seconds',
+            f'after update_catalog_metadata_task_seconds={time.perf_counter() - start_time} seconds',
             exc_info=e,
         )
         raise e
     logger.info(
         f'Finished update_catalog_metadata_task with {len(associated_content_keys)} '
         f'associated content keys for catalog {catalog_query_id} '
-        f'after update_catalog_metadata_task_seconds={time.perf_counter()-start_time} seconds'
+        f'after update_catalog_metadata_task_seconds={time.perf_counter() - start_time} seconds'
     )
 
 

diff --git a/pylintrc b/pylintrc
@@ -64,7 +64,7 @@
 # SERIOUSLY.
 #
 # ------------------------------
-# Generated by edx-lint version: 5.2.5
+# Generated by edx-lint version: 5.3.6
 # ------------------------------
 [MASTER]
 ignore = ,migrations, settings, wsgi.py
@@ -259,6 +259,7 @@ enable =
 	useless-suppression,
 disable = 
 	bad-indentation,
+	broad-exception-raised,
 	consider-using-f-string,
 	duplicate-code,
 	file-ignored,
@@ -380,6 +381,6 @@ ext-import-graph =
 int-import-graph = 
 
 [EXCEPTIONS]
-overgeneral-exceptions = Exception
+overgeneral-exceptions = builtins.Exception
 
-# b86606bb77e6a87fcf5f6893592d35d432a507f5
+# 43c0dc058c537a12f0149a84a44e307cd064feda
diff --git a/requirements/base.txt b/requirements/base.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.8
+# This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
 #    make upgrade
@@ -27,13 +27,6 @@ attrs==23.2.0
     #   referencing
 backoff==1.10.0
     # via analytics-python
-backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9"
-    # via
-    #   -c requirements/constraints.txt
-    #   celery
-    #   django
-    #   djangorestframework
-    #   kombu
 billiard==4.2.0
     # via celery
 celery==5.4.0
@@ -181,8 +174,6 @@ edx-rest-api-client==5.7.0
     # via -r requirements/base.in
 edx-toggles==5.2.0
     # via -r requirements/base.in
-exceptiongroup==1.2.1
-    # via anyio
 h11==0.14.0
     # via httpcore
 httpcore==1.0.5
@@ -194,10 +185,6 @@ idna==3.7
     #   anyio
     #   httpx
     #   requests
-importlib-resources==6.4.0
-    # via
-    #   jsonschema
-    #   jsonschema-specifications
 inflection==0.5.1
     # via drf-spectacular
 jinja2==3.1.4
@@ -222,7 +209,7 @@ mysqlclient==2.2.4
     # via -r requirements/base.in
 newrelic==9.9.0
     # via edx-django-utils
-numpy==1.24.4
+numpy==1.26.4
     # via
     #   scikit-learn
     #   scipy
@@ -236,8 +223,6 @@ openai==1.13.3
     #   -r requirements/base.in
 pbr==6.0.0
     # via stevedore
-pkgutil-resolve-name==1.3.10
-    # via jsonschema
 ply==3.11
     # via djangoql
 prompt-toolkit==3.0.43
@@ -297,15 +282,15 @@ requests==2.31.0
     #   social-auth-core
 requests-oauthlib==2.0.0
     # via social-auth-core
-rpds-py==0.18.0
+rpds-py==0.18.1
     # via
     #   jsonschema
     #   referencing
 rules==3.3
     # via -r requirements/base.in
-scikit-learn==1.3.2
+scikit-learn==1.4.2
     # via -r requirements/base.in
-scipy==1.10.1
+scipy==1.13.0
     # via scikit-learn
 semantic-version==2.10.0
     # via edx-drf-extensions
@@ -348,19 +333,12 @@ tqdm==4.66.4
     # via openai
 typing-extensions==4.11.0
     # via
-    #   annotated-types
-    #   anyio
-    #   asgiref
-    #   drf-spectacular
     #   edx-opaque-keys
-    #   kombu
     #   openai
     #   pydantic
     #   pydantic-core
 tzdata==2024.1
-    # via
-    #   backports-zoneinfo
-    #   celery
+    # via celery
 uritemplate==4.1.1
     # via drf-spectacular
 urllib3==2.2.1
@@ -375,9 +353,7 @@ wcwidth==0.2.13
 xlsxwriter==3.2.0
     # via -r requirements/base.in
 zipp==3.18.1
-    # via
-    #   -r requirements/base.in
-    #   importlib-resources
+    # via -r requirements/base.in
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools
diff --git a/requirements/constraints.txt b/requirements/constraints.txt
@@ -33,11 +33,6 @@ diff-cover==4.0.0
 # redis 4 client won't work with redis 3 server
 redis<4
 
-# edx-lint>=5.3.0 and/or pylint>=2.15 throws a bunch of unknown option value errors
-edx-lint<5.3
-pylint<2.15
-astroid<2.12
-
 # To avoid any breaking changes
 openai<=1.13.3