fix: Make the Dockerfile work and update requirements (#2)

Assorted getting-started fixes:

* Made the Dockerfile successfully build with a working Python 3.9 environment
* Added Anki dependency
* Fixed some requirements problems that were preventing use of Django 4.2
* Fixed all pycodestyle and pylint failures
* Got documentation builds working

Dockerfile

@@ -1,39 +1,54 @@
-FROM ubuntu:focal as app
-MAINTAINER [email protected]
-
+FROM ubuntu:focal as base
 
 # Packages installed:
 
 # language-pack-en locales; ubuntu locale support so that system utilities have a consistent
 # language and time zone.
 
-# python; ubuntu doesnt ship with python, so this is the python we will use to run the application
-
-# python3-pip; install pip to install application requirements.txt files
+# python3.9; version of python compatible with Anki
 
 # libmysqlclient-dev; to install header files needed to use native C implementation for
 # MySQL-python for performance gains.
 
 # libssl-dev; # mysqlclient wont install without this.
 
-# python3-dev; to install header files for python extensions; much wheel-building depends on this
+# make; needed to build the gevent wheel (unclear why a binary wheel isn't being found and used)
 
-# gcc; for compiling python extensions distributed with python packages like mysql-client
+# python3.9-dev; to install header files for python extensions; much wheel-building depends on this
 
-# If you add a package here please include a comment above describing what it is used for
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -qy install --no-install-recommends \
- language-pack-en locales \
- python3.8 python3-dev python3-pip \
- # The mysqlclient Python package has install-time dependencies
- libmysqlclient-dev libssl-dev pkg-config \
- gcc
+# python3.9-venv; install Python 3.9 version of venv for creating a virtualenv with pip
+
+# gcc; for compiling python extensions distributed with python packages like mysql-client
 
+# unzip; for extracting the downloaded watchman binary release
 
-RUN pip install --upgrade pip setuptools
-# delete apt package lists because we do not need them inflating our image
-RUN rm -rf /var/lib/apt/lists/*
+# wget; for fetching a local copy of common_constraints.txt to edit until the Django 4.2 upgrade is complete
 
-RUN ln -s /usr/bin/python3 /usr/bin/python
+# If you add a package here please include a comment above describing what it is used for
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get -qy install --no-install-recommends software-properties-common && \
+    apt-add-repository -y ppa:deadsnakes/ppa && \
+    apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get -qy install --no-install-recommends \
+    language-pack-en \
+    locales \
+    make \
+    python3.9 \
+    python3.9-dev \
+    python3.9-venv \
+    # The mysqlclient Python package has install-time dependencies
+    libmysqlclient-dev \
+    libssl-dev \
+    pkg-config \
+    gcc \
+    unzip \
+    wget && \
+    rm -rf /var/lib/apt/lists/*
+
+# Create Python env
+ENV VIRTUAL_ENV=/edx/app/venvs/flashcards
+RUN python3.9 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
 RUN locale-gen en_US.UTF-8
 ENV LANG en_US.UTF-8
@@ -48,10 +63,13 @@ WORKDIR /edx/app/flashcards
 
 # Copy the requirements explicitly even though we copy everything below
 # this prevents the image cache from busting unless the dependencies have changed.
+COPY requirements/pip.txt /edx/app/flashcards/requirements/pip.txt
 COPY requirements/production.txt /edx/app/flashcards/requirements/production.txt
 
 # Dependencies are installed as root so they cannot be modified by the application user.
-RUN pip install -r requirements/production.txt
+
+RUN pip install --no-cache-dir -r requirements/pip.txt
+RUN pip install --no-cache-dir -r requirements/production.txt
 
 RUN mkdir -p /edx/var/log
 
@@ -65,3 +83,22 @@ CMD gunicorn --workers=2 --name flashcards -c /edx/app/flashcards/flashcards/doc
 # This line is after the requirements so that changes to the code will not
 # bust the image cache
 COPY . /edx/app/flashcards
+
+# We don't switch back to the app user for devstack because we need devstack users to be
+# able to update requirements and generally run things as root.
+FROM base as dev
+USER root
+ENV DJANGO_SETTINGS_MODULE flashcards.settings.devstack
+
+# Install watchman
+RUN wget https://github.com/facebook/watchman/releases/download/v2023.10.23.00/watchman-v2023.10.23.00-linux.zip
+RUN unzip watchman-v2023.10.23.00-linux.zip
+RUN mkdir -p /usr/local/{bin,lib} /usr/local/var/run/watchman
+RUN cp watchman-v2023.10.23.00-linux/bin/* /usr/local/bin
+RUN cp watchman-v2023.10.23.00-linux/lib/* /usr/local/lib
+RUN chmod 755 /usr/local/bin/watchman
+RUN chmod 2777 /usr/local/var/run/watchman
+
+RUN pip install --no-cache-dir -r /edx/app/flashcards/requirements/dev.txt
+
+CMD while true; do python ./manage.py runserver 0.0.0.0:8491; sleep 2; done

Makefile

@@ -44,7 +44,7 @@ dev_requirements: clean_tox piptools ## sync to requirements for local developme
 validation_requirements: piptools ## sync to requirements for testing & code quality checking
 	pip-sync -q requirements/validation.txt
 
-doc_requirements: piptools
+doc_requirements: piptools ## install requirements for documentation builds
 	pip-sync -q requirements/doc.txt
 
 production-requirements: piptools ## install requirements for production
@@ -59,8 +59,7 @@ shell: ## run Django shell
 test: clean ## run tests and generate coverage report
 	pytest
 
-# To be run from CI context
-coverage: clean
+coverage: clean ## To be run from CI context
 	pytest --cov-report html
 	$(BROWSER)htmlcov/index.html
 
@@ -76,7 +75,7 @@ style: ## run Python style checker
 lint: ## run Python code linting
 	pylint --rcfile=pylintrc flashcards *.py
 
-quality:
+quality: ## run all quality checks
 	tox -e quality 
 
 pii_check: ## check for PII annotations on all Django models
@@ -97,8 +96,17 @@ html_coverage: ## generate and view HTML coverage report
 # Define PIP_COMPILE_OPTS=-v to get more information during make upgrade.
 PIP_COMPILE = pip-compile --upgrade $(PIP_COMPILE_OPTS)
 
+COMMON_CONSTRAINTS_TXT=requirements/common_constraints.txt
+.PHONY: $(COMMON_CONSTRAINTS_TXT)
+$(COMMON_CONSTRAINTS_TXT):
+	wget -O "$(@)" https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt || touch "$(@)"
+	echo "$(COMMON_CONSTRAINTS_TEMP_COMMENT)" | cat - $(@) > temp && mv temp $(@)
+
+
 upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade
-upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
+upgrade: $(COMMON_CONSTRAINTS_TXT) ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
+	sed 's/Django<4.0//g' requirements/common_constraints.txt > requirements/common_constraints.tmp
+	mv requirements/common_constraints.tmp requirements/common_constraints.txt
 	pip install -qr requirements/pip-tools.txt
 	# Make sure to compile files after any other files they include!
 	$(PIP_COMPILE) --allow-unsafe -o requirements/pip.txt requirements/pip.in
@@ -125,7 +133,7 @@ extract_translations: ## extract strings to be translated, outputting .mo files
 dummy_translations: ## generate dummy translation (.po) files
 	cd flashcards && i18n_tool dummy
 
-compile_translations: # compile translation files, outputting .po files for each supported language
+compile_translations: ## compile translation files, outputting .po files for each supported language
 	python manage.py compilemessages
 
 fake_translations: extract_translations dummy_translations compile_translations ## generate and compile dummy translation files
@@ -143,7 +151,7 @@ open-devstack: ## open a shell on the server started by start-devstack
 	docker exec -it flashcards /edx/app/flashcards/devstack.sh open
 
 pkg-devstack: ## build the flashcards image from the latest configuration and code
-	docker build -t flashcards:latest -f docker/build/flashcards/Dockerfile git://github.com/openedx/configuration
+	docker build -t flashcards:latest -f docker/build/flashcards/Dockerfile
 
 detect_changed_source_translations: ## check if translation files are up-to-date
 	cd flashcards && i18n_tool changed
@@ -154,28 +162,28 @@ docker_build:
 	docker build . -f Dockerfile -t openedx/flashcards
 
 # devstack-themed shortcuts
-dev.up: # Starts all containers
+dev.up: ## Starts all containers
 	docker-compose up -d
 
 dev.up.build:
 	docker-compose up -d --build
 
-dev.down: # Kills containers and all of their data that isn't in volumes
+dev.down: ## Kills containers and all of their data that isn't in volumes
 	docker-compose down
 
-dev.stop: # Stops containers so they can be restarted
+dev.stop: ## Stops containers so they can be restarted
 	docker-compose stop
 
-app-shell: # Run the app shell as root
+app-shell: ## Run the app shell as root
 	docker exec -u 0 -it flashcards.app bash
 
-db-shell: # Run the app shell as root, enter the app's database
+db-shell: ## Run the app shell as root, enter the app's database
 	docker exec -u 0 -it flashcards.db mysql -u root flashcards
 
-%-logs: # View the logs of the specified service container
+%-logs: ## View the logs of the specified service container
 	docker-compose logs -f --tail=500 $*
 
-%-restart: # Restart the specified service container
+%-restart: ## Restart the specified service container
 	docker-compose restart $*
 
 %-attach:

docker-compose.yml

@@ -1,7 +1,9 @@
 version: "2.1"
 services:
   db:
-    image: mysql:8.0
+    # Oracle-packaged version includes a `linux/arm64/v8` version, needed for
+    # machines with Apple Silicon CPUs (Mac M1, M2)
+    image: mysql:8.0.33-oracle
     container_name: flashcards.db
     environment:
       # See how these environment variables being used at https://github.com/mysql/mysql-docker/blob/mysql-server/8.0/docker-entrypoint.sh
@@ -27,7 +29,7 @@ services:
     environment:
       DJANGO_SETTINGS_MODULE: flashcards.settings.devstack
     ports:
-      - "8491:8491" # TODO: change this to your port
+      - "8491:8491"
     networks:
       - devstack_default
     stdin_open: true

docs/conf.py

@@ -39,6 +39,14 @@ def get_version(*file_paths):
 
 VERSION = get_version('../flashcards', '__init__.py')
 
+# Specify settings module (which will be picked up from the sandbox)
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'flashcards.settings.test')
+
+import django
+
+
+django.setup()
+
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.

flashcards/apps/core/tests/test_context_processors.py

@@ -2,7 +2,7 @@
 
 from django.test import TestCase, override_settings
 
-from flashcards.apps.core.context_processors import core
+# from flashcards.apps.core.context_processors import core
 
 PLATFORM_NAME = 'Test Platform'
 
@@ -12,6 +12,6 @@ class CoreContextProcessorTests(TestCase):
 
     @override_settings(PLATFORM_NAME=PLATFORM_NAME)
     def test_core(self):
-      return
+        return
         # request = RequestFactory().get('/')
         # self.assertDictEqual(core(request), {'platform_name': PLATFORM_NAME})

flashcards/apps/core/tests/test_models.py

@@ -1,10 +1,11 @@
 """ Tests for core models. """
 
 from django.test import TestCase
+
 # from django_dynamic_fixture import G
 # from social_django.models import UserSocialAuth
 
-from flashcards.apps.core.models import User
+# from flashcards.apps.core.models import User
 
 
 class UserTests(TestCase):

flashcards/apps/core/tests/test_views.py

@@ -6,6 +6,7 @@
 from django.contrib.auth import get_user_model
 # from django.db import DatabaseError
 from django.test import TestCase
+
 # from django.test.utils import override_settings
 # from django.urls import reverse
 

flashcards/settings/base.py

@@ -241,3 +241,6 @@ def root(*path_fragments):
 
 # Set up logging for development use (logging to stdout)
 LOGGING = get_logger_config(debug=DEBUG)
+
+# OpenAI API key, to be specified in the private settings file
+OPENAI_API_KEY = ''

flashcards/utils.py

@@ -1,12 +1,14 @@
 """
-Integration with openai to generate 'flashcards' in csv form
+OpenAI integration utilities.
+
+Integration with OpenAI to generate 'flashcards' in csv form
 based on course content
 """
 
 import openai
-from flashcards.settings.private import OPENAI_API_KEY # pylint: disable=import-error,no-name-in-module
+from django.conf import settings
 
-openai.api_key = OPENAI_API_KEY
+openai.api_key = settings.OPENAI_API_KEY
 
 
 content_prompt = """
@@ -25,7 +27,7 @@
 Lesson 5: Filamentous Fungi
 Lesson 6: Aged Meat and Cheese
 Lesson 7: Chocolate and Coffee
-"""
+"""  # noqa
 
 course_content = """
 ROBERTO KOLTER: While humans have been preparing and consuming
@@ -169,7 +171,7 @@
 Why can the Dead Sea keep swimmers afloat?,Due to high salt content
 Why is the Dead Sea called Dead?,Because only simple organisms can live in it
 Why only simple organisms can live in the Dead Sea?,Because of high salt content
-"""
+"""  # noqa
 
 messages = [
     {"role": "system",
@@ -178,19 +180,18 @@
      "content": content_prompt + course_content},
 ]
 
-
-c3 = openai.ChatCompletion.create(
-    model="gpt-3.5-turbo",
-    messages=messages,
-    temperature=1.0,
-)
-
-print(c3['choices'][0]['message']['content'])
-
-# c4 = openai.ChatCompletion.create(
-#     model="gpt-4",
-#     messages=messages,
-#     temperature=1.0,
-# )
-
-# print(c4)
+if openai.api_key:
+    c3 = openai.ChatCompletion.create(
+        model="gpt-3.5-turbo",
+        messages=messages,
+        temperature=1.0,
+    )
+    print(c3['choices'][0]['message']['content'])
+
+    # c4 = openai.ChatCompletion.create(
+    #     model="gpt-4",
+    #     messages=messages,
+    #     temperature=1.0,
+    # )
+
+    # print(c4)

requirements/base.in

@@ -1,7 +1,8 @@
 # Core requirements for using this application
 -c constraints.txt
 
-Django         # Web application framework
+anki                    # library/application for flashcards with spaced repetition
+Django                  # Web application framework
 django-cors-headers
 django-extensions
 django-rest-swagger
@@ -13,5 +14,5 @@ edx-django-release-util
 edx-drf-extensions
 edx-rest-api-client
 mysqlclient
-openai
+openai                  # OpenAI library used to generate flash card candidates from course content
 pytz