From cbadccda5f9f515835c9934c9766eba135d8c872 Mon Sep 17 00:00:00 2001 From: muhammad-ammar Date: Tue, 13 Feb 2024 17:25:03 +0500 Subject: [PATCH] feat: add streaming csv support --- analyticsdataserver/settings/base.py | 1 + analyticsdataserver/settings/local.py | 1 + analyticsdataserver/settings/test.py | 1 + requirements/base.txt | 36 +++++++++++----------- requirements/dev.txt | 38 ++++++++++++----------- requirements/django.txt | 2 +- requirements/doc.txt | 40 ++++++++++++------------ requirements/pip.txt | 4 +-- requirements/production.txt | 38 ++++++++++++----------- requirements/test.txt | 44 ++++++++++++++------------- requirements/tox.txt | 6 ++-- 11 files changed, 112 insertions(+), 99 deletions(-) diff --git a/analyticsdataserver/settings/base.py b/analyticsdataserver/settings/base.py index b77f5b3d..d6952bc6 100644 --- a/analyticsdataserver/settings/base.py +++ b/analyticsdataserver/settings/base.py @@ -358,6 +358,7 @@ ANALYTICS_DATABASE_V1 = None DATABASE_ROUTERS = ['analyticsdataserver.router.AnalyticsAPIRouter', 'analyticsdataserver.router.AnalyticsModelsRouter'] ENTERPRISE_REPORTING_DB_ALIAS = 'enterprise' +ENROLLMENTS_PAGE_SIZE = 10000 LMS_BASE_URL = None diff --git a/analyticsdataserver/settings/local.py b/analyticsdataserver/settings/local.py index acf56d67..bc7f1e91 100644 --- a/analyticsdataserver/settings/local.py +++ b/analyticsdataserver/settings/local.py @@ -75,6 +75,7 @@ ANALYTICS_DATABASE = 'analytics' ENTERPRISE_REPORTING_DB_ALIAS = 'analytics' ANALYTICS_DATABASE_V1 = 'analytics' +ENROLLMENTS_PAGE_SIZE = 10000 TEST_RUNNER = 'django_nose.NoseTestSuiteRunner' diff --git a/analyticsdataserver/settings/test.py b/analyticsdataserver/settings/test.py index 188863ad..69f2fa83 100644 --- a/analyticsdataserver/settings/test.py +++ b/analyticsdataserver/settings/test.py @@ -34,6 +34,7 @@ ANALYTICS_DATABASE = 'analytics' ANALYTICS_DATABASE_V1 = 'analytics_v1' ENTERPRISE_REPORTING_DB_ALIAS = 'default' +ENROLLMENTS_PAGE_SIZE = 10000 LMS_BASE_URL = 'http://lms-host' diff --git a/requirements/base.txt b/requirements/base.txt index 389b13ad..93cd5e61 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -13,13 +13,13 @@ backports-zoneinfo==0.2.1 # via django boto==2.49.0 # via -r requirements/base.in -boto3==1.34.16 +boto3==1.34.40 # via -r requirements/base.in -botocore==1.34.16 +botocore==1.34.40 # via # boto3 # s3transfer -certifi==2023.11.17 +certifi==2024.2.2 # via requests cffi==1.16.0 # via @@ -33,11 +33,11 @@ coreapi==2.3.3 # via -r requirements/base.in coreschema==0.0.4 # via coreapi -cryptography==41.0.7 +cryptography==42.0.2 # via # django-fernet-fields-v2 # pyjwt -django==4.2.9 +django==4.2.10 # via # -c requirements/constraints.txt # -r requirements/base.in @@ -69,7 +69,7 @@ django-fernet-fields-v2==0.9 # via edx-enterprise-data django-filter==23.5 # via edx-enterprise-data -django-model-utils==4.3.1 +django-model-utils==4.4.0 # via # edx-enterprise-data # edx-rbac @@ -90,7 +90,9 @@ djangorestframework==3.14.0 # edx-api-doc-tools # edx-drf-extensions djangorestframework-csv==3.0.2 - # via -r requirements/base.in + # via + # -r requirements/base.in + # edx-enterprise-data drf-jwt==1.19.2 # via edx-drf-extensions drf-yasg==1.21.7 @@ -101,18 +103,18 @@ edx-ccx-keys==1.2.1 # via -r requirements/base.in edx-django-release-util==1.3.0 # via -r requirements/base.in -edx-django-utils==5.9.0 +edx-django-utils==5.10.1 # via # -r requirements/base.in # edx-drf-extensions # edx-enterprise-data # edx-rest-api-client -edx-drf-extensions==9.1.2 +edx-drf-extensions==10.2.0 # via # -r requirements/base.in # edx-enterprise-data # edx-rbac -edx-enterprise-data==5.5.1 +edx-enterprise-data==6.0.0 # via -r requirements/base.in edx-opaque-keys==2.5.1 # via @@ -128,7 +130,7 @@ edx-rest-api-client==5.6.1 # edx-enterprise-data factory-boy==3.3.0 # via edx-enterprise-data -faker==22.2.0 +faker==23.1.0 # via factory-boy html5lib==1.1 # via -r requirements/base.in @@ -148,9 +150,9 @@ jmespath==1.0.1 # botocore markdown==3.5.2 # via -r requirements/base.in -markupsafe==2.1.3 +markupsafe==2.1.5 # via jinja2 -newrelic==9.4.0 +newrelic==9.6.0 # via edx-django-utils ordered-set==4.1.0 # via -r requirements/base.in @@ -158,7 +160,7 @@ packaging==23.2 # via drf-yasg pbr==6.0.0 # via stevedore -psutil==5.9.7 +psutil==5.9.8 # via edx-django-utils pycparser==2.21 # via cffi @@ -177,9 +179,9 @@ python-dateutil==2.8.2 # via # botocore # faker -python-memcached==1.61 +python-memcached==1.62 # via -r requirements/base.in -pytz==2023.3.post1 +pytz==2024.1 # via # djangorestframework # drf-yasg @@ -215,7 +217,7 @@ stevedore==5.1.0 # via # edx-django-utils # edx-opaque-keys -tqdm==4.66.1 +tqdm==4.66.2 # via -r requirements/base.in typing-extensions==4.9.0 # via diff --git a/requirements/dev.txt b/requirements/dev.txt index f3d97016..fb3ee8dc 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -13,13 +13,13 @@ backports-zoneinfo==0.2.1 # via django boto==2.49.0 # via -r requirements/base.in -boto3==1.34.16 +boto3==1.34.40 # via -r requirements/base.in -botocore==1.34.16 +botocore==1.34.40 # via # boto3 # s3transfer -certifi==2023.11.17 +certifi==2024.2.2 # via requests cffi==1.16.0 # via @@ -33,11 +33,11 @@ coreapi==2.3.3 # via -r requirements/base.in coreschema==0.0.4 # via coreapi -cryptography==41.0.7 +cryptography==42.0.2 # via # django-fernet-fields-v2 # pyjwt -django==4.2.9 +django==4.2.10 # via # -c requirements/constraints.txt # -r requirements/base.in @@ -69,7 +69,7 @@ django-fernet-fields-v2==0.9 # via edx-enterprise-data django-filter==23.5 # via edx-enterprise-data -django-model-utils==4.3.1 +django-model-utils==4.4.0 # via # edx-enterprise-data # edx-rbac @@ -90,7 +90,9 @@ djangorestframework==3.14.0 # edx-api-doc-tools # edx-drf-extensions djangorestframework-csv==3.0.2 - # via -r requirements/base.in + # via + # -r requirements/base.in + # edx-enterprise-data drf-jwt==1.19.2 # via edx-drf-extensions drf-yasg==1.21.7 @@ -101,18 +103,18 @@ edx-ccx-keys==1.2.1 # via -r requirements/base.in edx-django-release-util==1.3.0 # via -r requirements/base.in -edx-django-utils==5.9.0 +edx-django-utils==5.10.1 # via # -r requirements/base.in # edx-drf-extensions # edx-enterprise-data # edx-rest-api-client -edx-drf-extensions==9.1.2 +edx-drf-extensions==10.2.0 # via # -r requirements/base.in # edx-enterprise-data # edx-rbac -edx-enterprise-data==5.5.1 +edx-enterprise-data==6.0.0 # via -r requirements/base.in edx-opaque-keys==2.5.1 # via @@ -128,7 +130,7 @@ edx-rest-api-client==5.6.1 # edx-enterprise-data factory-boy==3.3.0 # via edx-enterprise-data -faker==22.2.0 +faker==23.1.0 # via factory-boy html5lib==1.1 # via -r requirements/base.in @@ -148,11 +150,11 @@ jmespath==1.0.1 # botocore markdown==3.5.2 # via -r requirements/base.in -markupsafe==2.1.3 +markupsafe==2.1.5 # via jinja2 -mysqlclient==2.2.1 +mysqlclient==2.2.4 # via -r requirements/dev.in -newrelic==9.4.0 +newrelic==9.6.0 # via edx-django-utils ordered-set==4.1.0 # via -r requirements/base.in @@ -160,7 +162,7 @@ packaging==23.2 # via drf-yasg pbr==6.0.0 # via stevedore -psutil==5.9.7 +psutil==5.9.8 # via edx-django-utils pycparser==2.21 # via cffi @@ -179,9 +181,9 @@ python-dateutil==2.8.2 # via # botocore # faker -python-memcached==1.61 +python-memcached==1.62 # via -r requirements/base.in -pytz==2023.3.post1 +pytz==2024.1 # via # djangorestframework # drf-yasg @@ -217,7 +219,7 @@ stevedore==5.1.0 # via # edx-django-utils # edx-opaque-keys -tqdm==4.66.1 +tqdm==4.66.2 # via -r requirements/base.in typing-extensions==4.9.0 # via diff --git a/requirements/django.txt b/requirements/django.txt index 01c70d8d..1facfe28 100644 --- a/requirements/django.txt +++ b/requirements/django.txt @@ -1 +1 @@ -django==4.2.9 +django==4.2.10 diff --git a/requirements/doc.txt b/requirements/doc.txt index a0f4175e..7ecaad9f 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -19,17 +19,17 @@ babel==2.14.0 # sphinx backports-zoneinfo==0.2.1 # via django -beautifulsoup4==4.12.2 +beautifulsoup4==4.12.3 # via pydata-sphinx-theme boto==2.49.0 # via -r requirements/base.in -boto3==1.34.16 +boto3==1.34.40 # via -r requirements/base.in -botocore==1.34.16 +botocore==1.34.40 # via # boto3 # s3transfer -certifi==2023.11.17 +certifi==2024.2.2 # via requests cffi==1.16.0 # via @@ -43,11 +43,11 @@ coreapi==2.3.3 # via -r requirements/base.in coreschema==0.0.4 # via coreapi -cryptography==41.0.7 +cryptography==42.0.2 # via # django-fernet-fields-v2 # pyjwt -django==4.2.9 +django==4.2.10 # via # -c requirements/constraints.txt # -r requirements/base.in @@ -79,7 +79,7 @@ django-fernet-fields-v2==0.9 # via edx-enterprise-data django-filter==23.5 # via edx-enterprise-data -django-model-utils==4.3.1 +django-model-utils==4.4.0 # via # edx-enterprise-data # edx-rbac @@ -100,7 +100,9 @@ djangorestframework==3.14.0 # edx-api-doc-tools # edx-drf-extensions djangorestframework-csv==3.0.2 - # via -r requirements/base.in + # via + # -r requirements/base.in + # edx-enterprise-data docutils==0.19 # via # pydata-sphinx-theme @@ -115,18 +117,18 @@ edx-ccx-keys==1.2.1 # via -r requirements/base.in edx-django-release-util==1.3.0 # via -r requirements/base.in -edx-django-utils==5.9.0 +edx-django-utils==5.10.1 # via # -r requirements/base.in # edx-drf-extensions # edx-enterprise-data # edx-rest-api-client -edx-drf-extensions==9.1.2 +edx-drf-extensions==10.2.0 # via # -r requirements/base.in # edx-enterprise-data # edx-rbac -edx-enterprise-data==5.5.1 +edx-enterprise-data==6.0.0 # via -r requirements/base.in edx-opaque-keys==2.5.1 # via @@ -142,7 +144,7 @@ edx-rest-api-client==5.6.1 # edx-enterprise-data factory-boy==3.3.0 # via edx-enterprise-data -faker==22.2.0 +faker==23.1.0 # via factory-boy html5lib==1.1 # via -r requirements/base.in @@ -168,9 +170,9 @@ jmespath==1.0.1 # botocore markdown==3.5.2 # via -r requirements/base.in -markupsafe==2.1.3 +markupsafe==2.1.5 # via jinja2 -newrelic==9.4.0 +newrelic==9.6.0 # via edx-django-utils ordered-set==4.1.0 # via -r requirements/base.in @@ -179,11 +181,11 @@ packaging==23.2 # drf-yasg # pydata-sphinx-theme # sphinx -path==16.9.0 +path==16.10.0 # via -r requirements/doc.in pbr==6.0.0 # via stevedore -psutil==5.9.7 +psutil==5.9.8 # via edx-django-utils pycparser==2.21 # via cffi @@ -209,9 +211,9 @@ python-dateutil==2.8.2 # via # botocore # faker -python-memcached==1.61 +python-memcached==1.62 # via -r requirements/base.in -pytz==2023.3.post1 +pytz==2024.1 # via # babel # djangorestframework @@ -272,7 +274,7 @@ stevedore==5.1.0 # via # edx-django-utils # edx-opaque-keys -tqdm==4.66.1 +tqdm==4.66.2 # via -r requirements/base.in typing-extensions==4.9.0 # via diff --git a/requirements/pip.txt b/requirements/pip.txt index 21283b7d..32e80486 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -8,7 +8,7 @@ wheel==0.42.0 # via -r requirements/pip.in # The following packages are considered to be unsafe in a requirements file: -pip==23.3.2 +pip==24.0 # via -r requirements/pip.in -setuptools==69.0.3 +setuptools==69.1.0 # via -r requirements/pip.in diff --git a/requirements/production.txt b/requirements/production.txt index 9d1d76a1..ee563458 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -13,13 +13,13 @@ backports-zoneinfo==0.2.1 # via django boto==2.49.0 # via -r requirements/base.in -boto3==1.34.16 +boto3==1.34.40 # via -r requirements/base.in -botocore==1.34.16 +botocore==1.34.40 # via # boto3 # s3transfer -certifi==2023.11.17 +certifi==2024.2.2 # via requests cffi==1.16.0 # via @@ -33,11 +33,11 @@ coreapi==2.3.3 # via -r requirements/base.in coreschema==0.0.4 # via coreapi -cryptography==41.0.7 +cryptography==42.0.2 # via # django-fernet-fields-v2 # pyjwt -django==4.2.9 +django==4.2.10 # via # -c requirements/constraints.txt # -r requirements/base.in @@ -69,7 +69,7 @@ django-fernet-fields-v2==0.9 # via edx-enterprise-data django-filter==23.5 # via edx-enterprise-data -django-model-utils==4.3.1 +django-model-utils==4.4.0 # via # edx-enterprise-data # edx-rbac @@ -90,7 +90,9 @@ djangorestframework==3.14.0 # edx-api-doc-tools # edx-drf-extensions djangorestframework-csv==3.0.2 - # via -r requirements/base.in + # via + # -r requirements/base.in + # edx-enterprise-data drf-jwt==1.19.2 # via edx-drf-extensions drf-yasg==1.21.7 @@ -101,18 +103,18 @@ edx-ccx-keys==1.2.1 # via -r requirements/base.in edx-django-release-util==1.3.0 # via -r requirements/base.in -edx-django-utils==5.9.0 +edx-django-utils==5.10.1 # via # -r requirements/base.in # edx-drf-extensions # edx-enterprise-data # edx-rest-api-client -edx-drf-extensions==9.1.2 +edx-drf-extensions==10.2.0 # via # -r requirements/base.in # edx-enterprise-data # edx-rbac -edx-enterprise-data==5.5.1 +edx-enterprise-data==6.0.0 # via -r requirements/base.in edx-opaque-keys==2.5.1 # via @@ -128,7 +130,7 @@ edx-rest-api-client==5.6.1 # edx-enterprise-data factory-boy==3.3.0 # via edx-enterprise-data -faker==22.2.0 +faker==23.1.0 # via factory-boy gevent==23.9.1 # via -r requirements/production.in @@ -154,11 +156,11 @@ jmespath==1.0.1 # botocore markdown==3.5.2 # via -r requirements/base.in -markupsafe==2.1.3 +markupsafe==2.1.5 # via jinja2 -mysqlclient==2.2.1 +mysqlclient==2.2.4 # via -r requirements/production.in -newrelic==9.4.0 +newrelic==9.6.0 # via # -r requirements/production.in # edx-django-utils @@ -172,7 +174,7 @@ path-py==8.2.1 # via -r requirements/production.in pbr==6.0.0 # via stevedore -psutil==5.9.7 +psutil==5.9.8 # via edx-django-utils pycparser==2.21 # via cffi @@ -191,9 +193,9 @@ python-dateutil==2.8.2 # via # botocore # faker -python-memcached==1.61 +python-memcached==1.62 # via -r requirements/base.in -pytz==2023.3.post1 +pytz==2024.1 # via # djangorestframework # drf-yasg @@ -230,7 +232,7 @@ stevedore==5.1.0 # via # edx-django-utils # edx-opaque-keys -tqdm==4.66.1 +tqdm==4.66.2 # via -r requirements/base.in typing-extensions==4.9.0 # via diff --git a/requirements/test.txt b/requirements/test.txt index b57248a9..502eb859 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -15,13 +15,13 @@ backports-zoneinfo==0.2.1 # via django boto==2.49.0 # via -r requirements/base.in -boto3==1.34.16 +boto3==1.34.40 # via -r requirements/base.in -botocore==1.34.16 +botocore==1.34.40 # via # boto3 # s3transfer -certifi==2023.11.17 +certifi==2024.2.2 # via requests cffi==1.16.0 # via @@ -37,17 +37,17 @@ coreapi==2.3.3 # via -r requirements/base.in coreschema==0.0.4 # via coreapi -coverage[toml]==7.4.0 +coverage[toml]==7.4.1 # via # -r requirements/test.in # pytest-cov -cryptography==41.0.7 +cryptography==42.0.2 # via # django-fernet-fields-v2 # pyjwt ddt==1.7.1 # via -r requirements/test.in -diff-cover==8.0.2 +diff-cover==8.0.3 # via -r requirements/test.in # via # -c requirements/constraints.txt @@ -82,7 +82,7 @@ django-fernet-fields-v2==0.9 # via edx-enterprise-data django-filter==23.5 # via edx-enterprise-data -django-model-utils==4.3.1 +django-model-utils==4.4.0 # via # edx-enterprise-data # edx-rbac @@ -103,7 +103,9 @@ djangorestframework==3.14.0 # edx-api-doc-tools # edx-drf-extensions djangorestframework-csv==3.0.2 - # via -r requirements/base.in + # via + # -r requirements/base.in + # edx-enterprise-data drf-jwt==1.19.2 # via edx-drf-extensions drf-yasg==1.21.7 @@ -114,18 +116,18 @@ edx-ccx-keys==1.2.1 # via -r requirements/base.in edx-django-release-util==1.3.0 # via -r requirements/base.in -edx-django-utils==5.9.0 +edx-django-utils==5.10.1 # via # -r requirements/base.in # edx-drf-extensions # edx-enterprise-data # edx-rest-api-client -edx-drf-extensions==9.1.2 +edx-drf-extensions==10.2.0 # via # -r requirements/base.in # edx-enterprise-data # edx-rbac -edx-enterprise-data==5.5.1 +edx-enterprise-data==6.0.0 # via -r requirements/base.in edx-opaque-keys==2.5.1 # via @@ -143,7 +145,7 @@ exceptiongroup==1.2.0 # via pytest factory-boy==3.3.0 # via edx-enterprise-data -faker==22.2.0 +faker==23.1.0 # via factory-boy freezegun==1.4.0 # via -r requirements/test.in @@ -173,11 +175,11 @@ lazy-object-proxy==1.4.3 # via astroid markdown==3.5.2 # via -r requirements/base.in -markupsafe==2.1.3 +markupsafe==2.1.5 # via jinja2 mccabe==0.6.1 # via pylint -newrelic==9.4.0 +newrelic==9.6.0 # via edx-django-utils ordered-set==4.1.0 # via -r requirements/base.in @@ -187,11 +189,11 @@ packaging==23.2 # pytest pbr==6.0.0 # via stevedore -pluggy==1.3.0 +pluggy==1.4.0 # via # diff-cover # pytest -psutil==5.9.7 +psutil==5.9.8 # via edx-django-utils pycodestyle==2.11.1 # via -r requirements/test.in @@ -216,22 +218,22 @@ pymongo==3.13.0 # via edx-opaque-keys pynacl==1.5.0 # via edx-django-utils -pytest==7.4.4 +pytest==8.0.0 # via # pytest-cov # pytest-django pytest-cov==4.1.0 # via -r requirements/test.in -pytest-django==4.7.0 +pytest-django==4.8.0 # via -r requirements/test.in python-dateutil==2.8.2 # via # botocore # faker # freezegun -python-memcached==1.61 +python-memcached==1.62 # via -r requirements/base.in -pytz==2023.3.post1 +pytz==2024.1 # via # -r requirements/test.in # djangorestframework @@ -279,7 +281,7 @@ tomli==2.0.1 # via # coverage # pytest -tqdm==4.66.1 +tqdm==4.66.2 # via -r requirements/base.in typing-extensions==4.9.0 # via diff --git a/requirements/tox.txt b/requirements/tox.txt index 58a57dde..28261f31 100644 --- a/requirements/tox.txt +++ b/requirements/tox.txt @@ -20,11 +20,11 @@ packaging==23.2 # via # pyproject-api # tox -platformdirs==4.1.0 +platformdirs==4.2.0 # via # tox # virtualenv -pluggy==1.3.0 +pluggy==1.4.0 # via tox pyproject-api==1.6.1 # via tox @@ -32,7 +32,7 @@ tomli==2.0.1 # via # pyproject-api # tox -tox==4.11.4 +tox==4.12.1 # via -r requirements/tox.in virtualenv==20.25.0 # via tox