feat: learners-count statistics

.github/workflows/ci.yml

@@ -1,27 +1,11 @@
 name: Python CI
 
 on:
-  push:
-    branches: [main]
-  issue_comment:
-    types: [created]
+  pull_request:
+    branches:
+      - '*'
 
 jobs:
-  check_comment:
-    runs-on: ubuntu-latest
-    if: github.event.issue.pull_request != '' # Only runs if the comment is on a PR
-    outputs:
-      should_run: ${{ steps.comment_check.outputs.should_run }}
-    steps:
-      - name: Check for 'run tests' comment
-        id: comment_check
-        uses: actions/github-script@v5
-        with:
-          github-token: ${{secrets.GITHUB_TOKEN}}
-          script: |
-            const isRunTestsComment = '${{ github.event.comment.body }}'.trim() === 'run tests';
-            core.setOutput('should_run', isRunTestsComment ? 'true' : 'false');
-
   run_tests:
     name: tests
     runs-on: ${{ matrix.os }}
@@ -32,9 +16,9 @@ jobs:
         toxenv: [quality, django32]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: setup python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 

.gitignore

@@ -63,3 +63,6 @@ docs/futurex_openedx_extensions.*.rst
 # Private requirements
 requirements/private.in
 requirements/private.txt
+
+# temporary tests migration files
+/test_utils/edx_platform_mocks/fake_models/migrations/

futurex_openedx_extensions/dashboard/apps.py

@@ -6,7 +6,7 @@
 class DashboardConfig(AppConfig):
     """Configuration for the dashboard Django application"""
 
-    name = 'dashboard'
+    name = 'futurex_openedx_extensions.dashboard'
 
     plugin_app = {
         'settings_config': {
@@ -23,7 +23,7 @@ class DashboardConfig(AppConfig):
         },
         'url_config': {
             'lms.djangoapp': {
-                'namespace': 'dashboard',
+                'namespace': 'fx_dashboard',
             },
         },
     }

futurex_openedx_extensions/dashboard/settings/common_production.py

@@ -5,4 +5,4 @@ def plugin_settings(settings):  # pylint: disable=unused-argument
     """
     plugin settings
     """
-    # Nothing to do here yet.
+    # Nothing to do here yet

futurex_openedx_extensions/dashboard/statistics/certificates.py

@@ -0,0 +1,36 @@
+"""functions for getting statistics about certificates"""
+from __future__ import annotations
+
+from typing import Dict, List
+
+from django.db.models import Count, OuterRef, Subquery
+from lms.djangoapps.certificates.models import GeneratedCertificate
+from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
+
+from futurex_openedx_extensions.helpers.tenants import get_course_org_filter_list
+
+
+def get_certificates_count(tenant_ids: List[int]) -> Dict[str, int]:
+    """
+    Get the count of issued certificates in the given tenants. The count is grouped by organization. Certificates
+    for admins, staff, and superusers are also included.
+
+    :param tenant_ids: List of tenant IDs to get the count for
+    :type tenant_ids: List[int]
+    :return: Count of certificates per organization
+    :rtype: Dict[str, int]
+    """
+    course_org_filter_list = get_course_org_filter_list(tenant_ids)['course_org_filter_list']
+
+    result = list(GeneratedCertificate.objects.filter(
+        status='downloadable',
+        course_id__in=CourseOverview.objects.filter(
+            org__in=course_org_filter_list
+        ),
+    ).annotate(course_org=Subquery(
+        CourseOverview.objects.filter(
+            id=OuterRef('course_id')
+        ).values('org')
+    )).values('course_org').annotate(certificates_count=Count('id')).values_list('course_org', 'certificates_count'))
+
+    return dict(result)

futurex_openedx_extensions/dashboard/statistics/courses.py

@@ -0,0 +1,41 @@
+"""functions for getting statistics about courses"""
+from __future__ import annotations
+
+from typing import List
+
+from django.db.models import Count, Q
+from django.db.models.query import QuerySet
+from django.utils.timezone import now
+from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
+
+from futurex_openedx_extensions.helpers.tenants import get_course_org_filter_list
+
+
+def get_courses_count(tenant_ids: List[int], only_active=False, only_visible=False) -> QuerySet:
+    """
+    Get the count of courses in the given tenants
+
+    :param tenant_ids: List of tenant IDs to get the count for
+    :type tenant_ids: List[int]
+    :param only_active: Whether to only count active courses (according to dates)
+    :type only_active: bool
+    :param only_visible: Whether to only count visible courses (according to staff-only visibility)
+    :type only_visible: bool
+    :return: QuerySet of courses count per organization
+    :rtype: QuerySet
+    """
+    course_org_filter_list = get_course_org_filter_list(tenant_ids)['course_org_filter_list']
+
+    q_set = CourseOverview.objects.filter(org__in=course_org_filter_list)
+    if only_active:
+        q_set = q_set.filter(
+            Q(start__isnull=True) | Q(start__lte=now()),
+        ).filter(
+            Q(end__isnull=True) | Q(end__gte=now()),
+        )
+    if only_visible:
+        q_set = q_set.filter(visible_to_staff_only=False)
+
+    return q_set.values('org').annotate(
+        courses_count=Count('id')
+    ).order_by('org')

futurex_openedx_extensions/dashboard/statistics/learners.py

@@ -0,0 +1,194 @@
+"""functions for getting statistics about learners"""
+from __future__ import annotations
+
+from typing import Dict, List
+
+from common.djangoapps.student.models import CourseAccessRole, CourseEnrollment, UserSignupSource
+from django.contrib.auth import get_user_model
+from django.db.models import Count, Exists, OuterRef, Q, Subquery
+from django.db.models.query import QuerySet
+from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
+
+from futurex_openedx_extensions.helpers.tenants import get_course_org_filter_list, get_tenant_site
+
+
+def get_learners_count_having_enrollment_per_org(tenant_id) -> QuerySet:
+    """
+    TODO: Cache the result of this function
+    Get the count of learners with enrollments per organization. Admins and staff are excluded from the count. This
+    function takes one tenant ID for performance reasons.
+
+
+    SELECT coc.org, COUNT(DISTINCT au.id)
+    FROM edxapp.auth_user au
+    INNER JOIN edxapp.student_courseenrollment sc ON
+        au.id = sc.user_id
+    INNER JOIN edxapp.course_overviews_courseoverview coc ON
+        sc.course_id = coc.id AND
+        coc.org IN ('ORG1', 'ORG2')  -- course_org_filter_list
+    WHERE au.id NOT IN (
+            SELECT DISTINCT cr.user_id
+            FROM edxapp.student_courseaccessrole cr
+            WHERE cr.org = coc.org
+        ) AND
+        au.is_superuser = 0 AND
+        au.is_staff = 0 AND
+        au.is_active = 1
+    GROUP BY coc.org
+
+
+    :return: QuerySet of learners count per organization
+    :rtype: QuerySet
+    """
+    course_org_filter_list = get_course_org_filter_list([tenant_id])['course_org_filter_list']
+
+    return CourseOverview.objects.filter(
+        org__in=course_org_filter_list
+    ).values('org').annotate(
+        learners_count=Count(
+            'courseenrollment__user_id',
+            filter=~Exists(
+                CourseAccessRole.objects.filter(
+                    user_id=OuterRef('courseenrollment__user_id'),
+                    org=OuterRef('org')
+                )
+            ) &
+            Q(courseenrollment__user__is_superuser=False) &
+            Q(courseenrollment__user__is_staff=False) &
+            Q(courseenrollment__user__is_active=True),
+            distinct=True
+        )
+    )
+
+
+def get_learners_count_having_enrollment_for_tenant(tenant_id) -> QuerySet:
+    """
+    TODO: Cache the result of this function
+    Get the count of learners with enrollments per organization. Admins and staff are excluded from the count
+
+
+    SELECT COUNT(DISTINCT au.id)
+    FROM edxapp.auth_user au
+    INNER JOIN edxapp.student_courseenrollment sc ON
+        au.id = sc.user_id
+    INNER JOIN edxapp.course_overviews_courseoverview coc ON
+        sc.course_id = coc.id AND
+        coc.org IN ('ORG1', 'ORG2')  -- course_org_filter_list
+    WHERE au.id NOT IN (
+            SELECT DISTINCT cr.user_id
+            FROM edxapp.student_courseaccessrole cr
+            WHERE cr.org = coc.org
+        ) AND
+        au.is_superuser = 0 AND
+        au.is_staff = 0 AND
+        au.is_active = 1
+
+
+    :return: QuerySet of learners count per organization
+    :rtype: QuerySet
+    """
+    course_org_filter_list = get_course_org_filter_list([tenant_id])['course_org_filter_list']
+
+    return get_user_model().objects.filter(
+        is_superuser=False,
+        is_staff=False,
+        is_active=True,
+        courseenrollment__course__org__in=course_org_filter_list,
+    ).exclude(
+        Exists(
+            CourseAccessRole.objects.filter(
+                user_id=OuterRef('id'),
+                org=OuterRef('courseenrollment__course__org'),
+            ).values('user_id')
+        )
+    ).values('id').distinct().count()
+
+
+def get_learners_count_having_no_enrollment(tenant_id) -> QuerySet:
+    """
+    TODO: Cache the result of this function
+    Get the count of learners with no enrollments per organization. Admins and staff are excluded from the count.
+    Since there is no enrollment, we'll use UserSignupSource
+
+    The function returns the count for one tenant for performance reasons.
+
+    SELECT COUNT(distinct su.id)
+    FROM edxapp.student_usersignupsource su
+    WHERE su.site = 'demo.example.com' -- tenant_site
+        AND su.user_id not in (
+            SELECT distinct au.id
+            FROM edxapp.auth_user au
+            INNER JOIN edxapp.student_courseenrollment sc ON
+                au.id = sc.user_id
+            INNER JOIN edxapp.course_overviews_courseoverview coc ON
+                sc.course_id = coc.id AND
+                coc.org IN ('ORG1', 'ORG2') -- course_org_filter_list
+            WHERE au.id NOT IN (
+                    SELECT DISTINCT cr.user_id
+                    FROM edxapp.student_courseaccessrole cr
+                    WHERE cr.org = coc.org
+                ) AND
+                au.is_superuser = 0 AND
+                au.is_staff = 0 AND
+                au.is_active = 1 AND
+        ) AND su.user_id NOT IN (
+            SELECT DISTINCT cr.user_id
+            FROM edxapp.student_courseaccessrole cr
+            WHERE cr.org IN ('ORG1', 'ORG2') -- course_org_filter_list
+        )
+
+    """
+    course_org_filter_list = get_course_org_filter_list([tenant_id])['course_org_filter_list']
+    tenant_site = get_tenant_site(tenant_id)
+
+    return UserSignupSource.objects.filter(
+        site=tenant_site
+    ).exclude(
+        user_id__in=Subquery(
+            CourseEnrollment.objects.filter(
+                user_id=OuterRef('user_id'),
+                course__org__in=course_org_filter_list,
+                user__is_superuser=False,
+                user__is_staff=False,
+                user__is_active=True,
+            ).exclude(
+                Exists(
+                    CourseAccessRole.objects.filter(
+                        user_id=OuterRef('user_id'),
+                        org=OuterRef('course__org'),
+                    ).values('user_id').distinct()
+                ),
+            ).values('user_id')
+        )
+    ).exclude(
+        user_id__in=Subquery(
+            CourseAccessRole.objects.filter(
+                org__in=course_org_filter_list,
+            ).values('user_id').distinct()
+        )
+    ).values('user_id').distinct().count()
+
+
+def get_learners_count(tenant_ids: List[int]) -> Dict[int, Dict[str, int]]:
+    """
+    Get the count of learners in the given list of tenants. Admins and staff are excluded from the count.
+
+    :param tenant_ids: List of tenant IDs to get the count for
+    :type tenant_ids: List[int]
+    :return: Dictionary of tenant ID and the count of learners
+    :rtype: Dict[int, Dict[str, int]]
+    """
+    result = {
+        tenant_id: {
+            'learners_count': get_learners_count_having_enrollment_for_tenant(tenant_id),
+            'learners_count_no_enrollment': get_learners_count_having_no_enrollment(tenant_id),
+            'learners_count_per_org': {},
+        }
+        for tenant_id in tenant_ids
+    }
+    for tenant_id in tenant_ids:
+        result[tenant_id]['learners_count_per_org'] = {
+            item['org']: item['learners_count']
+            for item in get_learners_count_having_enrollment_per_org(tenant_id)
+        }
+    return result

futurex_openedx_extensions/dashboard/urls.py

@@ -1,10 +1,12 @@
 """
 URLs for dashboard.
 """
-from django.urls import re_path  # pylint: disable=unused-import
-from django.views.generic import TemplateView  # pylint: disable=unused-import
+from django.urls import re_path
+
+from futurex_openedx_extensions.dashboard.views import TotalCountsView
+
+app_name = 'fx_dashboard'
 
 urlpatterns = [
-    # TODO: Fill in URL patterns and views here.
-    # re_path(r'', TemplateView.as_view(template_name="dashboard/base.html")),
+    re_path(r'^api/fx/statistics/v1/total_counts', TotalCountsView.as_view(), name='total-counts'),
 ]