From 233e8965a7f55ab0bef7dc851f4410ff45aa43e3 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Fri, 13 Sep 2024 16:57:52 +0530
Subject: [PATCH 001/165] added helper method

---
 .../0055_opportunityaccess_invited_date.py    | 17 ++++
 commcare_connect/opportunity/models.py        |  1 +
 commcare_connect/program/helpers.py           | 71 ++++++++++++++++
 commcare_connect/program/tables.py            | 85 ++++++++++++++++++-
 .../program/tests/test_helpers.py             | 37 ++++++++
 commcare_connect/program/urls.py              |  6 ++
 commcare_connect/program/views.py             | 33 ++++++-
 .../templates/program/dashboard.html          | 63 ++++++++++++++
 8 files changed, 310 insertions(+), 3 deletions(-)
 create mode 100644 commcare_connect/opportunity/migrations/0055_opportunityaccess_invited_date.py
 create mode 100644 commcare_connect/program/helpers.py
 create mode 100644 commcare_connect/program/tests/test_helpers.py
 create mode 100644 commcare_connect/templates/program/dashboard.html

diff --git a/commcare_connect/opportunity/migrations/0055_opportunityaccess_invited_date.py b/commcare_connect/opportunity/migrations/0055_opportunityaccess_invited_date.py
new file mode 100644
index 00000000..b55f94d5
--- /dev/null
+++ b/commcare_connect/opportunity/migrations/0055_opportunityaccess_invited_date.py
@@ -0,0 +1,17 @@
+# Generated by Django 4.2.5 on 2024-09-12 18:13
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("opportunity", "0054_opportunity_managed_alter_opportunity_organization"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="opportunityaccess",
+            name="invited_date",
+            field=models.DateTimeField(auto_now_add=True, null=True),
+        ),
+    ]
diff --git a/commcare_connect/opportunity/models.py b/commcare_connect/opportunity/models.py
index f02264f0..d2f7a266 100644
--- a/commcare_connect/opportunity/models.py
+++ b/commcare_connect/opportunity/models.py
@@ -236,6 +236,7 @@ class OpportunityAccess(models.Model):
     suspended = models.BooleanField(default=False)
     suspension_date = models.DateTimeField(null=True, blank=True)
     suspension_reason = models.CharField(max_length=300, null=True, blank=True)
+    invited_date = models.DateTimeField(auto_now_add=True, editable=False, null=True)
 
     class Meta:
         indexes = [models.Index(fields=["invite_id"])]
diff --git a/commcare_connect/program/helpers.py b/commcare_connect/program/helpers.py
new file mode 100644
index 00000000..7f961b74
--- /dev/null
+++ b/commcare_connect/program/helpers.py
@@ -0,0 +1,71 @@
+from django.db.models import Avg, Count, DurationField, ExpressionWrapper, F, OuterRef, Q, Subquery
+
+from commcare_connect.opportunity.models import UserVisit, VisitValidationStatus
+from commcare_connect.program.models import ManagedOpportunity, Program
+
+
+def get_annotated_managed_opportunity(program: Program):
+    filter_for_valid__visit_date = ~Q(
+        opportunityaccess__uservisit__status__in=[
+            VisitValidationStatus.over_limit,
+            VisitValidationStatus.trial,
+        ]
+    )
+
+    earliest_visits = (
+        UserVisit.objects.filter(
+            opportunity_access=OuterRef("opportunityaccess"),
+        )
+        .exclude(status__in=[VisitValidationStatus.over_limit, VisitValidationStatus.trial])
+        .order_by("visit_date")
+        .values("visit_date")[:1]
+    )
+
+    managed_opportunities = (
+        ManagedOpportunity.objects.filter(program=program)
+        .order_by("start_date")
+        .annotate(
+            workers_invited=Count("opportunityaccess"),
+            workers_passing_assessment=Count(
+                "opportunityaccess__assessment",
+                filter=Q(
+                    opportunityaccess__assessment__passed=True,
+                    opportunityaccess__assessment__opportunity=F("opportunityaccess__opportunity"),
+                ),
+            ),
+            workers_starting_delivery=Count(
+                "opportunityaccess__uservisit__user",
+                filter=filter_for_valid__visit_date,
+                distinct=True,
+            ),
+            percentage_conversion=F("workers_starting_delivery") / F("workers_invited") * 100,
+            average_time_to_convert=Avg(
+                ExpressionWrapper(
+                    Subquery(earliest_visits) - F("opportunityaccess__invited_date"), output_field=DurationField()
+                ),
+                filter=filter_for_valid__visit_date,
+            ),
+        )
+        .prefetch_related(
+            "opportunityaccess_set",
+            "opportunityaccess_set__uservisit_set",
+            "opportunityaccess_set__assessment_set",
+        )
+    )
+
+    return managed_opportunities
+
+
+def get_annotated_managed_opportunity_nm(program: Program, start_date=None, end_date=None):
+    managed_opportunities = (
+        ManagedOpportunity.objects.filter(program=program, start_date__gte=start_date)
+        .order_by("start_date")
+        .annotate()
+        .prefetch_related(
+            "opportunityaccess_set",
+            "opportunityaccess_set__uservisit_set",
+            "opportunityaccess_set__assessment_set",
+        )
+    )
+
+    return managed_opportunities
diff --git a/commcare_connect/program/tables.py b/commcare_connect/program/tables.py
index 2a360d3c..b3214703 100644
--- a/commcare_connect/program/tables.py
+++ b/commcare_connect/program/tables.py
@@ -4,7 +4,7 @@
 from django.utils.safestring import mark_safe
 from django.utils.translation import gettext_lazy as _
 
-from .models import Program, ProgramApplication, ProgramApplicationStatus
+from .models import ManagedOpportunity, Program, ProgramApplication, ProgramApplicationStatus
 
 TABLE_TEMPLATE = "django_tables2/bootstrap5.html"
 RESPONSIVE_TABLE_AND_LIGHT_HEADER = {
@@ -160,6 +160,14 @@ def render_manage(self, record):
                 "pk": record.id,
             },
         )
+
+        dashboard_url = reverse(
+            "program:dashboard",
+            kwargs={
+                "org_slug": self.context["request"].org.slug,
+                "pk": record.id,
+            },
+        )
         application_url = reverse(
             "program:applications",
             kwargs={
@@ -192,6 +200,7 @@ def render_manage(self, record):
                 "color": "success",
                 "icon": "bi bi-people-fill",
             },
+            {"post": False, "url": dashboard_url, "text": "Dashboard", "color": "info", "icon": "bi bi-graph-up"},
         ]
         return get_manage_buttons_html(buttons, self.context["request"])
 
@@ -221,3 +230,77 @@ def get_manage_buttons_html(buttons, request):
         request=request,
     )
     return mark_safe(html)
+
+
+class FunnelPerformanceTable(tables.Table):
+    organization = tables.Column()
+    start_date = tables.DateColumn()
+    workers_invited = tables.Column(verbose_name=_("Workers Invited"))
+    workers_passing_assessment = tables.Column(verbose_name=_("Workers Passing Assessment"))
+    workers_starting_delivery = tables.Column(verbose_name=_("Workers Starting Delivery"))
+    percentage_conversion = tables.Column(verbose_name=_("% Conversion"))
+    average_time_to_convert = tables.Column(verbose_name=_("Average Time To convert"))
+
+    class Meta:
+        model = ManagedOpportunity
+        empty_text = "No data available yet."
+        fields = (
+            "organization",
+            "start_date",
+            "workers_invited",
+            "workers_passing_assessment",
+            "workers_starting_delivery",
+            "percentage_conversion",
+            "average_time_to_convert",
+        )
+        orderable = False
+
+    def render_average_time_to_convert(self, record):
+        total_seconds = record.average_time_to_convert.total_seconds()
+        hours = total_seconds / 3600
+        return f"{round(hours, 2)}hr"
+
+
+class DeliveryPerformanceTable(tables.Table):
+    organization = tables.Column()
+    start_date = tables.DateColumn()
+    workers_invited = tables.Column(
+        empty_values=(),
+        verbose_name=_("Workers Invited"),
+    )
+    workers_starting_delivery = tables.Column(empty_values=(), verbose_name=_("Total Workers Starting Delivery"))
+    active_workers = tables.Column(empty_values=(), verbose_name=_("Active Workers"))
+    deliveries_per_day_per_worker = tables.Column(empty_values=(), verbose_name=_("Deliveries Per Day Per Worker"))
+    percentage_records_flagged = tables.Column(empty_values=(), verbose_name="% Records Flagged")
+
+    class Meta:
+        model = ManagedOpportunity
+        empty_text = "No data available yet."
+        fields = (
+            "organization",
+            "start_date",
+            "Total Workers Starting Delivery",
+            "Active Workers",
+            "Deliveries Per Day Per Worker",
+            "% Records Flagged",
+        )
+        orderable = False
+
+    @staticmethod
+    def get_queryset():
+        queryset = ManagedOpportunity.objects.prefetch_related("opportunityaccess_set")
+        return queryset
+
+    def precompute_data(self, record):
+        """Precompute the values for the record and store them in the record."""
+        if not hasattr(record, "_precomputed_data"):
+            data = {
+                "workers_starting_delivery": 0,
+            }
+            for access in record.opportunityaccess_set.all():
+                if access.last_visit_date:
+                    data["workers_starting_delivery"] += 1
+
+            record._precomputed_data = data
+
+        return record._precomputed_data
diff --git a/commcare_connect/program/tests/test_helpers.py b/commcare_connect/program/tests/test_helpers.py
new file mode 100644
index 00000000..04d4a168
--- /dev/null
+++ b/commcare_connect/program/tests/test_helpers.py
@@ -0,0 +1,37 @@
+from datetime import timedelta
+
+import pytest
+from django.urls import reverse
+from django_celery_beat.utils import now
+
+from commcare_connect.opportunity.models import VisitValidationStatus
+from commcare_connect.opportunity.tests.factories import AssessmentFactory, OpportunityAccessFactory, UserVisitFactory
+from commcare_connect.program.tests.factories import ManagedOpportunityFactory, ProgramFactory
+from commcare_connect.program.tests.test_views import BaseProgramTest
+from commcare_connect.users.tests.factories import OrganizationFactory, UserFactory
+
+
+class TestFunnelPerformanceTable(BaseProgramTest):
+    @pytest.mark.django_db
+    class TestProgramListView(BaseProgramTest):
+        @pytest.fixture(autouse=True)
+        def test_setup(self):
+            self.program = ProgramFactory.create(organization=self.organization)
+            self.list_url = reverse(
+                "program:funnel_performance_table", kwargs={"org_slug": self.organization.slug, "pk": self.program.id}
+            )
+
+            nm_org = OrganizationFactory.create()
+            opp = ManagedOpportunityFactory.create(program=self.program, organization=nm_org)
+            users = UserFactory.create_batch(5)
+            for index, user in enumerate(users):
+                access = OpportunityAccessFactory.create(opportunity=opp, user=user, invited_date=now())
+                AssessmentFactory.create(opportunity=opp, user=user, opportunity_access=access)
+                visit_status = VisitValidationStatus.pending if index < 9 else VisitValidationStatus.trial
+                UserVisitFactory.create(
+                    user=user,
+                    opportunity=opp,
+                    status=visit_status,
+                    opportunity_access=access,
+                    visit_date=now() + timedelta(3),
+                )
diff --git a/commcare_connect/program/urls.py b/commcare_connect/program/urls.py
index 53209232..4e93c995 100644
--- a/commcare_connect/program/urls.py
+++ b/commcare_connect/program/urls.py
@@ -1,12 +1,15 @@
 from django.urls import path
 
 from commcare_connect.program.views import (
+    FunnelPerformanceTableView,
     ManagedOpportunityInit,
     ManagedOpportunityList,
     ProgramApplicationList,
     ProgramCreateOrUpdate,
     ProgramList,
     apply_or_decline_application,
+    dashboard,
+    delivery_table,
     invite_organization,
     manage_application,
 )
@@ -26,4 +29,7 @@
         view=apply_or_decline_application,
         name="apply_or_decline_application",
     ),
+    path("<int:pk>/dashboard", dashboard, name="dashboard"),
+    path("<int:pk>/funnel-performance-table", FunnelPerformanceTableView.as_view(), name="funnel_performance_table"),
+    path("<int:pk>/delivery-performance-table", delivery_table, name="delivery_performance_table"),
 ]
diff --git a/commcare_connect/program/views.py b/commcare_connect/program/views.py
index 4c23d224..70850d9b 100644
--- a/commcare_connect/program/views.py
+++ b/commcare_connect/program/views.py
@@ -1,6 +1,6 @@
 from django.contrib import messages
 from django.contrib.auth.mixins import LoginRequiredMixin, UserPassesTestMixin
-from django.shortcuts import get_object_or_404, redirect
+from django.shortcuts import get_object_or_404, redirect, render
 from django.urls import reverse
 from django.views.decorators.http import require_POST
 from django.views.generic import ListView, UpdateView
@@ -10,8 +10,9 @@
 from commcare_connect.organization.decorators import org_admin_required, org_program_manager_required
 from commcare_connect.organization.models import Organization
 from commcare_connect.program.forms import ManagedOpportunityInitForm, ProgramForm
+from commcare_connect.program.helpers import get_annotated_managed_opportunity
 from commcare_connect.program.models import ManagedOpportunity, Program, ProgramApplication, ProgramApplicationStatus
-from commcare_connect.program.tables import ProgramApplicationTable, ProgramTable
+from commcare_connect.program.tables import FunnelPerformanceTable, ProgramApplicationTable, ProgramTable
 
 
 class ProgramManagerMixin(LoginRequiredMixin, UserPassesTestMixin):
@@ -232,3 +233,31 @@ def apply_or_decline_application(request, application_id, action, org_slug=None,
     messages.success(request, action_map[action]["message"])
 
     return redirect(redirect_url)
+
+
+@org_program_manager_required
+def dashboard(request, **kwargs):
+    program = get_object_or_404(Program, id=kwargs.get("pk"), organization=request.org)
+    context = {
+        "program": program,
+    }
+    return render(request, "program/dashboard.html", context)
+
+
+class FunnelPerformanceTableView(ProgramManagerMixin, SingleTableView):
+    model = ManagedOpportunity
+    paginate_by = 10
+    table_class = FunnelPerformanceTable
+    template_name = "tables/single_table.html"
+
+    def get_queryset(self):
+        program_id = self.kwargs["pk"]
+        program = get_object_or_404(Program, id=program_id)
+        return get_annotated_managed_opportunity(program)
+
+
+@org_program_manager_required
+def delivery_table(request, **kwargs):
+    manage_opps = ManagedOpportunity.objects.filter(program__id=kwargs.get("pk"))
+    delivery_performance_table = FunnelPerformanceTable(manage_opps)
+    return render(request, "tables/single_table.html", {"table": delivery_performance_table})
diff --git a/commcare_connect/templates/program/dashboard.html b/commcare_connect/templates/program/dashboard.html
new file mode 100644
index 00000000..7234964d
--- /dev/null
+++ b/commcare_connect/templates/program/dashboard.html
@@ -0,0 +1,63 @@
+{% extends "program/base.html" %}
+{% load static %}
+{% load i18n %}
+{% load django_tables2 %}
+{% block title %}{{ request.org }} - Programs{% endblock %}
+
+{% block breadcrumbs_inner %}
+{{ block.super }}
+<li class="breadcrumb-item" aria-current="page">{{ program.name }}</li>
+<li class="breadcrumb-item active">{% translate "Dashboard" %}</li>
+{% endblock %}
+{% block content %}
+<div class="container bg-white shadow pb-2">
+  <div class="mt-5 py-3">
+    <h1> Dashboard</h1>
+  </div>
+  <section class="mt-4 shadow mb-5">
+    <ul class="nav nav-tabs fw-bold bg-primary-subtle" role="tablist">
+      <li class="nav-item" role="presentation">
+        <button class="nav-link active" id="funnel-performance-tab" data-bs-toggle="tab"
+                data-bs-target="#funnel-performance-tab-pane"
+                type="button" role="tab" aria-controls="funnel-performance" aria-selected="true">
+          <i class="bi bi-filter-square"></i> {% trans "Funnel Performance" %}
+        </button>
+      </li>
+      <li class="nav-item" role="presentation">
+        <button class="nav-link" id="delivery-performance-tab" data-bs-toggle="tab"
+                data-bs-target="#delivery-performance-tab-pane"
+                type="button" role="tab" aria-controls="delivery-performance" aria-selected="false">
+          <i class="bi bi-journal-text"></i> {% trans "Delivery Performance" %}
+        </button>
+      </li>
+      <li class="nav-item" role="presentation">
+        <button class="nav-link" id="worker-performance-tab" data-bs-toggle="tab"
+                data-bs-target="#worker-performance-tab-pane"
+                type="button" role="tab" aria-controls="worker-performance-tab-pane" aria-selected="false">
+          <i class="bi bi-person-lines-fill"></i> {% trans "Worker Performance" %}
+        </button>
+      </li>
+    </ul>
+    <div class="tab-content">
+      <div class="tab-pane fade show active" id="funnel-performance-tab-pane" role="tabpanel"
+           aria-labelledby="funnel-performance-tab"
+           tabindex="0" hx-on::after-request="refreshTooltips()">
+        <div class="pb-4" id="funnel-performance-table-containers"
+             hx-get="{% url 'program:funnel_performance_table' request.org.slug program.id %}" hx-trigger="load"
+             hx-swap="outerHTML">
+          {% include "tables/table_placeholder.html" with num_cols=6 %}
+        </div>
+      </div>
+      <div class="tab-pane fade show active" id="delivery-performance-tab-pane" role="tabpanel"
+           aria-labelledby="funnel-performance-tab"
+           tabindex="0" hx-on::after-request="refreshTooltips()">
+        <div class="pb-4" id="delivery-performance-table-containers"
+             hx-get="{% url 'program:delivery_performance_table' request.org.slug program.id %}" hx-trigger="load"
+             hx-swap="outerHTML">
+          {% include "tables/table_placeholder.html" with num_cols=7 %}
+        </div>
+      </div>
+    </div>
+  </section>
+</div>
+{% endblock content %}

From b73e31fbfc8a577c4ec9af500b886d484f46da33 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Mon, 16 Sep 2024 16:05:28 +0530
Subject: [PATCH 002/165] refactor code

---
 commcare_connect/program/tables.py            | 47 +-----------------
 .../program/tests/test_helpers.py             | 49 +++++++++----------
 commcare_connect/program/urls.py              |  4 +-
 .../templates/program/dashboard.html          | 26 +---------
 4 files changed, 27 insertions(+), 99 deletions(-)

diff --git a/commcare_connect/program/tables.py b/commcare_connect/program/tables.py
index b3214703..437d4aa2 100644
--- a/commcare_connect/program/tables.py
+++ b/commcare_connect/program/tables.py
@@ -238,7 +238,7 @@ class FunnelPerformanceTable(tables.Table):
     workers_invited = tables.Column(verbose_name=_("Workers Invited"))
     workers_passing_assessment = tables.Column(verbose_name=_("Workers Passing Assessment"))
     workers_starting_delivery = tables.Column(verbose_name=_("Workers Starting Delivery"))
-    percentage_conversion = tables.Column(verbose_name=_("% Conversion"))
+    percentage_conversion = tables.Column(verbose_name=_("Percentage Conversion"))
     average_time_to_convert = tables.Column(verbose_name=_("Average Time To convert"))
 
     class Meta:
@@ -259,48 +259,3 @@ def render_average_time_to_convert(self, record):
         total_seconds = record.average_time_to_convert.total_seconds()
         hours = total_seconds / 3600
         return f"{round(hours, 2)}hr"
-
-
-class DeliveryPerformanceTable(tables.Table):
-    organization = tables.Column()
-    start_date = tables.DateColumn()
-    workers_invited = tables.Column(
-        empty_values=(),
-        verbose_name=_("Workers Invited"),
-    )
-    workers_starting_delivery = tables.Column(empty_values=(), verbose_name=_("Total Workers Starting Delivery"))
-    active_workers = tables.Column(empty_values=(), verbose_name=_("Active Workers"))
-    deliveries_per_day_per_worker = tables.Column(empty_values=(), verbose_name=_("Deliveries Per Day Per Worker"))
-    percentage_records_flagged = tables.Column(empty_values=(), verbose_name="% Records Flagged")
-
-    class Meta:
-        model = ManagedOpportunity
-        empty_text = "No data available yet."
-        fields = (
-            "organization",
-            "start_date",
-            "Total Workers Starting Delivery",
-            "Active Workers",
-            "Deliveries Per Day Per Worker",
-            "% Records Flagged",
-        )
-        orderable = False
-
-    @staticmethod
-    def get_queryset():
-        queryset = ManagedOpportunity.objects.prefetch_related("opportunityaccess_set")
-        return queryset
-
-    def precompute_data(self, record):
-        """Precompute the values for the record and store them in the record."""
-        if not hasattr(record, "_precomputed_data"):
-            data = {
-                "workers_starting_delivery": 0,
-            }
-            for access in record.opportunityaccess_set.all():
-                if access.last_visit_date:
-                    data["workers_starting_delivery"] += 1
-
-            record._precomputed_data = data
-
-        return record._precomputed_data
diff --git a/commcare_connect/program/tests/test_helpers.py b/commcare_connect/program/tests/test_helpers.py
index 04d4a168..8339976f 100644
--- a/commcare_connect/program/tests/test_helpers.py
+++ b/commcare_connect/program/tests/test_helpers.py
@@ -1,37 +1,34 @@
 from datetime import timedelta
 
-import pytest
-from django.urls import reverse
 from django_celery_beat.utils import now
 
 from commcare_connect.opportunity.models import VisitValidationStatus
 from commcare_connect.opportunity.tests.factories import AssessmentFactory, OpportunityAccessFactory, UserVisitFactory
+from commcare_connect.organization.models import Organization
+from commcare_connect.program.helpers import get_annotated_managed_opportunity
 from commcare_connect.program.tests.factories import ManagedOpportunityFactory, ProgramFactory
-from commcare_connect.program.tests.test_views import BaseProgramTest
 from commcare_connect.users.tests.factories import OrganizationFactory, UserFactory
 
 
-class TestFunnelPerformanceTable(BaseProgramTest):
-    @pytest.mark.django_db
-    class TestProgramListView(BaseProgramTest):
-        @pytest.fixture(autouse=True)
-        def test_setup(self):
-            self.program = ProgramFactory.create(organization=self.organization)
-            self.list_url = reverse(
-                "program:funnel_performance_table", kwargs={"org_slug": self.organization.slug, "pk": self.program.id}
-            )
+def test_get_annotated_managed_opportunity(program_manager_org: Organization):
+    program = ProgramFactory.create(organization=program_manager_org)
+    nm_org = OrganizationFactory.create()
+    opp = ManagedOpportunityFactory.create(program=program, organization=nm_org)
+    users = UserFactory.create_batch(5)
+    for index, user in enumerate(users):
+        access = OpportunityAccessFactory.create(opportunity=opp, user=user, invited_date=now())
+        AssessmentFactory.create(opportunity=opp, user=user, opportunity_access=access)
+        visit_status = VisitValidationStatus.pending if index < 3 else VisitValidationStatus.trial
+        UserVisitFactory.create(
+            user=user,
+            opportunity=opp,
+            status=visit_status,
+            opportunity_access=access,
+            visit_date=now() + timedelta(1),
+        )
 
-            nm_org = OrganizationFactory.create()
-            opp = ManagedOpportunityFactory.create(program=self.program, organization=nm_org)
-            users = UserFactory.create_batch(5)
-            for index, user in enumerate(users):
-                access = OpportunityAccessFactory.create(opportunity=opp, user=user, invited_date=now())
-                AssessmentFactory.create(opportunity=opp, user=user, opportunity_access=access)
-                visit_status = VisitValidationStatus.pending if index < 9 else VisitValidationStatus.trial
-                UserVisitFactory.create(
-                    user=user,
-                    opportunity=opp,
-                    status=visit_status,
-                    opportunity_access=access,
-                    visit_date=now() + timedelta(3),
-                )
+    opps = get_annotated_managed_opportunity(program)
+    for opp in opps:
+        assert nm_org.slug == opp.organization.slug
+        assert opp.workers_passing_assessment == 5
+        assert opp.workers_starting_delivery == 3
diff --git a/commcare_connect/program/urls.py b/commcare_connect/program/urls.py
index 4e93c995..339771bb 100644
--- a/commcare_connect/program/urls.py
+++ b/commcare_connect/program/urls.py
@@ -9,7 +9,6 @@
     ProgramList,
     apply_or_decline_application,
     dashboard,
-    delivery_table,
     invite_organization,
     manage_application,
 )
@@ -30,6 +29,5 @@
         name="apply_or_decline_application",
     ),
     path("<int:pk>/dashboard", dashboard, name="dashboard"),
-    path("<int:pk>/funnel-performance-table", FunnelPerformanceTableView.as_view(), name="funnel_performance_table"),
-    path("<int:pk>/delivery-performance-table", delivery_table, name="delivery_performance_table"),
+    path("<int:pk>/funnel_performance_table", FunnelPerformanceTableView.as_view(), name="funnel_performance_table"),
 ]
diff --git a/commcare_connect/templates/program/dashboard.html b/commcare_connect/templates/program/dashboard.html
index 7234964d..c5c7d895 100644
--- a/commcare_connect/templates/program/dashboard.html
+++ b/commcare_connect/templates/program/dashboard.html
@@ -12,7 +12,7 @@
 {% block content %}
 <div class="container bg-white shadow pb-2">
   <div class="mt-5 py-3">
-    <h1> Dashboard</h1>
+    <h1> {% trans "Dashboard" %}</h1>
   </div>
   <section class="mt-4 shadow mb-5">
     <ul class="nav nav-tabs fw-bold bg-primary-subtle" role="tablist">
@@ -23,20 +23,7 @@ <h1> Dashboard</h1>
           <i class="bi bi-filter-square"></i> {% trans "Funnel Performance" %}
         </button>
       </li>
-      <li class="nav-item" role="presentation">
-        <button class="nav-link" id="delivery-performance-tab" data-bs-toggle="tab"
-                data-bs-target="#delivery-performance-tab-pane"
-                type="button" role="tab" aria-controls="delivery-performance" aria-selected="false">
-          <i class="bi bi-journal-text"></i> {% trans "Delivery Performance" %}
-        </button>
-      </li>
-      <li class="nav-item" role="presentation">
-        <button class="nav-link" id="worker-performance-tab" data-bs-toggle="tab"
-                data-bs-target="#worker-performance-tab-pane"
-                type="button" role="tab" aria-controls="worker-performance-tab-pane" aria-selected="false">
-          <i class="bi bi-person-lines-fill"></i> {% trans "Worker Performance" %}
-        </button>
-      </li>
+
     </ul>
     <div class="tab-content">
       <div class="tab-pane fade show active" id="funnel-performance-tab-pane" role="tabpanel"
@@ -48,15 +35,6 @@ <h1> Dashboard</h1>
           {% include "tables/table_placeholder.html" with num_cols=6 %}
         </div>
       </div>
-      <div class="tab-pane fade show active" id="delivery-performance-tab-pane" role="tabpanel"
-           aria-labelledby="funnel-performance-tab"
-           tabindex="0" hx-on::after-request="refreshTooltips()">
-        <div class="pb-4" id="delivery-performance-table-containers"
-             hx-get="{% url 'program:delivery_performance_table' request.org.slug program.id %}" hx-trigger="load"
-             hx-swap="outerHTML">
-          {% include "tables/table_placeholder.html" with num_cols=7 %}
-        </div>
-      </div>
     </div>
   </section>
 </div>

From 0c02dc370db9617d5a6177af62d351a33947f9cf Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Mon, 16 Sep 2024 16:18:14 +0530
Subject: [PATCH 003/165] Removed unused code

---
 commcare_connect/program/helpers.py | 15 ---------------
 commcare_connect/program/views.py   |  7 -------
 2 files changed, 22 deletions(-)

diff --git a/commcare_connect/program/helpers.py b/commcare_connect/program/helpers.py
index 7f961b74..a9bcd9f7 100644
--- a/commcare_connect/program/helpers.py
+++ b/commcare_connect/program/helpers.py
@@ -54,18 +54,3 @@ def get_annotated_managed_opportunity(program: Program):
     )
 
     return managed_opportunities
-
-
-def get_annotated_managed_opportunity_nm(program: Program, start_date=None, end_date=None):
-    managed_opportunities = (
-        ManagedOpportunity.objects.filter(program=program, start_date__gte=start_date)
-        .order_by("start_date")
-        .annotate()
-        .prefetch_related(
-            "opportunityaccess_set",
-            "opportunityaccess_set__uservisit_set",
-            "opportunityaccess_set__assessment_set",
-        )
-    )
-
-    return managed_opportunities
diff --git a/commcare_connect/program/views.py b/commcare_connect/program/views.py
index 70850d9b..15db42a5 100644
--- a/commcare_connect/program/views.py
+++ b/commcare_connect/program/views.py
@@ -254,10 +254,3 @@ def get_queryset(self):
         program_id = self.kwargs["pk"]
         program = get_object_or_404(Program, id=program_id)
         return get_annotated_managed_opportunity(program)
-
-
-@org_program_manager_required
-def delivery_table(request, **kwargs):
-    manage_opps = ManagedOpportunity.objects.filter(program__id=kwargs.get("pk"))
-    delivery_performance_table = FunnelPerformanceTable(manage_opps)
-    return render(request, "tables/single_table.html", {"table": delivery_performance_table})

From be2ba7d2245a151456c255362355f9e273ca670a Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 18 Sep 2024 15:53:44 +0530
Subject: [PATCH 004/165] fixed migration sequence

---
 ...invited_date.py => 0059_opportunityaccess_invited_date.py} | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
 rename commcare_connect/opportunity/migrations/{0055_opportunityaccess_invited_date.py => 0059_opportunityaccess_invited_date.py} (70%)

diff --git a/commcare_connect/opportunity/migrations/0055_opportunityaccess_invited_date.py b/commcare_connect/opportunity/migrations/0059_opportunityaccess_invited_date.py
similarity index 70%
rename from commcare_connect/opportunity/migrations/0055_opportunityaccess_invited_date.py
rename to commcare_connect/opportunity/migrations/0059_opportunityaccess_invited_date.py
index b55f94d5..1382f847 100644
--- a/commcare_connect/opportunity/migrations/0055_opportunityaccess_invited_date.py
+++ b/commcare_connect/opportunity/migrations/0059_opportunityaccess_invited_date.py
@@ -1,11 +1,11 @@
-# Generated by Django 4.2.5 on 2024-09-12 18:13
+# Generated by Django 4.2.5 on 2024-09-18 10:16
 
 from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
     dependencies = [
-        ("opportunity", "0054_opportunity_managed_alter_opportunity_organization"),
+        ("opportunity", "0058_paymentinvoice_payment_invoice"),
     ]
 
     operations = [

From 568bb8e550af931e2949887f56281c56d93f7586 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 18 Sep 2024 16:23:13 +0530
Subject: [PATCH 005/165] Made improvements in function based views

---
 commcare_connect/opportunity/views.py | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/commcare_connect/opportunity/views.py b/commcare_connect/opportunity/views.py
index 9b6c00aa..a5b5a72d 100644
--- a/commcare_connect/opportunity/views.py
+++ b/commcare_connect/opportunity/views.py
@@ -357,8 +357,7 @@ def get_queryset(self):
 
 
 @org_member_required
-def export_user_visits(request, **kwargs):
-    opportunity_id = kwargs["pk"]
+def export_user_visits(request, org_slug, opportunity_id):
     get_opportunity_or_404(org_slug=request.org.slug, pk=opportunity_id)
     form = VisitExportForm(data=request.POST)
     if not form.is_valid():
@@ -483,8 +482,7 @@ def get_queryset(self):
 
 
 @org_member_required
-def export_users_for_payment(request, **kwargs):
-    opportunity_id = kwargs["pk"]
+def export_users_for_payment(request, org_slug, opportunity_id):
     get_opportunity_or_404(org_slug=request.org.slug, pk=opportunity_id)
     form = PaymentExportForm(data=request.POST)
     if not form.is_valid():
@@ -629,8 +627,7 @@ def get_queryset(self):
 
 
 @org_member_required
-def export_user_status(request, **kwargs):
-    opportunity_id = kwargs["pk"]
+def export_user_status(request, org_slug, opportunity_id):
     get_opportunity_or_404(org_slug=request.org.slug, pk=opportunity_id)
     form = PaymentExportForm(data=request.POST)
     if not form.is_valid():
@@ -658,8 +655,7 @@ def get_queryset(self):
 
 
 @org_member_required
-def export_deliver_status(request, **kwargs):
-    opportunity_id = kwargs["pk"]
+def export_deliver_status(request, org_slug, opportunity_id):
     get_opportunity_or_404(pk=opportunity_id, org_slug=request.org.slug)
     form = PaymentExportForm(data=request.POST)
     if not form.is_valid():
@@ -987,8 +983,7 @@ def get_queryset(self):
 
 
 @org_member_required
-def export_completed_work(request, **kwargs):
-    opportunity_id = kwargs["pk"]
+def export_completed_work(request, org_slug, opportunity_id):
     get_opportunity_or_404(org_slug=request.org.slug, pk=opportunity_id)
     form = PaymentExportForm(data=request.POST)
     if not form.is_valid():
@@ -1052,8 +1047,7 @@ def suspended_users_list(request, org_slug=None, pk=None):
 
 
 @org_member_required
-def export_catchment_area(request, **kwargs):
-    opportunity_id = kwargs["pk"]
+def export_catchment_area(request, org_slug, opportunity_id):
     get_opportunity_or_404(org_slug=request.org.slug, pk=opportunity_id)
     form = PaymentExportForm(data=request.POST)
     if not form.is_valid():

From 0cb0721ff9ac8669f31313a317450e954a58e088 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 18 Sep 2024 17:14:06 +0530
Subject: [PATCH 006/165] fixed the args name issue

---
 commcare_connect/opportunity/views.py | 60 +++++++++++++--------------
 1 file changed, 30 insertions(+), 30 deletions(-)

diff --git a/commcare_connect/opportunity/views.py b/commcare_connect/opportunity/views.py
index a5b5a72d..289448a3 100644
--- a/commcare_connect/opportunity/views.py
+++ b/commcare_connect/opportunity/views.py
@@ -357,20 +357,20 @@ def get_queryset(self):
 
 
 @org_member_required
-def export_user_visits(request, org_slug, opportunity_id):
-    get_opportunity_or_404(org_slug=request.org.slug, pk=opportunity_id)
+def export_user_visits(request, org_slug, pk):
+    get_opportunity_or_404(org_slug=request.org.slug, pk=pk)
     form = VisitExportForm(data=request.POST)
     if not form.is_valid():
         messages.error(request, form.errors)
-        return redirect("opportunity:detail", request.org.slug, opportunity_id)
+        return redirect("opportunity:detail", request.org.slug, pk)
 
     export_format = form.cleaned_data["format"]
     date_range = DateRanges(form.cleaned_data["date_range"])
     status = form.cleaned_data["status"]
     flatten = form.cleaned_data["flatten_form_data"]
 
-    result = generate_visit_export.delay(opportunity_id, date_range, status, export_format, flatten)
-    redirect_url = reverse("opportunity:detail", args=(request.org.slug, opportunity_id))
+    result = generate_visit_export.delay(pk, date_range, status, export_format, flatten)
+    redirect_url = reverse("opportunity:detail", args=(request.org.slug, pk))
     return redirect(f"{redirect_url}?export_task_id={result.id}")
 
 
@@ -482,16 +482,16 @@ def get_queryset(self):
 
 
 @org_member_required
-def export_users_for_payment(request, org_slug, opportunity_id):
-    get_opportunity_or_404(org_slug=request.org.slug, pk=opportunity_id)
+def export_users_for_payment(request, org_slug, pk):
+    get_opportunity_or_404(org_slug=request.org.slug, pk=pk)
     form = PaymentExportForm(data=request.POST)
     if not form.is_valid():
         messages.error(request, form.errors)
-        return redirect("opportunity:detail", request.org.slug, opportunity_id)
+        return redirect("opportunity:detail", request.org.slug, pk)
 
     export_format = form.cleaned_data["format"]
-    result = generate_payment_export.delay(opportunity_id, export_format)
-    redirect_url = reverse("opportunity:detail", args=(request.org.slug, opportunity_id))
+    result = generate_payment_export.delay(pk, export_format)
+    redirect_url = reverse("opportunity:detail", args=(request.org.slug, pk))
     return redirect(f"{redirect_url}?export_task_id={result.id}")
 
 
@@ -627,16 +627,16 @@ def get_queryset(self):
 
 
 @org_member_required
-def export_user_status(request, org_slug, opportunity_id):
-    get_opportunity_or_404(org_slug=request.org.slug, pk=opportunity_id)
+def export_user_status(request, org_slug, pk):
+    get_opportunity_or_404(org_slug=request.org.slug, pk=pk)
     form = PaymentExportForm(data=request.POST)
     if not form.is_valid():
         messages.error(request, form.errors)
-        return redirect("opportunity:detail", request.org.slug, opportunity_id)
+        return redirect("opportunity:detail", request.org.slug, pk)
 
     export_format = form.cleaned_data["format"]
-    result = generate_user_status_export.delay(opportunity_id, export_format)
-    redirect_url = reverse("opportunity:detail", args=(request.org.slug, opportunity_id))
+    result = generate_user_status_export.delay(pk, export_format)
+    redirect_url = reverse("opportunity:detail", args=(request.org.slug, pk))
     return redirect(f"{redirect_url}?export_task_id={result.id}")
 
 
@@ -655,16 +655,16 @@ def get_queryset(self):
 
 
 @org_member_required
-def export_deliver_status(request, org_slug, opportunity_id):
-    get_opportunity_or_404(pk=opportunity_id, org_slug=request.org.slug)
+def export_deliver_status(request, org_slug, pk):
+    get_opportunity_or_404(pk=pk, org_slug=request.org.slug)
     form = PaymentExportForm(data=request.POST)
     if not form.is_valid():
         messages.error(request, form.errors)
-        return redirect("opportunity:detail", request.org.slug, opportunity_id)
+        return redirect("opportunity:detail", request.org.slug, pk)
 
     export_format = form.cleaned_data["format"]
-    result = generate_deliver_status_export.delay(opportunity_id, export_format)
-    redirect_url = reverse("opportunity:detail", args=(request.org.slug, opportunity_id))
+    result = generate_deliver_status_export.delay(pk, export_format)
+    redirect_url = reverse("opportunity:detail", args=(request.org.slug, pk))
     return redirect(f"{redirect_url}?export_task_id={result.id}")
 
 
@@ -983,16 +983,16 @@ def get_queryset(self):
 
 
 @org_member_required
-def export_completed_work(request, org_slug, opportunity_id):
-    get_opportunity_or_404(org_slug=request.org.slug, pk=opportunity_id)
+def export_completed_work(request, org_slug, pk):
+    get_opportunity_or_404(org_slug=request.org.slug, pk=pk)
     form = PaymentExportForm(data=request.POST)
     if not form.is_valid():
         messages.error(request, form.errors)
-        return redirect("opportunity:detail", request.org.slug, opportunity_id)
+        return redirect("opportunity:detail", request.org.slug, pk)
 
     export_format = form.cleaned_data["format"]
-    result = generate_work_status_export.delay(opportunity_id, export_format)
-    redirect_url = reverse("opportunity:detail", args=(request.org.slug, opportunity_id))
+    result = generate_work_status_export.delay(pk, export_format)
+    redirect_url = reverse("opportunity:detail", args=(request.org.slug, pk))
     return redirect(f"{redirect_url}?export_task_id={result.id}")
 
 
@@ -1047,16 +1047,16 @@ def suspended_users_list(request, org_slug=None, pk=None):
 
 
 @org_member_required
-def export_catchment_area(request, org_slug, opportunity_id):
-    get_opportunity_or_404(org_slug=request.org.slug, pk=opportunity_id)
+def export_catchment_area(request, org_slug, pk):
+    get_opportunity_or_404(org_slug=request.org.slug, pk=pk)
     form = PaymentExportForm(data=request.POST)
     if not form.is_valid():
         messages.error(request, form.errors)
-        return redirect("opportunity:detail", request.org.slug, opportunity_id)
+        return redirect("opportunity:detail", request.org.slug, pk)
 
     export_format = form.cleaned_data["format"]
-    result = generate_catchment_area_export.delay(opportunity_id, export_format)
-    redirect_url = reverse("opportunity:detail", args=(request.org.slug, opportunity_id))
+    result = generate_catchment_area_export.delay(pk, export_format)
+    redirect_url = reverse("opportunity:detail", args=(request.org.slug, pk))
     return redirect(f"{redirect_url}?export_task_id={result.id}")
 
 

From a8f01206c6e5ee998645182ec92e02c3fbad9fb5 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 25 Sep 2024 11:28:05 +0530
Subject: [PATCH 007/165] added performace table query and table

---
 commcare_connect/program/helpers.py           | 58 ++++++++++++++++---
 commcare_connect/program/tables.py            | 22 +++++++
 .../program/tests/test_helpers.py             | 24 ++++++++
 commcare_connect/program/urls.py              |  9 ++-
 commcare_connect/program/views.py             | 24 +++++++-
 .../templates/program/dashboard.html          | 37 ++++++++++++
 6 files changed, 161 insertions(+), 13 deletions(-)

diff --git a/commcare_connect/program/helpers.py b/commcare_connect/program/helpers.py
index a9bcd9f7..6ee46f45 100644
--- a/commcare_connect/program/helpers.py
+++ b/commcare_connect/program/helpers.py
@@ -3,15 +3,15 @@
 from commcare_connect.opportunity.models import UserVisit, VisitValidationStatus
 from commcare_connect.program.models import ManagedOpportunity, Program
 
+FILTER_FOR_VALID_VISIT_DATE = ~Q(
+    opportunityaccess__uservisit__status__in=[
+        VisitValidationStatus.over_limit,
+        VisitValidationStatus.trial,
+    ]
+)
 
-def get_annotated_managed_opportunity(program: Program):
-    filter_for_valid__visit_date = ~Q(
-        opportunityaccess__uservisit__status__in=[
-            VisitValidationStatus.over_limit,
-            VisitValidationStatus.trial,
-        ]
-    )
 
+def get_annotated_managed_opportunity(program: Program):
     earliest_visits = (
         UserVisit.objects.filter(
             opportunity_access=OuterRef("opportunityaccess"),
@@ -35,7 +35,7 @@ def get_annotated_managed_opportunity(program: Program):
             ),
             workers_starting_delivery=Count(
                 "opportunityaccess__uservisit__user",
-                filter=filter_for_valid__visit_date,
+                filter=FILTER_FOR_VALID_VISIT_DATE,
                 distinct=True,
             ),
             percentage_conversion=F("workers_starting_delivery") / F("workers_invited") * 100,
@@ -43,7 +43,7 @@ def get_annotated_managed_opportunity(program: Program):
                 ExpressionWrapper(
                     Subquery(earliest_visits) - F("opportunityaccess__invited_date"), output_field=DurationField()
                 ),
-                filter=filter_for_valid__visit_date,
+                filter=FILTER_FOR_VALID_VISIT_DATE,
             ),
         )
         .prefetch_related(
@@ -54,3 +54,43 @@ def get_annotated_managed_opportunity(program: Program):
     )
 
     return managed_opportunities
+
+
+def get_delivery_performance_report(program: Program, start_date, end_date):
+    date_filter = Q()
+
+    if start_date:
+        date_filter &= Q(opportunityaccess__uservisit__visit_date__gte=start_date)
+
+    if end_date:
+        date_filter &= Q(opportunityaccess__uservisit__visit_date__lte=end_date)
+
+    active_workers_filter = Q(FILTER_FOR_VALID_VISIT_DATE, date_filter)
+
+    managed_opportunities = (
+        ManagedOpportunity.objects.filter(program=program)
+        .order_by("start_date")
+        .prefetch_related(
+            "opportunityaccess_set",
+            "opportunityaccess_set__uservisit_set",
+            "opportunityaccess_set__completedwork_set",
+        )
+        .annotate(
+            total_workers_starting_delivery=Count(
+                "opportunityaccess__uservisit__user",
+                filter=FILTER_FOR_VALID_VISIT_DATE,
+                distinct=True,
+            ),
+            active_workers=Count(
+                "opportunityaccess__uservisit__user",
+                filter=active_workers_filter,
+                distinct=True,
+            ),
+            total_payment_units=Count("opportunityaccess__completedwork", distinct=True),
+            total_payement_since_start_date=Count("opportunityaccess__completedwork", distinct=True),
+            deliveries_per_day=F("total_payement_since_start_date") / F("active_workers"),
+            records_flagged_percentage=F("total_payment_units") / F("total_payement_since_start_date"),
+        )
+    )
+
+    return managed_opportunities
diff --git a/commcare_connect/program/tables.py b/commcare_connect/program/tables.py
index 4e964d4c..bc5b68ea 100644
--- a/commcare_connect/program/tables.py
+++ b/commcare_connect/program/tables.py
@@ -262,3 +262,25 @@ def render_average_time_to_convert(self, record):
         total_seconds = record.average_time_to_convert.total_seconds()
         hours = total_seconds / 3600
         return f"{round(hours, 2)}hr"
+
+
+class DeliveryPerformanceTable(tables.Table):
+    organization = tables.Column()
+    start_date = tables.DateColumn()
+    workers_invited = tables.Column(verbose_name=_("Workers Starting Delivery"))
+    active_workers = tables.Column(verbose_name=_("Active Workers"))
+    delivery_per_day_per_worker = tables.Column(verbose_name=_("Delivery Per Day Per Worker"))
+    records_flagged = tables.Column(verbose_name=_("Records flagged"))
+
+    class Meta:
+        model = ManagedOpportunity
+        empty_text = "No data available yet."
+        fields = (
+            "organization",
+            "start_date",
+            "workers_starting_delivery",
+            "active_workers",
+            "delivery_per_day_per_worker",
+            "records_flagged",
+        )
+        orderable = False
diff --git a/commcare_connect/program/tests/test_helpers.py b/commcare_connect/program/tests/test_helpers.py
index 8339976f..e4f2ba6f 100644
--- a/commcare_connect/program/tests/test_helpers.py
+++ b/commcare_connect/program/tests/test_helpers.py
@@ -32,3 +32,27 @@ def test_get_annotated_managed_opportunity(program_manager_org: Organization):
         assert nm_org.slug == opp.organization.slug
         assert opp.workers_passing_assessment == 5
         assert opp.workers_starting_delivery == 3
+
+
+def test_delivery_performance(program_manager_org: Organization):
+    program = ProgramFactory.create(organization=program_manager_org)
+    nm_org = OrganizationFactory.create()
+    opp = ManagedOpportunityFactory.create(program=program, organization=nm_org)
+    users = UserFactory.create_batch(5)
+    for index, user in enumerate(users):
+        access = OpportunityAccessFactory.create(opportunity=opp, user=user, invited_date=now())
+        AssessmentFactory.create(opportunity=opp, user=user, opportunity_access=access)
+        visit_status = VisitValidationStatus.pending if index < 3 else VisitValidationStatus.trial
+        UserVisitFactory.create(
+            user=user,
+            opportunity=opp,
+            status=visit_status,
+            opportunity_access=access,
+            visit_date=now() + timedelta(1),
+        )
+
+    opps = get_annotated_managed_opportunity(program)
+    for opp in opps:
+        assert nm_org.slug == opp.organization.slug
+        assert opp.workers_passing_assessment == 5
+        assert opp.workers_starting_delivery == 3
diff --git a/commcare_connect/program/urls.py b/commcare_connect/program/urls.py
index 339771bb..7f96d4c9 100644
--- a/commcare_connect/program/urls.py
+++ b/commcare_connect/program/urls.py
@@ -1,7 +1,7 @@
 from django.urls import path
 
 from commcare_connect.program.views import (
-    FunnelPerformanceTableView,
+    DeliveryPerformanceTableView,
     ManagedOpportunityInit,
     ManagedOpportunityList,
     ProgramApplicationList,
@@ -29,5 +29,10 @@
         name="apply_or_decline_application",
     ),
     path("<int:pk>/dashboard", dashboard, name="dashboard"),
-    path("<int:pk>/funnel_performance_table", FunnelPerformanceTableView.as_view(), name="funnel_performance_table"),
+    path("<int:pk>/funnel_performance_table", DeliveryPerformanceTableView.as_view(), name="funnel_performance_table"),
+    path(
+        "<int:pk>/delivery_performance_table",
+        DeliveryPerformanceTableView.as_view(),
+        name="delivery_performance_table",
+    ),
 ]
diff --git a/commcare_connect/program/views.py b/commcare_connect/program/views.py
index c7b21f94..ab23240a 100644
--- a/commcare_connect/program/views.py
+++ b/commcare_connect/program/views.py
@@ -10,9 +10,14 @@
 from commcare_connect.organization.decorators import org_admin_required, org_program_manager_required
 from commcare_connect.organization.models import Organization
 from commcare_connect.program.forms import ManagedOpportunityInitForm, ProgramForm
-from commcare_connect.program.helpers import get_annotated_managed_opportunity
+from commcare_connect.program.helpers import get_annotated_managed_opportunity, get_delivery_performance_report
 from commcare_connect.program.models import ManagedOpportunity, Program, ProgramApplication, ProgramApplicationStatus
-from commcare_connect.program.tables import FunnelPerformanceTable, ProgramApplicationTable, ProgramTable
+from commcare_connect.program.tables import (
+    DeliveryPerformanceTable,
+    FunnelPerformanceTable,
+    ProgramApplicationTable,
+    ProgramTable,
+)
 
 
 class ProgramManagerMixin(LoginRequiredMixin, UserPassesTestMixin):
@@ -258,3 +263,18 @@ def get_queryset(self):
         program_id = self.kwargs["pk"]
         program = get_object_or_404(Program, id=program_id)
         return get_annotated_managed_opportunity(program)
+
+
+class DeliveryPerformanceTableView(ProgramManagerMixin, SingleTableView):
+    model = ManagedOpportunity
+    paginate_by = 10
+    table_class = DeliveryPerformanceTable
+    template_name = "tables/single_table.html"
+
+    def get_queryset(self):
+        program_id = self.kwargs["pk"]
+        program = get_object_or_404(Program, id=program_id)
+        start_date = self.request.GET.get("start_date") or None
+        end_date = self.request.GET.get("end_date") or None
+
+        return get_delivery_performance_report(program, start_date, end_date)
diff --git a/commcare_connect/templates/program/dashboard.html b/commcare_connect/templates/program/dashboard.html
index c5c7d895..57f3b461 100644
--- a/commcare_connect/templates/program/dashboard.html
+++ b/commcare_connect/templates/program/dashboard.html
@@ -24,6 +24,14 @@ <h1> {% trans "Dashboard" %}</h1>
         </button>
       </li>
 
+      <li class="nav-item" role="presentation">
+        <button class="nav-link" id="delivery-performance-tab" data-bs-toggle="tab"
+                data-bs-target="#delivery-performance-tab-pane"
+                type="button" role="tab" aria-controls="delivery-performance" aria-selected="false">
+          <i class="bi bi-journal-text"></i> {% trans "Delivery Performance" %}
+        </button>
+      </li>
+
     </ul>
     <div class="tab-content">
       <div class="tab-pane fade show active" id="funnel-performance-tab-pane" role="tabpanel"
@@ -35,6 +43,35 @@ <h1> {% trans "Dashboard" %}</h1>
           {% include "tables/table_placeholder.html" with num_cols=6 %}
         </div>
       </div>
+
+      <div class="tab-pane fade" id="delivery-performance-tab-pane" role="tabpanel"
+           aria-labelledby="funnel-performance-tab"
+           tabindex="0" hx-on::after-request="refreshTooltips()">
+        <div class="d-flex gap-3 justify-content-end align-items-end my-3 me-3">
+          <div>
+            <label for="startDate">Start Date</label>
+            <input id="startDate" name="start_date" class="form-control" type="date"/>
+          </div>
+          <div>
+            <label for="endDate">End Date</label>
+            <input id="endDate" name="end_date" class="form-control" type="date"/>
+          </div>
+          <button class="btn btn-primary h-50"
+                  hx-get="{% url "program:delivery_performance_table" org_slug=request.org.slug pk=program.id %}{% querystring %}"
+                  hx-include="[name='start_date'],[name='end_date']"
+                  hx-trigger="click"
+                  hx-target="#performanceTable">
+            <i class="bi bi-funnel-fill"></i> Filter
+          </button>
+        </div>
+        <div id="performanceTable">
+        <div class="pb-4" id="delivery-performance-table-containers"
+             hx-get="{% url "program:delivery_performance_table" org_slug=request.org.slug pk=program.id %}{% querystring %}"
+             hx-trigger="load"
+             hx-swap="outerHTML">
+          {% include "tables/table_placeholder.html" with num_cols=7 %}
+        </div>
+      </div>
     </div>
   </section>
 </div>

From d5ea308a314df5f84245c7862d0d40afbde65140 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 25 Sep 2024 13:56:14 +0530
Subject: [PATCH 008/165] Refactor logic and changed tests

---
 commcare_connect/program/helpers.py           | 32 ++++++++++++++++---
 commcare_connect/program/tables.py            |  8 ++---
 .../program/tests/test_helpers.py             | 23 ++++++++-----
 commcare_connect/program/views.py             |  2 ++
 4 files changed, 48 insertions(+), 17 deletions(-)

diff --git a/commcare_connect/program/helpers.py b/commcare_connect/program/helpers.py
index 6ee46f45..933b4a3a 100644
--- a/commcare_connect/program/helpers.py
+++ b/commcare_connect/program/helpers.py
@@ -1,4 +1,18 @@
-from django.db.models import Avg, Count, DurationField, ExpressionWrapper, F, OuterRef, Q, Subquery
+from django.db.models import (
+    Avg,
+    Case,
+    Count,
+    DurationField,
+    ExpressionWrapper,
+    F,
+    FloatField,
+    OuterRef,
+    Q,
+    Subquery,
+    Value,
+    When,
+)
+from django.db.models.functions import Round
 
 from commcare_connect.opportunity.models import UserVisit, VisitValidationStatus
 from commcare_connect.program.models import ManagedOpportunity, Program
@@ -86,10 +100,18 @@ def get_delivery_performance_report(program: Program, start_date, end_date):
                 filter=active_workers_filter,
                 distinct=True,
             ),
-            total_payment_units=Count("opportunityaccess__completedwork", distinct=True),
-            total_payement_since_start_date=Count("opportunityaccess__completedwork", distinct=True),
-            deliveries_per_day=F("total_payement_since_start_date") / F("active_workers"),
-            records_flagged_percentage=F("total_payment_units") / F("total_payement_since_start_date"),
+            total_payment_units=Count("opportunityaccess__completedwork"),
+            total_payement_since_start_date=Count("opportunityaccess__completedwork", filter=date_filter),
+            delivery_per_day_per_worker=Case(
+                When(active_workers=0, then=Value(0)),
+                default=Round(F("total_payement_since_start_date") / F("active_workers"), 2),
+                output_field=FloatField(),
+            ),
+            records_flagged_percentage=Case(
+                When(active_workers=0, then=Value(0)),
+                default=Round(F("total_payment_units") / F("total_payement_since_start_date")),
+                output_field=FloatField(),
+            ),
         )
     )
 
diff --git a/commcare_connect/program/tables.py b/commcare_connect/program/tables.py
index bc5b68ea..b36a3001 100644
--- a/commcare_connect/program/tables.py
+++ b/commcare_connect/program/tables.py
@@ -267,10 +267,10 @@ def render_average_time_to_convert(self, record):
 class DeliveryPerformanceTable(tables.Table):
     organization = tables.Column()
     start_date = tables.DateColumn()
-    workers_invited = tables.Column(verbose_name=_("Workers Starting Delivery"))
+    total_workers_starting_delivery = tables.Column(verbose_name=_("Workers Starting Delivery"))
     active_workers = tables.Column(verbose_name=_("Active Workers"))
     delivery_per_day_per_worker = tables.Column(verbose_name=_("Delivery Per Day Per Worker"))
-    records_flagged = tables.Column(verbose_name=_("Records flagged"))
+    records_flagged_percentage = tables.Column(verbose_name=_("Records flagged"))
 
     class Meta:
         model = ManagedOpportunity
@@ -278,9 +278,9 @@ class Meta:
         fields = (
             "organization",
             "start_date",
-            "workers_starting_delivery",
+            "total_workers_starting_delivery",
             "active_workers",
             "delivery_per_day_per_worker",
-            "records_flagged",
+            "records_flagged_percentage",
         )
         orderable = False
diff --git a/commcare_connect/program/tests/test_helpers.py b/commcare_connect/program/tests/test_helpers.py
index e4f2ba6f..50b8bf56 100644
--- a/commcare_connect/program/tests/test_helpers.py
+++ b/commcare_connect/program/tests/test_helpers.py
@@ -3,9 +3,14 @@
 from django_celery_beat.utils import now
 
 from commcare_connect.opportunity.models import VisitValidationStatus
-from commcare_connect.opportunity.tests.factories import AssessmentFactory, OpportunityAccessFactory, UserVisitFactory
+from commcare_connect.opportunity.tests.factories import (
+    AssessmentFactory,
+    CompletedWorkFactory,
+    OpportunityAccessFactory,
+    UserVisitFactory,
+)
 from commcare_connect.organization.models import Organization
-from commcare_connect.program.helpers import get_annotated_managed_opportunity
+from commcare_connect.program.helpers import get_annotated_managed_opportunity, get_delivery_performance_report
 from commcare_connect.program.tests.factories import ManagedOpportunityFactory, ProgramFactory
 from commcare_connect.users.tests.factories import OrganizationFactory, UserFactory
 
@@ -48,11 +53,13 @@ def test_delivery_performance(program_manager_org: Organization):
             opportunity=opp,
             status=visit_status,
             opportunity_access=access,
-            visit_date=now() + timedelta(1),
+            visit_date=now() + timedelta(3),
         )
+        CompletedWorkFactory.create(opportunity_access=access)
 
-    opps = get_annotated_managed_opportunity(program)
-    for opp in opps:
-        assert nm_org.slug == opp.organization.slug
-        assert opp.workers_passing_assessment == 5
-        assert opp.workers_starting_delivery == 3
+    opps = get_delivery_performance_report(program, None, None)
+
+    assert opps[0].total_workers_starting_delivery == 3, "Total workers starting delivery doesn't match"
+    assert opps[0].active_workers == 3, "Active workers count doesn't match"
+    assert opps[0].delivery_per_day_per_worker == 1.0, "Deliveries per day doesn't match"
+    assert opps[0].records_flagged_percentage == 1.0, "Records flagged percentage doesn't match"
diff --git a/commcare_connect/program/views.py b/commcare_connect/program/views.py
index ab23240a..5bb75d10 100644
--- a/commcare_connect/program/views.py
+++ b/commcare_connect/program/views.py
@@ -277,4 +277,6 @@ def get_queryset(self):
         start_date = self.request.GET.get("start_date") or None
         end_date = self.request.GET.get("end_date") or None
 
+        print("start_date", start_date)
+        print("end_date", end_date)
         return get_delivery_performance_report(program, start_date, end_date)

From 8d54051e69620c04fded13e5bcefc2f93401552f Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Thu, 26 Sep 2024 17:29:40 +0530
Subject: [PATCH 009/165] fixed url for delivery table

---
 commcare_connect/program/urls.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/commcare_connect/program/urls.py b/commcare_connect/program/urls.py
index 7f96d4c9..7d9cd6c1 100644
--- a/commcare_connect/program/urls.py
+++ b/commcare_connect/program/urls.py
@@ -2,6 +2,7 @@
 
 from commcare_connect.program.views import (
     DeliveryPerformanceTableView,
+    FunnelPerformanceTableView,
     ManagedOpportunityInit,
     ManagedOpportunityList,
     ProgramApplicationList,
@@ -29,7 +30,7 @@
         name="apply_or_decline_application",
     ),
     path("<int:pk>/dashboard", dashboard, name="dashboard"),
-    path("<int:pk>/funnel_performance_table", DeliveryPerformanceTableView.as_view(), name="funnel_performance_table"),
+    path("<int:pk>/funnel_performance_table", FunnelPerformanceTableView.as_view(), name="funnel_performance_table"),
     path(
         "<int:pk>/delivery_performance_table",
         DeliveryPerformanceTableView.as_view(),

From ecaecc4a40ae631eda66c4cac23b448e61ed218a Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Fri, 27 Sep 2024 12:55:04 +0530
Subject: [PATCH 010/165] fixed percent issue

---
 commcare_connect/program/helpers.py            | 2 +-
 commcare_connect/program/tests/test_helpers.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/commcare_connect/program/helpers.py b/commcare_connect/program/helpers.py
index 933b4a3a..6357e703 100644
--- a/commcare_connect/program/helpers.py
+++ b/commcare_connect/program/helpers.py
@@ -109,7 +109,7 @@ def get_delivery_performance_report(program: Program, start_date, end_date):
             ),
             records_flagged_percentage=Case(
                 When(active_workers=0, then=Value(0)),
-                default=Round(F("total_payment_units") / F("total_payement_since_start_date")),
+                default=Round(F("total_payment_units") / F("total_payement_since_start_date")) * 100,
                 output_field=FloatField(),
             ),
         )
diff --git a/commcare_connect/program/tests/test_helpers.py b/commcare_connect/program/tests/test_helpers.py
index 50b8bf56..440a4cf2 100644
--- a/commcare_connect/program/tests/test_helpers.py
+++ b/commcare_connect/program/tests/test_helpers.py
@@ -62,4 +62,4 @@ def test_delivery_performance(program_manager_org: Organization):
     assert opps[0].total_workers_starting_delivery == 3, "Total workers starting delivery doesn't match"
     assert opps[0].active_workers == 3, "Active workers count doesn't match"
     assert opps[0].delivery_per_day_per_worker == 1.0, "Deliveries per day doesn't match"
-    assert opps[0].records_flagged_percentage == 1.0, "Records flagged percentage doesn't match"
+    assert opps[0].records_flagged_percentage == 100.0, "Records flagged percentage doesn't match"

From 5fafc6b439bd0edd2332c733b7af19a98dbba8b6 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 2 Oct 2024 12:55:31 +0530
Subject: [PATCH 011/165] Fixed visit logic and added more test

---
 commcare_connect/program/helpers.py           |  26 ++--
 .../program/tests/test_helpers.py             | 126 +++++++++++++++---
 2 files changed, 128 insertions(+), 24 deletions(-)

diff --git a/commcare_connect/program/helpers.py b/commcare_connect/program/helpers.py
index 6357e703..1d44c728 100644
--- a/commcare_connect/program/helpers.py
+++ b/commcare_connect/program/helpers.py
@@ -12,7 +12,7 @@
     Value,
     When,
 )
-from django.db.models.functions import Round
+from django.db.models.functions import Cast, Round
 
 from commcare_connect.opportunity.models import UserVisit, VisitValidationStatus
 from commcare_connect.program.models import ManagedOpportunity, Program
@@ -79,7 +79,12 @@ def get_delivery_performance_report(program: Program, start_date, end_date):
     if end_date:
         date_filter &= Q(opportunityaccess__uservisit__visit_date__lte=end_date)
 
-    active_workers_filter = Q(FILTER_FOR_VALID_VISIT_DATE, date_filter)
+    flagged_visits_filter = Q(opportunityaccess__uservisit__flagged=True) & ~Q(
+        opportunityaccess__uservisit__status__in=[
+            VisitValidationStatus.rejected,
+            VisitValidationStatus.approved,
+        ]
+    )
 
     managed_opportunities = (
         ManagedOpportunity.objects.filter(program=program)
@@ -92,24 +97,29 @@ def get_delivery_performance_report(program: Program, start_date, end_date):
         .annotate(
             total_workers_starting_delivery=Count(
                 "opportunityaccess__uservisit__user",
-                filter=FILTER_FOR_VALID_VISIT_DATE,
                 distinct=True,
             ),
             active_workers=Count(
                 "opportunityaccess__uservisit__user",
-                filter=active_workers_filter,
+                filter=date_filter,
                 distinct=True,
             ),
             total_payment_units=Count("opportunityaccess__completedwork"),
-            total_payement_since_start_date=Count("opportunityaccess__completedwork", filter=date_filter),
+            total_payment_units_with_flags=Count("opportunityaccess__completedwork", filter=flagged_visits_filter),
+            total_payment_since_start_date=Count("opportunityaccess__completedwork", filter=date_filter),
             delivery_per_day_per_worker=Case(
                 When(active_workers=0, then=Value(0)),
-                default=Round(F("total_payement_since_start_date") / F("active_workers"), 2),
+                default=Round(F("total_payment_since_start_date") / F("active_workers"), 2),
                 output_field=FloatField(),
             ),
             records_flagged_percentage=Case(
-                When(active_workers=0, then=Value(0)),
-                default=Round(F("total_payment_units") / F("total_payement_since_start_date")) * 100,
+                When(total_payment_since_start_date=0, then=Value(0)),
+                default=Round(
+                    Cast(F("total_payment_units_with_flags"), FloatField())
+                    / Cast(F("total_payment_since_start_date"), FloatField())
+                    * 100,
+                    2,
+                ),
                 output_field=FloatField(),
             ),
         )
diff --git a/commcare_connect/program/tests/test_helpers.py b/commcare_connect/program/tests/test_helpers.py
index 440a4cf2..bb4c60c0 100644
--- a/commcare_connect/program/tests/test_helpers.py
+++ b/commcare_connect/program/tests/test_helpers.py
@@ -1,5 +1,6 @@
 from datetime import timedelta
 
+import pytest
 from django_celery_beat.utils import now
 
 from commcare_connect.opportunity.models import VisitValidationStatus
@@ -39,27 +40,120 @@ def test_get_annotated_managed_opportunity(program_manager_org: Organization):
         assert opp.workers_starting_delivery == 3
 
 
-def test_delivery_performance(program_manager_org: Organization):
-    program = ProgramFactory.create(organization=program_manager_org)
-    nm_org = OrganizationFactory.create()
-    opp = ManagedOpportunityFactory.create(program=program, organization=nm_org)
-    users = UserFactory.create_batch(5)
-    for index, user in enumerate(users):
-        access = OpportunityAccessFactory.create(opportunity=opp, user=user, invited_date=now())
-        AssessmentFactory.create(opportunity=opp, user=user, opportunity_access=access)
-        visit_status = VisitValidationStatus.pending if index < 3 else VisitValidationStatus.trial
+@pytest.mark.django_db
+class TestDeliveryPerformanceReport:
+    @pytest.fixture(autouse=True)
+    def setup(self):
+        self.program_manager_org = OrganizationFactory.create()
+        self.program = ProgramFactory.create(organization=self.program_manager_org)
+        self.nm_org = OrganizationFactory.create()
+        self.opp = ManagedOpportunityFactory.create(program=self.program, organization=self.nm_org)
+
+    def create_user_with_visit(self, visit_status, visit_date, flagged=False, create_completed_work=True):
+        user = UserFactory.create()
+        access = OpportunityAccessFactory.create(opportunity=self.opp, user=user, invited_date=now())
         UserVisitFactory.create(
             user=user,
-            opportunity=opp,
+            opportunity=self.opp,
             status=visit_status,
             opportunity_access=access,
-            visit_date=now() + timedelta(3),
+            visit_date=visit_date,
+            flagged=flagged,
+        )
+        if create_completed_work:
+            CompletedWorkFactory.create(opportunity_access=access)
+        return user
+
+    def test_basic_delivery_performance(self):
+        for _ in range(2):
+            self.create_user_with_visit(VisitValidationStatus.pending, now(), True)
+        for _ in range(3):
+            self.create_user_with_visit(VisitValidationStatus.approved, now(), False)
+
+        opps = get_delivery_performance_report(self.program, None, None)
+        assert len(opps) == 1
+        assert opps[0].total_workers_starting_delivery == 5
+        assert opps[0].active_workers == 5
+        assert opps[0].total_payment_units == 5
+        assert opps[0].total_payment_units_with_flags == 2
+        assert opps[0].total_payment_since_start_date == 5
+        assert opps[0].delivery_per_day_per_worker == 1.0
+        assert opps[0].records_flagged_percentage == 40.0
+
+    def test_delivery_performance_with_date_range(self):
+        start_date = now() - timedelta(10)
+        end_date = now() + timedelta(10)
+
+        self.create_user_with_visit(VisitValidationStatus.pending, start_date - timedelta(1))
+        self.create_user_with_visit(VisitValidationStatus.pending, start_date + timedelta(1))
+        self.create_user_with_visit(VisitValidationStatus.pending, end_date - timedelta(1))
+        self.create_user_with_visit(VisitValidationStatus.pending, end_date + timedelta(1))
+
+        opps = get_delivery_performance_report(self.program, start_date, end_date)
+        assert opps[0].active_workers == 2
+        assert opps[0].total_payment_since_start_date == 2
+
+    def test_delivery_performance_with_flagged_visits(self):
+        self.create_user_with_visit(VisitValidationStatus.pending, now())
+        self.create_user_with_visit(VisitValidationStatus.pending, now(), flagged=True)
+
+        opps = get_delivery_performance_report(self.program, None, None)
+        assert opps[0].total_payment_units_with_flags == 1
+        assert opps[0].records_flagged_percentage == 50.0
+
+    def test_delivery_performance_with_no_active_workers(self):
+        self.create_user_with_visit(VisitValidationStatus.over_limit, now())
+        self.create_user_with_visit(VisitValidationStatus.trial, now())
+
+        opps = get_delivery_performance_report(self.program, None, None)
+        assert opps[0].total_workers_starting_delivery == 2
+        assert opps[0].active_workers == 2
+        assert opps[0].delivery_per_day_per_worker == 1.0
+
+    def test_delivery_performance_with_multiple_opportunities(self):
+        opp2 = ManagedOpportunityFactory.create(program=self.program)
+
+        self.create_user_with_visit(VisitValidationStatus.pending, now())
+
+        user = UserFactory.create()
+        access = OpportunityAccessFactory.create(opportunity=opp2, user=user, invited_date=now())
+        UserVisitFactory.create(
+            user=user,
+            opportunity=opp2,
+            status=VisitValidationStatus.pending,
+            opportunity_access=access,
+            visit_date=now(),
         )
         CompletedWorkFactory.create(opportunity_access=access)
 
-    opps = get_delivery_performance_report(program, None, None)
+        opps = get_delivery_performance_report(self.program, None, None)
+        assert len(opps) == 2
+        assert all(o.active_workers == 1 for o in opps)
+
+    def test_delivery_performance_with_no_completed_work(self):
+        self.create_user_with_visit(VisitValidationStatus.pending, now(), create_completed_work=False)
+
+        opps = get_delivery_performance_report(self.program, None, None)
+        assert opps[0].total_payment_units == 0
+        assert opps[0].delivery_per_day_per_worker == 0
+
+    @pytest.mark.parametrize("visit_status", [VisitValidationStatus.rejected, VisitValidationStatus.approved])
+    def test_delivery_performance_excluded_statuses(self, visit_status):
+        self.create_user_with_visit(visit_status, now(), flagged=True)
+
+        opps = get_delivery_performance_report(self.program, None, None)
+        assert opps[0].total_workers_starting_delivery == 1
+        assert opps[0].active_workers == 1
+        assert opps[0].total_payment_units_with_flags == 0
+
+    def test_delivery_performance_with_mixed_statuses(self):
+        self.create_user_with_visit(VisitValidationStatus.pending, now(), flagged=True)
+        self.create_user_with_visit(VisitValidationStatus.approved, now(), flagged=True)
+        self.create_user_with_visit(VisitValidationStatus.rejected, now(), flagged=True)
+        self.create_user_with_visit(VisitValidationStatus.over_limit, now(), flagged=True)
 
-    assert opps[0].total_workers_starting_delivery == 3, "Total workers starting delivery doesn't match"
-    assert opps[0].active_workers == 3, "Active workers count doesn't match"
-    assert opps[0].delivery_per_day_per_worker == 1.0, "Deliveries per day doesn't match"
-    assert opps[0].records_flagged_percentage == 100.0, "Records flagged percentage doesn't match"
+        opps = get_delivery_performance_report(self.program, None, None)
+        assert opps[0].total_workers_starting_delivery == 4
+        assert opps[0].active_workers == 4
+        assert opps[0].total_payment_units_with_flags == 2
+        assert opps[0].records_flagged_percentage == 50.0

From 680b361a5c0f1df32c462d8cbd25935cd1acbc35 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 2 Oct 2024 13:23:52 +0530
Subject: [PATCH 012/165] Code refactor

---
 commcare_connect/program/helpers.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/commcare_connect/program/helpers.py b/commcare_connect/program/helpers.py
index 1d44c728..96aa1eb6 100644
--- a/commcare_connect/program/helpers.py
+++ b/commcare_connect/program/helpers.py
@@ -25,6 +25,14 @@
 )
 
 
+def calculate_safe_percentage(numerator, denominator):
+    return Case(
+        When(**{denominator: 0}, then=Value(0)),  # Handle division by zero
+        default=Round(Cast(F(numerator), FloatField()) / Cast(F(denominator), FloatField()) * 100, 2),
+        output_field=FloatField(),
+    )
+
+
 def get_annotated_managed_opportunity(program: Program):
     earliest_visits = (
         UserVisit.objects.filter(
@@ -52,7 +60,7 @@ def get_annotated_managed_opportunity(program: Program):
                 filter=FILTER_FOR_VALID_VISIT_DATE,
                 distinct=True,
             ),
-            percentage_conversion=F("workers_starting_delivery") / F("workers_invited") * 100,
+            percentage_conversion=calculate_safe_percentage("workers_starting_delivery", "workers_invited"),
             average_time_to_convert=Avg(
                 ExpressionWrapper(
                     Subquery(earliest_visits) - F("opportunityaccess__invited_date"), output_field=DurationField()
@@ -112,15 +120,8 @@ def get_delivery_performance_report(program: Program, start_date, end_date):
                 default=Round(F("total_payment_since_start_date") / F("active_workers"), 2),
                 output_field=FloatField(),
             ),
-            records_flagged_percentage=Case(
-                When(total_payment_since_start_date=0, then=Value(0)),
-                default=Round(
-                    Cast(F("total_payment_units_with_flags"), FloatField())
-                    / Cast(F("total_payment_since_start_date"), FloatField())
-                    * 100,
-                    2,
-                ),
-                output_field=FloatField(),
+            records_flagged_percentage=calculate_safe_percentage(
+                "total_payment_units_with_flags", "total_payment_since_start_date"
             ),
         )
     )

From 4f88e2aa8a84e76736c6954c1e540b7857143581 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Mon, 7 Oct 2024 10:27:00 +0530
Subject: [PATCH 013/165] fixed code review issues and added more tests

---
 commcare_connect/program/helpers.py           |  44 ++++--
 .../program/tests/test_helpers.py             | 139 +++++++++++++++---
 2 files changed, 155 insertions(+), 28 deletions(-)

diff --git a/commcare_connect/program/helpers.py b/commcare_connect/program/helpers.py
index a9bcd9f7..086b7e10 100644
--- a/commcare_connect/program/helpers.py
+++ b/commcare_connect/program/helpers.py
@@ -1,22 +1,44 @@
-from django.db.models import Avg, Count, DurationField, ExpressionWrapper, F, OuterRef, Q, Subquery
+from django.db.models import (
+    Avg,
+    Case,
+    Count,
+    DurationField,
+    ExpressionWrapper,
+    F,
+    FloatField,
+    OuterRef,
+    Q,
+    Subquery,
+    Value,
+    When,
+)
+from django.db.models.functions import Cast, Round
 
 from commcare_connect.opportunity.models import UserVisit, VisitValidationStatus
 from commcare_connect.program.models import ManagedOpportunity, Program
 
 
-def get_annotated_managed_opportunity(program: Program):
-    filter_for_valid__visit_date = ~Q(
-        opportunityaccess__uservisit__status__in=[
-            VisitValidationStatus.over_limit,
-            VisitValidationStatus.trial,
-        ]
+def calculate_safe_percentage(numerator, denominator):
+    return Case(
+        When(**{denominator: 0}, then=Value(0)),  # Handle division by zero
+        default=Round(Cast(F(numerator), FloatField()) / Cast(F(denominator), FloatField()) * 100, 2),
+        output_field=FloatField(),
     )
 
+
+def get_annotated_managed_opportunity(program: Program):
+    excluded_status = [
+        VisitValidationStatus.over_limit,
+        VisitValidationStatus.trial,
+    ]
+
+    filter_for_valid__visit_date = ~Q(opportunityaccess__uservisit__status__in=excluded_status)
+
     earliest_visits = (
         UserVisit.objects.filter(
             opportunity_access=OuterRef("opportunityaccess"),
         )
-        .exclude(status__in=[VisitValidationStatus.over_limit, VisitValidationStatus.trial])
+        .exclude(status__in=excluded_status)
         .order_by("visit_date")
         .values("visit_date")[:1]
     )
@@ -25,20 +47,20 @@ def get_annotated_managed_opportunity(program: Program):
         ManagedOpportunity.objects.filter(program=program)
         .order_by("start_date")
         .annotate(
-            workers_invited=Count("opportunityaccess"),
+            workers_invited=Count("opportunityaccess", distinct=True),
             workers_passing_assessment=Count(
                 "opportunityaccess__assessment",
                 filter=Q(
                     opportunityaccess__assessment__passed=True,
-                    opportunityaccess__assessment__opportunity=F("opportunityaccess__opportunity"),
                 ),
+                distinct=True,
             ),
             workers_starting_delivery=Count(
                 "opportunityaccess__uservisit__user",
                 filter=filter_for_valid__visit_date,
                 distinct=True,
             ),
-            percentage_conversion=F("workers_starting_delivery") / F("workers_invited") * 100,
+            percentage_conversion=calculate_safe_percentage("workers_starting_delivery", "workers_invited"),
             average_time_to_convert=Avg(
                 ExpressionWrapper(
                     Subquery(earliest_visits) - F("opportunityaccess__invited_date"), output_field=DurationField()
diff --git a/commcare_connect/program/tests/test_helpers.py b/commcare_connect/program/tests/test_helpers.py
index 8339976f..150aa305 100644
--- a/commcare_connect/program/tests/test_helpers.py
+++ b/commcare_connect/program/tests/test_helpers.py
@@ -1,34 +1,139 @@
 from datetime import timedelta
 
+import pytest
 from django_celery_beat.utils import now
 
 from commcare_connect.opportunity.models import VisitValidationStatus
 from commcare_connect.opportunity.tests.factories import AssessmentFactory, OpportunityAccessFactory, UserVisitFactory
-from commcare_connect.organization.models import Organization
 from commcare_connect.program.helpers import get_annotated_managed_opportunity
 from commcare_connect.program.tests.factories import ManagedOpportunityFactory, ProgramFactory
 from commcare_connect.users.tests.factories import OrganizationFactory, UserFactory
 
 
-def test_get_annotated_managed_opportunity(program_manager_org: Organization):
-    program = ProgramFactory.create(organization=program_manager_org)
-    nm_org = OrganizationFactory.create()
-    opp = ManagedOpportunityFactory.create(program=program, organization=nm_org)
-    users = UserFactory.create_batch(5)
-    for index, user in enumerate(users):
-        access = OpportunityAccessFactory.create(opportunity=opp, user=user, invited_date=now())
-        AssessmentFactory.create(opportunity=opp, user=user, opportunity_access=access)
-        visit_status = VisitValidationStatus.pending if index < 3 else VisitValidationStatus.trial
+class TestGetAnnotatedManagedOpportunity:
+    @pytest.fixture(autouse=True)
+    def setup(self, db):
+        self.program = ProgramFactory.create()
+        self.nm_org = OrganizationFactory.create()
+        self.opp = ManagedOpportunityFactory.create(program=self.program, organization=self.nm_org)
+
+    def create_user_with_access(self, visit_status=VisitValidationStatus.pending, passed_assessment=True):
+        user = UserFactory.create()
+        access = OpportunityAccessFactory.create(opportunity=self.opp, user=user, invited_date=now())
+        AssessmentFactory.create(opportunity=self.opp, user=user, opportunity_access=access, passed=passed_assessment)
         UserVisitFactory.create(
             user=user,
-            opportunity=opp,
+            opportunity=self.opp,
             status=visit_status,
             opportunity_access=access,
-            visit_date=now() + timedelta(1),
+            visit_date=now() + timedelta(days=1),
+        )
+        return user
+
+    def test_basic_scenario(self):
+        for i in range(5):
+            self.create_user_with_access(
+                visit_status=VisitValidationStatus.pending if i < 3 else VisitValidationStatus.trial
+            )
+
+        opps = get_annotated_managed_opportunity(self.program)
+        assert len(opps) == 1
+        annotated_opp = opps[0]
+        assert annotated_opp.organization.slug == self.nm_org.slug
+        assert annotated_opp.workers_invited == 5
+        assert annotated_opp.workers_passing_assessment == 5
+        assert annotated_opp.workers_starting_delivery == 3
+        assert annotated_opp.percentage_conversion == 60.0
+
+    def test_empty_scenario(self):
+        opps = get_annotated_managed_opportunity(self.program)
+        assert len(opps) == 1
+        annotated_opp = opps[0]
+        assert annotated_opp.workers_invited == 0
+        assert annotated_opp.workers_passing_assessment == 0
+        assert annotated_opp.workers_starting_delivery == 0
+        assert annotated_opp.percentage_conversion == 0.0
+        assert annotated_opp.average_time_to_convert is None
+
+    def test_multiple_visits(self):
+        user = self.create_user_with_access()
+        UserVisitFactory.create_batch(
+            2,
+            user=user,
+            opportunity=self.opp,
+            status=VisitValidationStatus.pending,
+            opportunity_access=user.opportunityaccess_set.first(),
+            visit_date=now() + timedelta(days=2),
         )
 
-    opps = get_annotated_managed_opportunity(program)
-    for opp in opps:
-        assert nm_org.slug == opp.organization.slug
-        assert opp.workers_passing_assessment == 5
-        assert opp.workers_starting_delivery == 3
+        opps = get_annotated_managed_opportunity(self.program)
+        assert len(opps) == 1
+        annotated_opp = opps[0]
+        assert annotated_opp.workers_invited == 1
+        assert annotated_opp.workers_passing_assessment == 1
+        assert annotated_opp.workers_starting_delivery == 1
+        assert annotated_opp.percentage_conversion == 100.0
+
+    def test_excluded_statuses(self):
+        self.create_user_with_access(visit_status=VisitValidationStatus.over_limit)
+        self.create_user_with_access(visit_status=VisitValidationStatus.trial)
+
+        opps = get_annotated_managed_opportunity(self.program)
+        assert len(opps) == 1
+        annotated_opp = opps[0]
+        assert annotated_opp.workers_invited == 2
+        assert annotated_opp.workers_passing_assessment == 2
+        assert annotated_opp.workers_starting_delivery == 0
+        assert annotated_opp.percentage_conversion == 0.0
+
+    def test_average_time_to_convert(self):
+        for i in range(3):
+            user = self.create_user_with_access()
+            user.opportunityaccess_set.update(invited_date=now() - timedelta(days=i))
+
+        opps = get_annotated_managed_opportunity(self.program)
+        assert len(opps) == 1
+        annotated_opp = opps[0]
+        expected_time = timedelta(days=2)
+        actual_time = annotated_opp.average_time_to_convert
+        assert abs(actual_time - expected_time) < timedelta(seconds=5)
+
+    def test_multiple_opportunities(self):
+        nm_org2 = OrganizationFactory.create()
+        opp2 = ManagedOpportunityFactory.create(
+            program=self.program, organization=nm_org2, start_date=now() + timedelta(days=1)
+        )
+
+        self.create_user_with_access()
+        user2 = UserFactory.create()
+        access2 = OpportunityAccessFactory.create(opportunity=opp2, user=user2, invited_date=now())
+        AssessmentFactory.create(opportunity=opp2, user=user2, opportunity_access=access2, passed=True)
+        UserVisitFactory.create(
+            user=user2,
+            opportunity=opp2,
+            status=VisitValidationStatus.pending,
+            opportunity_access=access2,
+            visit_date=now() + timedelta(days=1),
+        )
+
+        opps = get_annotated_managed_opportunity(self.program)
+        assert len(opps) == 2
+        assert opps[0].organization.slug == self.nm_org.slug
+        assert opps[1].organization.slug == nm_org2.slug
+        for annotated_opp in opps:
+            assert annotated_opp.workers_invited == 1
+            assert annotated_opp.workers_passing_assessment == 1
+            assert annotated_opp.workers_starting_delivery == 1
+            assert annotated_opp.percentage_conversion == 100.0
+
+    def test_failed_assessments(self):
+        self.create_user_with_access(passed_assessment=False)
+        self.create_user_with_access(passed_assessment=True)
+
+        opps = get_annotated_managed_opportunity(self.program)
+        assert len(opps) == 1
+        annotated_opp = opps[0]
+        assert annotated_opp.workers_invited == 2
+        assert annotated_opp.workers_passing_assessment == 1
+        assert annotated_opp.workers_starting_delivery == 2
+        assert annotated_opp.percentage_conversion == 100.0

From 7075ad4a8b23c1b10cc76d6151660659393d1283 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Mon, 7 Oct 2024 10:29:38 +0530
Subject: [PATCH 014/165] added check for none average time

---
 commcare_connect/program/tables.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/commcare_connect/program/tables.py b/commcare_connect/program/tables.py
index 4e964d4c..e9cf0296 100644
--- a/commcare_connect/program/tables.py
+++ b/commcare_connect/program/tables.py
@@ -259,6 +259,8 @@ class Meta:
         orderable = False
 
     def render_average_time_to_convert(self, record):
+        if not record.average_time_to_convert:
+            return "---"
         total_seconds = record.average_time_to_convert.total_seconds()
         hours = total_seconds / 3600
         return f"{round(hours, 2)}hr"

From 56651e5afcf82d984a4807833be35d6f7e9a2a91 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Mon, 7 Oct 2024 10:35:12 +0530
Subject: [PATCH 015/165] fixed migration sequence

---
 ...invited_date.py => 0060_opportunityaccess_invited_date.py} | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
 rename commcare_connect/opportunity/migrations/{0059_opportunityaccess_invited_date.py => 0060_opportunityaccess_invited_date.py} (74%)

diff --git a/commcare_connect/opportunity/migrations/0059_opportunityaccess_invited_date.py b/commcare_connect/opportunity/migrations/0060_opportunityaccess_invited_date.py
similarity index 74%
rename from commcare_connect/opportunity/migrations/0059_opportunityaccess_invited_date.py
rename to commcare_connect/opportunity/migrations/0060_opportunityaccess_invited_date.py
index 1382f847..d96766df 100644
--- a/commcare_connect/opportunity/migrations/0059_opportunityaccess_invited_date.py
+++ b/commcare_connect/opportunity/migrations/0060_opportunityaccess_invited_date.py
@@ -1,11 +1,11 @@
-# Generated by Django 4.2.5 on 2024-09-18 10:16
+# Generated by Django 4.2.5 on 2024-10-07 05:04
 
 from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
     dependencies = [
-        ("opportunity", "0058_paymentinvoice_payment_invoice"),
+        ("opportunity", "0059_payment_amount_usd"),
     ]
 
     operations = [

From 7b8ded4c3038f22cf5cd7de716543ebfd50c9add Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Mon, 7 Oct 2024 18:38:28 +0530
Subject: [PATCH 016/165] Fix spelling

---
 .../tests/test_receiver_integration.py           | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/commcare_connect/form_receiver/tests/test_receiver_integration.py b/commcare_connect/form_receiver/tests/test_receiver_integration.py
index c52af2a5..e75048ba 100644
--- a/commcare_connect/form_receiver/tests/test_receiver_integration.py
+++ b/commcare_connect/form_receiver/tests/test_receiver_integration.py
@@ -460,7 +460,7 @@ def test_auto_approve_visits_and_payments(
     assert access.payment_accrued == completed_work.payment_accrued
 
 
-def test_reciever_verification_flags_form_submission(
+def test_receiver_verification_flags_form_submission(
     user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
 ):
     verification_flags = OpportunityVerificationFlags.objects.get(opportunity=opportunity)
@@ -477,7 +477,7 @@ def test_reciever_verification_flags_form_submission(
     assert not visit.flagged
 
 
-def test_reciever_verification_flags_form_submission_start(
+def test_receiver_verification_flags_form_submission_start(
     user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
 ):
     verification_flags = OpportunityVerificationFlags.objects.get(opportunity=opportunity)
@@ -494,7 +494,7 @@ def test_reciever_verification_flags_form_submission_start(
     assert ["form_submission_period", "Form was submitted before the start time"] in visit.flag_reason.get("flags", [])
 
 
-def test_reciever_verification_flags_form_submission_end(
+def test_receiver_verification_flags_form_submission_end(
     user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
 ):
     verification_flags = OpportunityVerificationFlags.objects.get(opportunity=opportunity)
@@ -511,7 +511,7 @@ def test_reciever_verification_flags_form_submission_end(
     assert ["form_submission_period", "Form was submitted after the end time"] in visit.flag_reason.get("flags", [])
 
 
-def test_reciever_verification_flags_duration(
+def test_receiver_verification_flags_duration(
     user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
 ):
     form_json = _create_opp_and_form_json(opportunity, user=user_with_connectid_link)
@@ -524,7 +524,7 @@ def test_reciever_verification_flags_duration(
     assert ["duration", "The form was completed too quickly."] in visit.flag_reason.get("flags", [])
 
 
-def test_reciever_verification_flags_check_attachments(
+def test_receiver_verification_flags_check_attachments(
     user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
 ):
     form_json = _create_opp_and_form_json(opportunity, user=user_with_connectid_link)
@@ -537,7 +537,7 @@ def test_reciever_verification_flags_check_attachments(
     assert ["attachment_missing", "Form was submitted without attachements."] in visit.flag_reason.get("flags", [])
 
 
-def test_reciever_verification_flags_form_json_rule(
+def test_receiver_verification_flags_form_json_rule(
     user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
 ):
     form_json = _create_opp_and_form_json(opportunity, user=user_with_connectid_link)
@@ -555,7 +555,7 @@ def test_reciever_verification_flags_form_json_rule(
     assert not visit.flagged
 
 
-def test_reciever_verification_flags_form_json_rule_flagged(
+def test_receiver_verification_flags_form_json_rule_flagged(
     user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
 ):
     form_json = _create_opp_and_form_json(opportunity, user=user_with_connectid_link)
@@ -577,7 +577,7 @@ def test_reciever_verification_flags_form_json_rule_flagged(
     ] in visit.flag_reason.get("flags", [])
 
 
-def test_reciever_verification_flags_catchment_areas(
+def test_receiver_verification_flags_catchment_areas(
     user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
 ):
     verification_flags = OpportunityVerificationFlags.objects.get(opportunity=opportunity)

From 2fee3497e937f364c1befefa8588955cca3eaf0f Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Mon, 7 Oct 2024 18:43:48 +0530
Subject: [PATCH 017/165] Add tests for form receiver review status changes

---
 .../tests/test_receiver_integration.py        | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/commcare_connect/form_receiver/tests/test_receiver_integration.py b/commcare_connect/form_receiver/tests/test_receiver_integration.py
index e75048ba..6108c5bb 100644
--- a/commcare_connect/form_receiver/tests/test_receiver_integration.py
+++ b/commcare_connect/form_receiver/tests/test_receiver_integration.py
@@ -23,6 +23,7 @@
     OpportunityClaimLimit,
     OpportunityVerificationFlags,
     UserVisit,
+    VisitReviewStatus,
     VisitValidationStatus,
 )
 from commcare_connect.opportunity.tasks import bulk_approve_completed_work
@@ -596,6 +597,33 @@ def test_receiver_verification_flags_catchment_areas(
     assert ["catchment", "Visit outside worker catchment areas"] in visit.flag_reason.get("flags", [])
 
 
+def test_receiver_auto_agree_approved_visit(
+    user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
+):
+    opportunity.managed = True
+    opportunity.save()
+    form_json = _create_opp_and_form_json(opportunity, user=user_with_connectid_link)
+    make_request(api_client, form_json, user_with_connectid_link)
+    visit = UserVisit.objects.get(user=user_with_connectid_link)
+    assert not visit.flagged
+    assert visit.status == VisitValidationStatus.approved
+    assert visit.review_status == VisitReviewStatus.agree
+
+
+def test_receiver_flagged_visit_review_pending(
+    user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
+):
+    opportunity.managed = True
+    opportunity.save()
+    form_json = _create_opp_and_form_json(opportunity, user=user_with_connectid_link)
+    form_json["metadata"]["location"] = None
+    make_request(api_client, form_json, user_with_connectid_link)
+    visit = UserVisit.objects.get(user=user_with_connectid_link)
+    assert visit.flagged
+    assert visit.status == VisitValidationStatus.pending
+    assert visit.review_status == VisitReviewStatus.pending
+
+
 def _get_form_json(learn_app, module_id, form_block=None):
     form_json = get_form_json(
         form_block=form_block or LearnModuleJsonFactory(id=module_id).json,

From fbddab58c24b0b701969ab38da37a971717b5451 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Tue, 8 Oct 2024 15:11:52 +0530
Subject: [PATCH 018/165] Refactor tests to use fixture

---
 .../tests/test_receiver_integration.py        | 42 ++++++++++---------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/commcare_connect/form_receiver/tests/test_receiver_integration.py b/commcare_connect/form_receiver/tests/test_receiver_integration.py
index 6108c5bb..6e9a9941 100644
--- a/commcare_connect/form_receiver/tests/test_receiver_integration.py
+++ b/commcare_connect/form_receiver/tests/test_receiver_integration.py
@@ -173,24 +173,14 @@ def test_receiver_deliver_form_daily_visits_reached(
 def test_receiver_deliver_form_max_visits_reached(
     mobile_user_with_connect_link: User, api_client: APIClient, opportunity: Opportunity
 ):
-    def form_json(payment_unit):
-        deliver_unit = DeliverUnitFactory(app=opportunity.deliver_app, payment_unit=payment_unit)
-        stub = DeliverUnitStubFactory(id=deliver_unit.slug)
-        form_json = get_form_json(
-            form_block=stub.json,
-            domain=deliver_unit.app.cc_domain,
-            app_id=deliver_unit.app.cc_app_id,
-        )
-        return form_json
-
     def submit_form_for_random_entity(form_json):
         duplicate_json = deepcopy(form_json)
         duplicate_json["form"]["deliver"]["entity_id"] = str(uuid4())
         make_request(api_client, duplicate_json, mobile_user_with_connect_link)
 
     payment_units = opportunity.paymentunit_set.all()
-    form_json1 = form_json(payment_units[0])
-    form_json2 = form_json(payment_units[1])
+    form_json1 = get_form_json_for_payment_unit(payment_units[0])
+    form_json2 = get_form_json_for_payment_unit(payment_units[1])
     for _ in range(2):
         submit_form_for_random_entity(form_json1)
         submit_form_for_random_entity(form_json2)
@@ -598,32 +588,44 @@ def test_receiver_verification_flags_catchment_areas(
 
 
 def test_receiver_auto_agree_approved_visit(
-    user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
+    mobile_user_with_connect_link: User, api_client: APIClient, opportunity: Opportunity
 ):
     opportunity.managed = True
     opportunity.save()
-    form_json = _create_opp_and_form_json(opportunity, user=user_with_connectid_link)
-    make_request(api_client, form_json, user_with_connectid_link)
-    visit = UserVisit.objects.get(user=user_with_connectid_link)
+    form_json = get_form_json_for_payment_unit(opportunity.paymentunit_set.all()[0])
+    make_request(api_client, form_json, mobile_user_with_connect_link)
+    visit = UserVisit.objects.get(user=mobile_user_with_connect_link)
     assert not visit.flagged
     assert visit.status == VisitValidationStatus.approved
     assert visit.review_status == VisitReviewStatus.agree
 
 
+@pytest.mark.parametrize("paymentunit_options", [pytest.param({"max_daily": 2})])
 def test_receiver_flagged_visit_review_pending(
-    user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
+    mobile_user_with_connect_link: User, api_client: APIClient, opportunity: Opportunity
 ):
     opportunity.managed = True
     opportunity.save()
-    form_json = _create_opp_and_form_json(opportunity, user=user_with_connectid_link)
+    form_json = get_form_json_for_payment_unit(opportunity.paymentunit_set.all()[0])
     form_json["metadata"]["location"] = None
-    make_request(api_client, form_json, user_with_connectid_link)
-    visit = UserVisit.objects.get(user=user_with_connectid_link)
+    make_request(api_client, form_json, mobile_user_with_connect_link)
+    visit = UserVisit.objects.get(user=mobile_user_with_connect_link)
     assert visit.flagged
     assert visit.status == VisitValidationStatus.pending
     assert visit.review_status == VisitReviewStatus.pending
 
 
+def get_form_json_for_payment_unit(payment_unit):
+    deliver_unit = DeliverUnitFactory(app=payment_unit.opportunity.deliver_app, payment_unit=payment_unit)
+    stub = DeliverUnitStubFactory(id=deliver_unit.slug)
+    form_json = get_form_json(
+        form_block=stub.json,
+        domain=deliver_unit.app.cc_domain,
+        app_id=deliver_unit.app.cc_app_id,
+    )
+    return form_json
+
+
 def _get_form_json(learn_app, module_id, form_block=None):
     form_json = get_form_json(
         form_block=form_block or LearnModuleJsonFactory(id=module_id).json,

From d04c0af5f9f8420af029fb94f599bd8567440146 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Tue, 8 Oct 2024 16:24:47 +0530
Subject: [PATCH 019/165] Add config to change opportunity args

---
 commcare_connect/conftest.py                             | 8 +++++---
 .../form_receiver/tests/test_receiver_integration.py     | 9 ++++-----
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/commcare_connect/conftest.py b/commcare_connect/conftest.py
index 8c91a1df..2e8260ab 100644
--- a/commcare_connect/conftest.py
+++ b/commcare_connect/conftest.py
@@ -47,9 +47,11 @@ def user(db) -> User:
     return UserFactory()
 
 
-@pytest.fixture()
-def opportunity():
-    factory = OpportunityFactory(is_test=False)
+@pytest.fixture
+def opportunity(request):
+    opp_options = {"is_test": False}
+    opp_options.update(request.param if hasattr(request, "param") else {})
+    factory = OpportunityFactory(**opp_options)
     OpportunityVerificationFlagsFactory(opportunity=factory)
     return factory
 
diff --git a/commcare_connect/form_receiver/tests/test_receiver_integration.py b/commcare_connect/form_receiver/tests/test_receiver_integration.py
index 6e9a9941..be002639 100644
--- a/commcare_connect/form_receiver/tests/test_receiver_integration.py
+++ b/commcare_connect/form_receiver/tests/test_receiver_integration.py
@@ -587,11 +587,11 @@ def test_receiver_verification_flags_catchment_areas(
     assert ["catchment", "Visit outside worker catchment areas"] in visit.flag_reason.get("flags", [])
 
 
+@pytest.mark.parametrize("opportunity", [{"managed": True}], indirect=True)
 def test_receiver_auto_agree_approved_visit(
     mobile_user_with_connect_link: User, api_client: APIClient, opportunity: Opportunity
 ):
-    opportunity.managed = True
-    opportunity.save()
+    assert opportunity.managed
     form_json = get_form_json_for_payment_unit(opportunity.paymentunit_set.all()[0])
     make_request(api_client, form_json, mobile_user_with_connect_link)
     visit = UserVisit.objects.get(user=mobile_user_with_connect_link)
@@ -600,12 +600,11 @@ def test_receiver_auto_agree_approved_visit(
     assert visit.review_status == VisitReviewStatus.agree
 
 
-@pytest.mark.parametrize("paymentunit_options", [pytest.param({"max_daily": 2})])
+@pytest.mark.parametrize("opportunity", [{"managed": True}], indirect=True)
 def test_receiver_flagged_visit_review_pending(
     mobile_user_with_connect_link: User, api_client: APIClient, opportunity: Opportunity
 ):
-    opportunity.managed = True
-    opportunity.save()
+    assert opportunity.managed
     form_json = get_form_json_for_payment_unit(opportunity.paymentunit_set.all()[0])
     form_json["metadata"]["location"] = None
     make_request(api_client, form_json, mobile_user_with_connect_link)

From 17db33d01c6053daaaf8ddad736f2315aa9752a7 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Tue, 8 Oct 2024 18:27:01 +0530
Subject: [PATCH 020/165] Add visit import test for review visits

---
 .../opportunity/tests/test_visit_import.py    | 45 +++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/commcare_connect/opportunity/tests/test_visit_import.py b/commcare_connect/opportunity/tests/test_visit_import.py
index 4c5f10ee..729f05d7 100644
--- a/commcare_connect/opportunity/tests/test_visit_import.py
+++ b/commcare_connect/opportunity/tests/test_visit_import.py
@@ -19,6 +19,7 @@
     Payment,
     PaymentUnit,
     UserVisit,
+    VisitReviewStatus,
     VisitValidationStatus,
 )
 from commcare_connect.opportunity.tests.factories import (
@@ -539,3 +540,47 @@ def get_assignable_completed_work_count(access: OpportunityAccess) -> int:
             total_assigned_count += 1
 
     return total_assigned_count
+
+
+@pytest.mark.parametrize("opportunity", [{"managed": True}], indirect=True)
+def test_network_manager_approve_flagged_visit(mobile_user: User, opportunity: Opportunity):
+    assert opportunity.managed
+    access = OpportunityAccess.objects.get(user=mobile_user, opportunity=opportunity)
+    visits = UserVisitFactory.create_batch(
+        5, opportunity=opportunity, status=VisitValidationStatus.pending, user=mobile_user, opportunity_access=access
+    )
+    dataset = Dataset(headers=["visit id", "status", "rejected reason", "justification"])
+    dataset.extend([[visit.xform_id, VisitValidationStatus.approved.value, "", "justification"] for visit in visits])
+    before_update = now()
+    import_status = _bulk_update_visit_status(opportunity, dataset)
+    after_update = now()
+    assert not import_status.missing_visits
+    updated_visits = UserVisit.objects.filter(opportunity=opportunity)
+    for visit in updated_visits:
+        assert visit.status == VisitValidationStatus.approved
+        assert visit.status_modified_date is not None
+        assert before_update <= visit.status_modified_date <= after_update
+        assert before_update <= visit.review_created_on <= after_update
+        assert visit.review_status == VisitReviewStatus.pending
+        assert visit.justification == "justification"
+
+
+@pytest.mark.parametrize("opportunity", [{"managed": True}], indirect=True)
+def test_network_manager_reject_flagged_visit(mobile_user: User, opportunity: Opportunity):
+    assert opportunity.managed
+    access = OpportunityAccess.objects.get(user=mobile_user, opportunity=opportunity)
+    visits = UserVisitFactory.create_batch(
+        5, opportunity=opportunity, status=VisitValidationStatus.pending, user=mobile_user, opportunity_access=access
+    )
+    dataset = Dataset(headers=["visit id", "status", "rejected reason", "justification"])
+    dataset.extend([[visit.xform_id, VisitValidationStatus.rejected.value, "", "justification"] for visit in visits])
+    before_update = now()
+    import_status = _bulk_update_visit_status(opportunity, dataset)
+    after_update = now()
+    assert not import_status.missing_visits
+    updated_visits = UserVisit.objects.filter(opportunity=opportunity)
+    for visit in updated_visits:
+        assert visit.status == VisitValidationStatus.rejected
+        assert visit.status_modified_date is not None
+        assert before_update <= visit.status_modified_date <= after_update
+        assert visit.review_created_on is None

From 0598a635c477b892b28eb833a7c5c4e6ba82a600 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Wed, 9 Oct 2024 17:22:56 +0530
Subject: [PATCH 021/165] Add test for UserVisitReview view

---
 .../opportunity/tests/test_views.py           | 98 ++++++++++++++++++-
 1 file changed, 97 insertions(+), 1 deletion(-)

diff --git a/commcare_connect/opportunity/tests/test_views.py b/commcare_connect/opportunity/tests/test_views.py
index da418aae..a74432fd 100644
--- a/commcare_connect/opportunity/tests/test_views.py
+++ b/commcare_connect/opportunity/tests/test_views.py
@@ -1,14 +1,25 @@
 import pytest
 from django.test import Client
 from django.urls import reverse
+from django.utils.timezone import now
 
-from commcare_connect.opportunity.models import Opportunity, OpportunityAccess, OpportunityClaimLimit
+from commcare_connect.opportunity.models import (
+    Opportunity,
+    OpportunityAccess,
+    OpportunityClaimLimit,
+    UserVisit,
+    VisitReviewStatus,
+    VisitValidationStatus,
+)
 from commcare_connect.opportunity.tests.factories import (
+    OpportunityAccessFactory,
     OpportunityClaimFactory,
     OpportunityClaimLimitFactory,
     PaymentUnitFactory,
+    UserVisitFactory,
 )
 from commcare_connect.organization.models import Organization
+from commcare_connect.program.tests.factories import ManagedOpportunityFactory, ProgramFactory
 from commcare_connect.users.models import User
 
 
@@ -38,3 +49,88 @@ def test_add_budget_existing_users(
     assert opportunity.total_budget == 205
     assert opportunity.claimed_budget == 15
     assert OpportunityClaimLimit.objects.get(pk=ocl.pk).max_visits == 15
+
+
+class TestUserVisitReviewView:
+    @pytest.fixture(autouse=True)
+    @pytest.mark.parametrize("opportunity", [{"managed": True}], indirect=True)
+    def setup(
+        self,
+        client: Client,
+        program_manager_org: Organization,
+        program_manager_org_user_admin: User,
+        organization: Organization,
+        org_user_admin: User,
+        mobile_user: User,
+    ):
+        self.client = client
+        self.pm_org = program_manager_org
+        self.pm_user = program_manager_org_user_admin
+        self.nm_org = organization
+        self.nm_user = org_user_admin
+        self.program = ProgramFactory(organization=self.pm_org)
+        self.opportunity = ManagedOpportunityFactory(program=self.program, organization=self.nm_org)
+        self.mobile_user = mobile_user
+        access = OpportunityAccessFactory(user=mobile_user, opportunity=self.opportunity, accepted=True)
+        self.visits = UserVisitFactory.create_batch(
+            10,
+            user=mobile_user,
+            opportunity=self.opportunity,
+            status=VisitValidationStatus.approved,
+            review_created_on=now(),
+            review_status=VisitReviewStatus.pending,
+            opportunity_access=access,
+        )
+
+    def test_user_visit_review_program_manager_pending(self):
+        self.url = reverse("opportunity:user_visit_review", args=(self.pm_org.slug, self.opportunity.id))
+        self.client.force_login(self.pm_user)
+        response = self.client.get(self.url)
+        assert response.status_code == 200
+        table = response.context["table"]
+        assert len(table.rows) == 10
+        assert "pk" in table.columns.names()
+
+        visits = UserVisit.objects.filter(id__in=[visit.id for visit in self.visits])
+        for visit in visits:
+            assert visit.review_status == VisitReviewStatus.pending
+
+    def test_user_visit_review_program_manager_agree(self):
+        self.url = reverse("opportunity:user_visit_review", args=(self.pm_org.slug, self.opportunity.id))
+        self.client.force_login(self.pm_user)
+        response = self.client.post(self.url, {"pk": [], "review_status": VisitReviewStatus.agree.value})
+        assert response.status_code == 200
+        visits = UserVisit.objects.filter(id__in=[visit.id for visit in self.visits])
+        for visit in visits:
+            assert visit.review_status == VisitReviewStatus.pending
+
+        visit_ids = [visit.id for visit in self.visits][:5]
+        response = self.client.post(self.url, {"pk": visit_ids, "review_status": VisitReviewStatus.agree.value})
+        assert response.status_code == 200
+        visits = UserVisit.objects.filter(id__in=visit_ids)
+        for visit in visits:
+            assert visit.review_status == VisitReviewStatus.agree
+
+    def test_user_visit_review_program_manager_reject(self):
+        self.url = reverse("opportunity:user_visit_review", args=(self.pm_org.slug, self.opportunity.id))
+        self.client.force_login(self.pm_user)
+        response = self.client.post(self.url, {"pk": [], "review_status": VisitReviewStatus.agree.value})
+        assert response.status_code == 200
+        visits = UserVisit.objects.filter(id__in=[visit.id for visit in self.visits])
+        for visit in visits:
+            assert visit.review_status == VisitReviewStatus.pending
+
+        visit_ids = [visit.id for visit in self.visits][:5]
+        response = self.client.post(self.url, {"pk": visit_ids, "review_status": VisitReviewStatus.disagree.value})
+        assert response.status_code == 200
+        visits = UserVisit.objects.filter(id__in=visit_ids)
+        for visit in visits:
+            assert visit.review_status == VisitReviewStatus.disagree
+
+    def test_user_visit_review_network_manager(self):
+        self.url = reverse("opportunity:user_visit_review", args=(self.nm_org.slug, self.opportunity.id))
+        self.client.force_login(self.nm_user)
+        response = self.client.get(self.url)
+        table = response.context["table"]
+        assert len(table.rows) == 10
+        assert "pk" not in table.columns.names()

From dca64e5e07695145a66d87ec4d866178f9b157c1 Mon Sep 17 00:00:00 2001
From: Sravan Reddy <sravfeyn@gmail.com>
Date: Wed, 9 Oct 2024 22:17:22 +0530
Subject: [PATCH 022/165] include submission time slot in opportunity API

---
 .../tests/test_receiver_integration.py        | 15 +++------------
 .../opportunity/api/serializers.py            | 19 +++++++++++++++++++
 .../opportunity/tests/factories.py            |  4 +++-
 .../opportunity/tests/test_api_views.py       |  4 ++++
 4 files changed, 29 insertions(+), 13 deletions(-)

diff --git a/commcare_connect/form_receiver/tests/test_receiver_integration.py b/commcare_connect/form_receiver/tests/test_receiver_integration.py
index c52af2a5..908fa0df 100644
--- a/commcare_connect/form_receiver/tests/test_receiver_integration.py
+++ b/commcare_connect/form_receiver/tests/test_receiver_integration.py
@@ -464,12 +464,9 @@ def test_reciever_verification_flags_form_submission(
     user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
 ):
     verification_flags = OpportunityVerificationFlags.objects.get(opportunity=opportunity)
-    verification_flags.form_submission_start = datetime.time(hour=10, minute=0)
-    verification_flags.form_submission_end = datetime.time(hour=12, minute=0)
-    verification_flags.save()
 
     form_json = _create_opp_and_form_json(opportunity, user=user_with_connectid_link)
-    time = datetime.datetime(2024, 4, 17, 10, 0, 0)
+    time = datetime.datetime(2024, 4, 17, verification_flags.form_submission_start.hour, 0, 0)
     form_json["metadata"]["timeStart"] = time
     form_json["metadata"]["timeEnd"] = time + datetime.timedelta(minutes=10)
     make_request(api_client, form_json, user_with_connectid_link)
@@ -481,12 +478,9 @@ def test_reciever_verification_flags_form_submission_start(
     user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
 ):
     verification_flags = OpportunityVerificationFlags.objects.get(opportunity=opportunity)
-    verification_flags.form_submission_start = datetime.time(hour=10, minute=0)
-    verification_flags.form_submission_end = datetime.time(hour=12, minute=0)
-    verification_flags.save()
 
     form_json = _create_opp_and_form_json(opportunity, user=user_with_connectid_link)
-    time = datetime.datetime(2024, 4, 17, 9, 0, 0)
+    time = datetime.datetime(2024, 4, 17, verification_flags.form_submission_start.hour - 1, 0, 0)
     form_json["metadata"]["timeStart"] = time
     make_request(api_client, form_json, user_with_connectid_link)
     visit = UserVisit.objects.get(user=user_with_connectid_link)
@@ -498,12 +492,9 @@ def test_reciever_verification_flags_form_submission_end(
     user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
 ):
     verification_flags = OpportunityVerificationFlags.objects.get(opportunity=opportunity)
-    verification_flags.form_submission_start = datetime.time(hour=10, minute=0)
-    verification_flags.form_submission_end = datetime.time(hour=12, minute=0)
-    verification_flags.save()
 
     form_json = _create_opp_and_form_json(opportunity, user=user_with_connectid_link)
-    time = datetime.datetime(2024, 4, 17, 13, 0, 0)
+    time = datetime.datetime(2024, 4, 17, verification_flags.form_submission_end.hour + 1, 0, 0)
     form_json["metadata"]["timeStart"] = time
     make_request(api_client, form_json, user_with_connectid_link)
     visit = UserVisit.objects.get(user=user_with_connectid_link)
diff --git a/commcare_connect/opportunity/api/serializers.py b/commcare_connect/opportunity/api/serializers.py
index bf558935..cbeafe7d 100644
--- a/commcare_connect/opportunity/api/serializers.py
+++ b/commcare_connect/opportunity/api/serializers.py
@@ -1,3 +1,5 @@
+from functools import lru_cache
+
 from django.conf import settings
 from django.db.models import Sum
 from rest_framework import serializers
@@ -15,6 +17,7 @@
     OpportunityAccess,
     OpportunityClaim,
     OpportunityClaimLimit,
+    OpportunityVerificationFlags,
     Payment,
     PaymentUnit,
     UserVisit,
@@ -105,6 +108,8 @@ class OpportunitySerializer(serializers.ModelSerializer):
     payment_units = serializers.SerializerMethodField()
     is_user_suspended = serializers.SerializerMethodField()
     catchment_areas = serializers.SerializerMethodField()
+    start_time_threshold = serializers.SerializerMethodField()
+    end_time_threshold = serializers.SerializerMethodField()
 
     class Meta:
         model = Opportunity
@@ -133,6 +138,8 @@ class Meta:
             "payment_units",
             "is_user_suspended",
             "catchment_areas",
+            "start_time_threshold",
+            "end_time_threshold",
         ]
 
     def get_claim(self, obj):
@@ -178,6 +185,18 @@ def get_catchment_areas(self, obj):
         catchments = CatchmentArea.objects.filter(opportunity_access=opp_access)
         return CatchmentAreaSerializer(catchments, many=True).data
 
+    @lru_cache
+    def _get_flags(self, obj):
+        return OpportunityVerificationFlags.objects.filter(opportunity=obj).first()
+
+    def get_start_time_threshold(self, obj):
+        flags = self._get_flags(obj)
+        return flags.form_submission_start
+
+    def get_end_time_threshold(self, obj):
+        flags = self._get_flags(obj)
+        return flags.form_submission_end
+
 
 @quickcache(vary_on=["user.pk", "opportunity.pk"], timeout=60 * 60)
 def _get_opp_access(user, opportunity):
diff --git a/commcare_connect/opportunity/tests/factories.py b/commcare_connect/opportunity/tests/factories.py
index daa64837..1d7bf83a 100644
--- a/commcare_connect/opportunity/tests/factories.py
+++ b/commcare_connect/opportunity/tests/factories.py
@@ -1,4 +1,4 @@
-from datetime import timezone
+from datetime import time, timezone
 
 from factory import DictFactory, Faker, LazyAttribute, SelfAttribute, SubFactory
 from factory.django import DjangoModelFactory
@@ -72,6 +72,8 @@ class Meta:
 
 class OpportunityVerificationFlagsFactory(DjangoModelFactory):
     opportunity = SubFactory(OpportunityFactory)
+    form_submission_start = time(10, 0, 0)
+    form_submission_end = time(12, 0, 0)
 
     class Meta:
         model = "opportunity.OpportunityVerificationFlags"
diff --git a/commcare_connect/opportunity/tests/test_api_views.py b/commcare_connect/opportunity/tests/test_api_views.py
index e7d26a88..c574c934 100644
--- a/commcare_connect/opportunity/tests/test_api_views.py
+++ b/commcare_connect/opportunity/tests/test_api_views.py
@@ -15,6 +15,7 @@
     OpportunityAccess,
     OpportunityClaim,
     OpportunityClaimLimit,
+    OpportunityVerificationFlags,
     Payment,
     VisitValidationStatus,
 )
@@ -160,6 +161,9 @@ def test_opportunity_list_endpoint(
     assert response.data[0]["budget_per_visit"] == max([pu.amount for pu in payment_units])
     claim_limits = OpportunityClaimLimit.objects.filter(opportunity_claim__opportunity_access__opportunity=opportunity)
     assert response.data[0]["claim"]["max_payments"] == sum([cl.max_visits for cl in claim_limits])
+    verification_flags = OpportunityVerificationFlags.objects.get(opportunity=opportunity)
+    assert response.data[0]["start_time_threshold"] == verification_flags.form_submission_start
+    assert response.data[0]["end_time_threshold"] == verification_flags.form_submission_end
 
 
 def test_delivery_progress_endpoint(

From 78fe8cd983e6cb2381c1edd63f73eea1d9b4c2a3 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Tue, 15 Oct 2024 15:26:51 +0530
Subject: [PATCH 023/165] Refactor tests removed print statement.

---
 commcare_connect/program/helpers.py           |  5 --
 .../program/tests/test_helpers.py             | 46 +++++++++----------
 commcare_connect/program/views.py             |  3 --
 3 files changed, 21 insertions(+), 33 deletions(-)

diff --git a/commcare_connect/program/helpers.py b/commcare_connect/program/helpers.py
index d74e05f3..bdbe8665 100644
--- a/commcare_connect/program/helpers.py
+++ b/commcare_connect/program/helpers.py
@@ -97,11 +97,6 @@ def get_delivery_performance_report(program: Program, start_date, end_date):
     managed_opportunities = (
         ManagedOpportunity.objects.filter(program=program)
         .order_by("start_date")
-        .prefetch_related(
-            "opportunityaccess_set",
-            "opportunityaccess_set__uservisit_set",
-            "opportunityaccess_set__completedwork_set",
-        )
         .annotate(
             total_workers_starting_delivery=Count(
                 "opportunityaccess__uservisit__user",
diff --git a/commcare_connect/program/tests/test_helpers.py b/commcare_connect/program/tests/test_helpers.py
index 7be60a1c..986d3cf8 100644
--- a/commcare_connect/program/tests/test_helpers.py
+++ b/commcare_connect/program/tests/test_helpers.py
@@ -15,7 +15,8 @@
 from commcare_connect.users.tests.factories import OrganizationFactory, UserFactory
 
 
-class TestGetAnnotatedManagedOpportunity:
+@pytest.mark.django_db
+class BaseManagedOpportunityTest:
     @pytest.fixture(autouse=True)
     def setup(self, db):
         self.program = ProgramFactory.create()
@@ -35,6 +36,23 @@ def create_user_with_access(self, visit_status=VisitValidationStatus.pending, pa
         )
         return user
 
+    def create_user_with_visit(self, visit_status, visit_date, flagged=False, create_completed_work=True):
+        user = UserFactory.create()
+        access = OpportunityAccessFactory.create(opportunity=self.opp, user=user, invited_date=now())
+        UserVisitFactory.create(
+            user=user,
+            opportunity=self.opp,
+            status=visit_status,
+            opportunity_access=access,
+            visit_date=visit_date,
+            flagged=flagged,
+        )
+        if create_completed_work:
+            CompletedWorkFactory.create(opportunity_access=access)
+        return user
+
+
+class TestGetAnnotatedManagedOpportunity(BaseManagedOpportunityTest):
     def test_basic_scenario(self):
         for i in range(5):
             self.create_user_with_access(
@@ -103,7 +121,7 @@ def test_average_time_to_convert(self):
         assert abs(actual_time - expected_time) < timedelta(seconds=5)
 
     def test_multiple_opportunities(self):
-        nm_org2 = OrganizationFactory.create()
+        nm_org2 = OrganizationFactory.create(program_manager=True)
         opp2 = ManagedOpportunityFactory.create(
             program=self.program, organization=nm_org2, start_date=now() + timedelta(days=1)
         )
@@ -144,29 +162,7 @@ def test_failed_assessments(self):
 
 
 @pytest.mark.django_db
-class TestDeliveryPerformanceReport:
-    @pytest.fixture(autouse=True)
-    def setup(self):
-        self.program_manager_org = OrganizationFactory.create()
-        self.program = ProgramFactory.create(organization=self.program_manager_org)
-        self.nm_org = OrganizationFactory.create()
-        self.opp = ManagedOpportunityFactory.create(program=self.program, organization=self.nm_org)
-
-    def create_user_with_visit(self, visit_status, visit_date, flagged=False, create_completed_work=True):
-        user = UserFactory.create()
-        access = OpportunityAccessFactory.create(opportunity=self.opp, user=user, invited_date=now())
-        UserVisitFactory.create(
-            user=user,
-            opportunity=self.opp,
-            status=visit_status,
-            opportunity_access=access,
-            visit_date=visit_date,
-            flagged=flagged,
-        )
-        if create_completed_work:
-            CompletedWorkFactory.create(opportunity_access=access)
-        return user
-
+class TestDeliveryPerformanceReport(BaseManagedOpportunityTest):
     def test_basic_delivery_performance(self):
         for _ in range(2):
             self.create_user_with_visit(VisitValidationStatus.pending, now(), True)
diff --git a/commcare_connect/program/views.py b/commcare_connect/program/views.py
index 5bb75d10..ff60a8b5 100644
--- a/commcare_connect/program/views.py
+++ b/commcare_connect/program/views.py
@@ -276,7 +276,4 @@ def get_queryset(self):
         program = get_object_or_404(Program, id=program_id)
         start_date = self.request.GET.get("start_date") or None
         end_date = self.request.GET.get("end_date") or None
-
-        print("start_date", start_date)
-        print("end_date", end_date)
         return get_delivery_performance_report(program, start_date, end_date)

From f87aa0223bd7fcd6b7a3857c1051e023e876068e Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Tue, 15 Oct 2024 15:51:35 +0530
Subject: [PATCH 024/165] refactor tests

---
 commcare_connect/program/helpers.py           |   5 -
 .../program/tests/test_helpers.py             | 166 +++++++-----------
 2 files changed, 63 insertions(+), 108 deletions(-)

diff --git a/commcare_connect/program/helpers.py b/commcare_connect/program/helpers.py
index 086b7e10..78802a34 100644
--- a/commcare_connect/program/helpers.py
+++ b/commcare_connect/program/helpers.py
@@ -68,11 +68,6 @@ def get_annotated_managed_opportunity(program: Program):
                 filter=filter_for_valid__visit_date,
             ),
         )
-        .prefetch_related(
-            "opportunityaccess_set",
-            "opportunityaccess_set__uservisit_set",
-            "opportunityaccess_set__assessment_set",
-        )
     )
 
     return managed_opportunities
diff --git a/commcare_connect/program/tests/test_helpers.py b/commcare_connect/program/tests/test_helpers.py
index 150aa305..de67f02c 100644
--- a/commcare_connect/program/tests/test_helpers.py
+++ b/commcare_connect/program/tests/test_helpers.py
@@ -30,110 +30,70 @@ def create_user_with_access(self, visit_status=VisitValidationStatus.pending, pa
         )
         return user
 
-    def test_basic_scenario(self):
-        for i in range(5):
-            self.create_user_with_access(
-                visit_status=VisitValidationStatus.pending if i < 3 else VisitValidationStatus.trial
-            )
+    @pytest.mark.parametrize(
+        "scenario, visit_statuses, passing_assessments, expected_invited,"
+        " expected_passing, expected_delivery, expected_conversion",
+        [
+            (
+                "basic_scenario",
+                [VisitValidationStatus.pending, VisitValidationStatus.pending, VisitValidationStatus.trial],
+                [True, True, True],
+                3,
+                3,
+                2,
+                66.67,
+            ),
+            ("empty_scenario", [], [], 0, 0, 0, 0.0),
+            ("multiple_visits_scenario", [VisitValidationStatus.pending], [True], 1, 1, 1, 100.0),
+            (
+                "excluded_statuses",
+                [VisitValidationStatus.over_limit, VisitValidationStatus.trial],
+                [True, True],
+                2,
+                2,
+                0,
+                0.0,
+            ),
+            (
+                "failed_assessments",
+                [VisitValidationStatus.pending, VisitValidationStatus.pending],
+                [False, True],
+                2,
+                1,
+                2,
+                100.0,
+            ),
+        ],
+    )
+    def test_scenarios(
+        self,
+        scenario,
+        visit_statuses,
+        passing_assessments,
+        expected_invited,
+        expected_passing,
+        expected_delivery,
+        expected_conversion,
+    ):
+        for i, visit_status in enumerate(visit_statuses):
+            user = self.create_user_with_access(visit_status=visit_status, passed_assessment=passing_assessments[i])
+
+            # For the "multiple_visits_scenario", create additional visits for the same user
+            if scenario == "multiple_visits_scenario":
+                access = user.opportunityaccess_set.first()
+                UserVisitFactory.create_batch(
+                    2,
+                    user=user,
+                    opportunity=self.opp,
+                    status=VisitValidationStatus.pending,
+                    opportunity_access=access,
+                    visit_date=now() + timedelta(days=2),
+                )
 
         opps = get_annotated_managed_opportunity(self.program)
         assert len(opps) == 1
         annotated_opp = opps[0]
-        assert annotated_opp.organization.slug == self.nm_org.slug
-        assert annotated_opp.workers_invited == 5
-        assert annotated_opp.workers_passing_assessment == 5
-        assert annotated_opp.workers_starting_delivery == 3
-        assert annotated_opp.percentage_conversion == 60.0
-
-    def test_empty_scenario(self):
-        opps = get_annotated_managed_opportunity(self.program)
-        assert len(opps) == 1
-        annotated_opp = opps[0]
-        assert annotated_opp.workers_invited == 0
-        assert annotated_opp.workers_passing_assessment == 0
-        assert annotated_opp.workers_starting_delivery == 0
-        assert annotated_opp.percentage_conversion == 0.0
-        assert annotated_opp.average_time_to_convert is None
-
-    def test_multiple_visits(self):
-        user = self.create_user_with_access()
-        UserVisitFactory.create_batch(
-            2,
-            user=user,
-            opportunity=self.opp,
-            status=VisitValidationStatus.pending,
-            opportunity_access=user.opportunityaccess_set.first(),
-            visit_date=now() + timedelta(days=2),
-        )
-
-        opps = get_annotated_managed_opportunity(self.program)
-        assert len(opps) == 1
-        annotated_opp = opps[0]
-        assert annotated_opp.workers_invited == 1
-        assert annotated_opp.workers_passing_assessment == 1
-        assert annotated_opp.workers_starting_delivery == 1
-        assert annotated_opp.percentage_conversion == 100.0
-
-    def test_excluded_statuses(self):
-        self.create_user_with_access(visit_status=VisitValidationStatus.over_limit)
-        self.create_user_with_access(visit_status=VisitValidationStatus.trial)
-
-        opps = get_annotated_managed_opportunity(self.program)
-        assert len(opps) == 1
-        annotated_opp = opps[0]
-        assert annotated_opp.workers_invited == 2
-        assert annotated_opp.workers_passing_assessment == 2
-        assert annotated_opp.workers_starting_delivery == 0
-        assert annotated_opp.percentage_conversion == 0.0
-
-    def test_average_time_to_convert(self):
-        for i in range(3):
-            user = self.create_user_with_access()
-            user.opportunityaccess_set.update(invited_date=now() - timedelta(days=i))
-
-        opps = get_annotated_managed_opportunity(self.program)
-        assert len(opps) == 1
-        annotated_opp = opps[0]
-        expected_time = timedelta(days=2)
-        actual_time = annotated_opp.average_time_to_convert
-        assert abs(actual_time - expected_time) < timedelta(seconds=5)
-
-    def test_multiple_opportunities(self):
-        nm_org2 = OrganizationFactory.create()
-        opp2 = ManagedOpportunityFactory.create(
-            program=self.program, organization=nm_org2, start_date=now() + timedelta(days=1)
-        )
-
-        self.create_user_with_access()
-        user2 = UserFactory.create()
-        access2 = OpportunityAccessFactory.create(opportunity=opp2, user=user2, invited_date=now())
-        AssessmentFactory.create(opportunity=opp2, user=user2, opportunity_access=access2, passed=True)
-        UserVisitFactory.create(
-            user=user2,
-            opportunity=opp2,
-            status=VisitValidationStatus.pending,
-            opportunity_access=access2,
-            visit_date=now() + timedelta(days=1),
-        )
-
-        opps = get_annotated_managed_opportunity(self.program)
-        assert len(opps) == 2
-        assert opps[0].organization.slug == self.nm_org.slug
-        assert opps[1].organization.slug == nm_org2.slug
-        for annotated_opp in opps:
-            assert annotated_opp.workers_invited == 1
-            assert annotated_opp.workers_passing_assessment == 1
-            assert annotated_opp.workers_starting_delivery == 1
-            assert annotated_opp.percentage_conversion == 100.0
-
-    def test_failed_assessments(self):
-        self.create_user_with_access(passed_assessment=False)
-        self.create_user_with_access(passed_assessment=True)
-
-        opps = get_annotated_managed_opportunity(self.program)
-        assert len(opps) == 1
-        annotated_opp = opps[0]
-        assert annotated_opp.workers_invited == 2
-        assert annotated_opp.workers_passing_assessment == 1
-        assert annotated_opp.workers_starting_delivery == 2
-        assert annotated_opp.percentage_conversion == 100.0
+        assert annotated_opp.workers_invited == expected_invited, f"Failed in {scenario}"
+        assert annotated_opp.workers_passing_assessment == expected_passing, f"Failed in {scenario}"
+        assert annotated_opp.workers_starting_delivery == expected_delivery, f"Failed in {scenario}"
+        assert pytest.approx(annotated_opp.percentage_conversion, 0.01) == expected_conversion, f"Failed in {scenario}"

From 98f0a2ef5919eda10ed4f02ca7ee5abbbd4175f4 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Tue, 15 Oct 2024 15:56:19 +0530
Subject: [PATCH 025/165] Resolved migration conflict

---
 ...invited_date.py => 0061_opportunityaccess_invited_date.py} | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
 rename commcare_connect/opportunity/migrations/{0060_opportunityaccess_invited_date.py => 0061_opportunityaccess_invited_date.py} (74%)

diff --git a/commcare_connect/opportunity/migrations/0060_opportunityaccess_invited_date.py b/commcare_connect/opportunity/migrations/0061_opportunityaccess_invited_date.py
similarity index 74%
rename from commcare_connect/opportunity/migrations/0060_opportunityaccess_invited_date.py
rename to commcare_connect/opportunity/migrations/0061_opportunityaccess_invited_date.py
index d96766df..b5cebef6 100644
--- a/commcare_connect/opportunity/migrations/0060_opportunityaccess_invited_date.py
+++ b/commcare_connect/opportunity/migrations/0061_opportunityaccess_invited_date.py
@@ -1,11 +1,11 @@
-# Generated by Django 4.2.5 on 2024-10-07 05:04
+# Generated by Django 4.2.5 on 2024-10-15 10:25
 
 from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
     dependencies = [
-        ("opportunity", "0059_payment_amount_usd"),
+        ("opportunity", "0060_completedwork_payment_date"),
     ]
 
     operations = [

From e53d03fe056b2f201874ea830345a668609de15f Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Tue, 15 Oct 2024 16:36:30 +0530
Subject: [PATCH 026/165] Fix date time warning in tests

---
 commcare_connect/opportunity/tests/factories.py | 2 +-
 commcare_connect/reports/tests/test_reports.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/commcare_connect/opportunity/tests/factories.py b/commcare_connect/opportunity/tests/factories.py
index d69b3ffb..5f3316fd 100644
--- a/commcare_connect/opportunity/tests/factories.py
+++ b/commcare_connect/opportunity/tests/factories.py
@@ -230,7 +230,7 @@ class Meta:
 class PaymentFactory(DjangoModelFactory):
     opportunity_access = SubFactory(OpportunityAccessFactory)
     amount = Faker("pyint", min_value=1, max_value=10000)
-    date_paid = Faker("past_date")
+    date_paid = Faker("date_time", tzinfo=timezone.utc)
 
     class Meta:
         model = "opportunity.Payment"
diff --git a/commcare_connect/reports/tests/test_reports.py b/commcare_connect/reports/tests/test_reports.py
index 319d4100..df6e28a2 100644
--- a/commcare_connect/reports/tests/test_reports.py
+++ b/commcare_connect/reports/tests/test_reports.py
@@ -40,7 +40,7 @@ def test_delivery_stats(opportunity: Opportunity):
                     status=VisitValidationStatus.approved.value,
                     opportunity_access=access,
                     completed_work=completed_work,
-                    visit_date=Faker("date_this_month"),
+                    visit_date=Faker("date_time_this_month", tzinfo=datetime.UTC),
                 )
 
     quarter = math.ceil(datetime.datetime.utcnow().month / 12 * 4)

From 55ab9fc7090b588fcf2c01046c902975df6377c8 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 16 Oct 2024 08:29:58 +0530
Subject: [PATCH 027/165] refactor old migration to load historical model

---
 .../0060_completedwork_payment_date.py        |  6 ++++--
 .../opportunity/utils/completed_work.py       | 19 +++++++++++++++----
 2 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/commcare_connect/opportunity/migrations/0060_completedwork_payment_date.py b/commcare_connect/opportunity/migrations/0060_completedwork_payment_date.py
index d8c8622a..c8481f41 100644
--- a/commcare_connect/opportunity/migrations/0060_completedwork_payment_date.py
+++ b/commcare_connect/opportunity/migrations/0060_completedwork_payment_date.py
@@ -2,15 +2,17 @@
 
 from django.db import migrations, models, transaction
 
-from commcare_connect.opportunity.models import OpportunityAccess
 from commcare_connect.opportunity.utils.completed_work import update_work_payment_date
 
 
 @transaction.atomic
 def update_paid_date_from_payments(apps, schema_editor):
+    OpportunityAccess = apps.get_model("opportunity.OpportunityAccess")
+    Payment = apps.get_model("opportunity.Payment")
+    CompletedWork = apps.get_model("opportunity.CompletedWork")
     accesses = OpportunityAccess.objects.all()
     for access in accesses:
-        update_work_payment_date(access)
+        update_work_payment_date(access, Payment, CompletedWork)
 
 
 class Migration(migrations.Migration):
diff --git a/commcare_connect/opportunity/utils/completed_work.py b/commcare_connect/opportunity/utils/completed_work.py
index df2d9906..fa0fb459 100644
--- a/commcare_connect/opportunity/utils/completed_work.py
+++ b/commcare_connect/opportunity/utils/completed_work.py
@@ -44,9 +44,20 @@ def update_status(completed_works, opportunity_access, compute_payment=True):
         opportunity_access.save()
 
 
-def update_work_payment_date(access: OpportunityAccess):
-    payments = Payment.objects.filter(opportunity_access=access).order_by("date_paid")
-    completed_works = CompletedWork.objects.filter(opportunity_access=access).order_by("status_modified_date")
+def update_work_payment_date(access: OpportunityAccess, payment_model_ref=None, completed_work_model_ref=None):
+    """
+    Import models dynamically within the function helps us avoid issues with historical models during migrations.
+    """
+    if not payment_model_ref:
+        payment_model_ref = Payment
+
+    if not completed_work_model_ref:
+        completed_work_model_ref = CompletedWork
+
+    payments = payment_model_ref.objects.filter(opportunity_access=access).order_by("date_paid")
+    completed_works = completed_work_model_ref.objects.filter(opportunity_access=access).order_by(
+        "status_modified_date"
+    )
 
     if not payments or not completed_works:
         return
@@ -76,4 +87,4 @@ def update_work_payment_date(access: OpportunityAccess):
         break
 
     if works_to_update:
-        CompletedWork.objects.bulk_update(works_to_update, ["payment_date"])
+        completed_work_model_ref.objects.bulk_update(works_to_update, ["payment_date"])

From 979298c62f0a02365acb87326ee9c169f4cc7e3c Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 16 Oct 2024 08:33:40 +0530
Subject: [PATCH 028/165] updated the comment

---
 commcare_connect/opportunity/utils/completed_work.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/commcare_connect/opportunity/utils/completed_work.py b/commcare_connect/opportunity/utils/completed_work.py
index fa0fb459..e8d8c431 100644
--- a/commcare_connect/opportunity/utils/completed_work.py
+++ b/commcare_connect/opportunity/utils/completed_work.py
@@ -46,7 +46,9 @@ def update_status(completed_works, opportunity_access, compute_payment=True):
 
 def update_work_payment_date(access: OpportunityAccess, payment_model_ref=None, completed_work_model_ref=None):
     """
-    Import models dynamically within the function helps us avoid issues with historical models during migrations.
+    Dynamically assign models to avoid issues with historical models during migrations.
+    Top-level imports use the current model, which may not match the schema at migration
+    time. This ensures we use historical models during migrations and current models in normal execution.
     """
     if not payment_model_ref:
         payment_model_ref = Payment

From 61001288cac96be8c0db1ac22d587ae120bfcab7 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 16 Oct 2024 12:21:47 +0530
Subject: [PATCH 029/165] renamed variables

---
 commcare_connect/opportunity/utils/completed_work.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/commcare_connect/opportunity/utils/completed_work.py b/commcare_connect/opportunity/utils/completed_work.py
index e8d8c431..2050e30b 100644
--- a/commcare_connect/opportunity/utils/completed_work.py
+++ b/commcare_connect/opportunity/utils/completed_work.py
@@ -44,17 +44,14 @@ def update_status(completed_works, opportunity_access, compute_payment=True):
         opportunity_access.save()
 
 
-def update_work_payment_date(access: OpportunityAccess, payment_model_ref=None, completed_work_model_ref=None):
+def update_work_payment_date(access: OpportunityAccess, payment_model=None, completed_work_model=None):
     """
     Dynamically assign models to avoid issues with historical models during migrations.
     Top-level imports use the current model, which may not match the schema at migration
     time. This ensures we use historical models during migrations and current models in normal execution.
     """
-    if not payment_model_ref:
-        payment_model_ref = Payment
-
-    if not completed_work_model_ref:
-        completed_work_model_ref = CompletedWork
+    payment_model_ref = payment_model or Payment
+    completed_work_model_ref = completed_work_model or CompletedWork
 
     payments = payment_model_ref.objects.filter(opportunity_access=access).order_by("date_paid")
     completed_works = completed_work_model_ref.objects.filter(opportunity_access=access).order_by(

From 9ba026d3e968271ef3cd86768668388299fb3d57 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Wed, 16 Oct 2024 15:05:48 +0530
Subject: [PATCH 030/165] Refactor remove extra variables and use model choices

---
 commcare_connect/opportunity/tests/test_views.py | 6 +-----
 commcare_connect/opportunity/views.py            | 3 ++-
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/commcare_connect/opportunity/tests/test_views.py b/commcare_connect/opportunity/tests/test_views.py
index a74432fd..ab1f0700 100644
--- a/commcare_connect/opportunity/tests/test_views.py
+++ b/commcare_connect/opportunity/tests/test_views.py
@@ -53,7 +53,6 @@ def test_add_budget_existing_users(
 
 class TestUserVisitReviewView:
     @pytest.fixture(autouse=True)
-    @pytest.mark.parametrize("opportunity", [{"managed": True}], indirect=True)
     def setup(
         self,
         client: Client,
@@ -61,7 +60,6 @@ def setup(
         program_manager_org_user_admin: User,
         organization: Organization,
         org_user_admin: User,
-        mobile_user: User,
     ):
         self.client = client
         self.pm_org = program_manager_org
@@ -70,11 +68,9 @@ def setup(
         self.nm_user = org_user_admin
         self.program = ProgramFactory(organization=self.pm_org)
         self.opportunity = ManagedOpportunityFactory(program=self.program, organization=self.nm_org)
-        self.mobile_user = mobile_user
-        access = OpportunityAccessFactory(user=mobile_user, opportunity=self.opportunity, accepted=True)
+        access = OpportunityAccessFactory(opportunity=self.opportunity, accepted=True)
         self.visits = UserVisitFactory.create_batch(
             10,
-            user=mobile_user,
             opportunity=self.opportunity,
             status=VisitValidationStatus.approved,
             review_created_on=now(),
diff --git a/commcare_connect/opportunity/views.py b/commcare_connect/opportunity/views.py
index 9b6c00aa..082250a7 100644
--- a/commcare_connect/opportunity/views.py
+++ b/commcare_connect/opportunity/views.py
@@ -65,6 +65,7 @@
     PaymentInvoice,
     PaymentUnit,
     UserVisit,
+    VisitReviewStatus,
     VisitValidationStatus,
 )
 from commcare_connect.opportunity.tables import (
@@ -1119,7 +1120,7 @@ def user_visit_review(request, org_slug, opp_id):
         review_status = request.POST.get("review_status").lower()
         updated_reviews = request.POST.getlist("pk")
         user_visits = UserVisit.objects.filter(pk__in=updated_reviews)
-        if review_status in ["agree", "disagree"]:
+        if review_status in [VisitReviewStatus.agree.value, VisitReviewStatus.disagree.value]:
             user_visits.update(review_status=review_status)
             update_payment_accrued(opportunity=opportunity, users=[visit.user for visit in user_visits])
 

From 97c2f2515a09e5a19c368f6e39baab56fa90d47a Mon Sep 17 00:00:00 2001
From: Sravan Reddy <sravfeyn@gmail.com>
Date: Thu, 17 Oct 2024 22:58:00 +0530
Subject: [PATCH 031/165] move to nested fields

---
 .../opportunity/api/serializers.py            | 26 ++++++-------------
 .../opportunity/tests/test_api_views.py       |  9 ++++---
 2 files changed, 13 insertions(+), 22 deletions(-)

diff --git a/commcare_connect/opportunity/api/serializers.py b/commcare_connect/opportunity/api/serializers.py
index cbeafe7d..d293d5ab 100644
--- a/commcare_connect/opportunity/api/serializers.py
+++ b/commcare_connect/opportunity/api/serializers.py
@@ -1,5 +1,3 @@
-from functools import lru_cache
-
 from django.conf import settings
 from django.db.models import Sum
 from rest_framework import serializers
@@ -94,6 +92,12 @@ class Meta:
         fields = ["id", "name", "latitude", "longitude", "radius", "active"]
 
 
+class OpportunityVerificationFlagsSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = OpportunityVerificationFlags
+        fields = ["form_submission_start", "form_submission_end"]
+
+
 class OpportunitySerializer(serializers.ModelSerializer):
     organization = serializers.SlugRelatedField(read_only=True, slug_field="slug")
     learn_app = CommCareAppSerializer()
@@ -108,8 +112,7 @@ class OpportunitySerializer(serializers.ModelSerializer):
     payment_units = serializers.SerializerMethodField()
     is_user_suspended = serializers.SerializerMethodField()
     catchment_areas = serializers.SerializerMethodField()
-    start_time_threshold = serializers.SerializerMethodField()
-    end_time_threshold = serializers.SerializerMethodField()
+    verification_flags = OpportunityVerificationFlagsSerializer(source="opportunityverificationflags", read_only=True)
 
     class Meta:
         model = Opportunity
@@ -138,8 +141,7 @@ class Meta:
             "payment_units",
             "is_user_suspended",
             "catchment_areas",
-            "start_time_threshold",
-            "end_time_threshold",
+            "verification_flags",
         ]
 
     def get_claim(self, obj):
@@ -185,18 +187,6 @@ def get_catchment_areas(self, obj):
         catchments = CatchmentArea.objects.filter(opportunity_access=opp_access)
         return CatchmentAreaSerializer(catchments, many=True).data
 
-    @lru_cache
-    def _get_flags(self, obj):
-        return OpportunityVerificationFlags.objects.filter(opportunity=obj).first()
-
-    def get_start_time_threshold(self, obj):
-        flags = self._get_flags(obj)
-        return flags.form_submission_start
-
-    def get_end_time_threshold(self, obj):
-        flags = self._get_flags(obj)
-        return flags.form_submission_end
-
 
 @quickcache(vary_on=["user.pk", "opportunity.pk"], timeout=60 * 60)
 def _get_opp_access(user, opportunity):
diff --git a/commcare_connect/opportunity/tests/test_api_views.py b/commcare_connect/opportunity/tests/test_api_views.py
index c574c934..9afb6c37 100644
--- a/commcare_connect/opportunity/tests/test_api_views.py
+++ b/commcare_connect/opportunity/tests/test_api_views.py
@@ -15,7 +15,6 @@
     OpportunityAccess,
     OpportunityClaim,
     OpportunityClaimLimit,
-    OpportunityVerificationFlags,
     Payment,
     VisitValidationStatus,
 )
@@ -161,9 +160,11 @@ def test_opportunity_list_endpoint(
     assert response.data[0]["budget_per_visit"] == max([pu.amount for pu in payment_units])
     claim_limits = OpportunityClaimLimit.objects.filter(opportunity_claim__opportunity_access__opportunity=opportunity)
     assert response.data[0]["claim"]["max_payments"] == sum([cl.max_visits for cl in claim_limits])
-    verification_flags = OpportunityVerificationFlags.objects.get(opportunity=opportunity)
-    assert response.data[0]["start_time_threshold"] == verification_flags.form_submission_start
-    assert response.data[0]["end_time_threshold"] == verification_flags.form_submission_end
+    verification_flags = opportunity.opportunityverificationflags
+    assert response.data[0]["verification_flags"]["form_submission_start"] == str(
+        verification_flags.form_submission_start
+    )
+    assert response.data[0]["verification_flags"]["form_submission_end"] == str(verification_flags.form_submission_end)
 
 
 def test_delivery_progress_endpoint(

From 678dfae69217a104b099cedea076b25ece7e0f35 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Fri, 18 Oct 2024 17:16:40 +0530
Subject: [PATCH 032/165] removed redundant code

---
 commcare_connect/program/tests/test_helpers.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/commcare_connect/program/tests/test_helpers.py b/commcare_connect/program/tests/test_helpers.py
index 96f7d4e7..c03e0f1e 100644
--- a/commcare_connect/program/tests/test_helpers.py
+++ b/commcare_connect/program/tests/test_helpers.py
@@ -53,12 +53,6 @@ def create_user_with_visit(self, visit_status, visit_date, flagged=False, create
 
 
 class TestGetAnnotatedManagedOpportunity(BaseManagedOpportunityTest):
-    def test_basic_scenario(self):
-        for i in range(5):
-            self.create_user_with_access(
-                visit_status=VisitValidationStatus.pending if i < 3 else VisitValidationStatus.trial
-            )
-
     @pytest.mark.parametrize(
         "scenario, visit_statuses, passing_assessments, expected_invited,"
         " expected_passing, expected_delivery, expected_conversion",

From 76ea8d8524843a7794c4b2168f2eb4598d82caae Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Fri, 18 Oct 2024 17:44:55 +0530
Subject: [PATCH 033/165] Add combine test cases with parametrize

---
 .../opportunity/tests/test_views.py           | 33 ++++---------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/commcare_connect/opportunity/tests/test_views.py b/commcare_connect/opportunity/tests/test_views.py
index ab1f0700..064ee424 100644
--- a/commcare_connect/opportunity/tests/test_views.py
+++ b/commcare_connect/opportunity/tests/test_views.py
@@ -78,7 +78,7 @@ def setup(
             opportunity_access=access,
         )
 
-    def test_user_visit_review_program_manager_pending(self):
+    def test_user_visit_review_program_manager_table(self):
         self.url = reverse("opportunity:user_visit_review", args=(self.pm_org.slug, self.opportunity.id))
         self.client.force_login(self.pm_user)
         response = self.client.get(self.url)
@@ -87,43 +87,24 @@ def test_user_visit_review_program_manager_pending(self):
         assert len(table.rows) == 10
         assert "pk" in table.columns.names()
 
-        visits = UserVisit.objects.filter(id__in=[visit.id for visit in self.visits])
-        for visit in visits:
-            assert visit.review_status == VisitReviewStatus.pending
-
-    def test_user_visit_review_program_manager_agree(self):
-        self.url = reverse("opportunity:user_visit_review", args=(self.pm_org.slug, self.opportunity.id))
-        self.client.force_login(self.pm_user)
-        response = self.client.post(self.url, {"pk": [], "review_status": VisitReviewStatus.agree.value})
-        assert response.status_code == 200
-        visits = UserVisit.objects.filter(id__in=[visit.id for visit in self.visits])
-        for visit in visits:
-            assert visit.review_status == VisitReviewStatus.pending
-
-        visit_ids = [visit.id for visit in self.visits][:5]
-        response = self.client.post(self.url, {"pk": visit_ids, "review_status": VisitReviewStatus.agree.value})
-        assert response.status_code == 200
-        visits = UserVisit.objects.filter(id__in=visit_ids)
-        for visit in visits:
-            assert visit.review_status == VisitReviewStatus.agree
-
-    def test_user_visit_review_program_manager_reject(self):
+    @pytest.mark.parametrize("review_status", [(VisitReviewStatus.agree), (VisitReviewStatus.disagree)])
+    def test_user_visit_review_program_manager_approval(self, review_status):
         self.url = reverse("opportunity:user_visit_review", args=(self.pm_org.slug, self.opportunity.id))
         self.client.force_login(self.pm_user)
-        response = self.client.post(self.url, {"pk": [], "review_status": VisitReviewStatus.agree.value})
+        response = self.client.post(self.url, {"pk": [], "review_status": review_status.value})
         assert response.status_code == 200
         visits = UserVisit.objects.filter(id__in=[visit.id for visit in self.visits])
         for visit in visits:
             assert visit.review_status == VisitReviewStatus.pending
 
         visit_ids = [visit.id for visit in self.visits][:5]
-        response = self.client.post(self.url, {"pk": visit_ids, "review_status": VisitReviewStatus.disagree.value})
+        response = self.client.post(self.url, {"pk": visit_ids, "review_status": review_status.value})
         assert response.status_code == 200
         visits = UserVisit.objects.filter(id__in=visit_ids)
         for visit in visits:
-            assert visit.review_status == VisitReviewStatus.disagree
+            assert visit.review_status == review_status
 
-    def test_user_visit_review_network_manager(self):
+    def test_user_visit_review_network_manager_table(self):
         self.url = reverse("opportunity:user_visit_review", args=(self.nm_org.slug, self.opportunity.id))
         self.client.force_login(self.nm_user)
         response = self.client.get(self.url)

From 05b8f2f58977c4877ce6c755e0ebb922fe9b562e Mon Sep 17 00:00:00 2001
From: Sravan Reddy <sravfeyn@gmail.com>
Date: Mon, 21 Oct 2024 11:38:12 +0530
Subject: [PATCH 034/165] refactor tests parametrize

---
 commcare_connect/conftest.py                  |  5 +-
 .../tests/test_receiver_integration.py        | 67 ++++++++++---------
 .../opportunity/tests/factories.py            |  6 +-
 .../opportunity/tests/test_api_views.py       | 16 ++++-
 4 files changed, 55 insertions(+), 39 deletions(-)

diff --git a/commcare_connect/conftest.py b/commcare_connect/conftest.py
index 8c91a1df..1cbb6ff4 100644
--- a/commcare_connect/conftest.py
+++ b/commcare_connect/conftest.py
@@ -48,9 +48,10 @@ def user(db) -> User:
 
 
 @pytest.fixture()
-def opportunity():
+def opportunity(request):
+    verification_flags = getattr(request, "param", {}).get("verification_flags", {})
     factory = OpportunityFactory(is_test=False)
-    OpportunityVerificationFlagsFactory(opportunity=factory)
+    OpportunityVerificationFlagsFactory(opportunity=factory, **verification_flags)
     return factory
 
 
diff --git a/commcare_connect/form_receiver/tests/test_receiver_integration.py b/commcare_connect/form_receiver/tests/test_receiver_integration.py
index 908fa0df..731ef697 100644
--- a/commcare_connect/form_receiver/tests/test_receiver_integration.py
+++ b/commcare_connect/form_receiver/tests/test_receiver_integration.py
@@ -460,46 +460,47 @@ def test_auto_approve_visits_and_payments(
     assert access.payment_accrued == completed_work.payment_accrued
 
 
+@pytest.mark.parametrize(
+    "opportunity",
+    [
+        {
+            "verification_flags": {
+                "form_submission_start": datetime.time(10, 0),
+                "form_submission_end": datetime.time(14, 0),
+            }
+        }
+    ],
+    indirect=True,
+)
+@pytest.mark.parametrize(
+    "submission_time_hour, expected_message",
+    [
+        (11, None),
+        (9, "Form was submitted before the start time"),
+        (15, "Form was submitted after the end time"),
+    ],
+)
 def test_reciever_verification_flags_form_submission(
-    user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
+    user_with_connectid_link: User,
+    api_client: APIClient,
+    opportunity: Opportunity,
+    submission_time_hour,
+    expected_message,
 ):
-    verification_flags = OpportunityVerificationFlags.objects.get(opportunity=opportunity)
-
     form_json = _create_opp_and_form_json(opportunity, user=user_with_connectid_link)
-    time = datetime.datetime(2024, 4, 17, verification_flags.form_submission_start.hour, 0, 0)
-    form_json["metadata"]["timeStart"] = time
-    form_json["metadata"]["timeEnd"] = time + datetime.timedelta(minutes=10)
-    make_request(api_client, form_json, user_with_connectid_link)
-    visit = UserVisit.objects.get(user=user_with_connectid_link)
-    assert not visit.flagged
-
-
-def test_reciever_verification_flags_form_submission_start(
-    user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
-):
-    verification_flags = OpportunityVerificationFlags.objects.get(opportunity=opportunity)
+    submission_time = datetime.datetime(2024, 5, 17, hour=submission_time_hour, minute=0)
+    form_json["metadata"]["timeStart"] = submission_time
 
-    form_json = _create_opp_and_form_json(opportunity, user=user_with_connectid_link)
-    time = datetime.datetime(2024, 4, 17, verification_flags.form_submission_start.hour - 1, 0, 0)
-    form_json["metadata"]["timeStart"] = time
     make_request(api_client, form_json, user_with_connectid_link)
-    visit = UserVisit.objects.get(user=user_with_connectid_link)
-    assert visit.flagged
-    assert ["form_submission_period", "Form was submitted before the start time"] in visit.flag_reason.get("flags", [])
-
-
-def test_reciever_verification_flags_form_submission_end(
-    user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
-):
-    verification_flags = OpportunityVerificationFlags.objects.get(opportunity=opportunity)
 
-    form_json = _create_opp_and_form_json(opportunity, user=user_with_connectid_link)
-    time = datetime.datetime(2024, 4, 17, verification_flags.form_submission_end.hour + 1, 0, 0)
-    form_json["metadata"]["timeStart"] = time
-    make_request(api_client, form_json, user_with_connectid_link)
     visit = UserVisit.objects.get(user=user_with_connectid_link)
-    assert visit.flagged
-    assert ["form_submission_period", "Form was submitted after the end time"] in visit.flag_reason.get("flags", [])
+
+    # Assert based on the expected message
+    if expected_message is None:
+        assert not visit.flagged
+    else:
+        assert visit.flagged
+        assert ["form_submission_period", expected_message] in visit.flag_reason.get("flags", [])
 
 
 def test_reciever_verification_flags_duration(
diff --git a/commcare_connect/opportunity/tests/factories.py b/commcare_connect/opportunity/tests/factories.py
index 1d7bf83a..f65bb809 100644
--- a/commcare_connect/opportunity/tests/factories.py
+++ b/commcare_connect/opportunity/tests/factories.py
@@ -1,4 +1,4 @@
-from datetime import time, timezone
+from datetime import timezone
 
 from factory import DictFactory, Faker, LazyAttribute, SelfAttribute, SubFactory
 from factory.django import DjangoModelFactory
@@ -72,8 +72,8 @@ class Meta:
 
 class OpportunityVerificationFlagsFactory(DjangoModelFactory):
     opportunity = SubFactory(OpportunityFactory)
-    form_submission_start = time(10, 0, 0)
-    form_submission_end = time(12, 0, 0)
+    form_submission_start = None  # Default to None
+    form_submission_end = None  # Default to None
 
     class Meta:
         model = "opportunity.OpportunityVerificationFlags"
diff --git a/commcare_connect/opportunity/tests/test_api_views.py b/commcare_connect/opportunity/tests/test_api_views.py
index 9afb6c37..0471ebbd 100644
--- a/commcare_connect/opportunity/tests/test_api_views.py
+++ b/commcare_connect/opportunity/tests/test_api_views.py
@@ -146,8 +146,22 @@ def test_learn_progress_endpoint(mobile_user: User, api_client: APIClient):
     assert list(response.data["assessments"][0].keys()) == ["date", "score", "passing_score", "passed"]
 
 
+@pytest.mark.parametrize(
+    "opportunity",
+    [
+        {
+            "verification_flags": {
+                "form_submission_start": datetime.time(10, 0),
+                "form_submission_end": datetime.time(14, 0),
+            }
+        }
+    ],
+    indirect=True,
+)
 def test_opportunity_list_endpoint(
-    mobile_user_with_connect_link: User, api_client: APIClient, opportunity: Opportunity
+    mobile_user_with_connect_link: User,
+    api_client: APIClient,
+    opportunity: Opportunity,
 ):
     api_client.force_authenticate(mobile_user_with_connect_link)
     response = api_client.get("/api/opportunity/")

From efdb94c2aa90c099fb93bb4e8e0668d60dd8b62d Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Tue, 22 Oct 2024 16:13:08 +0530
Subject: [PATCH 035/165] Add review status completed work update tests

---
 .../opportunity/tests/factories.py            |  1 +
 .../opportunity/tests/test_visit_import.py    | 36 +++++++++++++++++++
 2 files changed, 37 insertions(+)

diff --git a/commcare_connect/opportunity/tests/factories.py b/commcare_connect/opportunity/tests/factories.py
index 5f3316fd..1646bc70 100644
--- a/commcare_connect/opportunity/tests/factories.py
+++ b/commcare_connect/opportunity/tests/factories.py
@@ -130,6 +130,7 @@ class UserVisitFactory(DjangoModelFactory):
     visit_date = Faker("date_time", tzinfo=timezone.utc)
     form_json = Faker("pydict", value_types=[str, int, float, bool])
     xform_id = Faker("uuid4")
+    completed_work = SubFactory(CompletedWorkFactory)
 
     class Meta:
         model = "opportunity.UserVisit"
diff --git a/commcare_connect/opportunity/tests/test_visit_import.py b/commcare_connect/opportunity/tests/test_visit_import.py
index 729f05d7..e41ddf5a 100644
--- a/commcare_connect/opportunity/tests/test_visit_import.py
+++ b/commcare_connect/opportunity/tests/test_visit_import.py
@@ -584,3 +584,39 @@ def test_network_manager_reject_flagged_visit(mobile_user: User, opportunity: Op
         assert visit.status_modified_date is not None
         assert before_update <= visit.status_modified_date <= after_update
         assert visit.review_created_on is None
+
+
+@pytest.mark.parametrize("opportunity", [{"managed": True}], indirect=True)
+@pytest.mark.parametrize(
+    "review_status, cw_status",
+    [
+        (VisitReviewStatus.pending, CompletedWorkStatus.pending),
+        (VisitReviewStatus.agree, CompletedWorkStatus.approved),
+        (VisitReviewStatus.disagree, CompletedWorkStatus.pending),
+    ],
+)
+def test_review_completed_work_status(
+    mobile_user_with_connect_link: User, opportunity: Opportunity, review_status, cw_status
+):
+    deliver_unit = DeliverUnitFactory(app=opportunity.deliver_app, payment_unit=opportunity.paymentunit_set.first())
+    access = OpportunityAccess.objects.get(user=mobile_user_with_connect_link, opportunity=opportunity)
+    UserVisitFactory.create_batch(
+        2,
+        opportunity_access=access,
+        status=VisitValidationStatus.approved,
+        review_status=review_status,
+        review_created_on=now(),
+        completed_work__status=CompletedWorkStatus.pending,
+        completed_work__opportunity_access=access,
+        completed_work__payment_unit=opportunity.paymentunit_set.first(),
+        deliver_unit=deliver_unit,
+    )
+    assert access.payment_accrued == 0
+    update_payment_accrued(opportunity, {mobile_user_with_connect_link.id})
+    completed_works = CompletedWork.objects.filter(opportunity_access=access)
+    payment_accrued = 0
+    for cw in completed_works:
+        assert cw.status == cw_status
+        payment_accrued += cw.payment_accrued
+    access.refresh_from_db()
+    assert access.payment_accrued == payment_accrued

From 74da1e22d28c50c65f0e37a21ab78e07eb48be1d Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Wed, 23 Oct 2024 15:52:17 +0530
Subject: [PATCH 036/165] Fix test payment_accrued calculation

---
 commcare_connect/opportunity/tests/test_visit_import.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/commcare_connect/opportunity/tests/test_visit_import.py b/commcare_connect/opportunity/tests/test_visit_import.py
index e41ddf5a..6fbd9579 100644
--- a/commcare_connect/opportunity/tests/test_visit_import.py
+++ b/commcare_connect/opportunity/tests/test_visit_import.py
@@ -617,6 +617,7 @@ def test_review_completed_work_status(
     payment_accrued = 0
     for cw in completed_works:
         assert cw.status == cw_status
-        payment_accrued += cw.payment_accrued
+        if cw.status == CompletedWorkStatus.approved:
+            payment_accrued += cw.payment_accrued
     access.refresh_from_db()
     assert access.payment_accrued == payment_accrued

From 161b11334d02d56b5475827f020c104a5cf55c60 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Wed, 23 Oct 2024 15:52:31 +0530
Subject: [PATCH 037/165] Add combine receiver tests with parametrize

---
 .../tests/test_receiver_integration.py        | 33 +++++++------------
 1 file changed, 12 insertions(+), 21 deletions(-)

diff --git a/commcare_connect/form_receiver/tests/test_receiver_integration.py b/commcare_connect/form_receiver/tests/test_receiver_integration.py
index be002639..2d2a93a4 100644
--- a/commcare_connect/form_receiver/tests/test_receiver_integration.py
+++ b/commcare_connect/form_receiver/tests/test_receiver_integration.py
@@ -588,30 +588,21 @@ def test_receiver_verification_flags_catchment_areas(
 
 
 @pytest.mark.parametrize("opportunity", [{"managed": True}], indirect=True)
-def test_receiver_auto_agree_approved_visit(
-    mobile_user_with_connect_link: User, api_client: APIClient, opportunity: Opportunity
-):
-    assert opportunity.managed
-    form_json = get_form_json_for_payment_unit(opportunity.paymentunit_set.all()[0])
-    make_request(api_client, form_json, mobile_user_with_connect_link)
-    visit = UserVisit.objects.get(user=mobile_user_with_connect_link)
-    assert not visit.flagged
-    assert visit.status == VisitValidationStatus.approved
-    assert visit.review_status == VisitReviewStatus.agree
-
-
-@pytest.mark.parametrize("opportunity", [{"managed": True}], indirect=True)
-def test_receiver_flagged_visit_review_pending(
-    mobile_user_with_connect_link: User, api_client: APIClient, opportunity: Opportunity
+@pytest.mark.parametrize(
+    "visit_status, review_status",
+    [
+        (VisitValidationStatus.approved, VisitReviewStatus.agree),
+        (VisitValidationStatus.pending, VisitReviewStatus.pending),
+    ],
+)
+def test_receiver_visit_review_status(
+    mobile_user_with_connect_link: User, api_client: APIClient, opportunity: Opportunity, visit_status, review_status
 ):
     assert opportunity.managed
-    form_json = get_form_json_for_payment_unit(opportunity.paymentunit_set.all()[0])
-    form_json["metadata"]["location"] = None
+    form_json = get_form_json_for_payment_unit(opportunity.paymentunit_set.first())
+    if visit_status != VisitValidationStatus.approved:
+        form_json["metadata"]["location"] = None
     make_request(api_client, form_json, mobile_user_with_connect_link)
-    visit = UserVisit.objects.get(user=mobile_user_with_connect_link)
-    assert visit.flagged
-    assert visit.status == VisitValidationStatus.pending
-    assert visit.review_status == VisitReviewStatus.pending
 
 
 def get_form_json_for_payment_unit(payment_unit):

From 543fa4515a52701d71467964e506c3151de3bd52 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Wed, 23 Oct 2024 15:57:19 +0530
Subject: [PATCH 038/165] Add combine network manager tests with parametrize

---
 .../opportunity/tests/test_visit_import.py    | 35 +++++--------------
 1 file changed, 8 insertions(+), 27 deletions(-)

diff --git a/commcare_connect/opportunity/tests/test_visit_import.py b/commcare_connect/opportunity/tests/test_visit_import.py
index 6fbd9579..d7e5d258 100644
--- a/commcare_connect/opportunity/tests/test_visit_import.py
+++ b/commcare_connect/opportunity/tests/test_visit_import.py
@@ -543,47 +543,28 @@ def get_assignable_completed_work_count(access: OpportunityAccess) -> int:
 
 
 @pytest.mark.parametrize("opportunity", [{"managed": True}], indirect=True)
-def test_network_manager_approve_flagged_visit(mobile_user: User, opportunity: Opportunity):
+@pytest.mark.parametrize("visit_status", [VisitValidationStatus.approved, VisitValidationStatus.rejected])
+def test_network_manager_flagged_visit_review_status(mobile_user: User, opportunity: Opportunity, visit_status):
     assert opportunity.managed
     access = OpportunityAccess.objects.get(user=mobile_user, opportunity=opportunity)
     visits = UserVisitFactory.create_batch(
         5, opportunity=opportunity, status=VisitValidationStatus.pending, user=mobile_user, opportunity_access=access
     )
     dataset = Dataset(headers=["visit id", "status", "rejected reason", "justification"])
-    dataset.extend([[visit.xform_id, VisitValidationStatus.approved.value, "", "justification"] for visit in visits])
+    dataset.extend([[visit.xform_id, visit_status.value, "", "justification"] for visit in visits])
     before_update = now()
     import_status = _bulk_update_visit_status(opportunity, dataset)
     after_update = now()
     assert not import_status.missing_visits
     updated_visits = UserVisit.objects.filter(opportunity=opportunity)
     for visit in updated_visits:
-        assert visit.status == VisitValidationStatus.approved
+        assert visit.status == visit_status
         assert visit.status_modified_date is not None
         assert before_update <= visit.status_modified_date <= after_update
-        assert before_update <= visit.review_created_on <= after_update
-        assert visit.review_status == VisitReviewStatus.pending
-        assert visit.justification == "justification"
-
-
-@pytest.mark.parametrize("opportunity", [{"managed": True}], indirect=True)
-def test_network_manager_reject_flagged_visit(mobile_user: User, opportunity: Opportunity):
-    assert opportunity.managed
-    access = OpportunityAccess.objects.get(user=mobile_user, opportunity=opportunity)
-    visits = UserVisitFactory.create_batch(
-        5, opportunity=opportunity, status=VisitValidationStatus.pending, user=mobile_user, opportunity_access=access
-    )
-    dataset = Dataset(headers=["visit id", "status", "rejected reason", "justification"])
-    dataset.extend([[visit.xform_id, VisitValidationStatus.rejected.value, "", "justification"] for visit in visits])
-    before_update = now()
-    import_status = _bulk_update_visit_status(opportunity, dataset)
-    after_update = now()
-    assert not import_status.missing_visits
-    updated_visits = UserVisit.objects.filter(opportunity=opportunity)
-    for visit in updated_visits:
-        assert visit.status == VisitValidationStatus.rejected
-        assert visit.status_modified_date is not None
-        assert before_update <= visit.status_modified_date <= after_update
-        assert visit.review_created_on is None
+        if visit.status == VisitValidationStatus.approved:
+            assert before_update <= visit.review_created_on <= after_update
+            assert visit.review_status == VisitReviewStatus.pending
+            assert visit.justification == "justification"
 
 
 @pytest.mark.parametrize("opportunity", [{"managed": True}], indirect=True)

From 7dbd8a0e1431c315b71e54a13334d42b52485ce4 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Thu, 24 Oct 2024 14:03:04 +0530
Subject: [PATCH 039/165] Refactor rearrange code to remove type warnings

---
 commcare_connect/form_receiver/processor.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/commcare_connect/form_receiver/processor.py b/commcare_connect/form_receiver/processor.py
index 641213c5..70ad0538 100644
--- a/commcare_connect/form_receiver/processor.py
+++ b/commcare_connect/form_receiver/processor.py
@@ -303,15 +303,12 @@ def process_deliver_unit(user, xform: XForm, app: CommCareApp, opportunity: Oppo
         user_visit.status = VisitValidationStatus.approved
         user_visit.review_status = VisitReviewStatus.agree
     user_visit.save()
-    if (
-        completed_work is not None
-        and completed_work.completed_count > 0
-        and completed_work.status == CompletedWorkStatus.incomplete
-    ):
-        completed_work.status = CompletedWorkStatus.pending
-        completed_work_needs_save = True
-    if completed_work_needs_save:
-        completed_work.save()
+    if completed_work is not None:
+        if completed_work.completed_count > 0 and completed_work.status == CompletedWorkStatus.incomplete:
+            completed_work.status = CompletedWorkStatus.pending
+            completed_work_needs_save = True
+        if completed_work_needs_save:
+            completed_work.save()
     download_user_visit_attachments.delay(user_visit.id)
 
 

From bc52eb64b269728d8c4b4b84edc8b451ed8c6007 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Thu, 24 Oct 2024 14:03:32 +0530
Subject: [PATCH 040/165] Add update_payment_accrued for submitted forms

---
 commcare_connect/form_receiver/processor.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/commcare_connect/form_receiver/processor.py b/commcare_connect/form_receiver/processor.py
index 70ad0538..b3d4a3d9 100644
--- a/commcare_connect/form_receiver/processor.py
+++ b/commcare_connect/form_receiver/processor.py
@@ -1,5 +1,6 @@
 import datetime
 
+from django.core.cache import cache
 from django.db.models import Count, Q
 from django.utils.timezone import now
 from geopy.distance import distance
@@ -29,6 +30,7 @@
     VisitValidationStatus,
 )
 from commcare_connect.opportunity.tasks import download_user_visit_attachments
+from commcare_connect.opportunity.visit_import import update_payment_accrued
 from commcare_connect.users.models import User
 
 LEARN_MODULE_JSONPATH = parse("$..module")
@@ -309,6 +311,8 @@ def process_deliver_unit(user, xform: XForm, app: CommCareApp, opportunity: Oppo
             completed_work_needs_save = True
         if completed_work_needs_save:
             completed_work.save()
+    with cache.lock(access.id):
+        update_payment_accrued(opportunity, [access.user_id])
     download_user_visit_attachments.delay(user_visit.id)
 
 

From 2569638208088024705da7807adb132cc84db033 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Thu, 24 Oct 2024 14:48:01 +0530
Subject: [PATCH 041/165] Fix failing tests, Refactor combine duplicate test
 cases with parametrize

---
 .../tests/test_receiver_integration.py        | 54 +++----------------
 1 file changed, 8 insertions(+), 46 deletions(-)

diff --git a/commcare_connect/form_receiver/tests/test_receiver_integration.py b/commcare_connect/form_receiver/tests/test_receiver_integration.py
index 731ef697..0be668c8 100644
--- a/commcare_connect/form_receiver/tests/test_receiver_integration.py
+++ b/commcare_connect/form_receiver/tests/test_receiver_integration.py
@@ -351,8 +351,11 @@ def test_auto_approve_payments_approved_visit(
     assert access.payment_accrued == completed_work.payment_accrued
 
 
-def test_auto_approve_payments_rejected_visit(
-    user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
+@pytest.mark.parametrize(
+    "update_func, args_required", [(update_payment_accrued, True), (bulk_approve_completed_work, False)]
+)
+def test_auto_approve_payments_rejected_visit_functions(
+    user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity, update_func, args_required
 ):
     form_json = _create_opp_and_form_json(opportunity, user=user_with_connectid_link)
     form_json["metadata"]["timeEnd"] = "2023-06-07T12:36:10.178000Z"
@@ -366,17 +369,10 @@ def test_auto_approve_payments_rejected_visit(
     rejected_reason.append(visit.reason)
     visit.save()
 
-    duplicate_json = deepcopy(form_json)
-    duplicate_json["id"] = str(uuid4())
-    make_request(api_client, duplicate_json, user_with_connectid_link)
-    visit = UserVisit.objects.get(xform_id=duplicate_json["id"])
-    visit.status = VisitValidationStatus.rejected
-    visit.reason = "duplicate"
-    rejected_reason.append(visit.reason)
-    visit.save()
-
     # Payment Approval
-    update_payment_accrued(opportunity, users=[user_with_connectid_link])
+    args = (opportunity, [user_with_connectid_link]) if args_required else ()
+    update_func(*args)
+
     access = OpportunityAccess.objects.get(user=user_with_connectid_link, opportunity=opportunity)
     completed_work = CompletedWork.objects.get(opportunity_access=access)
     assert completed_work.status == CompletedWorkStatus.rejected
@@ -406,40 +402,6 @@ def test_auto_approve_payments_approved_visit_task(
     assert access.payment_accrued == completed_work.payment_accrued
 
 
-def test_auto_approve_payments_rejected_visit_task(
-    user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
-):
-    form_json = _create_opp_and_form_json(opportunity, user=user_with_connectid_link)
-    form_json["metadata"]["timeEnd"] = "2023-06-07T12:36:10.178000Z"
-    opportunity.auto_approve_payments = True
-    opportunity.save()
-    make_request(api_client, form_json, user_with_connectid_link)
-    rejected_reason = []
-    visit = UserVisit.objects.get(user=user_with_connectid_link)
-    visit.status = VisitValidationStatus.rejected
-    visit.reason = "rejected"
-    rejected_reason.append(visit.reason)
-    visit.save()
-
-    duplicate_json = deepcopy(form_json)
-    duplicate_json["id"] = str(uuid4())
-    make_request(api_client, duplicate_json, user_with_connectid_link)
-    visit = UserVisit.objects.get(xform_id=duplicate_json["id"])
-    visit.status = VisitValidationStatus.rejected
-    visit.reason = "duplicate"
-    rejected_reason.append(visit.reason)
-    visit.save()
-
-    # Payment Approval
-    bulk_approve_completed_work()
-    access = OpportunityAccess.objects.get(user=user_with_connectid_link, opportunity=opportunity)
-    completed_work = CompletedWork.objects.get(opportunity_access=access)
-    assert completed_work.status == CompletedWorkStatus.rejected
-    for reason in rejected_reason:
-        assert reason in completed_work.reason
-    assert access.payment_accrued == completed_work.payment_accrued
-
-
 def test_auto_approve_visits_and_payments(
     user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
 ):

From 35049d33472a66aa3f4affbc06e2e16dcfb0d78d Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Thu, 24 Oct 2024 15:59:05 +0530
Subject: [PATCH 042/165] moved the logic to command to update the paid date

---
 .../update_completed_work_paid_date.py        | 23 +++++++++++++++++++
 .../0060_completedwork_payment_date.py        | 15 +-----------
 .../opportunity/utils/completed_work.py       | 18 ++++-----------
 3 files changed, 28 insertions(+), 28 deletions(-)
 create mode 100644 commcare_connect/opportunity/management/commands/update_completed_work_paid_date.py

diff --git a/commcare_connect/opportunity/management/commands/update_completed_work_paid_date.py b/commcare_connect/opportunity/management/commands/update_completed_work_paid_date.py
new file mode 100644
index 00000000..000d1e4f
--- /dev/null
+++ b/commcare_connect/opportunity/management/commands/update_completed_work_paid_date.py
@@ -0,0 +1,23 @@
+from django.core.management import BaseCommand
+from django.db import transaction
+
+from commcare_connect.opportunity.models import OpportunityAccess
+from commcare_connect.opportunity.utils.completed_work import update_work_payment_date
+
+
+class Command(BaseCommand):
+    help = "Updates paid dates from payments for all opportunity accesses"
+
+    def handle(self, *args, **kwargs):
+        try:
+            with transaction.atomic():
+                accesses = OpportunityAccess.objects.all()
+                self.stdout.write("Starting to process to update the paid date...")
+
+                for access in accesses:
+                    update_work_payment_date(access)
+
+                self.stdout.write("Process completed")
+
+        except Exception as e:
+            self.stdout.write(self.style.ERROR(f"An error occurred: {str(e)}"))
diff --git a/commcare_connect/opportunity/migrations/0060_completedwork_payment_date.py b/commcare_connect/opportunity/migrations/0060_completedwork_payment_date.py
index c8481f41..d1478347 100644
--- a/commcare_connect/opportunity/migrations/0060_completedwork_payment_date.py
+++ b/commcare_connect/opportunity/migrations/0060_completedwork_payment_date.py
@@ -1,18 +1,6 @@
 # Generated by Django 4.2.5 on 2024-10-07 08:54
 
-from django.db import migrations, models, transaction
-
-from commcare_connect.opportunity.utils.completed_work import update_work_payment_date
-
-
-@transaction.atomic
-def update_paid_date_from_payments(apps, schema_editor):
-    OpportunityAccess = apps.get_model("opportunity.OpportunityAccess")
-    Payment = apps.get_model("opportunity.Payment")
-    CompletedWork = apps.get_model("opportunity.CompletedWork")
-    accesses = OpportunityAccess.objects.all()
-    for access in accesses:
-        update_work_payment_date(access, Payment, CompletedWork)
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
@@ -26,5 +14,4 @@ class Migration(migrations.Migration):
             name="payment_date",
             field=models.DateTimeField(null=True),
         ),
-        migrations.RunPython(update_paid_date_from_payments, migrations.RunPython.noop),
     ]
diff --git a/commcare_connect/opportunity/utils/completed_work.py b/commcare_connect/opportunity/utils/completed_work.py
index 2050e30b..df2d9906 100644
--- a/commcare_connect/opportunity/utils/completed_work.py
+++ b/commcare_connect/opportunity/utils/completed_work.py
@@ -44,19 +44,9 @@ def update_status(completed_works, opportunity_access, compute_payment=True):
         opportunity_access.save()
 
 
-def update_work_payment_date(access: OpportunityAccess, payment_model=None, completed_work_model=None):
-    """
-    Dynamically assign models to avoid issues with historical models during migrations.
-    Top-level imports use the current model, which may not match the schema at migration
-    time. This ensures we use historical models during migrations and current models in normal execution.
-    """
-    payment_model_ref = payment_model or Payment
-    completed_work_model_ref = completed_work_model or CompletedWork
-
-    payments = payment_model_ref.objects.filter(opportunity_access=access).order_by("date_paid")
-    completed_works = completed_work_model_ref.objects.filter(opportunity_access=access).order_by(
-        "status_modified_date"
-    )
+def update_work_payment_date(access: OpportunityAccess):
+    payments = Payment.objects.filter(opportunity_access=access).order_by("date_paid")
+    completed_works = CompletedWork.objects.filter(opportunity_access=access).order_by("status_modified_date")
 
     if not payments or not completed_works:
         return
@@ -86,4 +76,4 @@ def update_work_payment_date(access: OpportunityAccess, payment_model=None, comp
         break
 
     if works_to_update:
-        completed_work_model_ref.objects.bulk_update(works_to_update, ["payment_date"])
+        CompletedWork.objects.bulk_update(works_to_update, ["payment_date"])

From 11d8e4ef9e94252e565bc0e9e6c992081ce01172 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Tue, 29 Oct 2024 12:07:16 +0530
Subject: [PATCH 043/165] Add lock in update_payment_accrued

---
 commcare_connect/form_receiver/processor.py  |  4 ----
 commcare_connect/opportunity/visit_import.py | 10 ++++++----
 2 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/commcare_connect/form_receiver/processor.py b/commcare_connect/form_receiver/processor.py
index b3d4a3d9..70ad0538 100644
--- a/commcare_connect/form_receiver/processor.py
+++ b/commcare_connect/form_receiver/processor.py
@@ -1,6 +1,5 @@
 import datetime
 
-from django.core.cache import cache
 from django.db.models import Count, Q
 from django.utils.timezone import now
 from geopy.distance import distance
@@ -30,7 +29,6 @@
     VisitValidationStatus,
 )
 from commcare_connect.opportunity.tasks import download_user_visit_attachments
-from commcare_connect.opportunity.visit_import import update_payment_accrued
 from commcare_connect.users.models import User
 
 LEARN_MODULE_JSONPATH = parse("$..module")
@@ -311,8 +309,6 @@ def process_deliver_unit(user, xform: XForm, app: CommCareApp, opportunity: Oppo
             completed_work_needs_save = True
         if completed_work_needs_save:
             completed_work.save()
-    with cache.lock(access.id):
-        update_payment_accrued(opportunity, [access.user_id])
     download_user_visit_attachments.delay(user_visit.id)
 
 
diff --git a/commcare_connect/opportunity/visit_import.py b/commcare_connect/opportunity/visit_import.py
index e57cbd57..c544da56 100644
--- a/commcare_connect/opportunity/visit_import.py
+++ b/commcare_connect/opportunity/visit_import.py
@@ -6,6 +6,7 @@
 from decimal import Decimal, InvalidOperation
 
 from django.conf import settings
+from django.core.cache import cache
 from django.core.files.uploadedfile import UploadedFile
 from django.db import transaction
 from django.utils.timezone import now
@@ -168,10 +169,11 @@ def update_payment_accrued(opportunity: Opportunity, users):
     """Updates payment accrued for completed and approved CompletedWork instances."""
     access_objects = OpportunityAccess.objects.filter(user__in=users, opportunity=opportunity, suspended=False)
     for access in access_objects:
-        completed_works = access.completedwork_set.exclude(
-            status__in=[CompletedWorkStatus.rejected, CompletedWorkStatus.over_limit]
-        ).select_related("payment_unit")
-        update_status(completed_works, access, True)
+        with cache.lock(f"update_payment_accrued_lock_{access.id}"):
+            completed_works = access.completedwork_set.exclude(
+                status__in=[CompletedWorkStatus.rejected, CompletedWorkStatus.over_limit]
+            ).select_related("payment_unit")
+            update_status(completed_works, access, True)
 
 
 def get_data_by_visit_id(dataset) -> dict[int, VisitData]:

From 9ab669eb03499aa4e6be86e6cab2cda9a0698f09 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Tue, 29 Oct 2024 13:22:09 +0530
Subject: [PATCH 044/165] updated budget per user for managed opp

---
 commcare_connect/opportunity/models.py        |  3 ++-
 commcare_connect/program/tests/factories.py   |  1 +
 commcare_connect/program/tests/test_models.py | 20 +++++++++++++++++++
 3 files changed, 23 insertions(+), 1 deletion(-)
 create mode 100644 commcare_connect/program/tests/test_models.py

diff --git a/commcare_connect/opportunity/models.py b/commcare_connect/opportunity/models.py
index 17f68978..f107d8e6 100644
--- a/commcare_connect/opportunity/models.py
+++ b/commcare_connect/opportunity/models.py
@@ -181,8 +181,9 @@ def budget_per_visit_new(self):
     def budget_per_user(self):
         payment_units = self.paymentunit_set.all()
         budget = 0
+        org_pay = self.managedopportunity.org_pay_per_visit if self.managed else 0
         for pu in payment_units:
-            budget += pu.max_total * pu.amount
+            budget += pu.max_total * (pu.amount + org_pay)
         return budget
 
     @property
diff --git a/commcare_connect/program/tests/factories.py b/commcare_connect/program/tests/factories.py
index f07baff6..00f22762 100644
--- a/commcare_connect/program/tests/factories.py
+++ b/commcare_connect/program/tests/factories.py
@@ -22,6 +22,7 @@ class Meta:
 
 class ManagedOpportunityFactory(OpportunityFactory):
     program = SubFactory(ProgramFactory)
+    org_pay_per_visit = Faker("random_int", min=500, max=1000)
 
     class Meta:
         model = ManagedOpportunity
diff --git a/commcare_connect/program/tests/test_models.py b/commcare_connect/program/tests/test_models.py
new file mode 100644
index 00000000..2b482778
--- /dev/null
+++ b/commcare_connect/program/tests/test_models.py
@@ -0,0 +1,20 @@
+import pytest
+
+from commcare_connect.opportunity.tests.factories import PaymentUnitFactory
+from commcare_connect.program.models import ManagedOpportunity
+from commcare_connect.program.tests.factories import ManagedOpportunityFactory
+
+
+@pytest.mark.django_db
+def test_managed_opportunity_stats():
+    opportunity = ManagedOpportunityFactory(total_budget=3600000, org_pay_per_visit=450)
+    PaymentUnitFactory(opportunity=opportunity, max_total=600, max_daily=5, amount=750, parent_payment_unit=None)
+
+    opportunity = ManagedOpportunity.objects.get(id=opportunity.id)
+
+    assert opportunity.budget_per_user == 720000
+    assert opportunity.allotted_visits == 3000
+    assert opportunity.number_of_users == 5
+    assert opportunity.max_visits_per_user_new == 600
+    assert opportunity.daily_max_visits_per_user_new == 5
+    assert opportunity.budget_per_visit_new == 750

From 0f7c83f0113cb3a6bee6975462e4a077751034b5 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Tue, 29 Oct 2024 15:02:14 +0530
Subject: [PATCH 045/165] Add missing test

---
 .../form_receiver/tests/test_receiver_integration.py       | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/commcare_connect/form_receiver/tests/test_receiver_integration.py b/commcare_connect/form_receiver/tests/test_receiver_integration.py
index e1e601f6..e24857da 100644
--- a/commcare_connect/form_receiver/tests/test_receiver_integration.py
+++ b/commcare_connect/form_receiver/tests/test_receiver_integration.py
@@ -579,7 +579,7 @@ def test_receiver_verification_flags_catchment_areas(
     assert ["catchment", "Visit outside worker catchment areas"] in visit.flag_reason.get("flags", [])
 
 
-@pytest.mark.parametrize("opportunity", [{"opp_options": {"managed": True}}], indirect=True)
+@pytest.mark.parametrize("opportunity", [{"opp_options": {"managed": True, "org_pay_per_visit": 2}}], indirect=True)
 @pytest.mark.parametrize(
     "visit_status, review_status",
     [
@@ -595,6 +595,11 @@ def test_receiver_visit_review_status(
     if visit_status != VisitValidationStatus.approved:
         form_json["metadata"]["location"] = None
     make_request(api_client, form_json, mobile_user_with_connect_link)
+    visit = UserVisit.objects.get(user=mobile_user_with_connect_link)
+    if visit_status != VisitValidationStatus.approved:
+        assert visit.flagged
+    assert visit.status == visit_status
+    assert visit.review_status == review_status
 
 
 def get_form_json_for_payment_unit(payment_unit):

From b389826dac400218ba412a6dce4303c1c0a4964b Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Tue, 29 Oct 2024 15:27:41 +0530
Subject: [PATCH 046/165] Add ManagedOpportunity in fixture

---
 commcare_connect/conftest.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/commcare_connect/conftest.py b/commcare_connect/conftest.py
index 415a6bba..c984262b 100644
--- a/commcare_connect/conftest.py
+++ b/commcare_connect/conftest.py
@@ -10,6 +10,7 @@
     PaymentUnitFactory,
 )
 from commcare_connect.organization.models import Organization
+from commcare_connect.program.tests.factories import ManagedOpportunityFactory
 from commcare_connect.users.models import User
 from commcare_connect.users.tests.factories import (
     ConnectIdUserLinkFactory,
@@ -52,7 +53,10 @@ def opportunity(request):
     verification_flags = getattr(request, "param", {}).get("verification_flags", {})
     opp_options = {"is_test": False}
     opp_options.update(getattr(request, "param", {}).get("opp_options", {}))
-    factory = OpportunityFactory(**opp_options)
+    if opp_options.get("managed", False):
+        factory = ManagedOpportunityFactory(**opp_options)
+    else:
+        factory = OpportunityFactory(**opp_options)
     OpportunityVerificationFlagsFactory(opportunity=factory, **verification_flags)
     return factory
 

From 68f5b45c12dc634f87d1e5013a4599cafb6ecf1f Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Tue, 29 Oct 2024 16:58:11 +0530
Subject: [PATCH 047/165] Add start, end date fields on model, migration

---
 ...entunit_end_date_paymentunit_start_date.py | 22 +++++++++++++++++++
 commcare_connect/opportunity/models.py        |  2 ++
 2 files changed, 24 insertions(+)
 create mode 100644 commcare_connect/opportunity/migrations/0061_paymentunit_end_date_paymentunit_start_date.py

diff --git a/commcare_connect/opportunity/migrations/0061_paymentunit_end_date_paymentunit_start_date.py b/commcare_connect/opportunity/migrations/0061_paymentunit_end_date_paymentunit_start_date.py
new file mode 100644
index 00000000..6e44bd1b
--- /dev/null
+++ b/commcare_connect/opportunity/migrations/0061_paymentunit_end_date_paymentunit_start_date.py
@@ -0,0 +1,22 @@
+# Generated by Django 4.2.5 on 2024-10-29 11:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("opportunity", "0060_completedwork_payment_date"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="paymentunit",
+            name="end_date",
+            field=models.DateField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="paymentunit",
+            name="start_date",
+            field=models.DateField(blank=True, null=True),
+        ),
+    ]
diff --git a/commcare_connect/opportunity/models.py b/commcare_connect/opportunity/models.py
index 17f68978..b18cabe5 100644
--- a/commcare_connect/opportunity/models.py
+++ b/commcare_connect/opportunity/models.py
@@ -354,6 +354,8 @@ class PaymentUnit(models.Model):
         blank=True,
         null=True,
     )
+    start_date = models.DateField(null=True, blank=True)
+    end_date = models.DateField(null=True, blank=True)
 
 
 class DeliverUnit(models.Model):

From 1064b98263c32f1ad3ad395a036bdcda3b8239c3 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 30 Oct 2024 10:43:04 +0530
Subject: [PATCH 048/165] filtered credentails on the basis of current org

---
 commcare_connect/connect_id_client/main.py    |   7 +-
 commcare_connect/opportunity/forms.py         |   8 +-
 .../opportunity/tests/test_forms.py           | 198 +++++++++++++++++-
 commcare_connect/opportunity/views.py         |   7 +-
 commcare_connect/organization/views.py        |   4 +-
 5 files changed, 215 insertions(+), 9 deletions(-)

diff --git a/commcare_connect/connect_id_client/main.py b/commcare_connect/connect_id_client/main.py
index 1f5307a1..7f117d9f 100644
--- a/commcare_connect/connect_id_client/main.py
+++ b/commcare_connect/connect_id_client/main.py
@@ -54,8 +54,11 @@ def add_credential(organization: Organization, credential: str, users: list[str]
     return
 
 
-def fetch_credentials():
-    response = _make_request(GET, "/users/fetch_credentials")
+def fetch_credentials(org_slug=None):
+    params = {}
+    if org_slug:
+        params["org_slug"] = org_slug
+    response = _make_request(GET, "/users/fetch_credentials", params=params)
     data = response.json()
     return [Credential(**c) for c in data["credentials"]]
 
diff --git a/commcare_connect/opportunity/forms.py b/commcare_connect/opportunity/forms.py
index 3e472450..7c567d89 100644
--- a/commcare_connect/opportunity/forms.py
+++ b/commcare_connect/opportunity/forms.py
@@ -33,7 +33,8 @@
 
 class OpportunityUserInviteForm(forms.Form):
     def __init__(self, *args, **kwargs):
-        credentials = connect_id_client.fetch_credentials()
+        org_slug = kwargs.pop("org_slug", None)
+        credentials = connect_id_client.fetch_credentials(org_slug)
         super().__init__(*args, **kwargs)
 
         self.helper = FormHelper(self)
@@ -73,7 +74,10 @@ def clean_users(self):
         return split_users
 
 
-class OpportunityChangeForm(forms.ModelForm, OpportunityUserInviteForm):
+class OpportunityChangeForm(
+    OpportunityUserInviteForm,
+    forms.ModelForm,
+):
     class Meta:
         model = Opportunity
         fields = [
diff --git a/commcare_connect/opportunity/tests/test_forms.py b/commcare_connect/opportunity/tests/test_forms.py
index a8f422be..58f6db95 100644
--- a/commcare_connect/opportunity/tests/test_forms.py
+++ b/commcare_connect/opportunity/tests/test_forms.py
@@ -5,8 +5,8 @@
 import pytest
 from factory.fuzzy import FuzzyDate, FuzzyText
 
-from commcare_connect.opportunity.forms import OpportunityCreationForm
-from commcare_connect.opportunity.tests.factories import ApplicationFactory
+from commcare_connect.opportunity.forms import OpportunityChangeForm, OpportunityCreationForm
+from commcare_connect.opportunity.tests.factories import ApplicationFactory, CommCareAppFactory, OpportunityFactory
 
 
 class TestOpportunityCreationForm:
@@ -111,3 +111,197 @@ def test_save(self, user, organization):
         )
         form.is_valid()
         form.save()
+
+
+@pytest.mark.django_db
+class TestOpportunityChangeForm:
+    @pytest.fixture(autouse=True)
+    def setup_credentials_mock(self, monkeypatch):
+        self.mock_credentials = [
+            type("Credential", (), {"slug": "cert1", "name": "Work for test"}),
+            type("Credential", (), {"slug": "cert2", "name": "Work for test"}),
+        ]
+        monkeypatch.setattr(
+            "commcare_connect.connect_id_client.fetch_credentials", lambda org_slug: self.mock_credentials
+        )
+
+    @pytest.fixture
+    def valid_opportunity(self, organization):
+        return OpportunityFactory(
+            organization=organization,
+            active=True,
+            learn_app=CommCareAppFactory(cc_app_id="test_learn_app"),
+            deliver_app=CommCareAppFactory(cc_app_id="test_deliver_app"),
+            name="Test Opportunity",
+            description="Test Description",
+            short_description="Short Description",
+            currency="USD",
+            is_test=False,
+            end_date=datetime.date.today() + datetime.timedelta(days=30),
+        )
+
+    @pytest.fixture
+    def base_form_data(self, valid_opportunity):
+        return {
+            "name": "Updated Opportunity",
+            "description": "Updated Description",
+            "short_description": "Updated Short Description",
+            "active": True,
+            "currency": "EUR",
+            "is_test": False,
+            "delivery_type": valid_opportunity.delivery_type.id,
+            "additional_users": 5,
+            "end_date": (datetime.date.today() + datetime.timedelta(days=60)).isoformat(),
+            "users": "+1234567890\n+9876543210",
+            "filter_country": "US",
+            "filter_credential": "cert1",
+        }
+
+    def test_form_initialization(self, valid_opportunity, organization):
+        form = OpportunityChangeForm(instance=valid_opportunity, org_slug=organization.slug)
+        expected_fields = {
+            "name",
+            "description",
+            "short_description",
+            "active",
+            "currency",
+            "is_test",
+            "delivery_type",
+            "additional_users",
+            "end_date",
+            "users",
+            "filter_country",
+            "filter_credential",
+        }
+        assert all(field in form.fields for field in expected_fields)
+
+        expected_initial = {
+            "name": valid_opportunity.name,
+            "description": valid_opportunity.description,
+            "short_description": valid_opportunity.short_description,
+            "active": valid_opportunity.active,
+            "currency": valid_opportunity.currency,
+            "is_test": valid_opportunity.is_test,
+            "delivery_type": valid_opportunity.delivery_type.id,
+            "end_date": valid_opportunity.end_date.isoformat(),
+            "filter_country": [""],
+            "filter_credential": [""],
+        }
+        assert all(form.initial.get(key) == value for key, value in expected_initial.items())
+
+    @pytest.mark.parametrize(
+        "field",
+        [
+            "name",
+            "description",
+            "short_description",
+            "currency",
+        ],
+    )
+    def test_required_fields(self, valid_opportunity, organization, field, base_form_data):
+        data = base_form_data.copy()
+        data[field] = ""
+        form = OpportunityChangeForm(data=data, instance=valid_opportunity, org_slug=organization.slug)
+        assert not form.is_valid()
+        assert field in form.errors
+
+    @pytest.mark.parametrize(
+        "test_data",
+        [
+            pytest.param(
+                {
+                    "field": "additional_users",
+                    "value": "invalid",
+                    "error_expected": True,
+                    "error_message": "Enter a whole number.",
+                },
+                id="invalid_additional_users",
+            ),
+            pytest.param(
+                {
+                    "field": "end_date",
+                    "value": "invalid-date",
+                    "error_expected": True,
+                    "error_message": "Enter a valid date.",
+                },
+                id="invalid_end_date",
+            ),
+            pytest.param(
+                {
+                    "field": "users",
+                    "value": "  +1234567890  \n  +9876543210  ",
+                    "error_expected": False,
+                    "expected_clean": ["+1234567890", "+9876543210"],
+                },
+                id="valid_users_with_whitespace",
+            ),
+        ],
+    )
+    def test_field_validation(self, valid_opportunity, organization, base_form_data, test_data):
+        data = base_form_data.copy()
+        data[test_data["field"]] = test_data["value"]
+        form = OpportunityChangeForm(data=data, instance=valid_opportunity, org_slug=organization.slug)
+        if test_data["error_expected"]:
+            assert not form.is_valid()
+            assert test_data["error_message"] in str(form.errors[test_data["field"]])
+        else:
+            assert form.is_valid()
+            if "expected_clean" in test_data:
+                assert form.cleaned_data[test_data["field"]] == test_data["expected_clean"]
+
+    @pytest.mark.parametrize(
+        "app_scenario",
+        [
+            pytest.param(
+                {
+                    "active_app_ids": ("unique_app1", "unique_app2"),
+                    "new_app_ids": ("different_app1", "different_app2"),
+                    "expected_valid": True,
+                },
+                id="unique_apps",
+            ),
+            pytest.param(
+                {
+                    "active_app_ids": ("shared_app1", "shared_app2"),
+                    "new_app_ids": ("shared_app1", "shared_app2"),
+                    "expected_valid": False,
+                },
+                id="reused_apps",
+            ),
+        ],
+    )
+    def test_app_reuse_validation(self, organization, base_form_data, app_scenario):
+        OpportunityFactory(
+            organization=organization,
+            active=True,
+            learn_app=CommCareAppFactory(cc_app_id=app_scenario["active_app_ids"][0]),
+            deliver_app=CommCareAppFactory(cc_app_id=app_scenario["active_app_ids"][1]),
+        )
+
+        inactive_opp = OpportunityFactory(
+            organization=organization,
+            active=False,
+            learn_app=CommCareAppFactory(cc_app_id=app_scenario["new_app_ids"][0]),
+            deliver_app=CommCareAppFactory(cc_app_id=app_scenario["new_app_ids"][1]),
+        )
+
+        form = OpportunityChangeForm(data=base_form_data, instance=inactive_opp, org_slug=organization.slug)
+
+        assert form.is_valid() == app_scenario["expected_valid"]
+        if not app_scenario["expected_valid"]:
+            assert "Cannot reactivate opportunity with reused applications" in str(form.errors["active"])
+
+    @pytest.mark.parametrize(
+        "data_updates,expected_valid",
+        [
+            ({"currency": "USD", "additional_users": 5}, True),
+            ({"currency": "EUR", "additional_users": 10}, True),
+            ({"currency": "INVALID", "additional_users": 5}, False),
+            ({"currency": "USD", "additional_users": -5}, True),
+        ],
+    )
+    def test_valid_combinations(self, valid_opportunity, organization, base_form_data, data_updates, expected_valid):
+        data = base_form_data.copy()
+        data.update(data_updates)
+        form = OpportunityChangeForm(data=data, instance=valid_opportunity, org_slug=organization.slug)
+        assert form.is_valid() == expected_valid
diff --git a/commcare_connect/opportunity/views.py b/commcare_connect/opportunity/views.py
index e9083536..a53c0050 100644
--- a/commcare_connect/opportunity/views.py
+++ b/commcare_connect/opportunity/views.py
@@ -212,6 +212,11 @@ class OpportunityEdit(OrganizationUserMemberRoleMixin, UpdateView):
     def get_success_url(self):
         return reverse("opportunity:detail", args=(self.request.org.slug, self.object.id))
 
+    def get_form_kwargs(self):
+        kwargs = super().get_form_kwargs()
+        kwargs["org_slug"] = self.request.org.slug
+        return kwargs
+
     def form_valid(self, form):
         opportunity = form.instance
         opportunity.modified_by = self.request.user.email
@@ -1079,7 +1084,7 @@ def import_catchment_area(request, org_slug=None, pk=None):
 @org_member_required
 def opportunity_user_invite(request, org_slug=None, pk=None):
     opportunity = get_object_or_404(Opportunity, organization=request.org, id=pk)
-    form = OpportunityUserInviteForm(data=request.POST or None)
+    form = OpportunityUserInviteForm(data=request.POST or None, org_slug=request.org.slug)
     if form.is_valid():
         users = form.cleaned_data["users"]
         filter_country = form.cleaned_data["filter_country"]
diff --git a/commcare_connect/organization/views.py b/commcare_connect/organization/views.py
index f2e5da30..fce5002c 100644
--- a/commcare_connect/organization/views.py
+++ b/commcare_connect/organization/views.py
@@ -47,7 +47,7 @@ def organization_home(request, org_slug):
     if not form:
         form = OrganizationChangeForm(instance=org)
 
-    credentials = connect_id_client.fetch_credentials()
+    credentials = connect_id_client.fetch_credentials(org_slug=request.org.slug)
     credential_name = f"Worked for {org.name}"
     if not any(c.name == credential_name for c in credentials):
         credentials.append(Credential(name=credential_name, slug=slugify(credential_name)))
@@ -96,7 +96,7 @@ def accept_invite(request, org_slug, invite_id):
 @require_POST
 def add_credential_view(request, org_slug):
     org = get_object_or_404(Organization, slug=org_slug)
-    credentials = connect_id_client.fetch_credentials()
+    credentials = connect_id_client.fetch_credentials(org_slug=request.org.slug)
     credential_name = f"Worked for {org.name}"
     if not any(c.name == credential_name for c in credentials):
         credentials.append(Credential(name=credential_name, slug=slugify(credential_name)))

From d38a8e11df9db1ca0b1c1da050612527a47fb245 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 30 Oct 2024 16:23:52 +0530
Subject: [PATCH 049/165] changed no of user calculation

---
 commcare_connect/opportunity/models.py        | 14 +++++++++++---
 commcare_connect/program/tests/test_models.py |  4 ++--
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/commcare_connect/opportunity/models.py b/commcare_connect/opportunity/models.py
index f107d8e6..cf763f5d 100644
--- a/commcare_connect/opportunity/models.py
+++ b/commcare_connect/opportunity/models.py
@@ -159,7 +159,16 @@ def approved_visits(self):
 
     @property
     def number_of_users(self):
-        return self.total_budget / self.budget_per_user
+        if not self.managed:
+            return self.total_budget / self.budget_per_user
+
+        budget_per_user = 0
+        payment_units = self.paymentunit_set.all()
+        org_pay = self.managedopportunity.org_pay_per_visit
+        for pu in payment_units:
+            budget_per_user += pu.max_total * (pu.amount + org_pay)
+
+        return self.total_budget / budget_per_user
 
     @property
     def allotted_visits(self):
@@ -181,9 +190,8 @@ def budget_per_visit_new(self):
     def budget_per_user(self):
         payment_units = self.paymentunit_set.all()
         budget = 0
-        org_pay = self.managedopportunity.org_pay_per_visit if self.managed else 0
         for pu in payment_units:
-            budget += pu.max_total * (pu.amount + org_pay)
+            budget += pu.max_total * pu.amount
         return budget
 
     @property
diff --git a/commcare_connect/program/tests/test_models.py b/commcare_connect/program/tests/test_models.py
index 2b482778..ef4243d3 100644
--- a/commcare_connect/program/tests/test_models.py
+++ b/commcare_connect/program/tests/test_models.py
@@ -8,11 +8,11 @@
 @pytest.mark.django_db
 def test_managed_opportunity_stats():
     opportunity = ManagedOpportunityFactory(total_budget=3600000, org_pay_per_visit=450)
-    PaymentUnitFactory(opportunity=opportunity, max_total=600, max_daily=5, amount=750, parent_payment_unit=None)
+    PaymentUnitFactory(opportunity=opportunity, max_total=600, max_daily=5, amount=750)
 
     opportunity = ManagedOpportunity.objects.get(id=opportunity.id)
 
-    assert opportunity.budget_per_user == 720000
+    assert opportunity.budget_per_user == 450000
     assert opportunity.allotted_visits == 3000
     assert opportunity.number_of_users == 5
     assert opportunity.max_visits_per_user_new == 600

From 88a0f52cc01ed9ceffbfda506b8710403507d315 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Wed, 30 Oct 2024 16:35:02 +0530
Subject: [PATCH 050/165] Add update_payment_accrued to form processor

---
 commcare_connect/form_receiver/processor.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/commcare_connect/form_receiver/processor.py b/commcare_connect/form_receiver/processor.py
index 70ad0538..83185a58 100644
--- a/commcare_connect/form_receiver/processor.py
+++ b/commcare_connect/form_receiver/processor.py
@@ -29,6 +29,7 @@
     VisitValidationStatus,
 )
 from commcare_connect.opportunity.tasks import download_user_visit_attachments
+from commcare_connect.opportunity.visit_import import update_payment_accrued
 from commcare_connect.users.models import User
 
 LEARN_MODULE_JSONPATH = parse("$..module")
@@ -309,6 +310,7 @@ def process_deliver_unit(user, xform: XForm, app: CommCareApp, opportunity: Oppo
             completed_work_needs_save = True
         if completed_work_needs_save:
             completed_work.save()
+    update_payment_accrued(opportunity, [user.id])
     download_user_visit_attachments.delay(user_visit.id)
 
 

From 02f634625a2c31e9b0452fd40f4d7df1dc8bd7d9 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Wed, 30 Oct 2024 19:41:56 +0530
Subject: [PATCH 051/165] Add end_date to opportunity claim limits, migration

---
 ...tyclaimlimit_end_date_paymentunit_end_date_and_more.py} | 7 ++++++-
 commcare_connect/opportunity/models.py                     | 2 ++
 2 files changed, 8 insertions(+), 1 deletion(-)
 rename commcare_connect/opportunity/migrations/{0061_paymentunit_end_date_paymentunit_start_date.py => 0061_opportunityclaimlimit_end_date_paymentunit_end_date_and_more.py} (70%)

diff --git a/commcare_connect/opportunity/migrations/0061_paymentunit_end_date_paymentunit_start_date.py b/commcare_connect/opportunity/migrations/0061_opportunityclaimlimit_end_date_paymentunit_end_date_and_more.py
similarity index 70%
rename from commcare_connect/opportunity/migrations/0061_paymentunit_end_date_paymentunit_start_date.py
rename to commcare_connect/opportunity/migrations/0061_opportunityclaimlimit_end_date_paymentunit_end_date_and_more.py
index 6e44bd1b..11e615a2 100644
--- a/commcare_connect/opportunity/migrations/0061_paymentunit_end_date_paymentunit_start_date.py
+++ b/commcare_connect/opportunity/migrations/0061_opportunityclaimlimit_end_date_paymentunit_end_date_and_more.py
@@ -1,4 +1,4 @@
-# Generated by Django 4.2.5 on 2024-10-29 11:28
+# Generated by Django 4.2.5 on 2024-10-30 14:11
 
 from django.db import migrations, models
 
@@ -9,6 +9,11 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
+        migrations.AddField(
+            model_name="opportunityclaimlimit",
+            name="end_date",
+            field=models.DateField(blank=True, null=True),
+        ),
         migrations.AddField(
             model_name="paymentunit",
             name="end_date",
diff --git a/commcare_connect/opportunity/models.py b/commcare_connect/opportunity/models.py
index b18cabe5..c734eff9 100644
--- a/commcare_connect/opportunity/models.py
+++ b/commcare_connect/opportunity/models.py
@@ -586,6 +586,7 @@ class OpportunityClaimLimit(models.Model):
     opportunity_claim = models.ForeignKey(OpportunityClaim, on_delete=models.CASCADE)
     payment_unit = models.ForeignKey(PaymentUnit, on_delete=models.CASCADE)
     max_visits = models.IntegerField()
+    end_date = models.DateField(null=True, blank=True)
 
     class Meta:
         unique_together = [
@@ -615,6 +616,7 @@ def create_claim_limits(cls, opportunity: Opportunity, claim: OpportunityClaim):
                 opportunity_claim=claim,
                 payment_unit=payment_unit,
                 defaults={"max_visits": min(remaining, payment_unit.max_total)},
+                end_date=payment_unit.end_date,
             )
 
 

From 4fc2b2ea68872b24f0e8e085b0558ef004c2158b Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 11:01:50 +0200
Subject: [PATCH 052/165] mocking out some dashboard updates

---
 .../templates/reports/dashboard.html          | 106 +++++++++++++++++-
 1 file changed, 102 insertions(+), 4 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 3e164d3a..e2def6e2 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -4,10 +4,108 @@
 {% load django_tables2 %}
 {% block title %}Admin Dashboard{% endblock %}
 {% block content %}
-  <h2 class="mb-2">Visit Dashboard</h2>
-  <div id="map" class="mt-3 mb-2 rounded border" style="height: 900px;"></div>
-
-
+  <h2 class="my-2">Program Dashboard</h2>
+  <div class="container">
+    <div class="row mb-3">
+      <div class="col-md-4 mt-4">
+        <div class="form-group">
+          <label for="program-select" class="form-label">Program</label>
+          <select class="form-select" id="program-select">
+            <option selected>Select a program...</option>
+            {% for program in programs %}
+              <option value="{{ program.id }}">{{ program.name }}</option>
+            {% endfor %}
+          </select>
+        </div>
+      </div>
+      <div class="col-md-4 mt-4">
+        <div class="form-group">
+          <label for="org-select" class="form-label">Organization</label>
+          <select class="form-select" id="org-select">
+            <option selected>Select an organization...</option>
+            {% for org in organizations %}
+              <option value="{{ org.id }}">{{ org.name }}</option>
+            {% endfor %}
+          </select>
+        </div>
+      </div>
+      <div class="col-md-4 mt-4">
+        <div class="form-group">
+          <label for="date-select" class="form-label">Date</label>
+          <input type="date" class="form-control" id="date-select">
+        </div>
+      </div>
+    </div>
+    <div class="row mb-4">
+      <div class="col-12 text-end">
+        <button type="submit" class="btn btn-primary">Update Dashboard</button>
+      </div>
+    </div>
+    <div class="row">
+      <div class="col-12 col-sm-6 col-md-3 mb-3">
+        <div class="card">
+          <div class="card-body ">
+            <div class="display-4 fw-bold">
+              0
+            </div>
+            <div class="small text-muted mt-1">
+              Total Visits
+            </div>
+          </div>
+        </div>
+      </div>
+      <div class="col-12 col-sm-6 col-md-3 mb-3">
+        <div class="card">
+          <div class="card-body">
+            <div class="display-4 fw-bold">
+              0
+            </div>
+            <div class="small text-muted mt-1">
+              Active Users
+            </div>
+          </div>
+        </div>
+      </div>
+      <div class="col-12 col-sm-6 col-md-3 mb-3">
+        <div class="card">
+          <div class="card-body">
+            <div class="display-4 fw-bold">
+              0
+            </div>
+            <div class="small text-muted mt-1">
+              Completed Visits
+            </div>
+          </div>
+        </div>
+      </div>
+      <div class="col-12 col-sm-6 col-md-3 mb-3">
+        <div class="card">
+          <div class="card-body">
+            <div class="display-4 fw-bold">
+              0
+            </div>
+            <div class="small text-muted mt-1">
+              Pending Visits
+            </div>
+          </div>
+        </div>
+      </div>
+    </div>
+  </div>
+  <div class="row">
+    <div class="col-12 col-lg-8">
+      <h3 class="my-2">Service Delivery Map</h3>
+      <div id="map" class="mt-3 mb-2 rounded border" style="height: 900px;"></div>
+    </div>
+    <div class="col-12 col-lg-4">
+      <h3 class="my-2">Additional Information</h3>
+      <div class="card">
+        <div class="card-body">
+          <p class="text-muted">Placeholder content</p>
+        </div>
+      </div>
+    </div>
+  </div>
 {% endblock content %}
 {% block inline_javascript %}
 {{ block.super }}

From 76563a23feffb4b2db9792ad59e7ebe7b9449de4 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 11:16:14 +0200
Subject: [PATCH 053/165] wire the front end of dashboard stats

---
 .../templates/reports/dashboard.html          | 51 +++++++++++++++----
 1 file changed, 42 insertions(+), 9 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index e2def6e2..f090edf4 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -5,13 +5,13 @@
 {% block title %}Admin Dashboard{% endblock %}
 {% block content %}
   <h2 class="my-2">Program Dashboard</h2>
-  <div class="container">
+  <div class="container" x-data="dashboardStats()">
     <div class="row mb-3">
       <div class="col-md-4 mt-4">
         <div class="form-group">
           <label for="program-select" class="form-label">Program</label>
           <select class="form-select" id="program-select">
-            <option selected>Select a program...</option>
+            <option selected value="">Select a program...</option>
             {% for program in programs %}
               <option value="{{ program.id }}">{{ program.name }}</option>
             {% endfor %}
@@ -22,7 +22,7 @@ <h2 class="my-2">Program Dashboard</h2>
         <div class="form-group">
           <label for="org-select" class="form-label">Organization</label>
           <select class="form-select" id="org-select">
-            <option selected>Select an organization...</option>
+            <option selected value="">Select an organization...</option>
             {% for org in organizations %}
               <option value="{{ org.id }}">{{ org.name }}</option>
             {% endfor %}
@@ -32,7 +32,7 @@ <h2 class="my-2">Program Dashboard</h2>
       <div class="col-md-4 mt-4">
         <div class="form-group">
           <label for="date-select" class="form-label">Date</label>
-          <input type="date" class="form-control" id="date-select">
+          <input type="date" class="form-control" id="date-select" value="">
         </div>
       </div>
     </div>
@@ -44,8 +44,8 @@ <h2 class="my-2">Program Dashboard</h2>
     <div class="row">
       <div class="col-12 col-sm-6 col-md-3 mb-3">
         <div class="card">
-          <div class="card-body ">
-            <div class="display-4 fw-bold">
+          <div class="card-body">
+            <div class="display-4 fw-bold" x-text="stats.total_visits || 0">
               0
             </div>
             <div class="small text-muted mt-1">
@@ -57,7 +57,7 @@ <h2 class="my-2">Program Dashboard</h2>
       <div class="col-12 col-sm-6 col-md-3 mb-3">
         <div class="card">
           <div class="card-body">
-            <div class="display-4 fw-bold">
+            <div class="display-4 fw-bold" x-text="stats.active_users || 0">
               0
             </div>
             <div class="small text-muted mt-1">
@@ -69,7 +69,7 @@ <h2 class="my-2">Program Dashboard</h2>
       <div class="col-12 col-sm-6 col-md-3 mb-3">
         <div class="card">
           <div class="card-body">
-            <div class="display-4 fw-bold">
+            <div class="display-4 fw-bold" x-text="stats.completed_visits || 0">
               0
             </div>
             <div class="small text-muted mt-1">
@@ -81,7 +81,7 @@ <h2 class="my-2">Program Dashboard</h2>
       <div class="col-12 col-sm-6 col-md-3 mb-3">
         <div class="card">
           <div class="card-body">
-            <div class="display-4 fw-bold">
+            <div class="display-4 fw-bold" x-text="stats.pending_visits || 0">
               0
             </div>
             <div class="small text-muted mt-1">
@@ -110,6 +110,39 @@ <h3 class="my-2">Additional Information</h3>
 {% block inline_javascript %}
 {{ block.super }}
   {{ user_visits|json_script:"userVisits" }}
+  <script>
+    function dashboardStats() {
+      return {
+        stats: {
+          total_visits: 0,
+          active_users: 0,
+          completed_visits: 0,
+          pending_visits: 0
+        },
+        async init() {
+          await this.loadStats();
+
+          // Set up event listeners for filter changes
+          document.getElementById('program-select').addEventListener('change', () => this.loadStats());
+          document.getElementById('org-select').addEventListener('change', () => this.loadStats());
+          document.getElementById('date-select').addEventListener('change', () => this.loadStats());
+        },
+        async loadStats() {
+          const program = document.getElementById('program-select').value;
+          const organization = document.getElementById('org-select').value;
+          const date = document.getElementById('date-select').value;
+
+          try {
+            const response = await fetch(`/api/dashboard-stats/?program=${program}&organization=${organization}&date=${date}`);
+            if (!response.ok) throw new Error('Failed to load stats');
+            this.stats = await response.json();
+          } catch (error) {
+            console.error('Error loading dashboard stats:', error);
+          }
+        }
+      }
+    }
+  </script>
   <script type="module">
     window.addEventListener('DOMContentLoaded', () => {
       mapboxgl.accessToken = "{{ mapbox_token }}";

From 9588c47d76da019892ce67bf1306b6948a0e21a5 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 11:32:36 +0200
Subject: [PATCH 054/165] wip: switch to use Django filters

---
 commcare_connect/reports/urls.py              |  3 +-
 commcare_connect/reports/views.py             | 74 +++++++++++++++-
 .../templates/reports/dashboard.html          | 85 ++++---------------
 3 files changed, 87 insertions(+), 75 deletions(-)

diff --git a/commcare_connect/reports/urls.py b/commcare_connect/reports/urls.py
index 0b780e73..cd05d994 100644
--- a/commcare_connect/reports/urls.py
+++ b/commcare_connect/reports/urls.py
@@ -6,6 +6,7 @@
 
 urlpatterns = [
     path("program_dashboard", views.program_dashboard_report, name="program_dashboard_report"),
-    path("api/visit_map_data/", views.visit_map_data, name="visit_map_data"),
     path("delivery_stats", view=views.DeliveryStatsReportView.as_view(), name="delivery_stats_report"),
+    path("api/visit_map_data/", views.visit_map_data, name="visit_map_data"),
+    path("api/dashboard_stats/", views.dashboard_stats_api, name="dashboard_stats_api"),
 ]
diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index 9cea2c50..75d490b5 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -2,6 +2,8 @@
 
 import django_filters
 import django_tables2 as tables
+from crispy_forms.helper import FormHelper
+from crispy_forms.layout import Column, Layout, Row
 from django import forms
 from django.conf import settings
 from django.contrib.auth.decorators import login_required, user_passes_test
@@ -16,7 +18,8 @@
 from django_filters.views import FilterView
 
 from commcare_connect.cache import quickcache
-from commcare_connect.opportunity.models import CompletedWork, CompletedWorkStatus, DeliveryType, Payment
+from commcare_connect.opportunity.models import CompletedWork, CompletedWorkStatus, DeliveryType, Payment, UserVisit
+from commcare_connect.organization.models import Organization
 
 from .tables import AdminReportTable
 
@@ -151,14 +154,51 @@ def _get_table_data_for_quarter(quarter, delivery_type, group_by_delivery_type=F
     return data
 
 
+class DashboardFilters(django_filters.FilterSet):
+    opportunity__delivery_type = django_filters.ModelChoiceFilter(
+        queryset=DeliveryType.objects.all(),
+        empty_label="All Programs",
+        required=False,
+    )
+    opportunity__organization = django_filters.ModelChoiceFilter(
+        queryset=Organization.objects.all(),
+        empty_label="All Organizations",
+        required=False,
+    )
+    visit_date = django_filters.DateFilter(
+        widget=forms.DateInput(attrs={"type": "date"}),
+        required=False,
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.form.helper = FormHelper()
+        self.form.helper.form_class = "form-inline"
+        self.form.helper.layout = Layout(
+            Row(
+                Column("opportunity__delivery_type", css_class="col-md-4"),
+                Column("opportunity__organization", css_class="col-md-4"),
+                Column("visit_date", css_class="col-md-4"),
+            )
+        )
+
+    class Meta:
+        model = UserVisit
+        fields = ["opportunity__delivery_type", "opportunity__organization", "visit_date"]
+
+
 @login_required
-@user_passes_test(lambda user: user.is_superuser)
-@require_GET
+@user_passes_test(lambda u: u.is_superuser)
 def program_dashboard_report(request):
+    filterset = DashboardFilters(request.GET)
+
     return render(
         request,
         "reports/dashboard.html",
-        context={"mapbox_token": settings.MAPBOX_TOKEN},
+        context={
+            "mapbox_token": settings.MAPBOX_TOKEN,
+            "filter": filterset,
+        },
     )
 
 
@@ -317,3 +357,29 @@ def object_list(self):
             data = _get_table_data_for_quarter(q, delivery_type, group_by_delivery_type)
             table_data += data
         return table_data
+
+
+@login_required
+@user_passes_test(lambda u: u.is_superuser)
+def dashboard_stats_api(request):
+    filterset = DashboardFilters(request.GET)
+
+    # Use the filtered queryset to calculate stats
+    queryset = UserVisit.objects.all()
+    if filterset.is_valid():
+        queryset = filterset.filter_queryset(queryset)
+
+    # Example stats calculation (adjust based on your needs)
+    total_visits = queryset.count()
+    completed_visits = queryset.filter(status=CompletedWorkStatus.approved).count()
+    pending_visits = total_visits - completed_visits
+    active_users = queryset.values("opportunity_access__user").distinct().count()
+
+    return JsonResponse(
+        {
+            "total_visits": total_visits,
+            "active_users": active_users,
+            "completed_visits": completed_visits,
+            "pending_visits": pending_visits,
+        }
+    )
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index f090edf4..96a4e1a2 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -6,87 +6,40 @@
 {% block content %}
   <h2 class="my-2">Program Dashboard</h2>
   <div class="container" x-data="dashboardStats()">
-    <div class="row mb-3">
-      <div class="col-md-4 mt-4">
-        <div class="form-group">
-          <label for="program-select" class="form-label">Program</label>
-          <select class="form-select" id="program-select">
-            <option selected value="">Select a program...</option>
-            {% for program in programs %}
-              <option value="{{ program.id }}">{{ program.name }}</option>
-            {% endfor %}
-          </select>
-        </div>
-      </div>
-      <div class="col-md-4 mt-4">
-        <div class="form-group">
-          <label for="org-select" class="form-label">Organization</label>
-          <select class="form-select" id="org-select">
-            <option selected value="">Select an organization...</option>
-            {% for org in organizations %}
-              <option value="{{ org.id }}">{{ org.name }}</option>
-            {% endfor %}
-          </select>
-        </div>
-      </div>
-      <div class="col-md-4 mt-4">
-        <div class="form-group">
-          <label for="date-select" class="form-label">Date</label>
-          <input type="date" class="form-control" id="date-select" value="">
-        </div>
-      </div>
-    </div>
-    <div class="row mb-4">
-      <div class="col-12 text-end">
-        <button type="submit" class="btn btn-primary">Update Dashboard</button>
-      </div>
-    </div>
+    <form class="row mb-3" id="filter-form" hx-get="{% url 'reports:dashboard_stats_api' %}"
+          hx-trigger="change from:#filter-form" hx-swap="none" x-ref="filterForm">
+      {% crispy filter.form %}
+    </form>
     <div class="row">
       <div class="col-12 col-sm-6 col-md-3 mb-3">
         <div class="card">
           <div class="card-body">
-            <div class="display-4 fw-bold" x-text="stats.total_visits || 0">
-              0
-            </div>
-            <div class="small text-muted mt-1">
-              Total Visits
-            </div>
+            <div class="display-4 fw-bold" x-text="stats.total_visits || 0">0</div>
+            <div class="small text-muted mt-1">Total Visits</div>
           </div>
         </div>
       </div>
       <div class="col-12 col-sm-6 col-md-3 mb-3">
         <div class="card">
           <div class="card-body">
-            <div class="display-4 fw-bold" x-text="stats.active_users || 0">
-              0
-            </div>
-            <div class="small text-muted mt-1">
-              Active Users
-            </div>
+            <div class="display-4 fw-bold" x-text="stats.active_users || 0">0</div>
+            <div class="small text-muted mt-1">Active Users</div>
           </div>
         </div>
       </div>
       <div class="col-12 col-sm-6 col-md-3 mb-3">
         <div class="card">
           <div class="card-body">
-            <div class="display-4 fw-bold" x-text="stats.completed_visits || 0">
-              0
-            </div>
-            <div class="small text-muted mt-1">
-              Completed Visits
-            </div>
+            <div class="display-4 fw-bold" x-text="stats.completed_visits || 0">0</div>
+            <div class="small text-muted mt-1">Completed Visits</div>
           </div>
         </div>
       </div>
       <div class="col-12 col-sm-6 col-md-3 mb-3">
         <div class="card">
           <div class="card-body">
-            <div class="display-4 fw-bold" x-text="stats.pending_visits || 0">
-              0
-            </div>
-            <div class="small text-muted mt-1">
-              Pending Visits
-            </div>
+            <div class="display-4 fw-bold" x-text="stats.pending_visits || 0">0</div>
+            <div class="small text-muted mt-1">Pending Visits</div>
           </div>
         </div>
       </div>
@@ -109,7 +62,6 @@ <h3 class="my-2">Additional Information</h3>
 {% endblock content %}
 {% block inline_javascript %}
 {{ block.super }}
-  {{ user_visits|json_script:"userVisits" }}
   <script>
     function dashboardStats() {
       return {
@@ -121,19 +73,12 @@ <h3 class="my-2">Additional Information</h3>
         },
         async init() {
           await this.loadStats();
-
-          // Set up event listeners for filter changes
-          document.getElementById('program-select').addEventListener('change', () => this.loadStats());
-          document.getElementById('org-select').addEventListener('change', () => this.loadStats());
-          document.getElementById('date-select').addEventListener('change', () => this.loadStats());
         },
         async loadStats() {
-          const program = document.getElementById('program-select').value;
-          const organization = document.getElementById('org-select').value;
-          const date = document.getElementById('date-select').value;
-
           try {
-            const response = await fetch(`/api/dashboard-stats/?program=${program}&organization=${organization}&date=${date}`);
+            const formData = new FormData(this.$refs.filterForm);
+            const queryString = new URLSearchParams(formData).toString();
+            const response = await fetch(`{% url 'reports:dashboard_stats_api' %}?${queryString}`);
             if (!response.ok) throw new Error('Failed to load stats');
             this.stats = await response.json();
           } catch (error) {

From 3e26dec8e4433a76eb793b50052c9410ee8e1175 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 12:19:05 +0200
Subject: [PATCH 055/165] fix form submission

---
 .../templates/reports/dashboard.html          | 41 +++++++++++++++----
 1 file changed, 33 insertions(+), 8 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 96a4e1a2..f78fd00d 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -5,11 +5,19 @@
 {% block title %}Admin Dashboard{% endblock %}
 {% block content %}
   <h2 class="my-2">Program Dashboard</h2>
-  <div class="container" x-data="dashboardStats()">
-    <form class="row mb-3" id="filter-form" hx-get="{% url 'reports:dashboard_stats_api' %}"
-          hx-trigger="change from:#filter-form" hx-swap="none" x-ref="filterForm">
-      {% crispy filter.form %}
-    </form>
+  <div class="container" x-data="dashboardStats()" x-init="$nextTick(() => console.log('Alpine initialized'))">
+    <div class="mb-4">
+        <form class="row mb-4 id="filter-form" x-ref="filterForm" x-on:submit.prevent="loadStats()">
+        {% crispy filter.form %}
+        <div class="col-12 text-end">
+            <button
+            type="button"
+            class="btn btn-primary"
+            @click="loadStats()"
+            >Update Dashboard</button>
+        </div>
+        </form>
+    </div>
     <div class="row">
       <div class="col-12 col-sm-6 col-md-3 mb-3">
         <div class="card">
@@ -72,15 +80,32 @@ <h3 class="my-2">Additional Information</h3>
           pending_visits: 0
         },
         async init() {
+          console.log('init called');
           await this.loadStats();
         },
-        async loadStats() {
+        async loadStats(event) {
+          console.log('loadStats called', {
+            event,
+            form: this.$refs.filterForm,
+            formData: new FormData(this.$refs.filterForm)
+          });
+
           try {
             const formData = new FormData(this.$refs.filterForm);
             const queryString = new URLSearchParams(formData).toString();
-            const response = await fetch(`{% url 'reports:dashboard_stats_api' %}?${queryString}`);
+            console.log('queryString:', queryString);
+
+            const url = `{% url 'reports:dashboard_stats_api' %}?${queryString}`;
+            console.log('fetching from:', url);
+
+            const response = await fetch(url);
+            console.log('response:', response);
+
             if (!response.ok) throw new Error('Failed to load stats');
-            this.stats = await response.json();
+            const data = await response.json();
+            console.log('received data:', data);
+
+            this.stats = data;
           } catch (error) {
             console.error('Error loading dashboard stats:', error);
           }

From 6ac84d59130903d0ff8d088f0ba76d0ea32b586f Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 12:21:48 +0200
Subject: [PATCH 056/165] run autoformat

---
 .../templates/reports/dashboard.html          | 424 +++++++++---------
 1 file changed, 210 insertions(+), 214 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index f78fd00d..04efd3a3 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -4,253 +4,249 @@
 {% load django_tables2 %}
 {% block title %}Admin Dashboard{% endblock %}
 {% block content %}
-  <h2 class="my-2">Program Dashboard</h2>
-  <div class="container" x-data="dashboardStats()" x-init="$nextTick(() => console.log('Alpine initialized'))">
-    <div class="mb-4">
-        <form class="row mb-4 id="filter-form" x-ref="filterForm" x-on:submit.prevent="loadStats()">
-        {% crispy filter.form %}
-        <div class="col-12 text-end">
-            <button
-            type="button"
-            class="btn btn-primary"
-            @click="loadStats()"
-            >Update Dashboard</button>
-        </div>
-        </form>
-    </div>
-    <div class="row">
-      <div class="col-12 col-sm-6 col-md-3 mb-3">
-        <div class="card">
-          <div class="card-body">
-            <div class="display-4 fw-bold" x-text="stats.total_visits || 0">0</div>
-            <div class="small text-muted mt-1">Total Visits</div>
-          </div>
-        </div>
+<h2 class="my-2">Program Dashboard</h2>
+<div class="container" x-data="dashboardStats()" x-init="$nextTick(() => console.log('Alpine initialized'))">
+  <div class="mb-4">
+    <form class="row mb-4 id=" filter-form" x-ref="filterForm" x-on:submit.prevent="loadStats()">
+      {% crispy filter.form %}
+      <div class="col-12 text-end">
+        <button type="button" class="btn btn-primary" @click="loadStats()">Update Dashboard</button>
       </div>
-      <div class="col-12 col-sm-6 col-md-3 mb-3">
-        <div class="card">
-          <div class="card-body">
-            <div class="display-4 fw-bold" x-text="stats.active_users || 0">0</div>
-            <div class="small text-muted mt-1">Active Users</div>
-          </div>
+    </form>
+  </div>
+  <div class="row">
+    <div class="col-12 col-sm-6 col-md-3 mb-3">
+      <div class="card">
+        <div class="card-body">
+          <div class="display-4 fw-bold" x-text="stats.total_visits || 0">0</div>
+          <div class="small text-muted mt-1">Total Visits</div>
         </div>
       </div>
-      <div class="col-12 col-sm-6 col-md-3 mb-3">
-        <div class="card">
-          <div class="card-body">
-            <div class="display-4 fw-bold" x-text="stats.completed_visits || 0">0</div>
-            <div class="small text-muted mt-1">Completed Visits</div>
-          </div>
+    </div>
+    <div class="col-12 col-sm-6 col-md-3 mb-3">
+      <div class="card">
+        <div class="card-body">
+          <div class="display-4 fw-bold" x-text="stats.active_users || 0">0</div>
+          <div class="small text-muted mt-1">Active Users</div>
         </div>
       </div>
-      <div class="col-12 col-sm-6 col-md-3 mb-3">
-        <div class="card">
-          <div class="card-body">
-            <div class="display-4 fw-bold" x-text="stats.pending_visits || 0">0</div>
-            <div class="small text-muted mt-1">Pending Visits</div>
-          </div>
+    </div>
+    <div class="col-12 col-sm-6 col-md-3 mb-3">
+      <div class="card">
+        <div class="card-body">
+          <div class="display-4 fw-bold" x-text="stats.completed_visits || 0">0</div>
+          <div class="small text-muted mt-1">Completed Visits</div>
         </div>
       </div>
     </div>
-  </div>
-  <div class="row">
-    <div class="col-12 col-lg-8">
-      <h3 class="my-2">Service Delivery Map</h3>
-      <div id="map" class="mt-3 mb-2 rounded border" style="height: 900px;"></div>
-    </div>
-    <div class="col-12 col-lg-4">
-      <h3 class="my-2">Additional Information</h3>
+    <div class="col-12 col-sm-6 col-md-3 mb-3">
       <div class="card">
         <div class="card-body">
-          <p class="text-muted">Placeholder content</p>
+          <div class="display-4 fw-bold" x-text="stats.pending_visits || 0">0</div>
+          <div class="small text-muted mt-1">Pending Visits</div>
         </div>
       </div>
     </div>
   </div>
+</div>
+<div class="row">
+  <div class="col-12 col-lg-8">
+    <h3 class="my-2">Service Delivery Map</h3>
+    <div id="map" class="mt-3 mb-2 rounded border" style="height: 900px;"></div>
+  </div>
+  <div class="col-12 col-lg-4">
+    <h3 class="my-2">Additional Information</h3>
+    <div class="card">
+      <div class="card-body">
+        <p class="text-muted">Placeholder content</p>
+      </div>
+    </div>
+  </div>
+</div>
 {% endblock content %}
 {% block inline_javascript %}
 {{ block.super }}
-  <script>
-    function dashboardStats() {
-      return {
-        stats: {
-          total_visits: 0,
-          active_users: 0,
-          completed_visits: 0,
-          pending_visits: 0
-        },
-        async init() {
-          console.log('init called');
-          await this.loadStats();
-        },
-        async loadStats(event) {
-          console.log('loadStats called', {
-            event,
-            form: this.$refs.filterForm,
-            formData: new FormData(this.$refs.filterForm)
-          });
+<script>
+  function dashboardStats() {
+    return {
+      stats: {
+        total_visits: 0,
+        active_users: 0,
+        completed_visits: 0,
+        pending_visits: 0
+      },
+      async init() {
+        console.log('init called');
+        await this.loadStats();
+      },
+      async loadStats(event) {
+        console.log('loadStats called', {
+          event,
+          form: this.$refs.filterForm,
+          formData: new FormData(this.$refs.filterForm)
+        });
 
-          try {
-            const formData = new FormData(this.$refs.filterForm);
-            const queryString = new URLSearchParams(formData).toString();
-            console.log('queryString:', queryString);
+        try {
+          const formData = new FormData(this.$refs.filterForm);
+          const queryString = new URLSearchParams(formData).toString();
+          console.log('queryString:', queryString);
 
-            const url = `{% url 'reports:dashboard_stats_api' %}?${queryString}`;
-            console.log('fetching from:', url);
+          const url = `{% url 'reports:dashboard_stats_api' %}?${queryString}`;
+          console.log('fetching from:', url);
 
-            const response = await fetch(url);
-            console.log('response:', response);
+          const response = await fetch(url);
+          console.log('response:', response);
 
-            if (!response.ok) throw new Error('Failed to load stats');
-            const data = await response.json();
-            console.log('received data:', data);
+          if (!response.ok) throw new Error('Failed to load stats');
+          const data = await response.json();
+          console.log('received data:', data);
 
-            this.stats = data;
-          } catch (error) {
-            console.error('Error loading dashboard stats:', error);
-          }
+          this.stats = data;
+        } catch (error) {
+          console.error('Error loading dashboard stats:', error);
         }
       }
     }
-  </script>
-  <script type="module">
-    window.addEventListener('DOMContentLoaded', () => {
-      mapboxgl.accessToken = "{{ mapbox_token }}";
+  }
+</script>
+<script type="module">
+  window.addEventListener('DOMContentLoaded', () => {
+    mapboxgl.accessToken = "{{ mapbox_token }}";
     const map = new mapboxgl.Map({
-        container: 'map',
-        // Choose from Mapbox's core styles, or make your own style with Mapbox Studio
-        style: 'mapbox://styles/mapbox/dark-v11',
-        center: [-103.5917, 40.6699],
-        zoom: 3
+      container: 'map',
+      // Choose from Mapbox's core styles, or make your own style with Mapbox Studio
+      style: 'mapbox://styles/mapbox/dark-v11',
+      center: [-103.5917, 40.6699],
+      zoom: 3
     });
 
     map.on('load', () => {
-        // Add a new source from our GeoJSON data and
-        // set the 'cluster' option to true. GL-JS will
-        // add the point_count property to your source data.
-        map.addSource('visits', {
-            type: 'geojson',
-            // Point to GeoJSON data from the backend.
-            data: '{% url "reports:visit_map_data" %}',
-            cluster: true,
-            clusterMaxZoom: 14, // Max zoom to cluster points on
-            clusterRadius: 50 // Radius of each cluster when clustering points (defaults to 50)
-        });
-
-        const clusterStep1 = 50;
-        const clusterStep2 = 100;
-
-        map.addLayer({
-            id: 'clusters',
-            type: 'circle',
-            source: 'visits',
-            filter: ['has', 'point_count'],
-            paint: {
-                // Use step expressions (https://docs.mapbox.com/style-spec/reference/expressions/#step)
-                // with three steps to implement three types of circles:
-                //   * Blue, 20px circles when point count is less than clusterStep1
-                //   * Yellow, 30px circles when point count is between clusterStep1 and clusterStep2
-                //   * Green, 40px circles when point count is greater than or equal to clusterStep2
-                'circle-color': [
-                    'step',
-                    ['get', 'point_count'],
-                    '#51bbd6',
-                    clusterStep1,
-                    '#f1f075',
-                    clusterStep2,
-                    '#00FF00'
-                ],
-                'circle-radius': [
-                    'step',
-                    ['get', 'point_count'],
-                    20,
-                    clusterStep1,
-                    30,
-                    clusterStep2,
-                    40
-                ]
-            }
-        });
-
-        map.addLayer({
-            id: 'cluster-count',
-            type: 'symbol',
-            source: 'visits',
-            filter: ['has', 'point_count'],
-            layout: {
-                'text-field': ['get', 'point_count_abbreviated'],
-                'text-font': ['DIN Offc Pro Medium', 'Arial Unicode MS Bold'],
-                'text-size': 12
-            }
-        });
+      // Add a new source from our GeoJSON data and
+      // set the 'cluster' option to true. GL-JS will
+      // add the point_count property to your source data.
+      map.addSource('visits', {
+        type: 'geojson',
+        // Point to GeoJSON data from the backend.
+        data: '{% url "reports:visit_map_data" %}',
+        cluster: true,
+        clusterMaxZoom: 14, // Max zoom to cluster points on
+        clusterRadius: 50 // Radius of each cluster when clustering points (defaults to 50)
+      });
+
+      const clusterStep1 = 50;
+      const clusterStep2 = 100;
+
+      map.addLayer({
+        id: 'clusters',
+        type: 'circle',
+        source: 'visits',
+        filter: ['has', 'point_count'],
+        paint: {
+          // Use step expressions (https://docs.mapbox.com/style-spec/reference/expressions/#step)
+          // with three steps to implement three types of circles:
+          //   * Blue, 20px circles when point count is less than clusterStep1
+          //   * Yellow, 30px circles when point count is between clusterStep1 and clusterStep2
+          //   * Green, 40px circles when point count is greater than or equal to clusterStep2
+          'circle-color': [
+            'step',
+            ['get', 'point_count'],
+            '#51bbd6',
+            clusterStep1,
+            '#f1f075',
+            clusterStep2,
+            '#00FF00'
+          ],
+          'circle-radius': [
+            'step',
+            ['get', 'point_count'],
+            20,
+            clusterStep1,
+            30,
+            clusterStep2,
+            40
+          ]
+        }
+      });
+
+      map.addLayer({
+        id: 'cluster-count',
+        type: 'symbol',
+        source: 'visits',
+        filter: ['has', 'point_count'],
+        layout: {
+          'text-field': ['get', 'point_count_abbreviated'],
+          'text-font': ['DIN Offc Pro Medium', 'Arial Unicode MS Bold'],
+          'text-size': 12
+        }
+      });
+
+      map.addLayer({
+        id: 'unclustered-point',
+        type: 'circle',
+        source: 'visits',
+        filter: ['!', ['has', 'point_count']],
+        paint: {
+          'circle-color': ['get', 'color'],
+          'circle-radius': 4,
+          'circle-stroke-width': 1,
+          'circle-stroke-color': '#fff'
+        }
+      });
 
-        map.addLayer({
-            id: 'unclustered-point',
-            type: 'circle',
-            source: 'visits',
-            filter: ['!', ['has', 'point_count']],
-            paint: {
-                'circle-color': ['get', 'color'],
-                'circle-radius': 4,
-                'circle-stroke-width': 1,
-                'circle-stroke-color': '#fff'
-            }
+      // inspect a cluster on click
+      map.on('click', 'clusters', (e) => {
+        const features = map.queryRenderedFeatures(e.point, {
+          layers: ['clusters']
         });
-
-        // inspect a cluster on click
-        map.on('click', 'clusters', (e) => {
-            const features = map.queryRenderedFeatures(e.point, {
-                layers: ['clusters']
+        const clusterId = features[0].properties.cluster_id;
+        map.getSource('visits').getClusterExpansionZoom(
+          clusterId,
+          (err, zoom) => {
+            if (err) return;
+
+            map.easeTo({
+              center: features[0].geometry.coordinates,
+              zoom: zoom
             });
-            const clusterId = features[0].properties.cluster_id;
-            map.getSource('visits').getClusterExpansionZoom(
-                clusterId,
-                (err, zoom) => {
-                    if (err) return;
-
-                    map.easeTo({
-                        center: features[0].geometry.coordinates,
-                        zoom: zoom
-                    });
-                }
-            );
-        });
-
-        // When a click event occurs on a feature in
-        // the unclustered-point layer, open a popup at
-        // the location of the feature, with
-        // description HTML from its properties.
-        map.on('click', 'unclustered-point', (e) => {
-            const coordinates = e.features[0].geometry.coordinates.slice();
-            const status = e.features[0].properties.status;
-            const rawDate = e.features[0].properties.visit_date;
-            const visitDate = new Date(rawDate).toLocaleDateString('en-US', { year: 'numeric', month: 'short', day: 'numeric' });
-
-
-            // Ensure that if the map is zoomed out such that
-            // multiple copies of the feature are visible, the
-            // popup appears over the copy being pointed to.
-            if (['mercator', 'equirectangular'].includes(map.getProjection().name)) {
-                while (Math.abs(e.lngLat.lng - coordinates[0]) > 180) {
-                    coordinates[0] += e.lngLat.lng > coordinates[0] ? 360 : -360;
-                }
-            }
-
-            new mapboxgl.Popup()
-                .setLngLat(coordinates)
-                .setHTML(
-                    `Visit Date: ${visitDate}<br>Status: ${status}`
-                )
-                .addTo(map);
-        });
+          }
+        );
+      });
+
+      // When a click event occurs on a feature in
+      // the unclustered-point layer, open a popup at
+      // the location of the feature, with
+      // description HTML from its properties.
+      map.on('click', 'unclustered-point', (e) => {
+        const coordinates = e.features[0].geometry.coordinates.slice();
+        const status = e.features[0].properties.status;
+        const rawDate = e.features[0].properties.visit_date;
+        const visitDate = new Date(rawDate).toLocaleDateString('en-US', { year: 'numeric', month: 'short', day: 'numeric' });
+
+
+        // Ensure that if the map is zoomed out such that
+        // multiple copies of the feature are visible, the
+        // popup appears over the copy being pointed to.
+        if (['mercator', 'equirectangular'].includes(map.getProjection().name)) {
+          while (Math.abs(e.lngLat.lng - coordinates[0]) > 180) {
+            coordinates[0] += e.lngLat.lng > coordinates[0] ? 360 : -360;
+          }
+        }
 
-        map.on('mouseenter', 'clusters', () => {
-            map.getCanvas().style.cursor = 'pointer';
-        });
-        map.on('mouseleave', 'clusters', () => {
-            map.getCanvas().style.cursor = '';
-        });
+        new mapboxgl.Popup()
+          .setLngLat(coordinates)
+          .setHTML(
+            `Visit Date: ${visitDate}<br>Status: ${status}`
+          )
+          .addTo(map);
+      });
+
+      map.on('mouseenter', 'clusters', () => {
+        map.getCanvas().style.cursor = 'pointer';
+      });
+      map.on('mouseleave', 'clusters', () => {
+        map.getCanvas().style.cursor = '';
+      });
     });
   });
-  </script>
+</script>
 {% endblock %}

From 2009b19113cbc38a6c131e689454cfe60519fd9f Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 12:27:16 +0200
Subject: [PATCH 057/165] fix html syntax

---
 commcare_connect/templates/reports/dashboard.html | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 04efd3a3..f4b66294 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -7,12 +7,12 @@
 <h2 class="my-2">Program Dashboard</h2>
 <div class="container" x-data="dashboardStats()" x-init="$nextTick(() => console.log('Alpine initialized'))">
   <div class="mb-4">
-    <form class="row mb-4 id=" filter-form" x-ref="filterForm" x-on:submit.prevent="loadStats()">
+    <div class="row mb-4" x-ref="filterForm">
       {% crispy filter.form %}
       <div class="col-12 text-end">
         <button type="button" class="btn btn-primary" @click="loadStats()">Update Dashboard</button>
       </div>
-    </form>
+    </div>
   </div>
   <div class="row">
     <div class="col-12 col-sm-6 col-md-3 mb-3">
@@ -80,14 +80,9 @@ <h3 class="my-2">Additional Information</h3>
         await this.loadStats();
       },
       async loadStats(event) {
-        console.log('loadStats called', {
-          event,
-          form: this.$refs.filterForm,
-          formData: new FormData(this.$refs.filterForm)
-        });
-
         try {
-          const formData = new FormData(this.$refs.filterForm);
+          const formElement = this.$refs.filterForm.querySelector('form');
+          const formData = new FormData(formElement);
           const queryString = new URLSearchParams(formData).toString();
           console.log('queryString:', queryString);
 

From cb4e471303734766d76524c2a961fa6a41bc2832 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 12:29:08 +0200
Subject: [PATCH 058/165] remove submit button entirely

---
 commcare_connect/templates/reports/dashboard.html | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index f4b66294..59e4ef7a 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -9,9 +9,6 @@ <h2 class="my-2">Program Dashboard</h2>
   <div class="mb-4">
     <div class="row mb-4" x-ref="filterForm">
       {% crispy filter.form %}
-      <div class="col-12 text-end">
-        <button type="button" class="btn btn-primary" @click="loadStats()">Update Dashboard</button>
-      </div>
     </div>
   </div>
   <div class="row">
@@ -78,6 +75,11 @@ <h3 class="my-2">Additional Information</h3>
       async init() {
         console.log('init called');
         await this.loadStats();
+
+        const formElement = this.$refs.filterForm.querySelector('form');
+        formElement.querySelectorAll('select, input').forEach(input => {
+          input.addEventListener('change', () => this.loadStats());
+        });
       },
       async loadStats(event) {
         try {

From 914f95e5d51defb9c3cc5bf573f639190003a4de Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 13:36:51 +0200
Subject: [PATCH 059/165] spinners!

---
 .../templates/reports/dashboard.html          | 36 ++++++++++++++++---
 1 file changed, 32 insertions(+), 4 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 59e4ef7a..0050bfea 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -15,7 +15,13 @@ <h2 class="my-2">Program Dashboard</h2>
     <div class="col-12 col-sm-6 col-md-3 mb-3">
       <div class="card">
         <div class="card-body">
-          <div class="display-4 fw-bold" x-text="stats.total_visits || 0">0</div>
+          <div class="fw-bold">
+            <span x-show="!isLoading" x-text="stats.total_visits || 0" class="display-4">0</span>
+            <span x-show="isLoading">
+              <span class="spinner-border text-secondary opacity-50" role="status"></span>
+              <span class="display-4">&nbsp;</span>
+            </span>
+          </div>
           <div class="small text-muted mt-1">Total Visits</div>
         </div>
       </div>
@@ -23,7 +29,13 @@ <h2 class="my-2">Program Dashboard</h2>
     <div class="col-12 col-sm-6 col-md-3 mb-3">
       <div class="card">
         <div class="card-body">
-          <div class="display-4 fw-bold" x-text="stats.active_users || 0">0</div>
+          <div class="fw-bold">
+            <span x-show="!isLoading" x-text="stats.active_users || 0" class="display-4">0</span>
+            <span x-show="isLoading">
+              <span class="spinner-border text-secondary opacity-50" role="status"></span>
+              <span class="display-4">&nbsp;</span>
+            </span>
+          </div>
           <div class="small text-muted mt-1">Active Users</div>
         </div>
       </div>
@@ -31,7 +43,13 @@ <h2 class="my-2">Program Dashboard</h2>
     <div class="col-12 col-sm-6 col-md-3 mb-3">
       <div class="card">
         <div class="card-body">
-          <div class="display-4 fw-bold" x-text="stats.completed_visits || 0">0</div>
+          <div class="fw-bold">
+            <span x-show="!isLoading" x-text="stats.completed_visits || 0" class="display-4">0</span>
+            <span x-show="isLoading">
+              <span class="spinner-border text-secondary opacity-50" role="status"></span>
+              <span class="display-4">&nbsp;</span>
+            </span>
+          </div>
           <div class="small text-muted mt-1">Completed Visits</div>
         </div>
       </div>
@@ -39,7 +57,13 @@ <h2 class="my-2">Program Dashboard</h2>
     <div class="col-12 col-sm-6 col-md-3 mb-3">
       <div class="card">
         <div class="card-body">
-          <div class="display-4 fw-bold" x-text="stats.pending_visits || 0">0</div>
+          <div class="fw-bold">
+            <span x-show="!isLoading" x-text="stats.pending_visits || 0" class="display-4">0</span>
+            <span x-show="isLoading">
+              <span class="spinner-border text-secondary opacity-50" role="status"></span>
+              <span class="display-4">&nbsp;</span>
+            </span>
+          </div>
           <div class="small text-muted mt-1">Pending Visits</div>
         </div>
       </div>
@@ -72,6 +96,7 @@ <h3 class="my-2">Additional Information</h3>
         completed_visits: 0,
         pending_visits: 0
       },
+      isLoading: false,
       async init() {
         console.log('init called');
         await this.loadStats();
@@ -83,6 +108,7 @@ <h3 class="my-2">Additional Information</h3>
       },
       async loadStats(event) {
         try {
+          this.isLoading = true;
           const formElement = this.$refs.filterForm.querySelector('form');
           const formData = new FormData(formElement);
           const queryString = new URLSearchParams(formData).toString();
@@ -101,6 +127,8 @@ <h3 class="my-2">Additional Information</h3>
           this.stats = data;
         } catch (error) {
           console.error('Error loading dashboard stats:', error);
+        } finally {
+          this.isLoading = false;
         }
       }
     }

From aa9da02536fc4f3cc9f4b46574f118ed5b793b55 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 13:50:42 +0200
Subject: [PATCH 060/165] improve filters

---
 commcare_connect/reports/views.py | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index 75d490b5..2ed9f971 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -157,16 +157,28 @@ def _get_table_data_for_quarter(quarter, delivery_type, group_by_delivery_type=F
 class DashboardFilters(django_filters.FilterSet):
     opportunity__delivery_type = django_filters.ModelChoiceFilter(
         queryset=DeliveryType.objects.all(),
+        label="Program",
         empty_label="All Programs",
         required=False,
     )
     opportunity__organization = django_filters.ModelChoiceFilter(
         queryset=Organization.objects.all(),
+        label="Organization",
         empty_label="All Organizations",
         required=False,
     )
-    visit_date = django_filters.DateFilter(
+    from_date = django_filters.DateTimeFilter(
         widget=forms.DateInput(attrs={"type": "date"}),
+        field_name="visit_date",
+        lookup_expr="gt",
+        label="From Date",
+        required=False,
+    )
+    to_date = django_filters.DateTimeFilter(
+        widget=forms.DateInput(attrs={"type": "date"}),
+        field_name="visit_date",
+        lookup_expr="lte",
+        label="To Date",
         required=False,
     )
 
@@ -176,15 +188,16 @@ def __init__(self, *args, **kwargs):
         self.form.helper.form_class = "form-inline"
         self.form.helper.layout = Layout(
             Row(
-                Column("opportunity__delivery_type", css_class="col-md-4"),
-                Column("opportunity__organization", css_class="col-md-4"),
-                Column("visit_date", css_class="col-md-4"),
+                Column("opportunity__delivery_type", css_class="col-md-3"),
+                Column("opportunity__organization", css_class="col-md-3"),
+                Column("from_date", css_class="col-md-3"),
+                Column("to_date", css_class="col-md-3"),
             )
         )
 
     class Meta:
         model = UserVisit
-        fields = ["opportunity__delivery_type", "opportunity__organization", "visit_date"]
+        fields = ["opportunity__delivery_type", "opportunity__organization", "from_date", "to_date"]
 
 
 @login_required
@@ -367,6 +380,7 @@ def dashboard_stats_api(request):
     # Use the filtered queryset to calculate stats
     queryset = UserVisit.objects.all()
     if filterset.is_valid():
+        print(filterset.form.cleaned_data)
         queryset = filterset.filter_queryset(queryset)
 
     # Example stats calculation (adjust based on your needs)

From bad9331be97a21f522c24af9326fcb09fc503ae0 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 13:50:47 +0200
Subject: [PATCH 061/165] tweak order

---
 commcare_connect/templates/reports/dashboard.html | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 0050bfea..4ec15842 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -16,13 +16,13 @@ <h2 class="my-2">Program Dashboard</h2>
       <div class="card">
         <div class="card-body">
           <div class="fw-bold">
-            <span x-show="!isLoading" x-text="stats.total_visits || 0" class="display-4">0</span>
+            <span x-show="!isLoading" x-text="stats.active_users || 0" class="display-4">0</span>
             <span x-show="isLoading">
               <span class="spinner-border text-secondary opacity-50" role="status"></span>
               <span class="display-4">&nbsp;</span>
             </span>
           </div>
-          <div class="small text-muted mt-1">Total Visits</div>
+          <div class="small text-muted mt-1">Active Front-Line Workers</div>
         </div>
       </div>
     </div>
@@ -30,13 +30,13 @@ <h2 class="my-2">Program Dashboard</h2>
       <div class="card">
         <div class="card-body">
           <div class="fw-bold">
-            <span x-show="!isLoading" x-text="stats.active_users || 0" class="display-4">0</span>
+            <span x-show="!isLoading" x-text="stats.total_visits || 0" class="display-4">0</span>
             <span x-show="isLoading">
               <span class="spinner-border text-secondary opacity-50" role="status"></span>
               <span class="display-4">&nbsp;</span>
             </span>
           </div>
-          <div class="small text-muted mt-1">Active Users</div>
+          <div class="small text-muted mt-1">Total Visits</div>
         </div>
       </div>
     </div>

From d369f2f4607655964e95bdc7cc517cfeafa38116 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 13:52:11 +0200
Subject: [PATCH 062/165] fix other filters

---
 commcare_connect/reports/views.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index 2ed9f971..0f7cd7f8 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -155,14 +155,16 @@ def _get_table_data_for_quarter(quarter, delivery_type, group_by_delivery_type=F
 
 
 class DashboardFilters(django_filters.FilterSet):
-    opportunity__delivery_type = django_filters.ModelChoiceFilter(
+    program = django_filters.ModelChoiceFilter(
         queryset=DeliveryType.objects.all(),
+        field_name="opportunity__delivery_type",
         label="Program",
         empty_label="All Programs",
         required=False,
     )
-    opportunity__organization = django_filters.ModelChoiceFilter(
+    organization = django_filters.ModelChoiceFilter(
         queryset=Organization.objects.all(),
+        field_name="opportunity__organization",
         label="Organization",
         empty_label="All Organizations",
         required=False,
@@ -188,8 +190,8 @@ def __init__(self, *args, **kwargs):
         self.form.helper.form_class = "form-inline"
         self.form.helper.layout = Layout(
             Row(
-                Column("opportunity__delivery_type", css_class="col-md-3"),
-                Column("opportunity__organization", css_class="col-md-3"),
+                Column("program", css_class="col-md-3"),
+                Column("organization", css_class="col-md-3"),
                 Column("from_date", css_class="col-md-3"),
                 Column("to_date", css_class="col-md-3"),
             )
@@ -197,7 +199,7 @@ def __init__(self, *args, **kwargs):
 
     class Meta:
         model = UserVisit
-        fields = ["opportunity__delivery_type", "opportunity__organization", "from_date", "to_date"]
+        fields = ["program", "organization", "from_date", "to_date"]
 
 
 @login_required

From 75f7278e46ec97e536b3249b34382e3db538e78d Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 13:53:45 +0200
Subject: [PATCH 063/165] remove console statements

---
 commcare_connect/templates/reports/dashboard.html | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 4ec15842..7b3d63a1 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -5,7 +5,7 @@
 {% block title %}Admin Dashboard{% endblock %}
 {% block content %}
 <h2 class="my-2">Program Dashboard</h2>
-<div class="container" x-data="dashboardStats()" x-init="$nextTick(() => console.log('Alpine initialized'))">
+<div class="container" x-data="dashboardStats()" >
   <div class="mb-4">
     <div class="row mb-4" x-ref="filterForm">
       {% crispy filter.form %}
@@ -98,7 +98,6 @@ <h3 class="my-2">Additional Information</h3>
       },
       isLoading: false,
       async init() {
-        console.log('init called');
         await this.loadStats();
 
         const formElement = this.$refs.filterForm.querySelector('form');
@@ -112,18 +111,10 @@ <h3 class="my-2">Additional Information</h3>
           const formElement = this.$refs.filterForm.querySelector('form');
           const formData = new FormData(formElement);
           const queryString = new URLSearchParams(formData).toString();
-          console.log('queryString:', queryString);
-
           const url = `{% url 'reports:dashboard_stats_api' %}?${queryString}`;
-          console.log('fetching from:', url);
-
           const response = await fetch(url);
-          console.log('response:', response);
-
           if (!response.ok) throw new Error('Failed to load stats');
           const data = await response.json();
-          console.log('received data:', data);
-
           this.stats = data;
         } catch (error) {
           console.error('Error loading dashboard stats:', error);

From 24505a3265c34922c0399659d9346b7b358e3bcb Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 14:10:18 +0200
Subject: [PATCH 064/165] wip: ai generated query to mimic the sql that was
 previously running

---
 commcare_connect/reports/queries.py | 58 +++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100644 commcare_connect/reports/queries.py

diff --git a/commcare_connect/reports/queries.py b/commcare_connect/reports/queries.py
new file mode 100644
index 00000000..5afbf2a9
--- /dev/null
+++ b/commcare_connect/reports/queries.py
@@ -0,0 +1,58 @@
+from django.db.models import Case, DateTimeField, ExpressionWrapper, F, FloatField, Value, When
+from django.db.models.fields.json import KeyTextTransform
+from django.db.models.functions import Cast, Extract
+
+from commcare_connect.opportunity.models import UserVisit
+
+
+def get_visit_map_queryset_base():
+    return (
+        UserVisit.objects.annotate(
+            username_connectid=F("form_json__metadata__username"),
+            deliver_unit_name=F("deliver_unit__name"),
+            days_since_opp_start=ExpressionWrapper(
+                Extract(F("visit_date") - F("opportunity__start_date"), "day"), output_field=FloatField()
+            ),
+            timestart_str=KeyTextTransform("timeStart", KeyTextTransform("metadata", F("form_json"))),
+            timeend_str=KeyTextTransform("timeEnd", KeyTextTransform("metadata", F("form_json"))),
+            visit_duration=ExpressionWrapper(
+                Extract(Cast("timeend_str", DateTimeField()) - Cast("timestart_str", DateTimeField()), "epoch") / 60,
+                output_field=FloatField(),
+            ),
+            gps_location_lat=Case(
+                When(
+                    form_json__metadata__location__isnull=False,
+                    then=ExpressionWrapper(F("form_json__metadata__location__0"), output_field=FloatField()),
+                ),
+                default=Value(None),
+                output_field=FloatField(),
+            ),
+            gps_location_long=Case(
+                When(
+                    form_json__metadata__location__isnull=False,
+                    then=ExpressionWrapper(F("form_json__metadata__location__1"), output_field=FloatField()),
+                ),
+                default=Value(None),
+                output_field=FloatField(),
+            ),
+        )
+        .select_related("deliver_unit", "opportunity")
+        .values(
+            "opportunity_id",
+            "xform_id",
+            "visit_date",
+            "username_connectid",
+            "deliver_unit_name",
+            "days_since_opp_start",
+            "entity_id",
+            "status",
+            "flagged",
+            "flag_reason",
+            "reason",
+            "timestart_str",
+            "timeend_str",
+            "visit_duration",
+            "gps_location_lat",
+            "gps_location_long",
+        )
+    )

From b13744476be643553678c0905c66330f92388d34 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 14:29:00 +0200
Subject: [PATCH 065/165] simplify lat/lon lookups

---
 commcare_connect/reports/queries.py | 22 +++-------------------
 1 file changed, 3 insertions(+), 19 deletions(-)

diff --git a/commcare_connect/reports/queries.py b/commcare_connect/reports/queries.py
index 5afbf2a9..ac9e83cf 100644
--- a/commcare_connect/reports/queries.py
+++ b/commcare_connect/reports/queries.py
@@ -1,4 +1,4 @@
-from django.db.models import Case, DateTimeField, ExpressionWrapper, F, FloatField, Value, When
+from django.db.models import DateTimeField, ExpressionWrapper, F, FloatField
 from django.db.models.fields.json import KeyTextTransform
 from django.db.models.functions import Cast, Extract
 
@@ -19,22 +19,7 @@ def get_visit_map_queryset_base():
                 Extract(Cast("timeend_str", DateTimeField()) - Cast("timestart_str", DateTimeField()), "epoch") / 60,
                 output_field=FloatField(),
             ),
-            gps_location_lat=Case(
-                When(
-                    form_json__metadata__location__isnull=False,
-                    then=ExpressionWrapper(F("form_json__metadata__location__0"), output_field=FloatField()),
-                ),
-                default=Value(None),
-                output_field=FloatField(),
-            ),
-            gps_location_long=Case(
-                When(
-                    form_json__metadata__location__isnull=False,
-                    then=ExpressionWrapper(F("form_json__metadata__location__1"), output_field=FloatField()),
-                ),
-                default=Value(None),
-                output_field=FloatField(),
-            ),
+            location_str=KeyTextTransform("location", KeyTextTransform("metadata", F("form_json"))),
         )
         .select_related("deliver_unit", "opportunity")
         .values(
@@ -52,7 +37,6 @@ def get_visit_map_queryset_base():
             "timestart_str",
             "timeend_str",
             "visit_duration",
-            "gps_location_lat",
-            "gps_location_long",
+            "location_str",
         )
     )

From c128beb11a50a4daf4d10fd13b917787014ce18e Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 14:32:00 +0200
Subject: [PATCH 066/165] pass filters to the map

---
 commcare_connect/reports/queries.py           |  6 +--
 commcare_connect/reports/views.py             | 42 +++++++++++--------
 .../templates/reports/dashboard.html          | 39 ++++++++++++-----
 3 files changed, 54 insertions(+), 33 deletions(-)

diff --git a/commcare_connect/reports/queries.py b/commcare_connect/reports/queries.py
index ac9e83cf..4b7d2d14 100644
--- a/commcare_connect/reports/queries.py
+++ b/commcare_connect/reports/queries.py
@@ -2,12 +2,10 @@
 from django.db.models.fields.json import KeyTextTransform
 from django.db.models.functions import Cast, Extract
 
-from commcare_connect.opportunity.models import UserVisit
 
-
-def get_visit_map_queryset_base():
+def get_visit_map_queryset(base_queryset):
     return (
-        UserVisit.objects.annotate(
+        base_queryset.annotate(
             username_connectid=F("form_json__metadata__username"),
             deliver_unit_name=F("deliver_unit__name"),
             days_since_opp_start=ExpressionWrapper(
diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index 0f7cd7f8..357fccd8 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -8,7 +8,6 @@
 from django.conf import settings
 from django.contrib.auth.decorators import login_required, user_passes_test
 from django.contrib.auth.mixins import LoginRequiredMixin, UserPassesTestMixin
-from django.db import connection
 from django.db.models import Max, Q, Sum
 from django.http import JsonResponse
 from django.shortcuts import render
@@ -20,6 +19,7 @@
 from commcare_connect.cache import quickcache
 from commcare_connect.opportunity.models import CompletedWork, CompletedWorkStatus, DeliveryType, Payment, UserVisit
 from commcare_connect.organization.models import Organization
+from commcare_connect.reports.queries import get_visit_map_queryset
 
 from .tables import AdminReportTable
 
@@ -221,20 +221,19 @@ def program_dashboard_report(request):
 @user_passes_test(lambda user: user.is_superuser)
 @require_GET
 def visit_map_data(request):
-    with connection.cursor() as cursor:
-        # Read the SQL file
-        with open("commcare_connect/reports/sql/visit_map.sql") as sql_file:
-            sql_query = sql_file.read()
+    filterset = DashboardFilters(request.GET)
+
+    # Use the filtered queryset to calculate stats
 
-        # Execute the query
-        cursor.execute(sql_query)
+    queryset = UserVisit.objects.all()
+    if filterset.is_valid():
+        print(filterset.form.cleaned_data)
+        queryset = filterset.filter_queryset(queryset)
 
-        # Fetch all results
-        columns = [col[0] for col in cursor.description]
-        results = [dict(zip(columns, row)) for row in cursor.fetchall()]
+    queryset = get_visit_map_queryset(queryset)
 
     # Convert to GeoJSON
-    geojson = _results_to_geojson(results)
+    geojson = _results_to_geojson(queryset)
 
     # Return the GeoJSON as JSON response
     return JsonResponse(geojson, safe=False)
@@ -246,14 +245,21 @@ def _results_to_geojson(results):
         "approved": "#00FF00",
         "rejected": "#FF0000",
     }
-    for result in results:
+    print("calling _results_to_geojson")
+    for i, result in enumerate(results.all()):
+        location_str = result.get("location_str")
         # Check if both latitude and longitude are not None and can be converted to float
-        if result.get("gps_location_long") and result.get("gps_location_lat"):
-            try:
-                longitude = float(result["gps_location_long"])
-                latitude = float(result["gps_location_lat"])
-            except ValueError:
-                # Skip this result if conversion to float fails
+        if location_str:
+            split_location = location_str.split(" ")
+            if len(split_location) >= 2:
+                try:
+                    longitude = float(split_location[1])
+                    latitude = float(split_location[0])
+                except ValueError:
+                    # Skip this result if conversion to float fails
+                    continue
+            else:
+                # Or if the location string is not in the expected format
                 continue
 
             feature = {
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 7b3d63a1..e26744e9 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -7,7 +7,7 @@
 <h2 class="my-2">Program Dashboard</h2>
 <div class="container" x-data="dashboardStats()" >
   <div class="mb-4">
-    <div class="row mb-4" x-ref="filterForm">
+    <div class="row mb-4" x-ref="filterForm" id="filterForm">
       {% crispy filter.form %}
     </div>
   </div>
@@ -99,10 +99,12 @@ <h3 class="my-2">Additional Information</h3>
       isLoading: false,
       async init() {
         await this.loadStats();
-
         const formElement = this.$refs.filterForm.querySelector('form');
         formElement.querySelectorAll('select, input').forEach(input => {
-          input.addEventListener('change', () => this.loadStats());
+          input.addEventListener('change', () => {
+            this.loadStats();
+            window.refreshMapData();
+          });
         });
       },
       async loadStats(event) {
@@ -126,9 +128,23 @@ <h3 class="my-2">Additional Information</h3>
   }
 </script>
 <script type="module">
+  let map;
+
+  // Add this function to handle map data refresh
+  window.refreshMapData = () => {
+    if (!map) return;
+
+    const formElement = document.querySelector('#filterForm form');
+    const formData = new FormData(formElement);
+    const queryString = new URLSearchParams(formData).toString();
+
+    // Update the source data with new filters
+    map.getSource('visits').setData(`{% url "reports:visit_map_data" %}?${queryString}`);
+  };
+
   window.addEventListener('DOMContentLoaded', () => {
     mapboxgl.accessToken = "{{ mapbox_token }}";
-    const map = new mapboxgl.Map({
+    map = new mapboxgl.Map({
       container: 'map',
       // Choose from Mapbox's core styles, or make your own style with Mapbox Studio
       style: 'mapbox://styles/mapbox/dark-v11',
@@ -137,16 +153,17 @@ <h3 class="my-2">Additional Information</h3>
     });
 
     map.on('load', () => {
-      // Add a new source from our GeoJSON data and
-      // set the 'cluster' option to true. GL-JS will
-      // add the point_count property to your source data.
+      // Modify the source configuration to include initial filters
+      const formElement = document.querySelector('#filterForm form');
+      const formData = new FormData(formElement);
+      const queryString = new URLSearchParams(formData).toString();
+
       map.addSource('visits', {
         type: 'geojson',
-        // Point to GeoJSON data from the backend.
-        data: '{% url "reports:visit_map_data" %}',
+        data: `{% url "reports:visit_map_data" %}?${queryString}`,
         cluster: true,
-        clusterMaxZoom: 14, // Max zoom to cluster points on
-        clusterRadius: 50 // Radius of each cluster when clustering points (defaults to 50)
+        clusterMaxZoom: 14,
+        clusterRadius: 50
       });
 
       const clusterStep1 = 50;

From 6518876143555347b27b3e8800289187f3051a64 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 14:36:33 +0200
Subject: [PATCH 067/165] completed --> verified

---
 commcare_connect/reports/views.py                 | 8 ++++----
 commcare_connect/templates/reports/dashboard.html | 6 +++---
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index 357fccd8..fb1e0279 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -392,16 +392,16 @@ def dashboard_stats_api(request):
         queryset = filterset.filter_queryset(queryset)
 
     # Example stats calculation (adjust based on your needs)
-    total_visits = queryset.count()
-    completed_visits = queryset.filter(status=CompletedWorkStatus.approved).count()
-    pending_visits = total_visits - completed_visits
     active_users = queryset.values("opportunity_access__user").distinct().count()
+    total_visits = queryset.count()
+    verified_visits = queryset.filter(status=CompletedWorkStatus.approved).count()
+    pending_visits = total_visits - verified_visits
 
     return JsonResponse(
         {
             "total_visits": total_visits,
             "active_users": active_users,
-            "completed_visits": completed_visits,
+            "verified_visits": verified_visits,
             "pending_visits": pending_visits,
         }
     )
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index e26744e9..d7b6ca5c 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -44,13 +44,13 @@ <h2 class="my-2">Program Dashboard</h2>
       <div class="card">
         <div class="card-body">
           <div class="fw-bold">
-            <span x-show="!isLoading" x-text="stats.completed_visits || 0" class="display-4">0</span>
+            <span x-show="!isLoading" x-text="stats.verified_visits || 0" class="display-4">0</span>
             <span x-show="isLoading">
               <span class="spinner-border text-secondary opacity-50" role="status"></span>
               <span class="display-4">&nbsp;</span>
             </span>
           </div>
-          <div class="small text-muted mt-1">Completed Visits</div>
+          <div class="small text-muted mt-1">Verified Visits</div>
         </div>
       </div>
     </div>
@@ -93,7 +93,7 @@ <h3 class="my-2">Additional Information</h3>
       stats: {
         total_visits: 0,
         active_users: 0,
-        completed_visits: 0,
+        verified_visits: 0,
         pending_visits: 0
       },
       isLoading: false,

From 2c7ed1245035c16f78bd0c50fe8768e79e8607c8 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 14:39:58 +0200
Subject: [PATCH 068/165] switch to percent

---
 commcare_connect/reports/views.py                 | 4 ++--
 commcare_connect/templates/reports/dashboard.html | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index fb1e0279..c6871efa 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -395,13 +395,13 @@ def dashboard_stats_api(request):
     active_users = queryset.values("opportunity_access__user").distinct().count()
     total_visits = queryset.count()
     verified_visits = queryset.filter(status=CompletedWorkStatus.approved).count()
-    pending_visits = total_visits - verified_visits
+    percent_verified = round(float(verified_visits / total_visits) * 100, 1) if total_visits > 0 else 0
 
     return JsonResponse(
         {
             "total_visits": total_visits,
             "active_users": active_users,
             "verified_visits": verified_visits,
-            "pending_visits": pending_visits,
+            "percent_verified": f"{percent_verified:.1f}%",
         }
     )
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index d7b6ca5c..12c459c0 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -22,7 +22,7 @@ <h2 class="my-2">Program Dashboard</h2>
               <span class="display-4">&nbsp;</span>
             </span>
           </div>
-          <div class="small text-muted mt-1">Active Front-Line Workers</div>
+          <div class="small text-muted mt-1">Active FLWs</div>
         </div>
       </div>
     </div>
@@ -58,13 +58,13 @@ <h2 class="my-2">Program Dashboard</h2>
       <div class="card">
         <div class="card-body">
           <div class="fw-bold">
-            <span x-show="!isLoading" x-text="stats.pending_visits || 0" class="display-4">0</span>
+            <span x-show="!isLoading" x-text="stats.percent_verified || '0.0%'" class="display-4">0</span>
             <span x-show="isLoading">
               <span class="spinner-border text-secondary opacity-50" role="status"></span>
               <span class="display-4">&nbsp;</span>
             </span>
           </div>
-          <div class="small text-muted mt-1">Pending Visits</div>
+          <div class="small text-muted mt-1">Percent Verified</div>
         </div>
       </div>
     </div>
@@ -94,7 +94,7 @@ <h3 class="my-2">Additional Information</h3>
         total_visits: 0,
         active_users: 0,
         verified_visits: 0,
-        pending_visits: 0
+        percent_verified: "0.0%"
       },
       isLoading: false,
       async init() {

From 1cea238e2168736c0c50df094158aa0531d83302 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 14:50:24 +0200
Subject: [PATCH 069/165] add default dates (last 90 days)

---
 commcare_connect/reports/views.py             | 22 +++++++++++++++----
 .../templates/reports/dashboard.html          |  3 +++
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index c6871efa..c91e8789 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -1,4 +1,4 @@
-from datetime import date, datetime
+from datetime import date, datetime, timedelta
 
 import django_filters
 import django_tables2 as tables
@@ -197,6 +197,23 @@ def __init__(self, *args, **kwargs):
             )
         )
 
+        # Set default values if no data is provided
+        if not self.data:
+            # Create a mutable copy of the QueryDict
+            self.data = self.data.copy() if self.data else {}
+
+            # Set default dates
+            today = date.today()
+            default_from = today - timedelta(days=90)
+
+            # Set the default values
+            self.data["to_date"] = today.strftime("%Y-%m-%d")
+            self.data["from_date"] = default_from.strftime("%Y-%m-%d")
+
+            # Force the form to bind with the default data
+            self.form.is_bound = True
+            self.form.data = self.data
+
     class Meta:
         model = UserVisit
         fields = ["program", "organization", "from_date", "to_date"]
@@ -227,7 +244,6 @@ def visit_map_data(request):
 
     queryset = UserVisit.objects.all()
     if filterset.is_valid():
-        print(filterset.form.cleaned_data)
         queryset = filterset.filter_queryset(queryset)
 
     queryset = get_visit_map_queryset(queryset)
@@ -245,7 +261,6 @@ def _results_to_geojson(results):
         "approved": "#00FF00",
         "rejected": "#FF0000",
     }
-    print("calling _results_to_geojson")
     for i, result in enumerate(results.all()):
         location_str = result.get("location_str")
         # Check if both latitude and longitude are not None and can be converted to float
@@ -388,7 +403,6 @@ def dashboard_stats_api(request):
     # Use the filtered queryset to calculate stats
     queryset = UserVisit.objects.all()
     if filterset.is_valid():
-        print(filterset.form.cleaned_data)
         queryset = filterset.filter_queryset(queryset)
 
     # Example stats calculation (adjust based on your needs)
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 12c459c0..f3117dba 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -101,6 +101,9 @@ <h3 class="my-2">Additional Information</h3>
         await this.loadStats();
         const formElement = this.$refs.filterForm.querySelector('form');
         formElement.querySelectorAll('select, input').forEach(input => {
+          if (input.value) {
+            input.dispatchEvent(new Event('change'));
+          }
           input.addEventListener('change', () => {
             this.loadStats();
             window.refreshMapData();

From 6bf9b0faeff7ec3a41d5f800c7f17738243bd61d Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 14:51:19 +0200
Subject: [PATCH 070/165] remove additional column for now

---
 commcare_connect/templates/reports/dashboard.html | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index f3117dba..e1973c7d 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -71,18 +71,10 @@ <h2 class="my-2">Program Dashboard</h2>
   </div>
 </div>
 <div class="row">
-  <div class="col-12 col-lg-8">
+  <div class="col-12">
     <h3 class="my-2">Service Delivery Map</h3>
     <div id="map" class="mt-3 mb-2 rounded border" style="height: 900px;"></div>
   </div>
-  <div class="col-12 col-lg-4">
-    <h3 class="my-2">Additional Information</h3>
-    <div class="card">
-      <div class="card-body">
-        <p class="text-muted">Placeholder content</p>
-      </div>
-    </div>
-  </div>
 </div>
 {% endblock content %}
 {% block inline_javascript %}

From 456c515406552df629ce229ac7faf19fc1a2acdd Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 15:02:35 +0200
Subject: [PATCH 071/165] fix tests

---
 .../reports/tests/test_reports.py             | 37 +++++++++++++------
 1 file changed, 25 insertions(+), 12 deletions(-)

diff --git a/commcare_connect/reports/tests/test_reports.py b/commcare_connect/reports/tests/test_reports.py
index 25710f41..f85ae168 100644
--- a/commcare_connect/reports/tests/test_reports.py
+++ b/commcare_connect/reports/tests/test_reports.py
@@ -65,19 +65,32 @@ def test_delivery_stats(opportunity: Opportunity):
 
 
 def test_results_to_geojson():
+    class MockQuerySet:
+        def __init__(self, results):
+            self.results = results
+
+        def all(self):
+            return self.results
+
     # Test input
-    results = [
-        {"gps_location_long": "10.123", "gps_location_lat": "20.456", "status": "approved", "other_field": "value1"},
-        {"gps_location_long": "30.789", "gps_location_lat": "40.012", "status": "rejected", "other_field": "value2"},
-        {"gps_location_long": "invalid", "gps_location_lat": "50.678", "status": "unknown", "other_field": "value3"},
-        {"status": "approved", "other_field": "value4"},  # Case where lat/lon are not present
-        {  # Case where lat/lon are null
-            "gps_location_long": None,
-            "gps_location_lat": None,
-            "status": "rejected",
-            "other_field": "value5",
-        },
-    ]
+    results = MockQuerySet(
+        [
+            {"location_str": "20.456 10.123 0 0", "status": "approved", "other_field": "value1"},
+            {"location_str": "40.012 30.789", "status": "rejected", "other_field": "value2"},
+            {"location_str": "invalid location", "status": "unknown", "other_field": "value3"},
+            {"location_str": "bad location", "status": "unknown", "other_field": "value4"},
+            {
+                "location_str": None,
+                "status": "approved",
+                "other_field": "value5",
+            },  # Case where lat/lon are not present
+            {  # Case where lat/lon are null
+                "location_str": None,
+                "status": "rejected",
+                "other_field": "value5",
+            },
+        ]
+    )
 
     # Call the function
     geojson = _results_to_geojson(results)

From 437b55be85fd6a7a9cb6f43f486e512a9d1df1ac Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Fri, 1 Nov 2024 11:41:26 +0200
Subject: [PATCH 072/165] document the staging deployment setup

---
 README.md | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/README.md b/README.md
index a2450d14..6e32ed21 100644
--- a/README.md
+++ b/README.md
@@ -146,6 +146,19 @@ For details on how this actions is configured see:
 - https://aws.amazon.com/blogs/security/use-iam-roles-to-connect-github-actions-to-actions-in-aws/
 - https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-amazon-web-services
 
+### Deploying to the staging environment
+
+The project has a staging environment at [https://connect-staging.dimagi.com/](https://connect-staging.dimagi.com/),
+which is connected to the staging environment of CommCare HQ at
+[https://staging.commcarehq.org/](https://staging.commcarehq.org/).
+
+By convention, the `pkv/staging` branch is used for changes that are on the staging environment.
+To put your own changes on the staging environment, you can create merge your own branch into
+`pkv/staging` and then push it to GitHub.
+
+After that, you can deploy to the staging environment by manually running the `deploy`
+[workflow from here](https://github.com/dimagi/commcare-connect/actions/workflows/deploy.yml).
+
 ### Custom Bootstrap Compilation
 
 The generated CSS is set up with automatic Bootstrap recompilation with variables of your choice.

From ccbe6da5803c2cb6cd6c80192b7fe01de44af4fd Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Fri, 1 Nov 2024 15:56:15 +0200
Subject: [PATCH 073/165] add more data to the map properties

---
 commcare_connect/reports/queries.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/commcare_connect/reports/queries.py b/commcare_connect/reports/queries.py
index 4b7d2d14..508c9341 100644
--- a/commcare_connect/reports/queries.py
+++ b/commcare_connect/reports/queries.py
@@ -19,9 +19,13 @@ def get_visit_map_queryset(base_queryset):
             ),
             location_str=KeyTextTransform("location", KeyTextTransform("metadata", F("form_json"))),
         )
-        .select_related("deliver_unit", "opportunity")
+        .select_related("deliver_unit", "opportunity", "opportunity__delivery_type", "opportunity__organization")
         .values(
             "opportunity_id",
+            "opportunity__delivery_type__name",
+            "opportunity__delivery_type__slug",
+            "opportunity__organization__slug",
+            "opportunity__organization__name",
             "xform_id",
             "visit_date",
             "username_connectid",

From 66f89762909faf4f1ff9b00e36b1599811dcff02 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Mon, 4 Nov 2024 14:42:27 +0530
Subject: [PATCH 074/165] Add start and end date checks to form processor

---
 commcare_connect/form_receiver/processor.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/commcare_connect/form_receiver/processor.py b/commcare_connect/form_receiver/processor.py
index 641213c5..2872e81e 100644
--- a/commcare_connect/form_receiver/processor.py
+++ b/commcare_connect/form_receiver/processor.py
@@ -245,6 +245,7 @@ def process_deliver_unit(user, xform: XForm, app: CommCareApp, opportunity: Oppo
     claim = OpportunityClaim.objects.get(opportunity_access=access)
     entity_id = deliver_unit_block.get("entity_id")
     entity_name = deliver_unit_block.get("entity_name")
+    payment_unit = deliver_unit.payment_unit
     user_visit = UserVisit(
         opportunity=opportunity,
         user=user,
@@ -260,26 +261,25 @@ def process_deliver_unit(user, xform: XForm, app: CommCareApp, opportunity: Oppo
         location=xform.metadata.location,
     )
     completed_work_needs_save = False
-    if opportunity.start_date > datetime.date.today():
+    today = datetime.date.today()
+    if opportunity.start_date > today or (payment_unit.start_date and payment_unit.start_date > today):
         completed_work = None
         user_visit.status = VisitValidationStatus.trial
     else:
         completed_work, _ = CompletedWork.objects.get_or_create(
             opportunity_access=access,
             entity_id=entity_id,
-            payment_unit=deliver_unit.payment_unit,
+            payment_unit=payment_unit,
             defaults={
                 "entity_name": entity_name,
             },
         )
         user_visit.completed_work = completed_work
-        claim_limit = OpportunityClaimLimit.objects.get(
-            opportunity_claim=claim, payment_unit=completed_work.payment_unit
-        )
+        claim_limit = OpportunityClaimLimit.objects.get(opportunity_claim=claim, payment_unit=payment_unit)
         if (
-            counts["daily"] >= deliver_unit.payment_unit.max_daily
+            counts["daily"] >= payment_unit.max_daily
             or counts["total"] >= claim_limit.max_visits
-            or datetime.date.today() > claim.end_date
+            or (today > claim.end_date or (claim_limit.end_date and today > claim_limit.end_date))
         ):
             user_visit.status = VisitValidationStatus.over_limit
             if not completed_work.status == CompletedWorkStatus.over_limit:

From 515521bd9122af419f2f1b1ecd64fc6066ab184c Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Mon, 4 Nov 2024 14:42:45 +0530
Subject: [PATCH 075/165] Add tests for start and end dates on payment units

---
 .../tests/test_receiver_integration.py        | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/commcare_connect/form_receiver/tests/test_receiver_integration.py b/commcare_connect/form_receiver/tests/test_receiver_integration.py
index e24857da..2757a57c 100644
--- a/commcare_connect/form_receiver/tests/test_receiver_integration.py
+++ b/commcare_connect/form_receiver/tests/test_receiver_integration.py
@@ -3,6 +3,7 @@
 from uuid import uuid4
 
 import pytest
+from django.utils.timezone import now
 from rest_framework.test import APIClient
 
 from commcare_connect.form_receiver.tests.test_receiver_endpoint import add_credentials
@@ -602,6 +603,25 @@ def test_receiver_visit_review_status(
     assert visit.review_status == review_status
 
 
+@pytest.mark.parametrize(
+    "paymentunit_options, visit_status",
+    [
+        ({"start_date": now().date()}, VisitValidationStatus.approved),
+        ({"start_date": now() + datetime.timedelta(days=2)}, VisitValidationStatus.trial),
+        ({"end_date": now().date()}, VisitValidationStatus.approved),
+        ({"end_date": now() - datetime.timedelta(days=2)}, VisitValidationStatus.over_limit),
+    ],
+)
+def test_receiver_visit_payment_unit_dates(
+    mobile_user_with_connect_link: User, api_client: APIClient, opportunity: Opportunity, visit_status
+):
+    form_json = get_form_json_for_payment_unit(opportunity.paymentunit_set.first())
+    form_json["metadata"]["timeStart"] = now() - datetime.timedelta(minutes=2)
+    make_request(api_client, form_json, mobile_user_with_connect_link)
+    visit = UserVisit.objects.get(user=mobile_user_with_connect_link)
+    assert visit.status == visit_status
+
+
 def get_form_json_for_payment_unit(payment_unit):
     deliver_unit = DeliverUnitFactory(app=payment_unit.opportunity.deliver_app, payment_unit=payment_unit)
     stub = DeliverUnitStubFactory(id=deliver_unit.slug)

From 28fe5917a7fa91a2dbeb66536acc3ebf6da916be Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Mon, 4 Nov 2024 15:56:54 +0530
Subject: [PATCH 076/165] Add start and end date to payment unit form

---
 commcare_connect/opportunity/forms.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/commcare_connect/opportunity/forms.py b/commcare_connect/opportunity/forms.py
index 3e472450..dc1f4fae 100644
--- a/commcare_connect/opportunity/forms.py
+++ b/commcare_connect/opportunity/forms.py
@@ -611,7 +611,15 @@ def __init__(self, *args, **kwargs):
 class PaymentUnitForm(forms.ModelForm):
     class Meta:
         model = PaymentUnit
-        fields = ["name", "description", "amount", "max_total", "max_daily"]
+        fields = ["name", "description", "amount", "max_total", "max_daily", "start_date", "end_date"]
+        help_texts = {
+            "start_date": "Optional. If not specified opportunity start date applies to form submissions.",
+            "end_date": "Optional. If not specified opportunity end date applies to form submissions.",
+        }
+        widgets = {
+            "start_date": forms.DateInput(attrs={"type": "date", "class": "form-input"}),
+            "end_date": forms.DateInput(attrs={"type": "date", "class": "form-input"}),
+        }
 
     def __init__(self, *args, **kwargs):
         deliver_units = kwargs.pop("deliver_units", [])
@@ -623,6 +631,7 @@ def __init__(self, *args, **kwargs):
             Row(Field("name")),
             Row(Field("description")),
             Row(Field("amount")),
+            Row(Column("start_date"), Column("end_date")),
             Row(Field("required_deliver_units")),
             Row(Field("optional_deliver_units")),
             Row(Field("payment_units")),
@@ -683,6 +692,8 @@ def clean(self):
                     "optional_deliver_units",
                     error=f"{deliver_unit_obj.name} cannot be marked both Required and Optional",
                 )
+            if cleaned_data["end_date"] < now().date():
+                self.add_error("end_date", "Please provide a valid end date.")
         return cleaned_data
 
 

From cd9269358ff44a953ce3d8125a77770bf8657533 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Tue, 5 Nov 2024 16:09:40 +0530
Subject: [PATCH 077/165] changed user field from select to text

---
 commcare_connect/organization/forms.py        | 36 ++++++++++---
 .../organization/tests/__init__.py            |  0
 .../organization/tests/test_forms.py          | 53 +++++++++++++++++++
 3 files changed, 82 insertions(+), 7 deletions(-)
 create mode 100644 commcare_connect/organization/tests/__init__.py
 create mode 100644 commcare_connect/organization/tests/test_forms.py

diff --git a/commcare_connect/organization/forms.py b/commcare_connect/organization/forms.py
index f22a4b7b..5500b267 100644
--- a/commcare_connect/organization/forms.py
+++ b/commcare_connect/organization/forms.py
@@ -1,5 +1,7 @@
 from crispy_forms import helper, layout
 from django import forms
+from django.core.exceptions import ValidationError
+from django.db.models import Q
 from django.utils.translation import gettext
 
 from commcare_connect.organization.models import Organization, UserOrganizationMembership
@@ -25,29 +27,49 @@ def __init__(self, *args, **kwargs):
 
 
 class MembershipForm(forms.ModelForm):
+    email_or_username = forms.CharField(
+        max_length=254,
+        required=True,
+        label="",
+        widget=forms.TextInput(attrs={"placeholder": "Enter email or username"}),
+    )
+
     class Meta:
         model = UserOrganizationMembership
-        fields = ("user", "role")
-        labels = {"user": "", "role": ""}
+        fields = ("role",)
+        labels = {"role": ""}
 
     def __init__(self, *args, **kwargs):
         self.organization = kwargs.pop("organization")
         super().__init__(*args, **kwargs)
 
-        self.fields["user"].queryset = User.objects.filter(email__isnull=False).exclude(
-            memberships__organization=self.organization
-        )
-
         self.helper = helper.FormHelper(self)
         self.helper.layout = layout.Layout(
             layout.Row(
                 layout.HTML("<h4>Add new member</h4>"),
-                layout.Field("user", wrapper_class="col-md-5"),
+                layout.Field("email_or_username", wrapper_class="col-md-5"),
                 layout.Field("role", wrapper_class="col-md-5"),
                 layout.Div(layout.Submit("submit", gettext("Submit")), css_class="col-md-2"),
             ),
         )
 
+    def clean_email_or_username(self):
+        email_or_username = self.cleaned_data["email_or_username"]
+        user = (
+            User.objects.filter(
+                Q(email=email_or_username) | Q(username=email_or_username),
+                email__isnull=False,
+            )
+            .exclude(memberships__organization=self.organization)
+            .first()
+        )
+
+        if not user:
+            raise ValidationError("User with this email/username does not exist or is already a member")
+
+        self.instance.user = user
+        return email_or_username
+
 
 class AddCredentialForm(forms.Form):
     credential = forms.CharField(widget=forms.Select)
diff --git a/commcare_connect/organization/tests/__init__.py b/commcare_connect/organization/tests/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/commcare_connect/organization/tests/test_forms.py b/commcare_connect/organization/tests/test_forms.py
new file mode 100644
index 00000000..d3b1a71b
--- /dev/null
+++ b/commcare_connect/organization/tests/test_forms.py
@@ -0,0 +1,53 @@
+import pytest
+from django.test import Client
+from django.urls import reverse
+
+from commcare_connect.organization.models import Organization
+from commcare_connect.users.tests.factories import UserFactory
+
+
+class TestAddMembersView:
+    @pytest.fixture(autouse=True)
+    def setup(self, organization: Organization, client: Client):
+        self.url = reverse("organization:add_members", kwargs={"org_slug": organization.slug})
+        self.user = organization.memberships.filter(role="admin").first().user
+        self.client = client
+        client.force_login(self.user)
+
+    @pytest.mark.django_db
+    def test_add_member_by_email(self, organization):
+        new_user = UserFactory(email="test@example.com")
+        data = {"email_or_username": new_user.email, "role": "member"}
+        response = self.client.post(self.url, data)
+        assert response.status_code == 302
+        assert response.url == reverse("organization:home", kwargs={"org_slug": organization.slug})
+        membership = organization.memberships.get(user=new_user)
+        assert membership.role == data["role"]
+
+    @pytest.mark.django_db
+    def test_add_member_by_username(self, organization):
+        new_user = UserFactory(username="test")
+        data = {"email_or_username": new_user.username, "role": "member"}
+        response = self.client.post(self.url, data)
+        assert response.status_code == 302
+        membership = organization.memberships.get(user=new_user)
+        assert membership.role == data["role"]
+        assert not membership.accepted
+
+    @pytest.mark.django_db
+    def test_add_member_nonexistent_user(self, organization):
+        data = {"email_or_username": "nonexistent@example.com", "role": "member"}
+        response = self.client.post(self.url, data)
+        assert response.status_code == 302
+        assert not organization.memberships.filter(user__email="nonexistent@example.com").exists()
+
+    @pytest.mark.django_db
+    def test_add_existing_member(self, organization):
+        existing_user = UserFactory(email="test@example.com")
+        organization.members.add(existing_user, through_defaults={"role": "member"})
+
+        data = {"email_or_username": existing_user.email, "role": "admin"}
+        response = self.client.post(self.url, data)
+        assert response.status_code == 302
+        assert organization.memberships.filter(user=existing_user).count() == 1
+        assert organization.memberships.get(user=existing_user).role == "member"

From 0fcf8448ea912d30a0be1d288a4f248f260a6dff Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 6 Nov 2024 09:16:28 +0530
Subject: [PATCH 078/165] changed label

---
 commcare_connect/organization/forms.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/commcare_connect/organization/forms.py b/commcare_connect/organization/forms.py
index 5500b267..743268e9 100644
--- a/commcare_connect/organization/forms.py
+++ b/commcare_connect/organization/forms.py
@@ -31,7 +31,7 @@ class MembershipForm(forms.ModelForm):
         max_length=254,
         required=True,
         label="",
-        widget=forms.TextInput(attrs={"placeholder": "Enter email or username"}),
+        widget=forms.TextInput(attrs={"placeholder": "Enter email address or username"}),
     )
 
     class Meta:
@@ -56,10 +56,7 @@ def __init__(self, *args, **kwargs):
     def clean_email_or_username(self):
         email_or_username = self.cleaned_data["email_or_username"]
         user = (
-            User.objects.filter(
-                Q(email=email_or_username) | Q(username=email_or_username),
-                email__isnull=False,
-            )
+            User.objects.filter(Q(email=email_or_username) | Q(username=email_or_username))
             .exclude(memberships__organization=self.organization)
             .first()
         )

From edc2b5a3d537b0f066d6571a9dc5cd721bd71141 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 6 Nov 2024 09:50:30 +0530
Subject: [PATCH 079/165] refactor tests

---
 .../organization/tests/test_forms.py          | 72 ++++++++++---------
 1 file changed, 40 insertions(+), 32 deletions(-)

diff --git a/commcare_connect/organization/tests/test_forms.py b/commcare_connect/organization/tests/test_forms.py
index d3b1a71b..c3ae9a52 100644
--- a/commcare_connect/organization/tests/test_forms.py
+++ b/commcare_connect/organization/tests/test_forms.py
@@ -15,39 +15,47 @@ def setup(self, organization: Organization, client: Client):
         client.force_login(self.user)
 
     @pytest.mark.django_db
-    def test_add_member_by_email(self, organization):
-        new_user = UserFactory(email="test@example.com")
-        data = {"email_or_username": new_user.email, "role": "member"}
+    @pytest.mark.parametrize(
+        "email_or_username, role, expected_status_code, create_user, expected_role, should_exist",
+        [
+            ("testformember@example.com", "member", 302, True, "member", True),
+            ("testforadmin@example.com", "admin", 302, True, "admin", True),
+            ("testforusername", "member", 302, True, "member", True),
+            ("testforusername", "admin", 302, True, "admin", True),
+            ("nonexistent@example.com", "member", 302, False, None, False),
+            ("existing@example.com", "admin", 302, True, "member", True),
+        ],
+    )
+    def test_add_member(
+        self,
+        email_or_username,
+        role,
+        expected_status_code,
+        create_user,
+        expected_role,
+        should_exist,
+        organization,
+    ):
+        if create_user:
+            user = UserFactory(
+                email=email_or_username if "@" in email_or_username else None,
+                username=email_or_username if "@" not in email_or_username else None,
+            )
+
+            if email_or_username == "existing@example.com":
+                organization.members.add(user, through_defaults={"role": expected_role})
+
+        data = {"email_or_username": email_or_username, "role": role}
         response = self.client.post(self.url, data)
-        assert response.status_code == 302
-        assert response.url == reverse("organization:home", kwargs={"org_slug": organization.slug})
-        membership = organization.memberships.get(user=new_user)
-        assert membership.role == data["role"]
 
-    @pytest.mark.django_db
-    def test_add_member_by_username(self, organization):
-        new_user = UserFactory(username="test")
-        data = {"email_or_username": new_user.username, "role": "member"}
-        response = self.client.post(self.url, data)
-        assert response.status_code == 302
-        membership = organization.memberships.get(user=new_user)
-        assert membership.role == data["role"]
-        assert not membership.accepted
+        membership_filter = (
+            {"user__email": email_or_username} if "@" in email_or_username else {"user__username": email_or_username}
+        )
 
-    @pytest.mark.django_db
-    def test_add_member_nonexistent_user(self, organization):
-        data = {"email_or_username": "nonexistent@example.com", "role": "member"}
-        response = self.client.post(self.url, data)
-        assert response.status_code == 302
-        assert not organization.memberships.filter(user__email="nonexistent@example.com").exists()
+        assert response.status_code == expected_status_code
+        membership_exists = organization.memberships.filter(**membership_filter).exists()
+        assert membership_exists == should_exist
 
-    @pytest.mark.django_db
-    def test_add_existing_member(self, organization):
-        existing_user = UserFactory(email="test@example.com")
-        organization.members.add(existing_user, through_defaults={"role": "member"})
-
-        data = {"email_or_username": existing_user.email, "role": "admin"}
-        response = self.client.post(self.url, data)
-        assert response.status_code == 302
-        assert organization.memberships.filter(user=existing_user).count() == 1
-        assert organization.memberships.get(user=existing_user).role == "member"
+        if should_exist and expected_role:
+            membership = organization.memberships.get(**membership_filter)
+            assert membership.role == expected_role

From 96d9a64b38681c43b4f8c631452815fe2dec11d4 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 6 Nov 2024 14:28:02 +0530
Subject: [PATCH 080/165] removed the username option

---
 commcare_connect/organization/forms.py        | 19 +++++++------------
 .../organization/tests/test_forms.py          | 19 ++++++-------------
 2 files changed, 13 insertions(+), 25 deletions(-)

diff --git a/commcare_connect/organization/forms.py b/commcare_connect/organization/forms.py
index 743268e9..1ca006ed 100644
--- a/commcare_connect/organization/forms.py
+++ b/commcare_connect/organization/forms.py
@@ -1,7 +1,6 @@
 from crispy_forms import helper, layout
 from django import forms
 from django.core.exceptions import ValidationError
-from django.db.models import Q
 from django.utils.translation import gettext
 
 from commcare_connect.organization.models import Organization, UserOrganizationMembership
@@ -27,11 +26,11 @@ def __init__(self, *args, **kwargs):
 
 
 class MembershipForm(forms.ModelForm):
-    email_or_username = forms.CharField(
+    email = forms.CharField(
         max_length=254,
         required=True,
         label="",
-        widget=forms.TextInput(attrs={"placeholder": "Enter email address or username"}),
+        widget=forms.TextInput(attrs={"placeholder": "Enter email address"}),
     )
 
     class Meta:
@@ -47,25 +46,21 @@ def __init__(self, *args, **kwargs):
         self.helper.layout = layout.Layout(
             layout.Row(
                 layout.HTML("<h4>Add new member</h4>"),
-                layout.Field("email_or_username", wrapper_class="col-md-5"),
+                layout.Field("email", wrapper_class="col-md-5"),
                 layout.Field("role", wrapper_class="col-md-5"),
                 layout.Div(layout.Submit("submit", gettext("Submit")), css_class="col-md-2"),
             ),
         )
 
-    def clean_email_or_username(self):
-        email_or_username = self.cleaned_data["email_or_username"]
-        user = (
-            User.objects.filter(Q(email=email_or_username) | Q(username=email_or_username))
-            .exclude(memberships__organization=self.organization)
-            .first()
-        )
+    def clean_email(self):
+        email = self.cleaned_data["email"]
+        user = User.objects.filter(email=email).exclude(memberships__organization=self.organization).first()
 
         if not user:
             raise ValidationError("User with this email/username does not exist or is already a member")
 
         self.instance.user = user
-        return email_or_username
+        return email
 
 
 class AddCredentialForm(forms.Form):
diff --git a/commcare_connect/organization/tests/test_forms.py b/commcare_connect/organization/tests/test_forms.py
index c3ae9a52..8f4381a4 100644
--- a/commcare_connect/organization/tests/test_forms.py
+++ b/commcare_connect/organization/tests/test_forms.py
@@ -16,19 +16,17 @@ def setup(self, organization: Organization, client: Client):
 
     @pytest.mark.django_db
     @pytest.mark.parametrize(
-        "email_or_username, role, expected_status_code, create_user, expected_role, should_exist",
+        "email, role, expected_status_code, create_user, expected_role, should_exist",
         [
             ("testformember@example.com", "member", 302, True, "member", True),
             ("testforadmin@example.com", "admin", 302, True, "admin", True),
-            ("testforusername", "member", 302, True, "member", True),
-            ("testforusername", "admin", 302, True, "admin", True),
             ("nonexistent@example.com", "member", 302, False, None, False),
             ("existing@example.com", "admin", 302, True, "member", True),
         ],
     )
     def test_add_member(
         self,
-        email_or_username,
+        email,
         role,
         expected_status_code,
         create_user,
@@ -37,20 +35,15 @@ def test_add_member(
         organization,
     ):
         if create_user:
-            user = UserFactory(
-                email=email_or_username if "@" in email_or_username else None,
-                username=email_or_username if "@" not in email_or_username else None,
-            )
+            user = UserFactory(email=email)
 
-            if email_or_username == "existing@example.com":
+            if email == "existing@example.com":
                 organization.members.add(user, through_defaults={"role": expected_role})
 
-        data = {"email_or_username": email_or_username, "role": role}
+        data = {"email": email, "role": role}
         response = self.client.post(self.url, data)
 
-        membership_filter = (
-            {"user__email": email_or_username} if "@" in email_or_username else {"user__username": email_or_username}
-        )
+        membership_filter = {"user__email": email}
 
         assert response.status_code == expected_status_code
         membership_exists = organization.memberships.filter(**membership_filter).exists()

From f4a56ca81d3caf4c210cececfd502e422d050245 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 6 Nov 2024 14:30:17 +0530
Subject: [PATCH 081/165] changed the msg

---
 commcare_connect/organization/forms.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/commcare_connect/organization/forms.py b/commcare_connect/organization/forms.py
index 1ca006ed..4bbde435 100644
--- a/commcare_connect/organization/forms.py
+++ b/commcare_connect/organization/forms.py
@@ -57,7 +57,7 @@ def clean_email(self):
         user = User.objects.filter(email=email).exclude(memberships__organization=self.organization).first()
 
         if not user:
-            raise ValidationError("User with this email/username does not exist or is already a member")
+            raise ValidationError("User with this email does not exist or is already a member")
 
         self.instance.user = user
         return email

From 76aee2f87ac9f882d36c45e5e212ec63f455393b Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 6 Nov 2024 14:33:10 +0200
Subject: [PATCH 082/165] simpliify queries

---
 commcare_connect/reports/queries.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/commcare_connect/reports/queries.py b/commcare_connect/reports/queries.py
index 508c9341..ed903bb1 100644
--- a/commcare_connect/reports/queries.py
+++ b/commcare_connect/reports/queries.py
@@ -1,5 +1,5 @@
 from django.db.models import DateTimeField, ExpressionWrapper, F, FloatField
-from django.db.models.fields.json import KeyTextTransform
+from django.db.models.fields.json import KT
 from django.db.models.functions import Cast, Extract
 
 
@@ -11,13 +11,13 @@ def get_visit_map_queryset(base_queryset):
             days_since_opp_start=ExpressionWrapper(
                 Extract(F("visit_date") - F("opportunity__start_date"), "day"), output_field=FloatField()
             ),
-            timestart_str=KeyTextTransform("timeStart", KeyTextTransform("metadata", F("form_json"))),
-            timeend_str=KeyTextTransform("timeEnd", KeyTextTransform("metadata", F("form_json"))),
+            timestart_str=KT("form_json__metadata__timeStart"),
+            timeend_str=KT("form_json__metadata__timeEnd"),
             visit_duration=ExpressionWrapper(
                 Extract(Cast("timeend_str", DateTimeField()) - Cast("timestart_str", DateTimeField()), "epoch") / 60,
                 output_field=FloatField(),
             ),
-            location_str=KeyTextTransform("location", KeyTextTransform("metadata", F("form_json"))),
+            location_str=KT("form_json__metadata__location"),
         )
         .select_related("deliver_unit", "opportunity", "opportunity__delivery_type", "opportunity__organization")
         .values(

From fc25d8f271be50d6d0f766b609233de5647453e2 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 6 Nov 2024 14:36:22 +0200
Subject: [PATCH 083/165] remove unused fields

---
 commcare_connect/reports/queries.py | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/commcare_connect/reports/queries.py b/commcare_connect/reports/queries.py
index ed903bb1..e54a50a8 100644
--- a/commcare_connect/reports/queries.py
+++ b/commcare_connect/reports/queries.py
@@ -1,6 +1,5 @@
-from django.db.models import DateTimeField, ExpressionWrapper, F, FloatField
+from django.db.models import F
 from django.db.models.fields.json import KT
-from django.db.models.functions import Cast, Extract
 
 
 def get_visit_map_queryset(base_queryset):
@@ -8,15 +7,8 @@ def get_visit_map_queryset(base_queryset):
         base_queryset.annotate(
             username_connectid=F("form_json__metadata__username"),
             deliver_unit_name=F("deliver_unit__name"),
-            days_since_opp_start=ExpressionWrapper(
-                Extract(F("visit_date") - F("opportunity__start_date"), "day"), output_field=FloatField()
-            ),
             timestart_str=KT("form_json__metadata__timeStart"),
             timeend_str=KT("form_json__metadata__timeEnd"),
-            visit_duration=ExpressionWrapper(
-                Extract(Cast("timeend_str", DateTimeField()) - Cast("timestart_str", DateTimeField()), "epoch") / 60,
-                output_field=FloatField(),
-            ),
             location_str=KT("form_json__metadata__location"),
         )
         .select_related("deliver_unit", "opportunity", "opportunity__delivery_type", "opportunity__organization")
@@ -30,7 +22,6 @@ def get_visit_map_queryset(base_queryset):
             "visit_date",
             "username_connectid",
             "deliver_unit_name",
-            "days_since_opp_start",
             "entity_id",
             "status",
             "flagged",
@@ -38,7 +29,6 @@ def get_visit_map_queryset(base_queryset):
             "reason",
             "timestart_str",
             "timeend_str",
-            "visit_duration",
             "location_str",
         )
     )

From 8166964ae9e191b4631e78821cc01e34752fd31b Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 6 Nov 2024 14:41:47 +0200
Subject: [PATCH 084/165] use kt on the last field

---
 commcare_connect/reports/queries.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/commcare_connect/reports/queries.py b/commcare_connect/reports/queries.py
index e54a50a8..ed9ddf91 100644
--- a/commcare_connect/reports/queries.py
+++ b/commcare_connect/reports/queries.py
@@ -5,8 +5,8 @@
 def get_visit_map_queryset(base_queryset):
     return (
         base_queryset.annotate(
-            username_connectid=F("form_json__metadata__username"),
             deliver_unit_name=F("deliver_unit__name"),
+            username_connectid=KT("form_json__metadata__username"),
             timestart_str=KT("form_json__metadata__timeStart"),
             timeend_str=KT("form_json__metadata__timeEnd"),
             location_str=KT("form_json__metadata__location"),

From 2830f98d27cb96eeb51d38fdf47a0e0578590086 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Thu, 7 Nov 2024 14:42:12 +0530
Subject: [PATCH 085/165] Add hide user phone numbers for invited users

---
 commcare_connect/opportunity/tables.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/commcare_connect/opportunity/tables.py b/commcare_connect/opportunity/tables.py
index a856650f..9734e168 100644
--- a/commcare_connect/opportunity/tables.py
+++ b/commcare_connect/opportunity/tables.py
@@ -173,9 +173,7 @@ class Meta:
         orderable = False
 
     def render_display_name(self, record):
-        if record.opportunity_access is None:
-            return record.phone_number
-        if not record.opportunity_access.accepted:
+        if not getattr(record.opportunity_access, "accepted", False):
             return "---"
         return record.opportunity_access.display_name
 

From a5489df56f0790048366e9a39f91e677893d0f20 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Thu, 7 Nov 2024 14:44:50 +0530
Subject: [PATCH 086/165] Add user invite delete button and url

---
 commcare_connect/opportunity/tables.py        | 19 ++++++++++++++-----
 commcare_connect/opportunity/urls.py          |  1 +
 commcare_connect/opportunity/views.py         | 11 +++++++++++
 .../opportunity/opportunity_detail.html       |  2 +-
 4 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/commcare_connect/opportunity/tables.py b/commcare_connect/opportunity/tables.py
index 9734e168..e86969a8 100644
--- a/commcare_connect/opportunity/tables.py
+++ b/commcare_connect/opportunity/tables.py
@@ -153,7 +153,7 @@ class UserStatusTable(OrgContextTable):
     passed_assessment = BooleanAggregateColumn(verbose_name="Passed Assessment")
     started_delivery = AggregateColumn(verbose_name="Started Delivery", accessor="date_deliver_started")
     last_visit_date = columns.Column(accessor="last_visit_date_d")
-    view_profile = columns.Column("View Profile", empty_values=(), footer=lambda table: f"Invited: {len(table.rows)}")
+    view_profile = columns.Column("", empty_values=(), footer=lambda table: f"Invited: {len(table.rows)}")
 
     class Meta:
         model = UserInvite
@@ -178,10 +178,19 @@ def render_display_name(self, record):
         return record.opportunity_access.display_name
 
     def render_view_profile(self, record):
-        if record.opportunity_access is None:
-            return "---"
-        if not record.opportunity_access.accepted:
-            return "---"
+        invite_delete_url = reverse(
+            "opportunity:user_invite_delete",
+            args=(self.org_slug, record.opportunity.id, record.id),
+        )
+        if not getattr(record.opportunity_access, "accepted", False):
+            return format_html(
+                (
+                    '<button hx-post="{}" hx-swap="none" '
+                    'hx-confirm="Please confirm to delete the User Invite." '
+                    'class="btn btn-danger">Delete</button>'
+                ),
+                invite_delete_url,
+            )
         url = reverse(
             "opportunity:user_profile",
             kwargs={"org_slug": self.org_slug, "opp_id": record.opportunity.id, "pk": record.opportunity_access_id},
diff --git a/commcare_connect/opportunity/urls.py b/commcare_connect/opportunity/urls.py
index 66b7a957..ef5cae31 100644
--- a/commcare_connect/opportunity/urls.py
+++ b/commcare_connect/opportunity/urls.py
@@ -112,4 +112,5 @@
     path("<int:pk>/invoice_table/", views.PaymentInvoiceTableView.as_view(), name="invoice_table"),
     path("<int:pk>/invoice/create/", views.invoice_create, name="invoice_create"),
     path("<int:pk>/invoice/approve/", views.invoice_approve, name="invoice_approve"),
+    path("<int:opp_id>/user_invite_delete/<int:pk>/", views.user_invite_delete, name="user_invite_delete"),
 ]
diff --git a/commcare_connect/opportunity/views.py b/commcare_connect/opportunity/views.py
index e9083536..d725bbe5 100644
--- a/commcare_connect/opportunity/views.py
+++ b/commcare_connect/opportunity/views.py
@@ -64,6 +64,7 @@
     Payment,
     PaymentInvoice,
     PaymentUnit,
+    UserInvite,
     UserVisit,
     VisitReviewStatus,
     VisitValidationStatus,
@@ -1217,3 +1218,13 @@ def invoice_approve(request, org_slug, pk):
         )
         payment.save()
     return HttpResponse(headers={"HX-Trigger": "newInvoice"})
+
+
+@org_member_required
+@require_POST
+@csrf_exempt
+def user_invite_delete(request, org_slug, opp_id, pk):
+    opportunity = get_opportunity_or_404(opp_id, org_slug)
+    invite = get_object_or_404(UserInvite, pk=pk, opportunity=opportunity)
+    invite.delete()
+    return HttpResponse(status=200, headers={"HX-Trigger": "userStatusReload"})
diff --git a/commcare_connect/templates/opportunity/opportunity_detail.html b/commcare_connect/templates/opportunity/opportunity_detail.html
index eeeb7dd8..b1aef96d 100644
--- a/commcare_connect/templates/opportunity/opportunity_detail.html
+++ b/commcare_connect/templates/opportunity/opportunity_detail.html
@@ -364,7 +364,7 @@ <h2 class="accordion-header" id="headingTwo">
       <div class="tab-pane fade show active" id="user-status-tab-pane" role="tabpanel" aria-labelledby="user-status-tab"
            tabindex="0" hx-on::after-request="refreshTooltips()">
         <div hx-get="{% url "opportunity:user_status_table" org_slug=request.org.slug pk=opportunity.pk %}{% querystring %}"
-             hx-trigger="load" hx-swap="outerHTML">
+             hx-trigger="load, userStatusReload from:body">
           {% include "tables/table_placeholder.html" with num_cols=4 %}
         </div>
       </div>

From 1187920419427ef169991bd24fe2107b22b968c1 Mon Sep 17 00:00:00 2001
From: Cal Ellowitz <calellowitz@gmail.com>
Date: Fri, 8 Nov 2024 09:58:05 -0500
Subject: [PATCH 087/165] allow search by username

---
 commcare_connect/opportunity/admin.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/commcare_connect/opportunity/admin.py b/commcare_connect/opportunity/admin.py
index 6bb084a5..86839d6e 100644
--- a/commcare_connect/opportunity/admin.py
+++ b/commcare_connect/opportunity/admin.py
@@ -48,6 +48,7 @@ class OpportunityAccessAdmin(admin.ModelAdmin):
     form = OpportunityAccessCreationForm
     list_display = ["get_opp_name", "get_username"]
     actions = ["clear_user_progress"]
+    search_fields = ["user__username"]
 
     @admin.display(description="Opportunity Name")
     def get_opp_name(self, obj):

From 1f283a9e5d5fab09ee81c7f0ab746c2b6742b91b Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Tue, 12 Nov 2024 12:44:36 +0530
Subject: [PATCH 088/165] added distinct true in queries

---
 commcare_connect/program/helpers.py            | 17 ++++++++---------
 commcare_connect/program/tests/test_helpers.py |  7 ++++++-
 2 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/commcare_connect/program/helpers.py b/commcare_connect/program/helpers.py
index 94382f33..e26c0cfc 100644
--- a/commcare_connect/program/helpers.py
+++ b/commcare_connect/program/helpers.py
@@ -82,12 +82,7 @@ def get_delivery_performance_report(program: Program, start_date, end_date):
     if end_date:
         date_filter &= Q(opportunityaccess__uservisit__visit_date__lte=end_date)
 
-    flagged_visits_filter = Q(opportunityaccess__uservisit__flagged=True) & ~Q(
-        opportunityaccess__uservisit__status__in=[
-            VisitValidationStatus.rejected,
-            VisitValidationStatus.approved,
-        ]
-    )
+    flagged_visits_filter = Q(opportunityaccess__uservisit__flagged=True) & FILTER_FOR_VALID_VISIT_DATE
 
     managed_opportunities = (
         ManagedOpportunity.objects.filter(program=program)
@@ -103,9 +98,13 @@ def get_delivery_performance_report(program: Program, start_date, end_date):
                 filter=date_filter,
                 distinct=True,
             ),
-            total_payment_units=Count("opportunityaccess__completedwork"),
-            total_payment_units_with_flags=Count("opportunityaccess__completedwork", filter=flagged_visits_filter),
-            total_payment_since_start_date=Count("opportunityaccess__completedwork", filter=date_filter),
+            total_payment_units=Count("opportunityaccess__completedwork", distinct=True),
+            total_payment_units_with_flags=Count(
+                "opportunityaccess__completedwork", distinct=True, filter=flagged_visits_filter
+            ),
+            total_payment_since_start_date=Count(
+                "opportunityaccess__completedwork", distinct=True, filter=date_filter
+            ),
             delivery_per_day_per_worker=Case(
                 When(active_workers=0, then=Value(0)),
                 default=Round(F("total_payment_since_start_date") / F("active_workers"), 2),
diff --git a/commcare_connect/program/tests/test_helpers.py b/commcare_connect/program/tests/test_helpers.py
index c03e0f1e..015c5fb2 100644
--- a/commcare_connect/program/tests/test_helpers.py
+++ b/commcare_connect/program/tests/test_helpers.py
@@ -27,13 +27,17 @@ def create_user_with_access(self, visit_status=VisitValidationStatus.pending, pa
         user = UserFactory.create()
         access = OpportunityAccessFactory.create(opportunity=self.opp, user=user, invited_date=now())
         AssessmentFactory.create(opportunity=self.opp, user=user, opportunity_access=access, passed=passed_assessment)
-        UserVisitFactory.create(
+        visit = UserVisitFactory.create(
             user=user,
             opportunity=self.opp,
             status=visit_status,
             opportunity_access=access,
             visit_date=now() + timedelta(days=1),
         )
+        print("invited date:", access.invited_date)
+        print("invited date:", visit.visit_date)
+        print(visit.visit_date - access.invited_date)
+        print("@@@@@")
         return user
 
     def create_user_with_visit(self, visit_status, visit_date, flagged=False, create_completed_work=True):
@@ -115,6 +119,7 @@ def test_scenarios(
         opps = get_annotated_managed_opportunity(self.program)
         assert len(opps) == 1
         annotated_opp = opps[0]
+        print("avergae ==>:", annotated_opp.average_time_to_convert)
         assert annotated_opp.workers_invited == expected_invited, f"Failed in {scenario}"
         assert annotated_opp.workers_passing_assessment == expected_passing, f"Failed in {scenario}"
         assert annotated_opp.workers_starting_delivery == expected_delivery, f"Failed in {scenario}"

From c78a3ba710bb94701fe0518e4b1817dd44ff72bd Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Tue, 12 Nov 2024 12:47:45 +0530
Subject: [PATCH 089/165] removed logs

---
 commcare_connect/program/tests/test_helpers.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/commcare_connect/program/tests/test_helpers.py b/commcare_connect/program/tests/test_helpers.py
index 015c5fb2..c03e0f1e 100644
--- a/commcare_connect/program/tests/test_helpers.py
+++ b/commcare_connect/program/tests/test_helpers.py
@@ -27,17 +27,13 @@ def create_user_with_access(self, visit_status=VisitValidationStatus.pending, pa
         user = UserFactory.create()
         access = OpportunityAccessFactory.create(opportunity=self.opp, user=user, invited_date=now())
         AssessmentFactory.create(opportunity=self.opp, user=user, opportunity_access=access, passed=passed_assessment)
-        visit = UserVisitFactory.create(
+        UserVisitFactory.create(
             user=user,
             opportunity=self.opp,
             status=visit_status,
             opportunity_access=access,
             visit_date=now() + timedelta(days=1),
         )
-        print("invited date:", access.invited_date)
-        print("invited date:", visit.visit_date)
-        print(visit.visit_date - access.invited_date)
-        print("@@@@@")
         return user
 
     def create_user_with_visit(self, visit_status, visit_date, flagged=False, create_completed_work=True):
@@ -119,7 +115,6 @@ def test_scenarios(
         opps = get_annotated_managed_opportunity(self.program)
         assert len(opps) == 1
         annotated_opp = opps[0]
-        print("avergae ==>:", annotated_opp.average_time_to_convert)
         assert annotated_opp.workers_invited == expected_invited, f"Failed in {scenario}"
         assert annotated_opp.workers_passing_assessment == expected_passing, f"Failed in {scenario}"
         assert annotated_opp.workers_starting_delivery == expected_delivery, f"Failed in {scenario}"

From 3068d294f18cce836864f14918abbea2e5e809fd Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 14:44:36 +0200
Subject: [PATCH 090/165] change center to africa

---
 commcare_connect/templates/reports/dashboard.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index e1973c7d..75de2f19 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -143,8 +143,8 @@ <h3 class="my-2">Service Delivery Map</h3>
       container: 'map',
       // Choose from Mapbox's core styles, or make your own style with Mapbox Studio
       style: 'mapbox://styles/mapbox/dark-v11',
-      center: [-103.5917, 40.6699],
-      zoom: 3
+      center: [20, 0], // Centered on Africa (roughly central coordinates)
+      zoom: 3,
     });
 
     map.on('load', () => {

From fbbafd14bb1f257999b5e0a5f073fbdd660f4991 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 14:44:52 +0200
Subject: [PATCH 091/165] add ability to enable/disable clustering by url param

---
 commcare_connect/reports/views.py                 | 3 ++-
 commcare_connect/templates/reports/dashboard.html | 6 +++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index c91e8789..6c602801 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -223,12 +223,13 @@ class Meta:
 @user_passes_test(lambda u: u.is_superuser)
 def program_dashboard_report(request):
     filterset = DashboardFilters(request.GET)
-
+    cluster = request.GET.get("cluster", "true") == "true"
     return render(
         request,
         "reports/dashboard.html",
         context={
             "mapbox_token": settings.MAPBOX_TOKEN,
+            "cluster_visits": cluster,
             "filter": filterset,
         },
     )
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 75de2f19..89807352 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -139,9 +139,9 @@ <h3 class="my-2">Service Delivery Map</h3>
 
   window.addEventListener('DOMContentLoaded', () => {
     mapboxgl.accessToken = "{{ mapbox_token }}";
+    const cluster = {% if cluster_visits %}true{% else %}false{% endif %};
     map = new mapboxgl.Map({
       container: 'map',
-      // Choose from Mapbox's core styles, or make your own style with Mapbox Studio
       style: 'mapbox://styles/mapbox/dark-v11',
       center: [20, 0], // Centered on Africa (roughly central coordinates)
       zoom: 3,
@@ -156,8 +156,8 @@ <h3 class="my-2">Service Delivery Map</h3>
       map.addSource('visits', {
         type: 'geojson',
         data: `{% url "reports:visit_map_data" %}?${queryString}`,
-        cluster: true,
-        clusterMaxZoom: 14,
+        cluster: cluster,
+        clusterMaxZoom: 12,
         clusterRadius: 50
       });
 

From 1d06e14535c468cc6cf7158a9d12592cc200a934 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 15:19:55 +0200
Subject: [PATCH 092/165] wip: switch to chart clusters

---
 .../templates/reports/dashboard.html          | 208 +++++++++++++++---
 1 file changed, 176 insertions(+), 32 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 89807352..f133fde2 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -147,6 +147,17 @@ <h3 class="my-2">Service Delivery Map</h3>
       zoom: 3,
     });
 
+    // filters for classifying visits by status
+    const approved = ['==', ['get', 'status'], 'approved'];
+    const pending = ['all',
+      ['!=', ['get', 'status'], 'approved'],
+      ['!=', ['get', 'status'], 'rejected']
+    ];
+    const rejected = ['==', ['get', 'status'], 'rejected'];
+
+    // colors to use for the categories
+    const colors = ['#00FF00', '#FFFF00', '#FF0000'];
+
     map.on('load', () => {
       // Modify the source configuration to include initial filters
       const formElement = document.querySelector('#filterForm form');
@@ -158,43 +169,106 @@ <h3 class="my-2">Service Delivery Map</h3>
         data: `{% url "reports:visit_map_data" %}?${queryString}`,
         cluster: cluster,
         clusterMaxZoom: 12,
-        clusterRadius: 50
+        clusterRadius: 80,
+        clusterProperties: {
+          // keep separate counts for each status category in a cluster
+          'approved': ['+', ['case', approved, 1, 0]],
+          'pending': ['+', ['case', pending, 1, 0]],
+          'rejected': ['+', ['case', rejected, 1, 0]]
+        }
       });
 
+      // circle and symbol layers for rendering individual visits (unclustered points)
+      map.addLayer({
+          'id': 'visit_circle',
+          'type': 'circle',
+          'source': 'visits',
+          'filter': ['!=', 'cluster', true],
+          'paint': {
+            'circle-color': [
+              'case',
+              approved,
+              colors[0],
+              pending,
+              colors[1],
+              rejected,
+              colors[2],
+              colors[2]
+            ],
+            'circle-opacity': 0.6,
+            'circle-radius': 12
+          }
+      });
+
+
+      // objects for caching and keeping track of HTML marker objects (for performance)
+      const markers = {};
+      let markersOnScreen = {};
+      function updateMarkers() {
+        console.log('updateMarkers');
+        const newMarkers = {};
+        const features = map.querySourceFeatures('visits');
+
+        // for every cluster on the screen, create an HTML marker for it (if we didn't yet),
+        // and add it to the map if it's not there already
+        for (const feature of features) {
+          const coords = feature.geometry.coordinates;
+          const props = feature.properties;
+          if (!props.cluster) continue;
+          console.log('cluster', feature);
+          const id = props.cluster_id;
+
+          let marker = markers[id];
+          if (!marker) {
+              const el = createDonutChart(props);
+              marker = markers[id] = new mapboxgl.Marker({
+                  element: el
+              }).setLngLat(coords);
+          }
+          newMarkers[id] = marker;
+
+          if (!markersOnScreen[id]) marker.addTo(map);
+        }
+        // for every marker we've added previously, remove those that are no longer visible
+        for (const id in markersOnScreen) {
+          if (!newMarkers[id]) markersOnScreen[id].remove();
+        }
+        markersOnScreen = newMarkers;
+      }
       const clusterStep1 = 50;
       const clusterStep2 = 100;
 
-      map.addLayer({
-        id: 'clusters',
-        type: 'circle',
-        source: 'visits',
-        filter: ['has', 'point_count'],
-        paint: {
-          // Use step expressions (https://docs.mapbox.com/style-spec/reference/expressions/#step)
-          // with three steps to implement three types of circles:
-          //   * Blue, 20px circles when point count is less than clusterStep1
-          //   * Yellow, 30px circles when point count is between clusterStep1 and clusterStep2
-          //   * Green, 40px circles when point count is greater than or equal to clusterStep2
-          'circle-color': [
-            'step',
-            ['get', 'point_count'],
-            '#51bbd6',
-            clusterStep1,
-            '#f1f075',
-            clusterStep2,
-            '#00FF00'
-          ],
-          'circle-radius': [
-            'step',
-            ['get', 'point_count'],
-            20,
-            clusterStep1,
-            30,
-            clusterStep2,
-            40
-          ]
-        }
-      });
+      // map.addlayer({
+      //   id: 'clusters',
+      //   type: 'circle',
+      //   source: 'visits',
+      //   filter: ['has', 'point_count'],
+      //   paint: {
+      //     // use step expressions (https://docs.mapbox.com/style-spec/reference/expressions/#step)
+      //     // with three steps to implement three types of circles:
+      //     //   * blue, 20px circles when point count is less than clusterstep1
+      //     //   * yellow, 30px circles when point count is between clusterstep1 and clusterstep2
+      //     //   * green, 40px circles when point count is greater than or equal to clusterstep2
+      //     'circle-color': [
+      //       'step',
+      //       ['get', 'point_count'],
+      //       '#51bbd6',
+      //       clusterstep1,
+      //       '#f1f075',
+      //       clusterstep2,
+      //       '#00ff00'
+      //     ],
+      //     'circle-radius': [
+      //       'step',
+      //       ['get', 'point_count'],
+      //       20,
+      //       clusterstep1,
+      //       30,
+      //       clusterstep2,
+      //       40
+      //     ]
+      //   }
+      // });
 
       map.addLayer({
         id: 'cluster-count',
@@ -274,7 +348,77 @@ <h3 class="my-2">Service Delivery Map</h3>
       map.on('mouseleave', 'clusters', () => {
         map.getCanvas().style.cursor = '';
       });
+      // after the GeoJSON data is loaded, update markers on the screen on every frame
+      map.on('render', () => {
+        if (!map.isSourceLoaded('visits')) return;
+        updateMarkers();
+      });
     });
+
+    // code for creating an SVG donut chart from feature properties
+    function createDonutChart(props) {
+        const offsets = [];
+        const counts = [
+            props.approved,
+            props.pending,
+            props.rejected
+        ];
+        let total = 0;
+        for (const count of counts) {
+            offsets.push(total);
+            total += count;
+        }
+        const fontSize =
+            total >= 1000 ? 22 : total >= 100 ? 20 : total >= 10 ? 18 : 16;
+        const r =
+            total >= 1000 ? 50 : total >= 100 ? 32 : total >= 10 ? 24 : 18;
+        const r0 = Math.round(r * 0.6);
+        const w = r * 2;
+
+        let html = `<div>
+            <svg width="${w}" height="${w}" viewbox="0 0 ${w} ${w}" text-anchor="middle" style="font: ${fontSize}px sans-serif; display: block">`;
+
+        for (let i = 0; i < counts.length; i++) {
+            html += donutSegment(
+                offsets[i] / total,
+                (offsets[i] + counts[i]) / total,
+                r,
+                r0,
+                colors[i]
+            );
+        }
+        html += `<circle cx="${r}" cy="${r}" r="${r0}" fill="white" />
+            <text dominant-baseline="central" transform="translate(${r}, ${r})">
+                ${total.toLocaleString()}
+            </text>
+            </svg>
+            </div>`;
+
+        const el = document.createElement('div');
+        el.innerHTML = html;
+        return el.firstChild;
+    }
+
+    function donutSegment(start, end, r, r0, color) {
+        if (end - start === 1) end -= 0.00001;
+        const a0 = 2 * Math.PI * (start - 0.25);
+        const a1 = 2 * Math.PI * (end - 0.25);
+        const x0 = Math.cos(a0),
+            y0 = Math.sin(a0);
+        const x1 = Math.cos(a1),
+            y1 = Math.sin(a1);
+        const largeArc = end - start > 0.5 ? 1 : 0;
+
+        // draw an SVG path
+        return `<path d="M ${r + r0 * x0} ${r + r0 * y0} L ${r + r * x0} ${
+            r + r * y0
+        } A ${r} ${r} 0 ${largeArc} 1 ${r + r * x1} ${r + r * y1} L ${
+            r + r0 * x1
+        } ${r + r0 * y1} A ${r0} ${r0} 0 ${largeArc} 0 ${r + r0 * x0} ${
+            r + r0 * y0
+        }" fill="${color}" />`;
+    }
   });
+
 </script>
 {% endblock %}

From 7720e2fbcfe4af14af8b59c5731c9c816f7e0da6 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 15:52:08 +0200
Subject: [PATCH 093/165] cleanup

---
 .../templates/reports/dashboard.html          | 162 ++++++------------
 1 file changed, 55 insertions(+), 107 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index f133fde2..e41e3bc6 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -178,110 +178,6 @@ <h3 class="my-2">Service Delivery Map</h3>
         }
       });
 
-      // circle and symbol layers for rendering individual visits (unclustered points)
-      map.addLayer({
-          'id': 'visit_circle',
-          'type': 'circle',
-          'source': 'visits',
-          'filter': ['!=', 'cluster', true],
-          'paint': {
-            'circle-color': [
-              'case',
-              approved,
-              colors[0],
-              pending,
-              colors[1],
-              rejected,
-              colors[2],
-              colors[2]
-            ],
-            'circle-opacity': 0.6,
-            'circle-radius': 12
-          }
-      });
-
-
-      // objects for caching and keeping track of HTML marker objects (for performance)
-      const markers = {};
-      let markersOnScreen = {};
-      function updateMarkers() {
-        console.log('updateMarkers');
-        const newMarkers = {};
-        const features = map.querySourceFeatures('visits');
-
-        // for every cluster on the screen, create an HTML marker for it (if we didn't yet),
-        // and add it to the map if it's not there already
-        for (const feature of features) {
-          const coords = feature.geometry.coordinates;
-          const props = feature.properties;
-          if (!props.cluster) continue;
-          console.log('cluster', feature);
-          const id = props.cluster_id;
-
-          let marker = markers[id];
-          if (!marker) {
-              const el = createDonutChart(props);
-              marker = markers[id] = new mapboxgl.Marker({
-                  element: el
-              }).setLngLat(coords);
-          }
-          newMarkers[id] = marker;
-
-          if (!markersOnScreen[id]) marker.addTo(map);
-        }
-        // for every marker we've added previously, remove those that are no longer visible
-        for (const id in markersOnScreen) {
-          if (!newMarkers[id]) markersOnScreen[id].remove();
-        }
-        markersOnScreen = newMarkers;
-      }
-      const clusterStep1 = 50;
-      const clusterStep2 = 100;
-
-      // map.addlayer({
-      //   id: 'clusters',
-      //   type: 'circle',
-      //   source: 'visits',
-      //   filter: ['has', 'point_count'],
-      //   paint: {
-      //     // use step expressions (https://docs.mapbox.com/style-spec/reference/expressions/#step)
-      //     // with three steps to implement three types of circles:
-      //     //   * blue, 20px circles when point count is less than clusterstep1
-      //     //   * yellow, 30px circles when point count is between clusterstep1 and clusterstep2
-      //     //   * green, 40px circles when point count is greater than or equal to clusterstep2
-      //     'circle-color': [
-      //       'step',
-      //       ['get', 'point_count'],
-      //       '#51bbd6',
-      //       clusterstep1,
-      //       '#f1f075',
-      //       clusterstep2,
-      //       '#00ff00'
-      //     ],
-      //     'circle-radius': [
-      //       'step',
-      //       ['get', 'point_count'],
-      //       20,
-      //       clusterstep1,
-      //       30,
-      //       clusterstep2,
-      //       40
-      //     ]
-      //   }
-      // });
-
-      map.addLayer({
-        id: 'cluster-count',
-        type: 'symbol',
-        source: 'visits',
-        filter: ['has', 'point_count'],
-        layout: {
-          'text-field': ['get', 'point_count_abbreviated'],
-          'text-font': ['DIN Offc Pro Medium', 'Arial Unicode MS Bold'],
-          'text-size': 12
-        }
-      });
-
       map.addLayer({
         id: 'unclustered-point',
         type: 'circle',
@@ -349,6 +245,39 @@ <h3 class="my-2">Service Delivery Map</h3>
         map.getCanvas().style.cursor = '';
       });
       // after the GeoJSON data is loaded, update markers on the screen on every frame
+      // objects for caching and keeping track of HTML marker objects (for performance)
+      const markers = {};
+      let markersOnScreen = {};
+      function updateMarkers() {
+        const newMarkers = {};
+        const features = map.querySourceFeatures('visits');
+
+        // for every cluster on the screen, create an HTML marker for it (if we didn't yet),
+        // and add it to the map if it's not there already
+        for (const feature of features) {
+          const coords = feature.geometry.coordinates;
+          const props = feature.properties;
+          if (!props.cluster) continue;
+          const id = props.cluster_id;
+
+          let marker = markers[id];
+          if (!marker) {
+              const el = createDonutChart(props);
+              marker = markers[id] = new mapboxgl.Marker({
+                  element: el
+              }).setLngLat(coords);
+          }
+          newMarkers[id] = marker;
+
+          if (!markersOnScreen[id]) marker.addTo(map);
+        }
+        // for every marker we've added previously, remove those that are no longer visible
+        for (const id in markersOnScreen) {
+          if (!newMarkers[id]) markersOnScreen[id].remove();
+        }
+        markersOnScreen = newMarkers;
+      }
+
       map.on('render', () => {
         if (!map.isSourceLoaded('visits')) return;
         updateMarkers();
@@ -372,7 +301,7 @@ <h3 class="my-2">Service Delivery Map</h3>
             total >= 1000 ? 22 : total >= 100 ? 20 : total >= 10 ? 18 : 16;
         const r =
             total >= 1000 ? 50 : total >= 100 ? 32 : total >= 10 ? 24 : 18;
-        const r0 = Math.round(r * 0.6);
+        const r0 = Math.round(r * 0.8);
         const w = r * 2;
 
         let html = `<div>
@@ -387,8 +316,8 @@ <h3 class="my-2">Service Delivery Map</h3>
                 colors[i]
             );
         }
-        html += `<circle cx="${r}" cy="${r}" r="${r0}" fill="white" />
-            <text dominant-baseline="central" transform="translate(${r}, ${r})">
+        html += `<circle cx="${r}" cy="${r}" r="${r0}" fill="transparent" />
+            <text dominant-baseline="central" transform="translate(${r}, ${r})" fill="white">
                 ${total.toLocaleString()}
             </text>
             </svg>
@@ -396,6 +325,25 @@ <h3 class="my-2">Service Delivery Map</h3>
 
         const el = document.createElement('div');
         el.innerHTML = html;
+
+        // Add mouse events to the donut chart
+        el.style.cursor = 'pointer';
+
+        // Add click handler to zoom to cluster
+        el.addEventListener('click', () => {
+          map.getSource('visits').getClusterExpansionZoom(
+            props.cluster_id,
+            (err, zoom) => {
+              if (err) return;
+
+              map.easeTo({
+                center: props.coordinates,
+                zoom: zoom
+              });
+            }
+          );
+        });
+
         return el.firstChild;
     }
 

From 6ee83a53de240634972ae8a5558a9b8737a1f720 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 15:59:23 +0200
Subject: [PATCH 094/165] cleanup

---
 .../templates/reports/dashboard.html          | 26 ++++++++-----------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index e41e3bc6..16226039 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -238,12 +238,6 @@ <h3 class="my-2">Service Delivery Map</h3>
           .addTo(map);
       });
 
-      map.on('mouseenter', 'clusters', () => {
-        map.getCanvas().style.cursor = 'pointer';
-      });
-      map.on('mouseleave', 'clusters', () => {
-        map.getCanvas().style.cursor = '';
-      });
       // after the GeoJSON data is loaded, update markers on the screen on every frame
       // objects for caching and keeping track of HTML marker objects (for performance)
       const markers = {};
@@ -262,10 +256,14 @@ <h3 class="my-2">Service Delivery Map</h3>
 
           let marker = markers[id];
           if (!marker) {
-              const el = createDonutChart(props);
-              marker = markers[id] = new mapboxgl.Marker({
-                  element: el
-              }).setLngLat(coords);
+            const el = createDonutChart({
+              ...props,
+              cluster_id: id,  // Make sure cluster_id is passed
+              coordinates: coords  // Pass the coordinates
+            });
+            marker = markers[id] = new mapboxgl.Marker({
+              element: el
+            }).setLngLat(coords);
           }
           newMarkers[id] = marker;
 
@@ -325,12 +323,10 @@ <h3 class="my-2">Service Delivery Map</h3>
 
         const el = document.createElement('div');
         el.innerHTML = html;
-
-        // Add mouse events to the donut chart
         el.style.cursor = 'pointer';
 
-        // Add click handler to zoom to cluster
-        el.addEventListener('click', () => {
+          // Click handler to zoom and navigate to the cluster
+        el.addEventListener('click', (e) => {
           map.getSource('visits').getClusterExpansionZoom(
             props.cluster_id,
             (err, zoom) => {
@@ -344,7 +340,7 @@ <h3 class="my-2">Service Delivery Map</h3>
           );
         });
 
-        return el.firstChild;
+        return el;
     }
 
     function donutSegment(start, end, r, r0, color) {

From 1bbdd8074053a642e10d667a4712049f29d68f5b Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 16:00:48 +0200
Subject: [PATCH 095/165] delete unused click handler

---
 .../templates/reports/dashboard.html          | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 16226039..01ec3aea 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -191,25 +191,6 @@ <h3 class="my-2">Service Delivery Map</h3>
         }
       });
 
-      // inspect a cluster on click
-      map.on('click', 'clusters', (e) => {
-        const features = map.queryRenderedFeatures(e.point, {
-          layers: ['clusters']
-        });
-        const clusterId = features[0].properties.cluster_id;
-        map.getSource('visits').getClusterExpansionZoom(
-          clusterId,
-          (err, zoom) => {
-            if (err) return;
-
-            map.easeTo({
-              center: features[0].geometry.coordinates,
-              zoom: zoom
-            });
-          }
-        );
-      });
-
       // When a click event occurs on a feature in
       // the unclustered-point layer, open a popup at
       // the location of the feature, with

From 3a57fe5b7bc85129311edf288bef133cab8fcaf6 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 16:03:37 +0200
Subject: [PATCH 096/165] format file

---
 .../templates/reports/dashboard.html          | 341 +++++++++---------
 1 file changed, 169 insertions(+), 172 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 01ec3aea..9273d8ab 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -5,7 +5,7 @@
 {% block title %}Admin Dashboard{% endblock %}
 {% block content %}
 <h2 class="my-2">Program Dashboard</h2>
-<div class="container" x-data="dashboardStats()" >
+<div class="container" x-data="dashboardStats()">
   <div class="mb-4">
     <div class="row mb-4" x-ref="filterForm" id="filterForm">
       {% crispy filter.form %}
@@ -140,209 +140,206 @@ <h3 class="my-2">Service Delivery Map</h3>
   window.addEventListener('DOMContentLoaded', () => {
     mapboxgl.accessToken = "{{ mapbox_token }}";
     const cluster = {% if cluster_visits %}true{% else %}false{% endif %};
-    map = new mapboxgl.Map({
-      container: 'map',
-      style: 'mapbox://styles/mapbox/dark-v11',
-      center: [20, 0], // Centered on Africa (roughly central coordinates)
-      zoom: 3,
-    });
+  map = new mapboxgl.Map({
+    container: 'map',
+    style: 'mapbox://styles/mapbox/dark-v11',
+    center: [20, 0], // Centered on Africa (roughly central coordinates)
+    zoom: 3,
+  });
 
-    // filters for classifying visits by status
-    const approved = ['==', ['get', 'status'], 'approved'];
-    const pending = ['all',
-      ['!=', ['get', 'status'], 'approved'],
-      ['!=', ['get', 'status'], 'rejected']
-    ];
-    const rejected = ['==', ['get', 'status'], 'rejected'];
+  // filters for classifying visits by status
+  const approved = ['==', ['get', 'status'], 'approved'];
+  const pending = ['all',
+    ['!=', ['get', 'status'], 'approved'],
+    ['!=', ['get', 'status'], 'rejected']
+  ];
+  const rejected = ['==', ['get', 'status'], 'rejected'];
 
-    // colors to use for the categories
-    const colors = ['#00FF00', '#FFFF00', '#FF0000'];
+  // colors to use for the categories
+  const colors = ['#00FF00', '#FFFF00', '#FF0000'];
 
-    map.on('load', () => {
-      // Modify the source configuration to include initial filters
-      const formElement = document.querySelector('#filterForm form');
-      const formData = new FormData(formElement);
-      const queryString = new URLSearchParams(formData).toString();
+  map.on('load', () => {
+    // Modify the source configuration to include initial filters
+    const formElement = document.querySelector('#filterForm form');
+    const formData = new FormData(formElement);
+    const queryString = new URLSearchParams(formData).toString();
 
-      map.addSource('visits', {
-        type: 'geojson',
-        data: `{% url "reports:visit_map_data" %}?${queryString}`,
-        cluster: cluster,
-        clusterMaxZoom: 12,
-        clusterRadius: 80,
-        clusterProperties: {
-          // keep separate counts for each status category in a cluster
-          'approved': ['+', ['case', approved, 1, 0]],
-          'pending': ['+', ['case', pending, 1, 0]],
-          'rejected': ['+', ['case', rejected, 1, 0]]
-        }
-      });
+    map.addSource('visits', {
+      type: 'geojson',
+      data: `{% url "reports:visit_map_data" %}?${queryString}`,
+      cluster: cluster,
+      clusterMaxZoom: 12,
+      clusterRadius: 80,
+      clusterProperties: {
+        // keep separate counts for each status category in a cluster
+        'approved': ['+', ['case', approved, 1, 0]],
+        'pending': ['+', ['case', pending, 1, 0]],
+        'rejected': ['+', ['case', rejected, 1, 0]]
+      }
+    });
 
-      map.addLayer({
-        id: 'unclustered-point',
-        type: 'circle',
-        source: 'visits',
-        filter: ['!', ['has', 'point_count']],
-        paint: {
-          'circle-color': ['get', 'color'],
-          'circle-radius': 4,
-          'circle-stroke-width': 1,
-          'circle-stroke-color': '#fff'
-        }
-      });
+    map.addLayer({
+      id: 'unclustered-point',
+      type: 'circle',
+      source: 'visits',
+      filter: ['!', ['has', 'point_count']],
+      paint: {
+        'circle-color': ['get', 'color'],
+        'circle-radius': 4,
+        'circle-stroke-width': 1,
+        'circle-stroke-color': '#fff'
+      }
+    });
 
-      // When a click event occurs on a feature in
-      // the unclustered-point layer, open a popup at
-      // the location of the feature, with
-      // description HTML from its properties.
-      map.on('click', 'unclustered-point', (e) => {
-        const coordinates = e.features[0].geometry.coordinates.slice();
-        const status = e.features[0].properties.status;
-        const rawDate = e.features[0].properties.visit_date;
-        const visitDate = new Date(rawDate).toLocaleDateString('en-US', { year: 'numeric', month: 'short', day: 'numeric' });
+    // When a click event occurs on a feature in
+    // the unclustered-point layer, open a popup at
+    // the location of the feature, with
+    // description HTML from its properties.
+    map.on('click', 'unclustered-point', (e) => {
+      const coordinates = e.features[0].geometry.coordinates.slice();
+      const status = e.features[0].properties.status;
+      const rawDate = e.features[0].properties.visit_date;
+      const visitDate = new Date(rawDate).toLocaleDateString('en-US', { year: 'numeric', month: 'short', day: 'numeric' });
 
 
-        // Ensure that if the map is zoomed out such that
-        // multiple copies of the feature are visible, the
-        // popup appears over the copy being pointed to.
-        if (['mercator', 'equirectangular'].includes(map.getProjection().name)) {
-          while (Math.abs(e.lngLat.lng - coordinates[0]) > 180) {
-            coordinates[0] += e.lngLat.lng > coordinates[0] ? 360 : -360;
-          }
+      // Ensure that if the map is zoomed out such that
+      // multiple copies of the feature are visible, the
+      // popup appears over the copy being pointed to.
+      if (['mercator', 'equirectangular'].includes(map.getProjection().name)) {
+        while (Math.abs(e.lngLat.lng - coordinates[0]) > 180) {
+          coordinates[0] += e.lngLat.lng > coordinates[0] ? 360 : -360;
         }
+      }
 
-        new mapboxgl.Popup()
-          .setLngLat(coordinates)
-          .setHTML(
-            `Visit Date: ${visitDate}<br>Status: ${status}`
-          )
-          .addTo(map);
-      });
-
-      // after the GeoJSON data is loaded, update markers on the screen on every frame
-      // objects for caching and keeping track of HTML marker objects (for performance)
-      const markers = {};
-      let markersOnScreen = {};
-      function updateMarkers() {
-        const newMarkers = {};
-        const features = map.querySourceFeatures('visits');
+      new mapboxgl.Popup()
+        .setLngLat(coordinates)
+        .setHTML(
+          `Visit Date: ${visitDate}<br>Status: ${status}`
+        )
+        .addTo(map);
+    });
 
-        // for every cluster on the screen, create an HTML marker for it (if we didn't yet),
-        // and add it to the map if it's not there already
-        for (const feature of features) {
-          const coords = feature.geometry.coordinates;
-          const props = feature.properties;
-          if (!props.cluster) continue;
-          const id = props.cluster_id;
+    // after the GeoJSON data is loaded, update markers on the screen on every frame
+    // objects for caching and keeping track of HTML marker objects (for performance)
+    const markers = {};
+    let markersOnScreen = {};
+    function updateMarkers() {
+      const newMarkers = {};
+      const features = map.querySourceFeatures('visits');
 
-          let marker = markers[id];
-          if (!marker) {
-            const el = createDonutChart({
-              ...props,
-              cluster_id: id,  // Make sure cluster_id is passed
-              coordinates: coords  // Pass the coordinates
-            });
-            marker = markers[id] = new mapboxgl.Marker({
-              element: el
-            }).setLngLat(coords);
-          }
-          newMarkers[id] = marker;
+      // for every cluster on the screen, create an HTML marker for it (if we didn't yet),
+      // and add it to the map if it's not there already
+      for (const feature of features) {
+        const coords = feature.geometry.coordinates;
+        const props = feature.properties;
+        if (!props.cluster) continue;
+        const id = props.cluster_id;
 
-          if (!markersOnScreen[id]) marker.addTo(map);
-        }
-        // for every marker we've added previously, remove those that are no longer visible
-        for (const id in markersOnScreen) {
-          if (!newMarkers[id]) markersOnScreen[id].remove();
+        let marker = markers[id];
+        if (!marker) {
+          const el = createDonutChart({
+            ...props,
+            cluster_id: id,  // Make sure cluster_id is passed
+            coordinates: coords  // Pass the coordinates
+          });
+          marker = markers[id] = new mapboxgl.Marker({
+            element: el
+          }).setLngLat(coords);
         }
-        markersOnScreen = newMarkers;
+        newMarkers[id] = marker;
+
+        if (!markersOnScreen[id]) marker.addTo(map);
+      }
+      // for every marker we've added previously, remove those that are no longer visible
+      for (const id in markersOnScreen) {
+        if (!newMarkers[id]) markersOnScreen[id].remove();
       }
+      markersOnScreen = newMarkers;
+    }
 
-      map.on('render', () => {
-        if (!map.isSourceLoaded('visits')) return;
-        updateMarkers();
-      });
+    map.on('render', () => {
+      if (!map.isSourceLoaded('visits')) return;
+      updateMarkers();
     });
+  });
 
-    // code for creating an SVG donut chart from feature properties
-    function createDonutChart(props) {
-        const offsets = [];
-        const counts = [
-            props.approved,
-            props.pending,
-            props.rejected
-        ];
-        let total = 0;
-        for (const count of counts) {
-            offsets.push(total);
-            total += count;
-        }
-        const fontSize =
-            total >= 1000 ? 22 : total >= 100 ? 20 : total >= 10 ? 18 : 16;
-        const r =
-            total >= 1000 ? 50 : total >= 100 ? 32 : total >= 10 ? 24 : 18;
-        const r0 = Math.round(r * 0.8);
-        const w = r * 2;
+  // code for creating an SVG donut chart from feature properties
+  function createDonutChart(props) {
+    const offsets = [];
+    const counts = [
+      props.approved,
+      props.pending,
+      props.rejected
+    ];
+    let total = 0;
+    for (const count of counts) {
+      offsets.push(total);
+      total += count;
+    }
+    const fontSize =
+      total >= 1000 ? 22 : total >= 100 ? 20 : total >= 10 ? 18 : 16;
+    const r =
+      total >= 1000 ? 50 : total >= 100 ? 32 : total >= 10 ? 24 : 18;
+    const r0 = Math.round(r * 0.8);
+    const w = r * 2;
 
-        let html = `<div>
+    let html = `<div>
             <svg width="${w}" height="${w}" viewbox="0 0 ${w} ${w}" text-anchor="middle" style="font: ${fontSize}px sans-serif; display: block">`;
 
-        for (let i = 0; i < counts.length; i++) {
-            html += donutSegment(
-                offsets[i] / total,
-                (offsets[i] + counts[i]) / total,
-                r,
-                r0,
-                colors[i]
-            );
-        }
-        html += `<circle cx="${r}" cy="${r}" r="${r0}" fill="transparent" />
+    for (let i = 0; i < counts.length; i++) {
+      html += donutSegment(
+        offsets[i] / total,
+        (offsets[i] + counts[i]) / total,
+        r,
+        r0,
+        colors[i]
+      );
+    }
+    html += `<circle cx="${r}" cy="${r}" r="${r0}" fill="transparent" />
             <text dominant-baseline="central" transform="translate(${r}, ${r})" fill="white">
                 ${total.toLocaleString()}
             </text>
             </svg>
             </div>`;
 
-        const el = document.createElement('div');
-        el.innerHTML = html;
-        el.style.cursor = 'pointer';
+    const el = document.createElement('div');
+    el.innerHTML = html;
+    el.style.cursor = 'pointer';
 
-          // Click handler to zoom and navigate to the cluster
-        el.addEventListener('click', (e) => {
-          map.getSource('visits').getClusterExpansionZoom(
-            props.cluster_id,
-            (err, zoom) => {
-              if (err) return;
+    // Click handler to zoom and navigate to the cluster
+    el.addEventListener('click', (e) => {
+      map.getSource('visits').getClusterExpansionZoom(
+        props.cluster_id,
+        (err, zoom) => {
+          if (err) return;
 
-              map.easeTo({
-                center: props.coordinates,
-                zoom: zoom
-              });
-            }
-          );
-        });
+          map.easeTo({
+            center: props.coordinates,
+            zoom: zoom
+          });
+        }
+      );
+    });
 
-        return el;
-    }
+    return el;
+  }
 
-    function donutSegment(start, end, r, r0, color) {
-        if (end - start === 1) end -= 0.00001;
-        const a0 = 2 * Math.PI * (start - 0.25);
-        const a1 = 2 * Math.PI * (end - 0.25);
-        const x0 = Math.cos(a0),
-            y0 = Math.sin(a0);
-        const x1 = Math.cos(a1),
-            y1 = Math.sin(a1);
-        const largeArc = end - start > 0.5 ? 1 : 0;
+  function donutSegment(start, end, r, r0, color) {
+    if (end - start === 1) end -= 0.00001;
+    const a0 = 2 * Math.PI * (start - 0.25);
+    const a1 = 2 * Math.PI * (end - 0.25);
+    const x0 = Math.cos(a0),
+      y0 = Math.sin(a0);
+    const x1 = Math.cos(a1),
+      y1 = Math.sin(a1);
+    const largeArc = end - start > 0.5 ? 1 : 0;
 
-        // draw an SVG path
-        return `<path d="M ${r + r0 * x0} ${r + r0 * y0} L ${r + r * x0} ${
-            r + r * y0
-        } A ${r} ${r} 0 ${largeArc} 1 ${r + r * x1} ${r + r * y1} L ${
-            r + r0 * x1
-        } ${r + r0 * y1} A ${r0} ${r0} 0 ${largeArc} 0 ${r + r0 * x0} ${
-            r + r0 * y0
-        }" fill="${color}" />`;
-    }
+    // draw an SVG path
+    return `<path d="M ${r + r0 * x0} ${r + r0 * y0} L ${r + r * x0} ${r + r * y0
+      } A ${r} ${r} 0 ${largeArc} 1 ${r + r * x1} ${r + r * y1} L ${r + r0 * x1
+      } ${r + r0 * y1} A ${r0} ${r0} 0 ${largeArc} 0 ${r + r0 * x0} ${r + r0 * y0
+      }" fill="${color}" />`;
+  }
   });
 
 </script>

From 9ccadbbad2f69830910bb3fca569b48a61d561b8 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 16:07:07 +0200
Subject: [PATCH 097/165] wip: loading state

---
 .../templates/reports/dashboard.html          | 26 +++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 9273d8ab..26d1b1b5 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -73,7 +73,14 @@ <h2 class="my-2">Program Dashboard</h2>
 <div class="row">
   <div class="col-12">
     <h3 class="my-2">Service Delivery Map</h3>
-    <div id="map" class="mt-3 mb-2 rounded border" style="height: 900px;"></div>
+    <div class="position-relative">
+      <div id="map" class="mt-3 mb-2 rounded border" style="height: 900px;"></div>
+      <div id="map-loading" class="position-absolute top-0 start-0 w-100 h-100 bg-dark bg-opacity-50 d-flex justify-content-center align-items-center">
+        <div class="spinner-border text-light" role="status">
+          <span class="visually-hidden">Loading map...</span>
+        </div>
+      </div>
+    </div>
   </div>
 </div>
 {% endblock content %}
@@ -124,17 +131,23 @@ <h3 class="my-2">Service Delivery Map</h3>
 </script>
 <script type="module">
   let map;
+  const mapLoading = document.getElementById('map-loading');
 
   // Add this function to handle map data refresh
   window.refreshMapData = () => {
     if (!map) return;
+    mapLoading.classList.remove('d-none');  // Show loading state
 
     const formElement = document.querySelector('#filterForm form');
     const formData = new FormData(formElement);
     const queryString = new URLSearchParams(formData).toString();
 
     // Update the source data with new filters
-    map.getSource('visits').setData(`{% url "reports:visit_map_data" %}?${queryString}`);
+    map.getSource('visits').setData(
+      `{% url "reports:visit_map_data" %}?${queryString}`,
+      // Hide loading state after data is loaded
+      () => mapLoading.classList.add('d-none')
+    );
   };
 
   window.addEventListener('DOMContentLoaded', () => {
@@ -261,6 +274,15 @@ <h3 class="my-2">Service Delivery Map</h3>
       if (!map.isSourceLoaded('visits')) return;
       updateMarkers();
     });
+
+    // Hide loading overlay when initial map load is complete
+    mapLoading.classList.add('d-none');
+  });
+
+  // Show loading state if map errors
+  map.on('error', () => {
+    mapLoading.classList.add('d-none');
+    // Optionally add error messaging here
   });
 
   // code for creating an SVG donut chart from feature properties

From 447bd629855173a4bf745d0baf1f653cba74b2be Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 16:19:00 +0200
Subject: [PATCH 098/165] debug statements

---
 commcare_connect/templates/reports/dashboard.html | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 26d1b1b5..c71c0b6d 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -136,7 +136,8 @@ <h3 class="my-2">Service Delivery Map</h3>
   // Add this function to handle map data refresh
   window.refreshMapData = () => {
     if (!map) return;
-    mapLoading.classList.remove('d-none');  // Show loading state
+    console.log('refreshing map data');
+    // mapLoading.classList.remove('d-none');  // Show loading state
 
     const formElement = document.querySelector('#filterForm form');
     const formData = new FormData(formElement);
@@ -145,8 +146,11 @@ <h3 class="my-2">Service Delivery Map</h3>
     // Update the source data with new filters
     map.getSource('visits').setData(
       `{% url "reports:visit_map_data" %}?${queryString}`,
-      // Hide loading state after data is loaded
-      () => mapLoading.classList.add('d-none')
+      () => {
+        console.log('setting loading to none after update');
+        mapLoading.classList.add('d-none');  // Hide loading state after data is loaded
+        console.log('setting loading to none after update');
+      }
     );
   };
 
@@ -276,11 +280,13 @@ <h3 class="my-2">Service Delivery Map</h3>
     });
 
     // Hide loading overlay when initial map load is complete
+    console.log('setting loading to none')
     mapLoading.classList.add('d-none');
   });
 
   // Show loading state if map errors
   map.on('error', () => {
+    console.log('setting loading to none for an error')
     mapLoading.classList.add('d-none');
     // Optionally add error messaging here
   });

From f3026d3bd66dda70702cbeaf2b4ecf239645ae7e Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 16:27:33 +0200
Subject: [PATCH 099/165] improve set data / loading issues

---
 .../templates/reports/dashboard.html          | 27 +++++++++++--------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index c71c0b6d..48a63da7 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -134,24 +134,29 @@ <h3 class="my-2">Service Delivery Map</h3>
   const mapLoading = document.getElementById('map-loading');
 
   // Add this function to handle map data refresh
-  window.refreshMapData = () => {
+  window.refreshMapData = async () => {
     if (!map) return;
     console.log('refreshing map data');
-    // mapLoading.classList.remove('d-none');  // Show loading state
+    mapLoading.classList.remove('d-none');  // Show loading state
 
     const formElement = document.querySelector('#filterForm form');
     const formData = new FormData(formElement);
     const queryString = new URLSearchParams(formData).toString();
 
-    // Update the source data with new filters
-    map.getSource('visits').setData(
-      `{% url "reports:visit_map_data" %}?${queryString}`,
-      () => {
-        console.log('setting loading to none after update');
-        mapLoading.classList.add('d-none');  // Hide loading state after data is loaded
-        console.log('setting loading to none after update');
-      }
-    );
+    try {
+      // Fetch the data first
+      const response = await fetch(`{% url "reports:visit_map_data" %}?${queryString}`);
+      const data = await response.json();
+
+      // Then set it on the map source
+      map.getSource('visits').setData(data);
+
+      // Hide loading state after data is set
+      mapLoading.classList.add('d-none');
+    } catch (error) {
+      console.error('Error refreshing map data:', error);
+      mapLoading.classList.add('d-none');
+    }
   };
 
   window.addEventListener('DOMContentLoaded', () => {

From 845580492a118ba73efa9578f960d52b32c34fe0 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 16:35:40 +0200
Subject: [PATCH 100/165] style tweaks

---
 .../templates/reports/dashboard.html             | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 48a63da7..c5d5fa71 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -178,7 +178,7 @@ <h3 class="my-2">Service Delivery Map</h3>
   const rejected = ['==', ['get', 'status'], 'rejected'];
 
   // colors to use for the categories
-  const colors = ['#00FF00', '#FFFF00', '#FF0000'];
+  const colors = ['#4ade80', '#fbbf24', '#f87171']; // softer green, yellow, red
 
   map.on('load', () => {
     // Modify the source configuration to include initial filters
@@ -317,7 +317,12 @@ <h3 class="my-2">Service Delivery Map</h3>
     const w = r * 2;
 
     let html = `<div>
-            <svg width="${w}" height="${w}" viewbox="0 0 ${w} ${w}" text-anchor="middle" style="font: ${fontSize}px sans-serif; display: block">`;
+            <svg width="${w}" height="${w}" viewbox="0 0 ${w} ${w}" text-anchor="middle" style="font: ${fontSize}px sans-serif; display: block">
+            <defs>
+              <filter id="shadow">
+                <feDropShadow dx="0" dy="1" stdDeviation="2" flood-opacity="0.3"/>
+              </filter>
+            </defs>`;
 
     for (let i = 0; i < counts.length; i++) {
       html += donutSegment(
@@ -328,8 +333,9 @@ <h3 class="my-2">Service Delivery Map</h3>
         colors[i]
       );
     }
-    html += `<circle cx="${r}" cy="${r}" r="${r0}" fill="transparent" />
-            <text dominant-baseline="central" transform="translate(${r}, ${r})" fill="white">
+    html += `<circle cx="${r}" cy="${r}" r="${r0}" fill="#374151" />
+            <text dominant-baseline="central" transform="translate(${r}, ${r})"
+                  fill="white" font-weight="500" filter="url(#shadow)">
                 ${total.toLocaleString()}
             </text>
             </svg>
@@ -371,7 +377,7 @@ <h3 class="my-2">Service Delivery Map</h3>
     return `<path d="M ${r + r0 * x0} ${r + r0 * y0} L ${r + r * x0} ${r + r * y0
       } A ${r} ${r} 0 ${largeArc} 1 ${r + r * x1} ${r + r * y1} L ${r + r0 * x1
       } ${r + r0 * y1} A ${r0} ${r0} 0 ${largeArc} 0 ${r + r0 * x0} ${r + r0 * y0
-      }" fill="${color}" />`;
+      }" fill="${color}" opacity="0.85" stroke="#1f2937" stroke-width="1" />`;
   }
   });
 

From 4bae7e3c06c97cfc0ffe906a135023c6fc7e4efe Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 16:46:26 +0200
Subject: [PATCH 101/165] remove cluster argument

---
 commcare_connect/reports/views.py                 | 2 --
 commcare_connect/templates/reports/dashboard.html | 3 +--
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index 6c602801..7c0305b0 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -223,13 +223,11 @@ class Meta:
 @user_passes_test(lambda u: u.is_superuser)
 def program_dashboard_report(request):
     filterset = DashboardFilters(request.GET)
-    cluster = request.GET.get("cluster", "true") == "true"
     return render(
         request,
         "reports/dashboard.html",
         context={
             "mapbox_token": settings.MAPBOX_TOKEN,
-            "cluster_visits": cluster,
             "filter": filterset,
         },
     )
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index c5d5fa71..0e97e0fa 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -161,7 +161,6 @@ <h3 class="my-2">Service Delivery Map</h3>
 
   window.addEventListener('DOMContentLoaded', () => {
     mapboxgl.accessToken = "{{ mapbox_token }}";
-    const cluster = {% if cluster_visits %}true{% else %}false{% endif %};
   map = new mapboxgl.Map({
     container: 'map',
     style: 'mapbox://styles/mapbox/dark-v11',
@@ -189,7 +188,7 @@ <h3 class="my-2">Service Delivery Map</h3>
     map.addSource('visits', {
       type: 'geojson',
       data: `{% url "reports:visit_map_data" %}?${queryString}`,
-      cluster: cluster,
+      cluster: true,
       clusterMaxZoom: 12,
       clusterRadius: 80,
       clusterProperties: {

From 882c2aa22461c12180cbdc0c8b86015b3267bcd6 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 16:46:36 +0200
Subject: [PATCH 102/165] fix formatting

---
 .../templates/reports/dashboard.html          | 383 +++++++++---------
 1 file changed, 192 insertions(+), 191 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 0e97e0fa..eddc7a9f 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -75,7 +75,8 @@ <h2 class="my-2">Program Dashboard</h2>
     <h3 class="my-2">Service Delivery Map</h3>
     <div class="position-relative">
       <div id="map" class="mt-3 mb-2 rounded border" style="height: 900px;"></div>
-      <div id="map-loading" class="position-absolute top-0 start-0 w-100 h-100 bg-dark bg-opacity-50 d-flex justify-content-center align-items-center">
+      <div id="map-loading"
+        class="position-absolute top-0 start-0 w-100 h-100 bg-dark bg-opacity-50 d-flex justify-content-center align-items-center">
         <div class="spinner-border text-light" role="status">
           <span class="visually-hidden">Loading map...</span>
         </div>
@@ -161,161 +162,161 @@ <h3 class="my-2">Service Delivery Map</h3>
 
   window.addEventListener('DOMContentLoaded', () => {
     mapboxgl.accessToken = "{{ mapbox_token }}";
-  map = new mapboxgl.Map({
-    container: 'map',
-    style: 'mapbox://styles/mapbox/dark-v11',
-    center: [20, 0], // Centered on Africa (roughly central coordinates)
-    zoom: 3,
-  });
-
-  // filters for classifying visits by status
-  const approved = ['==', ['get', 'status'], 'approved'];
-  const pending = ['all',
-    ['!=', ['get', 'status'], 'approved'],
-    ['!=', ['get', 'status'], 'rejected']
-  ];
-  const rejected = ['==', ['get', 'status'], 'rejected'];
-
-  // colors to use for the categories
-  const colors = ['#4ade80', '#fbbf24', '#f87171']; // softer green, yellow, red
-
-  map.on('load', () => {
-    // Modify the source configuration to include initial filters
-    const formElement = document.querySelector('#filterForm form');
-    const formData = new FormData(formElement);
-    const queryString = new URLSearchParams(formData).toString();
-
-    map.addSource('visits', {
-      type: 'geojson',
-      data: `{% url "reports:visit_map_data" %}?${queryString}`,
-      cluster: true,
-      clusterMaxZoom: 12,
-      clusterRadius: 80,
-      clusterProperties: {
-        // keep separate counts for each status category in a cluster
-        'approved': ['+', ['case', approved, 1, 0]],
-        'pending': ['+', ['case', pending, 1, 0]],
-        'rejected': ['+', ['case', rejected, 1, 0]]
-      }
-    });
-
-    map.addLayer({
-      id: 'unclustered-point',
-      type: 'circle',
-      source: 'visits',
-      filter: ['!', ['has', 'point_count']],
-      paint: {
-        'circle-color': ['get', 'color'],
-        'circle-radius': 4,
-        'circle-stroke-width': 1,
-        'circle-stroke-color': '#fff'
-      }
+    map = new mapboxgl.Map({
+      container: 'map',
+      style: 'mapbox://styles/mapbox/dark-v11',
+      center: [20, 0], // Centered on Africa (roughly central coordinates)
+      zoom: 3,
     });
 
-    // When a click event occurs on a feature in
-    // the unclustered-point layer, open a popup at
-    // the location of the feature, with
-    // description HTML from its properties.
-    map.on('click', 'unclustered-point', (e) => {
-      const coordinates = e.features[0].geometry.coordinates.slice();
-      const status = e.features[0].properties.status;
-      const rawDate = e.features[0].properties.visit_date;
-      const visitDate = new Date(rawDate).toLocaleDateString('en-US', { year: 'numeric', month: 'short', day: 'numeric' });
-
-
-      // Ensure that if the map is zoomed out such that
-      // multiple copies of the feature are visible, the
-      // popup appears over the copy being pointed to.
-      if (['mercator', 'equirectangular'].includes(map.getProjection().name)) {
-        while (Math.abs(e.lngLat.lng - coordinates[0]) > 180) {
-          coordinates[0] += e.lngLat.lng > coordinates[0] ? 360 : -360;
+    // filters for classifying visits by status
+    const approved = ['==', ['get', 'status'], 'approved'];
+    const pending = ['all',
+      ['!=', ['get', 'status'], 'approved'],
+      ['!=', ['get', 'status'], 'rejected']
+    ];
+    const rejected = ['==', ['get', 'status'], 'rejected'];
+
+    // colors to use for the categories
+    const colors = ['#4ade80', '#fbbf24', '#f87171']; // softer green, yellow, red
+
+    map.on('load', () => {
+      // Modify the source configuration to include initial filters
+      const formElement = document.querySelector('#filterForm form');
+      const formData = new FormData(formElement);
+      const queryString = new URLSearchParams(formData).toString();
+
+      map.addSource('visits', {
+        type: 'geojson',
+        data: `{% url "reports:visit_map_data" %}?${queryString}`,
+        cluster: true,
+        clusterMaxZoom: 12,
+        clusterRadius: 80,
+        clusterProperties: {
+          // keep separate counts for each status category in a cluster
+          'approved': ['+', ['case', approved, 1, 0]],
+          'pending': ['+', ['case', pending, 1, 0]],
+          'rejected': ['+', ['case', rejected, 1, 0]]
+        }
+      });
+
+      map.addLayer({
+        id: 'unclustered-point',
+        type: 'circle',
+        source: 'visits',
+        filter: ['!', ['has', 'point_count']],
+        paint: {
+          'circle-color': ['get', 'color'],
+          'circle-radius': 4,
+          'circle-stroke-width': 1,
+          'circle-stroke-color': '#fff'
+        }
+      });
+
+      // When a click event occurs on a feature in
+      // the unclustered-point layer, open a popup at
+      // the location of the feature, with
+      // description HTML from its properties.
+      map.on('click', 'unclustered-point', (e) => {
+        const coordinates = e.features[0].geometry.coordinates.slice();
+        const status = e.features[0].properties.status;
+        const rawDate = e.features[0].properties.visit_date;
+        const visitDate = new Date(rawDate).toLocaleDateString('en-US', { year: 'numeric', month: 'short', day: 'numeric' });
+
+
+        // Ensure that if the map is zoomed out such that
+        // multiple copies of the feature are visible, the
+        // popup appears over the copy being pointed to.
+        if (['mercator', 'equirectangular'].includes(map.getProjection().name)) {
+          while (Math.abs(e.lngLat.lng - coordinates[0]) > 180) {
+            coordinates[0] += e.lngLat.lng > coordinates[0] ? 360 : -360;
+          }
         }
-      }
 
-      new mapboxgl.Popup()
-        .setLngLat(coordinates)
-        .setHTML(
-          `Visit Date: ${visitDate}<br>Status: ${status}`
-        )
-        .addTo(map);
-    });
+        new mapboxgl.Popup()
+          .setLngLat(coordinates)
+          .setHTML(
+            `Visit Date: ${visitDate}<br>Status: ${status}`
+          )
+          .addTo(map);
+      });
+
+      // after the GeoJSON data is loaded, update markers on the screen on every frame
+      // objects for caching and keeping track of HTML marker objects (for performance)
+      const markers = {};
+      let markersOnScreen = {};
+      function updateMarkers() {
+        const newMarkers = {};
+        const features = map.querySourceFeatures('visits');
+
+        // for every cluster on the screen, create an HTML marker for it (if we didn't yet),
+        // and add it to the map if it's not there already
+        for (const feature of features) {
+          const coords = feature.geometry.coordinates;
+          const props = feature.properties;
+          if (!props.cluster) continue;
+          const id = props.cluster_id;
+
+          let marker = markers[id];
+          if (!marker) {
+            const el = createDonutChart({
+              ...props,
+              cluster_id: id,  // Make sure cluster_id is passed
+              coordinates: coords  // Pass the coordinates
+            });
+            marker = markers[id] = new mapboxgl.Marker({
+              element: el
+            }).setLngLat(coords);
+          }
+          newMarkers[id] = marker;
 
-    // after the GeoJSON data is loaded, update markers on the screen on every frame
-    // objects for caching and keeping track of HTML marker objects (for performance)
-    const markers = {};
-    let markersOnScreen = {};
-    function updateMarkers() {
-      const newMarkers = {};
-      const features = map.querySourceFeatures('visits');
-
-      // for every cluster on the screen, create an HTML marker for it (if we didn't yet),
-      // and add it to the map if it's not there already
-      for (const feature of features) {
-        const coords = feature.geometry.coordinates;
-        const props = feature.properties;
-        if (!props.cluster) continue;
-        const id = props.cluster_id;
-
-        let marker = markers[id];
-        if (!marker) {
-          const el = createDonutChart({
-            ...props,
-            cluster_id: id,  // Make sure cluster_id is passed
-            coordinates: coords  // Pass the coordinates
-          });
-          marker = markers[id] = new mapboxgl.Marker({
-            element: el
-          }).setLngLat(coords);
+          if (!markersOnScreen[id]) marker.addTo(map);
         }
-        newMarkers[id] = marker;
-
-        if (!markersOnScreen[id]) marker.addTo(map);
-      }
-      // for every marker we've added previously, remove those that are no longer visible
-      for (const id in markersOnScreen) {
-        if (!newMarkers[id]) markersOnScreen[id].remove();
+        // for every marker we've added previously, remove those that are no longer visible
+        for (const id in markersOnScreen) {
+          if (!newMarkers[id]) markersOnScreen[id].remove();
+        }
+        markersOnScreen = newMarkers;
       }
-      markersOnScreen = newMarkers;
-    }
 
-    map.on('render', () => {
-      if (!map.isSourceLoaded('visits')) return;
-      updateMarkers();
-    });
+      map.on('render', () => {
+        if (!map.isSourceLoaded('visits')) return;
+        updateMarkers();
+      });
 
-    // Hide loading overlay when initial map load is complete
-    console.log('setting loading to none')
-    mapLoading.classList.add('d-none');
-  });
+      // Hide loading overlay when initial map load is complete
+      console.log('setting loading to none')
+      mapLoading.classList.add('d-none');
+    });
 
-  // Show loading state if map errors
-  map.on('error', () => {
-    console.log('setting loading to none for an error')
-    mapLoading.classList.add('d-none');
-    // Optionally add error messaging here
-  });
+    // Show loading state if map errors
+    map.on('error', () => {
+      console.log('setting loading to none for an error')
+      mapLoading.classList.add('d-none');
+      // Optionally add error messaging here
+    });
 
-  // code for creating an SVG donut chart from feature properties
-  function createDonutChart(props) {
-    const offsets = [];
-    const counts = [
-      props.approved,
-      props.pending,
-      props.rejected
-    ];
-    let total = 0;
-    for (const count of counts) {
-      offsets.push(total);
-      total += count;
-    }
-    const fontSize =
-      total >= 1000 ? 22 : total >= 100 ? 20 : total >= 10 ? 18 : 16;
-    const r =
-      total >= 1000 ? 50 : total >= 100 ? 32 : total >= 10 ? 24 : 18;
-    const r0 = Math.round(r * 0.8);
-    const w = r * 2;
-
-    let html = `<div>
+    // code for creating an SVG donut chart from feature properties
+    function createDonutChart(props) {
+      const offsets = [];
+      const counts = [
+        props.approved,
+        props.pending,
+        props.rejected
+      ];
+      let total = 0;
+      for (const count of counts) {
+        offsets.push(total);
+        total += count;
+      }
+      const fontSize =
+        total >= 1000 ? 22 : total >= 100 ? 20 : total >= 10 ? 18 : 16;
+      const r =
+        total >= 1000 ? 50 : total >= 100 ? 32 : total >= 10 ? 24 : 18;
+      const r0 = Math.round(r * 0.8);
+      const w = r * 2;
+
+      let html = `<div>
             <svg width="${w}" height="${w}" viewbox="0 0 ${w} ${w}" text-anchor="middle" style="font: ${fontSize}px sans-serif; display: block">
             <defs>
               <filter id="shadow">
@@ -323,16 +324,16 @@ <h3 class="my-2">Service Delivery Map</h3>
               </filter>
             </defs>`;
 
-    for (let i = 0; i < counts.length; i++) {
-      html += donutSegment(
-        offsets[i] / total,
-        (offsets[i] + counts[i]) / total,
-        r,
-        r0,
-        colors[i]
-      );
-    }
-    html += `<circle cx="${r}" cy="${r}" r="${r0}" fill="#374151" />
+      for (let i = 0; i < counts.length; i++) {
+        html += donutSegment(
+          offsets[i] / total,
+          (offsets[i] + counts[i]) / total,
+          r,
+          r0,
+          colors[i]
+        );
+      }
+      html += `<circle cx="${r}" cy="${r}" r="${r0}" fill="#374151" />
             <text dominant-baseline="central" transform="translate(${r}, ${r})"
                   fill="white" font-weight="500" filter="url(#shadow)">
                 ${total.toLocaleString()}
@@ -340,44 +341,44 @@ <h3 class="my-2">Service Delivery Map</h3>
             </svg>
             </div>`;
 
-    const el = document.createElement('div');
-    el.innerHTML = html;
-    el.style.cursor = 'pointer';
-
-    // Click handler to zoom and navigate to the cluster
-    el.addEventListener('click', (e) => {
-      map.getSource('visits').getClusterExpansionZoom(
-        props.cluster_id,
-        (err, zoom) => {
-          if (err) return;
-
-          map.easeTo({
-            center: props.coordinates,
-            zoom: zoom
-          });
-        }
-      );
-    });
+      const el = document.createElement('div');
+      el.innerHTML = html;
+      el.style.cursor = 'pointer';
+
+      // Click handler to zoom and navigate to the cluster
+      el.addEventListener('click', (e) => {
+        map.getSource('visits').getClusterExpansionZoom(
+          props.cluster_id,
+          (err, zoom) => {
+            if (err) return;
+
+            map.easeTo({
+              center: props.coordinates,
+              zoom: zoom
+            });
+          }
+        );
+      });
 
-    return el;
-  }
+      return el;
+    }
 
-  function donutSegment(start, end, r, r0, color) {
-    if (end - start === 1) end -= 0.00001;
-    const a0 = 2 * Math.PI * (start - 0.25);
-    const a1 = 2 * Math.PI * (end - 0.25);
-    const x0 = Math.cos(a0),
-      y0 = Math.sin(a0);
-    const x1 = Math.cos(a1),
-      y1 = Math.sin(a1);
-    const largeArc = end - start > 0.5 ? 1 : 0;
-
-    // draw an SVG path
-    return `<path d="M ${r + r0 * x0} ${r + r0 * y0} L ${r + r * x0} ${r + r * y0
-      } A ${r} ${r} 0 ${largeArc} 1 ${r + r * x1} ${r + r * y1} L ${r + r0 * x1
-      } ${r + r0 * y1} A ${r0} ${r0} 0 ${largeArc} 0 ${r + r0 * x0} ${r + r0 * y0
-      }" fill="${color}" opacity="0.85" stroke="#1f2937" stroke-width="1" />`;
-  }
+    function donutSegment(start, end, r, r0, color) {
+      if (end - start === 1) end -= 0.00001;
+      const a0 = 2 * Math.PI * (start - 0.25);
+      const a1 = 2 * Math.PI * (end - 0.25);
+      const x0 = Math.cos(a0),
+        y0 = Math.sin(a0);
+      const x1 = Math.cos(a1),
+        y1 = Math.sin(a1);
+      const largeArc = end - start > 0.5 ? 1 : 0;
+
+      // draw an SVG path
+      return `<path d="M ${r + r0 * x0} ${r + r0 * y0} L ${r + r * x0} ${r + r * y0
+        } A ${r} ${r} 0 ${largeArc} 1 ${r + r * x1} ${r + r * y1} L ${r + r0 * x1
+        } ${r + r0 * y1} A ${r0} ${r0} 0 ${largeArc} 0 ${r + r0 * x0} ${r + r0 * y0
+        }" fill="${color}" opacity="0.85" stroke="#1f2937" stroke-width="1" />`;
+    }
   });
 
 </script>

From cb1d7291bce64f34767bba6df002ee8a2777a61f Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 16:52:35 +0200
Subject: [PATCH 103/165] consistent colors

---
 commcare_connect/reports/views.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index 7c0305b0..9a8dc980 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -257,8 +257,8 @@ def visit_map_data(request):
 def _results_to_geojson(results):
     geojson = {"type": "FeatureCollection", "features": []}
     status_to_color = {
-        "approved": "#00FF00",
-        "rejected": "#FF0000",
+        "approved": "#4ade80",
+        "rejected": "#f87171",
     }
     for i, result in enumerate(results.all()):
         location_str = result.get("location_str")
@@ -286,7 +286,7 @@ def _results_to_geojson(results):
                     key: value for key, value in result.items() if key not in ["gps_location_lat", "gps_location_long"]
                 },
             }
-            color = status_to_color.get(result.get("status", ""), "#FFFF00")
+            color = status_to_color.get(result.get("status", ""), "#fbbf24")
             feature["properties"]["color"] = color
             geojson["features"].append(feature)
 

From 08a6f312e1e66e8bd5f455b4433749fd78764152 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 16:52:41 +0200
Subject: [PATCH 104/165] better comments

---
 .../templates/reports/dashboard.html             | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index eddc7a9f..d4b7ee71 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -134,10 +134,9 @@ <h3 class="my-2">Service Delivery Map</h3>
   let map;
   const mapLoading = document.getElementById('map-loading');
 
-  // Add this function to handle map data refresh
   window.refreshMapData = async () => {
     if (!map) return;
-    console.log('refreshing map data');
+
     mapLoading.classList.remove('d-none');  // Show loading state
 
     const formElement = document.querySelector('#filterForm form');
@@ -145,11 +144,11 @@ <h3 class="my-2">Service Delivery Map</h3>
     const queryString = new URLSearchParams(formData).toString();
 
     try {
-      // Fetch the data first
+      // Fetch the data
       const response = await fetch(`{% url "reports:visit_map_data" %}?${queryString}`);
       const data = await response.json();
 
-      // Then set it on the map source
+      // Set it on the map source
       map.getSource('visits').setData(data);
 
       // Hide loading state after data is set
@@ -169,6 +168,12 @@ <h3 class="my-2">Service Delivery Map</h3>
       zoom: 3,
     });
 
+    // This loads a map with two layers, one is a cluster with donut
+    // charts based on the visit data, inspired heavily by this mapbox example:
+    // https://docs.mapbox.com/mapbox-gl-js/example/cluster-html/
+    // The other is a layer of unclustered points with clickable popups, based on
+    // this example: https://docs.mapbox.com/mapbox-gl-js/example/popup-on-click/
+
     // filters for classifying visits by status
     const approved = ['==', ['get', 'status'], 'approved'];
     const pending = ['all',
@@ -178,7 +183,8 @@ <h3 class="my-2">Service Delivery Map</h3>
     const rejected = ['==', ['get', 'status'], 'rejected'];
 
     // colors to use for the categories
-    const colors = ['#4ade80', '#fbbf24', '#f87171']; // softer green, yellow, red
+    // soft green, yellow, red
+    const colors = ['#4ade80', '#fbbf24', '#f87171'];
 
     map.on('load', () => {
       // Modify the source configuration to include initial filters

From 6b1868561182e7a0bbe7f2a4d333a518a610e22d Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 16:54:42 +0200
Subject: [PATCH 105/165] default to only 30 days of data

---
 commcare_connect/reports/views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index 9a8dc980..7e25f92b 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -204,7 +204,7 @@ def __init__(self, *args, **kwargs):
 
             # Set default dates
             today = date.today()
-            default_from = today - timedelta(days=90)
+            default_from = today - timedelta(days=30)
 
             # Set the default values
             self.data["to_date"] = today.strftime("%Y-%m-%d")

From b164959d835e22e803fb2de117fe31d5d4e53e9c Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 16:54:46 +0200
Subject: [PATCH 106/165] whitespace

---
 commcare_connect/templates/reports/dashboard.html | 1 +
 1 file changed, 1 insertion(+)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index d4b7ee71..3a0e1806 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -251,6 +251,7 @@ <h3 class="my-2">Service Delivery Map</h3>
       // objects for caching and keeping track of HTML marker objects (for performance)
       const markers = {};
       let markersOnScreen = {};
+
       function updateMarkers() {
         const newMarkers = {};
         const features = map.querySourceFeatures('visits');

From 2d8cd8f92225744786bd2fef276940014076d873 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 17:20:26 +0200
Subject: [PATCH 107/165] add minimum cluster size

---
 commcare_connect/templates/reports/dashboard.html | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 3a0e1806..7ab9879e 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -132,6 +132,7 @@ <h3 class="my-2">Service Delivery Map</h3>
 </script>
 <script type="module">
   let map;
+  const minClusterSize = 5;
   const mapLoading = document.getElementById('map-loading');
 
   window.refreshMapData = async () => {
@@ -198,6 +199,7 @@ <h3 class="my-2">Service Delivery Map</h3>
         cluster: true,
         clusterMaxZoom: 12,
         clusterRadius: 80,
+        clusterMinPoints: minClusterSize,
         clusterProperties: {
           // keep separate counts for each status category in a cluster
           'approved': ['+', ['case', approved, 1, 0]],

From 8896d5f3b04f42d109b9d3df9004ad42a863b3ce Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 17:23:34 +0200
Subject: [PATCH 108/165] increase maxZoom

---
 commcare_connect/templates/reports/dashboard.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 7ab9879e..1f0ca0f8 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -197,7 +197,7 @@ <h3 class="my-2">Service Delivery Map</h3>
         type: 'geojson',
         data: `{% url "reports:visit_map_data" %}?${queryString}`,
         cluster: true,
-        clusterMaxZoom: 12,
+        clusterMaxZoom: 14,
         clusterRadius: 80,
         clusterMinPoints: minClusterSize,
         clusterProperties: {

From 91f0f0bb93ea7009e479abcb88ae6c3c67806a86 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 17:49:42 +0200
Subject: [PATCH 109/165] extract some functions to external javascript

---
 commcare_connect/static/js/dashboard.js       | 90 ++++++++++++++++++
 .../templates/reports/dashboard.html          | 94 ++-----------------
 webpack/base.config.js                        |  4 +
 3 files changed, 100 insertions(+), 88 deletions(-)
 create mode 100644 commcare_connect/static/js/dashboard.js

diff --git a/commcare_connect/static/js/dashboard.js b/commcare_connect/static/js/dashboard.js
new file mode 100644
index 00000000..c1d8557f
--- /dev/null
+++ b/commcare_connect/static/js/dashboard.js
@@ -0,0 +1,90 @@
+console.log('dashboard.js loaded');
+
+// colors to use for the categories
+// soft green, yellow, red
+const visitColors = ['#4ade80', '#fbbf24', '#f87171'];
+
+// Function to create a donut chart
+function createDonutChart(props, map) {
+  console.log('createDonutChart', props);
+  const offsets = [];
+  const counts = [props.approved, props.pending, props.rejected];
+  let total = 0;
+  for (const count of counts) {
+    offsets.push(total);
+    total += count;
+  }
+  const fontSize =
+    total >= 1000 ? 22 : total >= 100 ? 20 : total >= 10 ? 18 : 16;
+  const r = total >= 1000 ? 50 : total >= 100 ? 32 : total >= 10 ? 24 : 18;
+  const r0 = Math.round(r * 0.8);
+  const w = r * 2;
+
+  let html = `<div>
+        <svg width="${w}" height="${w}" viewbox="0 0 ${w} ${w}" text-anchor="middle" style="font: ${fontSize}px sans-serif; display: block">
+        <defs>
+          <filter id="shadow">
+            <feDropShadow dx="0" dy="1" stdDeviation="2" flood-opacity="0.3"/>
+          </filter>
+        </defs>`;
+
+  for (let i = 0; i < counts.length; i++) {
+    html += donutSegment(
+      offsets[i] / total,
+      (offsets[i] + counts[i]) / total,
+      r,
+      r0,
+      visitColors[i],
+    );
+  }
+  html += `<circle cx="${r}" cy="${r}" r="${r0}" fill="#374151" />
+        <text dominant-baseline="central" transform="translate(${r}, ${r})"
+              fill="white" font-weight="500" filter="url(#shadow)">
+            ${total.toLocaleString()}
+        </text>
+        </svg>
+        </div>`;
+
+  const el = document.createElement('div');
+  el.innerHTML = html;
+  el.style.cursor = 'pointer';
+
+  // Click handler to zoom and navigate to the cluster
+  el.addEventListener('click', (e) => {
+    map
+      .getSource('visits')
+      .getClusterExpansionZoom(props.cluster_id, (err, zoom) => {
+        if (err) return;
+
+        map.easeTo({
+          center: props.coordinates,
+          zoom: zoom,
+        });
+      });
+  });
+
+  return el;
+}
+
+// Function to create a donut segment
+function donutSegment(start, end, r, r0, color) {
+  if (end - start === 1) end -= 0.00001;
+  const a0 = 2 * Math.PI * (start - 0.25);
+  const a1 = 2 * Math.PI * (end - 0.25);
+  const x0 = Math.cos(a0),
+    y0 = Math.sin(a0);
+  const x1 = Math.cos(a1),
+    y1 = Math.sin(a1);
+  const largeArc = end - start > 0.5 ? 1 : 0;
+
+  // draw an SVG path
+  return `<path d="M ${r + r0 * x0} ${r + r0 * y0} L ${r + r * x0} ${
+    r + r * y0
+  } A ${r} ${r} 0 ${largeArc} 1 ${r + r * x1} ${r + r * y1} L ${r + r0 * x1} ${
+    r + r0 * y1
+  } A ${r0} ${r0} 0 ${largeArc} 0 ${r + r0 * x0} ${
+    r + r0 * y0
+  }" fill="${color}" opacity="0.85" stroke="#1f2937" stroke-width="1" />`;
+}
+
+window.createDonutChart = createDonutChart;
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 1f0ca0f8..9cc75a1b 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -3,6 +3,10 @@
 {% load crispy_forms_tags %}
 {% load django_tables2 %}
 {% block title %}Admin Dashboard{% endblock %}
+{% block javascript %}
+  {{ block.super }}
+  <script src="{% static 'bundles/js/dashboard-bundle.js' %}" ></script>
+{% endblock %}
 {% block content %}
 <h2 class="my-2">Program Dashboard</h2>
 <div class="container" x-data="dashboardStats()">
@@ -183,9 +187,6 @@ <h3 class="my-2">Service Delivery Map</h3>
     ];
     const rejected = ['==', ['get', 'status'], 'rejected'];
 
-    // colors to use for the categories
-    // soft green, yellow, red
-    const colors = ['#4ade80', '#fbbf24', '#f87171'];
 
     map.on('load', () => {
       // Modify the source configuration to include initial filters
@@ -268,11 +269,12 @@ <h3 class="my-2">Service Delivery Map</h3>
 
           let marker = markers[id];
           if (!marker) {
+            console.log("creating donut chart for cluster", id)
             const el = createDonutChart({
               ...props,
               cluster_id: id,  // Make sure cluster_id is passed
               coordinates: coords  // Pass the coordinates
-            });
+            }, map);
             marker = markers[id] = new mapboxgl.Marker({
               element: el
             }).setLngLat(coords);
@@ -304,90 +306,6 @@ <h3 class="my-2">Service Delivery Map</h3>
       mapLoading.classList.add('d-none');
       // Optionally add error messaging here
     });
-
-    // code for creating an SVG donut chart from feature properties
-    function createDonutChart(props) {
-      const offsets = [];
-      const counts = [
-        props.approved,
-        props.pending,
-        props.rejected
-      ];
-      let total = 0;
-      for (const count of counts) {
-        offsets.push(total);
-        total += count;
-      }
-      const fontSize =
-        total >= 1000 ? 22 : total >= 100 ? 20 : total >= 10 ? 18 : 16;
-      const r =
-        total >= 1000 ? 50 : total >= 100 ? 32 : total >= 10 ? 24 : 18;
-      const r0 = Math.round(r * 0.8);
-      const w = r * 2;
-
-      let html = `<div>
-            <svg width="${w}" height="${w}" viewbox="0 0 ${w} ${w}" text-anchor="middle" style="font: ${fontSize}px sans-serif; display: block">
-            <defs>
-              <filter id="shadow">
-                <feDropShadow dx="0" dy="1" stdDeviation="2" flood-opacity="0.3"/>
-              </filter>
-            </defs>`;
-
-      for (let i = 0; i < counts.length; i++) {
-        html += donutSegment(
-          offsets[i] / total,
-          (offsets[i] + counts[i]) / total,
-          r,
-          r0,
-          colors[i]
-        );
-      }
-      html += `<circle cx="${r}" cy="${r}" r="${r0}" fill="#374151" />
-            <text dominant-baseline="central" transform="translate(${r}, ${r})"
-                  fill="white" font-weight="500" filter="url(#shadow)">
-                ${total.toLocaleString()}
-            </text>
-            </svg>
-            </div>`;
-
-      const el = document.createElement('div');
-      el.innerHTML = html;
-      el.style.cursor = 'pointer';
-
-      // Click handler to zoom and navigate to the cluster
-      el.addEventListener('click', (e) => {
-        map.getSource('visits').getClusterExpansionZoom(
-          props.cluster_id,
-          (err, zoom) => {
-            if (err) return;
-
-            map.easeTo({
-              center: props.coordinates,
-              zoom: zoom
-            });
-          }
-        );
-      });
-
-      return el;
-    }
-
-    function donutSegment(start, end, r, r0, color) {
-      if (end - start === 1) end -= 0.00001;
-      const a0 = 2 * Math.PI * (start - 0.25);
-      const a1 = 2 * Math.PI * (end - 0.25);
-      const x0 = Math.cos(a0),
-        y0 = Math.sin(a0);
-      const x1 = Math.cos(a1),
-        y1 = Math.sin(a1);
-      const largeArc = end - start > 0.5 ? 1 : 0;
-
-      // draw an SVG path
-      return `<path d="M ${r + r0 * x0} ${r + r0 * y0} L ${r + r * x0} ${r + r * y0
-        } A ${r} ${r} 0 ${largeArc} 1 ${r + r * x1} ${r + r * y1} L ${r + r0 * x1
-        } ${r + r0 * y1} A ${r0} ${r0} 0 ${largeArc} 0 ${r + r0 * x0} ${r + r0 * y0
-        }" fill="${color}" opacity="0.85" stroke="#1f2937" stroke-width="1" />`;
-    }
   });
 
 </script>
diff --git a/webpack/base.config.js b/webpack/base.config.js
index 74235cf8..8f1fe4de 100644
--- a/webpack/base.config.js
+++ b/webpack/base.config.js
@@ -8,6 +8,10 @@ module.exports = {
   context: path.join(__dirname, '../'),
   entry: {
     project: path.resolve(__dirname, '../commcare_connect/static/js/project'),
+    dashboard: path.resolve(
+      __dirname,
+      '../commcare_connect/static/js/dashboard',
+    ),
     vendors: path.resolve(__dirname, '../commcare_connect/static/js/vendors'),
   },
   output: {

From be0f47d3d880ba8fa74e726fd3b966aa840f85bc Mon Sep 17 00:00:00 2001
From: Sravan Reddy <sravfeyn@gmail.com>
Date: Tue, 12 Nov 2024 21:20:42 +0530
Subject: [PATCH 110/165] Publish android assetlinks for deeplinking

---
 config/urls.py  |  3 +++
 config/views.py | 30 ++++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+)
 create mode 100644 config/views.py

diff --git a/config/urls.py b/config/urls.py
index 9eca1758..4c368e0f 100644
--- a/config/urls.py
+++ b/config/urls.py
@@ -9,9 +9,12 @@
 
 from commcare_connect.organization.views import organization_create
 
+from . import views
+
 urlpatterns = [
     path("", TemplateView.as_view(template_name="pages/home.html"), name="home"),
     path("about/", TemplateView.as_view(template_name="pages/about.html"), name="about"),
+    path(".well-known/assetlinks.json", views.assetlinks_json, name="assetlinks_json"),
     # Django Admin, use {% url 'admin:index' %}
     path(settings.ADMIN_URL, admin.site.urls),
     path("o/", include("oauth2_provider.urls", namespace="oauth2_provider")),
diff --git a/config/views.py b/config/views.py
new file mode 100644
index 00000000..3b7fc766
--- /dev/null
+++ b/config/views.py
@@ -0,0 +1,30 @@
+from django.http import JsonResponse
+
+
+def assetlinks_json(request):
+    assetfile = [
+        {
+            "relation": ["delegate_permission/common.handle_all_urls"],
+            "target": {
+                "namespace": "android_app",
+                "package_name": "org.commcare.dalvik",
+                "sha256_cert_fingerprints":
+                [
+                    "88:57:18:F8:E8:7D:74:04:97:AE:83:65:74:ED:EF:10:40:D9:4C:E2:54:F0:E0:40:64:77:96:7F:D1:39:F9:81",
+                    "89:55:DF:D8:0E:66:63:06:D2:6D:88:A4:A3:88:A4:D9:16:5A:C4:1A:7E:E1:C6:78:87:00:37:55:93:03:7B:03"
+                ]
+            }
+        },
+        {
+            "relation": ["delegate_permission/common.handle_all_urls"],
+            "target": {
+                "namespace": "android_app",
+                "package_name": "org.commcare.dalvik.debug",
+                "sha256_cert_fingerprints":
+                [
+                    "88:57:18:F8:E8:7D:74:04:97:AE:83:65:74:ED:EF:10:40:D9:4C:E2:54:F0:E0:40:64:77:96:7F:D1:39:F9:81"
+                ]
+            }
+        },
+    ]
+    return JsonResponse(assetfile, safe=False)

From 4b30441429096705b4d0cbf9af9b430d7448a980 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 17:53:27 +0200
Subject: [PATCH 111/165] extract updateMarkers

---
 commcare_connect/static/js/dashboard.js       | 44 +++++++++++++++++++
 .../templates/reports/dashboard.html          | 41 +----------------
 2 files changed, 45 insertions(+), 40 deletions(-)

diff --git a/commcare_connect/static/js/dashboard.js b/commcare_connect/static/js/dashboard.js
index c1d8557f..9318a825 100644
--- a/commcare_connect/static/js/dashboard.js
+++ b/commcare_connect/static/js/dashboard.js
@@ -4,6 +4,49 @@ console.log('dashboard.js loaded');
 // soft green, yellow, red
 const visitColors = ['#4ade80', '#fbbf24', '#f87171'];
 
+// after the GeoJSON data is loaded, update markers on the screen on every frame
+// objects for caching and keeping track of HTML marker objects (for performance)
+const markers = {};
+let markersOnScreen = {};
+
+function updateMarkers(map) {
+  const newMarkers = {};
+  const features = map.querySourceFeatures('visits');
+
+  // for every cluster on the screen, create an HTML marker for it (if we didn't yet),
+  // and add it to the map if it's not there already
+  for (const feature of features) {
+    const coords = feature.geometry.coordinates;
+    const props = feature.properties;
+    if (!props.cluster) continue;
+    const id = props.cluster_id;
+
+    let marker = markers[id];
+    if (!marker) {
+      console.log('creating donut chart for cluster', id);
+      const el = createDonutChart(
+        {
+          ...props,
+          cluster_id: id, // Make sure cluster_id is passed
+          coordinates: coords, // Pass the coordinates
+        },
+        map,
+      );
+      marker = markers[id] = new mapboxgl.Marker({
+        element: el,
+      }).setLngLat(coords);
+    }
+    newMarkers[id] = marker;
+
+    if (!markersOnScreen[id]) marker.addTo(map);
+  }
+  // for every marker we've added previously, remove those that are no longer visible
+  for (const id in markersOnScreen) {
+    if (!newMarkers[id]) markersOnScreen[id].remove();
+  }
+  markersOnScreen = newMarkers;
+}
+
 // Function to create a donut chart
 function createDonutChart(props, map) {
   console.log('createDonutChart', props);
@@ -87,4 +130,5 @@ function donutSegment(start, end, r, r0, color) {
   }" fill="${color}" opacity="0.85" stroke="#1f2937" stroke-width="1" />`;
 }
 
+window.updateMarkers = updateMarkers;
 window.createDonutChart = createDonutChart;
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 9cc75a1b..2a457188 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -250,49 +250,10 @@ <h3 class="my-2">Service Delivery Map</h3>
           .addTo(map);
       });
 
-      // after the GeoJSON data is loaded, update markers on the screen on every frame
-      // objects for caching and keeping track of HTML marker objects (for performance)
-      const markers = {};
-      let markersOnScreen = {};
-
-      function updateMarkers() {
-        const newMarkers = {};
-        const features = map.querySourceFeatures('visits');
-
-        // for every cluster on the screen, create an HTML marker for it (if we didn't yet),
-        // and add it to the map if it's not there already
-        for (const feature of features) {
-          const coords = feature.geometry.coordinates;
-          const props = feature.properties;
-          if (!props.cluster) continue;
-          const id = props.cluster_id;
-
-          let marker = markers[id];
-          if (!marker) {
-            console.log("creating donut chart for cluster", id)
-            const el = createDonutChart({
-              ...props,
-              cluster_id: id,  // Make sure cluster_id is passed
-              coordinates: coords  // Pass the coordinates
-            }, map);
-            marker = markers[id] = new mapboxgl.Marker({
-              element: el
-            }).setLngLat(coords);
-          }
-          newMarkers[id] = marker;
-
-          if (!markersOnScreen[id]) marker.addTo(map);
-        }
-        // for every marker we've added previously, remove those that are no longer visible
-        for (const id in markersOnScreen) {
-          if (!newMarkers[id]) markersOnScreen[id].remove();
-        }
-        markersOnScreen = newMarkers;
-      }
 
       map.on('render', () => {
         if (!map.isSourceLoaded('visits')) return;
-        updateMarkers();
+        updateMarkers(map);
       });
 
       // Hide loading overlay when initial map load is complete

From ee37856a2577a4ee5582dffc455fce3e218f7e14 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 17:53:45 +0200
Subject: [PATCH 112/165] remove logging statments

---
 commcare_connect/static/js/dashboard.js | 2 --
 1 file changed, 2 deletions(-)

diff --git a/commcare_connect/static/js/dashboard.js b/commcare_connect/static/js/dashboard.js
index 9318a825..d73535b3 100644
--- a/commcare_connect/static/js/dashboard.js
+++ b/commcare_connect/static/js/dashboard.js
@@ -23,7 +23,6 @@ function updateMarkers(map) {
 
     let marker = markers[id];
     if (!marker) {
-      console.log('creating donut chart for cluster', id);
       const el = createDonutChart(
         {
           ...props,
@@ -49,7 +48,6 @@ function updateMarkers(map) {
 
 // Function to create a donut chart
 function createDonutChart(props, map) {
-  console.log('createDonutChart', props);
   const offsets = [];
   const counts = [props.approved, props.pending, props.rejected];
   let total = 0;

From 8330b101f22176a34a65eab52e380fde680efee7 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 17:56:36 +0200
Subject: [PATCH 113/165] fix tests

---
 commcare_connect/reports/tests/test_reports.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/commcare_connect/reports/tests/test_reports.py b/commcare_connect/reports/tests/test_reports.py
index f85ae168..441090f7 100644
--- a/commcare_connect/reports/tests/test_reports.py
+++ b/commcare_connect/reports/tests/test_reports.py
@@ -106,7 +106,7 @@ def all(self):
     assert feature1["geometry"]["coordinates"] == [10.123, 20.456]
     assert feature1["properties"]["status"] == "approved"
     assert feature1["properties"]["other_field"] == "value1"
-    assert feature1["properties"]["color"] == "#00FF00"
+    assert feature1["properties"]["color"] == "#4ade80"
 
     # Check the second feature
     feature2 = geojson["features"][1]
@@ -115,7 +115,7 @@ def all(self):
     assert feature2["geometry"]["coordinates"] == [30.789, 40.012]
     assert feature2["properties"]["status"] == "rejected"
     assert feature2["properties"]["other_field"] == "value2"
-    assert feature2["properties"]["color"] == "#FF0000"
+    assert feature2["properties"]["color"] == "#f87171"
 
     # Check that the other cases are not included
     assert all(f["properties"]["other_field"] not in ["value3", "value4", "value5"] for f in geojson["features"])

From fa092aa4a52df2beb49600f4e90d196167046e42 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 17:58:14 +0200
Subject: [PATCH 114/165] put map in a container

---
 .../templates/reports/dashboard.html          | 20 ++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 2a457188..bafdd0aa 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -74,15 +74,17 @@ <h2 class="my-2">Program Dashboard</h2>
     </div>
   </div>
 </div>
-<div class="row">
-  <div class="col-12">
-    <h3 class="my-2">Service Delivery Map</h3>
-    <div class="position-relative">
-      <div id="map" class="mt-3 mb-2 rounded border" style="height: 900px;"></div>
-      <div id="map-loading"
-        class="position-absolute top-0 start-0 w-100 h-100 bg-dark bg-opacity-50 d-flex justify-content-center align-items-center">
-        <div class="spinner-border text-light" role="status">
-          <span class="visually-hidden">Loading map...</span>
+<div class="container">
+  <div class="row">
+    <div class="col-12">
+      <h3 class="my-2">Service Delivery Map</h3>
+      <div class="position-relative">
+        <div id="map" class="mt-3 mb-2 rounded border" style="height: 500px;"></div>
+        <div id="map-loading"
+          class="position-absolute top-0 start-0 w-100 h-100 bg-dark bg-opacity-50 d-flex justify-content-center align-items-center">
+          <div class="spinner-border text-light" role="status">
+            <span class="visually-hidden">Loading map...</span>
+          </div>
         </div>
       </div>
     </div>

From 09639ce8b8cb3ba19069b7c5c0c7548e4c9171cf Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 13 Nov 2024 07:56:12 +0530
Subject: [PATCH 115/165] added test coverage

---
 .../program/tests/test_helpers.py             | 24 +++++++++++++------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/commcare_connect/program/tests/test_helpers.py b/commcare_connect/program/tests/test_helpers.py
index c03e0f1e..736ecd89 100644
--- a/commcare_connect/program/tests/test_helpers.py
+++ b/commcare_connect/program/tests/test_helpers.py
@@ -55,7 +55,7 @@ def create_user_with_visit(self, visit_status, visit_date, flagged=False, create
 class TestGetAnnotatedManagedOpportunity(BaseManagedOpportunityTest):
     @pytest.mark.parametrize(
         "scenario, visit_statuses, passing_assessments, expected_invited,"
-        " expected_passing, expected_delivery, expected_conversion",
+        " expected_passing, expected_delivery, expected_conversion, expected_avg_time_to_convert",
         [
             (
                 "basic_scenario",
@@ -65,9 +65,10 @@ class TestGetAnnotatedManagedOpportunity(BaseManagedOpportunityTest):
                 3,
                 2,
                 66.67,
+                timedelta(days=1),
             ),
-            ("empty_scenario", [], [], 0, 0, 0, 0.0),
-            ("multiple_visits_scenario", [VisitValidationStatus.pending], [True], 1, 1, 1, 100.0),
+            ("empty_scenario", [], [], 0, 0, 0, 0.0, None),
+            ("multiple_visits_scenario", [VisitValidationStatus.pending], [True], 1, 1, 1, 100.0, timedelta(days=1)),
             (
                 "excluded_statuses",
                 [VisitValidationStatus.over_limit, VisitValidationStatus.trial],
@@ -76,6 +77,7 @@ class TestGetAnnotatedManagedOpportunity(BaseManagedOpportunityTest):
                 2,
                 0,
                 0.0,
+                None,
             ),
             (
                 "failed_assessments",
@@ -85,6 +87,7 @@ class TestGetAnnotatedManagedOpportunity(BaseManagedOpportunityTest):
                 1,
                 2,
                 100.0,
+                timedelta(days=1),
             ),
         ],
     )
@@ -97,6 +100,7 @@ def test_scenarios(
         expected_passing,
         expected_delivery,
         expected_conversion,
+        expected_avg_time_to_convert,
     ):
         for i, visit_status in enumerate(visit_statuses):
             user = self.create_user_with_access(visit_status=visit_status, passed_assessment=passing_assessments[i])
@@ -115,10 +119,16 @@ def test_scenarios(
         opps = get_annotated_managed_opportunity(self.program)
         assert len(opps) == 1
         annotated_opp = opps[0]
-        assert annotated_opp.workers_invited == expected_invited, f"Failed in {scenario}"
-        assert annotated_opp.workers_passing_assessment == expected_passing, f"Failed in {scenario}"
-        assert annotated_opp.workers_starting_delivery == expected_delivery, f"Failed in {scenario}"
-        assert pytest.approx(annotated_opp.percentage_conversion, 0.01) == expected_conversion, f"Failed in {scenario}"
+        assert annotated_opp.workers_invited == expected_invited
+        assert annotated_opp.workers_passing_assessment == expected_passing
+        assert annotated_opp.workers_starting_delivery == expected_delivery
+        assert pytest.approx(annotated_opp.percentage_conversion, 0.01) == expected_conversion
+
+        if expected_avg_time_to_convert:
+            diff = abs(annotated_opp.average_time_to_convert - expected_avg_time_to_convert)
+            assert diff < timedelta(minutes=1)
+        else:
+            assert annotated_opp.average_time_to_convert is None
 
 
 @pytest.mark.django_db

From 89443e512c037d74fd530171011c7b944cf7498c Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 13 Nov 2024 08:29:14 +0530
Subject: [PATCH 116/165] changed test as per new changes

---
 commcare_connect/program/tests/test_helpers.py | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/commcare_connect/program/tests/test_helpers.py b/commcare_connect/program/tests/test_helpers.py
index 736ecd89..61bdaa8c 100644
--- a/commcare_connect/program/tests/test_helpers.py
+++ b/commcare_connect/program/tests/test_helpers.py
@@ -138,7 +138,7 @@ class TestDeliveryPerformanceReport(BaseManagedOpportunityTest):
 
     @pytest.mark.parametrize(
         "scenario, visit_statuses, visit_date, flagged_statuses, expected_active_workers, "
-        "expected_total_workers, expected_flags, expected_records_flagged_percentage,"
+        "expected_total_workers, expected_records_flagged_percentage,"
         "total_payment_units_with_flags,total_payment_since_start_date, delivery_per_day_per_worker",
         [
             (
@@ -148,9 +148,8 @@ class TestDeliveryPerformanceReport(BaseManagedOpportunityTest):
                 [True] * 3 + [False] * 2,
                 5,
                 5,
-                2,
-                40.0,
-                2,
+                60.0,
+                3,
                 5,
                 1.0,
             ),
@@ -166,7 +165,6 @@ class TestDeliveryPerformanceReport(BaseManagedOpportunityTest):
                 [False] * 4,
                 2,
                 4,
-                0,
                 0.0,
                 0,
                 2,
@@ -179,7 +177,6 @@ class TestDeliveryPerformanceReport(BaseManagedOpportunityTest):
                 [False, True],
                 2,
                 2,
-                1,
                 50.0,
                 1,
                 2,
@@ -192,7 +189,6 @@ class TestDeliveryPerformanceReport(BaseManagedOpportunityTest):
                 [False, False],
                 0,
                 0,
-                0,
                 0.0,
                 0,
                 0,
@@ -210,9 +206,8 @@ class TestDeliveryPerformanceReport(BaseManagedOpportunityTest):
                 [True] * 4,
                 3,
                 3,
-                2,
-                66.67,
-                2,
+                100,
+                3,
                 3,
                 1.0,
             ),
@@ -226,7 +221,6 @@ def test_delivery_performance_report_scenarios(
         flagged_statuses,
         expected_active_workers,
         expected_total_workers,
-        expected_flags,
         expected_records_flagged_percentage,
         total_payment_units_with_flags,
         total_payment_since_start_date,
@@ -247,7 +241,6 @@ def test_delivery_performance_report_scenarios(
         assert len(opps) == 1
         assert opps[0].active_workers == expected_active_workers
         assert opps[0].total_workers_starting_delivery == expected_total_workers
-        assert opps[0].total_payment_units_with_flags == expected_flags
         assert opps[0].records_flagged_percentage == expected_records_flagged_percentage
         assert opps[0].total_payment_units_with_flags == total_payment_units_with_flags
         assert opps[0].total_payment_since_start_date == total_payment_since_start_date

From f64430ad9d7ff06829b561ddb32ba67b336d7f18 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 14:40:58 +0200
Subject: [PATCH 117/165] dummy chart implementation

---
 .../templates/reports/dashboard.html          | 72 ++++++++++++++++++-
 1 file changed, 70 insertions(+), 2 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index bafdd0aa..f919dba1 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -6,6 +6,7 @@
 {% block javascript %}
   {{ block.super }}
   <script src="{% static 'bundles/js/dashboard-bundle.js' %}" ></script>
+  <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
 {% endblock %}
 {% block content %}
 <h2 class="my-2">Program Dashboard</h2>
@@ -76,7 +77,7 @@ <h2 class="my-2">Program Dashboard</h2>
 </div>
 <div class="container">
   <div class="row">
-    <div class="col-12">
+    <div class="col-12 col-md-6">
       <h3 class="my-2">Service Delivery Map</h3>
       <div class="position-relative">
         <div id="map" class="mt-3 mb-2 rounded border" style="height: 500px;"></div>
@@ -88,6 +89,10 @@ <h3 class="my-2">Service Delivery Map</h3>
         </div>
       </div>
     </div>
+    <div class="col-12 col-md-6">
+      <h3 class="my-2">Visits over Time</h3>
+      <canvas id="visits-over-time" class="mt-3 mb-2 rounded border" style="height: 500px;"></canvas>
+    </div>
   </div>
 </div>
 {% endblock content %}
@@ -172,7 +177,7 @@ <h3 class="my-2">Service Delivery Map</h3>
       container: 'map',
       style: 'mapbox://styles/mapbox/dark-v11',
       center: [20, 0], // Centered on Africa (roughly central coordinates)
-      zoom: 3,
+      zoom: 2,
     });
 
     // This loads a map with two layers, one is a cluster with donut
@@ -269,6 +274,69 @@ <h3 class="my-2">Service Delivery Map</h3>
       mapLoading.classList.add('d-none');
       // Optionally add error messaging here
     });
+
+    // Generate last 30 days of dummy data
+    const generateDummyData = () => {
+      const data = [];
+      const labels = [];
+      const today = new Date();
+
+      for (let i = 29; i >= 0; i--) {
+        const date = new Date(today);
+        date.setDate(date.getDate() - i);
+        labels.push(date.toLocaleDateString('en-US', { month: 'short', day: 'numeric' }));
+        // Random number between 10 and 50
+        data.push(Math.floor(Math.random() * 40) + 10);
+      }
+      return { labels, data };
+    };
+
+    const { labels, data } = generateDummyData();
+
+    const ctx = document.getElementById('visits-over-time');
+    const chart = new Chart(ctx, {
+      type: 'line',
+      data: {
+        labels: labels,
+        datasets: [{
+          label: 'Total Visits',
+          data: data,
+          borderColor: 'rgb(75, 192, 192)',
+          backgroundColor: 'rgba(75, 192, 192, 0.2)',
+          fill: true,
+          tension: 0.3
+        }]
+      },
+      options: {
+        responsive: false,
+        maintainAspectRatio: false,
+        plugins: {
+          title: {
+            display: true,
+            text: 'Visits by Date'
+          },
+          tooltip: {
+            mode: 'index',
+            intersect: false,
+          }
+        },
+        scales: {
+          y: {
+            beginAtZero: true,
+            title: {
+              display: true,
+              text: 'Number of Visits'
+            }
+          },
+          x: {
+            title: {
+              display: true,
+              text: 'Date'
+            }
+          }
+        }
+      }
+    });
   });
 
 </script>

From c9c76c641b80acb002b60c9069f5adbdf02113ea Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 14:47:44 +0200
Subject: [PATCH 118/165] fix sizing, responsiveness

---
 commcare_connect/templates/reports/dashboard.html | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index f919dba1..d9586f90 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -91,7 +91,9 @@ <h3 class="my-2">Service Delivery Map</h3>
     </div>
     <div class="col-12 col-md-6">
       <h3 class="my-2">Visits over Time</h3>
-      <canvas id="visits-over-time" class="mt-3 mb-2 rounded border" style="height: 500px;"></canvas>
+      <div style="position: relative; height: 500px; width: 100%;">
+        <canvas id="visits-over-time"></canvas>
+      </div>
     </div>
   </div>
 </div>
@@ -288,6 +290,7 @@ <h3 class="my-2">Visits over Time</h3>
         // Random number between 10 and 50
         data.push(Math.floor(Math.random() * 40) + 10);
       }
+      console.log({labels, data});
       return { labels, data };
     };
 
@@ -308,7 +311,7 @@ <h3 class="my-2">Visits over Time</h3>
         }]
       },
       options: {
-        responsive: false,
+        responsive: true,
         maintainAspectRatio: false,
         plugins: {
           title: {

From aabb39fcb7204d7fef0c1a43003e6fd03ace2c71 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 14:49:24 +0200
Subject: [PATCH 119/165] smaller cluster radius

---
 commcare_connect/templates/reports/dashboard.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index d9586f90..903a49d4 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -208,7 +208,7 @@ <h3 class="my-2">Visits over Time</h3>
         data: `{% url "reports:visit_map_data" %}?${queryString}`,
         cluster: true,
         clusterMaxZoom: 14,
-        clusterRadius: 80,
+        clusterRadius: 40,
         clusterMinPoints: minClusterSize,
         clusterProperties: {
           // keep separate counts for each status category in a cluster

From bcce9ac1a2f71157ec90f5271ff2130d36fd2aaf Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 15:00:23 +0200
Subject: [PATCH 120/165] add api for graphs

---
 commcare_connect/reports/urls.py              |   1 +
 commcare_connect/reports/views.py             |  41 +++++-
 .../templates/reports/dashboard.html          | 129 ++++++++++--------
 3 files changed, 115 insertions(+), 56 deletions(-)

diff --git a/commcare_connect/reports/urls.py b/commcare_connect/reports/urls.py
index cd05d994..bb8a9709 100644
--- a/commcare_connect/reports/urls.py
+++ b/commcare_connect/reports/urls.py
@@ -9,4 +9,5 @@
     path("delivery_stats", view=views.DeliveryStatsReportView.as_view(), name="delivery_stats_report"),
     path("api/visit_map_data/", views.visit_map_data, name="visit_map_data"),
     path("api/dashboard_stats/", views.dashboard_stats_api, name="dashboard_stats_api"),
+    path("api/visits_over_time/", views.visits_over_time_api, name="visits_over_time_api"),
 ]
diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index 7e25f92b..93cb9d27 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -8,7 +8,7 @@
 from django.conf import settings
 from django.contrib.auth.decorators import login_required, user_passes_test
 from django.contrib.auth.mixins import LoginRequiredMixin, UserPassesTestMixin
-from django.db.models import Max, Q, Sum
+from django.db.models import Count, Max, Q, Sum
 from django.http import JsonResponse
 from django.shortcuts import render
 from django.urls import reverse
@@ -418,3 +418,42 @@ def dashboard_stats_api(request):
             "percent_verified": f"{percent_verified:.1f}%",
         }
     )
+
+
+@login_required
+@user_passes_test(lambda u: u.is_superuser)
+def visits_over_time_api(request):
+    filterset = DashboardFilters(request.GET)
+
+    # Use the filtered queryset
+    queryset = UserVisit.objects.all()
+
+    if filterset.is_valid():
+        queryset = filterset.filter_queryset(queryset)
+        from_date = filterset.form.cleaned_data["from_date"]
+        to_date = filterset.form.cleaned_data["to_date"]
+    else:
+        to_date = datetime.now().date()
+        from_date = to_date - timedelta(days=30)
+        queryset = queryset.filter(visit_date__gte=from_date, visit_date__lte=to_date)
+
+    # Aggregate visits by date
+    visits_by_date = queryset.values("visit_date").annotate(count=Count("id")).order_by("visit_date")
+
+    # Create a complete date range with 0s for missing dates
+    date_counts = {result["visit_date"]: result["count"] for result in visits_by_date}
+
+    data = []
+    labels = []
+    current_date = from_date
+    while current_date <= to_date:
+        labels.append(current_date.strftime("%b %d"))
+        data.append(date_counts.get(current_date, 0))
+        current_date += timedelta(days=1)
+
+    return JsonResponse(
+        {
+            "labels": labels,
+            "data": data,
+        }
+    )
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 903a49d4..62de61d3 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -277,68 +277,87 @@ <h3 class="my-2">Visits over Time</h3>
       // Optionally add error messaging here
     });
 
-    // Generate last 30 days of dummy data
-    const generateDummyData = () => {
-      const data = [];
-      const labels = [];
-      const today = new Date();
 
-      for (let i = 29; i >= 0; i--) {
-        const date = new Date(today);
-        date.setDate(date.getDate() - i);
-        labels.push(date.toLocaleDateString('en-US', { month: 'short', day: 'numeric' }));
-        // Random number between 10 and 50
-        data.push(Math.floor(Math.random() * 40) + 10);
-      }
-      console.log({labels, data});
-      return { labels, data };
-    };
+  });
+  // Update chart when page loads
+  window.addEventListener('DOMContentLoaded', () => {
+        // Replace the chart initialization code with this:
+        const ctx = document.getElementById('visits-over-time');
+    let visitsChart;
 
-    const { labels, data } = generateDummyData();
+    async function updateVisitsChart() {
+      console.log('updating visits chart');
+      try {
+        // Get form data for filters
+        const formElement = document.querySelector('#filterForm form');
+        const formData = new FormData(formElement);
+        const queryString = new URLSearchParams(formData).toString();
 
-    const ctx = document.getElementById('visits-over-time');
-    const chart = new Chart(ctx, {
-      type: 'line',
-      data: {
-        labels: labels,
-        datasets: [{
-          label: 'Total Visits',
-          data: data,
-          borderColor: 'rgb(75, 192, 192)',
-          backgroundColor: 'rgba(75, 192, 192, 0.2)',
-          fill: true,
-          tension: 0.3
-        }]
-      },
-      options: {
-        responsive: true,
-        maintainAspectRatio: false,
-        plugins: {
-          title: {
-            display: true,
-            text: 'Visits by Date'
-          },
-          tooltip: {
-            mode: 'index',
-            intersect: false,
-          }
-        },
-        scales: {
-          y: {
-            beginAtZero: true,
-            title: {
-              display: true,
-              text: 'Number of Visits'
-            }
+        // Fetch data from API
+        const response = await fetch(`{% url 'reports:visits_over_time_api' %}?${queryString}`);
+        console.log('response', response);
+        const chartData = await response.json();
+
+        if (visitsChart) {
+          visitsChart.destroy();
+        }
+
+        visitsChart = new Chart(ctx, {
+          type: 'line',
+          data: {
+            labels: chartData.labels,
+            datasets: [{
+              label: 'Total Visits',
+              data: chartData.data,
+              borderColor: 'rgb(75, 192, 192)',
+              backgroundColor: 'rgba(75, 192, 192, 0.2)',
+              fill: true,
+              tension: 0.3
+            }]
           },
-          x: {
-            title: {
-              display: true,
-              text: 'Date'
+          options: {
+            responsive: true,
+            maintainAspectRatio: false,
+            plugins: {
+              title: {
+                display: true,
+                text: 'Visits by Date'
+              },
+              tooltip: {
+                mode: 'index',
+                intersect: false,
+              }
+            },
+            scales: {
+              y: {
+                beginAtZero: true,
+                title: {
+                  display: true,
+                  text: 'Number of Visits'
+                }
+              },
+              x: {
+                title: {
+                  display: true,
+                  text: 'Date'
+                }
+              }
             }
           }
-        }
+        });
+      } catch (error) {
+        console.error('Error updating visits chart:', error);
       }
+    }
+
+
+    console.log('updating visits chart on load');
+    updateVisitsChart();
+
+    // Update chart when filters change
+    const formElement = document.querySelector('#filterForm form');
+    formElement.querySelectorAll('select, input').forEach(input => {
+      input.addEventListener('change', updateVisitsChart);
     });
   });
 

From 1cc46a395b130c15cac96ed37bc695290ddfa452 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 15:08:57 +0200
Subject: [PATCH 121/165] get program breakdown kinda working

---
 commcare_connect/reports/views.py             | 44 ++++++++++++-------
 .../templates/reports/dashboard.html          | 35 ++++++++++-----
 2 files changed, 53 insertions(+), 26 deletions(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index 93cb9d27..ed197872 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -424,10 +424,8 @@ def dashboard_stats_api(request):
 @user_passes_test(lambda u: u.is_superuser)
 def visits_over_time_api(request):
     filterset = DashboardFilters(request.GET)
-
-    # Use the filtered queryset
     queryset = UserVisit.objects.all()
-
+    # Use the filtered queryset if available, else use last 30 days
     if filterset.is_valid():
         queryset = filterset.filter_queryset(queryset)
         from_date = filterset.form.cleaned_data["from_date"]
@@ -437,23 +435,39 @@ def visits_over_time_api(request):
         from_date = to_date - timedelta(days=30)
         queryset = queryset.filter(visit_date__gte=from_date, visit_date__lte=to_date)
 
-    # Aggregate visits by date
-    visits_by_date = queryset.values("visit_date").annotate(count=Count("id")).order_by("visit_date")
+    # Get visits by date and program in a single query
+    visits_by_program = (
+        queryset.values("visit_date", "opportunity__delivery_type__name")
+        .annotate(count=Count("id"))
+        .order_by("visit_date", "opportunity__delivery_type__name")
+    )
 
-    # Create a complete date range with 0s for missing dates
-    date_counts = {result["visit_date"]: result["count"] for result in visits_by_date}
+    # Create lookup dict for program data
+    program_data = {}
+    for visit in visits_by_program:
+        program_name = visit["opportunity__delivery_type__name"]
+        if program_name not in program_data:
+            program_data[program_name] = {}
+        program_data[program_name][visit["visit_date"]] = visit["count"]
 
-    data = []
+    # Create labels and datasets
     labels = []
+    datasets = []
     current_date = from_date
+
+    # Build labels array
     while current_date <= to_date:
         labels.append(current_date.strftime("%b %d"))
-        data.append(date_counts.get(current_date, 0))
         current_date += timedelta(days=1)
 
-    return JsonResponse(
-        {
-            "labels": labels,
-            "data": data,
-        }
-    )
+    # Build dataset for each program
+    for program_name in program_data.keys():
+        data = []
+        current_date = from_date
+        while current_date <= to_date:
+            data.append(program_data[program_name].get(current_date, 0))
+            current_date += timedelta(days=1)
+
+        datasets.append({"name": program_name, "data": data})
+
+    return JsonResponse({"labels": labels, "datasets": datasets})
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 62de61d3..4ec0329e 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -286,34 +286,46 @@ <h3 class="my-2">Visits over Time</h3>
     let visitsChart;
 
     async function updateVisitsChart() {
-      console.log('updating visits chart');
       try {
-        // Get form data for filters
         const formElement = document.querySelector('#filterForm form');
         const formData = new FormData(formElement);
         const queryString = new URLSearchParams(formData).toString();
 
-        // Fetch data from API
         const response = await fetch(`{% url 'reports:visits_over_time_api' %}?${queryString}`);
-        console.log('response', response);
         const chartData = await response.json();
 
         if (visitsChart) {
           visitsChart.destroy();
         }
 
+        // Assuming the API now returns data in this format:
+        // {
+        //   labels: ['2024-01', '2024-02', ...],
+        //   datasets: [
+        //     { name: 'Program A', data: [10, 20, ...] },
+        //     { name: 'Program B', data: [15, 25, ...] },
+        //   ]
+        // }
+
+        const colors = [
+          { border: 'rgb(75, 192, 192)', background: 'rgba(75, 192, 192, 0.2)' },
+          { border: 'rgb(255, 99, 132)', background: 'rgba(255, 99, 132, 0.2)' },
+          { border: 'rgb(255, 205, 86)', background: 'rgba(255, 205, 86, 0.2)' },
+          { border: 'rgb(54, 162, 235)', background: 'rgba(54, 162, 235, 0.2)' },
+        ];
+
         visitsChart = new Chart(ctx, {
           type: 'line',
           data: {
             labels: chartData.labels,
-            datasets: [{
-              label: 'Total Visits',
-              data: chartData.data,
-              borderColor: 'rgb(75, 192, 192)',
-              backgroundColor: 'rgba(75, 192, 192, 0.2)',
+            datasets: chartData.datasets.map((dataset, index) => ({
+              label: dataset.name,
+              data: dataset.data,
+              borderColor: colors[index % colors.length].border,
+              backgroundColor: colors[index % colors.length].background,
               fill: true,
               tension: 0.3
-            }]
+            }))
           },
           options: {
             responsive: true,
@@ -321,7 +333,7 @@ <h3 class="my-2">Visits over Time</h3>
             plugins: {
               title: {
                 display: true,
-                text: 'Visits by Date'
+                text: 'Visits by Program'
               },
               tooltip: {
                 mode: 'index',
@@ -331,6 +343,7 @@ <h3 class="my-2">Visits over Time</h3>
             scales: {
               y: {
                 beginAtZero: true,
+                stacked: true,
                 title: {
                   display: true,
                   text: 'Number of Visits'

From 38e0f134716c7c1d363d874b572a04dda9b68989 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 15:11:23 +0200
Subject: [PATCH 122/165] add "unknown" label

---
 commcare_connect/reports/views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index ed197872..e02d858a 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -468,6 +468,6 @@ def visits_over_time_api(request):
             data.append(program_data[program_name].get(current_date, 0))
             current_date += timedelta(days=1)
 
-        datasets.append({"name": program_name, "data": data})
+        datasets.append({"name": program_name or "Unknown", "data": data})
 
     return JsonResponse({"labels": labels, "datasets": datasets})

From 19870ab515ec49591f9596aa87ee15163048daf4 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 15:13:37 +0200
Subject: [PATCH 123/165] improve color overlaps

---
 commcare_connect/templates/reports/dashboard.html | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 4ec0329e..22338409 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -308,10 +308,10 @@ <h3 class="my-2">Visits over Time</h3>
         // }
 
         const colors = [
-          { border: 'rgb(75, 192, 192)', background: 'rgba(75, 192, 192, 0.2)' },
-          { border: 'rgb(255, 99, 132)', background: 'rgba(255, 99, 132, 0.2)' },
-          { border: 'rgb(255, 205, 86)', background: 'rgba(255, 205, 86, 0.2)' },
-          { border: 'rgb(54, 162, 235)', background: 'rgba(54, 162, 235, 0.2)' },
+          { border: 'rgb(75, 192, 192)', background: 'rgb(146, 219, 219)' },
+          { border: 'rgb(255, 99, 132)', background: 'rgb(255, 178, 193)' },
+          { border: 'rgb(255, 205, 86)', background: 'rgb(255, 227, 167)' },
+          { border: 'rgb(54, 162, 235)', background: 'rgb(157, 207, 245)' },
         ];
 
         visitsChart = new Chart(ctx, {

From f5746e54138631e96fbc3878c8cfc05a8874e73e Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 15:20:13 +0200
Subject: [PATCH 124/165] make it a bar chart

---
 .../templates/reports/dashboard.html             | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 22338409..3c488b72 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -315,7 +315,7 @@ <h3 class="my-2">Visits over Time</h3>
         ];
 
         visitsChart = new Chart(ctx, {
-          type: 'line',
+          type: 'bar',
           data: {
             labels: chartData.labels,
             datasets: chartData.datasets.map((dataset, index) => ({
@@ -323,8 +323,7 @@ <h3 class="my-2">Visits over Time</h3>
               data: dataset.data,
               borderColor: colors[index % colors.length].border,
               backgroundColor: colors[index % colors.length].background,
-              fill: true,
-              tension: 0.3
+              borderWidth: 1
             }))
           },
           options: {
@@ -341,18 +340,19 @@ <h3 class="my-2">Visits over Time</h3>
               }
             },
             scales: {
-              y: {
-                beginAtZero: true,
+              x: {
                 stacked: true,
                 title: {
                   display: true,
-                  text: 'Number of Visits'
+                  text: 'Date'
                 }
               },
-              x: {
+              y: {
+                stacked: true,
+                beginAtZero: true,
                 title: {
                   display: true,
-                  text: 'Date'
+                  text: 'Number of Visits'
                 }
               }
             }

From ac9ac103a52d39e82d5900213f11bed183be6d8f Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 15:26:31 +0200
Subject: [PATCH 125/165] mock out spots for pie charts

---
 .../templates/reports/dashboard.html             | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 3c488b72..2b99885d 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -91,9 +91,23 @@ <h3 class="my-2">Service Delivery Map</h3>
     </div>
     <div class="col-12 col-md-6">
       <h3 class="my-2">Visits over Time</h3>
-      <div style="position: relative; height: 500px; width: 100%;">
+      <div style="position: relative; height: 200px; width: 100%;">
         <canvas id="visits-over-time"></canvas>
       </div>
+      <div class="row mt-4">
+        <div class="col-12 col-md-6">
+          <h3 class="my-2">Visits by Program</h3>
+          <div style="position: relative; height: 200px; width: 100%;">
+            <canvas id="visits-by-program"></canvas>
+          </div>
+        </div>
+        <div class="col-12 col-md-6">
+          <h3 class="my-2">Visits by Status</h3>
+          <div style="position: relative; height: 200px; width: 100%;">
+            <canvas id="visits-by-status"></canvas>
+          </div>
+        </div>
+      </div>
     </div>
   </div>
 </div>

From 2eb33e048f241e6c38755e8814716315a41599f1 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 15:29:05 +0200
Subject: [PATCH 126/165] implement pie charts

---
 commcare_connect/reports/views.py             |  44 ++++--
 .../templates/reports/dashboard.html          | 133 +++++++++++++-----
 2 files changed, 132 insertions(+), 45 deletions(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index e02d858a..3439d419 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -435,32 +435,39 @@ def visits_over_time_api(request):
         from_date = to_date - timedelta(days=30)
         queryset = queryset.filter(visit_date__gte=from_date, visit_date__lte=to_date)
 
-    # Get visits by date and program in a single query
-    visits_by_program = (
+    # Get data for all three charts
+    # 1. Visits over time by program
+    visits_by_program_time = (
         queryset.values("visit_date", "opportunity__delivery_type__name")
         .annotate(count=Count("id"))
         .order_by("visit_date", "opportunity__delivery_type__name")
     )
 
-    # Create lookup dict for program data
+    # 2. Total visits by program
+    visits_by_program = (
+        queryset.values("opportunity__delivery_type__name").annotate(count=Count("id")).order_by("-count")
+    )
+
+    # 3. Visits by status
+    visits_by_status = queryset.values("status").annotate(count=Count("id")).order_by("-count")
+
+    # Process time series data
     program_data = {}
-    for visit in visits_by_program:
+    for visit in visits_by_program_time:
         program_name = visit["opportunity__delivery_type__name"]
         if program_name not in program_data:
             program_data[program_name] = {}
         program_data[program_name][visit["visit_date"]] = visit["count"]
 
-    # Create labels and datasets
+    # Create labels and datasets for time series
     labels = []
-    datasets = []
+    time_datasets = []
     current_date = from_date
 
-    # Build labels array
     while current_date <= to_date:
         labels.append(current_date.strftime("%b %d"))
         current_date += timedelta(days=1)
 
-    # Build dataset for each program
     for program_name in program_data.keys():
         data = []
         current_date = from_date
@@ -468,6 +475,23 @@ def visits_over_time_api(request):
             data.append(program_data[program_name].get(current_date, 0))
             current_date += timedelta(days=1)
 
-        datasets.append({"name": program_name or "Unknown", "data": data})
+        time_datasets.append({"name": program_name or "Unknown", "data": data})
+
+    # Process pie chart data
+    program_pie_data = {
+        "labels": [item["opportunity__delivery_type__name"] or "Unknown" for item in visits_by_program],
+        "data": [item["count"] for item in visits_by_program],
+    }
 
-    return JsonResponse({"labels": labels, "datasets": datasets})
+    status_pie_data = {
+        "labels": [item["status"] or "Unknown" for item in visits_by_status],
+        "data": [item["count"] for item in visits_by_status],
+    }
+
+    return JsonResponse(
+        {
+            "time_series": {"labels": labels, "datasets": time_datasets},
+            "program_pie": program_pie_data,
+            "status_pie": status_pie_data,
+        }
+    )
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 2b99885d..ee8e8f13 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -295,48 +295,42 @@ <h3 class="my-2">Visits by Status</h3>
   });
   // Update chart when page loads
   window.addEventListener('DOMContentLoaded', () => {
-        // Replace the chart initialization code with this:
-        const ctx = document.getElementById('visits-over-time');
-    let visitsChart;
+    const timeSeriesCtx = document.getElementById('visits-over-time');
+    const programPieCtx = document.getElementById('visits-by-program');
+    const statusPieCtx = document.getElementById('visits-by-status');
 
-    async function updateVisitsChart() {
+    let timeSeriesChart, programPieChart, statusPieChart;
+
+    const chartColors = [
+      { border: 'rgb(75, 192, 192)', background: 'rgba(75, 192, 192, 0.8)' },
+      { border: 'rgb(255, 99, 132)', background: 'rgba(255, 99, 132, 0.8)' },
+      { border: 'rgb(255, 205, 86)', background: 'rgba(255, 205, 86, 0.8)' },
+      { border: 'rgb(54, 162, 235)', background: 'rgba(54, 162, 235, 0.8)' },
+    ];
+
+    async function updateCharts() {
       try {
         const formElement = document.querySelector('#filterForm form');
         const formData = new FormData(formElement);
         const queryString = new URLSearchParams(formData).toString();
 
         const response = await fetch(`{% url 'reports:visits_over_time_api' %}?${queryString}`);
-        const chartData = await response.json();
+        const data = await response.json();
 
-        if (visitsChart) {
-          visitsChart.destroy();
+        // Update time series chart
+        if (timeSeriesChart) {
+          timeSeriesChart.destroy();
         }
 
-        // Assuming the API now returns data in this format:
-        // {
-        //   labels: ['2024-01', '2024-02', ...],
-        //   datasets: [
-        //     { name: 'Program A', data: [10, 20, ...] },
-        //     { name: 'Program B', data: [15, 25, ...] },
-        //   ]
-        // }
-
-        const colors = [
-          { border: 'rgb(75, 192, 192)', background: 'rgb(146, 219, 219)' },
-          { border: 'rgb(255, 99, 132)', background: 'rgb(255, 178, 193)' },
-          { border: 'rgb(255, 205, 86)', background: 'rgb(255, 227, 167)' },
-          { border: 'rgb(54, 162, 235)', background: 'rgb(157, 207, 245)' },
-        ];
-
-        visitsChart = new Chart(ctx, {
+        timeSeriesChart = new Chart(timeSeriesCtx, {
           type: 'bar',
           data: {
-            labels: chartData.labels,
-            datasets: chartData.datasets.map((dataset, index) => ({
+            labels: data.time_series.labels,
+            datasets: data.time_series.datasets.map((dataset, index) => ({
               label: dataset.name,
               data: dataset.data,
-              borderColor: colors[index % colors.length].border,
-              backgroundColor: colors[index % colors.length].background,
+              borderColor: chartColors[index % chartColors.length].border,
+              backgroundColor: chartColors[index % chartColors.length].background,
               borderWidth: 1
             }))
           },
@@ -346,7 +340,7 @@ <h3 class="my-2">Visits by Status</h3>
             plugins: {
               title: {
                 display: true,
-                text: 'Visits by Program'
+                text: 'Visits Over Time'
               },
               tooltip: {
                 mode: 'index',
@@ -372,19 +366,88 @@ <h3 class="my-2">Visits by Status</h3>
             }
           }
         });
+
+        // Update program pie chart
+        if (programPieChart) {
+          programPieChart.destroy();
+        }
+
+        programPieChart = new Chart(programPieCtx, {
+          type: 'pie',
+          data: {
+            labels: data.program_pie.labels,
+            datasets: [{
+              data: data.program_pie.data,
+              backgroundColor: chartColors.map(c => c.background),
+              borderColor: chartColors.map(c => c.border),
+              borderWidth: 1
+            }]
+          },
+          options: {
+            responsive: true,
+            maintainAspectRatio: false,
+            plugins: {
+              legend: {
+                position: 'right',
+                labels: {
+                  boxWidth: 12
+                }
+              }
+            }
+          }
+        });
+
+        // Update status pie chart
+        if (statusPieChart) {
+          statusPieChart.destroy();
+        }
+
+        const statusColors = {
+          approved: { background: 'rgba(74, 222, 128, 0.8)', border: 'rgb(74, 222, 128)' },
+          rejected: { background: 'rgba(248, 113, 113, 0.8)', border: 'rgb(248, 113, 113)' },
+          pending: { background: 'rgba(251, 191, 36, 0.8)', border: 'rgb(251, 191, 36)' }
+        };
+
+        statusPieChart = new Chart(statusPieCtx, {
+          type: 'pie',
+          data: {
+            labels: data.status_pie.labels,
+            datasets: [{
+              data: data.status_pie.data,
+              backgroundColor: data.status_pie.labels.map(status =>
+                statusColors[status]?.background || 'rgba(156, 163, 175, 0.8)'
+              ),
+              borderColor: data.status_pie.labels.map(status =>
+                statusColors[status]?.border || 'rgb(156, 163, 175)'
+              ),
+              borderWidth: 1
+            }]
+          },
+          options: {
+            responsive: true,
+            maintainAspectRatio: false,
+            plugins: {
+              legend: {
+                position: 'right',
+                labels: {
+                  boxWidth: 12
+                }
+              }
+            }
+          }
+        });
+
       } catch (error) {
-        console.error('Error updating visits chart:', error);
+        console.error('Error updating charts:', error);
       }
     }
 
+    updateCharts();
 
-    console.log('updating visits chart on load');
-    updateVisitsChart();
-
-    // Update chart when filters change
+    // Update charts when filters change
     const formElement = document.querySelector('#filterForm form');
     formElement.querySelectorAll('select, input').forEach(input => {
-      input.addEventListener('change', updateVisitsChart);
+      input.addEventListener('change', updateCharts);
     });
   });
 

From 14be29bf8f2341584336e77bbc514a4d54eac229 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 15:35:58 +0200
Subject: [PATCH 127/165] style tweaks

---
 .../templates/reports/dashboard.html          | 24 +++++++++----------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index ee8e8f13..5874858c 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -90,24 +90,26 @@ <h3 class="my-2">Service Delivery Map</h3>
       </div>
     </div>
     <div class="col-12 col-md-6">
-      <h3 class="my-2">Visits over Time</h3>
-      <div style="position: relative; height: 200px; width: 100%;">
-        <canvas id="visits-over-time"></canvas>
-      </div>
-      <div class="row mt-4">
+      <h3 class="my-2">Visit Breakdown</h3>
+      <div class="row mb-4">
         <div class="col-12 col-md-6">
-          <h3 class="my-2">Visits by Program</h3>
+          <h5 class="my-2">By Program</h3>
           <div style="position: relative; height: 200px; width: 100%;">
             <canvas id="visits-by-program"></canvas>
           </div>
         </div>
         <div class="col-12 col-md-6">
-          <h3 class="my-2">Visits by Status</h3>
+          <h5 class="my-2">By Status</h3>
           <div style="position: relative; height: 200px; width: 100%;">
             <canvas id="visits-by-status"></canvas>
           </div>
         </div>
       </div>
+      <h5>Over time</h5>
+      <div style="position: relative; height: 200px; width: 100%;">
+        <canvas id="visits-over-time"></canvas>
+      </div>
+
     </div>
   </div>
 </div>
@@ -338,10 +340,6 @@ <h3 class="my-2">Visits by Status</h3>
             responsive: true,
             maintainAspectRatio: false,
             plugins: {
-              title: {
-                display: true,
-                text: 'Visits Over Time'
-              },
               tooltip: {
                 mode: 'index',
                 intersect: false,
@@ -388,7 +386,7 @@ <h3 class="my-2">Visits by Status</h3>
             maintainAspectRatio: false,
             plugins: {
               legend: {
-                position: 'right',
+                position: 'bottom',
                 labels: {
                   boxWidth: 12
                 }
@@ -428,7 +426,7 @@ <h3 class="my-2">Visits by Status</h3>
             maintainAspectRatio: false,
             plugins: {
               legend: {
-                position: 'right',
+                position: 'bottom',
                 labels: {
                   boxWidth: 12
                 }

From 6748eaec1bb7786240d504033f895d020b61b8f6 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 15:38:56 +0200
Subject: [PATCH 128/165] use a better name

---
 commcare_connect/reports/urls.py                  | 2 +-
 commcare_connect/reports/views.py                 | 2 +-
 commcare_connect/templates/reports/dashboard.html | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/commcare_connect/reports/urls.py b/commcare_connect/reports/urls.py
index bb8a9709..de9ca690 100644
--- a/commcare_connect/reports/urls.py
+++ b/commcare_connect/reports/urls.py
@@ -9,5 +9,5 @@
     path("delivery_stats", view=views.DeliveryStatsReportView.as_view(), name="delivery_stats_report"),
     path("api/visit_map_data/", views.visit_map_data, name="visit_map_data"),
     path("api/dashboard_stats/", views.dashboard_stats_api, name="dashboard_stats_api"),
-    path("api/visits_over_time/", views.visits_over_time_api, name="visits_over_time_api"),
+    path("api/dashboard_charts/", views.dashboard_charts_api, name="dashboard_charts_api"),
 ]
diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index 3439d419..e42c9805 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -422,7 +422,7 @@ def dashboard_stats_api(request):
 
 @login_required
 @user_passes_test(lambda u: u.is_superuser)
-def visits_over_time_api(request):
+def dashboard_charts_api(request):
     filterset = DashboardFilters(request.GET)
     queryset = UserVisit.objects.all()
     # Use the filtered queryset if available, else use last 30 days
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 5874858c..1b08d494 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -316,7 +316,7 @@ <h5>Over time</h5>
         const formData = new FormData(formElement);
         const queryString = new URLSearchParams(formData).toString();
 
-        const response = await fetch(`{% url 'reports:visits_over_time_api' %}?${queryString}`);
+        const response = await fetch(`{% url 'reports:dashboard_charts_api' %}?${queryString}`);
         const data = await response.json();
 
         // Update time series chart

From c02a374cf394aa0f4e200eef9baa48e7c096760b Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 15:42:14 +0200
Subject: [PATCH 129/165] refactor each chart to its own function

---
 commcare_connect/reports/views.py | 44 ++++++++++++++++---------------
 1 file changed, 23 insertions(+), 21 deletions(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index e42c9805..623fdc33 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -435,22 +435,23 @@ def dashboard_charts_api(request):
         from_date = to_date - timedelta(days=30)
         queryset = queryset.filter(visit_date__gte=from_date, visit_date__lte=to_date)
 
-    # Get data for all three charts
-    # 1. Visits over time by program
+    return JsonResponse(
+        {
+            "time_series": _get_time_series_data(queryset, from_date, to_date),
+            "program_pie": _get_program_pie_data(queryset),
+            "status_pie": _get_status_pie_data(queryset),
+        }
+    )
+
+
+def _get_time_series_data(queryset, from_date, to_date):
+    # Get visits over time by program
     visits_by_program_time = (
         queryset.values("visit_date", "opportunity__delivery_type__name")
         .annotate(count=Count("id"))
         .order_by("visit_date", "opportunity__delivery_type__name")
     )
 
-    # 2. Total visits by program
-    visits_by_program = (
-        queryset.values("opportunity__delivery_type__name").annotate(count=Count("id")).order_by("-count")
-    )
-
-    # 3. Visits by status
-    visits_by_status = queryset.values("status").annotate(count=Count("id")).order_by("-count")
-
     # Process time series data
     program_data = {}
     for visit in visits_by_program_time:
@@ -477,21 +478,22 @@ def dashboard_charts_api(request):
 
         time_datasets.append({"name": program_name or "Unknown", "data": data})
 
-    # Process pie chart data
-    program_pie_data = {
+    return {"labels": labels, "datasets": time_datasets}
+
+
+def _get_program_pie_data(queryset):
+    visits_by_program = (
+        queryset.values("opportunity__delivery_type__name").annotate(count=Count("id")).order_by("-count")
+    )
+    return {
         "labels": [item["opportunity__delivery_type__name"] or "Unknown" for item in visits_by_program],
         "data": [item["count"] for item in visits_by_program],
     }
 
-    status_pie_data = {
+
+def _get_status_pie_data(queryset):
+    visits_by_status = queryset.values("status").annotate(count=Count("id")).order_by("-count")
+    return {
         "labels": [item["status"] or "Unknown" for item in visits_by_status],
         "data": [item["count"] for item in visits_by_status],
     }
-
-    return JsonResponse(
-        {
-            "time_series": {"labels": labels, "datasets": time_datasets},
-            "program_pie": program_pie_data,
-            "status_pie": status_pie_data,
-        }
-    )

From 684143e416ad09134eeff799c5f7c0b168ddd961 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 15:45:02 +0200
Subject: [PATCH 130/165] add doc strings

---
 commcare_connect/reports/views.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index 623fdc33..47e105b1 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -445,6 +445,21 @@ def dashboard_charts_api(request):
 
 
 def _get_time_series_data(queryset, from_date, to_date):
+    """Example output:
+    {
+        "labels": ["Jan 01", "Jan 02", "Jan 03"],
+        "datasets": [
+            {
+                "name": "Program A",
+                "data": [5, 3, 7]
+            },
+            {
+                "name": "Program B",
+                "data": [2, 4, 1]
+            }
+        ]
+    }
+    """
     # Get visits over time by program
     visits_by_program_time = (
         queryset.values("visit_date", "opportunity__delivery_type__name")
@@ -482,6 +497,12 @@ def _get_time_series_data(queryset, from_date, to_date):
 
 
 def _get_program_pie_data(queryset):
+    """Example output:
+    {
+        "labels": ["Program A", "Program B", "Unknown"],
+        "data": [10, 5, 2]
+    }
+    """
     visits_by_program = (
         queryset.values("opportunity__delivery_type__name").annotate(count=Count("id")).order_by("-count")
     )
@@ -492,6 +513,12 @@ def _get_program_pie_data(queryset):
 
 
 def _get_status_pie_data(queryset):
+    """Example output:
+    {
+        "labels": ["Approved", "Pending", "Rejected", "Unknown"],
+        "data": [15, 8, 4, 1]
+    }
+    """
     visits_by_status = queryset.values("status").annotate(count=Count("id")).order_by("-count")
     return {
         "labels": [item["status"] or "Unknown" for item in visits_by_status],

From fb11e70fb43141d850b668b5ec91e0a3329ad64e Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 15:59:42 +0200
Subject: [PATCH 131/165] externalize js to js file

---
 commcare_connect/static/js/dashboard.js       | 131 ++++++++++++++++++
 .../templates/reports/dashboard.html          | 111 +--------------
 2 files changed, 134 insertions(+), 108 deletions(-)

diff --git a/commcare_connect/static/js/dashboard.js b/commcare_connect/static/js/dashboard.js
index d73535b3..0a761c6a 100644
--- a/commcare_connect/static/js/dashboard.js
+++ b/commcare_connect/static/js/dashboard.js
@@ -128,5 +128,136 @@ function donutSegment(start, end, r, r0, color) {
   }" fill="${color}" opacity="0.85" stroke="#1f2937" stroke-width="1" />`;
 }
 
+const chartColors = [
+  { border: 'rgb(75, 192, 192)', background: 'rgba(75, 192, 192, 0.8)' },
+  { border: 'rgb(255, 99, 132)', background: 'rgba(255, 99, 132, 0.8)' },
+  { border: 'rgb(255, 205, 86)', background: 'rgba(255, 205, 86, 0.8)' },
+  { border: 'rgb(54, 162, 235)', background: 'rgba(54, 162, 235, 0.8)' },
+];
+
+const statusColors = {
+  approved: {
+    background: 'rgba(74, 222, 128, 0.8)',
+    border: 'rgb(74, 222, 128)',
+  },
+  rejected: {
+    background: 'rgba(248, 113, 113, 0.8)',
+    border: 'rgb(248, 113, 113)',
+  },
+  pending: {
+    background: 'rgba(251, 191, 36, 0.8)',
+    border: 'rgb(251, 191, 36)',
+  },
+};
+
+function createTimeSeriesChart(ctx, data) {
+  return new Chart(ctx, {
+    type: 'bar',
+    data: {
+      labels: data.labels,
+      datasets: data.datasets.map((dataset, index) => ({
+        label: dataset.name,
+        data: dataset.data,
+        borderColor: chartColors[index % chartColors.length].border,
+        backgroundColor: chartColors[index % chartColors.length].background,
+        borderWidth: 1,
+      })),
+    },
+    options: {
+      responsive: true,
+      maintainAspectRatio: false,
+      plugins: {
+        tooltip: {
+          mode: 'index',
+          intersect: false,
+        },
+      },
+      scales: {
+        x: {
+          stacked: true,
+          title: {
+            display: true,
+            text: 'Date',
+          },
+        },
+        y: {
+          stacked: true,
+          beginAtZero: true,
+          title: {
+            display: true,
+            text: 'Number of Visits',
+          },
+        },
+      },
+    },
+  });
+}
+
+function createProgramPieChart(ctx, data) {
+  return new Chart(ctx, {
+    type: 'pie',
+    data: {
+      labels: data.labels,
+      datasets: [
+        {
+          data: data.data,
+          backgroundColor: chartColors.map((c) => c.background),
+          borderColor: chartColors.map((c) => c.border),
+          borderWidth: 1,
+        },
+      ],
+    },
+    options: {
+      responsive: true,
+      maintainAspectRatio: false,
+      plugins: {
+        legend: {
+          position: 'bottom',
+          labels: {
+            boxWidth: 12,
+          },
+        },
+      },
+    },
+  });
+}
+
+function createStatusPieChart(ctx, data) {
+  return new Chart(ctx, {
+    type: 'pie',
+    data: {
+      labels: data.labels,
+      datasets: [
+        {
+          data: data.data,
+          backgroundColor: data.labels.map(
+            (status) =>
+              statusColors[status]?.background || 'rgba(156, 163, 175, 0.8)',
+          ),
+          borderColor: data.labels.map(
+            (status) => statusColors[status]?.border || 'rgb(156, 163, 175)',
+          ),
+          borderWidth: 1,
+        },
+      ],
+    },
+    options: {
+      responsive: true,
+      maintainAspectRatio: false,
+      plugins: {
+        legend: {
+          position: 'bottom',
+          labels: {
+            boxWidth: 12,
+          },
+        },
+      },
+    },
+  });
+}
+
 window.updateMarkers = updateMarkers;
 window.createDonutChart = createDonutChart;
+window.createTimeSeriesChart = createTimeSeriesChart;
+window.createProgramPieChart = createProgramPieChart;
+window.createStatusPieChart = createStatusPieChart;
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 1b08d494..5fe5a5b2 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -303,13 +303,6 @@ <h5>Over time</h5>
 
     let timeSeriesChart, programPieChart, statusPieChart;
 
-    const chartColors = [
-      { border: 'rgb(75, 192, 192)', background: 'rgba(75, 192, 192, 0.8)' },
-      { border: 'rgb(255, 99, 132)', background: 'rgba(255, 99, 132, 0.8)' },
-      { border: 'rgb(255, 205, 86)', background: 'rgba(255, 205, 86, 0.8)' },
-      { border: 'rgb(54, 162, 235)', background: 'rgba(54, 162, 235, 0.8)' },
-    ];
-
     async function updateCharts() {
       try {
         const formElement = document.querySelector('#filterForm form');
@@ -323,117 +316,19 @@ <h5>Over time</h5>
         if (timeSeriesChart) {
           timeSeriesChart.destroy();
         }
-
-        timeSeriesChart = new Chart(timeSeriesCtx, {
-          type: 'bar',
-          data: {
-            labels: data.time_series.labels,
-            datasets: data.time_series.datasets.map((dataset, index) => ({
-              label: dataset.name,
-              data: dataset.data,
-              borderColor: chartColors[index % chartColors.length].border,
-              backgroundColor: chartColors[index % chartColors.length].background,
-              borderWidth: 1
-            }))
-          },
-          options: {
-            responsive: true,
-            maintainAspectRatio: false,
-            plugins: {
-              tooltip: {
-                mode: 'index',
-                intersect: false,
-              }
-            },
-            scales: {
-              x: {
-                stacked: true,
-                title: {
-                  display: true,
-                  text: 'Date'
-                }
-              },
-              y: {
-                stacked: true,
-                beginAtZero: true,
-                title: {
-                  display: true,
-                  text: 'Number of Visits'
-                }
-              }
-            }
-          }
-        });
+        timeSeriesChart = createTimeSeriesChart(timeSeriesCtx, data.time_series);
 
         // Update program pie chart
         if (programPieChart) {
           programPieChart.destroy();
         }
-
-        programPieChart = new Chart(programPieCtx, {
-          type: 'pie',
-          data: {
-            labels: data.program_pie.labels,
-            datasets: [{
-              data: data.program_pie.data,
-              backgroundColor: chartColors.map(c => c.background),
-              borderColor: chartColors.map(c => c.border),
-              borderWidth: 1
-            }]
-          },
-          options: {
-            responsive: true,
-            maintainAspectRatio: false,
-            plugins: {
-              legend: {
-                position: 'bottom',
-                labels: {
-                  boxWidth: 12
-                }
-              }
-            }
-          }
-        });
+        programPieChart = createProgramPieChart(programPieCtx, data.program_pie);
 
         // Update status pie chart
         if (statusPieChart) {
           statusPieChart.destroy();
         }
-
-        const statusColors = {
-          approved: { background: 'rgba(74, 222, 128, 0.8)', border: 'rgb(74, 222, 128)' },
-          rejected: { background: 'rgba(248, 113, 113, 0.8)', border: 'rgb(248, 113, 113)' },
-          pending: { background: 'rgba(251, 191, 36, 0.8)', border: 'rgb(251, 191, 36)' }
-        };
-
-        statusPieChart = new Chart(statusPieCtx, {
-          type: 'pie',
-          data: {
-            labels: data.status_pie.labels,
-            datasets: [{
-              data: data.status_pie.data,
-              backgroundColor: data.status_pie.labels.map(status =>
-                statusColors[status]?.background || 'rgba(156, 163, 175, 0.8)'
-              ),
-              borderColor: data.status_pie.labels.map(status =>
-                statusColors[status]?.border || 'rgb(156, 163, 175)'
-              ),
-              borderWidth: 1
-            }]
-          },
-          options: {
-            responsive: true,
-            maintainAspectRatio: false,
-            plugins: {
-              legend: {
-                position: 'bottom',
-                labels: {
-                  boxWidth: 12
-                }
-              }
-            }
-          }
-        });
+        statusPieChart = createStatusPieChart(statusPieCtx, data.status_pie);
 
       } catch (error) {
         console.error('Error updating charts:', error);

From 9a426cee9821cdd463445959aae484ed05a244a2 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 16:04:10 +0200
Subject: [PATCH 132/165] add better empty states

---
 commcare_connect/static/js/dashboard.js | 62 +++++++++++++++++++++++++
 1 file changed, 62 insertions(+)

diff --git a/commcare_connect/static/js/dashboard.js b/commcare_connect/static/js/dashboard.js
index 0a761c6a..53ca2b65 100644
--- a/commcare_connect/static/js/dashboard.js
+++ b/commcare_connect/static/js/dashboard.js
@@ -194,6 +194,37 @@ function createTimeSeriesChart(ctx, data) {
 }
 
 function createProgramPieChart(ctx, data) {
+  // Check if there's no data or empty data
+  if (!data?.data?.length) {
+    return new Chart(ctx, {
+      type: 'pie',
+      data: {
+        labels: ['No data'],
+        datasets: [
+          {
+            data: [1],
+            backgroundColor: ['rgba(156, 163, 175, 0.3)'],
+            borderColor: ['rgb(156, 163, 175)'],
+            borderWidth: 1,
+          },
+        ],
+      },
+      options: {
+        responsive: true,
+        maintainAspectRatio: false,
+        plugins: {
+          legend: {
+            position: 'bottom',
+            labels: {
+              boxWidth: 12,
+              color: 'rgb(156, 163, 175)',
+            },
+          },
+        },
+      },
+    });
+  }
+
   return new Chart(ctx, {
     type: 'pie',
     data: {
@@ -223,6 +254,37 @@ function createProgramPieChart(ctx, data) {
 }
 
 function createStatusPieChart(ctx, data) {
+  // Check if there's no data or empty data
+  if (!data?.data?.length) {
+    return new Chart(ctx, {
+      type: 'pie',
+      data: {
+        labels: ['No data'],
+        datasets: [
+          {
+            data: [1],
+            backgroundColor: ['rgba(156, 163, 175, 0.3)'],
+            borderColor: ['rgb(156, 163, 175)'],
+            borderWidth: 1,
+          },
+        ],
+      },
+      options: {
+        responsive: true,
+        maintainAspectRatio: false,
+        plugins: {
+          legend: {
+            position: 'bottom',
+            labels: {
+              boxWidth: 12,
+              color: 'rgb(156, 163, 175)',
+            },
+          },
+        },
+      },
+    });
+  }
+
   return new Chart(ctx, {
     type: 'pie',
     data: {

From c424966ede84bdde1566471fb3db24b81939c9cb Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Thu, 14 Nov 2024 08:47:54 +0530
Subject: [PATCH 133/165] fixed average time to covert calc

---
 commcare_connect/program/helpers.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/commcare_connect/program/helpers.py b/commcare_connect/program/helpers.py
index e26c0cfc..7e0ca99a 100644
--- a/commcare_connect/program/helpers.py
+++ b/commcare_connect/program/helpers.py
@@ -37,6 +37,7 @@ def get_annotated_managed_opportunity(program: Program):
     earliest_visits = (
         UserVisit.objects.filter(
             opportunity_access=OuterRef("opportunityaccess"),
+            user=OuterRef("opportunityaccess__uservisit__user"),
         )
         .exclude(status__in=EXCLUDED_STATUS)
         .order_by("visit_date")
@@ -66,10 +67,10 @@ def get_annotated_managed_opportunity(program: Program):
                     Subquery(earliest_visits) - F("opportunityaccess__invited_date"), output_field=DurationField()
                 ),
                 filter=FILTER_FOR_VALID_VISIT_DATE,
+                distinct=True,
             ),
         )
     )
-
     return managed_opportunities
 
 

From 912d4c4baf90df2fa3b620c3880f9904aacfcbc0 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Thu, 14 Nov 2024 14:51:33 +0530
Subject: [PATCH 134/165] Add PaymentUnit model to admin

---
 commcare_connect/opportunity/admin.py  | 12 +++++++++++-
 commcare_connect/opportunity/models.py |  3 +++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/commcare_connect/opportunity/admin.py b/commcare_connect/opportunity/admin.py
index 86839d6e..f1a01712 100644
--- a/commcare_connect/opportunity/admin.py
+++ b/commcare_connect/opportunity/admin.py
@@ -26,7 +26,6 @@
 
 
 admin.site.register(CommCareApp)
-admin.site.register(PaymentUnit)
 admin.site.register(UserInvite)
 admin.site.register(DeliveryType)
 admin.site.register(DeliverUnitFlagRules)
@@ -113,6 +112,7 @@ class AssessmentAdmin(admin.ModelAdmin):
 @admin.register(CompletedWork)
 class CompletedWorkAdmin(admin.ModelAdmin):
     list_display = ["get_username", "get_opp_name", "opportunity_access", "payment_unit", "status"]
+    search_fields = ["get_username", "get_opp_name"]
 
     @admin.display(description="Opportunity Name")
     def get_opp_name(self, obj):
@@ -121,3 +121,13 @@ def get_opp_name(self, obj):
     @admin.display(description="Username")
     def get_username(self, obj):
         return obj.opportunity_access.user.username
+
+
+@admin.register(PaymentUnit)
+class PaymentUnitAdmin(admin.ModelAdmin):
+    list_display = ["name", "get_opp_name"]
+    search_fields = ["name"]
+
+    @admin.display(description="Opportunity Name")
+    def get_opp_name(self, obj):
+        return obj.opportunity_access.opportunity.name
diff --git a/commcare_connect/opportunity/models.py b/commcare_connect/opportunity/models.py
index cf763f5d..a6351e64 100644
--- a/commcare_connect/opportunity/models.py
+++ b/commcare_connect/opportunity/models.py
@@ -364,6 +364,9 @@ class PaymentUnit(models.Model):
         null=True,
     )
 
+    def __str__(self):
+        return self.name
+
 
 class DeliverUnit(models.Model):
     app = models.ForeignKey(

From 26cd175658e03bae3d69dde9fcb8ff0b86a2256c Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Fri, 15 Nov 2024 16:25:19 +0530
Subject: [PATCH 135/165] fixed record % issue

---
 commcare_connect/program/helpers.py            | 12 +++++++++---
 commcare_connect/program/tests/test_helpers.py |  6 ++++--
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/commcare_connect/program/helpers.py b/commcare_connect/program/helpers.py
index 7e0ca99a..390eaebf 100644
--- a/commcare_connect/program/helpers.py
+++ b/commcare_connect/program/helpers.py
@@ -83,7 +83,11 @@ def get_delivery_performance_report(program: Program, start_date, end_date):
     if end_date:
         date_filter &= Q(opportunityaccess__uservisit__visit_date__lte=end_date)
 
-    flagged_visits_filter = Q(opportunityaccess__uservisit__flagged=True) & FILTER_FOR_VALID_VISIT_DATE
+    flagged_visits_filter = (
+        Q(opportunityaccess__uservisit__flagged=True)
+        & date_filter
+        & Q(opportunityaccess__uservisit__completed_work__isnull=False)
+    )
 
     managed_opportunities = (
         ManagedOpportunity.objects.filter(program=program)
@@ -101,10 +105,12 @@ def get_delivery_performance_report(program: Program, start_date, end_date):
             ),
             total_payment_units=Count("opportunityaccess__completedwork", distinct=True),
             total_payment_units_with_flags=Count(
-                "opportunityaccess__completedwork", distinct=True, filter=flagged_visits_filter
+                "opportunityaccess__uservisit", distinct=True, filter=flagged_visits_filter
             ),
             total_payment_since_start_date=Count(
-                "opportunityaccess__completedwork", distinct=True, filter=date_filter
+                "opportunityaccess__uservisit",
+                distinct=True,
+                filter=date_filter & Q(opportunityaccess__uservisit__completed_work__isnull=False),
             ),
             delivery_per_day_per_worker=Case(
                 When(active_workers=0, then=Value(0)),
diff --git a/commcare_connect/program/tests/test_helpers.py b/commcare_connect/program/tests/test_helpers.py
index 61bdaa8c..fb7f07ea 100644
--- a/commcare_connect/program/tests/test_helpers.py
+++ b/commcare_connect/program/tests/test_helpers.py
@@ -39,7 +39,7 @@ def create_user_with_access(self, visit_status=VisitValidationStatus.pending, pa
     def create_user_with_visit(self, visit_status, visit_date, flagged=False, create_completed_work=True):
         user = UserFactory.create()
         access = OpportunityAccessFactory.create(opportunity=self.opp, user=user, invited_date=now())
-        UserVisitFactory.create(
+        visit = UserVisitFactory.create(
             user=user,
             opportunity=self.opp,
             status=visit_status,
@@ -48,7 +48,9 @@ def create_user_with_visit(self, visit_status, visit_date, flagged=False, create
             flagged=flagged,
         )
         if create_completed_work:
-            CompletedWorkFactory.create(opportunity_access=access)
+            work = CompletedWorkFactory.create(opportunity_access=access)
+            visit.completed_work = work
+            visit.save()
         return user
 
 

From 1644c3a83944a9e0ad78fe3bbc737eaa66592c2c Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Fri, 15 Nov 2024 17:20:52 +0530
Subject: [PATCH 136/165] Fix linter error

---
 config/views.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/config/views.py b/config/views.py
index 3b7fc766..58135f30 100644
--- a/config/views.py
+++ b/config/views.py
@@ -8,23 +8,21 @@ def assetlinks_json(request):
             "target": {
                 "namespace": "android_app",
                 "package_name": "org.commcare.dalvik",
-                "sha256_cert_fingerprints":
-                [
+                "sha256_cert_fingerprints": [
                     "88:57:18:F8:E8:7D:74:04:97:AE:83:65:74:ED:EF:10:40:D9:4C:E2:54:F0:E0:40:64:77:96:7F:D1:39:F9:81",
-                    "89:55:DF:D8:0E:66:63:06:D2:6D:88:A4:A3:88:A4:D9:16:5A:C4:1A:7E:E1:C6:78:87:00:37:55:93:03:7B:03"
-                ]
-            }
+                    "89:55:DF:D8:0E:66:63:06:D2:6D:88:A4:A3:88:A4:D9:16:5A:C4:1A:7E:E1:C6:78:87:00:37:55:93:03:7B:03",
+                ],
+            },
         },
         {
             "relation": ["delegate_permission/common.handle_all_urls"],
             "target": {
                 "namespace": "android_app",
                 "package_name": "org.commcare.dalvik.debug",
-                "sha256_cert_fingerprints":
-                [
+                "sha256_cert_fingerprints": [
                     "88:57:18:F8:E8:7D:74:04:97:AE:83:65:74:ED:EF:10:40:D9:4C:E2:54:F0:E0:40:64:77:96:7F:D1:39:F9:81"
-                ]
-            }
+                ],
+            },
         },
     ]
     return JsonResponse(assetfile, safe=False)

From 9fdfd5a54d7a6477366790707eb9a84ff1101a7e Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Tue, 19 Nov 2024 19:57:50 +0530
Subject: [PATCH 137/165] Add username and opportunity name to search fields

---
 commcare_connect/opportunity/admin.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/commcare_connect/opportunity/admin.py b/commcare_connect/opportunity/admin.py
index f1a01712..e833dd19 100644
--- a/commcare_connect/opportunity/admin.py
+++ b/commcare_connect/opportunity/admin.py
@@ -102,6 +102,7 @@ class CompletedModuleAdmin(admin.ModelAdmin):
 @admin.register(UserVisit)
 class UserVisitAdmin(admin.ModelAdmin):
     list_display = ["deliver_unit", "user", "opportunity", "status"]
+    search_fields = ["opportunity_access__user__username", "opportunity_access__opportunity__name"]
 
 
 @admin.register(Assessment)
@@ -112,7 +113,7 @@ class AssessmentAdmin(admin.ModelAdmin):
 @admin.register(CompletedWork)
 class CompletedWorkAdmin(admin.ModelAdmin):
     list_display = ["get_username", "get_opp_name", "opportunity_access", "payment_unit", "status"]
-    search_fields = ["get_username", "get_opp_name"]
+    search_fields = ["opportunity_access__user__username", "opportunity_access__opportunity__name"]
 
     @admin.display(description="Opportunity Name")
     def get_opp_name(self, obj):

From af496b1addb17a83a1ff0fb286bae602979564c9 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 20 Nov 2024 11:31:43 +0530
Subject: [PATCH 138/165] added the opportunity column

---
 commcare_connect/program/helpers.py           |  3 +-
 commcare_connect/program/tables.py            | 32 ++++++++++++++++---
 .../program/tests/test_helpers.py             |  2 +-
 3 files changed, 30 insertions(+), 7 deletions(-)

diff --git a/commcare_connect/program/helpers.py b/commcare_connect/program/helpers.py
index 390eaebf..0e83471e 100644
--- a/commcare_connect/program/helpers.py
+++ b/commcare_connect/program/helpers.py
@@ -103,7 +103,6 @@ def get_delivery_performance_report(program: Program, start_date, end_date):
                 filter=date_filter,
                 distinct=True,
             ),
-            total_payment_units=Count("opportunityaccess__completedwork", distinct=True),
             total_payment_units_with_flags=Count(
                 "opportunityaccess__uservisit", distinct=True, filter=flagged_visits_filter
             ),
@@ -112,7 +111,7 @@ def get_delivery_performance_report(program: Program, start_date, end_date):
                 distinct=True,
                 filter=date_filter & Q(opportunityaccess__uservisit__completed_work__isnull=False),
             ),
-            delivery_per_day_per_worker=Case(
+            deliveries_per_day_per_worker=Case(
                 When(active_workers=0, then=Value(0)),
                 default=Round(F("total_payment_since_start_date") / F("active_workers"), 2),
                 output_field=FloatField(),
diff --git a/commcare_connect/program/tables.py b/commcare_connect/program/tables.py
index 1207488a..a045700e 100644
--- a/commcare_connect/program/tables.py
+++ b/commcare_connect/program/tables.py
@@ -237,11 +237,12 @@ def get_manage_buttons_html(buttons, request):
 
 class FunnelPerformanceTable(tables.Table):
     organization = tables.Column()
+    opportunity = tables.Column()
     start_date = tables.DateColumn()
     workers_invited = tables.Column(verbose_name=_("Workers Invited"))
     workers_passing_assessment = tables.Column(verbose_name=_("Workers Passing Assessment"))
     workers_starting_delivery = tables.Column(verbose_name=_("Workers Starting Delivery"))
-    percentage_conversion = tables.Column(verbose_name=_("Percentage Conversion"))
+    percentage_conversion = tables.Column(verbose_name=_("% Conversion"))
     average_time_to_convert = tables.Column(verbose_name=_("Average Time To convert"))
 
     class Meta:
@@ -249,6 +250,7 @@ class Meta:
         empty_text = "No data available yet."
         fields = (
             "organization",
+            "opportunity",
             "start_date",
             "workers_invited",
             "workers_passing_assessment",
@@ -265,24 +267,46 @@ def render_average_time_to_convert(self, record):
         hours = total_seconds / 3600
         return f"{round(hours, 2)}hr"
 
+    def render_opportunity(self, record):
+        url = reverse(
+            "opportunity:detail",
+            kwargs={
+                "org_slug": record.organization.slug,
+                "pk": record.id,
+            },
+        )
+        return mark_safe(f'<a href="{url}">{record.name}</a>')
+
 
 class DeliveryPerformanceTable(tables.Table):
     organization = tables.Column()
+    opportunity = tables.Column()
     start_date = tables.DateColumn()
     total_workers_starting_delivery = tables.Column(verbose_name=_("Workers Starting Delivery"))
     active_workers = tables.Column(verbose_name=_("Active Workers"))
-    delivery_per_day_per_worker = tables.Column(verbose_name=_("Delivery Per Day Per Worker"))
-    records_flagged_percentage = tables.Column(verbose_name=_("Records flagged"))
+    deliveries_per_day_per_worker = tables.Column(verbose_name=_("Deliveries per Day per Worker"))
+    records_flagged_percentage = tables.Column(verbose_name=_("% Records flagged"))
 
     class Meta:
         model = ManagedOpportunity
         empty_text = "No data available yet."
         fields = (
             "organization",
+            "opportunity",
             "start_date",
             "total_workers_starting_delivery",
             "active_workers",
-            "delivery_per_day_per_worker",
+            "deliveries_per_day_per_worker",
             "records_flagged_percentage",
         )
         orderable = False
+
+    def render_opportunity(self, record):
+        url = reverse(
+            "opportunity:detail",
+            kwargs={
+                "org_slug": record.organization.slug,
+                "pk": record.id,
+            },
+        )
+        return mark_safe(f'<a href="{url}">{record.name}</a>')
diff --git a/commcare_connect/program/tests/test_helpers.py b/commcare_connect/program/tests/test_helpers.py
index fb7f07ea..cddfb6a2 100644
--- a/commcare_connect/program/tests/test_helpers.py
+++ b/commcare_connect/program/tests/test_helpers.py
@@ -246,4 +246,4 @@ def test_delivery_performance_report_scenarios(
         assert opps[0].records_flagged_percentage == expected_records_flagged_percentage
         assert opps[0].total_payment_units_with_flags == total_payment_units_with_flags
         assert opps[0].total_payment_since_start_date == total_payment_since_start_date
-        assert opps[0].delivery_per_day_per_worker == delivery_per_day_per_worker
+        assert opps[0].deliveries_per_day_per_worker == delivery_per_day_per_worker

From 0c233ba83f45b3ec67e01fafd8c6c1a9c61094ba Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 20 Nov 2024 12:18:49 +0530
Subject: [PATCH 139/165] fix the opporunity name not displayed issue

---
 commcare_connect/program/tables.py | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/commcare_connect/program/tables.py b/commcare_connect/program/tables.py
index a045700e..8ba34e79 100644
--- a/commcare_connect/program/tables.py
+++ b/commcare_connect/program/tables.py
@@ -1,6 +1,7 @@
 import django_tables2 as tables
 from django.template.loader import render_to_string
 from django.urls import reverse
+from django.utils.html import format_html
 from django.utils.safestring import mark_safe
 from django.utils.translation import gettext_lazy as _
 
@@ -237,7 +238,7 @@ def get_manage_buttons_html(buttons, request):
 
 class FunnelPerformanceTable(tables.Table):
     organization = tables.Column()
-    opportunity = tables.Column()
+    opportunity = tables.Column(accessor="name")
     start_date = tables.DateColumn()
     workers_invited = tables.Column(verbose_name=_("Workers Invited"))
     workers_passing_assessment = tables.Column(verbose_name=_("Workers Passing Assessment"))
@@ -260,14 +261,7 @@ class Meta:
         )
         orderable = False
 
-    def render_average_time_to_convert(self, record):
-        if not record.average_time_to_convert:
-            return "---"
-        total_seconds = record.average_time_to_convert.total_seconds()
-        hours = total_seconds / 3600
-        return f"{round(hours, 2)}hr"
-
-    def render_opportunity(self, record):
+    def render_opportunity(self, value, record):
         url = reverse(
             "opportunity:detail",
             kwargs={
@@ -275,12 +269,19 @@ def render_opportunity(self, record):
                 "pk": record.id,
             },
         )
-        return mark_safe(f'<a href="{url}">{record.name}</a>')
+        return format_html('<a href="{}">{}</a>', url, value)
+
+    def render_average_time_to_convert(self, record):
+        if not record.average_time_to_convert:
+            return "---"
+        total_seconds = record.average_time_to_convert.total_seconds()
+        hours = total_seconds / 3600
+        return f"{round(hours, 2)}hr"
 
 
 class DeliveryPerformanceTable(tables.Table):
     organization = tables.Column()
-    opportunity = tables.Column()
+    opportunity = tables.Column(accessor="name")
     start_date = tables.DateColumn()
     total_workers_starting_delivery = tables.Column(verbose_name=_("Workers Starting Delivery"))
     active_workers = tables.Column(verbose_name=_("Active Workers"))
@@ -301,7 +302,7 @@ class Meta:
         )
         orderable = False
 
-    def render_opportunity(self, record):
+    def render_opportunity(self, value, record):
         url = reverse(
             "opportunity:detail",
             kwargs={
@@ -309,4 +310,4 @@ def render_opportunity(self, record):
                 "pk": record.id,
             },
         )
-        return mark_safe(f'<a href="{url}">{record.name}</a>')
+        return format_html('<a href="{}">{}</a>', url, value)

From a7cca43cb757036d73e63ade7bac0f837a3d5eab Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Wed, 20 Nov 2024 13:25:08 +0530
Subject: [PATCH 140/165] Add move invite_delete_url inside if block

---
 commcare_connect/opportunity/tables.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/commcare_connect/opportunity/tables.py b/commcare_connect/opportunity/tables.py
index e86969a8..d722ad60 100644
--- a/commcare_connect/opportunity/tables.py
+++ b/commcare_connect/opportunity/tables.py
@@ -178,11 +178,11 @@ def render_display_name(self, record):
         return record.opportunity_access.display_name
 
     def render_view_profile(self, record):
-        invite_delete_url = reverse(
-            "opportunity:user_invite_delete",
-            args=(self.org_slug, record.opportunity.id, record.id),
-        )
         if not getattr(record.opportunity_access, "accepted", False):
+            invite_delete_url = reverse(
+                "opportunity:user_invite_delete",
+                args=(self.org_slug, record.opportunity.id, record.id),
+            )
             return format_html(
                 (
                     '<button hx-post="{}" hx-swap="none" '

From 9fb1dc4754621eded7604f18880de42584094b85 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Wed, 20 Nov 2024 14:12:36 +0530
Subject: [PATCH 141/165] Fix move claim limit fetch before completed work
 creation

---
 commcare_connect/form_receiver/processor.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/commcare_connect/form_receiver/processor.py b/commcare_connect/form_receiver/processor.py
index 641213c5..b36fe069 100644
--- a/commcare_connect/form_receiver/processor.py
+++ b/commcare_connect/form_receiver/processor.py
@@ -242,7 +242,9 @@ def process_deliver_unit(user, xform: XForm, app: CommCareApp, opportunity: Oppo
             entity=Count("pk", filter=Q(entity_id=deliver_unit_block.get("entity_id"), deliver_unit=deliver_unit)),
         )
     )
+    payment_unit = deliver_unit.payment_unit
     claim = OpportunityClaim.objects.get(opportunity_access=access)
+    claim_limit = OpportunityClaimLimit.objects.get(opportunity_claim=claim, payment_unit=payment_unit)
     entity_id = deliver_unit_block.get("entity_id")
     entity_name = deliver_unit_block.get("entity_name")
     user_visit = UserVisit(
@@ -267,17 +269,12 @@ def process_deliver_unit(user, xform: XForm, app: CommCareApp, opportunity: Oppo
         completed_work, _ = CompletedWork.objects.get_or_create(
             opportunity_access=access,
             entity_id=entity_id,
-            payment_unit=deliver_unit.payment_unit,
-            defaults={
-                "entity_name": entity_name,
-            },
+            payment_unit=payment_unit,
+            defaults={"entity_name": entity_name},
         )
         user_visit.completed_work = completed_work
-        claim_limit = OpportunityClaimLimit.objects.get(
-            opportunity_claim=claim, payment_unit=completed_work.payment_unit
-        )
         if (
-            counts["daily"] >= deliver_unit.payment_unit.max_daily
+            counts["daily"] >= payment_unit.max_daily
             or counts["total"] >= claim_limit.max_visits
             or datetime.date.today() > claim.end_date
         ):

From f8622c9c9bf33466d8e1149a8a53a502b6b53ddb Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 20 Nov 2024 18:32:39 +0530
Subject: [PATCH 142/165] chnaged column name to opportunity

---
 commcare_connect/program/tables.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/commcare_connect/program/tables.py b/commcare_connect/program/tables.py
index 8ba34e79..b899c19b 100644
--- a/commcare_connect/program/tables.py
+++ b/commcare_connect/program/tables.py
@@ -238,7 +238,7 @@ def get_manage_buttons_html(buttons, request):
 
 class FunnelPerformanceTable(tables.Table):
     organization = tables.Column()
-    opportunity = tables.Column(accessor="name")
+    opportunity = tables.Column(accessor="name", verbose_name="Opportunity")
     start_date = tables.DateColumn()
     workers_invited = tables.Column(verbose_name=_("Workers Invited"))
     workers_passing_assessment = tables.Column(verbose_name=_("Workers Passing Assessment"))
@@ -281,7 +281,7 @@ def render_average_time_to_convert(self, record):
 
 class DeliveryPerformanceTable(tables.Table):
     organization = tables.Column()
-    opportunity = tables.Column(accessor="name")
+    opportunity = tables.Column(accessor="name", verbose_name="Opportunity")
     start_date = tables.DateColumn()
     total_workers_starting_delivery = tables.Column(verbose_name=_("Workers Starting Delivery"))
     active_workers = tables.Column(verbose_name=_("Active Workers"))

From 1eff1c20095a2191c79c1a2d904af01a6440f4cd Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Thu, 21 Nov 2024 07:47:10 +0530
Subject: [PATCH 143/165] added street view and fixed opacity

---
 commcare_connect/static/js/project.js         |  2 +-
 .../templates/opportunity/user_profile.html   | 20 ++++++++++++++++++-
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/commcare_connect/static/js/project.js b/commcare_connect/static/js/project.js
index 83ba289b..b2665b39 100644
--- a/commcare_connect/static/js/project.js
+++ b/commcare_connect/static/js/project.js
@@ -67,7 +67,7 @@ window.addAccuracyCircles = addAccuracyCircles;
 function addCatchmentAreas(map, catchments) {
   const ACTIVE_COLOR = '#3366ff';
   const INACTIVE_COLOR = '#ff4d4d';
-  const CIRCLE_OPACITY = 0.3;
+  const CIRCLE_OPACITY = 0.15;
 
   map.on('load', () => {
     const catchmentCircles = catchments.map((catchment) =>
diff --git a/commcare_connect/templates/opportunity/user_profile.html b/commcare_connect/templates/opportunity/user_profile.html
index fe4fd4e0..084532d0 100644
--- a/commcare_connect/templates/opportunity/user_profile.html
+++ b/commcare_connect/templates/opportunity/user_profile.html
@@ -45,6 +45,14 @@ <h1 class="display-4 mb-0 text-center">{{access.display_name}}</h1>
           <div class="col">{{access.last_visit_date}}</div>
         </div>
       </div>
+      <div id="menu" class="btn-group btn-group-toggle" data-toggle="buttons" style="position: absolute; top: 0; left: 0;">
+      <label class="btn btn-secondary active">
+        <input type="radio" name="options" id="streets-v12" autocomplete="off" checked> Streets
+      </label>
+      <label class="btn btn-secondary">
+        <input type="radio" name="options" id="satellite-streets-v12" autocomplete="off"> Satellite
+      </label>
+    </div>
        <div class="position-relative">
         <div id="user-visit-map" class="mt-3 mb-2 rounded border" style="height: 300px;"></div>
         <div id="legend" class="card position-absolute bottom-0 end-0 m-2 z-1" x-data="{ show: false}" x-show="show">
@@ -81,7 +89,7 @@ <h6 class="card-title">Catchment Areas</h6>
   {{ user_catchments|json_script:"userCatchments" }}
   <script type="module">
     window.addEventListener('DOMContentLoaded', () => {
-      mapboxgl.accessToken = "{{ MAPBOX_TOKEN }}";
+      mapboxgl.accessToken = "pk.eyJ1IjoiaHlhZGF2MTAiLCJhIjoiY2x5ZHYxMDFuMGQzeDJrczZtamd5dXBqcSJ9.bjt_mM0gLycVpn_Q-B3zow";
       const map = new mapboxgl.Map({
         container: 'user-visit-map',
         style: 'mapbox://styles/mapbox/satellite-streets-v12',
@@ -98,6 +106,16 @@ <h6 class="card-title">Catchment Areas</h6>
       addAccuracyCircles(map, userVisits);
       const userCatchments = JSON.parse(document.getElementById('userCatchments').textContent);
       addCatchmentAreas(map, userCatchments)
+
+      const styleList = document.getElementById('menu');
+      const inputs = styleList.getElementsByTagName('input');
+
+      for (const input of inputs) {
+          input.onclick = (click) => {
+              const styleId = click.target.id;
+              map.setStyle('mapbox://styles/mapbox/' + styleId);
+          };
+      }
     });
   </script>
 {% endblock %}

From 76880ffb8cfaa985fbaa04b85f992d2f794ab727 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Thu, 21 Nov 2024 10:07:22 +0530
Subject: [PATCH 144/165] fixed catchement areas were disappering after change
 in map style

---
 commcare_connect/static/js/project.js         |  39 ++--
 .../templates/opportunity/user_profile.html   | 186 ++++++++++++------
 2 files changed, 145 insertions(+), 80 deletions(-)

diff --git a/commcare_connect/static/js/project.js b/commcare_connect/static/js/project.js
index b2665b39..d960110d 100644
--- a/commcare_connect/static/js/project.js
+++ b/commcare_connect/static/js/project.js
@@ -69,14 +69,19 @@ function addCatchmentAreas(map, catchments) {
   const INACTIVE_COLOR = '#ff4d4d';
   const CIRCLE_OPACITY = 0.15;
 
-  map.on('load', () => {
-    const catchmentCircles = catchments.map((catchment) =>
-      circle([catchment.lng, catchment.lat], catchment.radius, {
-        units: 'meters',
-        properties: { active: catchment.active },
-      }),
-    );
+  const catchmentCircles = catchments.map((catchment) =>
+    circle([catchment.lng, catchment.lat], catchment.radius, {
+      units: 'meters',
+      properties: { active: catchment.active },
+    }),
+  );
 
+  if (map.getSource('catchment_circles')) {
+    map.getSource('catchment_circles').setData({
+      type: 'FeatureCollection',
+      features: catchmentCircles,
+    });
+  } else {
     map.addSource('catchment_circles', {
       type: 'geojson',
       data: {
@@ -105,17 +110,17 @@ function addCatchmentAreas(map, catchments) {
         'line-opacity': 0.5,
       },
     });
+  }
 
-    if (catchments?.length) {
-      window.Alpine.nextTick(() => {
-        const legendElement = document.getElementById('legend');
-        if (legendElement) {
-          const legendData = window.Alpine.$data(legendElement);
-          legendData.show = true;
-        }
-      });
-    }
-  });
+  if (catchments?.length) {
+    window.Alpine.nextTick(() => {
+      const legendElement = document.getElementById('legend');
+      if (legendElement) {
+        const legendData = window.Alpine.$data(legendElement);
+        legendData.show = true;
+      }
+    });
+  }
 }
 
 window.addCatchmentAreas = addCatchmentAreas;
diff --git a/commcare_connect/templates/opportunity/user_profile.html b/commcare_connect/templates/opportunity/user_profile.html
index 084532d0..174c8cf4 100644
--- a/commcare_connect/templates/opportunity/user_profile.html
+++ b/commcare_connect/templates/opportunity/user_profile.html
@@ -22,63 +22,112 @@
       </ol>
     </nav>
 
-  <div class="card w-50 mx-auto">
-    <div class="card-body">
-      <h1 class="display-4 mb-0 text-center">{{access.display_name}}</h1>
-      <div class="text-uppercase mb-2 text-center"><small>{{access.user.username}}</small></div>
-      <hr />
-      <div class="text-center">
-        <div class="row mb-1 pb-1">
-          <div class="col fw-bold">{% translate "Phone" %}</div>
-          <div class="col">{{access.user.phone_number}}</div>
-        </div>
-        <div class="row mb-1 pb-1">
-          <div class="col fw-bold">{% translate "Learn Progress" %}</div>
-          <div class="col">{{access.learn_progress}}%</div>
-        </div>
-        <div class="row mb-1 pb-1">
-          <div class="col fw-bold">{% translate "Total Visits" %}</div>
-          <div class="col">{{access.visit_count}}</div>
+  <div class="container-fluid py-4">
+    <div class="card shadow-sm">
+      <div class="card-body">
+        <!-- Profile Header -->
+        <div class="text-center mb-4">
+          <div class="bg-primary rounded-circle d-inline-flex align-items-center justify-content-center mb-3"
+               style="width: 80px; height: 80px;">
+            <span class="text-white display-6">{{access.display_name|slice:":1"}}</span>
+          </div>
+          <h1 class="h2 mb-1">{{access.display_name}}</h1>
+          <div class="text-muted"><small>{{access.user.username}}</small></div>
         </div>
-        <div class="row">
-          <div class="col fw-bold">{% translate "Last Visit Date" %}</div>
-          <div class="col">{{access.last_visit_date}}</div>
+
+        <!-- Stats Grid -->
+        <div class="row g-3 mb-4">
+          <div class="col-md-6 col-lg-3">
+            <div class="card h-100 border-0 bg-light">
+              <div class="card-body text-center">
+                <i class="bi bi-telephone mb-2 text-primary fs-4"></i>
+                <h6 class="text-muted mb-1">{% translate "Phone" %}</h6>
+                <div>{{access.user.phone_number}}</div>
+              </div>
+            </div>
+          </div>
+          <div class="col-md-6 col-lg-3">
+            <div class="card h-100 border-0 bg-light">
+              <div class="card-body text-center">
+                <i class="bi bi-graph-up mb-2 text-success fs-4"></i>
+                <h6 class="text-muted mb-1">{% translate "Learn Progress" %}</h6>
+                <div class="d-flex align-items-center justify-content-center">
+                  <div class="progress w-75 bg-white">
+                    <div class="progress-bar bg-success" role="progressbar"
+                         style="width: {{access.learn_progress}}%" aria-valuenow="{{access.learn_progress}}"
+                         aria-valuemin="0" aria-valuemax="100"></div>
+                  </div>
+                  <span class="ms-2">{{access.learn_progress}}%</span>
+                </div>
+              </div>
+            </div>
+          </div>
+          <div class="col-md-6 col-lg-3">
+            <div class="card h-100 border-0 bg-light">
+              <div class="card-body text-center">
+                <i class="bi bi-pin-map mb-2 text-info fs-4"></i>
+                <h6 class="text-muted mb-1">{% translate "Total Visits" %}</h6>
+                <div>{{access.visit_count}}</div>
+              </div>
+            </div>
+          </div>
+          <div class="col-md-6 col-lg-3">
+            <div class="card h-100 border-0 bg-light">
+              <div class="card-body text-center">
+                <i class="bi bi-calendar mb-2 text-warning fs-4"></i>
+                <h6 class="text-muted mb-1">{% translate "Last Visit" %}</h6>
+                <div>{{access.last_visit_date}}</div>
+              </div>
+            </div>
+          </div>
         </div>
-      </div>
-      <div id="menu" class="btn-group btn-group-toggle" data-toggle="buttons" style="position: absolute; top: 0; left: 0;">
-      <label class="btn btn-secondary active">
-        <input type="radio" name="options" id="streets-v12" autocomplete="off" checked> Streets
-      </label>
-      <label class="btn btn-secondary">
-        <input type="radio" name="options" id="satellite-streets-v12" autocomplete="off"> Satellite
-      </label>
-    </div>
-       <div class="position-relative">
-        <div id="user-visit-map" class="mt-3 mb-2 rounded border" style="height: 300px;"></div>
-        <div id="legend" class="card position-absolute bottom-0 end-0 m-2 z-1" x-data="{ show: false}" x-show="show">
-          <div class="card-body">
-            <h6 class="card-title">Catchment Areas</h6>
-            <div class="mb-2 d-flex align-items-center">
-            <span class="d-inline-block me-2"
-                  style="width: 20px; height: 20px; opacity: 0.3; background-color: #3366ff;"></span>
-              <span>Active</span>
+
+        <!-- Map Section -->
+        <div class="position-relative">
+          <div class="card">
+            <div class="card-header bg-light d-flex justify-content-between align-items-center">
+              <h5 class="mb-0">Visit Locations</h5>
+              <div class="btn-group btn-group-sm">
+                <button type="button" class="btn btn-outline-primary active pe-none" id="streets-v12">
+                  <i class="bi bi-map me-1"></i>Streets
+                </button>
+                <button type="button" class="btn btn-outline-primary" id="satellite-streets-v12">
+                  <i class="bi bi-globe me-1"></i>Satellite
+                </button>
+              </div>
             </div>
-            <div class="d-flex align-items-center">
-            <span class="d-inline-block me-2"
-                  style="width: 20px; height: 20px; opacity: 0.3; background-color: #ff4d4d;"></span>
-              <span>Inactive</span>
+            <div class="card-body p-0">
+              <div id="user-visit-map" class="w-100" style="height: 600px;"></div>
+              <div id="legend" class="card position-absolute bottom-0 end-0 m-3 shadow-sm" x-data="{ show: false}" x-show="show">
+                <div class="card-body p-3">
+                  <h6 class="card-title mb-3">Catchment Areas</h6>
+                  <div class="mb-2 d-flex align-items-center">
+                    <span class="d-inline-block me-2 rounded bg-primary" style="width: 20px; height: 20px; opacity: 0.3;"></span>
+                    <span>Active</span>
+                  </div>
+                  <div class="d-flex align-items-center">
+                    <span class="d-inline-block me-2 rounded bg-danger" style="width: 20px; height: 20px; opacity: 0.3;"></span>
+                    <span>Inactive</span>
+                  </div>
+                </div>
+              </div>
             </div>
           </div>
         </div>
+
+        <!-- Action Buttons -->
+        <div class="text-center mt-4">
+          {% if access.suspended %}
+            <a class="btn btn-success btn-lg" href="{% url "opportunity:revoke_user_suspension" org_slug=request.org opp_id=access.opportunity_id pk=access.pk %}?next={{ request.path|urlencode }}">
+              <i class="bi bi-person-check me-1"></i>{% translate "Revoke Suspension" %}
+            </a>
+          {% else %}
+            <button class="btn btn-danger btn-lg" data-bs-toggle="modal" data-bs-target="#suspendUserModal">
+              <i class="bi bi-person-x me-1"></i>{% translate "Suspend" %}
+            </button>
+          {% endif %}
+        </div>
       </div>
-      {% if access.suspended %}
-        <a class="btn btn-success"
-           href="{% url "opportunity:revoke_user_suspension" org_slug=request.org opp_id=access.opportunity_id pk=access.pk %}?next={{ request.path|urlencode }}">
-          {% translate "Revoke Suspension" %}
-        </a>
-      {% else %}
-        <button class="btn btn-danger" data-bs-toggle="modal" data-bs-target="#suspendUserModal">{% translate "Suspend" %}</button>
-      {% endif %}
     </div>
   </div>
 {% endblock content %}
@@ -89,33 +138,44 @@ <h6 class="card-title">Catchment Areas</h6>
   {{ user_catchments|json_script:"userCatchments" }}
   <script type="module">
     window.addEventListener('DOMContentLoaded', () => {
-      mapboxgl.accessToken = "pk.eyJ1IjoiaHlhZGF2MTAiLCJhIjoiY2x5ZHYxMDFuMGQzeDJrczZtamd5dXBqcSJ9.bjt_mM0gLycVpn_Q-B3zow";
+      mapboxgl.accessToken = "{{ MAPBOX_TOKEN }}";
       const map = new mapboxgl.Map({
         container: 'user-visit-map',
-        style: 'mapbox://styles/mapbox/satellite-streets-v12',
+        style: 'mapbox://styles/mapbox/streets-v12', // Default to streets style
         center: [{{ lng_avg }}, {{ lat_avg }}],
         zoom: 14,
       });
+
       const userVisits = JSON.parse(document.getElementById('userVisits').textContent);
+      const userCatchments = JSON.parse(document.getElementById('userCatchments').textContent);
+
       userVisits.forEach(loc => {
         new mapboxgl.Marker()
           .setLngLat([loc.lng, loc.lat])
           .setPopup(new mapboxgl.Popup().setHTML(`${loc.entity_name}<br />${loc.visit_date}`))
           .addTo(map)
       })
-      addAccuracyCircles(map, userVisits);
-      const userCatchments = JSON.parse(document.getElementById('userCatchments').textContent);
-      addCatchmentAreas(map, userCatchments)
 
-      const styleList = document.getElementById('menu');
-      const inputs = styleList.getElementsByTagName('input');
+      document.getElementById('streets-v12').addEventListener('click', (e) => {
+        map.setStyle('mapbox://styles/mapbox/streets-v12');
+        e.target.classList.add('active');
+        e.target.classList.add('pe-none');
+        document.getElementById('satellite-streets-v12').classList.remove('active');
+        document.getElementById('satellite-streets-v12').classList.remove('pe-none');
+      });
+
+      document.getElementById('satellite-streets-v12').addEventListener('click', (e) => {
+        map.setStyle('mapbox://styles/mapbox/satellite-streets-v12');
+        e.target.classList.add('active');
+        e.target.classList.add('pe-none');
+        document.getElementById('streets-v12').classList.remove('active');
+        document.getElementById('streets-v12').classList.remove('pe-none');
+      });
 
-      for (const input of inputs) {
-          input.onclick = (click) => {
-              const styleId = click.target.id;
-              map.setStyle('mapbox://styles/mapbox/' + styleId);
-          };
-      }
+      map.on('style.load', () => {
+        addAccuracyCircles(map, userVisits);
+        addCatchmentAreas(map, userCatchments);
+      });
     });
   </script>
 {% endblock %}

From eecad814876217662cbe26f5366a2fff6749cdc2 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Thu, 21 Nov 2024 10:47:18 +0530
Subject: [PATCH 145/165] refactor code

---
 .../templates/opportunity/user_profile.html   | 27 ++++++++++---------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/commcare_connect/templates/opportunity/user_profile.html b/commcare_connect/templates/opportunity/user_profile.html
index 174c8cf4..a031f3f5 100644
--- a/commcare_connect/templates/opportunity/user_profile.html
+++ b/commcare_connect/templates/opportunity/user_profile.html
@@ -102,11 +102,13 @@ <h5 class="mb-0">Visit Locations</h5>
                 <div class="card-body p-3">
                   <h6 class="card-title mb-3">Catchment Areas</h6>
                   <div class="mb-2 d-flex align-items-center">
-                    <span class="d-inline-block me-2 rounded bg-primary" style="width: 20px; height: 20px; opacity: 0.3;"></span>
+                    <span class="d-inline-block me-2 rounded"
+                          style="width: 20px; height: 20px; opacity: 0.3; background-color: #3366ff;"></span>
                     <span>Active</span>
                   </div>
                   <div class="d-flex align-items-center">
-                    <span class="d-inline-block me-2 rounded bg-danger" style="width: 20px; height: 20px; opacity: 0.3;"></span>
+                    <span class="d-inline-block me-2 rounded"
+                          style="width: 20px; height: 20px; opacity: 0.3; background-color: #ff4d4d;"></span>
                     <span>Inactive</span>
                   </div>
                 </div>
@@ -156,20 +158,21 @@ <h6 class="card-title mb-3">Catchment Areas</h6>
           .addTo(map)
       })
 
+      const setMapStyle = (styleId, targetElementId, otherElementId) => {
+        map.setStyle(styleId);
+        const targetElement = document.getElementById(targetElementId);
+        targetElement.classList.add('active', 'pe-none');
+
+        const otherElement = document.getElementById(otherElementId);
+        otherElement.classList.remove('active', 'pe-none');
+      };
+
       document.getElementById('streets-v12').addEventListener('click', (e) => {
-        map.setStyle('mapbox://styles/mapbox/streets-v12');
-        e.target.classList.add('active');
-        e.target.classList.add('pe-none');
-        document.getElementById('satellite-streets-v12').classList.remove('active');
-        document.getElementById('satellite-streets-v12').classList.remove('pe-none');
+        setMapStyle('mapbox://styles/mapbox/streets-v12', 'streets-v12', 'satellite-streets-v12');
       });
 
       document.getElementById('satellite-streets-v12').addEventListener('click', (e) => {
-        map.setStyle('mapbox://styles/mapbox/satellite-streets-v12');
-        e.target.classList.add('active');
-        e.target.classList.add('pe-none');
-        document.getElementById('streets-v12').classList.remove('active');
-        document.getElementById('streets-v12').classList.remove('pe-none');
+        setMapStyle('mapbox://styles/mapbox/satellite-streets-v12', 'satellite-streets-v12', 'streets-v12');
       });
 
       map.on('style.load', () => {

From 5ce6cde06d0cba2dde5f7bede20cbae460579926 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Thu, 21 Nov 2024 13:40:04 +0530
Subject: [PATCH 146/165] Fix error in payment unit admin

---
 commcare_connect/opportunity/admin.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/commcare_connect/opportunity/admin.py b/commcare_connect/opportunity/admin.py
index e833dd19..2c026ec3 100644
--- a/commcare_connect/opportunity/admin.py
+++ b/commcare_connect/opportunity/admin.py
@@ -127,8 +127,8 @@ def get_username(self, obj):
 @admin.register(PaymentUnit)
 class PaymentUnitAdmin(admin.ModelAdmin):
     list_display = ["name", "get_opp_name"]
-    search_fields = ["name"]
+    search_fields = ["name", "opportunity__name"]
 
     @admin.display(description="Opportunity Name")
     def get_opp_name(self, obj):
-        return obj.opportunity_access.opportunity.name
+        return obj.opportunity.name

From 4ff0ae9065d0ff0b383baba0039c480be0cfc6c6 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Thu, 21 Nov 2024 14:30:31 +0530
Subject: [PATCH 147/165] used alpine @click event

---
 .../templates/opportunity/user_profile.html   | 57 +++++++++----------
 1 file changed, 28 insertions(+), 29 deletions(-)

diff --git a/commcare_connect/templates/opportunity/user_profile.html b/commcare_connect/templates/opportunity/user_profile.html
index a031f3f5..714a1c68 100644
--- a/commcare_connect/templates/opportunity/user_profile.html
+++ b/commcare_connect/templates/opportunity/user_profile.html
@@ -83,15 +83,14 @@ <h6 class="text-muted mb-1">{% translate "Last Visit" %}</h6>
         </div>
 
         <!-- Map Section -->
-        <div class="position-relative">
-          <div class="card">
+        <div class="card position-relative" x-data="{ currentStyle: 'streets-v12' }">
             <div class="card-header bg-light d-flex justify-content-between align-items-center">
               <h5 class="mb-0">Visit Locations</h5>
               <div class="btn-group btn-group-sm">
-                <button type="button" class="btn btn-outline-primary active pe-none" id="streets-v12">
+                <button type="button" @click="currentStyle = 'streets-v12'" class="btn btn-outline-primary" :class="{ 'active pe-none': currentStyle === 'streets-v12' }" id="streets-v12">
                   <i class="bi bi-map me-1"></i>Streets
                 </button>
-                <button type="button" class="btn btn-outline-primary" id="satellite-streets-v12">
+                <button type="button" @click="currentStyle = 'satellite-streets-v12'" class="btn btn-outline-primary" :class="{ 'active pe-none': currentStyle === 'satellite-streets-v12' }" id="satellite-streets-v12">
                   <i class="bi bi-globe me-1"></i>Satellite
                 </button>
               </div>
@@ -115,7 +114,6 @@ <h6 class="card-title mb-3">Catchment Areas</h6>
               </div>
             </div>
           </div>
-        </div>
 
         <!-- Action Buttons -->
         <div class="text-center mt-4">
@@ -143,7 +141,7 @@ <h6 class="card-title mb-3">Catchment Areas</h6>
       mapboxgl.accessToken = "{{ MAPBOX_TOKEN }}";
       const map = new mapboxgl.Map({
         container: 'user-visit-map',
-        style: 'mapbox://styles/mapbox/streets-v12', // Default to streets style
+        style: 'mapbox://styles/mapbox/streets-v12',
         center: [{{ lng_avg }}, {{ lat_avg }}],
         zoom: 14,
       });
@@ -151,33 +149,34 @@ <h6 class="card-title mb-3">Catchment Areas</h6>
       const userVisits = JSON.parse(document.getElementById('userVisits').textContent);
       const userCatchments = JSON.parse(document.getElementById('userCatchments').textContent);
 
-      userVisits.forEach(loc => {
-        new mapboxgl.Marker()
-          .setLngLat([loc.lng, loc.lat])
-          .setPopup(new mapboxgl.Popup().setHTML(`${loc.entity_name}<br />${loc.visit_date}`))
-          .addTo(map)
-      })
-
-      const setMapStyle = (styleId, targetElementId, otherElementId) => {
-        map.setStyle(styleId);
-        const targetElement = document.getElementById(targetElementId);
-        targetElement.classList.add('active', 'pe-none');
+      map.on('load', () => {
+        userVisits.forEach(loc => {
+          new mapboxgl.Marker()
+            .setLngLat([loc.lng, loc.lat])
+            .setPopup(new mapboxgl.Popup().setHTML(`${loc.entity_name}<br />${loc.visit_date}`))
+            .addTo(map)
+        });
 
-        const otherElement = document.getElementById(otherElementId);
-        otherElement.classList.remove('active', 'pe-none');
-      };
-
-      document.getElementById('streets-v12').addEventListener('click', (e) => {
-        setMapStyle('mapbox://styles/mapbox/streets-v12', 'streets-v12', 'satellite-streets-v12');
+        addAccuracyCircles(map, userVisits);
+        addCatchmentAreas(map, userCatchments);
       });
 
-      document.getElementById('satellite-streets-v12').addEventListener('click', (e) => {
-        setMapStyle('mapbox://styles/mapbox/satellite-streets-v12', 'satellite-streets-v12', 'streets-v12');
-      });
+      // Watch for Alpine.js style changes
+      Alpine.effect(() => {
+        const alpineData = Alpine.$data(document.querySelector('[x-data]'));
+        const currentStyle = alpineData.currentStyle;
+        const styles = {
+          'streets-v12': 'mapbox://styles/mapbox/streets-v12',
+          'satellite-streets-v12': 'mapbox://styles/mapbox/satellite-streets-v12'
+        };
+        map.setStyle(styles[currentStyle]);
 
-      map.on('style.load', () => {
-        addAccuracyCircles(map, userVisits);
-        addCatchmentAreas(map, userCatchments);
+        // Re-add circles and catchments after style changes
+        map.once('style.load', () => {
+          alpineData.currentStyle = currentStyle;
+          addAccuracyCircles(map, userVisits);
+          addCatchmentAreas(map, userCatchments);
+        });
       });
     });
   </script>

From 1b57aec98ba6f1861f9babe62f9a8a060ef75a69 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Thu, 21 Nov 2024 21:43:03 +0530
Subject: [PATCH 148/165] fix for accuracy circles

---
 commcare_connect/static/js/project.js | 32 ++++++++++++++++++---------
 1 file changed, 21 insertions(+), 11 deletions(-)

diff --git a/commcare_connect/static/js/project.js b/commcare_connect/static/js/project.js
index d960110d..ef6e4946 100644
--- a/commcare_connect/static/js/project.js
+++ b/commcare_connect/static/js/project.js
@@ -24,13 +24,22 @@ window.circle = circle;
  * @param {Array.<{lng: float, lat: float, precision: float}> visit_data - Visit location data for User
  */
 function addAccuracyCircles(map, visit_data) {
-  map.on('load', () => {
-    const visit_accuracy_circles = [];
-    visit_data.forEach((loc) => {
-      visit_accuracy_circles.push(
-        circle([loc.lng, loc.lat], loc.precision, { units: 'meters' }),
-      );
+  const FILL_OPACITY = 0.1;
+  const OUTLINE_COLOR = '#fcbf49';
+  const OUTLINE_WIDTH = 3;
+  const OUTLINE_OPACITY = 0.5;
+
+  const visit_accuracy_circles = visit_data.map((loc) =>
+    circle([loc.lng, loc.lat], loc.precision, { units: 'meters' }),
+  );
+
+  // Check if the source exists, then update or add the source
+  if (map.getSource('visit_accuracy_circles')) {
+    map.getSource('visit_accuracy_circles').setData({
+      type: 'FeatureCollection',
+      features: visit_accuracy_circles,
     });
+  } else {
     map.addSource('visit_accuracy_circles', {
       type: 'geojson',
       data: {
@@ -45,21 +54,22 @@ function addAccuracyCircles(map, visit_data) {
       type: 'fill',
       paint: {
         'fill-antialias': true,
-        'fill-opacity': 0.3,
+        'fill-opacity': FILL_OPACITY,
       },
     });
 
+    // Add the outline layer
     map.addLayer({
       id: 'visit-accuracy-circle-outlines-layer',
       source: 'visit_accuracy_circles',
       type: 'line',
       paint: {
-        'line-color': '#fcbf49',
-        'line-width': 3,
-        'line-opacity': 0.5,
+        'line-color': OUTLINE_COLOR,
+        'line-width': OUTLINE_WIDTH,
+        'line-opacity': OUTLINE_OPACITY,
       },
     });
-  });
+  }
 }
 
 window.addAccuracyCircles = addAccuracyCircles;

From e171f22ecaa04805ea9b7af016cb02cc1debe497 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Mon, 25 Nov 2024 14:00:19 +0530
Subject: [PATCH 149/165] Add opportunity date tests

---
 .../tests/test_receiver_integration.py          | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/commcare_connect/form_receiver/tests/test_receiver_integration.py b/commcare_connect/form_receiver/tests/test_receiver_integration.py
index 2757a57c..1c1f79ed 100644
--- a/commcare_connect/form_receiver/tests/test_receiver_integration.py
+++ b/commcare_connect/form_receiver/tests/test_receiver_integration.py
@@ -604,13 +604,20 @@ def test_receiver_visit_review_status(
 
 
 @pytest.mark.parametrize(
-    "paymentunit_options, visit_status",
+    "opportunity, paymentunit_options, visit_status",
     [
-        ({"start_date": now().date()}, VisitValidationStatus.approved),
-        ({"start_date": now() + datetime.timedelta(days=2)}, VisitValidationStatus.trial),
-        ({"end_date": now().date()}, VisitValidationStatus.approved),
-        ({"end_date": now() - datetime.timedelta(days=2)}, VisitValidationStatus.over_limit),
+        ({}, {"start_date": now().date()}, VisitValidationStatus.approved),
+        ({}, {"start_date": now() + datetime.timedelta(days=2)}, VisitValidationStatus.trial),
+        ({}, {"end_date": now().date()}, VisitValidationStatus.approved),
+        ({}, {"end_date": now() - datetime.timedelta(days=2)}, VisitValidationStatus.over_limit),
+        ({"opp_options": {"start_date": now().date()}}, {}, VisitValidationStatus.approved),
+        ({"opp_options": {"start_date": now() + datetime.timedelta(days=2)}}, {}, VisitValidationStatus.trial),
+        ({"opp_options": {"end_date": now().date()}}, {}, VisitValidationStatus.approved),
+        # NOTE: this test case fails as opportunities with past end_date are marked
+        # as inactive, and are not processed in the form processor
+        # ({"opp_options": {"end_date": now() - datetime.timedelta(days=2)}}, {}, VisitValidationStatus.over_limit),
     ],
+    indirect=["opportunity"],
 )
 def test_receiver_visit_payment_unit_dates(
     mobile_user_with_connect_link: User, api_client: APIClient, opportunity: Opportunity, visit_status

From 98e848c6df9a77c18acf7d532dc02344a5bb6a9e Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Tue, 26 Nov 2024 12:58:34 +0530
Subject: [PATCH 150/165] Add smaller button

---
 commcare_connect/opportunity/tables.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/commcare_connect/opportunity/tables.py b/commcare_connect/opportunity/tables.py
index d722ad60..6fd3953d 100644
--- a/commcare_connect/opportunity/tables.py
+++ b/commcare_connect/opportunity/tables.py
@@ -187,7 +187,7 @@ def render_view_profile(self, record):
                 (
                     '<button hx-post="{}" hx-swap="none" '
                     'hx-confirm="Please confirm to delete the User Invite." '
-                    'class="btn btn-danger">Delete</button>'
+                    'class="btn btn-danger btn-sm">Delete</button>'
                 ),
                 invite_delete_url,
             )

From 9d7991a553d0aa550c3fb28ad7875f7070ee0b0f Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Tue, 26 Nov 2024 13:27:19 +0530
Subject: [PATCH 151/165] Fix wrong text check causing issues in hq user
 creation

---
 commcare_connect/users/helpers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/commcare_connect/users/helpers.py b/commcare_connect/users/helpers.py
index 5cfbe50b..04264954 100644
--- a/commcare_connect/users/helpers.py
+++ b/commcare_connect/users/helpers.py
@@ -34,7 +34,7 @@ def create_hq_user(user, domain, api_key):
     try:
         hq_request.raise_for_status()
     except httpx.HTTPStatusError as e:
-        if e.response.status_code == 400 and "already exists" in e.response.text:
+        if e.response.status_code == 400 and "already taken" in e.response.text:
             return True
         raise CommCareHQAPIException(
             f"{e.response.status_code} Error response {e.response.text} while creating user {user.username}"

From e4deacd0d334e1b38a7c891c64b8bf894ce37e27 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Tue, 26 Nov 2024 17:20:47 +0530
Subject: [PATCH 152/165] Add make view profile button

---
 commcare_connect/opportunity/tables.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/commcare_connect/opportunity/tables.py b/commcare_connect/opportunity/tables.py
index 6fd3953d..e361f4c0 100644
--- a/commcare_connect/opportunity/tables.py
+++ b/commcare_connect/opportunity/tables.py
@@ -195,7 +195,7 @@ def render_view_profile(self, record):
             "opportunity:user_profile",
             kwargs={"org_slug": self.org_slug, "opp_id": record.opportunity.id, "pk": record.opportunity_access_id},
         )
-        return format_html('<a href="{}">View Profile</a>', url)
+        return format_html('<a class="btn btn-primary btn-sm" href="{}">View Profile</a>', url)
 
     def render_started_learning(self, record, value):
         return date_with_time_popup(self, value)

From fe18fc6fd656d3f303f7bb4919a4461d4d68f9e2 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Wed, 27 Nov 2024 17:45:08 +0530
Subject: [PATCH 153/165] Fix add check for no end_date

---
 commcare_connect/opportunity/forms.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/commcare_connect/opportunity/forms.py b/commcare_connect/opportunity/forms.py
index dc1f4fae..056c8453 100644
--- a/commcare_connect/opportunity/forms.py
+++ b/commcare_connect/opportunity/forms.py
@@ -692,7 +692,7 @@ def clean(self):
                     "optional_deliver_units",
                     error=f"{deliver_unit_obj.name} cannot be marked both Required and Optional",
                 )
-            if cleaned_data["end_date"] < now().date():
+            if cleaned_data["end_date"] and cleaned_data["end_date"] < now().date():
                 self.add_error("end_date", "Please provide a valid end date.")
         return cleaned_data
 

From 0d7b3589e04a747f356b8ad059cfcea1b652b6f0 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Wed, 27 Nov 2024 19:23:46 +0530
Subject: [PATCH 154/165] Remove commented code

---
 .../form_receiver/tests/test_receiver_integration.py           | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/commcare_connect/form_receiver/tests/test_receiver_integration.py b/commcare_connect/form_receiver/tests/test_receiver_integration.py
index 1c1f79ed..7af90d3a 100644
--- a/commcare_connect/form_receiver/tests/test_receiver_integration.py
+++ b/commcare_connect/form_receiver/tests/test_receiver_integration.py
@@ -613,9 +613,6 @@ def test_receiver_visit_review_status(
         ({"opp_options": {"start_date": now().date()}}, {}, VisitValidationStatus.approved),
         ({"opp_options": {"start_date": now() + datetime.timedelta(days=2)}}, {}, VisitValidationStatus.trial),
         ({"opp_options": {"end_date": now().date()}}, {}, VisitValidationStatus.approved),
-        # NOTE: this test case fails as opportunities with past end_date are marked
-        # as inactive, and are not processed in the form processor
-        # ({"opp_options": {"end_date": now() - datetime.timedelta(days=2)}}, {}, VisitValidationStatus.over_limit),
     ],
     indirect=["opportunity"],
 )

From 7ce05e763dd84e0402f376103543bc7db6a64760 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Thu, 28 Nov 2024 12:53:08 +0530
Subject: [PATCH 155/165] fix migration order

---
 ...invited_date.py => 0062_opportunityaccess_invited_date.py} | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
 rename commcare_connect/opportunity/migrations/{0061_opportunityaccess_invited_date.py => 0062_opportunityaccess_invited_date.py} (69%)

diff --git a/commcare_connect/opportunity/migrations/0061_opportunityaccess_invited_date.py b/commcare_connect/opportunity/migrations/0062_opportunityaccess_invited_date.py
similarity index 69%
rename from commcare_connect/opportunity/migrations/0061_opportunityaccess_invited_date.py
rename to commcare_connect/opportunity/migrations/0062_opportunityaccess_invited_date.py
index b5cebef6..12ef7484 100644
--- a/commcare_connect/opportunity/migrations/0061_opportunityaccess_invited_date.py
+++ b/commcare_connect/opportunity/migrations/0062_opportunityaccess_invited_date.py
@@ -1,11 +1,11 @@
-# Generated by Django 4.2.5 on 2024-10-15 10:25
+# Generated by Django 4.2.5 on 2024-11-28 07:22
 
 from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
     dependencies = [
-        ("opportunity", "0060_completedwork_payment_date"),
+        ("opportunity", "0061_opportunityclaimlimit_end_date_paymentunit_end_date_and_more"),
     ]
 
     operations = [

From d33b0af2d8a452dab6068aad6a37da20883a7acb Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Thu, 28 Nov 2024 16:53:02 +0530
Subject: [PATCH 156/165] Add 15 minute timeout on cache lock

---
 commcare_connect/opportunity/visit_import.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/commcare_connect/opportunity/visit_import.py b/commcare_connect/opportunity/visit_import.py
index c544da56..425b0454 100644
--- a/commcare_connect/opportunity/visit_import.py
+++ b/commcare_connect/opportunity/visit_import.py
@@ -169,7 +169,7 @@ def update_payment_accrued(opportunity: Opportunity, users):
     """Updates payment accrued for completed and approved CompletedWork instances."""
     access_objects = OpportunityAccess.objects.filter(user__in=users, opportunity=opportunity, suspended=False)
     for access in access_objects:
-        with cache.lock(f"update_payment_accrued_lock_{access.id}"):
+        with cache.lock(f"update_payment_accrued_lock_{access.id}", timeout=900):
             completed_works = access.completedwork_set.exclude(
                 status__in=[CompletedWorkStatus.rejected, CompletedWorkStatus.over_limit]
             ).select_related("payment_unit")

From 8c161e6e31a6c4baaf95053061429369f66d7f2b Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Fri, 29 Nov 2024 13:49:17 +0530
Subject: [PATCH 157/165] added the org slug in table

---
 commcare_connect/opportunity/tables.py | 16 +++++++++-------
 commcare_connect/opportunity/views.py  |  2 +-
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/commcare_connect/opportunity/tables.py b/commcare_connect/opportunity/tables.py
index a856650f..ca9586ad 100644
--- a/commcare_connect/opportunity/tables.py
+++ b/commcare_connect/opportunity/tables.py
@@ -374,7 +374,7 @@ class Meta:
         )
 
 
-class UserVisitReviewTable(tables.Table):
+class UserVisitReviewTable(OrgContextTable):
     pk = columns.CheckBoxColumn(
         accessor="pk",
         verbose_name="",
@@ -389,12 +389,7 @@ class UserVisitReviewTable(tables.Table):
     visit_date = columns.Column()
     created_on = columns.Column(accessor="review_created_on", verbose_name="Review Requested On")
     review_status = columns.Column(verbose_name="Program Manager Review")
-    user_visit = columns.LinkColumn(
-        "opportunity:visit_verification",
-        verbose_name="User Visit",
-        text="View",
-        args=[utils.A("opportunity__organization__slug"), utils.A("pk")],
-    )
+    user_visit = columns.Column(verbose_name="User Visit", empty_values=())
 
     class Meta:
         model = UserVisit
@@ -412,6 +407,13 @@ class Meta:
         )
         empty_text = "No visits submitted for review."
 
+    def render_user_visit(self, record):
+        url = reverse(
+            "opportunity:visit_verification",
+            kwargs={"org_slug": self.org_slug, "pk": record.pk},
+        )
+        return mark_safe(f'<a href="{url}">View</a>')
+
 
 class PaymentReportTable(tables.Table):
     payment_unit = columns.Column(verbose_name="Payment Unit")
diff --git a/commcare_connect/opportunity/views.py b/commcare_connect/opportunity/views.py
index a53c0050..645ba4ef 100644
--- a/commcare_connect/opportunity/views.py
+++ b/commcare_connect/opportunity/views.py
@@ -1112,7 +1112,7 @@ def user_visit_review(request, org_slug, opp_id):
     user_visit_reviews = UserVisit.objects.filter(opportunity=opportunity, review_created_on__isnull=False).order_by(
         "visit_date"
     )
-    table = UserVisitReviewTable(user_visit_reviews)
+    table = UserVisitReviewTable(user_visit_reviews, org_slug=request.org.slug)
     if not is_program_manager:
         table.exclude = ("pk",)
     if request.POST and is_program_manager:

From 41f5c25e1e24c1fd9c2e3931332d757d7211f597 Mon Sep 17 00:00:00 2001
From: Sravan Reddy <sravfeyn@gmail.com>
Date: Fri, 29 Nov 2024 16:15:38 +0530
Subject: [PATCH 158/165] Check for None value

---
 commcare_connect/form_receiver/processor.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/commcare_connect/form_receiver/processor.py b/commcare_connect/form_receiver/processor.py
index 2872e81e..aa8c844a 100644
--- a/commcare_connect/form_receiver/processor.py
+++ b/commcare_connect/form_receiver/processor.py
@@ -262,7 +262,8 @@ def process_deliver_unit(user, xform: XForm, app: CommCareApp, opportunity: Oppo
     )
     completed_work_needs_save = False
     today = datetime.date.today()
-    if opportunity.start_date > today or (payment_unit.start_date and payment_unit.start_date > today):
+    paymentunit_startdate = payment_unit.start_date if payment_unit else None
+    if opportunity.start_date > today or (paymentunit_startdate and paymentunit_startdate > today):
         completed_work = None
         user_visit.status = VisitValidationStatus.trial
     else:

From 7dc1f53af863239f74c446069c31a6d5e5c44839 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Thu, 5 Dec 2024 14:29:29 +0530
Subject: [PATCH 159/165] command to delete duplicate visits

---
 .../commands/delete_duplicate_visits.py       | 62 +++++++++++++++++
 .../opportunity/tests/test_commands.py        | 66 +++++++++++++++++++
 2 files changed, 128 insertions(+)
 create mode 100644 commcare_connect/opportunity/management/commands/delete_duplicate_visits.py
 create mode 100644 commcare_connect/opportunity/tests/test_commands.py

diff --git a/commcare_connect/opportunity/management/commands/delete_duplicate_visits.py b/commcare_connect/opportunity/management/commands/delete_duplicate_visits.py
new file mode 100644
index 00000000..0dc0d5d7
--- /dev/null
+++ b/commcare_connect/opportunity/management/commands/delete_duplicate_visits.py
@@ -0,0 +1,62 @@
+from django.core.management import BaseCommand
+from django.db import transaction
+from django.db.models import Count
+
+from commcare_connect.opportunity.models import UserVisit
+
+
+class Command(BaseCommand):
+    help = "Clean up duplicate visits."
+
+    def add_arguments(self, parser, *args, **kwargs):
+        parser.add_argument("--opp", type=int, help="Opportunity ID to clean up duplicate visits.")
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            help="If set, just print the visits that would be deleted without actually deleting them.",
+        )
+
+    def handle(self, *args, **options):
+        opportunity_id = options.get("opp")
+        dry_run = options.get("dry_run")
+
+        duplicates = (
+            UserVisit.objects.filter(opportunity_id=opportunity_id)
+            .values("opportunity", "entity_id", "deliver_unit", "xform_id")
+            .annotate(visit_count=Count("id"))
+            .filter(visit_count__gt=1)
+        )
+
+        if dry_run:
+            self.stdout.write("Running in dry-run mode. No records will be deleted.")
+        else:
+            self.stdout.write("Attention: Records will be deleted!!")
+
+        with transaction.atomic():
+            for duplicate in duplicates:
+                visits = UserVisit.objects.filter(
+                    opportunity_id=opportunity_id,
+                    entity_id=duplicate["entity_id"],
+                    deliver_unit=duplicate["deliver_unit"],
+                    xform_id=duplicate["xform_id"],
+                ).order_by("id")
+
+                visits_to_delete = visits[1:]
+
+                for visit in visits_to_delete:
+                    message = (
+                        f"Identified duplicate visit: id={visit.id}, "
+                        f"xform_id={visit.xform_id}, entity_id={visit.entity_id}, "
+                        f"deliver_unit={visit.deliver_unit}, status={visit.status}"
+                    )
+                    self.stdout.write(message)
+
+                    if not dry_run:
+                        visit.delete()
+
+        if not dry_run:
+            self.stdout.write(
+                self.style.SUCCESS(f"Duplicate visits for opportunity {opportunity_id} deleted successfully.")
+            )
+        else:
+            self.stdout.write(f"Dry-run complete for opportunity {opportunity_id}")
diff --git a/commcare_connect/opportunity/tests/test_commands.py b/commcare_connect/opportunity/tests/test_commands.py
new file mode 100644
index 00000000..ace7e60e
--- /dev/null
+++ b/commcare_connect/opportunity/tests/test_commands.py
@@ -0,0 +1,66 @@
+from io import StringIO
+from uuid import uuid4
+
+import pytest
+from django.core.management import call_command
+
+from commcare_connect.opportunity.models import UserVisit
+from commcare_connect.opportunity.tests.factories import DeliverUnitFactory, UserVisitFactory
+
+
+@pytest.fixture
+def setup_opportunity_with_duplicates(db):
+    def _setup(opportunity, xform_id, entity_id, deliver_unit, num_duplicates):
+        first_visit = UserVisitFactory.create(
+            opportunity=opportunity, deliver_unit=deliver_unit, xform_id=xform_id, entity_id=entity_id
+        )
+        for _ in range(num_duplicates - 1):
+            UserVisitFactory.create(
+                opportunity=opportunity, deliver_unit=deliver_unit, xform_id=xform_id, entity_id=entity_id
+            )
+        return first_visit
+
+    return _setup
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize(
+    "num_duplicates,expected_remaining,dry_run",
+    [
+        (1, 1, True),  # No duplicates, dry-run mode
+        (2, 2, True),  # One duplicate, dry-run mode
+        (3, 3, True),  # Two duplicates, dry-run mode
+        (2, 1, False),  # One duplicate, actual deletion
+        (3, 1, False),  # Two duplicates, actual deletion
+    ],
+)
+def test_delete_duplicate_visits(
+    opportunity, setup_opportunity_with_duplicates, num_duplicates, expected_remaining, dry_run
+):
+    xform_id = str(uuid4())
+    entity_id = str(uuid4())
+    deliver_unit = DeliverUnitFactory()
+
+    first_visit = setup_opportunity_with_duplicates(opportunity, xform_id, entity_id, deliver_unit, num_duplicates)
+
+    out = StringIO()
+
+    if dry_run:
+        call_command("delete_duplicate_visits", "--opp", str(opportunity.id), "--dry-run", stdout=out)
+    else:
+        call_command("delete_duplicate_visits", "--opp", str(opportunity.id), stdout=out)
+
+    remaining_visits = UserVisit.objects.filter(
+        opportunity=opportunity, entity_id=entity_id, deliver_unit=deliver_unit, xform_id=xform_id
+    )
+
+    # Verify the count of remaining visits matches the expectation
+    assert remaining_visits.count() == expected_remaining
+
+    if not dry_run:
+        # Ensure the first visit is still present after actual deletion
+        assert remaining_visits.filter(id=first_visit.id).exists()
+        assert f"Duplicate visits for opportunity {opportunity.id} deleted successfully." in out.getvalue()
+    else:
+        assert remaining_visits.count() == num_duplicates
+        assert f"Dry-run complete for opportunity {opportunity.id}" in out.getvalue()

From 81c7471566776ec8ff56b917a5ab364ed8f4395b Mon Sep 17 00:00:00 2001
From: Sravan Reddy <sravfeyn@gmail.com>
Date: Thu, 5 Dec 2024 21:24:39 +0530
Subject: [PATCH 160/165] hqapikey to admin

---
 commcare_connect/opportunity/admin.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/commcare_connect/opportunity/admin.py b/commcare_connect/opportunity/admin.py
index 2c026ec3..0e407574 100644
--- a/commcare_connect/opportunity/admin.py
+++ b/commcare_connect/opportunity/admin.py
@@ -10,6 +10,7 @@
     DeliverUnitFlagRules,
     DeliveryType,
     FormJsonValidationRules,
+    HQApiKey,
     LearnModule,
     Opportunity,
     OpportunityAccess,
@@ -30,6 +31,7 @@
 admin.site.register(DeliveryType)
 admin.site.register(DeliverUnitFlagRules)
 admin.site.register(FormJsonValidationRules)
+admin.site.register(HQApiKey)
 
 
 @admin.register(Opportunity)

From 5b48ce157dae7f002b90370c92a482bd026c4a30 Mon Sep 17 00:00:00 2001
From: Cal Ellowitz <calellowitz@gmail.com>
Date: Thu, 5 Dec 2024 22:15:08 -0500
Subject: [PATCH 161/165] add option to include over limit works

---
 .../commands/auto_approval_opportunities.py      | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/commcare_connect/opportunity/management/commands/auto_approval_opportunities.py b/commcare_connect/opportunity/management/commands/auto_approval_opportunities.py
index 1292700c..d4d9448c 100644
--- a/commcare_connect/opportunity/management/commands/auto_approval_opportunities.py
+++ b/commcare_connect/opportunity/management/commands/auto_approval_opportunities.py
@@ -9,19 +9,27 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument(
-            "--opp", type=int, required=True, help="ID of the opportunity to run auto-approval logic on"
+            "--opp",
+            type=int,
+            required=True,
+            help="ID of the opportunity to run auto-approval logic on",
+        )
+        parser.add_argument(
+            "--include-over-limit", action="store_true", help="Also run auto-approval logic on over limit works"
         )
 
     def handle(self, *args, opp: int, **options):
+        include_over_limit = options.get("include_over_limit", False)
+        excluded = [CompletedWorkStatus.rejected]
+        if not include_over_limit:
+            excluded.append(CompletedWorkStatus.over_limit)
         try:
             opportunity = Opportunity.objects.get(id=opp)
             access_objects = OpportunityAccess.objects.filter(
                 opportunity=opportunity, suspended=False, opportunity__auto_approve_payments=True
             )
             for access in access_objects:
-                completed_works = access.completedwork_set.exclude(
-                    status__in=[CompletedWorkStatus.rejected, CompletedWorkStatus.over_limit]
-                )
+                completed_works = access.completedwork_set.exclude(status__in=excluded)
                 update_status(completed_works, access, False)
 
             self.stdout.write(self.style.SUCCESS(f"Successfully processed opportunity with id {opp}"))

From 8314a3a61612bd137230c9127eaae5dcd2e7f464 Mon Sep 17 00:00:00 2001
From: Cal Ellowitz <calellowitz@gmail.com>
Date: Thu, 5 Dec 2024 22:25:58 -0500
Subject: [PATCH 162/165] add arg to update payment

---
 .../management/commands/auto_approval_opportunities.py        | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/commcare_connect/opportunity/management/commands/auto_approval_opportunities.py b/commcare_connect/opportunity/management/commands/auto_approval_opportunities.py
index d4d9448c..48002a9e 100644
--- a/commcare_connect/opportunity/management/commands/auto_approval_opportunities.py
+++ b/commcare_connect/opportunity/management/commands/auto_approval_opportunities.py
@@ -17,9 +17,11 @@ def add_arguments(self, parser):
         parser.add_argument(
             "--include-over-limit", action="store_true", help="Also run auto-approval logic on over limit works"
         )
+        parser.add_argument("--update-payment", action="store_true", help="Update payment accrued")
 
     def handle(self, *args, opp: int, **options):
         include_over_limit = options.get("include_over_limit", False)
+        update_payment = options.get("update_payment", False)
         excluded = [CompletedWorkStatus.rejected]
         if not include_over_limit:
             excluded.append(CompletedWorkStatus.over_limit)
@@ -30,7 +32,7 @@ def handle(self, *args, opp: int, **options):
             )
             for access in access_objects:
                 completed_works = access.completedwork_set.exclude(status__in=excluded)
-                update_status(completed_works, access, False)
+                update_status(completed_works, access, update_payment)
 
             self.stdout.write(self.style.SUCCESS(f"Successfully processed opportunity with id {opp}"))
 

From 75e5a8c505271c705d78642a2624437197fb9ee4 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Mon, 9 Dec 2024 12:43:03 +0530
Subject: [PATCH 163/165] Fix dont create review for already approved visits

---
 commcare_connect/opportunity/views.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/commcare_connect/opportunity/views.py b/commcare_connect/opportunity/views.py
index 645ba4ef..c8036c37 100644
--- a/commcare_connect/opportunity/views.py
+++ b/commcare_connect/opportunity/views.py
@@ -866,8 +866,9 @@ def visit_verification(request, org_slug=None, pk=None):
 @org_member_required
 def approve_visit(request, org_slug=None, pk=None):
     user_visit = UserVisit.objects.get(pk=pk)
+    old_status = user_visit.status
     user_visit.status = VisitValidationStatus.approved
-    if user_visit.opportunity.managed:
+    if user_visit.opportunity.managed and old_status != VisitValidationStatus.approved:
         user_visit.review_created_on = now()
     user_visit.save()
     opp_id = user_visit.opportunity_id

From d04163e48806424ee913c0a1a3bed3581d289fc2 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Tue, 10 Dec 2024 12:45:58 +0530
Subject: [PATCH 164/165] Refactor save only when status changed

---
 commcare_connect/opportunity/views.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/commcare_connect/opportunity/views.py b/commcare_connect/opportunity/views.py
index c8036c37..a9503f6d 100644
--- a/commcare_connect/opportunity/views.py
+++ b/commcare_connect/opportunity/views.py
@@ -866,17 +866,18 @@ def visit_verification(request, org_slug=None, pk=None):
 @org_member_required
 def approve_visit(request, org_slug=None, pk=None):
     user_visit = UserVisit.objects.get(pk=pk)
-    old_status = user_visit.status
-    user_visit.status = VisitValidationStatus.approved
-    if user_visit.opportunity.managed and old_status != VisitValidationStatus.approved:
-        user_visit.review_created_on = now()
-    user_visit.save()
     opp_id = user_visit.opportunity_id
-    access = OpportunityAccess.objects.get(user_id=user_visit.user_id, opportunity_id=opp_id)
-    update_payment_accrued(opportunity=access.opportunity, users=[access.user])
+    if user_visit.status != VisitValidationStatus.approved:
+        user_visit.status = VisitValidationStatus.approved
+        if user_visit.opportunity.managed:
+            user_visit.review_created_on = now()
+        user_visit.save()
+        update_payment_accrued(opportunity=user_visit.opportunity, users=[user_visit.user])
     if user_visit.opportunity.managed:
         return redirect("opportunity:user_visit_review", org_slug, opp_id)
-    return redirect("opportunity:user_visits_list", org_slug=org_slug, opp_id=user_visit.opportunity.id, pk=access.id)
+    return redirect(
+        "opportunity:user_visits_list", org_slug=org_slug, opp_id=opp_id, pk=user_visit.opportunity_access_id
+    )
 
 
 @org_member_required

From 7011da6e1aadd52101bcdbe717122e08babd7dc8 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 10 Dec 2024 10:20:52 +0200
Subject: [PATCH 165/165] fix date comparison issues on date chart

---
 commcare_connect/reports/views.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index 47e105b1..3cd94483 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -9,6 +9,7 @@
 from django.contrib.auth.decorators import login_required, user_passes_test
 from django.contrib.auth.mixins import LoginRequiredMixin, UserPassesTestMixin
 from django.db.models import Count, Max, Q, Sum
+from django.db.models.functions import TruncDate
 from django.http import JsonResponse
 from django.shortcuts import render
 from django.urls import reverse
@@ -462,9 +463,12 @@ def _get_time_series_data(queryset, from_date, to_date):
     """
     # Get visits over time by program
     visits_by_program_time = (
-        queryset.values("visit_date", "opportunity__delivery_type__name")
+        queryset.values(
+            "opportunity__delivery_type__name",
+            visit_date_date=TruncDate("visit_date"),
+        )
         .annotate(count=Count("id"))
-        .order_by("visit_date", "opportunity__delivery_type__name")
+        .order_by("visit_date_date", "opportunity__delivery_type__name")
     )
 
     # Process time series data
@@ -473,8 +477,7 @@ def _get_time_series_data(queryset, from_date, to_date):
         program_name = visit["opportunity__delivery_type__name"]
         if program_name not in program_data:
             program_data[program_name] = {}
-        program_data[program_name][visit["visit_date"]] = visit["count"]
-
+        program_data[program_name][visit["visit_date_date"]] = visit["count"]
     # Create labels and datasets for time series
     labels = []
     time_datasets = []
@@ -488,7 +491,9 @@ def _get_time_series_data(queryset, from_date, to_date):
         data = []
         current_date = from_date
         while current_date <= to_date:
-            data.append(program_data[program_name].get(current_date, 0))
+            # Convert current_date to a date object to avoid timezones making comparisons fail
+            current_date_date = current_date.date()
+            data.append(program_data[program_name].get(current_date_date, 0))
             current_date += timedelta(days=1)
 
         time_datasets.append({"name": program_name or "Unknown", "data": data})