From 568bb8e550af931e2949887f56281c56d93f7586 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 18 Sep 2024 16:23:13 +0530
Subject: [PATCH 01/96] Made improvements in function based views

---
 commcare_connect/opportunity/views.py | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/commcare_connect/opportunity/views.py b/commcare_connect/opportunity/views.py
index 9b6c00aa..a5b5a72d 100644
--- a/commcare_connect/opportunity/views.py
+++ b/commcare_connect/opportunity/views.py
@@ -357,8 +357,7 @@ def get_queryset(self):
 
 
 @org_member_required
-def export_user_visits(request, **kwargs):
-    opportunity_id = kwargs["pk"]
+def export_user_visits(request, org_slug, opportunity_id):
     get_opportunity_or_404(org_slug=request.org.slug, pk=opportunity_id)
     form = VisitExportForm(data=request.POST)
     if not form.is_valid():
@@ -483,8 +482,7 @@ def get_queryset(self):
 
 
 @org_member_required
-def export_users_for_payment(request, **kwargs):
-    opportunity_id = kwargs["pk"]
+def export_users_for_payment(request, org_slug, opportunity_id):
     get_opportunity_or_404(org_slug=request.org.slug, pk=opportunity_id)
     form = PaymentExportForm(data=request.POST)
     if not form.is_valid():
@@ -629,8 +627,7 @@ def get_queryset(self):
 
 
 @org_member_required
-def export_user_status(request, **kwargs):
-    opportunity_id = kwargs["pk"]
+def export_user_status(request, org_slug, opportunity_id):
     get_opportunity_or_404(org_slug=request.org.slug, pk=opportunity_id)
     form = PaymentExportForm(data=request.POST)
     if not form.is_valid():
@@ -658,8 +655,7 @@ def get_queryset(self):
 
 
 @org_member_required
-def export_deliver_status(request, **kwargs):
-    opportunity_id = kwargs["pk"]
+def export_deliver_status(request, org_slug, opportunity_id):
     get_opportunity_or_404(pk=opportunity_id, org_slug=request.org.slug)
     form = PaymentExportForm(data=request.POST)
     if not form.is_valid():
@@ -987,8 +983,7 @@ def get_queryset(self):
 
 
 @org_member_required
-def export_completed_work(request, **kwargs):
-    opportunity_id = kwargs["pk"]
+def export_completed_work(request, org_slug, opportunity_id):
     get_opportunity_or_404(org_slug=request.org.slug, pk=opportunity_id)
     form = PaymentExportForm(data=request.POST)
     if not form.is_valid():
@@ -1052,8 +1047,7 @@ def suspended_users_list(request, org_slug=None, pk=None):
 
 
 @org_member_required
-def export_catchment_area(request, **kwargs):
-    opportunity_id = kwargs["pk"]
+def export_catchment_area(request, org_slug, opportunity_id):
     get_opportunity_or_404(org_slug=request.org.slug, pk=opportunity_id)
     form = PaymentExportForm(data=request.POST)
     if not form.is_valid():

From 0cb0721ff9ac8669f31313a317450e954a58e088 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 18 Sep 2024 17:14:06 +0530
Subject: [PATCH 02/96] fixed the args name issue

---
 commcare_connect/opportunity/views.py | 60 +++++++++++++--------------
 1 file changed, 30 insertions(+), 30 deletions(-)

diff --git a/commcare_connect/opportunity/views.py b/commcare_connect/opportunity/views.py
index a5b5a72d..289448a3 100644
--- a/commcare_connect/opportunity/views.py
+++ b/commcare_connect/opportunity/views.py
@@ -357,20 +357,20 @@ def get_queryset(self):
 
 
 @org_member_required
-def export_user_visits(request, org_slug, opportunity_id):
-    get_opportunity_or_404(org_slug=request.org.slug, pk=opportunity_id)
+def export_user_visits(request, org_slug, pk):
+    get_opportunity_or_404(org_slug=request.org.slug, pk=pk)
     form = VisitExportForm(data=request.POST)
     if not form.is_valid():
         messages.error(request, form.errors)
-        return redirect("opportunity:detail", request.org.slug, opportunity_id)
+        return redirect("opportunity:detail", request.org.slug, pk)
 
     export_format = form.cleaned_data["format"]
     date_range = DateRanges(form.cleaned_data["date_range"])
     status = form.cleaned_data["status"]
     flatten = form.cleaned_data["flatten_form_data"]
 
-    result = generate_visit_export.delay(opportunity_id, date_range, status, export_format, flatten)
-    redirect_url = reverse("opportunity:detail", args=(request.org.slug, opportunity_id))
+    result = generate_visit_export.delay(pk, date_range, status, export_format, flatten)
+    redirect_url = reverse("opportunity:detail", args=(request.org.slug, pk))
     return redirect(f"{redirect_url}?export_task_id={result.id}")
 
 
@@ -482,16 +482,16 @@ def get_queryset(self):
 
 
 @org_member_required
-def export_users_for_payment(request, org_slug, opportunity_id):
-    get_opportunity_or_404(org_slug=request.org.slug, pk=opportunity_id)
+def export_users_for_payment(request, org_slug, pk):
+    get_opportunity_or_404(org_slug=request.org.slug, pk=pk)
     form = PaymentExportForm(data=request.POST)
     if not form.is_valid():
         messages.error(request, form.errors)
-        return redirect("opportunity:detail", request.org.slug, opportunity_id)
+        return redirect("opportunity:detail", request.org.slug, pk)
 
     export_format = form.cleaned_data["format"]
-    result = generate_payment_export.delay(opportunity_id, export_format)
-    redirect_url = reverse("opportunity:detail", args=(request.org.slug, opportunity_id))
+    result = generate_payment_export.delay(pk, export_format)
+    redirect_url = reverse("opportunity:detail", args=(request.org.slug, pk))
     return redirect(f"{redirect_url}?export_task_id={result.id}")
 
 
@@ -627,16 +627,16 @@ def get_queryset(self):
 
 
 @org_member_required
-def export_user_status(request, org_slug, opportunity_id):
-    get_opportunity_or_404(org_slug=request.org.slug, pk=opportunity_id)
+def export_user_status(request, org_slug, pk):
+    get_opportunity_or_404(org_slug=request.org.slug, pk=pk)
     form = PaymentExportForm(data=request.POST)
     if not form.is_valid():
         messages.error(request, form.errors)
-        return redirect("opportunity:detail", request.org.slug, opportunity_id)
+        return redirect("opportunity:detail", request.org.slug, pk)
 
     export_format = form.cleaned_data["format"]
-    result = generate_user_status_export.delay(opportunity_id, export_format)
-    redirect_url = reverse("opportunity:detail", args=(request.org.slug, opportunity_id))
+    result = generate_user_status_export.delay(pk, export_format)
+    redirect_url = reverse("opportunity:detail", args=(request.org.slug, pk))
     return redirect(f"{redirect_url}?export_task_id={result.id}")
 
 
@@ -655,16 +655,16 @@ def get_queryset(self):
 
 
 @org_member_required
-def export_deliver_status(request, org_slug, opportunity_id):
-    get_opportunity_or_404(pk=opportunity_id, org_slug=request.org.slug)
+def export_deliver_status(request, org_slug, pk):
+    get_opportunity_or_404(pk=pk, org_slug=request.org.slug)
     form = PaymentExportForm(data=request.POST)
     if not form.is_valid():
         messages.error(request, form.errors)
-        return redirect("opportunity:detail", request.org.slug, opportunity_id)
+        return redirect("opportunity:detail", request.org.slug, pk)
 
     export_format = form.cleaned_data["format"]
-    result = generate_deliver_status_export.delay(opportunity_id, export_format)
-    redirect_url = reverse("opportunity:detail", args=(request.org.slug, opportunity_id))
+    result = generate_deliver_status_export.delay(pk, export_format)
+    redirect_url = reverse("opportunity:detail", args=(request.org.slug, pk))
     return redirect(f"{redirect_url}?export_task_id={result.id}")
 
 
@@ -983,16 +983,16 @@ def get_queryset(self):
 
 
 @org_member_required
-def export_completed_work(request, org_slug, opportunity_id):
-    get_opportunity_or_404(org_slug=request.org.slug, pk=opportunity_id)
+def export_completed_work(request, org_slug, pk):
+    get_opportunity_or_404(org_slug=request.org.slug, pk=pk)
     form = PaymentExportForm(data=request.POST)
     if not form.is_valid():
         messages.error(request, form.errors)
-        return redirect("opportunity:detail", request.org.slug, opportunity_id)
+        return redirect("opportunity:detail", request.org.slug, pk)
 
     export_format = form.cleaned_data["format"]
-    result = generate_work_status_export.delay(opportunity_id, export_format)
-    redirect_url = reverse("opportunity:detail", args=(request.org.slug, opportunity_id))
+    result = generate_work_status_export.delay(pk, export_format)
+    redirect_url = reverse("opportunity:detail", args=(request.org.slug, pk))
     return redirect(f"{redirect_url}?export_task_id={result.id}")
 
 
@@ -1047,16 +1047,16 @@ def suspended_users_list(request, org_slug=None, pk=None):
 
 
 @org_member_required
-def export_catchment_area(request, org_slug, opportunity_id):
-    get_opportunity_or_404(org_slug=request.org.slug, pk=opportunity_id)
+def export_catchment_area(request, org_slug, pk):
+    get_opportunity_or_404(org_slug=request.org.slug, pk=pk)
     form = PaymentExportForm(data=request.POST)
     if not form.is_valid():
         messages.error(request, form.errors)
-        return redirect("opportunity:detail", request.org.slug, opportunity_id)
+        return redirect("opportunity:detail", request.org.slug, pk)
 
     export_format = form.cleaned_data["format"]
-    result = generate_catchment_area_export.delay(opportunity_id, export_format)
-    redirect_url = reverse("opportunity:detail", args=(request.org.slug, opportunity_id))
+    result = generate_catchment_area_export.delay(pk, export_format)
+    redirect_url = reverse("opportunity:detail", args=(request.org.slug, pk))
     return redirect(f"{redirect_url}?export_task_id={result.id}")
 
 

From 7b8ded4c3038f22cf5cd7de716543ebfd50c9add Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Mon, 7 Oct 2024 18:38:28 +0530
Subject: [PATCH 03/96] Fix spelling

---
 .../tests/test_receiver_integration.py           | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/commcare_connect/form_receiver/tests/test_receiver_integration.py b/commcare_connect/form_receiver/tests/test_receiver_integration.py
index c52af2a5..e75048ba 100644
--- a/commcare_connect/form_receiver/tests/test_receiver_integration.py
+++ b/commcare_connect/form_receiver/tests/test_receiver_integration.py
@@ -460,7 +460,7 @@ def test_auto_approve_visits_and_payments(
     assert access.payment_accrued == completed_work.payment_accrued
 
 
-def test_reciever_verification_flags_form_submission(
+def test_receiver_verification_flags_form_submission(
     user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
 ):
     verification_flags = OpportunityVerificationFlags.objects.get(opportunity=opportunity)
@@ -477,7 +477,7 @@ def test_reciever_verification_flags_form_submission(
     assert not visit.flagged
 
 
-def test_reciever_verification_flags_form_submission_start(
+def test_receiver_verification_flags_form_submission_start(
     user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
 ):
     verification_flags = OpportunityVerificationFlags.objects.get(opportunity=opportunity)
@@ -494,7 +494,7 @@ def test_reciever_verification_flags_form_submission_start(
     assert ["form_submission_period", "Form was submitted before the start time"] in visit.flag_reason.get("flags", [])
 
 
-def test_reciever_verification_flags_form_submission_end(
+def test_receiver_verification_flags_form_submission_end(
     user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
 ):
     verification_flags = OpportunityVerificationFlags.objects.get(opportunity=opportunity)
@@ -511,7 +511,7 @@ def test_reciever_verification_flags_form_submission_end(
     assert ["form_submission_period", "Form was submitted after the end time"] in visit.flag_reason.get("flags", [])
 
 
-def test_reciever_verification_flags_duration(
+def test_receiver_verification_flags_duration(
     user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
 ):
     form_json = _create_opp_and_form_json(opportunity, user=user_with_connectid_link)
@@ -524,7 +524,7 @@ def test_reciever_verification_flags_duration(
     assert ["duration", "The form was completed too quickly."] in visit.flag_reason.get("flags", [])
 
 
-def test_reciever_verification_flags_check_attachments(
+def test_receiver_verification_flags_check_attachments(
     user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
 ):
     form_json = _create_opp_and_form_json(opportunity, user=user_with_connectid_link)
@@ -537,7 +537,7 @@ def test_reciever_verification_flags_check_attachments(
     assert ["attachment_missing", "Form was submitted without attachements."] in visit.flag_reason.get("flags", [])
 
 
-def test_reciever_verification_flags_form_json_rule(
+def test_receiver_verification_flags_form_json_rule(
     user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
 ):
     form_json = _create_opp_and_form_json(opportunity, user=user_with_connectid_link)
@@ -555,7 +555,7 @@ def test_reciever_verification_flags_form_json_rule(
     assert not visit.flagged
 
 
-def test_reciever_verification_flags_form_json_rule_flagged(
+def test_receiver_verification_flags_form_json_rule_flagged(
     user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
 ):
     form_json = _create_opp_and_form_json(opportunity, user=user_with_connectid_link)
@@ -577,7 +577,7 @@ def test_reciever_verification_flags_form_json_rule_flagged(
     ] in visit.flag_reason.get("flags", [])
 
 
-def test_reciever_verification_flags_catchment_areas(
+def test_receiver_verification_flags_catchment_areas(
     user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
 ):
     verification_flags = OpportunityVerificationFlags.objects.get(opportunity=opportunity)

From 2fee3497e937f364c1befefa8588955cca3eaf0f Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Mon, 7 Oct 2024 18:43:48 +0530
Subject: [PATCH 04/96] Add tests for form receiver review status changes

---
 .../tests/test_receiver_integration.py        | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/commcare_connect/form_receiver/tests/test_receiver_integration.py b/commcare_connect/form_receiver/tests/test_receiver_integration.py
index e75048ba..6108c5bb 100644
--- a/commcare_connect/form_receiver/tests/test_receiver_integration.py
+++ b/commcare_connect/form_receiver/tests/test_receiver_integration.py
@@ -23,6 +23,7 @@
     OpportunityClaimLimit,
     OpportunityVerificationFlags,
     UserVisit,
+    VisitReviewStatus,
     VisitValidationStatus,
 )
 from commcare_connect.opportunity.tasks import bulk_approve_completed_work
@@ -596,6 +597,33 @@ def test_receiver_verification_flags_catchment_areas(
     assert ["catchment", "Visit outside worker catchment areas"] in visit.flag_reason.get("flags", [])
 
 
+def test_receiver_auto_agree_approved_visit(
+    user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
+):
+    opportunity.managed = True
+    opportunity.save()
+    form_json = _create_opp_and_form_json(opportunity, user=user_with_connectid_link)
+    make_request(api_client, form_json, user_with_connectid_link)
+    visit = UserVisit.objects.get(user=user_with_connectid_link)
+    assert not visit.flagged
+    assert visit.status == VisitValidationStatus.approved
+    assert visit.review_status == VisitReviewStatus.agree
+
+
+def test_receiver_flagged_visit_review_pending(
+    user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
+):
+    opportunity.managed = True
+    opportunity.save()
+    form_json = _create_opp_and_form_json(opportunity, user=user_with_connectid_link)
+    form_json["metadata"]["location"] = None
+    make_request(api_client, form_json, user_with_connectid_link)
+    visit = UserVisit.objects.get(user=user_with_connectid_link)
+    assert visit.flagged
+    assert visit.status == VisitValidationStatus.pending
+    assert visit.review_status == VisitReviewStatus.pending
+
+
 def _get_form_json(learn_app, module_id, form_block=None):
     form_json = get_form_json(
         form_block=form_block or LearnModuleJsonFactory(id=module_id).json,

From fbddab58c24b0b701969ab38da37a971717b5451 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Tue, 8 Oct 2024 15:11:52 +0530
Subject: [PATCH 05/96] Refactor tests to use fixture

---
 .../tests/test_receiver_integration.py        | 42 ++++++++++---------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/commcare_connect/form_receiver/tests/test_receiver_integration.py b/commcare_connect/form_receiver/tests/test_receiver_integration.py
index 6108c5bb..6e9a9941 100644
--- a/commcare_connect/form_receiver/tests/test_receiver_integration.py
+++ b/commcare_connect/form_receiver/tests/test_receiver_integration.py
@@ -173,24 +173,14 @@ def test_receiver_deliver_form_daily_visits_reached(
 def test_receiver_deliver_form_max_visits_reached(
     mobile_user_with_connect_link: User, api_client: APIClient, opportunity: Opportunity
 ):
-    def form_json(payment_unit):
-        deliver_unit = DeliverUnitFactory(app=opportunity.deliver_app, payment_unit=payment_unit)
-        stub = DeliverUnitStubFactory(id=deliver_unit.slug)
-        form_json = get_form_json(
-            form_block=stub.json,
-            domain=deliver_unit.app.cc_domain,
-            app_id=deliver_unit.app.cc_app_id,
-        )
-        return form_json
-
     def submit_form_for_random_entity(form_json):
         duplicate_json = deepcopy(form_json)
         duplicate_json["form"]["deliver"]["entity_id"] = str(uuid4())
         make_request(api_client, duplicate_json, mobile_user_with_connect_link)
 
     payment_units = opportunity.paymentunit_set.all()
-    form_json1 = form_json(payment_units[0])
-    form_json2 = form_json(payment_units[1])
+    form_json1 = get_form_json_for_payment_unit(payment_units[0])
+    form_json2 = get_form_json_for_payment_unit(payment_units[1])
     for _ in range(2):
         submit_form_for_random_entity(form_json1)
         submit_form_for_random_entity(form_json2)
@@ -598,32 +588,44 @@ def test_receiver_verification_flags_catchment_areas(
 
 
 def test_receiver_auto_agree_approved_visit(
-    user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
+    mobile_user_with_connect_link: User, api_client: APIClient, opportunity: Opportunity
 ):
     opportunity.managed = True
     opportunity.save()
-    form_json = _create_opp_and_form_json(opportunity, user=user_with_connectid_link)
-    make_request(api_client, form_json, user_with_connectid_link)
-    visit = UserVisit.objects.get(user=user_with_connectid_link)
+    form_json = get_form_json_for_payment_unit(opportunity.paymentunit_set.all()[0])
+    make_request(api_client, form_json, mobile_user_with_connect_link)
+    visit = UserVisit.objects.get(user=mobile_user_with_connect_link)
     assert not visit.flagged
     assert visit.status == VisitValidationStatus.approved
     assert visit.review_status == VisitReviewStatus.agree
 
 
+@pytest.mark.parametrize("paymentunit_options", [pytest.param({"max_daily": 2})])
 def test_receiver_flagged_visit_review_pending(
-    user_with_connectid_link: User, api_client: APIClient, opportunity: Opportunity
+    mobile_user_with_connect_link: User, api_client: APIClient, opportunity: Opportunity
 ):
     opportunity.managed = True
     opportunity.save()
-    form_json = _create_opp_and_form_json(opportunity, user=user_with_connectid_link)
+    form_json = get_form_json_for_payment_unit(opportunity.paymentunit_set.all()[0])
     form_json["metadata"]["location"] = None
-    make_request(api_client, form_json, user_with_connectid_link)
-    visit = UserVisit.objects.get(user=user_with_connectid_link)
+    make_request(api_client, form_json, mobile_user_with_connect_link)
+    visit = UserVisit.objects.get(user=mobile_user_with_connect_link)
     assert visit.flagged
     assert visit.status == VisitValidationStatus.pending
     assert visit.review_status == VisitReviewStatus.pending
 
 
+def get_form_json_for_payment_unit(payment_unit):
+    deliver_unit = DeliverUnitFactory(app=payment_unit.opportunity.deliver_app, payment_unit=payment_unit)
+    stub = DeliverUnitStubFactory(id=deliver_unit.slug)
+    form_json = get_form_json(
+        form_block=stub.json,
+        domain=deliver_unit.app.cc_domain,
+        app_id=deliver_unit.app.cc_app_id,
+    )
+    return form_json
+
+
 def _get_form_json(learn_app, module_id, form_block=None):
     form_json = get_form_json(
         form_block=form_block or LearnModuleJsonFactory(id=module_id).json,

From d04c0af5f9f8420af029fb94f599bd8567440146 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Tue, 8 Oct 2024 16:24:47 +0530
Subject: [PATCH 06/96] Add config to change opportunity args

---
 commcare_connect/conftest.py                             | 8 +++++---
 .../form_receiver/tests/test_receiver_integration.py     | 9 ++++-----
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/commcare_connect/conftest.py b/commcare_connect/conftest.py
index 8c91a1df..2e8260ab 100644
--- a/commcare_connect/conftest.py
+++ b/commcare_connect/conftest.py
@@ -47,9 +47,11 @@ def user(db) -> User:
     return UserFactory()
 
 
-@pytest.fixture()
-def opportunity():
-    factory = OpportunityFactory(is_test=False)
+@pytest.fixture
+def opportunity(request):
+    opp_options = {"is_test": False}
+    opp_options.update(request.param if hasattr(request, "param") else {})
+    factory = OpportunityFactory(**opp_options)
     OpportunityVerificationFlagsFactory(opportunity=factory)
     return factory
 
diff --git a/commcare_connect/form_receiver/tests/test_receiver_integration.py b/commcare_connect/form_receiver/tests/test_receiver_integration.py
index 6e9a9941..be002639 100644
--- a/commcare_connect/form_receiver/tests/test_receiver_integration.py
+++ b/commcare_connect/form_receiver/tests/test_receiver_integration.py
@@ -587,11 +587,11 @@ def test_receiver_verification_flags_catchment_areas(
     assert ["catchment", "Visit outside worker catchment areas"] in visit.flag_reason.get("flags", [])
 
 
+@pytest.mark.parametrize("opportunity", [{"managed": True}], indirect=True)
 def test_receiver_auto_agree_approved_visit(
     mobile_user_with_connect_link: User, api_client: APIClient, opportunity: Opportunity
 ):
-    opportunity.managed = True
-    opportunity.save()
+    assert opportunity.managed
     form_json = get_form_json_for_payment_unit(opportunity.paymentunit_set.all()[0])
     make_request(api_client, form_json, mobile_user_with_connect_link)
     visit = UserVisit.objects.get(user=mobile_user_with_connect_link)
@@ -600,12 +600,11 @@ def test_receiver_auto_agree_approved_visit(
     assert visit.review_status == VisitReviewStatus.agree
 
 
-@pytest.mark.parametrize("paymentunit_options", [pytest.param({"max_daily": 2})])
+@pytest.mark.parametrize("opportunity", [{"managed": True}], indirect=True)
 def test_receiver_flagged_visit_review_pending(
     mobile_user_with_connect_link: User, api_client: APIClient, opportunity: Opportunity
 ):
-    opportunity.managed = True
-    opportunity.save()
+    assert opportunity.managed
     form_json = get_form_json_for_payment_unit(opportunity.paymentunit_set.all()[0])
     form_json["metadata"]["location"] = None
     make_request(api_client, form_json, mobile_user_with_connect_link)

From 17db33d01c6053daaaf8ddad736f2315aa9752a7 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Tue, 8 Oct 2024 18:27:01 +0530
Subject: [PATCH 07/96] Add visit import test for review visits

---
 .../opportunity/tests/test_visit_import.py    | 45 +++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/commcare_connect/opportunity/tests/test_visit_import.py b/commcare_connect/opportunity/tests/test_visit_import.py
index 4c5f10ee..729f05d7 100644
--- a/commcare_connect/opportunity/tests/test_visit_import.py
+++ b/commcare_connect/opportunity/tests/test_visit_import.py
@@ -19,6 +19,7 @@
     Payment,
     PaymentUnit,
     UserVisit,
+    VisitReviewStatus,
     VisitValidationStatus,
 )
 from commcare_connect.opportunity.tests.factories import (
@@ -539,3 +540,47 @@ def get_assignable_completed_work_count(access: OpportunityAccess) -> int:
             total_assigned_count += 1
 
     return total_assigned_count
+
+
+@pytest.mark.parametrize("opportunity", [{"managed": True}], indirect=True)
+def test_network_manager_approve_flagged_visit(mobile_user: User, opportunity: Opportunity):
+    assert opportunity.managed
+    access = OpportunityAccess.objects.get(user=mobile_user, opportunity=opportunity)
+    visits = UserVisitFactory.create_batch(
+        5, opportunity=opportunity, status=VisitValidationStatus.pending, user=mobile_user, opportunity_access=access
+    )
+    dataset = Dataset(headers=["visit id", "status", "rejected reason", "justification"])
+    dataset.extend([[visit.xform_id, VisitValidationStatus.approved.value, "", "justification"] for visit in visits])
+    before_update = now()
+    import_status = _bulk_update_visit_status(opportunity, dataset)
+    after_update = now()
+    assert not import_status.missing_visits
+    updated_visits = UserVisit.objects.filter(opportunity=opportunity)
+    for visit in updated_visits:
+        assert visit.status == VisitValidationStatus.approved
+        assert visit.status_modified_date is not None
+        assert before_update <= visit.status_modified_date <= after_update
+        assert before_update <= visit.review_created_on <= after_update
+        assert visit.review_status == VisitReviewStatus.pending
+        assert visit.justification == "justification"
+
+
+@pytest.mark.parametrize("opportunity", [{"managed": True}], indirect=True)
+def test_network_manager_reject_flagged_visit(mobile_user: User, opportunity: Opportunity):
+    assert opportunity.managed
+    access = OpportunityAccess.objects.get(user=mobile_user, opportunity=opportunity)
+    visits = UserVisitFactory.create_batch(
+        5, opportunity=opportunity, status=VisitValidationStatus.pending, user=mobile_user, opportunity_access=access
+    )
+    dataset = Dataset(headers=["visit id", "status", "rejected reason", "justification"])
+    dataset.extend([[visit.xform_id, VisitValidationStatus.rejected.value, "", "justification"] for visit in visits])
+    before_update = now()
+    import_status = _bulk_update_visit_status(opportunity, dataset)
+    after_update = now()
+    assert not import_status.missing_visits
+    updated_visits = UserVisit.objects.filter(opportunity=opportunity)
+    for visit in updated_visits:
+        assert visit.status == VisitValidationStatus.rejected
+        assert visit.status_modified_date is not None
+        assert before_update <= visit.status_modified_date <= after_update
+        assert visit.review_created_on is None

From 0598a635c477b892b28eb833a7c5c4e6ba82a600 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Wed, 9 Oct 2024 17:22:56 +0530
Subject: [PATCH 08/96] Add test for UserVisitReview view

---
 .../opportunity/tests/test_views.py           | 98 ++++++++++++++++++-
 1 file changed, 97 insertions(+), 1 deletion(-)

diff --git a/commcare_connect/opportunity/tests/test_views.py b/commcare_connect/opportunity/tests/test_views.py
index da418aae..a74432fd 100644
--- a/commcare_connect/opportunity/tests/test_views.py
+++ b/commcare_connect/opportunity/tests/test_views.py
@@ -1,14 +1,25 @@
 import pytest
 from django.test import Client
 from django.urls import reverse
+from django.utils.timezone import now
 
-from commcare_connect.opportunity.models import Opportunity, OpportunityAccess, OpportunityClaimLimit
+from commcare_connect.opportunity.models import (
+    Opportunity,
+    OpportunityAccess,
+    OpportunityClaimLimit,
+    UserVisit,
+    VisitReviewStatus,
+    VisitValidationStatus,
+)
 from commcare_connect.opportunity.tests.factories import (
+    OpportunityAccessFactory,
     OpportunityClaimFactory,
     OpportunityClaimLimitFactory,
     PaymentUnitFactory,
+    UserVisitFactory,
 )
 from commcare_connect.organization.models import Organization
+from commcare_connect.program.tests.factories import ManagedOpportunityFactory, ProgramFactory
 from commcare_connect.users.models import User
 
 
@@ -38,3 +49,88 @@ def test_add_budget_existing_users(
     assert opportunity.total_budget == 205
     assert opportunity.claimed_budget == 15
     assert OpportunityClaimLimit.objects.get(pk=ocl.pk).max_visits == 15
+
+
+class TestUserVisitReviewView:
+    @pytest.fixture(autouse=True)
+    @pytest.mark.parametrize("opportunity", [{"managed": True}], indirect=True)
+    def setup(
+        self,
+        client: Client,
+        program_manager_org: Organization,
+        program_manager_org_user_admin: User,
+        organization: Organization,
+        org_user_admin: User,
+        mobile_user: User,
+    ):
+        self.client = client
+        self.pm_org = program_manager_org
+        self.pm_user = program_manager_org_user_admin
+        self.nm_org = organization
+        self.nm_user = org_user_admin
+        self.program = ProgramFactory(organization=self.pm_org)
+        self.opportunity = ManagedOpportunityFactory(program=self.program, organization=self.nm_org)
+        self.mobile_user = mobile_user
+        access = OpportunityAccessFactory(user=mobile_user, opportunity=self.opportunity, accepted=True)
+        self.visits = UserVisitFactory.create_batch(
+            10,
+            user=mobile_user,
+            opportunity=self.opportunity,
+            status=VisitValidationStatus.approved,
+            review_created_on=now(),
+            review_status=VisitReviewStatus.pending,
+            opportunity_access=access,
+        )
+
+    def test_user_visit_review_program_manager_pending(self):
+        self.url = reverse("opportunity:user_visit_review", args=(self.pm_org.slug, self.opportunity.id))
+        self.client.force_login(self.pm_user)
+        response = self.client.get(self.url)
+        assert response.status_code == 200
+        table = response.context["table"]
+        assert len(table.rows) == 10
+        assert "pk" in table.columns.names()
+
+        visits = UserVisit.objects.filter(id__in=[visit.id for visit in self.visits])
+        for visit in visits:
+            assert visit.review_status == VisitReviewStatus.pending
+
+    def test_user_visit_review_program_manager_agree(self):
+        self.url = reverse("opportunity:user_visit_review", args=(self.pm_org.slug, self.opportunity.id))
+        self.client.force_login(self.pm_user)
+        response = self.client.post(self.url, {"pk": [], "review_status": VisitReviewStatus.agree.value})
+        assert response.status_code == 200
+        visits = UserVisit.objects.filter(id__in=[visit.id for visit in self.visits])
+        for visit in visits:
+            assert visit.review_status == VisitReviewStatus.pending
+
+        visit_ids = [visit.id for visit in self.visits][:5]
+        response = self.client.post(self.url, {"pk": visit_ids, "review_status": VisitReviewStatus.agree.value})
+        assert response.status_code == 200
+        visits = UserVisit.objects.filter(id__in=visit_ids)
+        for visit in visits:
+            assert visit.review_status == VisitReviewStatus.agree
+
+    def test_user_visit_review_program_manager_reject(self):
+        self.url = reverse("opportunity:user_visit_review", args=(self.pm_org.slug, self.opportunity.id))
+        self.client.force_login(self.pm_user)
+        response = self.client.post(self.url, {"pk": [], "review_status": VisitReviewStatus.agree.value})
+        assert response.status_code == 200
+        visits = UserVisit.objects.filter(id__in=[visit.id for visit in self.visits])
+        for visit in visits:
+            assert visit.review_status == VisitReviewStatus.pending
+
+        visit_ids = [visit.id for visit in self.visits][:5]
+        response = self.client.post(self.url, {"pk": visit_ids, "review_status": VisitReviewStatus.disagree.value})
+        assert response.status_code == 200
+        visits = UserVisit.objects.filter(id__in=visit_ids)
+        for visit in visits:
+            assert visit.review_status == VisitReviewStatus.disagree
+
+    def test_user_visit_review_network_manager(self):
+        self.url = reverse("opportunity:user_visit_review", args=(self.nm_org.slug, self.opportunity.id))
+        self.client.force_login(self.nm_user)
+        response = self.client.get(self.url)
+        table = response.context["table"]
+        assert len(table.rows) == 10
+        assert "pk" not in table.columns.names()

From e53d03fe056b2f201874ea830345a668609de15f Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Tue, 15 Oct 2024 16:36:30 +0530
Subject: [PATCH 09/96] Fix date time warning in tests

---
 commcare_connect/opportunity/tests/factories.py | 2 +-
 commcare_connect/reports/tests/test_reports.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/commcare_connect/opportunity/tests/factories.py b/commcare_connect/opportunity/tests/factories.py
index d69b3ffb..5f3316fd 100644
--- a/commcare_connect/opportunity/tests/factories.py
+++ b/commcare_connect/opportunity/tests/factories.py
@@ -230,7 +230,7 @@ class Meta:
 class PaymentFactory(DjangoModelFactory):
     opportunity_access = SubFactory(OpportunityAccessFactory)
     amount = Faker("pyint", min_value=1, max_value=10000)
-    date_paid = Faker("past_date")
+    date_paid = Faker("date_time", tzinfo=timezone.utc)
 
     class Meta:
         model = "opportunity.Payment"
diff --git a/commcare_connect/reports/tests/test_reports.py b/commcare_connect/reports/tests/test_reports.py
index 319d4100..df6e28a2 100644
--- a/commcare_connect/reports/tests/test_reports.py
+++ b/commcare_connect/reports/tests/test_reports.py
@@ -40,7 +40,7 @@ def test_delivery_stats(opportunity: Opportunity):
                     status=VisitValidationStatus.approved.value,
                     opportunity_access=access,
                     completed_work=completed_work,
-                    visit_date=Faker("date_this_month"),
+                    visit_date=Faker("date_time_this_month", tzinfo=datetime.UTC),
                 )
 
     quarter = math.ceil(datetime.datetime.utcnow().month / 12 * 4)

From 9ba026d3e968271ef3cd86768668388299fb3d57 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Wed, 16 Oct 2024 15:05:48 +0530
Subject: [PATCH 10/96] Refactor remove extra variables and use model choices

---
 commcare_connect/opportunity/tests/test_views.py | 6 +-----
 commcare_connect/opportunity/views.py            | 3 ++-
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/commcare_connect/opportunity/tests/test_views.py b/commcare_connect/opportunity/tests/test_views.py
index a74432fd..ab1f0700 100644
--- a/commcare_connect/opportunity/tests/test_views.py
+++ b/commcare_connect/opportunity/tests/test_views.py
@@ -53,7 +53,6 @@ def test_add_budget_existing_users(
 
 class TestUserVisitReviewView:
     @pytest.fixture(autouse=True)
-    @pytest.mark.parametrize("opportunity", [{"managed": True}], indirect=True)
     def setup(
         self,
         client: Client,
@@ -61,7 +60,6 @@ def setup(
         program_manager_org_user_admin: User,
         organization: Organization,
         org_user_admin: User,
-        mobile_user: User,
     ):
         self.client = client
         self.pm_org = program_manager_org
@@ -70,11 +68,9 @@ def setup(
         self.nm_user = org_user_admin
         self.program = ProgramFactory(organization=self.pm_org)
         self.opportunity = ManagedOpportunityFactory(program=self.program, organization=self.nm_org)
-        self.mobile_user = mobile_user
-        access = OpportunityAccessFactory(user=mobile_user, opportunity=self.opportunity, accepted=True)
+        access = OpportunityAccessFactory(opportunity=self.opportunity, accepted=True)
         self.visits = UserVisitFactory.create_batch(
             10,
-            user=mobile_user,
             opportunity=self.opportunity,
             status=VisitValidationStatus.approved,
             review_created_on=now(),
diff --git a/commcare_connect/opportunity/views.py b/commcare_connect/opportunity/views.py
index 9b6c00aa..082250a7 100644
--- a/commcare_connect/opportunity/views.py
+++ b/commcare_connect/opportunity/views.py
@@ -65,6 +65,7 @@
     PaymentInvoice,
     PaymentUnit,
     UserVisit,
+    VisitReviewStatus,
     VisitValidationStatus,
 )
 from commcare_connect.opportunity.tables import (
@@ -1119,7 +1120,7 @@ def user_visit_review(request, org_slug, opp_id):
         review_status = request.POST.get("review_status").lower()
         updated_reviews = request.POST.getlist("pk")
         user_visits = UserVisit.objects.filter(pk__in=updated_reviews)
-        if review_status in ["agree", "disagree"]:
+        if review_status in [VisitReviewStatus.agree.value, VisitReviewStatus.disagree.value]:
             user_visits.update(review_status=review_status)
             update_payment_accrued(opportunity=opportunity, users=[visit.user for visit in user_visits])
 

From 76ea8d8524843a7794c4b2168f2eb4598d82caae Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Fri, 18 Oct 2024 17:44:55 +0530
Subject: [PATCH 11/96] Add combine test cases with parametrize

---
 .../opportunity/tests/test_views.py           | 33 ++++---------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/commcare_connect/opportunity/tests/test_views.py b/commcare_connect/opportunity/tests/test_views.py
index ab1f0700..064ee424 100644
--- a/commcare_connect/opportunity/tests/test_views.py
+++ b/commcare_connect/opportunity/tests/test_views.py
@@ -78,7 +78,7 @@ def setup(
             opportunity_access=access,
         )
 
-    def test_user_visit_review_program_manager_pending(self):
+    def test_user_visit_review_program_manager_table(self):
         self.url = reverse("opportunity:user_visit_review", args=(self.pm_org.slug, self.opportunity.id))
         self.client.force_login(self.pm_user)
         response = self.client.get(self.url)
@@ -87,43 +87,24 @@ def test_user_visit_review_program_manager_pending(self):
         assert len(table.rows) == 10
         assert "pk" in table.columns.names()
 
-        visits = UserVisit.objects.filter(id__in=[visit.id for visit in self.visits])
-        for visit in visits:
-            assert visit.review_status == VisitReviewStatus.pending
-
-    def test_user_visit_review_program_manager_agree(self):
-        self.url = reverse("opportunity:user_visit_review", args=(self.pm_org.slug, self.opportunity.id))
-        self.client.force_login(self.pm_user)
-        response = self.client.post(self.url, {"pk": [], "review_status": VisitReviewStatus.agree.value})
-        assert response.status_code == 200
-        visits = UserVisit.objects.filter(id__in=[visit.id for visit in self.visits])
-        for visit in visits:
-            assert visit.review_status == VisitReviewStatus.pending
-
-        visit_ids = [visit.id for visit in self.visits][:5]
-        response = self.client.post(self.url, {"pk": visit_ids, "review_status": VisitReviewStatus.agree.value})
-        assert response.status_code == 200
-        visits = UserVisit.objects.filter(id__in=visit_ids)
-        for visit in visits:
-            assert visit.review_status == VisitReviewStatus.agree
-
-    def test_user_visit_review_program_manager_reject(self):
+    @pytest.mark.parametrize("review_status", [(VisitReviewStatus.agree), (VisitReviewStatus.disagree)])
+    def test_user_visit_review_program_manager_approval(self, review_status):
         self.url = reverse("opportunity:user_visit_review", args=(self.pm_org.slug, self.opportunity.id))
         self.client.force_login(self.pm_user)
-        response = self.client.post(self.url, {"pk": [], "review_status": VisitReviewStatus.agree.value})
+        response = self.client.post(self.url, {"pk": [], "review_status": review_status.value})
         assert response.status_code == 200
         visits = UserVisit.objects.filter(id__in=[visit.id for visit in self.visits])
         for visit in visits:
             assert visit.review_status == VisitReviewStatus.pending
 
         visit_ids = [visit.id for visit in self.visits][:5]
-        response = self.client.post(self.url, {"pk": visit_ids, "review_status": VisitReviewStatus.disagree.value})
+        response = self.client.post(self.url, {"pk": visit_ids, "review_status": review_status.value})
         assert response.status_code == 200
         visits = UserVisit.objects.filter(id__in=visit_ids)
         for visit in visits:
-            assert visit.review_status == VisitReviewStatus.disagree
+            assert visit.review_status == review_status
 
-    def test_user_visit_review_network_manager(self):
+    def test_user_visit_review_network_manager_table(self):
         self.url = reverse("opportunity:user_visit_review", args=(self.nm_org.slug, self.opportunity.id))
         self.client.force_login(self.nm_user)
         response = self.client.get(self.url)

From efdb94c2aa90c099fb93bb4e8e0668d60dd8b62d Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Tue, 22 Oct 2024 16:13:08 +0530
Subject: [PATCH 12/96] Add review status completed work update tests

---
 .../opportunity/tests/factories.py            |  1 +
 .../opportunity/tests/test_visit_import.py    | 36 +++++++++++++++++++
 2 files changed, 37 insertions(+)

diff --git a/commcare_connect/opportunity/tests/factories.py b/commcare_connect/opportunity/tests/factories.py
index 5f3316fd..1646bc70 100644
--- a/commcare_connect/opportunity/tests/factories.py
+++ b/commcare_connect/opportunity/tests/factories.py
@@ -130,6 +130,7 @@ class UserVisitFactory(DjangoModelFactory):
     visit_date = Faker("date_time", tzinfo=timezone.utc)
     form_json = Faker("pydict", value_types=[str, int, float, bool])
     xform_id = Faker("uuid4")
+    completed_work = SubFactory(CompletedWorkFactory)
 
     class Meta:
         model = "opportunity.UserVisit"
diff --git a/commcare_connect/opportunity/tests/test_visit_import.py b/commcare_connect/opportunity/tests/test_visit_import.py
index 729f05d7..e41ddf5a 100644
--- a/commcare_connect/opportunity/tests/test_visit_import.py
+++ b/commcare_connect/opportunity/tests/test_visit_import.py
@@ -584,3 +584,39 @@ def test_network_manager_reject_flagged_visit(mobile_user: User, opportunity: Op
         assert visit.status_modified_date is not None
         assert before_update <= visit.status_modified_date <= after_update
         assert visit.review_created_on is None
+
+
+@pytest.mark.parametrize("opportunity", [{"managed": True}], indirect=True)
+@pytest.mark.parametrize(
+    "review_status, cw_status",
+    [
+        (VisitReviewStatus.pending, CompletedWorkStatus.pending),
+        (VisitReviewStatus.agree, CompletedWorkStatus.approved),
+        (VisitReviewStatus.disagree, CompletedWorkStatus.pending),
+    ],
+)
+def test_review_completed_work_status(
+    mobile_user_with_connect_link: User, opportunity: Opportunity, review_status, cw_status
+):
+    deliver_unit = DeliverUnitFactory(app=opportunity.deliver_app, payment_unit=opportunity.paymentunit_set.first())
+    access = OpportunityAccess.objects.get(user=mobile_user_with_connect_link, opportunity=opportunity)
+    UserVisitFactory.create_batch(
+        2,
+        opportunity_access=access,
+        status=VisitValidationStatus.approved,
+        review_status=review_status,
+        review_created_on=now(),
+        completed_work__status=CompletedWorkStatus.pending,
+        completed_work__opportunity_access=access,
+        completed_work__payment_unit=opportunity.paymentunit_set.first(),
+        deliver_unit=deliver_unit,
+    )
+    assert access.payment_accrued == 0
+    update_payment_accrued(opportunity, {mobile_user_with_connect_link.id})
+    completed_works = CompletedWork.objects.filter(opportunity_access=access)
+    payment_accrued = 0
+    for cw in completed_works:
+        assert cw.status == cw_status
+        payment_accrued += cw.payment_accrued
+    access.refresh_from_db()
+    assert access.payment_accrued == payment_accrued

From 74da1e22d28c50c65f0e37a21ab78e07eb48be1d Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Wed, 23 Oct 2024 15:52:17 +0530
Subject: [PATCH 13/96] Fix test payment_accrued calculation

---
 commcare_connect/opportunity/tests/test_visit_import.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/commcare_connect/opportunity/tests/test_visit_import.py b/commcare_connect/opportunity/tests/test_visit_import.py
index e41ddf5a..6fbd9579 100644
--- a/commcare_connect/opportunity/tests/test_visit_import.py
+++ b/commcare_connect/opportunity/tests/test_visit_import.py
@@ -617,6 +617,7 @@ def test_review_completed_work_status(
     payment_accrued = 0
     for cw in completed_works:
         assert cw.status == cw_status
-        payment_accrued += cw.payment_accrued
+        if cw.status == CompletedWorkStatus.approved:
+            payment_accrued += cw.payment_accrued
     access.refresh_from_db()
     assert access.payment_accrued == payment_accrued

From 161b11334d02d56b5475827f020c104a5cf55c60 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Wed, 23 Oct 2024 15:52:31 +0530
Subject: [PATCH 14/96] Add combine receiver tests with parametrize

---
 .../tests/test_receiver_integration.py        | 33 +++++++------------
 1 file changed, 12 insertions(+), 21 deletions(-)

diff --git a/commcare_connect/form_receiver/tests/test_receiver_integration.py b/commcare_connect/form_receiver/tests/test_receiver_integration.py
index be002639..2d2a93a4 100644
--- a/commcare_connect/form_receiver/tests/test_receiver_integration.py
+++ b/commcare_connect/form_receiver/tests/test_receiver_integration.py
@@ -588,30 +588,21 @@ def test_receiver_verification_flags_catchment_areas(
 
 
 @pytest.mark.parametrize("opportunity", [{"managed": True}], indirect=True)
-def test_receiver_auto_agree_approved_visit(
-    mobile_user_with_connect_link: User, api_client: APIClient, opportunity: Opportunity
-):
-    assert opportunity.managed
-    form_json = get_form_json_for_payment_unit(opportunity.paymentunit_set.all()[0])
-    make_request(api_client, form_json, mobile_user_with_connect_link)
-    visit = UserVisit.objects.get(user=mobile_user_with_connect_link)
-    assert not visit.flagged
-    assert visit.status == VisitValidationStatus.approved
-    assert visit.review_status == VisitReviewStatus.agree
-
-
-@pytest.mark.parametrize("opportunity", [{"managed": True}], indirect=True)
-def test_receiver_flagged_visit_review_pending(
-    mobile_user_with_connect_link: User, api_client: APIClient, opportunity: Opportunity
+@pytest.mark.parametrize(
+    "visit_status, review_status",
+    [
+        (VisitValidationStatus.approved, VisitReviewStatus.agree),
+        (VisitValidationStatus.pending, VisitReviewStatus.pending),
+    ],
+)
+def test_receiver_visit_review_status(
+    mobile_user_with_connect_link: User, api_client: APIClient, opportunity: Opportunity, visit_status, review_status
 ):
     assert opportunity.managed
-    form_json = get_form_json_for_payment_unit(opportunity.paymentunit_set.all()[0])
-    form_json["metadata"]["location"] = None
+    form_json = get_form_json_for_payment_unit(opportunity.paymentunit_set.first())
+    if visit_status != VisitValidationStatus.approved:
+        form_json["metadata"]["location"] = None
     make_request(api_client, form_json, mobile_user_with_connect_link)
-    visit = UserVisit.objects.get(user=mobile_user_with_connect_link)
-    assert visit.flagged
-    assert visit.status == VisitValidationStatus.pending
-    assert visit.review_status == VisitReviewStatus.pending
 
 
 def get_form_json_for_payment_unit(payment_unit):

From 543fa4515a52701d71467964e506c3151de3bd52 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Wed, 23 Oct 2024 15:57:19 +0530
Subject: [PATCH 15/96] Add combine network manager tests with parametrize

---
 .../opportunity/tests/test_visit_import.py    | 35 +++++--------------
 1 file changed, 8 insertions(+), 27 deletions(-)

diff --git a/commcare_connect/opportunity/tests/test_visit_import.py b/commcare_connect/opportunity/tests/test_visit_import.py
index 6fbd9579..d7e5d258 100644
--- a/commcare_connect/opportunity/tests/test_visit_import.py
+++ b/commcare_connect/opportunity/tests/test_visit_import.py
@@ -543,47 +543,28 @@ def get_assignable_completed_work_count(access: OpportunityAccess) -> int:
 
 
 @pytest.mark.parametrize("opportunity", [{"managed": True}], indirect=True)
-def test_network_manager_approve_flagged_visit(mobile_user: User, opportunity: Opportunity):
+@pytest.mark.parametrize("visit_status", [VisitValidationStatus.approved, VisitValidationStatus.rejected])
+def test_network_manager_flagged_visit_review_status(mobile_user: User, opportunity: Opportunity, visit_status):
     assert opportunity.managed
     access = OpportunityAccess.objects.get(user=mobile_user, opportunity=opportunity)
     visits = UserVisitFactory.create_batch(
         5, opportunity=opportunity, status=VisitValidationStatus.pending, user=mobile_user, opportunity_access=access
     )
     dataset = Dataset(headers=["visit id", "status", "rejected reason", "justification"])
-    dataset.extend([[visit.xform_id, VisitValidationStatus.approved.value, "", "justification"] for visit in visits])
+    dataset.extend([[visit.xform_id, visit_status.value, "", "justification"] for visit in visits])
     before_update = now()
     import_status = _bulk_update_visit_status(opportunity, dataset)
     after_update = now()
     assert not import_status.missing_visits
     updated_visits = UserVisit.objects.filter(opportunity=opportunity)
     for visit in updated_visits:
-        assert visit.status == VisitValidationStatus.approved
+        assert visit.status == visit_status
         assert visit.status_modified_date is not None
         assert before_update <= visit.status_modified_date <= after_update
-        assert before_update <= visit.review_created_on <= after_update
-        assert visit.review_status == VisitReviewStatus.pending
-        assert visit.justification == "justification"
-
-
-@pytest.mark.parametrize("opportunity", [{"managed": True}], indirect=True)
-def test_network_manager_reject_flagged_visit(mobile_user: User, opportunity: Opportunity):
-    assert opportunity.managed
-    access = OpportunityAccess.objects.get(user=mobile_user, opportunity=opportunity)
-    visits = UserVisitFactory.create_batch(
-        5, opportunity=opportunity, status=VisitValidationStatus.pending, user=mobile_user, opportunity_access=access
-    )
-    dataset = Dataset(headers=["visit id", "status", "rejected reason", "justification"])
-    dataset.extend([[visit.xform_id, VisitValidationStatus.rejected.value, "", "justification"] for visit in visits])
-    before_update = now()
-    import_status = _bulk_update_visit_status(opportunity, dataset)
-    after_update = now()
-    assert not import_status.missing_visits
-    updated_visits = UserVisit.objects.filter(opportunity=opportunity)
-    for visit in updated_visits:
-        assert visit.status == VisitValidationStatus.rejected
-        assert visit.status_modified_date is not None
-        assert before_update <= visit.status_modified_date <= after_update
-        assert visit.review_created_on is None
+        if visit.status == VisitValidationStatus.approved:
+            assert before_update <= visit.review_created_on <= after_update
+            assert visit.review_status == VisitReviewStatus.pending
+            assert visit.justification == "justification"
 
 
 @pytest.mark.parametrize("opportunity", [{"managed": True}], indirect=True)

From 35049d33472a66aa3f4affbc06e2e16dcfb0d78d Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Thu, 24 Oct 2024 15:59:05 +0530
Subject: [PATCH 16/96] moved the logic to command to update the paid date

---
 .../update_completed_work_paid_date.py        | 23 +++++++++++++++++++
 .../0060_completedwork_payment_date.py        | 15 +-----------
 .../opportunity/utils/completed_work.py       | 18 ++++-----------
 3 files changed, 28 insertions(+), 28 deletions(-)
 create mode 100644 commcare_connect/opportunity/management/commands/update_completed_work_paid_date.py

diff --git a/commcare_connect/opportunity/management/commands/update_completed_work_paid_date.py b/commcare_connect/opportunity/management/commands/update_completed_work_paid_date.py
new file mode 100644
index 00000000..000d1e4f
--- /dev/null
+++ b/commcare_connect/opportunity/management/commands/update_completed_work_paid_date.py
@@ -0,0 +1,23 @@
+from django.core.management import BaseCommand
+from django.db import transaction
+
+from commcare_connect.opportunity.models import OpportunityAccess
+from commcare_connect.opportunity.utils.completed_work import update_work_payment_date
+
+
+class Command(BaseCommand):
+    help = "Updates paid dates from payments for all opportunity accesses"
+
+    def handle(self, *args, **kwargs):
+        try:
+            with transaction.atomic():
+                accesses = OpportunityAccess.objects.all()
+                self.stdout.write("Starting to process to update the paid date...")
+
+                for access in accesses:
+                    update_work_payment_date(access)
+
+                self.stdout.write("Process completed")
+
+        except Exception as e:
+            self.stdout.write(self.style.ERROR(f"An error occurred: {str(e)}"))
diff --git a/commcare_connect/opportunity/migrations/0060_completedwork_payment_date.py b/commcare_connect/opportunity/migrations/0060_completedwork_payment_date.py
index c8481f41..d1478347 100644
--- a/commcare_connect/opportunity/migrations/0060_completedwork_payment_date.py
+++ b/commcare_connect/opportunity/migrations/0060_completedwork_payment_date.py
@@ -1,18 +1,6 @@
 # Generated by Django 4.2.5 on 2024-10-07 08:54
 
-from django.db import migrations, models, transaction
-
-from commcare_connect.opportunity.utils.completed_work import update_work_payment_date
-
-
-@transaction.atomic
-def update_paid_date_from_payments(apps, schema_editor):
-    OpportunityAccess = apps.get_model("opportunity.OpportunityAccess")
-    Payment = apps.get_model("opportunity.Payment")
-    CompletedWork = apps.get_model("opportunity.CompletedWork")
-    accesses = OpportunityAccess.objects.all()
-    for access in accesses:
-        update_work_payment_date(access, Payment, CompletedWork)
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
@@ -26,5 +14,4 @@ class Migration(migrations.Migration):
             name="payment_date",
             field=models.DateTimeField(null=True),
         ),
-        migrations.RunPython(update_paid_date_from_payments, migrations.RunPython.noop),
     ]
diff --git a/commcare_connect/opportunity/utils/completed_work.py b/commcare_connect/opportunity/utils/completed_work.py
index 2050e30b..df2d9906 100644
--- a/commcare_connect/opportunity/utils/completed_work.py
+++ b/commcare_connect/opportunity/utils/completed_work.py
@@ -44,19 +44,9 @@ def update_status(completed_works, opportunity_access, compute_payment=True):
         opportunity_access.save()
 
 
-def update_work_payment_date(access: OpportunityAccess, payment_model=None, completed_work_model=None):
-    """
-    Dynamically assign models to avoid issues with historical models during migrations.
-    Top-level imports use the current model, which may not match the schema at migration
-    time. This ensures we use historical models during migrations and current models in normal execution.
-    """
-    payment_model_ref = payment_model or Payment
-    completed_work_model_ref = completed_work_model or CompletedWork
-
-    payments = payment_model_ref.objects.filter(opportunity_access=access).order_by("date_paid")
-    completed_works = completed_work_model_ref.objects.filter(opportunity_access=access).order_by(
-        "status_modified_date"
-    )
+def update_work_payment_date(access: OpportunityAccess):
+    payments = Payment.objects.filter(opportunity_access=access).order_by("date_paid")
+    completed_works = CompletedWork.objects.filter(opportunity_access=access).order_by("status_modified_date")
 
     if not payments or not completed_works:
         return
@@ -86,4 +76,4 @@ def update_work_payment_date(access: OpportunityAccess, payment_model=None, comp
         break
 
     if works_to_update:
-        completed_work_model_ref.objects.bulk_update(works_to_update, ["payment_date"])
+        CompletedWork.objects.bulk_update(works_to_update, ["payment_date"])

From 9ab669eb03499aa4e6be86e6cab2cda9a0698f09 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Tue, 29 Oct 2024 13:22:09 +0530
Subject: [PATCH 17/96] updated budget per user for managed opp

---
 commcare_connect/opportunity/models.py        |  3 ++-
 commcare_connect/program/tests/factories.py   |  1 +
 commcare_connect/program/tests/test_models.py | 20 +++++++++++++++++++
 3 files changed, 23 insertions(+), 1 deletion(-)
 create mode 100644 commcare_connect/program/tests/test_models.py

diff --git a/commcare_connect/opportunity/models.py b/commcare_connect/opportunity/models.py
index 17f68978..f107d8e6 100644
--- a/commcare_connect/opportunity/models.py
+++ b/commcare_connect/opportunity/models.py
@@ -181,8 +181,9 @@ def budget_per_visit_new(self):
     def budget_per_user(self):
         payment_units = self.paymentunit_set.all()
         budget = 0
+        org_pay = self.managedopportunity.org_pay_per_visit if self.managed else 0
         for pu in payment_units:
-            budget += pu.max_total * pu.amount
+            budget += pu.max_total * (pu.amount + org_pay)
         return budget
 
     @property
diff --git a/commcare_connect/program/tests/factories.py b/commcare_connect/program/tests/factories.py
index f07baff6..00f22762 100644
--- a/commcare_connect/program/tests/factories.py
+++ b/commcare_connect/program/tests/factories.py
@@ -22,6 +22,7 @@ class Meta:
 
 class ManagedOpportunityFactory(OpportunityFactory):
     program = SubFactory(ProgramFactory)
+    org_pay_per_visit = Faker("random_int", min=500, max=1000)
 
     class Meta:
         model = ManagedOpportunity
diff --git a/commcare_connect/program/tests/test_models.py b/commcare_connect/program/tests/test_models.py
new file mode 100644
index 00000000..2b482778
--- /dev/null
+++ b/commcare_connect/program/tests/test_models.py
@@ -0,0 +1,20 @@
+import pytest
+
+from commcare_connect.opportunity.tests.factories import PaymentUnitFactory
+from commcare_connect.program.models import ManagedOpportunity
+from commcare_connect.program.tests.factories import ManagedOpportunityFactory
+
+
+@pytest.mark.django_db
+def test_managed_opportunity_stats():
+    opportunity = ManagedOpportunityFactory(total_budget=3600000, org_pay_per_visit=450)
+    PaymentUnitFactory(opportunity=opportunity, max_total=600, max_daily=5, amount=750, parent_payment_unit=None)
+
+    opportunity = ManagedOpportunity.objects.get(id=opportunity.id)
+
+    assert opportunity.budget_per_user == 720000
+    assert opportunity.allotted_visits == 3000
+    assert opportunity.number_of_users == 5
+    assert opportunity.max_visits_per_user_new == 600
+    assert opportunity.daily_max_visits_per_user_new == 5
+    assert opportunity.budget_per_visit_new == 750

From 0f7c83f0113cb3a6bee6975462e4a077751034b5 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Tue, 29 Oct 2024 15:02:14 +0530
Subject: [PATCH 18/96] Add missing test

---
 .../form_receiver/tests/test_receiver_integration.py       | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/commcare_connect/form_receiver/tests/test_receiver_integration.py b/commcare_connect/form_receiver/tests/test_receiver_integration.py
index e1e601f6..e24857da 100644
--- a/commcare_connect/form_receiver/tests/test_receiver_integration.py
+++ b/commcare_connect/form_receiver/tests/test_receiver_integration.py
@@ -579,7 +579,7 @@ def test_receiver_verification_flags_catchment_areas(
     assert ["catchment", "Visit outside worker catchment areas"] in visit.flag_reason.get("flags", [])
 
 
-@pytest.mark.parametrize("opportunity", [{"opp_options": {"managed": True}}], indirect=True)
+@pytest.mark.parametrize("opportunity", [{"opp_options": {"managed": True, "org_pay_per_visit": 2}}], indirect=True)
 @pytest.mark.parametrize(
     "visit_status, review_status",
     [
@@ -595,6 +595,11 @@ def test_receiver_visit_review_status(
     if visit_status != VisitValidationStatus.approved:
         form_json["metadata"]["location"] = None
     make_request(api_client, form_json, mobile_user_with_connect_link)
+    visit = UserVisit.objects.get(user=mobile_user_with_connect_link)
+    if visit_status != VisitValidationStatus.approved:
+        assert visit.flagged
+    assert visit.status == visit_status
+    assert visit.review_status == review_status
 
 
 def get_form_json_for_payment_unit(payment_unit):

From b389826dac400218ba412a6dce4303c1c0a4964b Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Tue, 29 Oct 2024 15:27:41 +0530
Subject: [PATCH 19/96] Add ManagedOpportunity in fixture

---
 commcare_connect/conftest.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/commcare_connect/conftest.py b/commcare_connect/conftest.py
index 415a6bba..c984262b 100644
--- a/commcare_connect/conftest.py
+++ b/commcare_connect/conftest.py
@@ -10,6 +10,7 @@
     PaymentUnitFactory,
 )
 from commcare_connect.organization.models import Organization
+from commcare_connect.program.tests.factories import ManagedOpportunityFactory
 from commcare_connect.users.models import User
 from commcare_connect.users.tests.factories import (
     ConnectIdUserLinkFactory,
@@ -52,7 +53,10 @@ def opportunity(request):
     verification_flags = getattr(request, "param", {}).get("verification_flags", {})
     opp_options = {"is_test": False}
     opp_options.update(getattr(request, "param", {}).get("opp_options", {}))
-    factory = OpportunityFactory(**opp_options)
+    if opp_options.get("managed", False):
+        factory = ManagedOpportunityFactory(**opp_options)
+    else:
+        factory = OpportunityFactory(**opp_options)
     OpportunityVerificationFlagsFactory(opportunity=factory, **verification_flags)
     return factory
 

From 1064b98263c32f1ad3ad395a036bdcda3b8239c3 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 30 Oct 2024 10:43:04 +0530
Subject: [PATCH 20/96] filtered credentails on the basis of current org

---
 commcare_connect/connect_id_client/main.py    |   7 +-
 commcare_connect/opportunity/forms.py         |   8 +-
 .../opportunity/tests/test_forms.py           | 198 +++++++++++++++++-
 commcare_connect/opportunity/views.py         |   7 +-
 commcare_connect/organization/views.py        |   4 +-
 5 files changed, 215 insertions(+), 9 deletions(-)

diff --git a/commcare_connect/connect_id_client/main.py b/commcare_connect/connect_id_client/main.py
index 1f5307a1..7f117d9f 100644
--- a/commcare_connect/connect_id_client/main.py
+++ b/commcare_connect/connect_id_client/main.py
@@ -54,8 +54,11 @@ def add_credential(organization: Organization, credential: str, users: list[str]
     return
 
 
-def fetch_credentials():
-    response = _make_request(GET, "/users/fetch_credentials")
+def fetch_credentials(org_slug=None):
+    params = {}
+    if org_slug:
+        params["org_slug"] = org_slug
+    response = _make_request(GET, "/users/fetch_credentials", params=params)
     data = response.json()
     return [Credential(**c) for c in data["credentials"]]
 
diff --git a/commcare_connect/opportunity/forms.py b/commcare_connect/opportunity/forms.py
index 3e472450..7c567d89 100644
--- a/commcare_connect/opportunity/forms.py
+++ b/commcare_connect/opportunity/forms.py
@@ -33,7 +33,8 @@
 
 class OpportunityUserInviteForm(forms.Form):
     def __init__(self, *args, **kwargs):
-        credentials = connect_id_client.fetch_credentials()
+        org_slug = kwargs.pop("org_slug", None)
+        credentials = connect_id_client.fetch_credentials(org_slug)
         super().__init__(*args, **kwargs)
 
         self.helper = FormHelper(self)
@@ -73,7 +74,10 @@ def clean_users(self):
         return split_users
 
 
-class OpportunityChangeForm(forms.ModelForm, OpportunityUserInviteForm):
+class OpportunityChangeForm(
+    OpportunityUserInviteForm,
+    forms.ModelForm,
+):
     class Meta:
         model = Opportunity
         fields = [
diff --git a/commcare_connect/opportunity/tests/test_forms.py b/commcare_connect/opportunity/tests/test_forms.py
index a8f422be..58f6db95 100644
--- a/commcare_connect/opportunity/tests/test_forms.py
+++ b/commcare_connect/opportunity/tests/test_forms.py
@@ -5,8 +5,8 @@
 import pytest
 from factory.fuzzy import FuzzyDate, FuzzyText
 
-from commcare_connect.opportunity.forms import OpportunityCreationForm
-from commcare_connect.opportunity.tests.factories import ApplicationFactory
+from commcare_connect.opportunity.forms import OpportunityChangeForm, OpportunityCreationForm
+from commcare_connect.opportunity.tests.factories import ApplicationFactory, CommCareAppFactory, OpportunityFactory
 
 
 class TestOpportunityCreationForm:
@@ -111,3 +111,197 @@ def test_save(self, user, organization):
         )
         form.is_valid()
         form.save()
+
+
+@pytest.mark.django_db
+class TestOpportunityChangeForm:
+    @pytest.fixture(autouse=True)
+    def setup_credentials_mock(self, monkeypatch):
+        self.mock_credentials = [
+            type("Credential", (), {"slug": "cert1", "name": "Work for test"}),
+            type("Credential", (), {"slug": "cert2", "name": "Work for test"}),
+        ]
+        monkeypatch.setattr(
+            "commcare_connect.connect_id_client.fetch_credentials", lambda org_slug: self.mock_credentials
+        )
+
+    @pytest.fixture
+    def valid_opportunity(self, organization):
+        return OpportunityFactory(
+            organization=organization,
+            active=True,
+            learn_app=CommCareAppFactory(cc_app_id="test_learn_app"),
+            deliver_app=CommCareAppFactory(cc_app_id="test_deliver_app"),
+            name="Test Opportunity",
+            description="Test Description",
+            short_description="Short Description",
+            currency="USD",
+            is_test=False,
+            end_date=datetime.date.today() + datetime.timedelta(days=30),
+        )
+
+    @pytest.fixture
+    def base_form_data(self, valid_opportunity):
+        return {
+            "name": "Updated Opportunity",
+            "description": "Updated Description",
+            "short_description": "Updated Short Description",
+            "active": True,
+            "currency": "EUR",
+            "is_test": False,
+            "delivery_type": valid_opportunity.delivery_type.id,
+            "additional_users": 5,
+            "end_date": (datetime.date.today() + datetime.timedelta(days=60)).isoformat(),
+            "users": "+1234567890\n+9876543210",
+            "filter_country": "US",
+            "filter_credential": "cert1",
+        }
+
+    def test_form_initialization(self, valid_opportunity, organization):
+        form = OpportunityChangeForm(instance=valid_opportunity, org_slug=organization.slug)
+        expected_fields = {
+            "name",
+            "description",
+            "short_description",
+            "active",
+            "currency",
+            "is_test",
+            "delivery_type",
+            "additional_users",
+            "end_date",
+            "users",
+            "filter_country",
+            "filter_credential",
+        }
+        assert all(field in form.fields for field in expected_fields)
+
+        expected_initial = {
+            "name": valid_opportunity.name,
+            "description": valid_opportunity.description,
+            "short_description": valid_opportunity.short_description,
+            "active": valid_opportunity.active,
+            "currency": valid_opportunity.currency,
+            "is_test": valid_opportunity.is_test,
+            "delivery_type": valid_opportunity.delivery_type.id,
+            "end_date": valid_opportunity.end_date.isoformat(),
+            "filter_country": [""],
+            "filter_credential": [""],
+        }
+        assert all(form.initial.get(key) == value for key, value in expected_initial.items())
+
+    @pytest.mark.parametrize(
+        "field",
+        [
+            "name",
+            "description",
+            "short_description",
+            "currency",
+        ],
+    )
+    def test_required_fields(self, valid_opportunity, organization, field, base_form_data):
+        data = base_form_data.copy()
+        data[field] = ""
+        form = OpportunityChangeForm(data=data, instance=valid_opportunity, org_slug=organization.slug)
+        assert not form.is_valid()
+        assert field in form.errors
+
+    @pytest.mark.parametrize(
+        "test_data",
+        [
+            pytest.param(
+                {
+                    "field": "additional_users",
+                    "value": "invalid",
+                    "error_expected": True,
+                    "error_message": "Enter a whole number.",
+                },
+                id="invalid_additional_users",
+            ),
+            pytest.param(
+                {
+                    "field": "end_date",
+                    "value": "invalid-date",
+                    "error_expected": True,
+                    "error_message": "Enter a valid date.",
+                },
+                id="invalid_end_date",
+            ),
+            pytest.param(
+                {
+                    "field": "users",
+                    "value": "  +1234567890  \n  +9876543210  ",
+                    "error_expected": False,
+                    "expected_clean": ["+1234567890", "+9876543210"],
+                },
+                id="valid_users_with_whitespace",
+            ),
+        ],
+    )
+    def test_field_validation(self, valid_opportunity, organization, base_form_data, test_data):
+        data = base_form_data.copy()
+        data[test_data["field"]] = test_data["value"]
+        form = OpportunityChangeForm(data=data, instance=valid_opportunity, org_slug=organization.slug)
+        if test_data["error_expected"]:
+            assert not form.is_valid()
+            assert test_data["error_message"] in str(form.errors[test_data["field"]])
+        else:
+            assert form.is_valid()
+            if "expected_clean" in test_data:
+                assert form.cleaned_data[test_data["field"]] == test_data["expected_clean"]
+
+    @pytest.mark.parametrize(
+        "app_scenario",
+        [
+            pytest.param(
+                {
+                    "active_app_ids": ("unique_app1", "unique_app2"),
+                    "new_app_ids": ("different_app1", "different_app2"),
+                    "expected_valid": True,
+                },
+                id="unique_apps",
+            ),
+            pytest.param(
+                {
+                    "active_app_ids": ("shared_app1", "shared_app2"),
+                    "new_app_ids": ("shared_app1", "shared_app2"),
+                    "expected_valid": False,
+                },
+                id="reused_apps",
+            ),
+        ],
+    )
+    def test_app_reuse_validation(self, organization, base_form_data, app_scenario):
+        OpportunityFactory(
+            organization=organization,
+            active=True,
+            learn_app=CommCareAppFactory(cc_app_id=app_scenario["active_app_ids"][0]),
+            deliver_app=CommCareAppFactory(cc_app_id=app_scenario["active_app_ids"][1]),
+        )
+
+        inactive_opp = OpportunityFactory(
+            organization=organization,
+            active=False,
+            learn_app=CommCareAppFactory(cc_app_id=app_scenario["new_app_ids"][0]),
+            deliver_app=CommCareAppFactory(cc_app_id=app_scenario["new_app_ids"][1]),
+        )
+
+        form = OpportunityChangeForm(data=base_form_data, instance=inactive_opp, org_slug=organization.slug)
+
+        assert form.is_valid() == app_scenario["expected_valid"]
+        if not app_scenario["expected_valid"]:
+            assert "Cannot reactivate opportunity with reused applications" in str(form.errors["active"])
+
+    @pytest.mark.parametrize(
+        "data_updates,expected_valid",
+        [
+            ({"currency": "USD", "additional_users": 5}, True),
+            ({"currency": "EUR", "additional_users": 10}, True),
+            ({"currency": "INVALID", "additional_users": 5}, False),
+            ({"currency": "USD", "additional_users": -5}, True),
+        ],
+    )
+    def test_valid_combinations(self, valid_opportunity, organization, base_form_data, data_updates, expected_valid):
+        data = base_form_data.copy()
+        data.update(data_updates)
+        form = OpportunityChangeForm(data=data, instance=valid_opportunity, org_slug=organization.slug)
+        assert form.is_valid() == expected_valid
diff --git a/commcare_connect/opportunity/views.py b/commcare_connect/opportunity/views.py
index e9083536..a53c0050 100644
--- a/commcare_connect/opportunity/views.py
+++ b/commcare_connect/opportunity/views.py
@@ -212,6 +212,11 @@ class OpportunityEdit(OrganizationUserMemberRoleMixin, UpdateView):
     def get_success_url(self):
         return reverse("opportunity:detail", args=(self.request.org.slug, self.object.id))
 
+    def get_form_kwargs(self):
+        kwargs = super().get_form_kwargs()
+        kwargs["org_slug"] = self.request.org.slug
+        return kwargs
+
     def form_valid(self, form):
         opportunity = form.instance
         opportunity.modified_by = self.request.user.email
@@ -1079,7 +1084,7 @@ def import_catchment_area(request, org_slug=None, pk=None):
 @org_member_required
 def opportunity_user_invite(request, org_slug=None, pk=None):
     opportunity = get_object_or_404(Opportunity, organization=request.org, id=pk)
-    form = OpportunityUserInviteForm(data=request.POST or None)
+    form = OpportunityUserInviteForm(data=request.POST or None, org_slug=request.org.slug)
     if form.is_valid():
         users = form.cleaned_data["users"]
         filter_country = form.cleaned_data["filter_country"]
diff --git a/commcare_connect/organization/views.py b/commcare_connect/organization/views.py
index f2e5da30..fce5002c 100644
--- a/commcare_connect/organization/views.py
+++ b/commcare_connect/organization/views.py
@@ -47,7 +47,7 @@ def organization_home(request, org_slug):
     if not form:
         form = OrganizationChangeForm(instance=org)
 
-    credentials = connect_id_client.fetch_credentials()
+    credentials = connect_id_client.fetch_credentials(org_slug=request.org.slug)
     credential_name = f"Worked for {org.name}"
     if not any(c.name == credential_name for c in credentials):
         credentials.append(Credential(name=credential_name, slug=slugify(credential_name)))
@@ -96,7 +96,7 @@ def accept_invite(request, org_slug, invite_id):
 @require_POST
 def add_credential_view(request, org_slug):
     org = get_object_or_404(Organization, slug=org_slug)
-    credentials = connect_id_client.fetch_credentials()
+    credentials = connect_id_client.fetch_credentials(org_slug=request.org.slug)
     credential_name = f"Worked for {org.name}"
     if not any(c.name == credential_name for c in credentials):
         credentials.append(Credential(name=credential_name, slug=slugify(credential_name)))

From d38a8e11df9db1ca0b1c1da050612527a47fb245 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 30 Oct 2024 16:23:52 +0530
Subject: [PATCH 21/96] changed no of user calculation

---
 commcare_connect/opportunity/models.py        | 14 +++++++++++---
 commcare_connect/program/tests/test_models.py |  4 ++--
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/commcare_connect/opportunity/models.py b/commcare_connect/opportunity/models.py
index f107d8e6..cf763f5d 100644
--- a/commcare_connect/opportunity/models.py
+++ b/commcare_connect/opportunity/models.py
@@ -159,7 +159,16 @@ def approved_visits(self):
 
     @property
     def number_of_users(self):
-        return self.total_budget / self.budget_per_user
+        if not self.managed:
+            return self.total_budget / self.budget_per_user
+
+        budget_per_user = 0
+        payment_units = self.paymentunit_set.all()
+        org_pay = self.managedopportunity.org_pay_per_visit
+        for pu in payment_units:
+            budget_per_user += pu.max_total * (pu.amount + org_pay)
+
+        return self.total_budget / budget_per_user
 
     @property
     def allotted_visits(self):
@@ -181,9 +190,8 @@ def budget_per_visit_new(self):
     def budget_per_user(self):
         payment_units = self.paymentunit_set.all()
         budget = 0
-        org_pay = self.managedopportunity.org_pay_per_visit if self.managed else 0
         for pu in payment_units:
-            budget += pu.max_total * (pu.amount + org_pay)
+            budget += pu.max_total * pu.amount
         return budget
 
     @property
diff --git a/commcare_connect/program/tests/test_models.py b/commcare_connect/program/tests/test_models.py
index 2b482778..ef4243d3 100644
--- a/commcare_connect/program/tests/test_models.py
+++ b/commcare_connect/program/tests/test_models.py
@@ -8,11 +8,11 @@
 @pytest.mark.django_db
 def test_managed_opportunity_stats():
     opportunity = ManagedOpportunityFactory(total_budget=3600000, org_pay_per_visit=450)
-    PaymentUnitFactory(opportunity=opportunity, max_total=600, max_daily=5, amount=750, parent_payment_unit=None)
+    PaymentUnitFactory(opportunity=opportunity, max_total=600, max_daily=5, amount=750)
 
     opportunity = ManagedOpportunity.objects.get(id=opportunity.id)
 
-    assert opportunity.budget_per_user == 720000
+    assert opportunity.budget_per_user == 450000
     assert opportunity.allotted_visits == 3000
     assert opportunity.number_of_users == 5
     assert opportunity.max_visits_per_user_new == 600

From 4fc2b2ea68872b24f0e8e085b0558ef004c2158b Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 11:01:50 +0200
Subject: [PATCH 22/96] mocking out some dashboard updates

---
 .../templates/reports/dashboard.html          | 106 +++++++++++++++++-
 1 file changed, 102 insertions(+), 4 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 3e164d3a..e2def6e2 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -4,10 +4,108 @@
 {% load django_tables2 %}
 {% block title %}Admin Dashboard{% endblock %}
 {% block content %}
-  <h2 class="mb-2">Visit Dashboard</h2>
-  <div id="map" class="mt-3 mb-2 rounded border" style="height: 900px;"></div>
-
-
+  <h2 class="my-2">Program Dashboard</h2>
+  <div class="container">
+    <div class="row mb-3">
+      <div class="col-md-4 mt-4">
+        <div class="form-group">
+          <label for="program-select" class="form-label">Program</label>
+          <select class="form-select" id="program-select">
+            <option selected>Select a program...</option>
+            {% for program in programs %}
+              <option value="{{ program.id }}">{{ program.name }}</option>
+            {% endfor %}
+          </select>
+        </div>
+      </div>
+      <div class="col-md-4 mt-4">
+        <div class="form-group">
+          <label for="org-select" class="form-label">Organization</label>
+          <select class="form-select" id="org-select">
+            <option selected>Select an organization...</option>
+            {% for org in organizations %}
+              <option value="{{ org.id }}">{{ org.name }}</option>
+            {% endfor %}
+          </select>
+        </div>
+      </div>
+      <div class="col-md-4 mt-4">
+        <div class="form-group">
+          <label for="date-select" class="form-label">Date</label>
+          <input type="date" class="form-control" id="date-select">
+        </div>
+      </div>
+    </div>
+    <div class="row mb-4">
+      <div class="col-12 text-end">
+        <button type="submit" class="btn btn-primary">Update Dashboard</button>
+      </div>
+    </div>
+    <div class="row">
+      <div class="col-12 col-sm-6 col-md-3 mb-3">
+        <div class="card">
+          <div class="card-body ">
+            <div class="display-4 fw-bold">
+              0
+            </div>
+            <div class="small text-muted mt-1">
+              Total Visits
+            </div>
+          </div>
+        </div>
+      </div>
+      <div class="col-12 col-sm-6 col-md-3 mb-3">
+        <div class="card">
+          <div class="card-body">
+            <div class="display-4 fw-bold">
+              0
+            </div>
+            <div class="small text-muted mt-1">
+              Active Users
+            </div>
+          </div>
+        </div>
+      </div>
+      <div class="col-12 col-sm-6 col-md-3 mb-3">
+        <div class="card">
+          <div class="card-body">
+            <div class="display-4 fw-bold">
+              0
+            </div>
+            <div class="small text-muted mt-1">
+              Completed Visits
+            </div>
+          </div>
+        </div>
+      </div>
+      <div class="col-12 col-sm-6 col-md-3 mb-3">
+        <div class="card">
+          <div class="card-body">
+            <div class="display-4 fw-bold">
+              0
+            </div>
+            <div class="small text-muted mt-1">
+              Pending Visits
+            </div>
+          </div>
+        </div>
+      </div>
+    </div>
+  </div>
+  <div class="row">
+    <div class="col-12 col-lg-8">
+      <h3 class="my-2">Service Delivery Map</h3>
+      <div id="map" class="mt-3 mb-2 rounded border" style="height: 900px;"></div>
+    </div>
+    <div class="col-12 col-lg-4">
+      <h3 class="my-2">Additional Information</h3>
+      <div class="card">
+        <div class="card-body">
+          <p class="text-muted">Placeholder content</p>
+        </div>
+      </div>
+    </div>
+  </div>
 {% endblock content %}
 {% block inline_javascript %}
 {{ block.super }}

From 76563a23feffb4b2db9792ad59e7ebe7b9449de4 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 11:16:14 +0200
Subject: [PATCH 23/96] wire the front end of dashboard stats

---
 .../templates/reports/dashboard.html          | 51 +++++++++++++++----
 1 file changed, 42 insertions(+), 9 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index e2def6e2..f090edf4 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -5,13 +5,13 @@
 {% block title %}Admin Dashboard{% endblock %}
 {% block content %}
   <h2 class="my-2">Program Dashboard</h2>
-  <div class="container">
+  <div class="container" x-data="dashboardStats()">
     <div class="row mb-3">
       <div class="col-md-4 mt-4">
         <div class="form-group">
           <label for="program-select" class="form-label">Program</label>
           <select class="form-select" id="program-select">
-            <option selected>Select a program...</option>
+            <option selected value="">Select a program...</option>
             {% for program in programs %}
               <option value="{{ program.id }}">{{ program.name }}</option>
             {% endfor %}
@@ -22,7 +22,7 @@ <h2 class="my-2">Program Dashboard</h2>
         <div class="form-group">
           <label for="org-select" class="form-label">Organization</label>
           <select class="form-select" id="org-select">
-            <option selected>Select an organization...</option>
+            <option selected value="">Select an organization...</option>
             {% for org in organizations %}
               <option value="{{ org.id }}">{{ org.name }}</option>
             {% endfor %}
@@ -32,7 +32,7 @@ <h2 class="my-2">Program Dashboard</h2>
       <div class="col-md-4 mt-4">
         <div class="form-group">
           <label for="date-select" class="form-label">Date</label>
-          <input type="date" class="form-control" id="date-select">
+          <input type="date" class="form-control" id="date-select" value="">
         </div>
       </div>
     </div>
@@ -44,8 +44,8 @@ <h2 class="my-2">Program Dashboard</h2>
     <div class="row">
       <div class="col-12 col-sm-6 col-md-3 mb-3">
         <div class="card">
-          <div class="card-body ">
-            <div class="display-4 fw-bold">
+          <div class="card-body">
+            <div class="display-4 fw-bold" x-text="stats.total_visits || 0">
               0
             </div>
             <div class="small text-muted mt-1">
@@ -57,7 +57,7 @@ <h2 class="my-2">Program Dashboard</h2>
       <div class="col-12 col-sm-6 col-md-3 mb-3">
         <div class="card">
           <div class="card-body">
-            <div class="display-4 fw-bold">
+            <div class="display-4 fw-bold" x-text="stats.active_users || 0">
               0
             </div>
             <div class="small text-muted mt-1">
@@ -69,7 +69,7 @@ <h2 class="my-2">Program Dashboard</h2>
       <div class="col-12 col-sm-6 col-md-3 mb-3">
         <div class="card">
           <div class="card-body">
-            <div class="display-4 fw-bold">
+            <div class="display-4 fw-bold" x-text="stats.completed_visits || 0">
               0
             </div>
             <div class="small text-muted mt-1">
@@ -81,7 +81,7 @@ <h2 class="my-2">Program Dashboard</h2>
       <div class="col-12 col-sm-6 col-md-3 mb-3">
         <div class="card">
           <div class="card-body">
-            <div class="display-4 fw-bold">
+            <div class="display-4 fw-bold" x-text="stats.pending_visits || 0">
               0
             </div>
             <div class="small text-muted mt-1">
@@ -110,6 +110,39 @@ <h3 class="my-2">Additional Information</h3>
 {% block inline_javascript %}
 {{ block.super }}
   {{ user_visits|json_script:"userVisits" }}
+  <script>
+    function dashboardStats() {
+      return {
+        stats: {
+          total_visits: 0,
+          active_users: 0,
+          completed_visits: 0,
+          pending_visits: 0
+        },
+        async init() {
+          await this.loadStats();
+
+          // Set up event listeners for filter changes
+          document.getElementById('program-select').addEventListener('change', () => this.loadStats());
+          document.getElementById('org-select').addEventListener('change', () => this.loadStats());
+          document.getElementById('date-select').addEventListener('change', () => this.loadStats());
+        },
+        async loadStats() {
+          const program = document.getElementById('program-select').value;
+          const organization = document.getElementById('org-select').value;
+          const date = document.getElementById('date-select').value;
+
+          try {
+            const response = await fetch(`/api/dashboard-stats/?program=${program}&organization=${organization}&date=${date}`);
+            if (!response.ok) throw new Error('Failed to load stats');
+            this.stats = await response.json();
+          } catch (error) {
+            console.error('Error loading dashboard stats:', error);
+          }
+        }
+      }
+    }
+  </script>
   <script type="module">
     window.addEventListener('DOMContentLoaded', () => {
       mapboxgl.accessToken = "{{ mapbox_token }}";

From 9588c47d76da019892ce67bf1306b6948a0e21a5 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 11:32:36 +0200
Subject: [PATCH 24/96] wip: switch to use Django filters

---
 commcare_connect/reports/urls.py              |  3 +-
 commcare_connect/reports/views.py             | 74 +++++++++++++++-
 .../templates/reports/dashboard.html          | 85 ++++---------------
 3 files changed, 87 insertions(+), 75 deletions(-)

diff --git a/commcare_connect/reports/urls.py b/commcare_connect/reports/urls.py
index 0b780e73..cd05d994 100644
--- a/commcare_connect/reports/urls.py
+++ b/commcare_connect/reports/urls.py
@@ -6,6 +6,7 @@
 
 urlpatterns = [
     path("program_dashboard", views.program_dashboard_report, name="program_dashboard_report"),
-    path("api/visit_map_data/", views.visit_map_data, name="visit_map_data"),
     path("delivery_stats", view=views.DeliveryStatsReportView.as_view(), name="delivery_stats_report"),
+    path("api/visit_map_data/", views.visit_map_data, name="visit_map_data"),
+    path("api/dashboard_stats/", views.dashboard_stats_api, name="dashboard_stats_api"),
 ]
diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index 9cea2c50..75d490b5 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -2,6 +2,8 @@
 
 import django_filters
 import django_tables2 as tables
+from crispy_forms.helper import FormHelper
+from crispy_forms.layout import Column, Layout, Row
 from django import forms
 from django.conf import settings
 from django.contrib.auth.decorators import login_required, user_passes_test
@@ -16,7 +18,8 @@
 from django_filters.views import FilterView
 
 from commcare_connect.cache import quickcache
-from commcare_connect.opportunity.models import CompletedWork, CompletedWorkStatus, DeliveryType, Payment
+from commcare_connect.opportunity.models import CompletedWork, CompletedWorkStatus, DeliveryType, Payment, UserVisit
+from commcare_connect.organization.models import Organization
 
 from .tables import AdminReportTable
 
@@ -151,14 +154,51 @@ def _get_table_data_for_quarter(quarter, delivery_type, group_by_delivery_type=F
     return data
 
 
+class DashboardFilters(django_filters.FilterSet):
+    opportunity__delivery_type = django_filters.ModelChoiceFilter(
+        queryset=DeliveryType.objects.all(),
+        empty_label="All Programs",
+        required=False,
+    )
+    opportunity__organization = django_filters.ModelChoiceFilter(
+        queryset=Organization.objects.all(),
+        empty_label="All Organizations",
+        required=False,
+    )
+    visit_date = django_filters.DateFilter(
+        widget=forms.DateInput(attrs={"type": "date"}),
+        required=False,
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.form.helper = FormHelper()
+        self.form.helper.form_class = "form-inline"
+        self.form.helper.layout = Layout(
+            Row(
+                Column("opportunity__delivery_type", css_class="col-md-4"),
+                Column("opportunity__organization", css_class="col-md-4"),
+                Column("visit_date", css_class="col-md-4"),
+            )
+        )
+
+    class Meta:
+        model = UserVisit
+        fields = ["opportunity__delivery_type", "opportunity__organization", "visit_date"]
+
+
 @login_required
-@user_passes_test(lambda user: user.is_superuser)
-@require_GET
+@user_passes_test(lambda u: u.is_superuser)
 def program_dashboard_report(request):
+    filterset = DashboardFilters(request.GET)
+
     return render(
         request,
         "reports/dashboard.html",
-        context={"mapbox_token": settings.MAPBOX_TOKEN},
+        context={
+            "mapbox_token": settings.MAPBOX_TOKEN,
+            "filter": filterset,
+        },
     )
 
 
@@ -317,3 +357,29 @@ def object_list(self):
             data = _get_table_data_for_quarter(q, delivery_type, group_by_delivery_type)
             table_data += data
         return table_data
+
+
+@login_required
+@user_passes_test(lambda u: u.is_superuser)
+def dashboard_stats_api(request):
+    filterset = DashboardFilters(request.GET)
+
+    # Use the filtered queryset to calculate stats
+    queryset = UserVisit.objects.all()
+    if filterset.is_valid():
+        queryset = filterset.filter_queryset(queryset)
+
+    # Example stats calculation (adjust based on your needs)
+    total_visits = queryset.count()
+    completed_visits = queryset.filter(status=CompletedWorkStatus.approved).count()
+    pending_visits = total_visits - completed_visits
+    active_users = queryset.values("opportunity_access__user").distinct().count()
+
+    return JsonResponse(
+        {
+            "total_visits": total_visits,
+            "active_users": active_users,
+            "completed_visits": completed_visits,
+            "pending_visits": pending_visits,
+        }
+    )
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index f090edf4..96a4e1a2 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -6,87 +6,40 @@
 {% block content %}
   <h2 class="my-2">Program Dashboard</h2>
   <div class="container" x-data="dashboardStats()">
-    <div class="row mb-3">
-      <div class="col-md-4 mt-4">
-        <div class="form-group">
-          <label for="program-select" class="form-label">Program</label>
-          <select class="form-select" id="program-select">
-            <option selected value="">Select a program...</option>
-            {% for program in programs %}
-              <option value="{{ program.id }}">{{ program.name }}</option>
-            {% endfor %}
-          </select>
-        </div>
-      </div>
-      <div class="col-md-4 mt-4">
-        <div class="form-group">
-          <label for="org-select" class="form-label">Organization</label>
-          <select class="form-select" id="org-select">
-            <option selected value="">Select an organization...</option>
-            {% for org in organizations %}
-              <option value="{{ org.id }}">{{ org.name }}</option>
-            {% endfor %}
-          </select>
-        </div>
-      </div>
-      <div class="col-md-4 mt-4">
-        <div class="form-group">
-          <label for="date-select" class="form-label">Date</label>
-          <input type="date" class="form-control" id="date-select" value="">
-        </div>
-      </div>
-    </div>
-    <div class="row mb-4">
-      <div class="col-12 text-end">
-        <button type="submit" class="btn btn-primary">Update Dashboard</button>
-      </div>
-    </div>
+    <form class="row mb-3" id="filter-form" hx-get="{% url 'reports:dashboard_stats_api' %}"
+          hx-trigger="change from:#filter-form" hx-swap="none" x-ref="filterForm">
+      {% crispy filter.form %}
+    </form>
     <div class="row">
       <div class="col-12 col-sm-6 col-md-3 mb-3">
         <div class="card">
           <div class="card-body">
-            <div class="display-4 fw-bold" x-text="stats.total_visits || 0">
-              0
-            </div>
-            <div class="small text-muted mt-1">
-              Total Visits
-            </div>
+            <div class="display-4 fw-bold" x-text="stats.total_visits || 0">0</div>
+            <div class="small text-muted mt-1">Total Visits</div>
           </div>
         </div>
       </div>
       <div class="col-12 col-sm-6 col-md-3 mb-3">
         <div class="card">
           <div class="card-body">
-            <div class="display-4 fw-bold" x-text="stats.active_users || 0">
-              0
-            </div>
-            <div class="small text-muted mt-1">
-              Active Users
-            </div>
+            <div class="display-4 fw-bold" x-text="stats.active_users || 0">0</div>
+            <div class="small text-muted mt-1">Active Users</div>
           </div>
         </div>
       </div>
       <div class="col-12 col-sm-6 col-md-3 mb-3">
         <div class="card">
           <div class="card-body">
-            <div class="display-4 fw-bold" x-text="stats.completed_visits || 0">
-              0
-            </div>
-            <div class="small text-muted mt-1">
-              Completed Visits
-            </div>
+            <div class="display-4 fw-bold" x-text="stats.completed_visits || 0">0</div>
+            <div class="small text-muted mt-1">Completed Visits</div>
           </div>
         </div>
       </div>
       <div class="col-12 col-sm-6 col-md-3 mb-3">
         <div class="card">
           <div class="card-body">
-            <div class="display-4 fw-bold" x-text="stats.pending_visits || 0">
-              0
-            </div>
-            <div class="small text-muted mt-1">
-              Pending Visits
-            </div>
+            <div class="display-4 fw-bold" x-text="stats.pending_visits || 0">0</div>
+            <div class="small text-muted mt-1">Pending Visits</div>
           </div>
         </div>
       </div>
@@ -109,7 +62,6 @@ <h3 class="my-2">Additional Information</h3>
 {% endblock content %}
 {% block inline_javascript %}
 {{ block.super }}
-  {{ user_visits|json_script:"userVisits" }}
   <script>
     function dashboardStats() {
       return {
@@ -121,19 +73,12 @@ <h3 class="my-2">Additional Information</h3>
         },
         async init() {
           await this.loadStats();
-
-          // Set up event listeners for filter changes
-          document.getElementById('program-select').addEventListener('change', () => this.loadStats());
-          document.getElementById('org-select').addEventListener('change', () => this.loadStats());
-          document.getElementById('date-select').addEventListener('change', () => this.loadStats());
         },
         async loadStats() {
-          const program = document.getElementById('program-select').value;
-          const organization = document.getElementById('org-select').value;
-          const date = document.getElementById('date-select').value;
-
           try {
-            const response = await fetch(`/api/dashboard-stats/?program=${program}&organization=${organization}&date=${date}`);
+            const formData = new FormData(this.$refs.filterForm);
+            const queryString = new URLSearchParams(formData).toString();
+            const response = await fetch(`{% url 'reports:dashboard_stats_api' %}?${queryString}`);
             if (!response.ok) throw new Error('Failed to load stats');
             this.stats = await response.json();
           } catch (error) {

From 3e26dec8e4433a76eb793b50052c9410ee8e1175 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 12:19:05 +0200
Subject: [PATCH 25/96] fix form submission

---
 .../templates/reports/dashboard.html          | 41 +++++++++++++++----
 1 file changed, 33 insertions(+), 8 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 96a4e1a2..f78fd00d 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -5,11 +5,19 @@
 {% block title %}Admin Dashboard{% endblock %}
 {% block content %}
   <h2 class="my-2">Program Dashboard</h2>
-  <div class="container" x-data="dashboardStats()">
-    <form class="row mb-3" id="filter-form" hx-get="{% url 'reports:dashboard_stats_api' %}"
-          hx-trigger="change from:#filter-form" hx-swap="none" x-ref="filterForm">
-      {% crispy filter.form %}
-    </form>
+  <div class="container" x-data="dashboardStats()" x-init="$nextTick(() => console.log('Alpine initialized'))">
+    <div class="mb-4">
+        <form class="row mb-4 id="filter-form" x-ref="filterForm" x-on:submit.prevent="loadStats()">
+        {% crispy filter.form %}
+        <div class="col-12 text-end">
+            <button
+            type="button"
+            class="btn btn-primary"
+            @click="loadStats()"
+            >Update Dashboard</button>
+        </div>
+        </form>
+    </div>
     <div class="row">
       <div class="col-12 col-sm-6 col-md-3 mb-3">
         <div class="card">
@@ -72,15 +80,32 @@ <h3 class="my-2">Additional Information</h3>
           pending_visits: 0
         },
         async init() {
+          console.log('init called');
           await this.loadStats();
         },
-        async loadStats() {
+        async loadStats(event) {
+          console.log('loadStats called', {
+            event,
+            form: this.$refs.filterForm,
+            formData: new FormData(this.$refs.filterForm)
+          });
+
           try {
             const formData = new FormData(this.$refs.filterForm);
             const queryString = new URLSearchParams(formData).toString();
-            const response = await fetch(`{% url 'reports:dashboard_stats_api' %}?${queryString}`);
+            console.log('queryString:', queryString);
+
+            const url = `{% url 'reports:dashboard_stats_api' %}?${queryString}`;
+            console.log('fetching from:', url);
+
+            const response = await fetch(url);
+            console.log('response:', response);
+
             if (!response.ok) throw new Error('Failed to load stats');
-            this.stats = await response.json();
+            const data = await response.json();
+            console.log('received data:', data);
+
+            this.stats = data;
           } catch (error) {
             console.error('Error loading dashboard stats:', error);
           }

From 6ac84d59130903d0ff8d088f0ba76d0ea32b586f Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 12:21:48 +0200
Subject: [PATCH 26/96] run autoformat

---
 .../templates/reports/dashboard.html          | 424 +++++++++---------
 1 file changed, 210 insertions(+), 214 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index f78fd00d..04efd3a3 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -4,253 +4,249 @@
 {% load django_tables2 %}
 {% block title %}Admin Dashboard{% endblock %}
 {% block content %}
-  <h2 class="my-2">Program Dashboard</h2>
-  <div class="container" x-data="dashboardStats()" x-init="$nextTick(() => console.log('Alpine initialized'))">
-    <div class="mb-4">
-        <form class="row mb-4 id="filter-form" x-ref="filterForm" x-on:submit.prevent="loadStats()">
-        {% crispy filter.form %}
-        <div class="col-12 text-end">
-            <button
-            type="button"
-            class="btn btn-primary"
-            @click="loadStats()"
-            >Update Dashboard</button>
-        </div>
-        </form>
-    </div>
-    <div class="row">
-      <div class="col-12 col-sm-6 col-md-3 mb-3">
-        <div class="card">
-          <div class="card-body">
-            <div class="display-4 fw-bold" x-text="stats.total_visits || 0">0</div>
-            <div class="small text-muted mt-1">Total Visits</div>
-          </div>
-        </div>
+<h2 class="my-2">Program Dashboard</h2>
+<div class="container" x-data="dashboardStats()" x-init="$nextTick(() => console.log('Alpine initialized'))">
+  <div class="mb-4">
+    <form class="row mb-4 id=" filter-form" x-ref="filterForm" x-on:submit.prevent="loadStats()">
+      {% crispy filter.form %}
+      <div class="col-12 text-end">
+        <button type="button" class="btn btn-primary" @click="loadStats()">Update Dashboard</button>
       </div>
-      <div class="col-12 col-sm-6 col-md-3 mb-3">
-        <div class="card">
-          <div class="card-body">
-            <div class="display-4 fw-bold" x-text="stats.active_users || 0">0</div>
-            <div class="small text-muted mt-1">Active Users</div>
-          </div>
+    </form>
+  </div>
+  <div class="row">
+    <div class="col-12 col-sm-6 col-md-3 mb-3">
+      <div class="card">
+        <div class="card-body">
+          <div class="display-4 fw-bold" x-text="stats.total_visits || 0">0</div>
+          <div class="small text-muted mt-1">Total Visits</div>
         </div>
       </div>
-      <div class="col-12 col-sm-6 col-md-3 mb-3">
-        <div class="card">
-          <div class="card-body">
-            <div class="display-4 fw-bold" x-text="stats.completed_visits || 0">0</div>
-            <div class="small text-muted mt-1">Completed Visits</div>
-          </div>
+    </div>
+    <div class="col-12 col-sm-6 col-md-3 mb-3">
+      <div class="card">
+        <div class="card-body">
+          <div class="display-4 fw-bold" x-text="stats.active_users || 0">0</div>
+          <div class="small text-muted mt-1">Active Users</div>
         </div>
       </div>
-      <div class="col-12 col-sm-6 col-md-3 mb-3">
-        <div class="card">
-          <div class="card-body">
-            <div class="display-4 fw-bold" x-text="stats.pending_visits || 0">0</div>
-            <div class="small text-muted mt-1">Pending Visits</div>
-          </div>
+    </div>
+    <div class="col-12 col-sm-6 col-md-3 mb-3">
+      <div class="card">
+        <div class="card-body">
+          <div class="display-4 fw-bold" x-text="stats.completed_visits || 0">0</div>
+          <div class="small text-muted mt-1">Completed Visits</div>
         </div>
       </div>
     </div>
-  </div>
-  <div class="row">
-    <div class="col-12 col-lg-8">
-      <h3 class="my-2">Service Delivery Map</h3>
-      <div id="map" class="mt-3 mb-2 rounded border" style="height: 900px;"></div>
-    </div>
-    <div class="col-12 col-lg-4">
-      <h3 class="my-2">Additional Information</h3>
+    <div class="col-12 col-sm-6 col-md-3 mb-3">
       <div class="card">
         <div class="card-body">
-          <p class="text-muted">Placeholder content</p>
+          <div class="display-4 fw-bold" x-text="stats.pending_visits || 0">0</div>
+          <div class="small text-muted mt-1">Pending Visits</div>
         </div>
       </div>
     </div>
   </div>
+</div>
+<div class="row">
+  <div class="col-12 col-lg-8">
+    <h3 class="my-2">Service Delivery Map</h3>
+    <div id="map" class="mt-3 mb-2 rounded border" style="height: 900px;"></div>
+  </div>
+  <div class="col-12 col-lg-4">
+    <h3 class="my-2">Additional Information</h3>
+    <div class="card">
+      <div class="card-body">
+        <p class="text-muted">Placeholder content</p>
+      </div>
+    </div>
+  </div>
+</div>
 {% endblock content %}
 {% block inline_javascript %}
 {{ block.super }}
-  <script>
-    function dashboardStats() {
-      return {
-        stats: {
-          total_visits: 0,
-          active_users: 0,
-          completed_visits: 0,
-          pending_visits: 0
-        },
-        async init() {
-          console.log('init called');
-          await this.loadStats();
-        },
-        async loadStats(event) {
-          console.log('loadStats called', {
-            event,
-            form: this.$refs.filterForm,
-            formData: new FormData(this.$refs.filterForm)
-          });
+<script>
+  function dashboardStats() {
+    return {
+      stats: {
+        total_visits: 0,
+        active_users: 0,
+        completed_visits: 0,
+        pending_visits: 0
+      },
+      async init() {
+        console.log('init called');
+        await this.loadStats();
+      },
+      async loadStats(event) {
+        console.log('loadStats called', {
+          event,
+          form: this.$refs.filterForm,
+          formData: new FormData(this.$refs.filterForm)
+        });
 
-          try {
-            const formData = new FormData(this.$refs.filterForm);
-            const queryString = new URLSearchParams(formData).toString();
-            console.log('queryString:', queryString);
+        try {
+          const formData = new FormData(this.$refs.filterForm);
+          const queryString = new URLSearchParams(formData).toString();
+          console.log('queryString:', queryString);
 
-            const url = `{% url 'reports:dashboard_stats_api' %}?${queryString}`;
-            console.log('fetching from:', url);
+          const url = `{% url 'reports:dashboard_stats_api' %}?${queryString}`;
+          console.log('fetching from:', url);
 
-            const response = await fetch(url);
-            console.log('response:', response);
+          const response = await fetch(url);
+          console.log('response:', response);
 
-            if (!response.ok) throw new Error('Failed to load stats');
-            const data = await response.json();
-            console.log('received data:', data);
+          if (!response.ok) throw new Error('Failed to load stats');
+          const data = await response.json();
+          console.log('received data:', data);
 
-            this.stats = data;
-          } catch (error) {
-            console.error('Error loading dashboard stats:', error);
-          }
+          this.stats = data;
+        } catch (error) {
+          console.error('Error loading dashboard stats:', error);
         }
       }
     }
-  </script>
-  <script type="module">
-    window.addEventListener('DOMContentLoaded', () => {
-      mapboxgl.accessToken = "{{ mapbox_token }}";
+  }
+</script>
+<script type="module">
+  window.addEventListener('DOMContentLoaded', () => {
+    mapboxgl.accessToken = "{{ mapbox_token }}";
     const map = new mapboxgl.Map({
-        container: 'map',
-        // Choose from Mapbox's core styles, or make your own style with Mapbox Studio
-        style: 'mapbox://styles/mapbox/dark-v11',
-        center: [-103.5917, 40.6699],
-        zoom: 3
+      container: 'map',
+      // Choose from Mapbox's core styles, or make your own style with Mapbox Studio
+      style: 'mapbox://styles/mapbox/dark-v11',
+      center: [-103.5917, 40.6699],
+      zoom: 3
     });
 
     map.on('load', () => {
-        // Add a new source from our GeoJSON data and
-        // set the 'cluster' option to true. GL-JS will
-        // add the point_count property to your source data.
-        map.addSource('visits', {
-            type: 'geojson',
-            // Point to GeoJSON data from the backend.
-            data: '{% url "reports:visit_map_data" %}',
-            cluster: true,
-            clusterMaxZoom: 14, // Max zoom to cluster points on
-            clusterRadius: 50 // Radius of each cluster when clustering points (defaults to 50)
-        });
-
-        const clusterStep1 = 50;
-        const clusterStep2 = 100;
-
-        map.addLayer({
-            id: 'clusters',
-            type: 'circle',
-            source: 'visits',
-            filter: ['has', 'point_count'],
-            paint: {
-                // Use step expressions (https://docs.mapbox.com/style-spec/reference/expressions/#step)
-                // with three steps to implement three types of circles:
-                //   * Blue, 20px circles when point count is less than clusterStep1
-                //   * Yellow, 30px circles when point count is between clusterStep1 and clusterStep2
-                //   * Green, 40px circles when point count is greater than or equal to clusterStep2
-                'circle-color': [
-                    'step',
-                    ['get', 'point_count'],
-                    '#51bbd6',
-                    clusterStep1,
-                    '#f1f075',
-                    clusterStep2,
-                    '#00FF00'
-                ],
-                'circle-radius': [
-                    'step',
-                    ['get', 'point_count'],
-                    20,
-                    clusterStep1,
-                    30,
-                    clusterStep2,
-                    40
-                ]
-            }
-        });
-
-        map.addLayer({
-            id: 'cluster-count',
-            type: 'symbol',
-            source: 'visits',
-            filter: ['has', 'point_count'],
-            layout: {
-                'text-field': ['get', 'point_count_abbreviated'],
-                'text-font': ['DIN Offc Pro Medium', 'Arial Unicode MS Bold'],
-                'text-size': 12
-            }
-        });
+      // Add a new source from our GeoJSON data and
+      // set the 'cluster' option to true. GL-JS will
+      // add the point_count property to your source data.
+      map.addSource('visits', {
+        type: 'geojson',
+        // Point to GeoJSON data from the backend.
+        data: '{% url "reports:visit_map_data" %}',
+        cluster: true,
+        clusterMaxZoom: 14, // Max zoom to cluster points on
+        clusterRadius: 50 // Radius of each cluster when clustering points (defaults to 50)
+      });
+
+      const clusterStep1 = 50;
+      const clusterStep2 = 100;
+
+      map.addLayer({
+        id: 'clusters',
+        type: 'circle',
+        source: 'visits',
+        filter: ['has', 'point_count'],
+        paint: {
+          // Use step expressions (https://docs.mapbox.com/style-spec/reference/expressions/#step)
+          // with three steps to implement three types of circles:
+          //   * Blue, 20px circles when point count is less than clusterStep1
+          //   * Yellow, 30px circles when point count is between clusterStep1 and clusterStep2
+          //   * Green, 40px circles when point count is greater than or equal to clusterStep2
+          'circle-color': [
+            'step',
+            ['get', 'point_count'],
+            '#51bbd6',
+            clusterStep1,
+            '#f1f075',
+            clusterStep2,
+            '#00FF00'
+          ],
+          'circle-radius': [
+            'step',
+            ['get', 'point_count'],
+            20,
+            clusterStep1,
+            30,
+            clusterStep2,
+            40
+          ]
+        }
+      });
+
+      map.addLayer({
+        id: 'cluster-count',
+        type: 'symbol',
+        source: 'visits',
+        filter: ['has', 'point_count'],
+        layout: {
+          'text-field': ['get', 'point_count_abbreviated'],
+          'text-font': ['DIN Offc Pro Medium', 'Arial Unicode MS Bold'],
+          'text-size': 12
+        }
+      });
+
+      map.addLayer({
+        id: 'unclustered-point',
+        type: 'circle',
+        source: 'visits',
+        filter: ['!', ['has', 'point_count']],
+        paint: {
+          'circle-color': ['get', 'color'],
+          'circle-radius': 4,
+          'circle-stroke-width': 1,
+          'circle-stroke-color': '#fff'
+        }
+      });
 
-        map.addLayer({
-            id: 'unclustered-point',
-            type: 'circle',
-            source: 'visits',
-            filter: ['!', ['has', 'point_count']],
-            paint: {
-                'circle-color': ['get', 'color'],
-                'circle-radius': 4,
-                'circle-stroke-width': 1,
-                'circle-stroke-color': '#fff'
-            }
+      // inspect a cluster on click
+      map.on('click', 'clusters', (e) => {
+        const features = map.queryRenderedFeatures(e.point, {
+          layers: ['clusters']
         });
-
-        // inspect a cluster on click
-        map.on('click', 'clusters', (e) => {
-            const features = map.queryRenderedFeatures(e.point, {
-                layers: ['clusters']
+        const clusterId = features[0].properties.cluster_id;
+        map.getSource('visits').getClusterExpansionZoom(
+          clusterId,
+          (err, zoom) => {
+            if (err) return;
+
+            map.easeTo({
+              center: features[0].geometry.coordinates,
+              zoom: zoom
             });
-            const clusterId = features[0].properties.cluster_id;
-            map.getSource('visits').getClusterExpansionZoom(
-                clusterId,
-                (err, zoom) => {
-                    if (err) return;
-
-                    map.easeTo({
-                        center: features[0].geometry.coordinates,
-                        zoom: zoom
-                    });
-                }
-            );
-        });
-
-        // When a click event occurs on a feature in
-        // the unclustered-point layer, open a popup at
-        // the location of the feature, with
-        // description HTML from its properties.
-        map.on('click', 'unclustered-point', (e) => {
-            const coordinates = e.features[0].geometry.coordinates.slice();
-            const status = e.features[0].properties.status;
-            const rawDate = e.features[0].properties.visit_date;
-            const visitDate = new Date(rawDate).toLocaleDateString('en-US', { year: 'numeric', month: 'short', day: 'numeric' });
-
-
-            // Ensure that if the map is zoomed out such that
-            // multiple copies of the feature are visible, the
-            // popup appears over the copy being pointed to.
-            if (['mercator', 'equirectangular'].includes(map.getProjection().name)) {
-                while (Math.abs(e.lngLat.lng - coordinates[0]) > 180) {
-                    coordinates[0] += e.lngLat.lng > coordinates[0] ? 360 : -360;
-                }
-            }
-
-            new mapboxgl.Popup()
-                .setLngLat(coordinates)
-                .setHTML(
-                    `Visit Date: ${visitDate}<br>Status: ${status}`
-                )
-                .addTo(map);
-        });
+          }
+        );
+      });
+
+      // When a click event occurs on a feature in
+      // the unclustered-point layer, open a popup at
+      // the location of the feature, with
+      // description HTML from its properties.
+      map.on('click', 'unclustered-point', (e) => {
+        const coordinates = e.features[0].geometry.coordinates.slice();
+        const status = e.features[0].properties.status;
+        const rawDate = e.features[0].properties.visit_date;
+        const visitDate = new Date(rawDate).toLocaleDateString('en-US', { year: 'numeric', month: 'short', day: 'numeric' });
+
+
+        // Ensure that if the map is zoomed out such that
+        // multiple copies of the feature are visible, the
+        // popup appears over the copy being pointed to.
+        if (['mercator', 'equirectangular'].includes(map.getProjection().name)) {
+          while (Math.abs(e.lngLat.lng - coordinates[0]) > 180) {
+            coordinates[0] += e.lngLat.lng > coordinates[0] ? 360 : -360;
+          }
+        }
 
-        map.on('mouseenter', 'clusters', () => {
-            map.getCanvas().style.cursor = 'pointer';
-        });
-        map.on('mouseleave', 'clusters', () => {
-            map.getCanvas().style.cursor = '';
-        });
+        new mapboxgl.Popup()
+          .setLngLat(coordinates)
+          .setHTML(
+            `Visit Date: ${visitDate}<br>Status: ${status}`
+          )
+          .addTo(map);
+      });
+
+      map.on('mouseenter', 'clusters', () => {
+        map.getCanvas().style.cursor = 'pointer';
+      });
+      map.on('mouseleave', 'clusters', () => {
+        map.getCanvas().style.cursor = '';
+      });
     });
   });
-  </script>
+</script>
 {% endblock %}

From 2009b19113cbc38a6c131e689454cfe60519fd9f Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 12:27:16 +0200
Subject: [PATCH 27/96] fix html syntax

---
 commcare_connect/templates/reports/dashboard.html | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 04efd3a3..f4b66294 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -7,12 +7,12 @@
 <h2 class="my-2">Program Dashboard</h2>
 <div class="container" x-data="dashboardStats()" x-init="$nextTick(() => console.log('Alpine initialized'))">
   <div class="mb-4">
-    <form class="row mb-4 id=" filter-form" x-ref="filterForm" x-on:submit.prevent="loadStats()">
+    <div class="row mb-4" x-ref="filterForm">
       {% crispy filter.form %}
       <div class="col-12 text-end">
         <button type="button" class="btn btn-primary" @click="loadStats()">Update Dashboard</button>
       </div>
-    </form>
+    </div>
   </div>
   <div class="row">
     <div class="col-12 col-sm-6 col-md-3 mb-3">
@@ -80,14 +80,9 @@ <h3 class="my-2">Additional Information</h3>
         await this.loadStats();
       },
       async loadStats(event) {
-        console.log('loadStats called', {
-          event,
-          form: this.$refs.filterForm,
-          formData: new FormData(this.$refs.filterForm)
-        });
-
         try {
-          const formData = new FormData(this.$refs.filterForm);
+          const formElement = this.$refs.filterForm.querySelector('form');
+          const formData = new FormData(formElement);
           const queryString = new URLSearchParams(formData).toString();
           console.log('queryString:', queryString);
 

From cb4e471303734766d76524c2a961fa6a41bc2832 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 12:29:08 +0200
Subject: [PATCH 28/96] remove submit button entirely

---
 commcare_connect/templates/reports/dashboard.html | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index f4b66294..59e4ef7a 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -9,9 +9,6 @@ <h2 class="my-2">Program Dashboard</h2>
   <div class="mb-4">
     <div class="row mb-4" x-ref="filterForm">
       {% crispy filter.form %}
-      <div class="col-12 text-end">
-        <button type="button" class="btn btn-primary" @click="loadStats()">Update Dashboard</button>
-      </div>
     </div>
   </div>
   <div class="row">
@@ -78,6 +75,11 @@ <h3 class="my-2">Additional Information</h3>
       async init() {
         console.log('init called');
         await this.loadStats();
+
+        const formElement = this.$refs.filterForm.querySelector('form');
+        formElement.querySelectorAll('select, input').forEach(input => {
+          input.addEventListener('change', () => this.loadStats());
+        });
       },
       async loadStats(event) {
         try {

From 914f95e5d51defb9c3cc5bf573f639190003a4de Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 13:36:51 +0200
Subject: [PATCH 29/96] spinners!

---
 .../templates/reports/dashboard.html          | 36 ++++++++++++++++---
 1 file changed, 32 insertions(+), 4 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 59e4ef7a..0050bfea 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -15,7 +15,13 @@ <h2 class="my-2">Program Dashboard</h2>
     <div class="col-12 col-sm-6 col-md-3 mb-3">
       <div class="card">
         <div class="card-body">
-          <div class="display-4 fw-bold" x-text="stats.total_visits || 0">0</div>
+          <div class="fw-bold">
+            <span x-show="!isLoading" x-text="stats.total_visits || 0" class="display-4">0</span>
+            <span x-show="isLoading">
+              <span class="spinner-border text-secondary opacity-50" role="status"></span>
+              <span class="display-4">&nbsp;</span>
+            </span>
+          </div>
           <div class="small text-muted mt-1">Total Visits</div>
         </div>
       </div>
@@ -23,7 +29,13 @@ <h2 class="my-2">Program Dashboard</h2>
     <div class="col-12 col-sm-6 col-md-3 mb-3">
       <div class="card">
         <div class="card-body">
-          <div class="display-4 fw-bold" x-text="stats.active_users || 0">0</div>
+          <div class="fw-bold">
+            <span x-show="!isLoading" x-text="stats.active_users || 0" class="display-4">0</span>
+            <span x-show="isLoading">
+              <span class="spinner-border text-secondary opacity-50" role="status"></span>
+              <span class="display-4">&nbsp;</span>
+            </span>
+          </div>
           <div class="small text-muted mt-1">Active Users</div>
         </div>
       </div>
@@ -31,7 +43,13 @@ <h2 class="my-2">Program Dashboard</h2>
     <div class="col-12 col-sm-6 col-md-3 mb-3">
       <div class="card">
         <div class="card-body">
-          <div class="display-4 fw-bold" x-text="stats.completed_visits || 0">0</div>
+          <div class="fw-bold">
+            <span x-show="!isLoading" x-text="stats.completed_visits || 0" class="display-4">0</span>
+            <span x-show="isLoading">
+              <span class="spinner-border text-secondary opacity-50" role="status"></span>
+              <span class="display-4">&nbsp;</span>
+            </span>
+          </div>
           <div class="small text-muted mt-1">Completed Visits</div>
         </div>
       </div>
@@ -39,7 +57,13 @@ <h2 class="my-2">Program Dashboard</h2>
     <div class="col-12 col-sm-6 col-md-3 mb-3">
       <div class="card">
         <div class="card-body">
-          <div class="display-4 fw-bold" x-text="stats.pending_visits || 0">0</div>
+          <div class="fw-bold">
+            <span x-show="!isLoading" x-text="stats.pending_visits || 0" class="display-4">0</span>
+            <span x-show="isLoading">
+              <span class="spinner-border text-secondary opacity-50" role="status"></span>
+              <span class="display-4">&nbsp;</span>
+            </span>
+          </div>
           <div class="small text-muted mt-1">Pending Visits</div>
         </div>
       </div>
@@ -72,6 +96,7 @@ <h3 class="my-2">Additional Information</h3>
         completed_visits: 0,
         pending_visits: 0
       },
+      isLoading: false,
       async init() {
         console.log('init called');
         await this.loadStats();
@@ -83,6 +108,7 @@ <h3 class="my-2">Additional Information</h3>
       },
       async loadStats(event) {
         try {
+          this.isLoading = true;
           const formElement = this.$refs.filterForm.querySelector('form');
           const formData = new FormData(formElement);
           const queryString = new URLSearchParams(formData).toString();
@@ -101,6 +127,8 @@ <h3 class="my-2">Additional Information</h3>
           this.stats = data;
         } catch (error) {
           console.error('Error loading dashboard stats:', error);
+        } finally {
+          this.isLoading = false;
         }
       }
     }

From aa9da02536fc4f3cc9f4b46574f118ed5b793b55 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 13:50:42 +0200
Subject: [PATCH 30/96] improve filters

---
 commcare_connect/reports/views.py | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index 75d490b5..2ed9f971 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -157,16 +157,28 @@ def _get_table_data_for_quarter(quarter, delivery_type, group_by_delivery_type=F
 class DashboardFilters(django_filters.FilterSet):
     opportunity__delivery_type = django_filters.ModelChoiceFilter(
         queryset=DeliveryType.objects.all(),
+        label="Program",
         empty_label="All Programs",
         required=False,
     )
     opportunity__organization = django_filters.ModelChoiceFilter(
         queryset=Organization.objects.all(),
+        label="Organization",
         empty_label="All Organizations",
         required=False,
     )
-    visit_date = django_filters.DateFilter(
+    from_date = django_filters.DateTimeFilter(
         widget=forms.DateInput(attrs={"type": "date"}),
+        field_name="visit_date",
+        lookup_expr="gt",
+        label="From Date",
+        required=False,
+    )
+    to_date = django_filters.DateTimeFilter(
+        widget=forms.DateInput(attrs={"type": "date"}),
+        field_name="visit_date",
+        lookup_expr="lte",
+        label="To Date",
         required=False,
     )
 
@@ -176,15 +188,16 @@ def __init__(self, *args, **kwargs):
         self.form.helper.form_class = "form-inline"
         self.form.helper.layout = Layout(
             Row(
-                Column("opportunity__delivery_type", css_class="col-md-4"),
-                Column("opportunity__organization", css_class="col-md-4"),
-                Column("visit_date", css_class="col-md-4"),
+                Column("opportunity__delivery_type", css_class="col-md-3"),
+                Column("opportunity__organization", css_class="col-md-3"),
+                Column("from_date", css_class="col-md-3"),
+                Column("to_date", css_class="col-md-3"),
             )
         )
 
     class Meta:
         model = UserVisit
-        fields = ["opportunity__delivery_type", "opportunity__organization", "visit_date"]
+        fields = ["opportunity__delivery_type", "opportunity__organization", "from_date", "to_date"]
 
 
 @login_required
@@ -367,6 +380,7 @@ def dashboard_stats_api(request):
     # Use the filtered queryset to calculate stats
     queryset = UserVisit.objects.all()
     if filterset.is_valid():
+        print(filterset.form.cleaned_data)
         queryset = filterset.filter_queryset(queryset)
 
     # Example stats calculation (adjust based on your needs)

From bad9331be97a21f522c24af9326fcb09fc503ae0 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 13:50:47 +0200
Subject: [PATCH 31/96] tweak order

---
 commcare_connect/templates/reports/dashboard.html | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 0050bfea..4ec15842 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -16,13 +16,13 @@ <h2 class="my-2">Program Dashboard</h2>
       <div class="card">
         <div class="card-body">
           <div class="fw-bold">
-            <span x-show="!isLoading" x-text="stats.total_visits || 0" class="display-4">0</span>
+            <span x-show="!isLoading" x-text="stats.active_users || 0" class="display-4">0</span>
             <span x-show="isLoading">
               <span class="spinner-border text-secondary opacity-50" role="status"></span>
               <span class="display-4">&nbsp;</span>
             </span>
           </div>
-          <div class="small text-muted mt-1">Total Visits</div>
+          <div class="small text-muted mt-1">Active Front-Line Workers</div>
         </div>
       </div>
     </div>
@@ -30,13 +30,13 @@ <h2 class="my-2">Program Dashboard</h2>
       <div class="card">
         <div class="card-body">
           <div class="fw-bold">
-            <span x-show="!isLoading" x-text="stats.active_users || 0" class="display-4">0</span>
+            <span x-show="!isLoading" x-text="stats.total_visits || 0" class="display-4">0</span>
             <span x-show="isLoading">
               <span class="spinner-border text-secondary opacity-50" role="status"></span>
               <span class="display-4">&nbsp;</span>
             </span>
           </div>
-          <div class="small text-muted mt-1">Active Users</div>
+          <div class="small text-muted mt-1">Total Visits</div>
         </div>
       </div>
     </div>

From d369f2f4607655964e95bdc7cc517cfeafa38116 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 13:52:11 +0200
Subject: [PATCH 32/96] fix other filters

---
 commcare_connect/reports/views.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index 2ed9f971..0f7cd7f8 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -155,14 +155,16 @@ def _get_table_data_for_quarter(quarter, delivery_type, group_by_delivery_type=F
 
 
 class DashboardFilters(django_filters.FilterSet):
-    opportunity__delivery_type = django_filters.ModelChoiceFilter(
+    program = django_filters.ModelChoiceFilter(
         queryset=DeliveryType.objects.all(),
+        field_name="opportunity__delivery_type",
         label="Program",
         empty_label="All Programs",
         required=False,
     )
-    opportunity__organization = django_filters.ModelChoiceFilter(
+    organization = django_filters.ModelChoiceFilter(
         queryset=Organization.objects.all(),
+        field_name="opportunity__organization",
         label="Organization",
         empty_label="All Organizations",
         required=False,
@@ -188,8 +190,8 @@ def __init__(self, *args, **kwargs):
         self.form.helper.form_class = "form-inline"
         self.form.helper.layout = Layout(
             Row(
-                Column("opportunity__delivery_type", css_class="col-md-3"),
-                Column("opportunity__organization", css_class="col-md-3"),
+                Column("program", css_class="col-md-3"),
+                Column("organization", css_class="col-md-3"),
                 Column("from_date", css_class="col-md-3"),
                 Column("to_date", css_class="col-md-3"),
             )
@@ -197,7 +199,7 @@ def __init__(self, *args, **kwargs):
 
     class Meta:
         model = UserVisit
-        fields = ["opportunity__delivery_type", "opportunity__organization", "from_date", "to_date"]
+        fields = ["program", "organization", "from_date", "to_date"]
 
 
 @login_required

From 75f7278e46ec97e536b3249b34382e3db538e78d Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 13:53:45 +0200
Subject: [PATCH 33/96] remove console statements

---
 commcare_connect/templates/reports/dashboard.html | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 4ec15842..7b3d63a1 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -5,7 +5,7 @@
 {% block title %}Admin Dashboard{% endblock %}
 {% block content %}
 <h2 class="my-2">Program Dashboard</h2>
-<div class="container" x-data="dashboardStats()" x-init="$nextTick(() => console.log('Alpine initialized'))">
+<div class="container" x-data="dashboardStats()" >
   <div class="mb-4">
     <div class="row mb-4" x-ref="filterForm">
       {% crispy filter.form %}
@@ -98,7 +98,6 @@ <h3 class="my-2">Additional Information</h3>
       },
       isLoading: false,
       async init() {
-        console.log('init called');
         await this.loadStats();
 
         const formElement = this.$refs.filterForm.querySelector('form');
@@ -112,18 +111,10 @@ <h3 class="my-2">Additional Information</h3>
           const formElement = this.$refs.filterForm.querySelector('form');
           const formData = new FormData(formElement);
           const queryString = new URLSearchParams(formData).toString();
-          console.log('queryString:', queryString);
-
           const url = `{% url 'reports:dashboard_stats_api' %}?${queryString}`;
-          console.log('fetching from:', url);
-
           const response = await fetch(url);
-          console.log('response:', response);
-
           if (!response.ok) throw new Error('Failed to load stats');
           const data = await response.json();
-          console.log('received data:', data);
-
           this.stats = data;
         } catch (error) {
           console.error('Error loading dashboard stats:', error);

From 24505a3265c34922c0399659d9346b7b358e3bcb Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 14:10:18 +0200
Subject: [PATCH 34/96] wip: ai generated query to mimic the sql that was
 previously running

---
 commcare_connect/reports/queries.py | 58 +++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100644 commcare_connect/reports/queries.py

diff --git a/commcare_connect/reports/queries.py b/commcare_connect/reports/queries.py
new file mode 100644
index 00000000..5afbf2a9
--- /dev/null
+++ b/commcare_connect/reports/queries.py
@@ -0,0 +1,58 @@
+from django.db.models import Case, DateTimeField, ExpressionWrapper, F, FloatField, Value, When
+from django.db.models.fields.json import KeyTextTransform
+from django.db.models.functions import Cast, Extract
+
+from commcare_connect.opportunity.models import UserVisit
+
+
+def get_visit_map_queryset_base():
+    return (
+        UserVisit.objects.annotate(
+            username_connectid=F("form_json__metadata__username"),
+            deliver_unit_name=F("deliver_unit__name"),
+            days_since_opp_start=ExpressionWrapper(
+                Extract(F("visit_date") - F("opportunity__start_date"), "day"), output_field=FloatField()
+            ),
+            timestart_str=KeyTextTransform("timeStart", KeyTextTransform("metadata", F("form_json"))),
+            timeend_str=KeyTextTransform("timeEnd", KeyTextTransform("metadata", F("form_json"))),
+            visit_duration=ExpressionWrapper(
+                Extract(Cast("timeend_str", DateTimeField()) - Cast("timestart_str", DateTimeField()), "epoch") / 60,
+                output_field=FloatField(),
+            ),
+            gps_location_lat=Case(
+                When(
+                    form_json__metadata__location__isnull=False,
+                    then=ExpressionWrapper(F("form_json__metadata__location__0"), output_field=FloatField()),
+                ),
+                default=Value(None),
+                output_field=FloatField(),
+            ),
+            gps_location_long=Case(
+                When(
+                    form_json__metadata__location__isnull=False,
+                    then=ExpressionWrapper(F("form_json__metadata__location__1"), output_field=FloatField()),
+                ),
+                default=Value(None),
+                output_field=FloatField(),
+            ),
+        )
+        .select_related("deliver_unit", "opportunity")
+        .values(
+            "opportunity_id",
+            "xform_id",
+            "visit_date",
+            "username_connectid",
+            "deliver_unit_name",
+            "days_since_opp_start",
+            "entity_id",
+            "status",
+            "flagged",
+            "flag_reason",
+            "reason",
+            "timestart_str",
+            "timeend_str",
+            "visit_duration",
+            "gps_location_lat",
+            "gps_location_long",
+        )
+    )

From b13744476be643553678c0905c66330f92388d34 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 14:29:00 +0200
Subject: [PATCH 35/96] simplify lat/lon lookups

---
 commcare_connect/reports/queries.py | 22 +++-------------------
 1 file changed, 3 insertions(+), 19 deletions(-)

diff --git a/commcare_connect/reports/queries.py b/commcare_connect/reports/queries.py
index 5afbf2a9..ac9e83cf 100644
--- a/commcare_connect/reports/queries.py
+++ b/commcare_connect/reports/queries.py
@@ -1,4 +1,4 @@
-from django.db.models import Case, DateTimeField, ExpressionWrapper, F, FloatField, Value, When
+from django.db.models import DateTimeField, ExpressionWrapper, F, FloatField
 from django.db.models.fields.json import KeyTextTransform
 from django.db.models.functions import Cast, Extract
 
@@ -19,22 +19,7 @@ def get_visit_map_queryset_base():
                 Extract(Cast("timeend_str", DateTimeField()) - Cast("timestart_str", DateTimeField()), "epoch") / 60,
                 output_field=FloatField(),
             ),
-            gps_location_lat=Case(
-                When(
-                    form_json__metadata__location__isnull=False,
-                    then=ExpressionWrapper(F("form_json__metadata__location__0"), output_field=FloatField()),
-                ),
-                default=Value(None),
-                output_field=FloatField(),
-            ),
-            gps_location_long=Case(
-                When(
-                    form_json__metadata__location__isnull=False,
-                    then=ExpressionWrapper(F("form_json__metadata__location__1"), output_field=FloatField()),
-                ),
-                default=Value(None),
-                output_field=FloatField(),
-            ),
+            location_str=KeyTextTransform("location", KeyTextTransform("metadata", F("form_json"))),
         )
         .select_related("deliver_unit", "opportunity")
         .values(
@@ -52,7 +37,6 @@ def get_visit_map_queryset_base():
             "timestart_str",
             "timeend_str",
             "visit_duration",
-            "gps_location_lat",
-            "gps_location_long",
+            "location_str",
         )
     )

From c128beb11a50a4daf4d10fd13b917787014ce18e Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 14:32:00 +0200
Subject: [PATCH 36/96] pass filters to the map

---
 commcare_connect/reports/queries.py           |  6 +--
 commcare_connect/reports/views.py             | 42 +++++++++++--------
 .../templates/reports/dashboard.html          | 39 ++++++++++++-----
 3 files changed, 54 insertions(+), 33 deletions(-)

diff --git a/commcare_connect/reports/queries.py b/commcare_connect/reports/queries.py
index ac9e83cf..4b7d2d14 100644
--- a/commcare_connect/reports/queries.py
+++ b/commcare_connect/reports/queries.py
@@ -2,12 +2,10 @@
 from django.db.models.fields.json import KeyTextTransform
 from django.db.models.functions import Cast, Extract
 
-from commcare_connect.opportunity.models import UserVisit
 
-
-def get_visit_map_queryset_base():
+def get_visit_map_queryset(base_queryset):
     return (
-        UserVisit.objects.annotate(
+        base_queryset.annotate(
             username_connectid=F("form_json__metadata__username"),
             deliver_unit_name=F("deliver_unit__name"),
             days_since_opp_start=ExpressionWrapper(
diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index 0f7cd7f8..357fccd8 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -8,7 +8,6 @@
 from django.conf import settings
 from django.contrib.auth.decorators import login_required, user_passes_test
 from django.contrib.auth.mixins import LoginRequiredMixin, UserPassesTestMixin
-from django.db import connection
 from django.db.models import Max, Q, Sum
 from django.http import JsonResponse
 from django.shortcuts import render
@@ -20,6 +19,7 @@
 from commcare_connect.cache import quickcache
 from commcare_connect.opportunity.models import CompletedWork, CompletedWorkStatus, DeliveryType, Payment, UserVisit
 from commcare_connect.organization.models import Organization
+from commcare_connect.reports.queries import get_visit_map_queryset
 
 from .tables import AdminReportTable
 
@@ -221,20 +221,19 @@ def program_dashboard_report(request):
 @user_passes_test(lambda user: user.is_superuser)
 @require_GET
 def visit_map_data(request):
-    with connection.cursor() as cursor:
-        # Read the SQL file
-        with open("commcare_connect/reports/sql/visit_map.sql") as sql_file:
-            sql_query = sql_file.read()
+    filterset = DashboardFilters(request.GET)
+
+    # Use the filtered queryset to calculate stats
 
-        # Execute the query
-        cursor.execute(sql_query)
+    queryset = UserVisit.objects.all()
+    if filterset.is_valid():
+        print(filterset.form.cleaned_data)
+        queryset = filterset.filter_queryset(queryset)
 
-        # Fetch all results
-        columns = [col[0] for col in cursor.description]
-        results = [dict(zip(columns, row)) for row in cursor.fetchall()]
+    queryset = get_visit_map_queryset(queryset)
 
     # Convert to GeoJSON
-    geojson = _results_to_geojson(results)
+    geojson = _results_to_geojson(queryset)
 
     # Return the GeoJSON as JSON response
     return JsonResponse(geojson, safe=False)
@@ -246,14 +245,21 @@ def _results_to_geojson(results):
         "approved": "#00FF00",
         "rejected": "#FF0000",
     }
-    for result in results:
+    print("calling _results_to_geojson")
+    for i, result in enumerate(results.all()):
+        location_str = result.get("location_str")
         # Check if both latitude and longitude are not None and can be converted to float
-        if result.get("gps_location_long") and result.get("gps_location_lat"):
-            try:
-                longitude = float(result["gps_location_long"])
-                latitude = float(result["gps_location_lat"])
-            except ValueError:
-                # Skip this result if conversion to float fails
+        if location_str:
+            split_location = location_str.split(" ")
+            if len(split_location) >= 2:
+                try:
+                    longitude = float(split_location[1])
+                    latitude = float(split_location[0])
+                except ValueError:
+                    # Skip this result if conversion to float fails
+                    continue
+            else:
+                # Or if the location string is not in the expected format
                 continue
 
             feature = {
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 7b3d63a1..e26744e9 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -7,7 +7,7 @@
 <h2 class="my-2">Program Dashboard</h2>
 <div class="container" x-data="dashboardStats()" >
   <div class="mb-4">
-    <div class="row mb-4" x-ref="filterForm">
+    <div class="row mb-4" x-ref="filterForm" id="filterForm">
       {% crispy filter.form %}
     </div>
   </div>
@@ -99,10 +99,12 @@ <h3 class="my-2">Additional Information</h3>
       isLoading: false,
       async init() {
         await this.loadStats();
-
         const formElement = this.$refs.filterForm.querySelector('form');
         formElement.querySelectorAll('select, input').forEach(input => {
-          input.addEventListener('change', () => this.loadStats());
+          input.addEventListener('change', () => {
+            this.loadStats();
+            window.refreshMapData();
+          });
         });
       },
       async loadStats(event) {
@@ -126,9 +128,23 @@ <h3 class="my-2">Additional Information</h3>
   }
 </script>
 <script type="module">
+  let map;
+
+  // Add this function to handle map data refresh
+  window.refreshMapData = () => {
+    if (!map) return;
+
+    const formElement = document.querySelector('#filterForm form');
+    const formData = new FormData(formElement);
+    const queryString = new URLSearchParams(formData).toString();
+
+    // Update the source data with new filters
+    map.getSource('visits').setData(`{% url "reports:visit_map_data" %}?${queryString}`);
+  };
+
   window.addEventListener('DOMContentLoaded', () => {
     mapboxgl.accessToken = "{{ mapbox_token }}";
-    const map = new mapboxgl.Map({
+    map = new mapboxgl.Map({
       container: 'map',
       // Choose from Mapbox's core styles, or make your own style with Mapbox Studio
       style: 'mapbox://styles/mapbox/dark-v11',
@@ -137,16 +153,17 @@ <h3 class="my-2">Additional Information</h3>
     });
 
     map.on('load', () => {
-      // Add a new source from our GeoJSON data and
-      // set the 'cluster' option to true. GL-JS will
-      // add the point_count property to your source data.
+      // Modify the source configuration to include initial filters
+      const formElement = document.querySelector('#filterForm form');
+      const formData = new FormData(formElement);
+      const queryString = new URLSearchParams(formData).toString();
+
       map.addSource('visits', {
         type: 'geojson',
-        // Point to GeoJSON data from the backend.
-        data: '{% url "reports:visit_map_data" %}',
+        data: `{% url "reports:visit_map_data" %}?${queryString}`,
         cluster: true,
-        clusterMaxZoom: 14, // Max zoom to cluster points on
-        clusterRadius: 50 // Radius of each cluster when clustering points (defaults to 50)
+        clusterMaxZoom: 14,
+        clusterRadius: 50
       });
 
       const clusterStep1 = 50;

From 6518876143555347b27b3e8800289187f3051a64 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 14:36:33 +0200
Subject: [PATCH 37/96] completed --> verified

---
 commcare_connect/reports/views.py                 | 8 ++++----
 commcare_connect/templates/reports/dashboard.html | 6 +++---
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index 357fccd8..fb1e0279 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -392,16 +392,16 @@ def dashboard_stats_api(request):
         queryset = filterset.filter_queryset(queryset)
 
     # Example stats calculation (adjust based on your needs)
-    total_visits = queryset.count()
-    completed_visits = queryset.filter(status=CompletedWorkStatus.approved).count()
-    pending_visits = total_visits - completed_visits
     active_users = queryset.values("opportunity_access__user").distinct().count()
+    total_visits = queryset.count()
+    verified_visits = queryset.filter(status=CompletedWorkStatus.approved).count()
+    pending_visits = total_visits - verified_visits
 
     return JsonResponse(
         {
             "total_visits": total_visits,
             "active_users": active_users,
-            "completed_visits": completed_visits,
+            "verified_visits": verified_visits,
             "pending_visits": pending_visits,
         }
     )
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index e26744e9..d7b6ca5c 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -44,13 +44,13 @@ <h2 class="my-2">Program Dashboard</h2>
       <div class="card">
         <div class="card-body">
           <div class="fw-bold">
-            <span x-show="!isLoading" x-text="stats.completed_visits || 0" class="display-4">0</span>
+            <span x-show="!isLoading" x-text="stats.verified_visits || 0" class="display-4">0</span>
             <span x-show="isLoading">
               <span class="spinner-border text-secondary opacity-50" role="status"></span>
               <span class="display-4">&nbsp;</span>
             </span>
           </div>
-          <div class="small text-muted mt-1">Completed Visits</div>
+          <div class="small text-muted mt-1">Verified Visits</div>
         </div>
       </div>
     </div>
@@ -93,7 +93,7 @@ <h3 class="my-2">Additional Information</h3>
       stats: {
         total_visits: 0,
         active_users: 0,
-        completed_visits: 0,
+        verified_visits: 0,
         pending_visits: 0
       },
       isLoading: false,

From 2c7ed1245035c16f78bd0c50fe8768e79e8607c8 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 14:39:58 +0200
Subject: [PATCH 38/96] switch to percent

---
 commcare_connect/reports/views.py                 | 4 ++--
 commcare_connect/templates/reports/dashboard.html | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index fb1e0279..c6871efa 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -395,13 +395,13 @@ def dashboard_stats_api(request):
     active_users = queryset.values("opportunity_access__user").distinct().count()
     total_visits = queryset.count()
     verified_visits = queryset.filter(status=CompletedWorkStatus.approved).count()
-    pending_visits = total_visits - verified_visits
+    percent_verified = round(float(verified_visits / total_visits) * 100, 1) if total_visits > 0 else 0
 
     return JsonResponse(
         {
             "total_visits": total_visits,
             "active_users": active_users,
             "verified_visits": verified_visits,
-            "pending_visits": pending_visits,
+            "percent_verified": f"{percent_verified:.1f}%",
         }
     )
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index d7b6ca5c..12c459c0 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -22,7 +22,7 @@ <h2 class="my-2">Program Dashboard</h2>
               <span class="display-4">&nbsp;</span>
             </span>
           </div>
-          <div class="small text-muted mt-1">Active Front-Line Workers</div>
+          <div class="small text-muted mt-1">Active FLWs</div>
         </div>
       </div>
     </div>
@@ -58,13 +58,13 @@ <h2 class="my-2">Program Dashboard</h2>
       <div class="card">
         <div class="card-body">
           <div class="fw-bold">
-            <span x-show="!isLoading" x-text="stats.pending_visits || 0" class="display-4">0</span>
+            <span x-show="!isLoading" x-text="stats.percent_verified || '0.0%'" class="display-4">0</span>
             <span x-show="isLoading">
               <span class="spinner-border text-secondary opacity-50" role="status"></span>
               <span class="display-4">&nbsp;</span>
             </span>
           </div>
-          <div class="small text-muted mt-1">Pending Visits</div>
+          <div class="small text-muted mt-1">Percent Verified</div>
         </div>
       </div>
     </div>
@@ -94,7 +94,7 @@ <h3 class="my-2">Additional Information</h3>
         total_visits: 0,
         active_users: 0,
         verified_visits: 0,
-        pending_visits: 0
+        percent_verified: "0.0%"
       },
       isLoading: false,
       async init() {

From 1cea238e2168736c0c50df094158aa0531d83302 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 14:50:24 +0200
Subject: [PATCH 39/96] add default dates (last 90 days)

---
 commcare_connect/reports/views.py             | 22 +++++++++++++++----
 .../templates/reports/dashboard.html          |  3 +++
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index c6871efa..c91e8789 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -1,4 +1,4 @@
-from datetime import date, datetime
+from datetime import date, datetime, timedelta
 
 import django_filters
 import django_tables2 as tables
@@ -197,6 +197,23 @@ def __init__(self, *args, **kwargs):
             )
         )
 
+        # Set default values if no data is provided
+        if not self.data:
+            # Create a mutable copy of the QueryDict
+            self.data = self.data.copy() if self.data else {}
+
+            # Set default dates
+            today = date.today()
+            default_from = today - timedelta(days=90)
+
+            # Set the default values
+            self.data["to_date"] = today.strftime("%Y-%m-%d")
+            self.data["from_date"] = default_from.strftime("%Y-%m-%d")
+
+            # Force the form to bind with the default data
+            self.form.is_bound = True
+            self.form.data = self.data
+
     class Meta:
         model = UserVisit
         fields = ["program", "organization", "from_date", "to_date"]
@@ -227,7 +244,6 @@ def visit_map_data(request):
 
     queryset = UserVisit.objects.all()
     if filterset.is_valid():
-        print(filterset.form.cleaned_data)
         queryset = filterset.filter_queryset(queryset)
 
     queryset = get_visit_map_queryset(queryset)
@@ -245,7 +261,6 @@ def _results_to_geojson(results):
         "approved": "#00FF00",
         "rejected": "#FF0000",
     }
-    print("calling _results_to_geojson")
     for i, result in enumerate(results.all()):
         location_str = result.get("location_str")
         # Check if both latitude and longitude are not None and can be converted to float
@@ -388,7 +403,6 @@ def dashboard_stats_api(request):
     # Use the filtered queryset to calculate stats
     queryset = UserVisit.objects.all()
     if filterset.is_valid():
-        print(filterset.form.cleaned_data)
         queryset = filterset.filter_queryset(queryset)
 
     # Example stats calculation (adjust based on your needs)
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 12c459c0..f3117dba 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -101,6 +101,9 @@ <h3 class="my-2">Additional Information</h3>
         await this.loadStats();
         const formElement = this.$refs.filterForm.querySelector('form');
         formElement.querySelectorAll('select, input').forEach(input => {
+          if (input.value) {
+            input.dispatchEvent(new Event('change'));
+          }
           input.addEventListener('change', () => {
             this.loadStats();
             window.refreshMapData();

From 6bf9b0faeff7ec3a41d5f800c7f17738243bd61d Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 14:51:19 +0200
Subject: [PATCH 40/96] remove additional column for now

---
 commcare_connect/templates/reports/dashboard.html | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index f3117dba..e1973c7d 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -71,18 +71,10 @@ <h2 class="my-2">Program Dashboard</h2>
   </div>
 </div>
 <div class="row">
-  <div class="col-12 col-lg-8">
+  <div class="col-12">
     <h3 class="my-2">Service Delivery Map</h3>
     <div id="map" class="mt-3 mb-2 rounded border" style="height: 900px;"></div>
   </div>
-  <div class="col-12 col-lg-4">
-    <h3 class="my-2">Additional Information</h3>
-    <div class="card">
-      <div class="card-body">
-        <p class="text-muted">Placeholder content</p>
-      </div>
-    </div>
-  </div>
 </div>
 {% endblock content %}
 {% block inline_javascript %}

From 456c515406552df629ce229ac7faf19fc1a2acdd Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Thu, 31 Oct 2024 15:02:35 +0200
Subject: [PATCH 41/96] fix tests

---
 .../reports/tests/test_reports.py             | 37 +++++++++++++------
 1 file changed, 25 insertions(+), 12 deletions(-)

diff --git a/commcare_connect/reports/tests/test_reports.py b/commcare_connect/reports/tests/test_reports.py
index 25710f41..f85ae168 100644
--- a/commcare_connect/reports/tests/test_reports.py
+++ b/commcare_connect/reports/tests/test_reports.py
@@ -65,19 +65,32 @@ def test_delivery_stats(opportunity: Opportunity):
 
 
 def test_results_to_geojson():
+    class MockQuerySet:
+        def __init__(self, results):
+            self.results = results
+
+        def all(self):
+            return self.results
+
     # Test input
-    results = [
-        {"gps_location_long": "10.123", "gps_location_lat": "20.456", "status": "approved", "other_field": "value1"},
-        {"gps_location_long": "30.789", "gps_location_lat": "40.012", "status": "rejected", "other_field": "value2"},
-        {"gps_location_long": "invalid", "gps_location_lat": "50.678", "status": "unknown", "other_field": "value3"},
-        {"status": "approved", "other_field": "value4"},  # Case where lat/lon are not present
-        {  # Case where lat/lon are null
-            "gps_location_long": None,
-            "gps_location_lat": None,
-            "status": "rejected",
-            "other_field": "value5",
-        },
-    ]
+    results = MockQuerySet(
+        [
+            {"location_str": "20.456 10.123 0 0", "status": "approved", "other_field": "value1"},
+            {"location_str": "40.012 30.789", "status": "rejected", "other_field": "value2"},
+            {"location_str": "invalid location", "status": "unknown", "other_field": "value3"},
+            {"location_str": "bad location", "status": "unknown", "other_field": "value4"},
+            {
+                "location_str": None,
+                "status": "approved",
+                "other_field": "value5",
+            },  # Case where lat/lon are not present
+            {  # Case where lat/lon are null
+                "location_str": None,
+                "status": "rejected",
+                "other_field": "value5",
+            },
+        ]
+    )
 
     # Call the function
     geojson = _results_to_geojson(results)

From 437b55be85fd6a7a9cb6f43f486e512a9d1df1ac Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Fri, 1 Nov 2024 11:41:26 +0200
Subject: [PATCH 42/96] document the staging deployment setup

---
 README.md | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/README.md b/README.md
index a2450d14..6e32ed21 100644
--- a/README.md
+++ b/README.md
@@ -146,6 +146,19 @@ For details on how this actions is configured see:
 - https://aws.amazon.com/blogs/security/use-iam-roles-to-connect-github-actions-to-actions-in-aws/
 - https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-amazon-web-services
 
+### Deploying to the staging environment
+
+The project has a staging environment at [https://connect-staging.dimagi.com/](https://connect-staging.dimagi.com/),
+which is connected to the staging environment of CommCare HQ at
+[https://staging.commcarehq.org/](https://staging.commcarehq.org/).
+
+By convention, the `pkv/staging` branch is used for changes that are on the staging environment.
+To put your own changes on the staging environment, you can create merge your own branch into
+`pkv/staging` and then push it to GitHub.
+
+After that, you can deploy to the staging environment by manually running the `deploy`
+[workflow from here](https://github.com/dimagi/commcare-connect/actions/workflows/deploy.yml).
+
 ### Custom Bootstrap Compilation
 
 The generated CSS is set up with automatic Bootstrap recompilation with variables of your choice.

From ccbe6da5803c2cb6cd6c80192b7fe01de44af4fd Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Fri, 1 Nov 2024 15:56:15 +0200
Subject: [PATCH 43/96] add more data to the map properties

---
 commcare_connect/reports/queries.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/commcare_connect/reports/queries.py b/commcare_connect/reports/queries.py
index 4b7d2d14..508c9341 100644
--- a/commcare_connect/reports/queries.py
+++ b/commcare_connect/reports/queries.py
@@ -19,9 +19,13 @@ def get_visit_map_queryset(base_queryset):
             ),
             location_str=KeyTextTransform("location", KeyTextTransform("metadata", F("form_json"))),
         )
-        .select_related("deliver_unit", "opportunity")
+        .select_related("deliver_unit", "opportunity", "opportunity__delivery_type", "opportunity__organization")
         .values(
             "opportunity_id",
+            "opportunity__delivery_type__name",
+            "opportunity__delivery_type__slug",
+            "opportunity__organization__slug",
+            "opportunity__organization__name",
             "xform_id",
             "visit_date",
             "username_connectid",

From cd9269358ff44a953ce3d8125a77770bf8657533 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Tue, 5 Nov 2024 16:09:40 +0530
Subject: [PATCH 44/96] changed user field from select to text

---
 commcare_connect/organization/forms.py        | 36 ++++++++++---
 .../organization/tests/__init__.py            |  0
 .../organization/tests/test_forms.py          | 53 +++++++++++++++++++
 3 files changed, 82 insertions(+), 7 deletions(-)
 create mode 100644 commcare_connect/organization/tests/__init__.py
 create mode 100644 commcare_connect/organization/tests/test_forms.py

diff --git a/commcare_connect/organization/forms.py b/commcare_connect/organization/forms.py
index f22a4b7b..5500b267 100644
--- a/commcare_connect/organization/forms.py
+++ b/commcare_connect/organization/forms.py
@@ -1,5 +1,7 @@
 from crispy_forms import helper, layout
 from django import forms
+from django.core.exceptions import ValidationError
+from django.db.models import Q
 from django.utils.translation import gettext
 
 from commcare_connect.organization.models import Organization, UserOrganizationMembership
@@ -25,29 +27,49 @@ def __init__(self, *args, **kwargs):
 
 
 class MembershipForm(forms.ModelForm):
+    email_or_username = forms.CharField(
+        max_length=254,
+        required=True,
+        label="",
+        widget=forms.TextInput(attrs={"placeholder": "Enter email or username"}),
+    )
+
     class Meta:
         model = UserOrganizationMembership
-        fields = ("user", "role")
-        labels = {"user": "", "role": ""}
+        fields = ("role",)
+        labels = {"role": ""}
 
     def __init__(self, *args, **kwargs):
         self.organization = kwargs.pop("organization")
         super().__init__(*args, **kwargs)
 
-        self.fields["user"].queryset = User.objects.filter(email__isnull=False).exclude(
-            memberships__organization=self.organization
-        )
-
         self.helper = helper.FormHelper(self)
         self.helper.layout = layout.Layout(
             layout.Row(
                 layout.HTML("<h4>Add new member</h4>"),
-                layout.Field("user", wrapper_class="col-md-5"),
+                layout.Field("email_or_username", wrapper_class="col-md-5"),
                 layout.Field("role", wrapper_class="col-md-5"),
                 layout.Div(layout.Submit("submit", gettext("Submit")), css_class="col-md-2"),
             ),
         )
 
+    def clean_email_or_username(self):
+        email_or_username = self.cleaned_data["email_or_username"]
+        user = (
+            User.objects.filter(
+                Q(email=email_or_username) | Q(username=email_or_username),
+                email__isnull=False,
+            )
+            .exclude(memberships__organization=self.organization)
+            .first()
+        )
+
+        if not user:
+            raise ValidationError("User with this email/username does not exist or is already a member")
+
+        self.instance.user = user
+        return email_or_username
+
 
 class AddCredentialForm(forms.Form):
     credential = forms.CharField(widget=forms.Select)
diff --git a/commcare_connect/organization/tests/__init__.py b/commcare_connect/organization/tests/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/commcare_connect/organization/tests/test_forms.py b/commcare_connect/organization/tests/test_forms.py
new file mode 100644
index 00000000..d3b1a71b
--- /dev/null
+++ b/commcare_connect/organization/tests/test_forms.py
@@ -0,0 +1,53 @@
+import pytest
+from django.test import Client
+from django.urls import reverse
+
+from commcare_connect.organization.models import Organization
+from commcare_connect.users.tests.factories import UserFactory
+
+
+class TestAddMembersView:
+    @pytest.fixture(autouse=True)
+    def setup(self, organization: Organization, client: Client):
+        self.url = reverse("organization:add_members", kwargs={"org_slug": organization.slug})
+        self.user = organization.memberships.filter(role="admin").first().user
+        self.client = client
+        client.force_login(self.user)
+
+    @pytest.mark.django_db
+    def test_add_member_by_email(self, organization):
+        new_user = UserFactory(email="test@example.com")
+        data = {"email_or_username": new_user.email, "role": "member"}
+        response = self.client.post(self.url, data)
+        assert response.status_code == 302
+        assert response.url == reverse("organization:home", kwargs={"org_slug": organization.slug})
+        membership = organization.memberships.get(user=new_user)
+        assert membership.role == data["role"]
+
+    @pytest.mark.django_db
+    def test_add_member_by_username(self, organization):
+        new_user = UserFactory(username="test")
+        data = {"email_or_username": new_user.username, "role": "member"}
+        response = self.client.post(self.url, data)
+        assert response.status_code == 302
+        membership = organization.memberships.get(user=new_user)
+        assert membership.role == data["role"]
+        assert not membership.accepted
+
+    @pytest.mark.django_db
+    def test_add_member_nonexistent_user(self, organization):
+        data = {"email_or_username": "nonexistent@example.com", "role": "member"}
+        response = self.client.post(self.url, data)
+        assert response.status_code == 302
+        assert not organization.memberships.filter(user__email="nonexistent@example.com").exists()
+
+    @pytest.mark.django_db
+    def test_add_existing_member(self, organization):
+        existing_user = UserFactory(email="test@example.com")
+        organization.members.add(existing_user, through_defaults={"role": "member"})
+
+        data = {"email_or_username": existing_user.email, "role": "admin"}
+        response = self.client.post(self.url, data)
+        assert response.status_code == 302
+        assert organization.memberships.filter(user=existing_user).count() == 1
+        assert organization.memberships.get(user=existing_user).role == "member"

From 0fcf8448ea912d30a0be1d288a4f248f260a6dff Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 6 Nov 2024 09:16:28 +0530
Subject: [PATCH 45/96] changed label

---
 commcare_connect/organization/forms.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/commcare_connect/organization/forms.py b/commcare_connect/organization/forms.py
index 5500b267..743268e9 100644
--- a/commcare_connect/organization/forms.py
+++ b/commcare_connect/organization/forms.py
@@ -31,7 +31,7 @@ class MembershipForm(forms.ModelForm):
         max_length=254,
         required=True,
         label="",
-        widget=forms.TextInput(attrs={"placeholder": "Enter email or username"}),
+        widget=forms.TextInput(attrs={"placeholder": "Enter email address or username"}),
     )
 
     class Meta:
@@ -56,10 +56,7 @@ def __init__(self, *args, **kwargs):
     def clean_email_or_username(self):
         email_or_username = self.cleaned_data["email_or_username"]
         user = (
-            User.objects.filter(
-                Q(email=email_or_username) | Q(username=email_or_username),
-                email__isnull=False,
-            )
+            User.objects.filter(Q(email=email_or_username) | Q(username=email_or_username))
             .exclude(memberships__organization=self.organization)
             .first()
         )

From edc2b5a3d537b0f066d6571a9dc5cd721bd71141 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 6 Nov 2024 09:50:30 +0530
Subject: [PATCH 46/96] refactor tests

---
 .../organization/tests/test_forms.py          | 72 ++++++++++---------
 1 file changed, 40 insertions(+), 32 deletions(-)

diff --git a/commcare_connect/organization/tests/test_forms.py b/commcare_connect/organization/tests/test_forms.py
index d3b1a71b..c3ae9a52 100644
--- a/commcare_connect/organization/tests/test_forms.py
+++ b/commcare_connect/organization/tests/test_forms.py
@@ -15,39 +15,47 @@ def setup(self, organization: Organization, client: Client):
         client.force_login(self.user)
 
     @pytest.mark.django_db
-    def test_add_member_by_email(self, organization):
-        new_user = UserFactory(email="test@example.com")
-        data = {"email_or_username": new_user.email, "role": "member"}
+    @pytest.mark.parametrize(
+        "email_or_username, role, expected_status_code, create_user, expected_role, should_exist",
+        [
+            ("testformember@example.com", "member", 302, True, "member", True),
+            ("testforadmin@example.com", "admin", 302, True, "admin", True),
+            ("testforusername", "member", 302, True, "member", True),
+            ("testforusername", "admin", 302, True, "admin", True),
+            ("nonexistent@example.com", "member", 302, False, None, False),
+            ("existing@example.com", "admin", 302, True, "member", True),
+        ],
+    )
+    def test_add_member(
+        self,
+        email_or_username,
+        role,
+        expected_status_code,
+        create_user,
+        expected_role,
+        should_exist,
+        organization,
+    ):
+        if create_user:
+            user = UserFactory(
+                email=email_or_username if "@" in email_or_username else None,
+                username=email_or_username if "@" not in email_or_username else None,
+            )
+
+            if email_or_username == "existing@example.com":
+                organization.members.add(user, through_defaults={"role": expected_role})
+
+        data = {"email_or_username": email_or_username, "role": role}
         response = self.client.post(self.url, data)
-        assert response.status_code == 302
-        assert response.url == reverse("organization:home", kwargs={"org_slug": organization.slug})
-        membership = organization.memberships.get(user=new_user)
-        assert membership.role == data["role"]
 
-    @pytest.mark.django_db
-    def test_add_member_by_username(self, organization):
-        new_user = UserFactory(username="test")
-        data = {"email_or_username": new_user.username, "role": "member"}
-        response = self.client.post(self.url, data)
-        assert response.status_code == 302
-        membership = organization.memberships.get(user=new_user)
-        assert membership.role == data["role"]
-        assert not membership.accepted
+        membership_filter = (
+            {"user__email": email_or_username} if "@" in email_or_username else {"user__username": email_or_username}
+        )
 
-    @pytest.mark.django_db
-    def test_add_member_nonexistent_user(self, organization):
-        data = {"email_or_username": "nonexistent@example.com", "role": "member"}
-        response = self.client.post(self.url, data)
-        assert response.status_code == 302
-        assert not organization.memberships.filter(user__email="nonexistent@example.com").exists()
+        assert response.status_code == expected_status_code
+        membership_exists = organization.memberships.filter(**membership_filter).exists()
+        assert membership_exists == should_exist
 
-    @pytest.mark.django_db
-    def test_add_existing_member(self, organization):
-        existing_user = UserFactory(email="test@example.com")
-        organization.members.add(existing_user, through_defaults={"role": "member"})
-
-        data = {"email_or_username": existing_user.email, "role": "admin"}
-        response = self.client.post(self.url, data)
-        assert response.status_code == 302
-        assert organization.memberships.filter(user=existing_user).count() == 1
-        assert organization.memberships.get(user=existing_user).role == "member"
+        if should_exist and expected_role:
+            membership = organization.memberships.get(**membership_filter)
+            assert membership.role == expected_role

From 96d9a64b38681c43b4f8c631452815fe2dec11d4 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 6 Nov 2024 14:28:02 +0530
Subject: [PATCH 47/96] removed the username option

---
 commcare_connect/organization/forms.py        | 19 +++++++------------
 .../organization/tests/test_forms.py          | 19 ++++++-------------
 2 files changed, 13 insertions(+), 25 deletions(-)

diff --git a/commcare_connect/organization/forms.py b/commcare_connect/organization/forms.py
index 743268e9..1ca006ed 100644
--- a/commcare_connect/organization/forms.py
+++ b/commcare_connect/organization/forms.py
@@ -1,7 +1,6 @@
 from crispy_forms import helper, layout
 from django import forms
 from django.core.exceptions import ValidationError
-from django.db.models import Q
 from django.utils.translation import gettext
 
 from commcare_connect.organization.models import Organization, UserOrganizationMembership
@@ -27,11 +26,11 @@ def __init__(self, *args, **kwargs):
 
 
 class MembershipForm(forms.ModelForm):
-    email_or_username = forms.CharField(
+    email = forms.CharField(
         max_length=254,
         required=True,
         label="",
-        widget=forms.TextInput(attrs={"placeholder": "Enter email address or username"}),
+        widget=forms.TextInput(attrs={"placeholder": "Enter email address"}),
     )
 
     class Meta:
@@ -47,25 +46,21 @@ def __init__(self, *args, **kwargs):
         self.helper.layout = layout.Layout(
             layout.Row(
                 layout.HTML("<h4>Add new member</h4>"),
-                layout.Field("email_or_username", wrapper_class="col-md-5"),
+                layout.Field("email", wrapper_class="col-md-5"),
                 layout.Field("role", wrapper_class="col-md-5"),
                 layout.Div(layout.Submit("submit", gettext("Submit")), css_class="col-md-2"),
             ),
         )
 
-    def clean_email_or_username(self):
-        email_or_username = self.cleaned_data["email_or_username"]
-        user = (
-            User.objects.filter(Q(email=email_or_username) | Q(username=email_or_username))
-            .exclude(memberships__organization=self.organization)
-            .first()
-        )
+    def clean_email(self):
+        email = self.cleaned_data["email"]
+        user = User.objects.filter(email=email).exclude(memberships__organization=self.organization).first()
 
         if not user:
             raise ValidationError("User with this email/username does not exist or is already a member")
 
         self.instance.user = user
-        return email_or_username
+        return email
 
 
 class AddCredentialForm(forms.Form):
diff --git a/commcare_connect/organization/tests/test_forms.py b/commcare_connect/organization/tests/test_forms.py
index c3ae9a52..8f4381a4 100644
--- a/commcare_connect/organization/tests/test_forms.py
+++ b/commcare_connect/organization/tests/test_forms.py
@@ -16,19 +16,17 @@ def setup(self, organization: Organization, client: Client):
 
     @pytest.mark.django_db
     @pytest.mark.parametrize(
-        "email_or_username, role, expected_status_code, create_user, expected_role, should_exist",
+        "email, role, expected_status_code, create_user, expected_role, should_exist",
         [
             ("testformember@example.com", "member", 302, True, "member", True),
             ("testforadmin@example.com", "admin", 302, True, "admin", True),
-            ("testforusername", "member", 302, True, "member", True),
-            ("testforusername", "admin", 302, True, "admin", True),
             ("nonexistent@example.com", "member", 302, False, None, False),
             ("existing@example.com", "admin", 302, True, "member", True),
         ],
     )
     def test_add_member(
         self,
-        email_or_username,
+        email,
         role,
         expected_status_code,
         create_user,
@@ -37,20 +35,15 @@ def test_add_member(
         organization,
     ):
         if create_user:
-            user = UserFactory(
-                email=email_or_username if "@" in email_or_username else None,
-                username=email_or_username if "@" not in email_or_username else None,
-            )
+            user = UserFactory(email=email)
 
-            if email_or_username == "existing@example.com":
+            if email == "existing@example.com":
                 organization.members.add(user, through_defaults={"role": expected_role})
 
-        data = {"email_or_username": email_or_username, "role": role}
+        data = {"email": email, "role": role}
         response = self.client.post(self.url, data)
 
-        membership_filter = (
-            {"user__email": email_or_username} if "@" in email_or_username else {"user__username": email_or_username}
-        )
+        membership_filter = {"user__email": email}
 
         assert response.status_code == expected_status_code
         membership_exists = organization.memberships.filter(**membership_filter).exists()

From f4a56ca81d3caf4c210cececfd502e422d050245 Mon Sep 17 00:00:00 2001
From: hemant10yadav <hemant.10.yadav@gmail.com>
Date: Wed, 6 Nov 2024 14:30:17 +0530
Subject: [PATCH 48/96] changed the msg

---
 commcare_connect/organization/forms.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/commcare_connect/organization/forms.py b/commcare_connect/organization/forms.py
index 1ca006ed..4bbde435 100644
--- a/commcare_connect/organization/forms.py
+++ b/commcare_connect/organization/forms.py
@@ -57,7 +57,7 @@ def clean_email(self):
         user = User.objects.filter(email=email).exclude(memberships__organization=self.organization).first()
 
         if not user:
-            raise ValidationError("User with this email/username does not exist or is already a member")
+            raise ValidationError("User with this email does not exist or is already a member")
 
         self.instance.user = user
         return email

From 76aee2f87ac9f882d36c45e5e212ec63f455393b Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 6 Nov 2024 14:33:10 +0200
Subject: [PATCH 49/96] simpliify queries

---
 commcare_connect/reports/queries.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/commcare_connect/reports/queries.py b/commcare_connect/reports/queries.py
index 508c9341..ed903bb1 100644
--- a/commcare_connect/reports/queries.py
+++ b/commcare_connect/reports/queries.py
@@ -1,5 +1,5 @@
 from django.db.models import DateTimeField, ExpressionWrapper, F, FloatField
-from django.db.models.fields.json import KeyTextTransform
+from django.db.models.fields.json import KT
 from django.db.models.functions import Cast, Extract
 
 
@@ -11,13 +11,13 @@ def get_visit_map_queryset(base_queryset):
             days_since_opp_start=ExpressionWrapper(
                 Extract(F("visit_date") - F("opportunity__start_date"), "day"), output_field=FloatField()
             ),
-            timestart_str=KeyTextTransform("timeStart", KeyTextTransform("metadata", F("form_json"))),
-            timeend_str=KeyTextTransform("timeEnd", KeyTextTransform("metadata", F("form_json"))),
+            timestart_str=KT("form_json__metadata__timeStart"),
+            timeend_str=KT("form_json__metadata__timeEnd"),
             visit_duration=ExpressionWrapper(
                 Extract(Cast("timeend_str", DateTimeField()) - Cast("timestart_str", DateTimeField()), "epoch") / 60,
                 output_field=FloatField(),
             ),
-            location_str=KeyTextTransform("location", KeyTextTransform("metadata", F("form_json"))),
+            location_str=KT("form_json__metadata__location"),
         )
         .select_related("deliver_unit", "opportunity", "opportunity__delivery_type", "opportunity__organization")
         .values(

From fc25d8f271be50d6d0f766b609233de5647453e2 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 6 Nov 2024 14:36:22 +0200
Subject: [PATCH 50/96] remove unused fields

---
 commcare_connect/reports/queries.py | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/commcare_connect/reports/queries.py b/commcare_connect/reports/queries.py
index ed903bb1..e54a50a8 100644
--- a/commcare_connect/reports/queries.py
+++ b/commcare_connect/reports/queries.py
@@ -1,6 +1,5 @@
-from django.db.models import DateTimeField, ExpressionWrapper, F, FloatField
+from django.db.models import F
 from django.db.models.fields.json import KT
-from django.db.models.functions import Cast, Extract
 
 
 def get_visit_map_queryset(base_queryset):
@@ -8,15 +7,8 @@ def get_visit_map_queryset(base_queryset):
         base_queryset.annotate(
             username_connectid=F("form_json__metadata__username"),
             deliver_unit_name=F("deliver_unit__name"),
-            days_since_opp_start=ExpressionWrapper(
-                Extract(F("visit_date") - F("opportunity__start_date"), "day"), output_field=FloatField()
-            ),
             timestart_str=KT("form_json__metadata__timeStart"),
             timeend_str=KT("form_json__metadata__timeEnd"),
-            visit_duration=ExpressionWrapper(
-                Extract(Cast("timeend_str", DateTimeField()) - Cast("timestart_str", DateTimeField()), "epoch") / 60,
-                output_field=FloatField(),
-            ),
             location_str=KT("form_json__metadata__location"),
         )
         .select_related("deliver_unit", "opportunity", "opportunity__delivery_type", "opportunity__organization")
@@ -30,7 +22,6 @@ def get_visit_map_queryset(base_queryset):
             "visit_date",
             "username_connectid",
             "deliver_unit_name",
-            "days_since_opp_start",
             "entity_id",
             "status",
             "flagged",
@@ -38,7 +29,6 @@ def get_visit_map_queryset(base_queryset):
             "reason",
             "timestart_str",
             "timeend_str",
-            "visit_duration",
             "location_str",
         )
     )

From 8166964ae9e191b4631e78821cc01e34752fd31b Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 6 Nov 2024 14:41:47 +0200
Subject: [PATCH 51/96] use kt on the last field

---
 commcare_connect/reports/queries.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/commcare_connect/reports/queries.py b/commcare_connect/reports/queries.py
index e54a50a8..ed9ddf91 100644
--- a/commcare_connect/reports/queries.py
+++ b/commcare_connect/reports/queries.py
@@ -5,8 +5,8 @@
 def get_visit_map_queryset(base_queryset):
     return (
         base_queryset.annotate(
-            username_connectid=F("form_json__metadata__username"),
             deliver_unit_name=F("deliver_unit__name"),
+            username_connectid=KT("form_json__metadata__username"),
             timestart_str=KT("form_json__metadata__timeStart"),
             timeend_str=KT("form_json__metadata__timeEnd"),
             location_str=KT("form_json__metadata__location"),

From 1187920419427ef169991bd24fe2107b22b968c1 Mon Sep 17 00:00:00 2001
From: Cal Ellowitz <calellowitz@gmail.com>
Date: Fri, 8 Nov 2024 09:58:05 -0500
Subject: [PATCH 52/96] allow search by username

---
 commcare_connect/opportunity/admin.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/commcare_connect/opportunity/admin.py b/commcare_connect/opportunity/admin.py
index 6bb084a5..86839d6e 100644
--- a/commcare_connect/opportunity/admin.py
+++ b/commcare_connect/opportunity/admin.py
@@ -48,6 +48,7 @@ class OpportunityAccessAdmin(admin.ModelAdmin):
     form = OpportunityAccessCreationForm
     list_display = ["get_opp_name", "get_username"]
     actions = ["clear_user_progress"]
+    search_fields = ["user__username"]
 
     @admin.display(description="Opportunity Name")
     def get_opp_name(self, obj):

From 3068d294f18cce836864f14918abbea2e5e809fd Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 14:44:36 +0200
Subject: [PATCH 53/96] change center to africa

---
 commcare_connect/templates/reports/dashboard.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index e1973c7d..75de2f19 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -143,8 +143,8 @@ <h3 class="my-2">Service Delivery Map</h3>
       container: 'map',
       // Choose from Mapbox's core styles, or make your own style with Mapbox Studio
       style: 'mapbox://styles/mapbox/dark-v11',
-      center: [-103.5917, 40.6699],
-      zoom: 3
+      center: [20, 0], // Centered on Africa (roughly central coordinates)
+      zoom: 3,
     });
 
     map.on('load', () => {

From fbbafd14bb1f257999b5e0a5f073fbdd660f4991 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 14:44:52 +0200
Subject: [PATCH 54/96] add ability to enable/disable clustering by url param

---
 commcare_connect/reports/views.py                 | 3 ++-
 commcare_connect/templates/reports/dashboard.html | 6 +++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index c91e8789..6c602801 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -223,12 +223,13 @@ class Meta:
 @user_passes_test(lambda u: u.is_superuser)
 def program_dashboard_report(request):
     filterset = DashboardFilters(request.GET)
-
+    cluster = request.GET.get("cluster", "true") == "true"
     return render(
         request,
         "reports/dashboard.html",
         context={
             "mapbox_token": settings.MAPBOX_TOKEN,
+            "cluster_visits": cluster,
             "filter": filterset,
         },
     )
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 75de2f19..89807352 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -139,9 +139,9 @@ <h3 class="my-2">Service Delivery Map</h3>
 
   window.addEventListener('DOMContentLoaded', () => {
     mapboxgl.accessToken = "{{ mapbox_token }}";
+    const cluster = {% if cluster_visits %}true{% else %}false{% endif %};
     map = new mapboxgl.Map({
       container: 'map',
-      // Choose from Mapbox's core styles, or make your own style with Mapbox Studio
       style: 'mapbox://styles/mapbox/dark-v11',
       center: [20, 0], // Centered on Africa (roughly central coordinates)
       zoom: 3,
@@ -156,8 +156,8 @@ <h3 class="my-2">Service Delivery Map</h3>
       map.addSource('visits', {
         type: 'geojson',
         data: `{% url "reports:visit_map_data" %}?${queryString}`,
-        cluster: true,
-        clusterMaxZoom: 14,
+        cluster: cluster,
+        clusterMaxZoom: 12,
         clusterRadius: 50
       });
 

From 1d06e14535c468cc6cf7158a9d12592cc200a934 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 15:19:55 +0200
Subject: [PATCH 55/96] wip: switch to chart clusters

---
 .../templates/reports/dashboard.html          | 208 +++++++++++++++---
 1 file changed, 176 insertions(+), 32 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 89807352..f133fde2 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -147,6 +147,17 @@ <h3 class="my-2">Service Delivery Map</h3>
       zoom: 3,
     });
 
+    // filters for classifying visits by status
+    const approved = ['==', ['get', 'status'], 'approved'];
+    const pending = ['all',
+      ['!=', ['get', 'status'], 'approved'],
+      ['!=', ['get', 'status'], 'rejected']
+    ];
+    const rejected = ['==', ['get', 'status'], 'rejected'];
+
+    // colors to use for the categories
+    const colors = ['#00FF00', '#FFFF00', '#FF0000'];
+
     map.on('load', () => {
       // Modify the source configuration to include initial filters
       const formElement = document.querySelector('#filterForm form');
@@ -158,43 +169,106 @@ <h3 class="my-2">Service Delivery Map</h3>
         data: `{% url "reports:visit_map_data" %}?${queryString}`,
         cluster: cluster,
         clusterMaxZoom: 12,
-        clusterRadius: 50
+        clusterRadius: 80,
+        clusterProperties: {
+          // keep separate counts for each status category in a cluster
+          'approved': ['+', ['case', approved, 1, 0]],
+          'pending': ['+', ['case', pending, 1, 0]],
+          'rejected': ['+', ['case', rejected, 1, 0]]
+        }
       });
 
+      // circle and symbol layers for rendering individual visits (unclustered points)
+      map.addLayer({
+          'id': 'visit_circle',
+          'type': 'circle',
+          'source': 'visits',
+          'filter': ['!=', 'cluster', true],
+          'paint': {
+            'circle-color': [
+              'case',
+              approved,
+              colors[0],
+              pending,
+              colors[1],
+              rejected,
+              colors[2],
+              colors[2]
+            ],
+            'circle-opacity': 0.6,
+            'circle-radius': 12
+          }
+      });
+
+
+      // objects for caching and keeping track of HTML marker objects (for performance)
+      const markers = {};
+      let markersOnScreen = {};
+      function updateMarkers() {
+        console.log('updateMarkers');
+        const newMarkers = {};
+        const features = map.querySourceFeatures('visits');
+
+        // for every cluster on the screen, create an HTML marker for it (if we didn't yet),
+        // and add it to the map if it's not there already
+        for (const feature of features) {
+          const coords = feature.geometry.coordinates;
+          const props = feature.properties;
+          if (!props.cluster) continue;
+          console.log('cluster', feature);
+          const id = props.cluster_id;
+
+          let marker = markers[id];
+          if (!marker) {
+              const el = createDonutChart(props);
+              marker = markers[id] = new mapboxgl.Marker({
+                  element: el
+              }).setLngLat(coords);
+          }
+          newMarkers[id] = marker;
+
+          if (!markersOnScreen[id]) marker.addTo(map);
+        }
+        // for every marker we've added previously, remove those that are no longer visible
+        for (const id in markersOnScreen) {
+          if (!newMarkers[id]) markersOnScreen[id].remove();
+        }
+        markersOnScreen = newMarkers;
+      }
       const clusterStep1 = 50;
       const clusterStep2 = 100;
 
-      map.addLayer({
-        id: 'clusters',
-        type: 'circle',
-        source: 'visits',
-        filter: ['has', 'point_count'],
-        paint: {
-          // Use step expressions (https://docs.mapbox.com/style-spec/reference/expressions/#step)
-          // with three steps to implement three types of circles:
-          //   * Blue, 20px circles when point count is less than clusterStep1
-          //   * Yellow, 30px circles when point count is between clusterStep1 and clusterStep2
-          //   * Green, 40px circles when point count is greater than or equal to clusterStep2
-          'circle-color': [
-            'step',
-            ['get', 'point_count'],
-            '#51bbd6',
-            clusterStep1,
-            '#f1f075',
-            clusterStep2,
-            '#00FF00'
-          ],
-          'circle-radius': [
-            'step',
-            ['get', 'point_count'],
-            20,
-            clusterStep1,
-            30,
-            clusterStep2,
-            40
-          ]
-        }
-      });
+      // map.addlayer({
+      //   id: 'clusters',
+      //   type: 'circle',
+      //   source: 'visits',
+      //   filter: ['has', 'point_count'],
+      //   paint: {
+      //     // use step expressions (https://docs.mapbox.com/style-spec/reference/expressions/#step)
+      //     // with three steps to implement three types of circles:
+      //     //   * blue, 20px circles when point count is less than clusterstep1
+      //     //   * yellow, 30px circles when point count is between clusterstep1 and clusterstep2
+      //     //   * green, 40px circles when point count is greater than or equal to clusterstep2
+      //     'circle-color': [
+      //       'step',
+      //       ['get', 'point_count'],
+      //       '#51bbd6',
+      //       clusterstep1,
+      //       '#f1f075',
+      //       clusterstep2,
+      //       '#00ff00'
+      //     ],
+      //     'circle-radius': [
+      //       'step',
+      //       ['get', 'point_count'],
+      //       20,
+      //       clusterstep1,
+      //       30,
+      //       clusterstep2,
+      //       40
+      //     ]
+      //   }
+      // });
 
       map.addLayer({
         id: 'cluster-count',
@@ -274,7 +348,77 @@ <h3 class="my-2">Service Delivery Map</h3>
       map.on('mouseleave', 'clusters', () => {
         map.getCanvas().style.cursor = '';
       });
+      // after the GeoJSON data is loaded, update markers on the screen on every frame
+      map.on('render', () => {
+        if (!map.isSourceLoaded('visits')) return;
+        updateMarkers();
+      });
     });
+
+    // code for creating an SVG donut chart from feature properties
+    function createDonutChart(props) {
+        const offsets = [];
+        const counts = [
+            props.approved,
+            props.pending,
+            props.rejected
+        ];
+        let total = 0;
+        for (const count of counts) {
+            offsets.push(total);
+            total += count;
+        }
+        const fontSize =
+            total >= 1000 ? 22 : total >= 100 ? 20 : total >= 10 ? 18 : 16;
+        const r =
+            total >= 1000 ? 50 : total >= 100 ? 32 : total >= 10 ? 24 : 18;
+        const r0 = Math.round(r * 0.6);
+        const w = r * 2;
+
+        let html = `<div>
+            <svg width="${w}" height="${w}" viewbox="0 0 ${w} ${w}" text-anchor="middle" style="font: ${fontSize}px sans-serif; display: block">`;
+
+        for (let i = 0; i < counts.length; i++) {
+            html += donutSegment(
+                offsets[i] / total,
+                (offsets[i] + counts[i]) / total,
+                r,
+                r0,
+                colors[i]
+            );
+        }
+        html += `<circle cx="${r}" cy="${r}" r="${r0}" fill="white" />
+            <text dominant-baseline="central" transform="translate(${r}, ${r})">
+                ${total.toLocaleString()}
+            </text>
+            </svg>
+            </div>`;
+
+        const el = document.createElement('div');
+        el.innerHTML = html;
+        return el.firstChild;
+    }
+
+    function donutSegment(start, end, r, r0, color) {
+        if (end - start === 1) end -= 0.00001;
+        const a0 = 2 * Math.PI * (start - 0.25);
+        const a1 = 2 * Math.PI * (end - 0.25);
+        const x0 = Math.cos(a0),
+            y0 = Math.sin(a0);
+        const x1 = Math.cos(a1),
+            y1 = Math.sin(a1);
+        const largeArc = end - start > 0.5 ? 1 : 0;
+
+        // draw an SVG path
+        return `<path d="M ${r + r0 * x0} ${r + r0 * y0} L ${r + r * x0} ${
+            r + r * y0
+        } A ${r} ${r} 0 ${largeArc} 1 ${r + r * x1} ${r + r * y1} L ${
+            r + r0 * x1
+        } ${r + r0 * y1} A ${r0} ${r0} 0 ${largeArc} 0 ${r + r0 * x0} ${
+            r + r0 * y0
+        }" fill="${color}" />`;
+    }
   });
+
 </script>
 {% endblock %}

From 7720e2fbcfe4af14af8b59c5731c9c816f7e0da6 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 15:52:08 +0200
Subject: [PATCH 56/96] cleanup

---
 .../templates/reports/dashboard.html          | 162 ++++++------------
 1 file changed, 55 insertions(+), 107 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index f133fde2..e41e3bc6 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -178,110 +178,6 @@ <h3 class="my-2">Service Delivery Map</h3>
         }
       });
 
-      // circle and symbol layers for rendering individual visits (unclustered points)
-      map.addLayer({
-          'id': 'visit_circle',
-          'type': 'circle',
-          'source': 'visits',
-          'filter': ['!=', 'cluster', true],
-          'paint': {
-            'circle-color': [
-              'case',
-              approved,
-              colors[0],
-              pending,
-              colors[1],
-              rejected,
-              colors[2],
-              colors[2]
-            ],
-            'circle-opacity': 0.6,
-            'circle-radius': 12
-          }
-      });
-
-
-      // objects for caching and keeping track of HTML marker objects (for performance)
-      const markers = {};
-      let markersOnScreen = {};
-      function updateMarkers() {
-        console.log('updateMarkers');
-        const newMarkers = {};
-        const features = map.querySourceFeatures('visits');
-
-        // for every cluster on the screen, create an HTML marker for it (if we didn't yet),
-        // and add it to the map if it's not there already
-        for (const feature of features) {
-          const coords = feature.geometry.coordinates;
-          const props = feature.properties;
-          if (!props.cluster) continue;
-          console.log('cluster', feature);
-          const id = props.cluster_id;
-
-          let marker = markers[id];
-          if (!marker) {
-              const el = createDonutChart(props);
-              marker = markers[id] = new mapboxgl.Marker({
-                  element: el
-              }).setLngLat(coords);
-          }
-          newMarkers[id] = marker;
-
-          if (!markersOnScreen[id]) marker.addTo(map);
-        }
-        // for every marker we've added previously, remove those that are no longer visible
-        for (const id in markersOnScreen) {
-          if (!newMarkers[id]) markersOnScreen[id].remove();
-        }
-        markersOnScreen = newMarkers;
-      }
-      const clusterStep1 = 50;
-      const clusterStep2 = 100;
-
-      // map.addlayer({
-      //   id: 'clusters',
-      //   type: 'circle',
-      //   source: 'visits',
-      //   filter: ['has', 'point_count'],
-      //   paint: {
-      //     // use step expressions (https://docs.mapbox.com/style-spec/reference/expressions/#step)
-      //     // with three steps to implement three types of circles:
-      //     //   * blue, 20px circles when point count is less than clusterstep1
-      //     //   * yellow, 30px circles when point count is between clusterstep1 and clusterstep2
-      //     //   * green, 40px circles when point count is greater than or equal to clusterstep2
-      //     'circle-color': [
-      //       'step',
-      //       ['get', 'point_count'],
-      //       '#51bbd6',
-      //       clusterstep1,
-      //       '#f1f075',
-      //       clusterstep2,
-      //       '#00ff00'
-      //     ],
-      //     'circle-radius': [
-      //       'step',
-      //       ['get', 'point_count'],
-      //       20,
-      //       clusterstep1,
-      //       30,
-      //       clusterstep2,
-      //       40
-      //     ]
-      //   }
-      // });
-
-      map.addLayer({
-        id: 'cluster-count',
-        type: 'symbol',
-        source: 'visits',
-        filter: ['has', 'point_count'],
-        layout: {
-          'text-field': ['get', 'point_count_abbreviated'],
-          'text-font': ['DIN Offc Pro Medium', 'Arial Unicode MS Bold'],
-          'text-size': 12
-        }
-      });
-
       map.addLayer({
         id: 'unclustered-point',
         type: 'circle',
@@ -349,6 +245,39 @@ <h3 class="my-2">Service Delivery Map</h3>
         map.getCanvas().style.cursor = '';
       });
       // after the GeoJSON data is loaded, update markers on the screen on every frame
+      // objects for caching and keeping track of HTML marker objects (for performance)
+      const markers = {};
+      let markersOnScreen = {};
+      function updateMarkers() {
+        const newMarkers = {};
+        const features = map.querySourceFeatures('visits');
+
+        // for every cluster on the screen, create an HTML marker for it (if we didn't yet),
+        // and add it to the map if it's not there already
+        for (const feature of features) {
+          const coords = feature.geometry.coordinates;
+          const props = feature.properties;
+          if (!props.cluster) continue;
+          const id = props.cluster_id;
+
+          let marker = markers[id];
+          if (!marker) {
+              const el = createDonutChart(props);
+              marker = markers[id] = new mapboxgl.Marker({
+                  element: el
+              }).setLngLat(coords);
+          }
+          newMarkers[id] = marker;
+
+          if (!markersOnScreen[id]) marker.addTo(map);
+        }
+        // for every marker we've added previously, remove those that are no longer visible
+        for (const id in markersOnScreen) {
+          if (!newMarkers[id]) markersOnScreen[id].remove();
+        }
+        markersOnScreen = newMarkers;
+      }
+
       map.on('render', () => {
         if (!map.isSourceLoaded('visits')) return;
         updateMarkers();
@@ -372,7 +301,7 @@ <h3 class="my-2">Service Delivery Map</h3>
             total >= 1000 ? 22 : total >= 100 ? 20 : total >= 10 ? 18 : 16;
         const r =
             total >= 1000 ? 50 : total >= 100 ? 32 : total >= 10 ? 24 : 18;
-        const r0 = Math.round(r * 0.6);
+        const r0 = Math.round(r * 0.8);
         const w = r * 2;
 
         let html = `<div>
@@ -387,8 +316,8 @@ <h3 class="my-2">Service Delivery Map</h3>
                 colors[i]
             );
         }
-        html += `<circle cx="${r}" cy="${r}" r="${r0}" fill="white" />
-            <text dominant-baseline="central" transform="translate(${r}, ${r})">
+        html += `<circle cx="${r}" cy="${r}" r="${r0}" fill="transparent" />
+            <text dominant-baseline="central" transform="translate(${r}, ${r})" fill="white">
                 ${total.toLocaleString()}
             </text>
             </svg>
@@ -396,6 +325,25 @@ <h3 class="my-2">Service Delivery Map</h3>
 
         const el = document.createElement('div');
         el.innerHTML = html;
+
+        // Add mouse events to the donut chart
+        el.style.cursor = 'pointer';
+
+        // Add click handler to zoom to cluster
+        el.addEventListener('click', () => {
+          map.getSource('visits').getClusterExpansionZoom(
+            props.cluster_id,
+            (err, zoom) => {
+              if (err) return;
+
+              map.easeTo({
+                center: props.coordinates,
+                zoom: zoom
+              });
+            }
+          );
+        });
+
         return el.firstChild;
     }
 

From 6ee83a53de240634972ae8a5558a9b8737a1f720 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 15:59:23 +0200
Subject: [PATCH 57/96] cleanup

---
 .../templates/reports/dashboard.html          | 26 ++++++++-----------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index e41e3bc6..16226039 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -238,12 +238,6 @@ <h3 class="my-2">Service Delivery Map</h3>
           .addTo(map);
       });
 
-      map.on('mouseenter', 'clusters', () => {
-        map.getCanvas().style.cursor = 'pointer';
-      });
-      map.on('mouseleave', 'clusters', () => {
-        map.getCanvas().style.cursor = '';
-      });
       // after the GeoJSON data is loaded, update markers on the screen on every frame
       // objects for caching and keeping track of HTML marker objects (for performance)
       const markers = {};
@@ -262,10 +256,14 @@ <h3 class="my-2">Service Delivery Map</h3>
 
           let marker = markers[id];
           if (!marker) {
-              const el = createDonutChart(props);
-              marker = markers[id] = new mapboxgl.Marker({
-                  element: el
-              }).setLngLat(coords);
+            const el = createDonutChart({
+              ...props,
+              cluster_id: id,  // Make sure cluster_id is passed
+              coordinates: coords  // Pass the coordinates
+            });
+            marker = markers[id] = new mapboxgl.Marker({
+              element: el
+            }).setLngLat(coords);
           }
           newMarkers[id] = marker;
 
@@ -325,12 +323,10 @@ <h3 class="my-2">Service Delivery Map</h3>
 
         const el = document.createElement('div');
         el.innerHTML = html;
-
-        // Add mouse events to the donut chart
         el.style.cursor = 'pointer';
 
-        // Add click handler to zoom to cluster
-        el.addEventListener('click', () => {
+          // Click handler to zoom and navigate to the cluster
+        el.addEventListener('click', (e) => {
           map.getSource('visits').getClusterExpansionZoom(
             props.cluster_id,
             (err, zoom) => {
@@ -344,7 +340,7 @@ <h3 class="my-2">Service Delivery Map</h3>
           );
         });
 
-        return el.firstChild;
+        return el;
     }
 
     function donutSegment(start, end, r, r0, color) {

From 1bbdd8074053a642e10d667a4712049f29d68f5b Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 16:00:48 +0200
Subject: [PATCH 58/96] delete unused click handler

---
 .../templates/reports/dashboard.html          | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 16226039..01ec3aea 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -191,25 +191,6 @@ <h3 class="my-2">Service Delivery Map</h3>
         }
       });
 
-      // inspect a cluster on click
-      map.on('click', 'clusters', (e) => {
-        const features = map.queryRenderedFeatures(e.point, {
-          layers: ['clusters']
-        });
-        const clusterId = features[0].properties.cluster_id;
-        map.getSource('visits').getClusterExpansionZoom(
-          clusterId,
-          (err, zoom) => {
-            if (err) return;
-
-            map.easeTo({
-              center: features[0].geometry.coordinates,
-              zoom: zoom
-            });
-          }
-        );
-      });
-
       // When a click event occurs on a feature in
       // the unclustered-point layer, open a popup at
       // the location of the feature, with

From 3a57fe5b7bc85129311edf288bef133cab8fcaf6 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 16:03:37 +0200
Subject: [PATCH 59/96] format file

---
 .../templates/reports/dashboard.html          | 341 +++++++++---------
 1 file changed, 169 insertions(+), 172 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 01ec3aea..9273d8ab 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -5,7 +5,7 @@
 {% block title %}Admin Dashboard{% endblock %}
 {% block content %}
 <h2 class="my-2">Program Dashboard</h2>
-<div class="container" x-data="dashboardStats()" >
+<div class="container" x-data="dashboardStats()">
   <div class="mb-4">
     <div class="row mb-4" x-ref="filterForm" id="filterForm">
       {% crispy filter.form %}
@@ -140,209 +140,206 @@ <h3 class="my-2">Service Delivery Map</h3>
   window.addEventListener('DOMContentLoaded', () => {
     mapboxgl.accessToken = "{{ mapbox_token }}";
     const cluster = {% if cluster_visits %}true{% else %}false{% endif %};
-    map = new mapboxgl.Map({
-      container: 'map',
-      style: 'mapbox://styles/mapbox/dark-v11',
-      center: [20, 0], // Centered on Africa (roughly central coordinates)
-      zoom: 3,
-    });
+  map = new mapboxgl.Map({
+    container: 'map',
+    style: 'mapbox://styles/mapbox/dark-v11',
+    center: [20, 0], // Centered on Africa (roughly central coordinates)
+    zoom: 3,
+  });
 
-    // filters for classifying visits by status
-    const approved = ['==', ['get', 'status'], 'approved'];
-    const pending = ['all',
-      ['!=', ['get', 'status'], 'approved'],
-      ['!=', ['get', 'status'], 'rejected']
-    ];
-    const rejected = ['==', ['get', 'status'], 'rejected'];
+  // filters for classifying visits by status
+  const approved = ['==', ['get', 'status'], 'approved'];
+  const pending = ['all',
+    ['!=', ['get', 'status'], 'approved'],
+    ['!=', ['get', 'status'], 'rejected']
+  ];
+  const rejected = ['==', ['get', 'status'], 'rejected'];
 
-    // colors to use for the categories
-    const colors = ['#00FF00', '#FFFF00', '#FF0000'];
+  // colors to use for the categories
+  const colors = ['#00FF00', '#FFFF00', '#FF0000'];
 
-    map.on('load', () => {
-      // Modify the source configuration to include initial filters
-      const formElement = document.querySelector('#filterForm form');
-      const formData = new FormData(formElement);
-      const queryString = new URLSearchParams(formData).toString();
+  map.on('load', () => {
+    // Modify the source configuration to include initial filters
+    const formElement = document.querySelector('#filterForm form');
+    const formData = new FormData(formElement);
+    const queryString = new URLSearchParams(formData).toString();
 
-      map.addSource('visits', {
-        type: 'geojson',
-        data: `{% url "reports:visit_map_data" %}?${queryString}`,
-        cluster: cluster,
-        clusterMaxZoom: 12,
-        clusterRadius: 80,
-        clusterProperties: {
-          // keep separate counts for each status category in a cluster
-          'approved': ['+', ['case', approved, 1, 0]],
-          'pending': ['+', ['case', pending, 1, 0]],
-          'rejected': ['+', ['case', rejected, 1, 0]]
-        }
-      });
+    map.addSource('visits', {
+      type: 'geojson',
+      data: `{% url "reports:visit_map_data" %}?${queryString}`,
+      cluster: cluster,
+      clusterMaxZoom: 12,
+      clusterRadius: 80,
+      clusterProperties: {
+        // keep separate counts for each status category in a cluster
+        'approved': ['+', ['case', approved, 1, 0]],
+        'pending': ['+', ['case', pending, 1, 0]],
+        'rejected': ['+', ['case', rejected, 1, 0]]
+      }
+    });
 
-      map.addLayer({
-        id: 'unclustered-point',
-        type: 'circle',
-        source: 'visits',
-        filter: ['!', ['has', 'point_count']],
-        paint: {
-          'circle-color': ['get', 'color'],
-          'circle-radius': 4,
-          'circle-stroke-width': 1,
-          'circle-stroke-color': '#fff'
-        }
-      });
+    map.addLayer({
+      id: 'unclustered-point',
+      type: 'circle',
+      source: 'visits',
+      filter: ['!', ['has', 'point_count']],
+      paint: {
+        'circle-color': ['get', 'color'],
+        'circle-radius': 4,
+        'circle-stroke-width': 1,
+        'circle-stroke-color': '#fff'
+      }
+    });
 
-      // When a click event occurs on a feature in
-      // the unclustered-point layer, open a popup at
-      // the location of the feature, with
-      // description HTML from its properties.
-      map.on('click', 'unclustered-point', (e) => {
-        const coordinates = e.features[0].geometry.coordinates.slice();
-        const status = e.features[0].properties.status;
-        const rawDate = e.features[0].properties.visit_date;
-        const visitDate = new Date(rawDate).toLocaleDateString('en-US', { year: 'numeric', month: 'short', day: 'numeric' });
+    // When a click event occurs on a feature in
+    // the unclustered-point layer, open a popup at
+    // the location of the feature, with
+    // description HTML from its properties.
+    map.on('click', 'unclustered-point', (e) => {
+      const coordinates = e.features[0].geometry.coordinates.slice();
+      const status = e.features[0].properties.status;
+      const rawDate = e.features[0].properties.visit_date;
+      const visitDate = new Date(rawDate).toLocaleDateString('en-US', { year: 'numeric', month: 'short', day: 'numeric' });
 
 
-        // Ensure that if the map is zoomed out such that
-        // multiple copies of the feature are visible, the
-        // popup appears over the copy being pointed to.
-        if (['mercator', 'equirectangular'].includes(map.getProjection().name)) {
-          while (Math.abs(e.lngLat.lng - coordinates[0]) > 180) {
-            coordinates[0] += e.lngLat.lng > coordinates[0] ? 360 : -360;
-          }
+      // Ensure that if the map is zoomed out such that
+      // multiple copies of the feature are visible, the
+      // popup appears over the copy being pointed to.
+      if (['mercator', 'equirectangular'].includes(map.getProjection().name)) {
+        while (Math.abs(e.lngLat.lng - coordinates[0]) > 180) {
+          coordinates[0] += e.lngLat.lng > coordinates[0] ? 360 : -360;
         }
+      }
 
-        new mapboxgl.Popup()
-          .setLngLat(coordinates)
-          .setHTML(
-            `Visit Date: ${visitDate}<br>Status: ${status}`
-          )
-          .addTo(map);
-      });
-
-      // after the GeoJSON data is loaded, update markers on the screen on every frame
-      // objects for caching and keeping track of HTML marker objects (for performance)
-      const markers = {};
-      let markersOnScreen = {};
-      function updateMarkers() {
-        const newMarkers = {};
-        const features = map.querySourceFeatures('visits');
+      new mapboxgl.Popup()
+        .setLngLat(coordinates)
+        .setHTML(
+          `Visit Date: ${visitDate}<br>Status: ${status}`
+        )
+        .addTo(map);
+    });
 
-        // for every cluster on the screen, create an HTML marker for it (if we didn't yet),
-        // and add it to the map if it's not there already
-        for (const feature of features) {
-          const coords = feature.geometry.coordinates;
-          const props = feature.properties;
-          if (!props.cluster) continue;
-          const id = props.cluster_id;
+    // after the GeoJSON data is loaded, update markers on the screen on every frame
+    // objects for caching and keeping track of HTML marker objects (for performance)
+    const markers = {};
+    let markersOnScreen = {};
+    function updateMarkers() {
+      const newMarkers = {};
+      const features = map.querySourceFeatures('visits');
 
-          let marker = markers[id];
-          if (!marker) {
-            const el = createDonutChart({
-              ...props,
-              cluster_id: id,  // Make sure cluster_id is passed
-              coordinates: coords  // Pass the coordinates
-            });
-            marker = markers[id] = new mapboxgl.Marker({
-              element: el
-            }).setLngLat(coords);
-          }
-          newMarkers[id] = marker;
+      // for every cluster on the screen, create an HTML marker for it (if we didn't yet),
+      // and add it to the map if it's not there already
+      for (const feature of features) {
+        const coords = feature.geometry.coordinates;
+        const props = feature.properties;
+        if (!props.cluster) continue;
+        const id = props.cluster_id;
 
-          if (!markersOnScreen[id]) marker.addTo(map);
-        }
-        // for every marker we've added previously, remove those that are no longer visible
-        for (const id in markersOnScreen) {
-          if (!newMarkers[id]) markersOnScreen[id].remove();
+        let marker = markers[id];
+        if (!marker) {
+          const el = createDonutChart({
+            ...props,
+            cluster_id: id,  // Make sure cluster_id is passed
+            coordinates: coords  // Pass the coordinates
+          });
+          marker = markers[id] = new mapboxgl.Marker({
+            element: el
+          }).setLngLat(coords);
         }
-        markersOnScreen = newMarkers;
+        newMarkers[id] = marker;
+
+        if (!markersOnScreen[id]) marker.addTo(map);
+      }
+      // for every marker we've added previously, remove those that are no longer visible
+      for (const id in markersOnScreen) {
+        if (!newMarkers[id]) markersOnScreen[id].remove();
       }
+      markersOnScreen = newMarkers;
+    }
 
-      map.on('render', () => {
-        if (!map.isSourceLoaded('visits')) return;
-        updateMarkers();
-      });
+    map.on('render', () => {
+      if (!map.isSourceLoaded('visits')) return;
+      updateMarkers();
     });
+  });
 
-    // code for creating an SVG donut chart from feature properties
-    function createDonutChart(props) {
-        const offsets = [];
-        const counts = [
-            props.approved,
-            props.pending,
-            props.rejected
-        ];
-        let total = 0;
-        for (const count of counts) {
-            offsets.push(total);
-            total += count;
-        }
-        const fontSize =
-            total >= 1000 ? 22 : total >= 100 ? 20 : total >= 10 ? 18 : 16;
-        const r =
-            total >= 1000 ? 50 : total >= 100 ? 32 : total >= 10 ? 24 : 18;
-        const r0 = Math.round(r * 0.8);
-        const w = r * 2;
+  // code for creating an SVG donut chart from feature properties
+  function createDonutChart(props) {
+    const offsets = [];
+    const counts = [
+      props.approved,
+      props.pending,
+      props.rejected
+    ];
+    let total = 0;
+    for (const count of counts) {
+      offsets.push(total);
+      total += count;
+    }
+    const fontSize =
+      total >= 1000 ? 22 : total >= 100 ? 20 : total >= 10 ? 18 : 16;
+    const r =
+      total >= 1000 ? 50 : total >= 100 ? 32 : total >= 10 ? 24 : 18;
+    const r0 = Math.round(r * 0.8);
+    const w = r * 2;
 
-        let html = `<div>
+    let html = `<div>
             <svg width="${w}" height="${w}" viewbox="0 0 ${w} ${w}" text-anchor="middle" style="font: ${fontSize}px sans-serif; display: block">`;
 
-        for (let i = 0; i < counts.length; i++) {
-            html += donutSegment(
-                offsets[i] / total,
-                (offsets[i] + counts[i]) / total,
-                r,
-                r0,
-                colors[i]
-            );
-        }
-        html += `<circle cx="${r}" cy="${r}" r="${r0}" fill="transparent" />
+    for (let i = 0; i < counts.length; i++) {
+      html += donutSegment(
+        offsets[i] / total,
+        (offsets[i] + counts[i]) / total,
+        r,
+        r0,
+        colors[i]
+      );
+    }
+    html += `<circle cx="${r}" cy="${r}" r="${r0}" fill="transparent" />
             <text dominant-baseline="central" transform="translate(${r}, ${r})" fill="white">
                 ${total.toLocaleString()}
             </text>
             </svg>
             </div>`;
 
-        const el = document.createElement('div');
-        el.innerHTML = html;
-        el.style.cursor = 'pointer';
+    const el = document.createElement('div');
+    el.innerHTML = html;
+    el.style.cursor = 'pointer';
 
-          // Click handler to zoom and navigate to the cluster
-        el.addEventListener('click', (e) => {
-          map.getSource('visits').getClusterExpansionZoom(
-            props.cluster_id,
-            (err, zoom) => {
-              if (err) return;
+    // Click handler to zoom and navigate to the cluster
+    el.addEventListener('click', (e) => {
+      map.getSource('visits').getClusterExpansionZoom(
+        props.cluster_id,
+        (err, zoom) => {
+          if (err) return;
 
-              map.easeTo({
-                center: props.coordinates,
-                zoom: zoom
-              });
-            }
-          );
-        });
+          map.easeTo({
+            center: props.coordinates,
+            zoom: zoom
+          });
+        }
+      );
+    });
 
-        return el;
-    }
+    return el;
+  }
 
-    function donutSegment(start, end, r, r0, color) {
-        if (end - start === 1) end -= 0.00001;
-        const a0 = 2 * Math.PI * (start - 0.25);
-        const a1 = 2 * Math.PI * (end - 0.25);
-        const x0 = Math.cos(a0),
-            y0 = Math.sin(a0);
-        const x1 = Math.cos(a1),
-            y1 = Math.sin(a1);
-        const largeArc = end - start > 0.5 ? 1 : 0;
+  function donutSegment(start, end, r, r0, color) {
+    if (end - start === 1) end -= 0.00001;
+    const a0 = 2 * Math.PI * (start - 0.25);
+    const a1 = 2 * Math.PI * (end - 0.25);
+    const x0 = Math.cos(a0),
+      y0 = Math.sin(a0);
+    const x1 = Math.cos(a1),
+      y1 = Math.sin(a1);
+    const largeArc = end - start > 0.5 ? 1 : 0;
 
-        // draw an SVG path
-        return `<path d="M ${r + r0 * x0} ${r + r0 * y0} L ${r + r * x0} ${
-            r + r * y0
-        } A ${r} ${r} 0 ${largeArc} 1 ${r + r * x1} ${r + r * y1} L ${
-            r + r0 * x1
-        } ${r + r0 * y1} A ${r0} ${r0} 0 ${largeArc} 0 ${r + r0 * x0} ${
-            r + r0 * y0
-        }" fill="${color}" />`;
-    }
+    // draw an SVG path
+    return `<path d="M ${r + r0 * x0} ${r + r0 * y0} L ${r + r * x0} ${r + r * y0
+      } A ${r} ${r} 0 ${largeArc} 1 ${r + r * x1} ${r + r * y1} L ${r + r0 * x1
+      } ${r + r0 * y1} A ${r0} ${r0} 0 ${largeArc} 0 ${r + r0 * x0} ${r + r0 * y0
+      }" fill="${color}" />`;
+  }
   });
 
 </script>

From 9ccadbbad2f69830910bb3fca569b48a61d561b8 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 16:07:07 +0200
Subject: [PATCH 60/96] wip: loading state

---
 .../templates/reports/dashboard.html          | 26 +++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 9273d8ab..26d1b1b5 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -73,7 +73,14 @@ <h2 class="my-2">Program Dashboard</h2>
 <div class="row">
   <div class="col-12">
     <h3 class="my-2">Service Delivery Map</h3>
-    <div id="map" class="mt-3 mb-2 rounded border" style="height: 900px;"></div>
+    <div class="position-relative">
+      <div id="map" class="mt-3 mb-2 rounded border" style="height: 900px;"></div>
+      <div id="map-loading" class="position-absolute top-0 start-0 w-100 h-100 bg-dark bg-opacity-50 d-flex justify-content-center align-items-center">
+        <div class="spinner-border text-light" role="status">
+          <span class="visually-hidden">Loading map...</span>
+        </div>
+      </div>
+    </div>
   </div>
 </div>
 {% endblock content %}
@@ -124,17 +131,23 @@ <h3 class="my-2">Service Delivery Map</h3>
 </script>
 <script type="module">
   let map;
+  const mapLoading = document.getElementById('map-loading');
 
   // Add this function to handle map data refresh
   window.refreshMapData = () => {
     if (!map) return;
+    mapLoading.classList.remove('d-none');  // Show loading state
 
     const formElement = document.querySelector('#filterForm form');
     const formData = new FormData(formElement);
     const queryString = new URLSearchParams(formData).toString();
 
     // Update the source data with new filters
-    map.getSource('visits').setData(`{% url "reports:visit_map_data" %}?${queryString}`);
+    map.getSource('visits').setData(
+      `{% url "reports:visit_map_data" %}?${queryString}`,
+      // Hide loading state after data is loaded
+      () => mapLoading.classList.add('d-none')
+    );
   };
 
   window.addEventListener('DOMContentLoaded', () => {
@@ -261,6 +274,15 @@ <h3 class="my-2">Service Delivery Map</h3>
       if (!map.isSourceLoaded('visits')) return;
       updateMarkers();
     });
+
+    // Hide loading overlay when initial map load is complete
+    mapLoading.classList.add('d-none');
+  });
+
+  // Show loading state if map errors
+  map.on('error', () => {
+    mapLoading.classList.add('d-none');
+    // Optionally add error messaging here
   });
 
   // code for creating an SVG donut chart from feature properties

From 447bd629855173a4bf745d0baf1f653cba74b2be Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 16:19:00 +0200
Subject: [PATCH 61/96] debug statements

---
 commcare_connect/templates/reports/dashboard.html | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 26d1b1b5..c71c0b6d 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -136,7 +136,8 @@ <h3 class="my-2">Service Delivery Map</h3>
   // Add this function to handle map data refresh
   window.refreshMapData = () => {
     if (!map) return;
-    mapLoading.classList.remove('d-none');  // Show loading state
+    console.log('refreshing map data');
+    // mapLoading.classList.remove('d-none');  // Show loading state
 
     const formElement = document.querySelector('#filterForm form');
     const formData = new FormData(formElement);
@@ -145,8 +146,11 @@ <h3 class="my-2">Service Delivery Map</h3>
     // Update the source data with new filters
     map.getSource('visits').setData(
       `{% url "reports:visit_map_data" %}?${queryString}`,
-      // Hide loading state after data is loaded
-      () => mapLoading.classList.add('d-none')
+      () => {
+        console.log('setting loading to none after update');
+        mapLoading.classList.add('d-none');  // Hide loading state after data is loaded
+        console.log('setting loading to none after update');
+      }
     );
   };
 
@@ -276,11 +280,13 @@ <h3 class="my-2">Service Delivery Map</h3>
     });
 
     // Hide loading overlay when initial map load is complete
+    console.log('setting loading to none')
     mapLoading.classList.add('d-none');
   });
 
   // Show loading state if map errors
   map.on('error', () => {
+    console.log('setting loading to none for an error')
     mapLoading.classList.add('d-none');
     // Optionally add error messaging here
   });

From f3026d3bd66dda70702cbeaf2b4ecf239645ae7e Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 16:27:33 +0200
Subject: [PATCH 62/96] improve set data / loading issues

---
 .../templates/reports/dashboard.html          | 27 +++++++++++--------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index c71c0b6d..48a63da7 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -134,24 +134,29 @@ <h3 class="my-2">Service Delivery Map</h3>
   const mapLoading = document.getElementById('map-loading');
 
   // Add this function to handle map data refresh
-  window.refreshMapData = () => {
+  window.refreshMapData = async () => {
     if (!map) return;
     console.log('refreshing map data');
-    // mapLoading.classList.remove('d-none');  // Show loading state
+    mapLoading.classList.remove('d-none');  // Show loading state
 
     const formElement = document.querySelector('#filterForm form');
     const formData = new FormData(formElement);
     const queryString = new URLSearchParams(formData).toString();
 
-    // Update the source data with new filters
-    map.getSource('visits').setData(
-      `{% url "reports:visit_map_data" %}?${queryString}`,
-      () => {
-        console.log('setting loading to none after update');
-        mapLoading.classList.add('d-none');  // Hide loading state after data is loaded
-        console.log('setting loading to none after update');
-      }
-    );
+    try {
+      // Fetch the data first
+      const response = await fetch(`{% url "reports:visit_map_data" %}?${queryString}`);
+      const data = await response.json();
+
+      // Then set it on the map source
+      map.getSource('visits').setData(data);
+
+      // Hide loading state after data is set
+      mapLoading.classList.add('d-none');
+    } catch (error) {
+      console.error('Error refreshing map data:', error);
+      mapLoading.classList.add('d-none');
+    }
   };
 
   window.addEventListener('DOMContentLoaded', () => {

From 845580492a118ba73efa9578f960d52b32c34fe0 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 16:35:40 +0200
Subject: [PATCH 63/96] style tweaks

---
 .../templates/reports/dashboard.html             | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 48a63da7..c5d5fa71 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -178,7 +178,7 @@ <h3 class="my-2">Service Delivery Map</h3>
   const rejected = ['==', ['get', 'status'], 'rejected'];
 
   // colors to use for the categories
-  const colors = ['#00FF00', '#FFFF00', '#FF0000'];
+  const colors = ['#4ade80', '#fbbf24', '#f87171']; // softer green, yellow, red
 
   map.on('load', () => {
     // Modify the source configuration to include initial filters
@@ -317,7 +317,12 @@ <h3 class="my-2">Service Delivery Map</h3>
     const w = r * 2;
 
     let html = `<div>
-            <svg width="${w}" height="${w}" viewbox="0 0 ${w} ${w}" text-anchor="middle" style="font: ${fontSize}px sans-serif; display: block">`;
+            <svg width="${w}" height="${w}" viewbox="0 0 ${w} ${w}" text-anchor="middle" style="font: ${fontSize}px sans-serif; display: block">
+            <defs>
+              <filter id="shadow">
+                <feDropShadow dx="0" dy="1" stdDeviation="2" flood-opacity="0.3"/>
+              </filter>
+            </defs>`;
 
     for (let i = 0; i < counts.length; i++) {
       html += donutSegment(
@@ -328,8 +333,9 @@ <h3 class="my-2">Service Delivery Map</h3>
         colors[i]
       );
     }
-    html += `<circle cx="${r}" cy="${r}" r="${r0}" fill="transparent" />
-            <text dominant-baseline="central" transform="translate(${r}, ${r})" fill="white">
+    html += `<circle cx="${r}" cy="${r}" r="${r0}" fill="#374151" />
+            <text dominant-baseline="central" transform="translate(${r}, ${r})"
+                  fill="white" font-weight="500" filter="url(#shadow)">
                 ${total.toLocaleString()}
             </text>
             </svg>
@@ -371,7 +377,7 @@ <h3 class="my-2">Service Delivery Map</h3>
     return `<path d="M ${r + r0 * x0} ${r + r0 * y0} L ${r + r * x0} ${r + r * y0
       } A ${r} ${r} 0 ${largeArc} 1 ${r + r * x1} ${r + r * y1} L ${r + r0 * x1
       } ${r + r0 * y1} A ${r0} ${r0} 0 ${largeArc} 0 ${r + r0 * x0} ${r + r0 * y0
-      }" fill="${color}" />`;
+      }" fill="${color}" opacity="0.85" stroke="#1f2937" stroke-width="1" />`;
   }
   });
 

From 4bae7e3c06c97cfc0ffe906a135023c6fc7e4efe Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 16:46:26 +0200
Subject: [PATCH 64/96] remove cluster argument

---
 commcare_connect/reports/views.py                 | 2 --
 commcare_connect/templates/reports/dashboard.html | 3 +--
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index 6c602801..7c0305b0 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -223,13 +223,11 @@ class Meta:
 @user_passes_test(lambda u: u.is_superuser)
 def program_dashboard_report(request):
     filterset = DashboardFilters(request.GET)
-    cluster = request.GET.get("cluster", "true") == "true"
     return render(
         request,
         "reports/dashboard.html",
         context={
             "mapbox_token": settings.MAPBOX_TOKEN,
-            "cluster_visits": cluster,
             "filter": filterset,
         },
     )
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index c5d5fa71..0e97e0fa 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -161,7 +161,6 @@ <h3 class="my-2">Service Delivery Map</h3>
 
   window.addEventListener('DOMContentLoaded', () => {
     mapboxgl.accessToken = "{{ mapbox_token }}";
-    const cluster = {% if cluster_visits %}true{% else %}false{% endif %};
   map = new mapboxgl.Map({
     container: 'map',
     style: 'mapbox://styles/mapbox/dark-v11',
@@ -189,7 +188,7 @@ <h3 class="my-2">Service Delivery Map</h3>
     map.addSource('visits', {
       type: 'geojson',
       data: `{% url "reports:visit_map_data" %}?${queryString}`,
-      cluster: cluster,
+      cluster: true,
       clusterMaxZoom: 12,
       clusterRadius: 80,
       clusterProperties: {

From 882c2aa22461c12180cbdc0c8b86015b3267bcd6 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 16:46:36 +0200
Subject: [PATCH 65/96] fix formatting

---
 .../templates/reports/dashboard.html          | 383 +++++++++---------
 1 file changed, 192 insertions(+), 191 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 0e97e0fa..eddc7a9f 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -75,7 +75,8 @@ <h2 class="my-2">Program Dashboard</h2>
     <h3 class="my-2">Service Delivery Map</h3>
     <div class="position-relative">
       <div id="map" class="mt-3 mb-2 rounded border" style="height: 900px;"></div>
-      <div id="map-loading" class="position-absolute top-0 start-0 w-100 h-100 bg-dark bg-opacity-50 d-flex justify-content-center align-items-center">
+      <div id="map-loading"
+        class="position-absolute top-0 start-0 w-100 h-100 bg-dark bg-opacity-50 d-flex justify-content-center align-items-center">
         <div class="spinner-border text-light" role="status">
           <span class="visually-hidden">Loading map...</span>
         </div>
@@ -161,161 +162,161 @@ <h3 class="my-2">Service Delivery Map</h3>
 
   window.addEventListener('DOMContentLoaded', () => {
     mapboxgl.accessToken = "{{ mapbox_token }}";
-  map = new mapboxgl.Map({
-    container: 'map',
-    style: 'mapbox://styles/mapbox/dark-v11',
-    center: [20, 0], // Centered on Africa (roughly central coordinates)
-    zoom: 3,
-  });
-
-  // filters for classifying visits by status
-  const approved = ['==', ['get', 'status'], 'approved'];
-  const pending = ['all',
-    ['!=', ['get', 'status'], 'approved'],
-    ['!=', ['get', 'status'], 'rejected']
-  ];
-  const rejected = ['==', ['get', 'status'], 'rejected'];
-
-  // colors to use for the categories
-  const colors = ['#4ade80', '#fbbf24', '#f87171']; // softer green, yellow, red
-
-  map.on('load', () => {
-    // Modify the source configuration to include initial filters
-    const formElement = document.querySelector('#filterForm form');
-    const formData = new FormData(formElement);
-    const queryString = new URLSearchParams(formData).toString();
-
-    map.addSource('visits', {
-      type: 'geojson',
-      data: `{% url "reports:visit_map_data" %}?${queryString}`,
-      cluster: true,
-      clusterMaxZoom: 12,
-      clusterRadius: 80,
-      clusterProperties: {
-        // keep separate counts for each status category in a cluster
-        'approved': ['+', ['case', approved, 1, 0]],
-        'pending': ['+', ['case', pending, 1, 0]],
-        'rejected': ['+', ['case', rejected, 1, 0]]
-      }
-    });
-
-    map.addLayer({
-      id: 'unclustered-point',
-      type: 'circle',
-      source: 'visits',
-      filter: ['!', ['has', 'point_count']],
-      paint: {
-        'circle-color': ['get', 'color'],
-        'circle-radius': 4,
-        'circle-stroke-width': 1,
-        'circle-stroke-color': '#fff'
-      }
+    map = new mapboxgl.Map({
+      container: 'map',
+      style: 'mapbox://styles/mapbox/dark-v11',
+      center: [20, 0], // Centered on Africa (roughly central coordinates)
+      zoom: 3,
     });
 
-    // When a click event occurs on a feature in
-    // the unclustered-point layer, open a popup at
-    // the location of the feature, with
-    // description HTML from its properties.
-    map.on('click', 'unclustered-point', (e) => {
-      const coordinates = e.features[0].geometry.coordinates.slice();
-      const status = e.features[0].properties.status;
-      const rawDate = e.features[0].properties.visit_date;
-      const visitDate = new Date(rawDate).toLocaleDateString('en-US', { year: 'numeric', month: 'short', day: 'numeric' });
-
-
-      // Ensure that if the map is zoomed out such that
-      // multiple copies of the feature are visible, the
-      // popup appears over the copy being pointed to.
-      if (['mercator', 'equirectangular'].includes(map.getProjection().name)) {
-        while (Math.abs(e.lngLat.lng - coordinates[0]) > 180) {
-          coordinates[0] += e.lngLat.lng > coordinates[0] ? 360 : -360;
+    // filters for classifying visits by status
+    const approved = ['==', ['get', 'status'], 'approved'];
+    const pending = ['all',
+      ['!=', ['get', 'status'], 'approved'],
+      ['!=', ['get', 'status'], 'rejected']
+    ];
+    const rejected = ['==', ['get', 'status'], 'rejected'];
+
+    // colors to use for the categories
+    const colors = ['#4ade80', '#fbbf24', '#f87171']; // softer green, yellow, red
+
+    map.on('load', () => {
+      // Modify the source configuration to include initial filters
+      const formElement = document.querySelector('#filterForm form');
+      const formData = new FormData(formElement);
+      const queryString = new URLSearchParams(formData).toString();
+
+      map.addSource('visits', {
+        type: 'geojson',
+        data: `{% url "reports:visit_map_data" %}?${queryString}`,
+        cluster: true,
+        clusterMaxZoom: 12,
+        clusterRadius: 80,
+        clusterProperties: {
+          // keep separate counts for each status category in a cluster
+          'approved': ['+', ['case', approved, 1, 0]],
+          'pending': ['+', ['case', pending, 1, 0]],
+          'rejected': ['+', ['case', rejected, 1, 0]]
+        }
+      });
+
+      map.addLayer({
+        id: 'unclustered-point',
+        type: 'circle',
+        source: 'visits',
+        filter: ['!', ['has', 'point_count']],
+        paint: {
+          'circle-color': ['get', 'color'],
+          'circle-radius': 4,
+          'circle-stroke-width': 1,
+          'circle-stroke-color': '#fff'
+        }
+      });
+
+      // When a click event occurs on a feature in
+      // the unclustered-point layer, open a popup at
+      // the location of the feature, with
+      // description HTML from its properties.
+      map.on('click', 'unclustered-point', (e) => {
+        const coordinates = e.features[0].geometry.coordinates.slice();
+        const status = e.features[0].properties.status;
+        const rawDate = e.features[0].properties.visit_date;
+        const visitDate = new Date(rawDate).toLocaleDateString('en-US', { year: 'numeric', month: 'short', day: 'numeric' });
+
+
+        // Ensure that if the map is zoomed out such that
+        // multiple copies of the feature are visible, the
+        // popup appears over the copy being pointed to.
+        if (['mercator', 'equirectangular'].includes(map.getProjection().name)) {
+          while (Math.abs(e.lngLat.lng - coordinates[0]) > 180) {
+            coordinates[0] += e.lngLat.lng > coordinates[0] ? 360 : -360;
+          }
         }
-      }
 
-      new mapboxgl.Popup()
-        .setLngLat(coordinates)
-        .setHTML(
-          `Visit Date: ${visitDate}<br>Status: ${status}`
-        )
-        .addTo(map);
-    });
+        new mapboxgl.Popup()
+          .setLngLat(coordinates)
+          .setHTML(
+            `Visit Date: ${visitDate}<br>Status: ${status}`
+          )
+          .addTo(map);
+      });
+
+      // after the GeoJSON data is loaded, update markers on the screen on every frame
+      // objects for caching and keeping track of HTML marker objects (for performance)
+      const markers = {};
+      let markersOnScreen = {};
+      function updateMarkers() {
+        const newMarkers = {};
+        const features = map.querySourceFeatures('visits');
+
+        // for every cluster on the screen, create an HTML marker for it (if we didn't yet),
+        // and add it to the map if it's not there already
+        for (const feature of features) {
+          const coords = feature.geometry.coordinates;
+          const props = feature.properties;
+          if (!props.cluster) continue;
+          const id = props.cluster_id;
+
+          let marker = markers[id];
+          if (!marker) {
+            const el = createDonutChart({
+              ...props,
+              cluster_id: id,  // Make sure cluster_id is passed
+              coordinates: coords  // Pass the coordinates
+            });
+            marker = markers[id] = new mapboxgl.Marker({
+              element: el
+            }).setLngLat(coords);
+          }
+          newMarkers[id] = marker;
 
-    // after the GeoJSON data is loaded, update markers on the screen on every frame
-    // objects for caching and keeping track of HTML marker objects (for performance)
-    const markers = {};
-    let markersOnScreen = {};
-    function updateMarkers() {
-      const newMarkers = {};
-      const features = map.querySourceFeatures('visits');
-
-      // for every cluster on the screen, create an HTML marker for it (if we didn't yet),
-      // and add it to the map if it's not there already
-      for (const feature of features) {
-        const coords = feature.geometry.coordinates;
-        const props = feature.properties;
-        if (!props.cluster) continue;
-        const id = props.cluster_id;
-
-        let marker = markers[id];
-        if (!marker) {
-          const el = createDonutChart({
-            ...props,
-            cluster_id: id,  // Make sure cluster_id is passed
-            coordinates: coords  // Pass the coordinates
-          });
-          marker = markers[id] = new mapboxgl.Marker({
-            element: el
-          }).setLngLat(coords);
+          if (!markersOnScreen[id]) marker.addTo(map);
         }
-        newMarkers[id] = marker;
-
-        if (!markersOnScreen[id]) marker.addTo(map);
-      }
-      // for every marker we've added previously, remove those that are no longer visible
-      for (const id in markersOnScreen) {
-        if (!newMarkers[id]) markersOnScreen[id].remove();
+        // for every marker we've added previously, remove those that are no longer visible
+        for (const id in markersOnScreen) {
+          if (!newMarkers[id]) markersOnScreen[id].remove();
+        }
+        markersOnScreen = newMarkers;
       }
-      markersOnScreen = newMarkers;
-    }
 
-    map.on('render', () => {
-      if (!map.isSourceLoaded('visits')) return;
-      updateMarkers();
-    });
+      map.on('render', () => {
+        if (!map.isSourceLoaded('visits')) return;
+        updateMarkers();
+      });
 
-    // Hide loading overlay when initial map load is complete
-    console.log('setting loading to none')
-    mapLoading.classList.add('d-none');
-  });
+      // Hide loading overlay when initial map load is complete
+      console.log('setting loading to none')
+      mapLoading.classList.add('d-none');
+    });
 
-  // Show loading state if map errors
-  map.on('error', () => {
-    console.log('setting loading to none for an error')
-    mapLoading.classList.add('d-none');
-    // Optionally add error messaging here
-  });
+    // Show loading state if map errors
+    map.on('error', () => {
+      console.log('setting loading to none for an error')
+      mapLoading.classList.add('d-none');
+      // Optionally add error messaging here
+    });
 
-  // code for creating an SVG donut chart from feature properties
-  function createDonutChart(props) {
-    const offsets = [];
-    const counts = [
-      props.approved,
-      props.pending,
-      props.rejected
-    ];
-    let total = 0;
-    for (const count of counts) {
-      offsets.push(total);
-      total += count;
-    }
-    const fontSize =
-      total >= 1000 ? 22 : total >= 100 ? 20 : total >= 10 ? 18 : 16;
-    const r =
-      total >= 1000 ? 50 : total >= 100 ? 32 : total >= 10 ? 24 : 18;
-    const r0 = Math.round(r * 0.8);
-    const w = r * 2;
-
-    let html = `<div>
+    // code for creating an SVG donut chart from feature properties
+    function createDonutChart(props) {
+      const offsets = [];
+      const counts = [
+        props.approved,
+        props.pending,
+        props.rejected
+      ];
+      let total = 0;
+      for (const count of counts) {
+        offsets.push(total);
+        total += count;
+      }
+      const fontSize =
+        total >= 1000 ? 22 : total >= 100 ? 20 : total >= 10 ? 18 : 16;
+      const r =
+        total >= 1000 ? 50 : total >= 100 ? 32 : total >= 10 ? 24 : 18;
+      const r0 = Math.round(r * 0.8);
+      const w = r * 2;
+
+      let html = `<div>
             <svg width="${w}" height="${w}" viewbox="0 0 ${w} ${w}" text-anchor="middle" style="font: ${fontSize}px sans-serif; display: block">
             <defs>
               <filter id="shadow">
@@ -323,16 +324,16 @@ <h3 class="my-2">Service Delivery Map</h3>
               </filter>
             </defs>`;
 
-    for (let i = 0; i < counts.length; i++) {
-      html += donutSegment(
-        offsets[i] / total,
-        (offsets[i] + counts[i]) / total,
-        r,
-        r0,
-        colors[i]
-      );
-    }
-    html += `<circle cx="${r}" cy="${r}" r="${r0}" fill="#374151" />
+      for (let i = 0; i < counts.length; i++) {
+        html += donutSegment(
+          offsets[i] / total,
+          (offsets[i] + counts[i]) / total,
+          r,
+          r0,
+          colors[i]
+        );
+      }
+      html += `<circle cx="${r}" cy="${r}" r="${r0}" fill="#374151" />
             <text dominant-baseline="central" transform="translate(${r}, ${r})"
                   fill="white" font-weight="500" filter="url(#shadow)">
                 ${total.toLocaleString()}
@@ -340,44 +341,44 @@ <h3 class="my-2">Service Delivery Map</h3>
             </svg>
             </div>`;
 
-    const el = document.createElement('div');
-    el.innerHTML = html;
-    el.style.cursor = 'pointer';
-
-    // Click handler to zoom and navigate to the cluster
-    el.addEventListener('click', (e) => {
-      map.getSource('visits').getClusterExpansionZoom(
-        props.cluster_id,
-        (err, zoom) => {
-          if (err) return;
-
-          map.easeTo({
-            center: props.coordinates,
-            zoom: zoom
-          });
-        }
-      );
-    });
+      const el = document.createElement('div');
+      el.innerHTML = html;
+      el.style.cursor = 'pointer';
+
+      // Click handler to zoom and navigate to the cluster
+      el.addEventListener('click', (e) => {
+        map.getSource('visits').getClusterExpansionZoom(
+          props.cluster_id,
+          (err, zoom) => {
+            if (err) return;
+
+            map.easeTo({
+              center: props.coordinates,
+              zoom: zoom
+            });
+          }
+        );
+      });
 
-    return el;
-  }
+      return el;
+    }
 
-  function donutSegment(start, end, r, r0, color) {
-    if (end - start === 1) end -= 0.00001;
-    const a0 = 2 * Math.PI * (start - 0.25);
-    const a1 = 2 * Math.PI * (end - 0.25);
-    const x0 = Math.cos(a0),
-      y0 = Math.sin(a0);
-    const x1 = Math.cos(a1),
-      y1 = Math.sin(a1);
-    const largeArc = end - start > 0.5 ? 1 : 0;
-
-    // draw an SVG path
-    return `<path d="M ${r + r0 * x0} ${r + r0 * y0} L ${r + r * x0} ${r + r * y0
-      } A ${r} ${r} 0 ${largeArc} 1 ${r + r * x1} ${r + r * y1} L ${r + r0 * x1
-      } ${r + r0 * y1} A ${r0} ${r0} 0 ${largeArc} 0 ${r + r0 * x0} ${r + r0 * y0
-      }" fill="${color}" opacity="0.85" stroke="#1f2937" stroke-width="1" />`;
-  }
+    function donutSegment(start, end, r, r0, color) {
+      if (end - start === 1) end -= 0.00001;
+      const a0 = 2 * Math.PI * (start - 0.25);
+      const a1 = 2 * Math.PI * (end - 0.25);
+      const x0 = Math.cos(a0),
+        y0 = Math.sin(a0);
+      const x1 = Math.cos(a1),
+        y1 = Math.sin(a1);
+      const largeArc = end - start > 0.5 ? 1 : 0;
+
+      // draw an SVG path
+      return `<path d="M ${r + r0 * x0} ${r + r0 * y0} L ${r + r * x0} ${r + r * y0
+        } A ${r} ${r} 0 ${largeArc} 1 ${r + r * x1} ${r + r * y1} L ${r + r0 * x1
+        } ${r + r0 * y1} A ${r0} ${r0} 0 ${largeArc} 0 ${r + r0 * x0} ${r + r0 * y0
+        }" fill="${color}" opacity="0.85" stroke="#1f2937" stroke-width="1" />`;
+    }
   });
 
 </script>

From cb1d7291bce64f34767bba6df002ee8a2777a61f Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 16:52:35 +0200
Subject: [PATCH 66/96] consistent colors

---
 commcare_connect/reports/views.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index 7c0305b0..9a8dc980 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -257,8 +257,8 @@ def visit_map_data(request):
 def _results_to_geojson(results):
     geojson = {"type": "FeatureCollection", "features": []}
     status_to_color = {
-        "approved": "#00FF00",
-        "rejected": "#FF0000",
+        "approved": "#4ade80",
+        "rejected": "#f87171",
     }
     for i, result in enumerate(results.all()):
         location_str = result.get("location_str")
@@ -286,7 +286,7 @@ def _results_to_geojson(results):
                     key: value for key, value in result.items() if key not in ["gps_location_lat", "gps_location_long"]
                 },
             }
-            color = status_to_color.get(result.get("status", ""), "#FFFF00")
+            color = status_to_color.get(result.get("status", ""), "#fbbf24")
             feature["properties"]["color"] = color
             geojson["features"].append(feature)
 

From 08a6f312e1e66e8bd5f455b4433749fd78764152 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 16:52:41 +0200
Subject: [PATCH 67/96] better comments

---
 .../templates/reports/dashboard.html             | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index eddc7a9f..d4b7ee71 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -134,10 +134,9 @@ <h3 class="my-2">Service Delivery Map</h3>
   let map;
   const mapLoading = document.getElementById('map-loading');
 
-  // Add this function to handle map data refresh
   window.refreshMapData = async () => {
     if (!map) return;
-    console.log('refreshing map data');
+
     mapLoading.classList.remove('d-none');  // Show loading state
 
     const formElement = document.querySelector('#filterForm form');
@@ -145,11 +144,11 @@ <h3 class="my-2">Service Delivery Map</h3>
     const queryString = new URLSearchParams(formData).toString();
 
     try {
-      // Fetch the data first
+      // Fetch the data
       const response = await fetch(`{% url "reports:visit_map_data" %}?${queryString}`);
       const data = await response.json();
 
-      // Then set it on the map source
+      // Set it on the map source
       map.getSource('visits').setData(data);
 
       // Hide loading state after data is set
@@ -169,6 +168,12 @@ <h3 class="my-2">Service Delivery Map</h3>
       zoom: 3,
     });
 
+    // This loads a map with two layers, one is a cluster with donut
+    // charts based on the visit data, inspired heavily by this mapbox example:
+    // https://docs.mapbox.com/mapbox-gl-js/example/cluster-html/
+    // The other is a layer of unclustered points with clickable popups, based on
+    // this example: https://docs.mapbox.com/mapbox-gl-js/example/popup-on-click/
+
     // filters for classifying visits by status
     const approved = ['==', ['get', 'status'], 'approved'];
     const pending = ['all',
@@ -178,7 +183,8 @@ <h3 class="my-2">Service Delivery Map</h3>
     const rejected = ['==', ['get', 'status'], 'rejected'];
 
     // colors to use for the categories
-    const colors = ['#4ade80', '#fbbf24', '#f87171']; // softer green, yellow, red
+    // soft green, yellow, red
+    const colors = ['#4ade80', '#fbbf24', '#f87171'];
 
     map.on('load', () => {
       // Modify the source configuration to include initial filters

From 6b1868561182e7a0bbe7f2a4d333a518a610e22d Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 16:54:42 +0200
Subject: [PATCH 68/96] default to only 30 days of data

---
 commcare_connect/reports/views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index 9a8dc980..7e25f92b 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -204,7 +204,7 @@ def __init__(self, *args, **kwargs):
 
             # Set default dates
             today = date.today()
-            default_from = today - timedelta(days=90)
+            default_from = today - timedelta(days=30)
 
             # Set the default values
             self.data["to_date"] = today.strftime("%Y-%m-%d")

From b164959d835e22e803fb2de117fe31d5d4e53e9c Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 16:54:46 +0200
Subject: [PATCH 69/96] whitespace

---
 commcare_connect/templates/reports/dashboard.html | 1 +
 1 file changed, 1 insertion(+)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index d4b7ee71..3a0e1806 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -251,6 +251,7 @@ <h3 class="my-2">Service Delivery Map</h3>
       // objects for caching and keeping track of HTML marker objects (for performance)
       const markers = {};
       let markersOnScreen = {};
+
       function updateMarkers() {
         const newMarkers = {};
         const features = map.querySourceFeatures('visits');

From 2d8cd8f92225744786bd2fef276940014076d873 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 17:20:26 +0200
Subject: [PATCH 70/96] add minimum cluster size

---
 commcare_connect/templates/reports/dashboard.html | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 3a0e1806..7ab9879e 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -132,6 +132,7 @@ <h3 class="my-2">Service Delivery Map</h3>
 </script>
 <script type="module">
   let map;
+  const minClusterSize = 5;
   const mapLoading = document.getElementById('map-loading');
 
   window.refreshMapData = async () => {
@@ -198,6 +199,7 @@ <h3 class="my-2">Service Delivery Map</h3>
         cluster: true,
         clusterMaxZoom: 12,
         clusterRadius: 80,
+        clusterMinPoints: minClusterSize,
         clusterProperties: {
           // keep separate counts for each status category in a cluster
           'approved': ['+', ['case', approved, 1, 0]],

From 8896d5f3b04f42d109b9d3df9004ad42a863b3ce Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 17:23:34 +0200
Subject: [PATCH 71/96] increase maxZoom

---
 commcare_connect/templates/reports/dashboard.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 7ab9879e..1f0ca0f8 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -197,7 +197,7 @@ <h3 class="my-2">Service Delivery Map</h3>
         type: 'geojson',
         data: `{% url "reports:visit_map_data" %}?${queryString}`,
         cluster: true,
-        clusterMaxZoom: 12,
+        clusterMaxZoom: 14,
         clusterRadius: 80,
         clusterMinPoints: minClusterSize,
         clusterProperties: {

From 91f0f0bb93ea7009e479abcb88ae6c3c67806a86 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 17:49:42 +0200
Subject: [PATCH 72/96] extract some functions to external javascript

---
 commcare_connect/static/js/dashboard.js       | 90 ++++++++++++++++++
 .../templates/reports/dashboard.html          | 94 ++-----------------
 webpack/base.config.js                        |  4 +
 3 files changed, 100 insertions(+), 88 deletions(-)
 create mode 100644 commcare_connect/static/js/dashboard.js

diff --git a/commcare_connect/static/js/dashboard.js b/commcare_connect/static/js/dashboard.js
new file mode 100644
index 00000000..c1d8557f
--- /dev/null
+++ b/commcare_connect/static/js/dashboard.js
@@ -0,0 +1,90 @@
+console.log('dashboard.js loaded');
+
+// colors to use for the categories
+// soft green, yellow, red
+const visitColors = ['#4ade80', '#fbbf24', '#f87171'];
+
+// Function to create a donut chart
+function createDonutChart(props, map) {
+  console.log('createDonutChart', props);
+  const offsets = [];
+  const counts = [props.approved, props.pending, props.rejected];
+  let total = 0;
+  for (const count of counts) {
+    offsets.push(total);
+    total += count;
+  }
+  const fontSize =
+    total >= 1000 ? 22 : total >= 100 ? 20 : total >= 10 ? 18 : 16;
+  const r = total >= 1000 ? 50 : total >= 100 ? 32 : total >= 10 ? 24 : 18;
+  const r0 = Math.round(r * 0.8);
+  const w = r * 2;
+
+  let html = `<div>
+        <svg width="${w}" height="${w}" viewbox="0 0 ${w} ${w}" text-anchor="middle" style="font: ${fontSize}px sans-serif; display: block">
+        <defs>
+          <filter id="shadow">
+            <feDropShadow dx="0" dy="1" stdDeviation="2" flood-opacity="0.3"/>
+          </filter>
+        </defs>`;
+
+  for (let i = 0; i < counts.length; i++) {
+    html += donutSegment(
+      offsets[i] / total,
+      (offsets[i] + counts[i]) / total,
+      r,
+      r0,
+      visitColors[i],
+    );
+  }
+  html += `<circle cx="${r}" cy="${r}" r="${r0}" fill="#374151" />
+        <text dominant-baseline="central" transform="translate(${r}, ${r})"
+              fill="white" font-weight="500" filter="url(#shadow)">
+            ${total.toLocaleString()}
+        </text>
+        </svg>
+        </div>`;
+
+  const el = document.createElement('div');
+  el.innerHTML = html;
+  el.style.cursor = 'pointer';
+
+  // Click handler to zoom and navigate to the cluster
+  el.addEventListener('click', (e) => {
+    map
+      .getSource('visits')
+      .getClusterExpansionZoom(props.cluster_id, (err, zoom) => {
+        if (err) return;
+
+        map.easeTo({
+          center: props.coordinates,
+          zoom: zoom,
+        });
+      });
+  });
+
+  return el;
+}
+
+// Function to create a donut segment
+function donutSegment(start, end, r, r0, color) {
+  if (end - start === 1) end -= 0.00001;
+  const a0 = 2 * Math.PI * (start - 0.25);
+  const a1 = 2 * Math.PI * (end - 0.25);
+  const x0 = Math.cos(a0),
+    y0 = Math.sin(a0);
+  const x1 = Math.cos(a1),
+    y1 = Math.sin(a1);
+  const largeArc = end - start > 0.5 ? 1 : 0;
+
+  // draw an SVG path
+  return `<path d="M ${r + r0 * x0} ${r + r0 * y0} L ${r + r * x0} ${
+    r + r * y0
+  } A ${r} ${r} 0 ${largeArc} 1 ${r + r * x1} ${r + r * y1} L ${r + r0 * x1} ${
+    r + r0 * y1
+  } A ${r0} ${r0} 0 ${largeArc} 0 ${r + r0 * x0} ${
+    r + r0 * y0
+  }" fill="${color}" opacity="0.85" stroke="#1f2937" stroke-width="1" />`;
+}
+
+window.createDonutChart = createDonutChart;
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 1f0ca0f8..9cc75a1b 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -3,6 +3,10 @@
 {% load crispy_forms_tags %}
 {% load django_tables2 %}
 {% block title %}Admin Dashboard{% endblock %}
+{% block javascript %}
+  {{ block.super }}
+  <script src="{% static 'bundles/js/dashboard-bundle.js' %}" ></script>
+{% endblock %}
 {% block content %}
 <h2 class="my-2">Program Dashboard</h2>
 <div class="container" x-data="dashboardStats()">
@@ -183,9 +187,6 @@ <h3 class="my-2">Service Delivery Map</h3>
     ];
     const rejected = ['==', ['get', 'status'], 'rejected'];
 
-    // colors to use for the categories
-    // soft green, yellow, red
-    const colors = ['#4ade80', '#fbbf24', '#f87171'];
 
     map.on('load', () => {
       // Modify the source configuration to include initial filters
@@ -268,11 +269,12 @@ <h3 class="my-2">Service Delivery Map</h3>
 
           let marker = markers[id];
           if (!marker) {
+            console.log("creating donut chart for cluster", id)
             const el = createDonutChart({
               ...props,
               cluster_id: id,  // Make sure cluster_id is passed
               coordinates: coords  // Pass the coordinates
-            });
+            }, map);
             marker = markers[id] = new mapboxgl.Marker({
               element: el
             }).setLngLat(coords);
@@ -304,90 +306,6 @@ <h3 class="my-2">Service Delivery Map</h3>
       mapLoading.classList.add('d-none');
       // Optionally add error messaging here
     });
-
-    // code for creating an SVG donut chart from feature properties
-    function createDonutChart(props) {
-      const offsets = [];
-      const counts = [
-        props.approved,
-        props.pending,
-        props.rejected
-      ];
-      let total = 0;
-      for (const count of counts) {
-        offsets.push(total);
-        total += count;
-      }
-      const fontSize =
-        total >= 1000 ? 22 : total >= 100 ? 20 : total >= 10 ? 18 : 16;
-      const r =
-        total >= 1000 ? 50 : total >= 100 ? 32 : total >= 10 ? 24 : 18;
-      const r0 = Math.round(r * 0.8);
-      const w = r * 2;
-
-      let html = `<div>
-            <svg width="${w}" height="${w}" viewbox="0 0 ${w} ${w}" text-anchor="middle" style="font: ${fontSize}px sans-serif; display: block">
-            <defs>
-              <filter id="shadow">
-                <feDropShadow dx="0" dy="1" stdDeviation="2" flood-opacity="0.3"/>
-              </filter>
-            </defs>`;
-
-      for (let i = 0; i < counts.length; i++) {
-        html += donutSegment(
-          offsets[i] / total,
-          (offsets[i] + counts[i]) / total,
-          r,
-          r0,
-          colors[i]
-        );
-      }
-      html += `<circle cx="${r}" cy="${r}" r="${r0}" fill="#374151" />
-            <text dominant-baseline="central" transform="translate(${r}, ${r})"
-                  fill="white" font-weight="500" filter="url(#shadow)">
-                ${total.toLocaleString()}
-            </text>
-            </svg>
-            </div>`;
-
-      const el = document.createElement('div');
-      el.innerHTML = html;
-      el.style.cursor = 'pointer';
-
-      // Click handler to zoom and navigate to the cluster
-      el.addEventListener('click', (e) => {
-        map.getSource('visits').getClusterExpansionZoom(
-          props.cluster_id,
-          (err, zoom) => {
-            if (err) return;
-
-            map.easeTo({
-              center: props.coordinates,
-              zoom: zoom
-            });
-          }
-        );
-      });
-
-      return el;
-    }
-
-    function donutSegment(start, end, r, r0, color) {
-      if (end - start === 1) end -= 0.00001;
-      const a0 = 2 * Math.PI * (start - 0.25);
-      const a1 = 2 * Math.PI * (end - 0.25);
-      const x0 = Math.cos(a0),
-        y0 = Math.sin(a0);
-      const x1 = Math.cos(a1),
-        y1 = Math.sin(a1);
-      const largeArc = end - start > 0.5 ? 1 : 0;
-
-      // draw an SVG path
-      return `<path d="M ${r + r0 * x0} ${r + r0 * y0} L ${r + r * x0} ${r + r * y0
-        } A ${r} ${r} 0 ${largeArc} 1 ${r + r * x1} ${r + r * y1} L ${r + r0 * x1
-        } ${r + r0 * y1} A ${r0} ${r0} 0 ${largeArc} 0 ${r + r0 * x0} ${r + r0 * y0
-        }" fill="${color}" opacity="0.85" stroke="#1f2937" stroke-width="1" />`;
-    }
   });
 
 </script>
diff --git a/webpack/base.config.js b/webpack/base.config.js
index 74235cf8..8f1fe4de 100644
--- a/webpack/base.config.js
+++ b/webpack/base.config.js
@@ -8,6 +8,10 @@ module.exports = {
   context: path.join(__dirname, '../'),
   entry: {
     project: path.resolve(__dirname, '../commcare_connect/static/js/project'),
+    dashboard: path.resolve(
+      __dirname,
+      '../commcare_connect/static/js/dashboard',
+    ),
     vendors: path.resolve(__dirname, '../commcare_connect/static/js/vendors'),
   },
   output: {

From be0f47d3d880ba8fa74e726fd3b966aa840f85bc Mon Sep 17 00:00:00 2001
From: Sravan Reddy <sravfeyn@gmail.com>
Date: Tue, 12 Nov 2024 21:20:42 +0530
Subject: [PATCH 73/96] Publish android assetlinks for deeplinking

---
 config/urls.py  |  3 +++
 config/views.py | 30 ++++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+)
 create mode 100644 config/views.py

diff --git a/config/urls.py b/config/urls.py
index 9eca1758..4c368e0f 100644
--- a/config/urls.py
+++ b/config/urls.py
@@ -9,9 +9,12 @@
 
 from commcare_connect.organization.views import organization_create
 
+from . import views
+
 urlpatterns = [
     path("", TemplateView.as_view(template_name="pages/home.html"), name="home"),
     path("about/", TemplateView.as_view(template_name="pages/about.html"), name="about"),
+    path(".well-known/assetlinks.json", views.assetlinks_json, name="assetlinks_json"),
     # Django Admin, use {% url 'admin:index' %}
     path(settings.ADMIN_URL, admin.site.urls),
     path("o/", include("oauth2_provider.urls", namespace="oauth2_provider")),
diff --git a/config/views.py b/config/views.py
new file mode 100644
index 00000000..3b7fc766
--- /dev/null
+++ b/config/views.py
@@ -0,0 +1,30 @@
+from django.http import JsonResponse
+
+
+def assetlinks_json(request):
+    assetfile = [
+        {
+            "relation": ["delegate_permission/common.handle_all_urls"],
+            "target": {
+                "namespace": "android_app",
+                "package_name": "org.commcare.dalvik",
+                "sha256_cert_fingerprints":
+                [
+                    "88:57:18:F8:E8:7D:74:04:97:AE:83:65:74:ED:EF:10:40:D9:4C:E2:54:F0:E0:40:64:77:96:7F:D1:39:F9:81",
+                    "89:55:DF:D8:0E:66:63:06:D2:6D:88:A4:A3:88:A4:D9:16:5A:C4:1A:7E:E1:C6:78:87:00:37:55:93:03:7B:03"
+                ]
+            }
+        },
+        {
+            "relation": ["delegate_permission/common.handle_all_urls"],
+            "target": {
+                "namespace": "android_app",
+                "package_name": "org.commcare.dalvik.debug",
+                "sha256_cert_fingerprints":
+                [
+                    "88:57:18:F8:E8:7D:74:04:97:AE:83:65:74:ED:EF:10:40:D9:4C:E2:54:F0:E0:40:64:77:96:7F:D1:39:F9:81"
+                ]
+            }
+        },
+    ]
+    return JsonResponse(assetfile, safe=False)

From 4b30441429096705b4d0cbf9af9b430d7448a980 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 17:53:27 +0200
Subject: [PATCH 74/96] extract updateMarkers

---
 commcare_connect/static/js/dashboard.js       | 44 +++++++++++++++++++
 .../templates/reports/dashboard.html          | 41 +----------------
 2 files changed, 45 insertions(+), 40 deletions(-)

diff --git a/commcare_connect/static/js/dashboard.js b/commcare_connect/static/js/dashboard.js
index c1d8557f..9318a825 100644
--- a/commcare_connect/static/js/dashboard.js
+++ b/commcare_connect/static/js/dashboard.js
@@ -4,6 +4,49 @@ console.log('dashboard.js loaded');
 // soft green, yellow, red
 const visitColors = ['#4ade80', '#fbbf24', '#f87171'];
 
+// after the GeoJSON data is loaded, update markers on the screen on every frame
+// objects for caching and keeping track of HTML marker objects (for performance)
+const markers = {};
+let markersOnScreen = {};
+
+function updateMarkers(map) {
+  const newMarkers = {};
+  const features = map.querySourceFeatures('visits');
+
+  // for every cluster on the screen, create an HTML marker for it (if we didn't yet),
+  // and add it to the map if it's not there already
+  for (const feature of features) {
+    const coords = feature.geometry.coordinates;
+    const props = feature.properties;
+    if (!props.cluster) continue;
+    const id = props.cluster_id;
+
+    let marker = markers[id];
+    if (!marker) {
+      console.log('creating donut chart for cluster', id);
+      const el = createDonutChart(
+        {
+          ...props,
+          cluster_id: id, // Make sure cluster_id is passed
+          coordinates: coords, // Pass the coordinates
+        },
+        map,
+      );
+      marker = markers[id] = new mapboxgl.Marker({
+        element: el,
+      }).setLngLat(coords);
+    }
+    newMarkers[id] = marker;
+
+    if (!markersOnScreen[id]) marker.addTo(map);
+  }
+  // for every marker we've added previously, remove those that are no longer visible
+  for (const id in markersOnScreen) {
+    if (!newMarkers[id]) markersOnScreen[id].remove();
+  }
+  markersOnScreen = newMarkers;
+}
+
 // Function to create a donut chart
 function createDonutChart(props, map) {
   console.log('createDonutChart', props);
@@ -87,4 +130,5 @@ function donutSegment(start, end, r, r0, color) {
   }" fill="${color}" opacity="0.85" stroke="#1f2937" stroke-width="1" />`;
 }
 
+window.updateMarkers = updateMarkers;
 window.createDonutChart = createDonutChart;
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 9cc75a1b..2a457188 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -250,49 +250,10 @@ <h3 class="my-2">Service Delivery Map</h3>
           .addTo(map);
       });
 
-      // after the GeoJSON data is loaded, update markers on the screen on every frame
-      // objects for caching and keeping track of HTML marker objects (for performance)
-      const markers = {};
-      let markersOnScreen = {};
-
-      function updateMarkers() {
-        const newMarkers = {};
-        const features = map.querySourceFeatures('visits');
-
-        // for every cluster on the screen, create an HTML marker for it (if we didn't yet),
-        // and add it to the map if it's not there already
-        for (const feature of features) {
-          const coords = feature.geometry.coordinates;
-          const props = feature.properties;
-          if (!props.cluster) continue;
-          const id = props.cluster_id;
-
-          let marker = markers[id];
-          if (!marker) {
-            console.log("creating donut chart for cluster", id)
-            const el = createDonutChart({
-              ...props,
-              cluster_id: id,  // Make sure cluster_id is passed
-              coordinates: coords  // Pass the coordinates
-            }, map);
-            marker = markers[id] = new mapboxgl.Marker({
-              element: el
-            }).setLngLat(coords);
-          }
-          newMarkers[id] = marker;
-
-          if (!markersOnScreen[id]) marker.addTo(map);
-        }
-        // for every marker we've added previously, remove those that are no longer visible
-        for (const id in markersOnScreen) {
-          if (!newMarkers[id]) markersOnScreen[id].remove();
-        }
-        markersOnScreen = newMarkers;
-      }
 
       map.on('render', () => {
         if (!map.isSourceLoaded('visits')) return;
-        updateMarkers();
+        updateMarkers(map);
       });
 
       // Hide loading overlay when initial map load is complete

From ee37856a2577a4ee5582dffc455fce3e218f7e14 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 17:53:45 +0200
Subject: [PATCH 75/96] remove logging statments

---
 commcare_connect/static/js/dashboard.js | 2 --
 1 file changed, 2 deletions(-)

diff --git a/commcare_connect/static/js/dashboard.js b/commcare_connect/static/js/dashboard.js
index 9318a825..d73535b3 100644
--- a/commcare_connect/static/js/dashboard.js
+++ b/commcare_connect/static/js/dashboard.js
@@ -23,7 +23,6 @@ function updateMarkers(map) {
 
     let marker = markers[id];
     if (!marker) {
-      console.log('creating donut chart for cluster', id);
       const el = createDonutChart(
         {
           ...props,
@@ -49,7 +48,6 @@ function updateMarkers(map) {
 
 // Function to create a donut chart
 function createDonutChart(props, map) {
-  console.log('createDonutChart', props);
   const offsets = [];
   const counts = [props.approved, props.pending, props.rejected];
   let total = 0;

From 8330b101f22176a34a65eab52e380fde680efee7 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 17:56:36 +0200
Subject: [PATCH 76/96] fix tests

---
 commcare_connect/reports/tests/test_reports.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/commcare_connect/reports/tests/test_reports.py b/commcare_connect/reports/tests/test_reports.py
index f85ae168..441090f7 100644
--- a/commcare_connect/reports/tests/test_reports.py
+++ b/commcare_connect/reports/tests/test_reports.py
@@ -106,7 +106,7 @@ def all(self):
     assert feature1["geometry"]["coordinates"] == [10.123, 20.456]
     assert feature1["properties"]["status"] == "approved"
     assert feature1["properties"]["other_field"] == "value1"
-    assert feature1["properties"]["color"] == "#00FF00"
+    assert feature1["properties"]["color"] == "#4ade80"
 
     # Check the second feature
     feature2 = geojson["features"][1]
@@ -115,7 +115,7 @@ def all(self):
     assert feature2["geometry"]["coordinates"] == [30.789, 40.012]
     assert feature2["properties"]["status"] == "rejected"
     assert feature2["properties"]["other_field"] == "value2"
-    assert feature2["properties"]["color"] == "#FF0000"
+    assert feature2["properties"]["color"] == "#f87171"
 
     # Check that the other cases are not included
     assert all(f["properties"]["other_field"] not in ["value3", "value4", "value5"] for f in geojson["features"])

From fa092aa4a52df2beb49600f4e90d196167046e42 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Tue, 12 Nov 2024 17:58:14 +0200
Subject: [PATCH 77/96] put map in a container

---
 .../templates/reports/dashboard.html          | 20 ++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 2a457188..bafdd0aa 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -74,15 +74,17 @@ <h2 class="my-2">Program Dashboard</h2>
     </div>
   </div>
 </div>
-<div class="row">
-  <div class="col-12">
-    <h3 class="my-2">Service Delivery Map</h3>
-    <div class="position-relative">
-      <div id="map" class="mt-3 mb-2 rounded border" style="height: 900px;"></div>
-      <div id="map-loading"
-        class="position-absolute top-0 start-0 w-100 h-100 bg-dark bg-opacity-50 d-flex justify-content-center align-items-center">
-        <div class="spinner-border text-light" role="status">
-          <span class="visually-hidden">Loading map...</span>
+<div class="container">
+  <div class="row">
+    <div class="col-12">
+      <h3 class="my-2">Service Delivery Map</h3>
+      <div class="position-relative">
+        <div id="map" class="mt-3 mb-2 rounded border" style="height: 500px;"></div>
+        <div id="map-loading"
+          class="position-absolute top-0 start-0 w-100 h-100 bg-dark bg-opacity-50 d-flex justify-content-center align-items-center">
+          <div class="spinner-border text-light" role="status">
+            <span class="visually-hidden">Loading map...</span>
+          </div>
         </div>
       </div>
     </div>

From f64430ad9d7ff06829b561ddb32ba67b336d7f18 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 14:40:58 +0200
Subject: [PATCH 78/96] dummy chart implementation

---
 .../templates/reports/dashboard.html          | 72 ++++++++++++++++++-
 1 file changed, 70 insertions(+), 2 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index bafdd0aa..f919dba1 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -6,6 +6,7 @@
 {% block javascript %}
   {{ block.super }}
   <script src="{% static 'bundles/js/dashboard-bundle.js' %}" ></script>
+  <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
 {% endblock %}
 {% block content %}
 <h2 class="my-2">Program Dashboard</h2>
@@ -76,7 +77,7 @@ <h2 class="my-2">Program Dashboard</h2>
 </div>
 <div class="container">
   <div class="row">
-    <div class="col-12">
+    <div class="col-12 col-md-6">
       <h3 class="my-2">Service Delivery Map</h3>
       <div class="position-relative">
         <div id="map" class="mt-3 mb-2 rounded border" style="height: 500px;"></div>
@@ -88,6 +89,10 @@ <h3 class="my-2">Service Delivery Map</h3>
         </div>
       </div>
     </div>
+    <div class="col-12 col-md-6">
+      <h3 class="my-2">Visits over Time</h3>
+      <canvas id="visits-over-time" class="mt-3 mb-2 rounded border" style="height: 500px;"></canvas>
+    </div>
   </div>
 </div>
 {% endblock content %}
@@ -172,7 +177,7 @@ <h3 class="my-2">Service Delivery Map</h3>
       container: 'map',
       style: 'mapbox://styles/mapbox/dark-v11',
       center: [20, 0], // Centered on Africa (roughly central coordinates)
-      zoom: 3,
+      zoom: 2,
     });
 
     // This loads a map with two layers, one is a cluster with donut
@@ -269,6 +274,69 @@ <h3 class="my-2">Service Delivery Map</h3>
       mapLoading.classList.add('d-none');
       // Optionally add error messaging here
     });
+
+    // Generate last 30 days of dummy data
+    const generateDummyData = () => {
+      const data = [];
+      const labels = [];
+      const today = new Date();
+
+      for (let i = 29; i >= 0; i--) {
+        const date = new Date(today);
+        date.setDate(date.getDate() - i);
+        labels.push(date.toLocaleDateString('en-US', { month: 'short', day: 'numeric' }));
+        // Random number between 10 and 50
+        data.push(Math.floor(Math.random() * 40) + 10);
+      }
+      return { labels, data };
+    };
+
+    const { labels, data } = generateDummyData();
+
+    const ctx = document.getElementById('visits-over-time');
+    const chart = new Chart(ctx, {
+      type: 'line',
+      data: {
+        labels: labels,
+        datasets: [{
+          label: 'Total Visits',
+          data: data,
+          borderColor: 'rgb(75, 192, 192)',
+          backgroundColor: 'rgba(75, 192, 192, 0.2)',
+          fill: true,
+          tension: 0.3
+        }]
+      },
+      options: {
+        responsive: false,
+        maintainAspectRatio: false,
+        plugins: {
+          title: {
+            display: true,
+            text: 'Visits by Date'
+          },
+          tooltip: {
+            mode: 'index',
+            intersect: false,
+          }
+        },
+        scales: {
+          y: {
+            beginAtZero: true,
+            title: {
+              display: true,
+              text: 'Number of Visits'
+            }
+          },
+          x: {
+            title: {
+              display: true,
+              text: 'Date'
+            }
+          }
+        }
+      }
+    });
   });
 
 </script>

From c9c76c641b80acb002b60c9069f5adbdf02113ea Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 14:47:44 +0200
Subject: [PATCH 79/96] fix sizing, responsiveness

---
 commcare_connect/templates/reports/dashboard.html | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index f919dba1..d9586f90 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -91,7 +91,9 @@ <h3 class="my-2">Service Delivery Map</h3>
     </div>
     <div class="col-12 col-md-6">
       <h3 class="my-2">Visits over Time</h3>
-      <canvas id="visits-over-time" class="mt-3 mb-2 rounded border" style="height: 500px;"></canvas>
+      <div style="position: relative; height: 500px; width: 100%;">
+        <canvas id="visits-over-time"></canvas>
+      </div>
     </div>
   </div>
 </div>
@@ -288,6 +290,7 @@ <h3 class="my-2">Visits over Time</h3>
         // Random number between 10 and 50
         data.push(Math.floor(Math.random() * 40) + 10);
       }
+      console.log({labels, data});
       return { labels, data };
     };
 
@@ -308,7 +311,7 @@ <h3 class="my-2">Visits over Time</h3>
         }]
       },
       options: {
-        responsive: false,
+        responsive: true,
         maintainAspectRatio: false,
         plugins: {
           title: {

From aabb39fcb7204d7fef0c1a43003e6fd03ace2c71 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 14:49:24 +0200
Subject: [PATCH 80/96] smaller cluster radius

---
 commcare_connect/templates/reports/dashboard.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index d9586f90..903a49d4 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -208,7 +208,7 @@ <h3 class="my-2">Visits over Time</h3>
         data: `{% url "reports:visit_map_data" %}?${queryString}`,
         cluster: true,
         clusterMaxZoom: 14,
-        clusterRadius: 80,
+        clusterRadius: 40,
         clusterMinPoints: minClusterSize,
         clusterProperties: {
           // keep separate counts for each status category in a cluster

From bcce9ac1a2f71157ec90f5271ff2130d36fd2aaf Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 15:00:23 +0200
Subject: [PATCH 81/96] add api for graphs

---
 commcare_connect/reports/urls.py              |   1 +
 commcare_connect/reports/views.py             |  41 +++++-
 .../templates/reports/dashboard.html          | 129 ++++++++++--------
 3 files changed, 115 insertions(+), 56 deletions(-)

diff --git a/commcare_connect/reports/urls.py b/commcare_connect/reports/urls.py
index cd05d994..bb8a9709 100644
--- a/commcare_connect/reports/urls.py
+++ b/commcare_connect/reports/urls.py
@@ -9,4 +9,5 @@
     path("delivery_stats", view=views.DeliveryStatsReportView.as_view(), name="delivery_stats_report"),
     path("api/visit_map_data/", views.visit_map_data, name="visit_map_data"),
     path("api/dashboard_stats/", views.dashboard_stats_api, name="dashboard_stats_api"),
+    path("api/visits_over_time/", views.visits_over_time_api, name="visits_over_time_api"),
 ]
diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index 7e25f92b..93cb9d27 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -8,7 +8,7 @@
 from django.conf import settings
 from django.contrib.auth.decorators import login_required, user_passes_test
 from django.contrib.auth.mixins import LoginRequiredMixin, UserPassesTestMixin
-from django.db.models import Max, Q, Sum
+from django.db.models import Count, Max, Q, Sum
 from django.http import JsonResponse
 from django.shortcuts import render
 from django.urls import reverse
@@ -418,3 +418,42 @@ def dashboard_stats_api(request):
             "percent_verified": f"{percent_verified:.1f}%",
         }
     )
+
+
+@login_required
+@user_passes_test(lambda u: u.is_superuser)
+def visits_over_time_api(request):
+    filterset = DashboardFilters(request.GET)
+
+    # Use the filtered queryset
+    queryset = UserVisit.objects.all()
+
+    if filterset.is_valid():
+        queryset = filterset.filter_queryset(queryset)
+        from_date = filterset.form.cleaned_data["from_date"]
+        to_date = filterset.form.cleaned_data["to_date"]
+    else:
+        to_date = datetime.now().date()
+        from_date = to_date - timedelta(days=30)
+        queryset = queryset.filter(visit_date__gte=from_date, visit_date__lte=to_date)
+
+    # Aggregate visits by date
+    visits_by_date = queryset.values("visit_date").annotate(count=Count("id")).order_by("visit_date")
+
+    # Create a complete date range with 0s for missing dates
+    date_counts = {result["visit_date"]: result["count"] for result in visits_by_date}
+
+    data = []
+    labels = []
+    current_date = from_date
+    while current_date <= to_date:
+        labels.append(current_date.strftime("%b %d"))
+        data.append(date_counts.get(current_date, 0))
+        current_date += timedelta(days=1)
+
+    return JsonResponse(
+        {
+            "labels": labels,
+            "data": data,
+        }
+    )
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 903a49d4..62de61d3 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -277,68 +277,87 @@ <h3 class="my-2">Visits over Time</h3>
       // Optionally add error messaging here
     });
 
-    // Generate last 30 days of dummy data
-    const generateDummyData = () => {
-      const data = [];
-      const labels = [];
-      const today = new Date();
 
-      for (let i = 29; i >= 0; i--) {
-        const date = new Date(today);
-        date.setDate(date.getDate() - i);
-        labels.push(date.toLocaleDateString('en-US', { month: 'short', day: 'numeric' }));
-        // Random number between 10 and 50
-        data.push(Math.floor(Math.random() * 40) + 10);
-      }
-      console.log({labels, data});
-      return { labels, data };
-    };
+  });
+  // Update chart when page loads
+  window.addEventListener('DOMContentLoaded', () => {
+        // Replace the chart initialization code with this:
+        const ctx = document.getElementById('visits-over-time');
+    let visitsChart;
 
-    const { labels, data } = generateDummyData();
+    async function updateVisitsChart() {
+      console.log('updating visits chart');
+      try {
+        // Get form data for filters
+        const formElement = document.querySelector('#filterForm form');
+        const formData = new FormData(formElement);
+        const queryString = new URLSearchParams(formData).toString();
 
-    const ctx = document.getElementById('visits-over-time');
-    const chart = new Chart(ctx, {
-      type: 'line',
-      data: {
-        labels: labels,
-        datasets: [{
-          label: 'Total Visits',
-          data: data,
-          borderColor: 'rgb(75, 192, 192)',
-          backgroundColor: 'rgba(75, 192, 192, 0.2)',
-          fill: true,
-          tension: 0.3
-        }]
-      },
-      options: {
-        responsive: true,
-        maintainAspectRatio: false,
-        plugins: {
-          title: {
-            display: true,
-            text: 'Visits by Date'
-          },
-          tooltip: {
-            mode: 'index',
-            intersect: false,
-          }
-        },
-        scales: {
-          y: {
-            beginAtZero: true,
-            title: {
-              display: true,
-              text: 'Number of Visits'
-            }
+        // Fetch data from API
+        const response = await fetch(`{% url 'reports:visits_over_time_api' %}?${queryString}`);
+        console.log('response', response);
+        const chartData = await response.json();
+
+        if (visitsChart) {
+          visitsChart.destroy();
+        }
+
+        visitsChart = new Chart(ctx, {
+          type: 'line',
+          data: {
+            labels: chartData.labels,
+            datasets: [{
+              label: 'Total Visits',
+              data: chartData.data,
+              borderColor: 'rgb(75, 192, 192)',
+              backgroundColor: 'rgba(75, 192, 192, 0.2)',
+              fill: true,
+              tension: 0.3
+            }]
           },
-          x: {
-            title: {
-              display: true,
-              text: 'Date'
+          options: {
+            responsive: true,
+            maintainAspectRatio: false,
+            plugins: {
+              title: {
+                display: true,
+                text: 'Visits by Date'
+              },
+              tooltip: {
+                mode: 'index',
+                intersect: false,
+              }
+            },
+            scales: {
+              y: {
+                beginAtZero: true,
+                title: {
+                  display: true,
+                  text: 'Number of Visits'
+                }
+              },
+              x: {
+                title: {
+                  display: true,
+                  text: 'Date'
+                }
+              }
             }
           }
-        }
+        });
+      } catch (error) {
+        console.error('Error updating visits chart:', error);
       }
+    }
+
+
+    console.log('updating visits chart on load');
+    updateVisitsChart();
+
+    // Update chart when filters change
+    const formElement = document.querySelector('#filterForm form');
+    formElement.querySelectorAll('select, input').forEach(input => {
+      input.addEventListener('change', updateVisitsChart);
     });
   });
 

From 1cc46a395b130c15cac96ed37bc695290ddfa452 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 15:08:57 +0200
Subject: [PATCH 82/96] get program breakdown kinda working

---
 commcare_connect/reports/views.py             | 44 ++++++++++++-------
 .../templates/reports/dashboard.html          | 35 ++++++++++-----
 2 files changed, 53 insertions(+), 26 deletions(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index 93cb9d27..ed197872 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -424,10 +424,8 @@ def dashboard_stats_api(request):
 @user_passes_test(lambda u: u.is_superuser)
 def visits_over_time_api(request):
     filterset = DashboardFilters(request.GET)
-
-    # Use the filtered queryset
     queryset = UserVisit.objects.all()
-
+    # Use the filtered queryset if available, else use last 30 days
     if filterset.is_valid():
         queryset = filterset.filter_queryset(queryset)
         from_date = filterset.form.cleaned_data["from_date"]
@@ -437,23 +435,39 @@ def visits_over_time_api(request):
         from_date = to_date - timedelta(days=30)
         queryset = queryset.filter(visit_date__gte=from_date, visit_date__lte=to_date)
 
-    # Aggregate visits by date
-    visits_by_date = queryset.values("visit_date").annotate(count=Count("id")).order_by("visit_date")
+    # Get visits by date and program in a single query
+    visits_by_program = (
+        queryset.values("visit_date", "opportunity__delivery_type__name")
+        .annotate(count=Count("id"))
+        .order_by("visit_date", "opportunity__delivery_type__name")
+    )
 
-    # Create a complete date range with 0s for missing dates
-    date_counts = {result["visit_date"]: result["count"] for result in visits_by_date}
+    # Create lookup dict for program data
+    program_data = {}
+    for visit in visits_by_program:
+        program_name = visit["opportunity__delivery_type__name"]
+        if program_name not in program_data:
+            program_data[program_name] = {}
+        program_data[program_name][visit["visit_date"]] = visit["count"]
 
-    data = []
+    # Create labels and datasets
     labels = []
+    datasets = []
     current_date = from_date
+
+    # Build labels array
     while current_date <= to_date:
         labels.append(current_date.strftime("%b %d"))
-        data.append(date_counts.get(current_date, 0))
         current_date += timedelta(days=1)
 
-    return JsonResponse(
-        {
-            "labels": labels,
-            "data": data,
-        }
-    )
+    # Build dataset for each program
+    for program_name in program_data.keys():
+        data = []
+        current_date = from_date
+        while current_date <= to_date:
+            data.append(program_data[program_name].get(current_date, 0))
+            current_date += timedelta(days=1)
+
+        datasets.append({"name": program_name, "data": data})
+
+    return JsonResponse({"labels": labels, "datasets": datasets})
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 62de61d3..4ec0329e 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -286,34 +286,46 @@ <h3 class="my-2">Visits over Time</h3>
     let visitsChart;
 
     async function updateVisitsChart() {
-      console.log('updating visits chart');
       try {
-        // Get form data for filters
         const formElement = document.querySelector('#filterForm form');
         const formData = new FormData(formElement);
         const queryString = new URLSearchParams(formData).toString();
 
-        // Fetch data from API
         const response = await fetch(`{% url 'reports:visits_over_time_api' %}?${queryString}`);
-        console.log('response', response);
         const chartData = await response.json();
 
         if (visitsChart) {
           visitsChart.destroy();
         }
 
+        // Assuming the API now returns data in this format:
+        // {
+        //   labels: ['2024-01', '2024-02', ...],
+        //   datasets: [
+        //     { name: 'Program A', data: [10, 20, ...] },
+        //     { name: 'Program B', data: [15, 25, ...] },
+        //   ]
+        // }
+
+        const colors = [
+          { border: 'rgb(75, 192, 192)', background: 'rgba(75, 192, 192, 0.2)' },
+          { border: 'rgb(255, 99, 132)', background: 'rgba(255, 99, 132, 0.2)' },
+          { border: 'rgb(255, 205, 86)', background: 'rgba(255, 205, 86, 0.2)' },
+          { border: 'rgb(54, 162, 235)', background: 'rgba(54, 162, 235, 0.2)' },
+        ];
+
         visitsChart = new Chart(ctx, {
           type: 'line',
           data: {
             labels: chartData.labels,
-            datasets: [{
-              label: 'Total Visits',
-              data: chartData.data,
-              borderColor: 'rgb(75, 192, 192)',
-              backgroundColor: 'rgba(75, 192, 192, 0.2)',
+            datasets: chartData.datasets.map((dataset, index) => ({
+              label: dataset.name,
+              data: dataset.data,
+              borderColor: colors[index % colors.length].border,
+              backgroundColor: colors[index % colors.length].background,
               fill: true,
               tension: 0.3
-            }]
+            }))
           },
           options: {
             responsive: true,
@@ -321,7 +333,7 @@ <h3 class="my-2">Visits over Time</h3>
             plugins: {
               title: {
                 display: true,
-                text: 'Visits by Date'
+                text: 'Visits by Program'
               },
               tooltip: {
                 mode: 'index',
@@ -331,6 +343,7 @@ <h3 class="my-2">Visits over Time</h3>
             scales: {
               y: {
                 beginAtZero: true,
+                stacked: true,
                 title: {
                   display: true,
                   text: 'Number of Visits'

From 38e0f134716c7c1d363d874b572a04dda9b68989 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 15:11:23 +0200
Subject: [PATCH 83/96] add "unknown" label

---
 commcare_connect/reports/views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index ed197872..e02d858a 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -468,6 +468,6 @@ def visits_over_time_api(request):
             data.append(program_data[program_name].get(current_date, 0))
             current_date += timedelta(days=1)
 
-        datasets.append({"name": program_name, "data": data})
+        datasets.append({"name": program_name or "Unknown", "data": data})
 
     return JsonResponse({"labels": labels, "datasets": datasets})

From 19870ab515ec49591f9596aa87ee15163048daf4 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 15:13:37 +0200
Subject: [PATCH 84/96] improve color overlaps

---
 commcare_connect/templates/reports/dashboard.html | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 4ec0329e..22338409 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -308,10 +308,10 @@ <h3 class="my-2">Visits over Time</h3>
         // }
 
         const colors = [
-          { border: 'rgb(75, 192, 192)', background: 'rgba(75, 192, 192, 0.2)' },
-          { border: 'rgb(255, 99, 132)', background: 'rgba(255, 99, 132, 0.2)' },
-          { border: 'rgb(255, 205, 86)', background: 'rgba(255, 205, 86, 0.2)' },
-          { border: 'rgb(54, 162, 235)', background: 'rgba(54, 162, 235, 0.2)' },
+          { border: 'rgb(75, 192, 192)', background: 'rgb(146, 219, 219)' },
+          { border: 'rgb(255, 99, 132)', background: 'rgb(255, 178, 193)' },
+          { border: 'rgb(255, 205, 86)', background: 'rgb(255, 227, 167)' },
+          { border: 'rgb(54, 162, 235)', background: 'rgb(157, 207, 245)' },
         ];
 
         visitsChart = new Chart(ctx, {

From f5746e54138631e96fbc3878c8cfc05a8874e73e Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 15:20:13 +0200
Subject: [PATCH 85/96] make it a bar chart

---
 .../templates/reports/dashboard.html             | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 22338409..3c488b72 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -315,7 +315,7 @@ <h3 class="my-2">Visits over Time</h3>
         ];
 
         visitsChart = new Chart(ctx, {
-          type: 'line',
+          type: 'bar',
           data: {
             labels: chartData.labels,
             datasets: chartData.datasets.map((dataset, index) => ({
@@ -323,8 +323,7 @@ <h3 class="my-2">Visits over Time</h3>
               data: dataset.data,
               borderColor: colors[index % colors.length].border,
               backgroundColor: colors[index % colors.length].background,
-              fill: true,
-              tension: 0.3
+              borderWidth: 1
             }))
           },
           options: {
@@ -341,18 +340,19 @@ <h3 class="my-2">Visits over Time</h3>
               }
             },
             scales: {
-              y: {
-                beginAtZero: true,
+              x: {
                 stacked: true,
                 title: {
                   display: true,
-                  text: 'Number of Visits'
+                  text: 'Date'
                 }
               },
-              x: {
+              y: {
+                stacked: true,
+                beginAtZero: true,
                 title: {
                   display: true,
-                  text: 'Date'
+                  text: 'Number of Visits'
                 }
               }
             }

From ac9ac103a52d39e82d5900213f11bed183be6d8f Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 15:26:31 +0200
Subject: [PATCH 86/96] mock out spots for pie charts

---
 .../templates/reports/dashboard.html             | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 3c488b72..2b99885d 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -91,9 +91,23 @@ <h3 class="my-2">Service Delivery Map</h3>
     </div>
     <div class="col-12 col-md-6">
       <h3 class="my-2">Visits over Time</h3>
-      <div style="position: relative; height: 500px; width: 100%;">
+      <div style="position: relative; height: 200px; width: 100%;">
         <canvas id="visits-over-time"></canvas>
       </div>
+      <div class="row mt-4">
+        <div class="col-12 col-md-6">
+          <h3 class="my-2">Visits by Program</h3>
+          <div style="position: relative; height: 200px; width: 100%;">
+            <canvas id="visits-by-program"></canvas>
+          </div>
+        </div>
+        <div class="col-12 col-md-6">
+          <h3 class="my-2">Visits by Status</h3>
+          <div style="position: relative; height: 200px; width: 100%;">
+            <canvas id="visits-by-status"></canvas>
+          </div>
+        </div>
+      </div>
     </div>
   </div>
 </div>

From 2eb33e048f241e6c38755e8814716315a41599f1 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 15:29:05 +0200
Subject: [PATCH 87/96] implement pie charts

---
 commcare_connect/reports/views.py             |  44 ++++--
 .../templates/reports/dashboard.html          | 133 +++++++++++++-----
 2 files changed, 132 insertions(+), 45 deletions(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index e02d858a..3439d419 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -435,32 +435,39 @@ def visits_over_time_api(request):
         from_date = to_date - timedelta(days=30)
         queryset = queryset.filter(visit_date__gte=from_date, visit_date__lte=to_date)
 
-    # Get visits by date and program in a single query
-    visits_by_program = (
+    # Get data for all three charts
+    # 1. Visits over time by program
+    visits_by_program_time = (
         queryset.values("visit_date", "opportunity__delivery_type__name")
         .annotate(count=Count("id"))
         .order_by("visit_date", "opportunity__delivery_type__name")
     )
 
-    # Create lookup dict for program data
+    # 2. Total visits by program
+    visits_by_program = (
+        queryset.values("opportunity__delivery_type__name").annotate(count=Count("id")).order_by("-count")
+    )
+
+    # 3. Visits by status
+    visits_by_status = queryset.values("status").annotate(count=Count("id")).order_by("-count")
+
+    # Process time series data
     program_data = {}
-    for visit in visits_by_program:
+    for visit in visits_by_program_time:
         program_name = visit["opportunity__delivery_type__name"]
         if program_name not in program_data:
             program_data[program_name] = {}
         program_data[program_name][visit["visit_date"]] = visit["count"]
 
-    # Create labels and datasets
+    # Create labels and datasets for time series
     labels = []
-    datasets = []
+    time_datasets = []
     current_date = from_date
 
-    # Build labels array
     while current_date <= to_date:
         labels.append(current_date.strftime("%b %d"))
         current_date += timedelta(days=1)
 
-    # Build dataset for each program
     for program_name in program_data.keys():
         data = []
         current_date = from_date
@@ -468,6 +475,23 @@ def visits_over_time_api(request):
             data.append(program_data[program_name].get(current_date, 0))
             current_date += timedelta(days=1)
 
-        datasets.append({"name": program_name or "Unknown", "data": data})
+        time_datasets.append({"name": program_name or "Unknown", "data": data})
+
+    # Process pie chart data
+    program_pie_data = {
+        "labels": [item["opportunity__delivery_type__name"] or "Unknown" for item in visits_by_program],
+        "data": [item["count"] for item in visits_by_program],
+    }
 
-    return JsonResponse({"labels": labels, "datasets": datasets})
+    status_pie_data = {
+        "labels": [item["status"] or "Unknown" for item in visits_by_status],
+        "data": [item["count"] for item in visits_by_status],
+    }
+
+    return JsonResponse(
+        {
+            "time_series": {"labels": labels, "datasets": time_datasets},
+            "program_pie": program_pie_data,
+            "status_pie": status_pie_data,
+        }
+    )
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 2b99885d..ee8e8f13 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -295,48 +295,42 @@ <h3 class="my-2">Visits by Status</h3>
   });
   // Update chart when page loads
   window.addEventListener('DOMContentLoaded', () => {
-        // Replace the chart initialization code with this:
-        const ctx = document.getElementById('visits-over-time');
-    let visitsChart;
+    const timeSeriesCtx = document.getElementById('visits-over-time');
+    const programPieCtx = document.getElementById('visits-by-program');
+    const statusPieCtx = document.getElementById('visits-by-status');
 
-    async function updateVisitsChart() {
+    let timeSeriesChart, programPieChart, statusPieChart;
+
+    const chartColors = [
+      { border: 'rgb(75, 192, 192)', background: 'rgba(75, 192, 192, 0.8)' },
+      { border: 'rgb(255, 99, 132)', background: 'rgba(255, 99, 132, 0.8)' },
+      { border: 'rgb(255, 205, 86)', background: 'rgba(255, 205, 86, 0.8)' },
+      { border: 'rgb(54, 162, 235)', background: 'rgba(54, 162, 235, 0.8)' },
+    ];
+
+    async function updateCharts() {
       try {
         const formElement = document.querySelector('#filterForm form');
         const formData = new FormData(formElement);
         const queryString = new URLSearchParams(formData).toString();
 
         const response = await fetch(`{% url 'reports:visits_over_time_api' %}?${queryString}`);
-        const chartData = await response.json();
+        const data = await response.json();
 
-        if (visitsChart) {
-          visitsChart.destroy();
+        // Update time series chart
+        if (timeSeriesChart) {
+          timeSeriesChart.destroy();
         }
 
-        // Assuming the API now returns data in this format:
-        // {
-        //   labels: ['2024-01', '2024-02', ...],
-        //   datasets: [
-        //     { name: 'Program A', data: [10, 20, ...] },
-        //     { name: 'Program B', data: [15, 25, ...] },
-        //   ]
-        // }
-
-        const colors = [
-          { border: 'rgb(75, 192, 192)', background: 'rgb(146, 219, 219)' },
-          { border: 'rgb(255, 99, 132)', background: 'rgb(255, 178, 193)' },
-          { border: 'rgb(255, 205, 86)', background: 'rgb(255, 227, 167)' },
-          { border: 'rgb(54, 162, 235)', background: 'rgb(157, 207, 245)' },
-        ];
-
-        visitsChart = new Chart(ctx, {
+        timeSeriesChart = new Chart(timeSeriesCtx, {
           type: 'bar',
           data: {
-            labels: chartData.labels,
-            datasets: chartData.datasets.map((dataset, index) => ({
+            labels: data.time_series.labels,
+            datasets: data.time_series.datasets.map((dataset, index) => ({
               label: dataset.name,
               data: dataset.data,
-              borderColor: colors[index % colors.length].border,
-              backgroundColor: colors[index % colors.length].background,
+              borderColor: chartColors[index % chartColors.length].border,
+              backgroundColor: chartColors[index % chartColors.length].background,
               borderWidth: 1
             }))
           },
@@ -346,7 +340,7 @@ <h3 class="my-2">Visits by Status</h3>
             plugins: {
               title: {
                 display: true,
-                text: 'Visits by Program'
+                text: 'Visits Over Time'
               },
               tooltip: {
                 mode: 'index',
@@ -372,19 +366,88 @@ <h3 class="my-2">Visits by Status</h3>
             }
           }
         });
+
+        // Update program pie chart
+        if (programPieChart) {
+          programPieChart.destroy();
+        }
+
+        programPieChart = new Chart(programPieCtx, {
+          type: 'pie',
+          data: {
+            labels: data.program_pie.labels,
+            datasets: [{
+              data: data.program_pie.data,
+              backgroundColor: chartColors.map(c => c.background),
+              borderColor: chartColors.map(c => c.border),
+              borderWidth: 1
+            }]
+          },
+          options: {
+            responsive: true,
+            maintainAspectRatio: false,
+            plugins: {
+              legend: {
+                position: 'right',
+                labels: {
+                  boxWidth: 12
+                }
+              }
+            }
+          }
+        });
+
+        // Update status pie chart
+        if (statusPieChart) {
+          statusPieChart.destroy();
+        }
+
+        const statusColors = {
+          approved: { background: 'rgba(74, 222, 128, 0.8)', border: 'rgb(74, 222, 128)' },
+          rejected: { background: 'rgba(248, 113, 113, 0.8)', border: 'rgb(248, 113, 113)' },
+          pending: { background: 'rgba(251, 191, 36, 0.8)', border: 'rgb(251, 191, 36)' }
+        };
+
+        statusPieChart = new Chart(statusPieCtx, {
+          type: 'pie',
+          data: {
+            labels: data.status_pie.labels,
+            datasets: [{
+              data: data.status_pie.data,
+              backgroundColor: data.status_pie.labels.map(status =>
+                statusColors[status]?.background || 'rgba(156, 163, 175, 0.8)'
+              ),
+              borderColor: data.status_pie.labels.map(status =>
+                statusColors[status]?.border || 'rgb(156, 163, 175)'
+              ),
+              borderWidth: 1
+            }]
+          },
+          options: {
+            responsive: true,
+            maintainAspectRatio: false,
+            plugins: {
+              legend: {
+                position: 'right',
+                labels: {
+                  boxWidth: 12
+                }
+              }
+            }
+          }
+        });
+
       } catch (error) {
-        console.error('Error updating visits chart:', error);
+        console.error('Error updating charts:', error);
       }
     }
 
+    updateCharts();
 
-    console.log('updating visits chart on load');
-    updateVisitsChart();
-
-    // Update chart when filters change
+    // Update charts when filters change
     const formElement = document.querySelector('#filterForm form');
     formElement.querySelectorAll('select, input').forEach(input => {
-      input.addEventListener('change', updateVisitsChart);
+      input.addEventListener('change', updateCharts);
     });
   });
 

From 14be29bf8f2341584336e77bbc514a4d54eac229 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 15:35:58 +0200
Subject: [PATCH 88/96] style tweaks

---
 .../templates/reports/dashboard.html          | 24 +++++++++----------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index ee8e8f13..5874858c 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -90,24 +90,26 @@ <h3 class="my-2">Service Delivery Map</h3>
       </div>
     </div>
     <div class="col-12 col-md-6">
-      <h3 class="my-2">Visits over Time</h3>
-      <div style="position: relative; height: 200px; width: 100%;">
-        <canvas id="visits-over-time"></canvas>
-      </div>
-      <div class="row mt-4">
+      <h3 class="my-2">Visit Breakdown</h3>
+      <div class="row mb-4">
         <div class="col-12 col-md-6">
-          <h3 class="my-2">Visits by Program</h3>
+          <h5 class="my-2">By Program</h3>
           <div style="position: relative; height: 200px; width: 100%;">
             <canvas id="visits-by-program"></canvas>
           </div>
         </div>
         <div class="col-12 col-md-6">
-          <h3 class="my-2">Visits by Status</h3>
+          <h5 class="my-2">By Status</h3>
           <div style="position: relative; height: 200px; width: 100%;">
             <canvas id="visits-by-status"></canvas>
           </div>
         </div>
       </div>
+      <h5>Over time</h5>
+      <div style="position: relative; height: 200px; width: 100%;">
+        <canvas id="visits-over-time"></canvas>
+      </div>
+
     </div>
   </div>
 </div>
@@ -338,10 +340,6 @@ <h3 class="my-2">Visits by Status</h3>
             responsive: true,
             maintainAspectRatio: false,
             plugins: {
-              title: {
-                display: true,
-                text: 'Visits Over Time'
-              },
               tooltip: {
                 mode: 'index',
                 intersect: false,
@@ -388,7 +386,7 @@ <h3 class="my-2">Visits by Status</h3>
             maintainAspectRatio: false,
             plugins: {
               legend: {
-                position: 'right',
+                position: 'bottom',
                 labels: {
                   boxWidth: 12
                 }
@@ -428,7 +426,7 @@ <h3 class="my-2">Visits by Status</h3>
             maintainAspectRatio: false,
             plugins: {
               legend: {
-                position: 'right',
+                position: 'bottom',
                 labels: {
                   boxWidth: 12
                 }

From 6748eaec1bb7786240d504033f895d020b61b8f6 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 15:38:56 +0200
Subject: [PATCH 89/96] use a better name

---
 commcare_connect/reports/urls.py                  | 2 +-
 commcare_connect/reports/views.py                 | 2 +-
 commcare_connect/templates/reports/dashboard.html | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/commcare_connect/reports/urls.py b/commcare_connect/reports/urls.py
index bb8a9709..de9ca690 100644
--- a/commcare_connect/reports/urls.py
+++ b/commcare_connect/reports/urls.py
@@ -9,5 +9,5 @@
     path("delivery_stats", view=views.DeliveryStatsReportView.as_view(), name="delivery_stats_report"),
     path("api/visit_map_data/", views.visit_map_data, name="visit_map_data"),
     path("api/dashboard_stats/", views.dashboard_stats_api, name="dashboard_stats_api"),
-    path("api/visits_over_time/", views.visits_over_time_api, name="visits_over_time_api"),
+    path("api/dashboard_charts/", views.dashboard_charts_api, name="dashboard_charts_api"),
 ]
diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index 3439d419..e42c9805 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -422,7 +422,7 @@ def dashboard_stats_api(request):
 
 @login_required
 @user_passes_test(lambda u: u.is_superuser)
-def visits_over_time_api(request):
+def dashboard_charts_api(request):
     filterset = DashboardFilters(request.GET)
     queryset = UserVisit.objects.all()
     # Use the filtered queryset if available, else use last 30 days
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 5874858c..1b08d494 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -316,7 +316,7 @@ <h5>Over time</h5>
         const formData = new FormData(formElement);
         const queryString = new URLSearchParams(formData).toString();
 
-        const response = await fetch(`{% url 'reports:visits_over_time_api' %}?${queryString}`);
+        const response = await fetch(`{% url 'reports:dashboard_charts_api' %}?${queryString}`);
         const data = await response.json();
 
         // Update time series chart

From c02a374cf394aa0f4e200eef9baa48e7c096760b Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 15:42:14 +0200
Subject: [PATCH 90/96] refactor each chart to its own function

---
 commcare_connect/reports/views.py | 44 ++++++++++++++++---------------
 1 file changed, 23 insertions(+), 21 deletions(-)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index e42c9805..623fdc33 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -435,22 +435,23 @@ def dashboard_charts_api(request):
         from_date = to_date - timedelta(days=30)
         queryset = queryset.filter(visit_date__gte=from_date, visit_date__lte=to_date)
 
-    # Get data for all three charts
-    # 1. Visits over time by program
+    return JsonResponse(
+        {
+            "time_series": _get_time_series_data(queryset, from_date, to_date),
+            "program_pie": _get_program_pie_data(queryset),
+            "status_pie": _get_status_pie_data(queryset),
+        }
+    )
+
+
+def _get_time_series_data(queryset, from_date, to_date):
+    # Get visits over time by program
     visits_by_program_time = (
         queryset.values("visit_date", "opportunity__delivery_type__name")
         .annotate(count=Count("id"))
         .order_by("visit_date", "opportunity__delivery_type__name")
     )
 
-    # 2. Total visits by program
-    visits_by_program = (
-        queryset.values("opportunity__delivery_type__name").annotate(count=Count("id")).order_by("-count")
-    )
-
-    # 3. Visits by status
-    visits_by_status = queryset.values("status").annotate(count=Count("id")).order_by("-count")
-
     # Process time series data
     program_data = {}
     for visit in visits_by_program_time:
@@ -477,21 +478,22 @@ def dashboard_charts_api(request):
 
         time_datasets.append({"name": program_name or "Unknown", "data": data})
 
-    # Process pie chart data
-    program_pie_data = {
+    return {"labels": labels, "datasets": time_datasets}
+
+
+def _get_program_pie_data(queryset):
+    visits_by_program = (
+        queryset.values("opportunity__delivery_type__name").annotate(count=Count("id")).order_by("-count")
+    )
+    return {
         "labels": [item["opportunity__delivery_type__name"] or "Unknown" for item in visits_by_program],
         "data": [item["count"] for item in visits_by_program],
     }
 
-    status_pie_data = {
+
+def _get_status_pie_data(queryset):
+    visits_by_status = queryset.values("status").annotate(count=Count("id")).order_by("-count")
+    return {
         "labels": [item["status"] or "Unknown" for item in visits_by_status],
         "data": [item["count"] for item in visits_by_status],
     }
-
-    return JsonResponse(
-        {
-            "time_series": {"labels": labels, "datasets": time_datasets},
-            "program_pie": program_pie_data,
-            "status_pie": status_pie_data,
-        }
-    )

From 684143e416ad09134eeff799c5f7c0b168ddd961 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 15:45:02 +0200
Subject: [PATCH 91/96] add doc strings

---
 commcare_connect/reports/views.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/commcare_connect/reports/views.py b/commcare_connect/reports/views.py
index 623fdc33..47e105b1 100644
--- a/commcare_connect/reports/views.py
+++ b/commcare_connect/reports/views.py
@@ -445,6 +445,21 @@ def dashboard_charts_api(request):
 
 
 def _get_time_series_data(queryset, from_date, to_date):
+    """Example output:
+    {
+        "labels": ["Jan 01", "Jan 02", "Jan 03"],
+        "datasets": [
+            {
+                "name": "Program A",
+                "data": [5, 3, 7]
+            },
+            {
+                "name": "Program B",
+                "data": [2, 4, 1]
+            }
+        ]
+    }
+    """
     # Get visits over time by program
     visits_by_program_time = (
         queryset.values("visit_date", "opportunity__delivery_type__name")
@@ -482,6 +497,12 @@ def _get_time_series_data(queryset, from_date, to_date):
 
 
 def _get_program_pie_data(queryset):
+    """Example output:
+    {
+        "labels": ["Program A", "Program B", "Unknown"],
+        "data": [10, 5, 2]
+    }
+    """
     visits_by_program = (
         queryset.values("opportunity__delivery_type__name").annotate(count=Count("id")).order_by("-count")
     )
@@ -492,6 +513,12 @@ def _get_program_pie_data(queryset):
 
 
 def _get_status_pie_data(queryset):
+    """Example output:
+    {
+        "labels": ["Approved", "Pending", "Rejected", "Unknown"],
+        "data": [15, 8, 4, 1]
+    }
+    """
     visits_by_status = queryset.values("status").annotate(count=Count("id")).order_by("-count")
     return {
         "labels": [item["status"] or "Unknown" for item in visits_by_status],

From fb11e70fb43141d850b668b5ec91e0a3329ad64e Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 15:59:42 +0200
Subject: [PATCH 92/96] externalize js to js file

---
 commcare_connect/static/js/dashboard.js       | 131 ++++++++++++++++++
 .../templates/reports/dashboard.html          | 111 +--------------
 2 files changed, 134 insertions(+), 108 deletions(-)

diff --git a/commcare_connect/static/js/dashboard.js b/commcare_connect/static/js/dashboard.js
index d73535b3..0a761c6a 100644
--- a/commcare_connect/static/js/dashboard.js
+++ b/commcare_connect/static/js/dashboard.js
@@ -128,5 +128,136 @@ function donutSegment(start, end, r, r0, color) {
   }" fill="${color}" opacity="0.85" stroke="#1f2937" stroke-width="1" />`;
 }
 
+const chartColors = [
+  { border: 'rgb(75, 192, 192)', background: 'rgba(75, 192, 192, 0.8)' },
+  { border: 'rgb(255, 99, 132)', background: 'rgba(255, 99, 132, 0.8)' },
+  { border: 'rgb(255, 205, 86)', background: 'rgba(255, 205, 86, 0.8)' },
+  { border: 'rgb(54, 162, 235)', background: 'rgba(54, 162, 235, 0.8)' },
+];
+
+const statusColors = {
+  approved: {
+    background: 'rgba(74, 222, 128, 0.8)',
+    border: 'rgb(74, 222, 128)',
+  },
+  rejected: {
+    background: 'rgba(248, 113, 113, 0.8)',
+    border: 'rgb(248, 113, 113)',
+  },
+  pending: {
+    background: 'rgba(251, 191, 36, 0.8)',
+    border: 'rgb(251, 191, 36)',
+  },
+};
+
+function createTimeSeriesChart(ctx, data) {
+  return new Chart(ctx, {
+    type: 'bar',
+    data: {
+      labels: data.labels,
+      datasets: data.datasets.map((dataset, index) => ({
+        label: dataset.name,
+        data: dataset.data,
+        borderColor: chartColors[index % chartColors.length].border,
+        backgroundColor: chartColors[index % chartColors.length].background,
+        borderWidth: 1,
+      })),
+    },
+    options: {
+      responsive: true,
+      maintainAspectRatio: false,
+      plugins: {
+        tooltip: {
+          mode: 'index',
+          intersect: false,
+        },
+      },
+      scales: {
+        x: {
+          stacked: true,
+          title: {
+            display: true,
+            text: 'Date',
+          },
+        },
+        y: {
+          stacked: true,
+          beginAtZero: true,
+          title: {
+            display: true,
+            text: 'Number of Visits',
+          },
+        },
+      },
+    },
+  });
+}
+
+function createProgramPieChart(ctx, data) {
+  return new Chart(ctx, {
+    type: 'pie',
+    data: {
+      labels: data.labels,
+      datasets: [
+        {
+          data: data.data,
+          backgroundColor: chartColors.map((c) => c.background),
+          borderColor: chartColors.map((c) => c.border),
+          borderWidth: 1,
+        },
+      ],
+    },
+    options: {
+      responsive: true,
+      maintainAspectRatio: false,
+      plugins: {
+        legend: {
+          position: 'bottom',
+          labels: {
+            boxWidth: 12,
+          },
+        },
+      },
+    },
+  });
+}
+
+function createStatusPieChart(ctx, data) {
+  return new Chart(ctx, {
+    type: 'pie',
+    data: {
+      labels: data.labels,
+      datasets: [
+        {
+          data: data.data,
+          backgroundColor: data.labels.map(
+            (status) =>
+              statusColors[status]?.background || 'rgba(156, 163, 175, 0.8)',
+          ),
+          borderColor: data.labels.map(
+            (status) => statusColors[status]?.border || 'rgb(156, 163, 175)',
+          ),
+          borderWidth: 1,
+        },
+      ],
+    },
+    options: {
+      responsive: true,
+      maintainAspectRatio: false,
+      plugins: {
+        legend: {
+          position: 'bottom',
+          labels: {
+            boxWidth: 12,
+          },
+        },
+      },
+    },
+  });
+}
+
 window.updateMarkers = updateMarkers;
 window.createDonutChart = createDonutChart;
+window.createTimeSeriesChart = createTimeSeriesChart;
+window.createProgramPieChart = createProgramPieChart;
+window.createStatusPieChart = createStatusPieChart;
diff --git a/commcare_connect/templates/reports/dashboard.html b/commcare_connect/templates/reports/dashboard.html
index 1b08d494..5fe5a5b2 100644
--- a/commcare_connect/templates/reports/dashboard.html
+++ b/commcare_connect/templates/reports/dashboard.html
@@ -303,13 +303,6 @@ <h5>Over time</h5>
 
     let timeSeriesChart, programPieChart, statusPieChart;
 
-    const chartColors = [
-      { border: 'rgb(75, 192, 192)', background: 'rgba(75, 192, 192, 0.8)' },
-      { border: 'rgb(255, 99, 132)', background: 'rgba(255, 99, 132, 0.8)' },
-      { border: 'rgb(255, 205, 86)', background: 'rgba(255, 205, 86, 0.8)' },
-      { border: 'rgb(54, 162, 235)', background: 'rgba(54, 162, 235, 0.8)' },
-    ];
-
     async function updateCharts() {
       try {
         const formElement = document.querySelector('#filterForm form');
@@ -323,117 +316,19 @@ <h5>Over time</h5>
         if (timeSeriesChart) {
           timeSeriesChart.destroy();
         }
-
-        timeSeriesChart = new Chart(timeSeriesCtx, {
-          type: 'bar',
-          data: {
-            labels: data.time_series.labels,
-            datasets: data.time_series.datasets.map((dataset, index) => ({
-              label: dataset.name,
-              data: dataset.data,
-              borderColor: chartColors[index % chartColors.length].border,
-              backgroundColor: chartColors[index % chartColors.length].background,
-              borderWidth: 1
-            }))
-          },
-          options: {
-            responsive: true,
-            maintainAspectRatio: false,
-            plugins: {
-              tooltip: {
-                mode: 'index',
-                intersect: false,
-              }
-            },
-            scales: {
-              x: {
-                stacked: true,
-                title: {
-                  display: true,
-                  text: 'Date'
-                }
-              },
-              y: {
-                stacked: true,
-                beginAtZero: true,
-                title: {
-                  display: true,
-                  text: 'Number of Visits'
-                }
-              }
-            }
-          }
-        });
+        timeSeriesChart = createTimeSeriesChart(timeSeriesCtx, data.time_series);
 
         // Update program pie chart
         if (programPieChart) {
           programPieChart.destroy();
         }
-
-        programPieChart = new Chart(programPieCtx, {
-          type: 'pie',
-          data: {
-            labels: data.program_pie.labels,
-            datasets: [{
-              data: data.program_pie.data,
-              backgroundColor: chartColors.map(c => c.background),
-              borderColor: chartColors.map(c => c.border),
-              borderWidth: 1
-            }]
-          },
-          options: {
-            responsive: true,
-            maintainAspectRatio: false,
-            plugins: {
-              legend: {
-                position: 'bottom',
-                labels: {
-                  boxWidth: 12
-                }
-              }
-            }
-          }
-        });
+        programPieChart = createProgramPieChart(programPieCtx, data.program_pie);
 
         // Update status pie chart
         if (statusPieChart) {
           statusPieChart.destroy();
         }
-
-        const statusColors = {
-          approved: { background: 'rgba(74, 222, 128, 0.8)', border: 'rgb(74, 222, 128)' },
-          rejected: { background: 'rgba(248, 113, 113, 0.8)', border: 'rgb(248, 113, 113)' },
-          pending: { background: 'rgba(251, 191, 36, 0.8)', border: 'rgb(251, 191, 36)' }
-        };
-
-        statusPieChart = new Chart(statusPieCtx, {
-          type: 'pie',
-          data: {
-            labels: data.status_pie.labels,
-            datasets: [{
-              data: data.status_pie.data,
-              backgroundColor: data.status_pie.labels.map(status =>
-                statusColors[status]?.background || 'rgba(156, 163, 175, 0.8)'
-              ),
-              borderColor: data.status_pie.labels.map(status =>
-                statusColors[status]?.border || 'rgb(156, 163, 175)'
-              ),
-              borderWidth: 1
-            }]
-          },
-          options: {
-            responsive: true,
-            maintainAspectRatio: false,
-            plugins: {
-              legend: {
-                position: 'bottom',
-                labels: {
-                  boxWidth: 12
-                }
-              }
-            }
-          }
-        });
+        statusPieChart = createStatusPieChart(statusPieCtx, data.status_pie);
 
       } catch (error) {
         console.error('Error updating charts:', error);

From 9a426cee9821cdd463445959aae484ed05a244a2 Mon Sep 17 00:00:00 2001
From: Cory Zue <cory@coryzue.com>
Date: Wed, 13 Nov 2024 16:04:10 +0200
Subject: [PATCH 93/96] add better empty states

---
 commcare_connect/static/js/dashboard.js | 62 +++++++++++++++++++++++++
 1 file changed, 62 insertions(+)

diff --git a/commcare_connect/static/js/dashboard.js b/commcare_connect/static/js/dashboard.js
index 0a761c6a..53ca2b65 100644
--- a/commcare_connect/static/js/dashboard.js
+++ b/commcare_connect/static/js/dashboard.js
@@ -194,6 +194,37 @@ function createTimeSeriesChart(ctx, data) {
 }
 
 function createProgramPieChart(ctx, data) {
+  // Check if there's no data or empty data
+  if (!data?.data?.length) {
+    return new Chart(ctx, {
+      type: 'pie',
+      data: {
+        labels: ['No data'],
+        datasets: [
+          {
+            data: [1],
+            backgroundColor: ['rgba(156, 163, 175, 0.3)'],
+            borderColor: ['rgb(156, 163, 175)'],
+            borderWidth: 1,
+          },
+        ],
+      },
+      options: {
+        responsive: true,
+        maintainAspectRatio: false,
+        plugins: {
+          legend: {
+            position: 'bottom',
+            labels: {
+              boxWidth: 12,
+              color: 'rgb(156, 163, 175)',
+            },
+          },
+        },
+      },
+    });
+  }
+
   return new Chart(ctx, {
     type: 'pie',
     data: {
@@ -223,6 +254,37 @@ function createProgramPieChart(ctx, data) {
 }
 
 function createStatusPieChart(ctx, data) {
+  // Check if there's no data or empty data
+  if (!data?.data?.length) {
+    return new Chart(ctx, {
+      type: 'pie',
+      data: {
+        labels: ['No data'],
+        datasets: [
+          {
+            data: [1],
+            backgroundColor: ['rgba(156, 163, 175, 0.3)'],
+            borderColor: ['rgb(156, 163, 175)'],
+            borderWidth: 1,
+          },
+        ],
+      },
+      options: {
+        responsive: true,
+        maintainAspectRatio: false,
+        plugins: {
+          legend: {
+            position: 'bottom',
+            labels: {
+              boxWidth: 12,
+              color: 'rgb(156, 163, 175)',
+            },
+          },
+        },
+      },
+    });
+  }
+
   return new Chart(ctx, {
     type: 'pie',
     data: {

From 912d4c4baf90df2fa3b620c3880f9904aacfcbc0 Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Thu, 14 Nov 2024 14:51:33 +0530
Subject: [PATCH 94/96] Add PaymentUnit model to admin

---
 commcare_connect/opportunity/admin.py  | 12 +++++++++++-
 commcare_connect/opportunity/models.py |  3 +++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/commcare_connect/opportunity/admin.py b/commcare_connect/opportunity/admin.py
index 86839d6e..f1a01712 100644
--- a/commcare_connect/opportunity/admin.py
+++ b/commcare_connect/opportunity/admin.py
@@ -26,7 +26,6 @@
 
 
 admin.site.register(CommCareApp)
-admin.site.register(PaymentUnit)
 admin.site.register(UserInvite)
 admin.site.register(DeliveryType)
 admin.site.register(DeliverUnitFlagRules)
@@ -113,6 +112,7 @@ class AssessmentAdmin(admin.ModelAdmin):
 @admin.register(CompletedWork)
 class CompletedWorkAdmin(admin.ModelAdmin):
     list_display = ["get_username", "get_opp_name", "opportunity_access", "payment_unit", "status"]
+    search_fields = ["get_username", "get_opp_name"]
 
     @admin.display(description="Opportunity Name")
     def get_opp_name(self, obj):
@@ -121,3 +121,13 @@ def get_opp_name(self, obj):
     @admin.display(description="Username")
     def get_username(self, obj):
         return obj.opportunity_access.user.username
+
+
+@admin.register(PaymentUnit)
+class PaymentUnitAdmin(admin.ModelAdmin):
+    list_display = ["name", "get_opp_name"]
+    search_fields = ["name"]
+
+    @admin.display(description="Opportunity Name")
+    def get_opp_name(self, obj):
+        return obj.opportunity_access.opportunity.name
diff --git a/commcare_connect/opportunity/models.py b/commcare_connect/opportunity/models.py
index cf763f5d..a6351e64 100644
--- a/commcare_connect/opportunity/models.py
+++ b/commcare_connect/opportunity/models.py
@@ -364,6 +364,9 @@ class PaymentUnit(models.Model):
         null=True,
     )
 
+    def __str__(self):
+        return self.name
+
 
 class DeliverUnit(models.Model):
     app = models.ForeignKey(

From 1644c3a83944a9e0ad78fe3bbc737eaa66592c2c Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Fri, 15 Nov 2024 17:20:52 +0530
Subject: [PATCH 95/96] Fix linter error

---
 config/views.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/config/views.py b/config/views.py
index 3b7fc766..58135f30 100644
--- a/config/views.py
+++ b/config/views.py
@@ -8,23 +8,21 @@ def assetlinks_json(request):
             "target": {
                 "namespace": "android_app",
                 "package_name": "org.commcare.dalvik",
-                "sha256_cert_fingerprints":
-                [
+                "sha256_cert_fingerprints": [
                     "88:57:18:F8:E8:7D:74:04:97:AE:83:65:74:ED:EF:10:40:D9:4C:E2:54:F0:E0:40:64:77:96:7F:D1:39:F9:81",
-                    "89:55:DF:D8:0E:66:63:06:D2:6D:88:A4:A3:88:A4:D9:16:5A:C4:1A:7E:E1:C6:78:87:00:37:55:93:03:7B:03"
-                ]
-            }
+                    "89:55:DF:D8:0E:66:63:06:D2:6D:88:A4:A3:88:A4:D9:16:5A:C4:1A:7E:E1:C6:78:87:00:37:55:93:03:7B:03",
+                ],
+            },
         },
         {
             "relation": ["delegate_permission/common.handle_all_urls"],
             "target": {
                 "namespace": "android_app",
                 "package_name": "org.commcare.dalvik.debug",
-                "sha256_cert_fingerprints":
-                [
+                "sha256_cert_fingerprints": [
                     "88:57:18:F8:E8:7D:74:04:97:AE:83:65:74:ED:EF:10:40:D9:4C:E2:54:F0:E0:40:64:77:96:7F:D1:39:F9:81"
-                ]
-            }
+                ],
+            },
         },
     ]
     return JsonResponse(assetfile, safe=False)

From 9fdfd5a54d7a6477366790707eb9a84ff1101a7e Mon Sep 17 00:00:00 2001
From: Pawan Verma <pverma@dimagi.com>
Date: Tue, 19 Nov 2024 19:57:50 +0530
Subject: [PATCH 96/96] Add username and opportunity name to search fields

---
 commcare_connect/opportunity/admin.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/commcare_connect/opportunity/admin.py b/commcare_connect/opportunity/admin.py
index f1a01712..e833dd19 100644
--- a/commcare_connect/opportunity/admin.py
+++ b/commcare_connect/opportunity/admin.py
@@ -102,6 +102,7 @@ class CompletedModuleAdmin(admin.ModelAdmin):
 @admin.register(UserVisit)
 class UserVisitAdmin(admin.ModelAdmin):
     list_display = ["deliver_unit", "user", "opportunity", "status"]
+    search_fields = ["opportunity_access__user__username", "opportunity_access__opportunity__name"]
 
 
 @admin.register(Assessment)
@@ -112,7 +113,7 @@ class AssessmentAdmin(admin.ModelAdmin):
 @admin.register(CompletedWork)
 class CompletedWorkAdmin(admin.ModelAdmin):
     list_display = ["get_username", "get_opp_name", "opportunity_access", "payment_unit", "status"]
-    search_fields = ["get_username", "get_opp_name"]
+    search_fields = ["opportunity_access__user__username", "opportunity_access__opportunity__name"]
 
     @admin.display(description="Opportunity Name")
     def get_opp_name(self, obj):