Merge pull request #139 from dimagi/sk/deliver-units

Use Deliver Units instead of 'forms'

commcare_connect/form_receiver/processor.py

@@ -8,7 +8,7 @@
     Assessment,
     CommCareApp,
     CompletedModule,
-    DeliverForm,
+    DeliverUnit,
     LearnModule,
     Opportunity,
     UserVisit,
@@ -17,20 +17,21 @@
 
 LEARN_MODULE_JSONPATH = parse("$..module")
 ASSESSMENT_JSONPATH = parse("$..assessment")
+DELIVER_UNIT_JSONPATH = parse("$..deliver")
 
 
 def process_xform(xform: XForm):
     """Process a form received from CommCare HQ."""
     app = get_app(xform.domain, xform.app_id)
     user = get_user(xform)
 
-    if process_deliver_form(user, xform):
-        return
+    opportunity = get_opportunity(deliver_app=app)
+    if opportunity:
+        process_deliver_form(user, xform, app, opportunity)
 
-    opportunity = get_opportunity_for_learn_app(app)
-    if not opportunity:
-        raise ProcessingError(f"No active opportunities found for CommCare app {app.cc_app_id}.")
-    process_learn_form(user, xform, app, opportunity)
+    opportunity = get_opportunity(learn_app=app)
+    if opportunity:
+        process_learn_form(user, xform, app, opportunity)
 
 
 def process_learn_form(user, xform: XForm, app: CommCareApp, opportunity: Opportunity):
@@ -121,35 +122,58 @@ def process_assessments(user, xform: XForm, app: CommCareApp, opportunity: Oppor
             return ProcessingError("Learn Assessment is already completed")
 
 
-def process_deliver_form(user, xform):
-    try:
-        deliver_form = DeliverForm.objects.filter(
-            xmlns=xform.xmlns, app__cc_domain=xform.domain, app__cc_app_id=xform.app_id
-        ).get()
-    except DeliverForm.DoesNotExist:
-        return False
-    except DeliverForm.MultipleObjectsReturned:
-        raise ProcessingError(f"Multiple deliver forms found for this app and XMLNS: {xform.app_id}, {xform.xmlns}")
+def process_deliver_form(user, xform: XForm, app: CommCareApp, opportunity: Opportunity):
+    matches = [
+        match.value for match in DELIVER_UNIT_JSONPATH.find(xform.form) if match.value["@xmlns"] == CCC_LEARN_XMLNS
+    ]
+    if matches:
+        for deliver_unit_block in matches:
+            process_deliver_unit(user, xform, app, opportunity, deliver_unit_block)
+
 
+def process_deliver_unit(user, xform: XForm, app: CommCareApp, opportunity: Opportunity, deliver_unit_block: dict):
+    deliver_unit = get_or_create_deliver_unit(app, deliver_unit_block)
     UserVisit.objects.create(
-        opportunity=deliver_form.opportunity,
+        opportunity=opportunity,
         user=user,
-        deliver_form=deliver_form,
+        deliver_unit=deliver_unit,
+        entity_id=deliver_unit_block.get("entity_id"),
+        entity_name=deliver_unit_block.get("entity_name"),
         visit_date=xform.metadata.timeStart,
         xform_id=xform.id,
         app_build_id=xform.build_id,
         app_build_version=xform.metadata.app_build_version,
         form_json=xform.raw_form,
     )
-    return True
 
 
-def get_opportunity_for_learn_app(app):
+def get_or_create_deliver_unit(app, unit_data):
+    unit, _ = DeliverUnit.objects.get_or_create(
+        app=app,
+        slug=unit_data["@id"],
+        defaults={
+            "name": unit_data["name"],
+        },
+    )
+    return unit
+
+
+def get_opportunity(*, learn_app=None, deliver_app=None):
+    if not learn_app and not deliver_app:
+        raise ValueError("One of learn_app or deliver_app must be provided")
+
+    kwargs = {}
+    if learn_app:
+        kwargs = {"learn_app": learn_app}
+    if deliver_app:
+        kwargs = {"deliver_app": deliver_app}
+
     try:
-        return Opportunity.objects.get(learn_app=app, active=True)
+        return Opportunity.objects.get(active=True, **kwargs)
     except Opportunity.DoesNotExist:
         pass
     except Opportunity.MultipleObjectsReturned:
+        app = learn_app or deliver_app
         raise ProcessingError(f"Multiple active opportunities found for CommCare app {app.cc_app_id}.")
 
 

commcare_connect/form_receiver/tests/test_process_xform.py

@@ -2,48 +2,56 @@
 from unittest import mock
 
 from commcare_connect.form_receiver.processor import process_deliver_form, process_learn_form
-from commcare_connect.form_receiver.tests.xforms import AssessmentStubFactory, LearnModuleJsonFactory, get_form_model
-from commcare_connect.opportunity.models import UserVisit
-from commcare_connect.opportunity.tests.factories import OpportunityFactory
-from commcare_connect.users.models import User
+from commcare_connect.form_receiver.tests.xforms import (
+    AssessmentStubFactory,
+    DeliverUnitStubFactory,
+    LearnModuleJsonFactory,
+    get_form_model,
+)
 
 LEARN_PROCESSOR_PATCHES = [
     "commcare_connect.form_receiver.processor.process_learn_modules",
     "commcare_connect.form_receiver.processor.process_assessments",
 ]
 
 
-def test_process_learn_form_no_matching_blocks(user: User):
+def test_process_learn_form_no_matching_blocks():
     with mock.patch("commcare_connect.form_receiver.processor.process_learn_modules") as process_learn_modules:
-        process_learn_form(user, get_form_model(), None, None)
+        process_learn_form(None, get_form_model(), None, None)
     assert process_learn_modules.call_count == 0
 
 
-def test_process_learn_module(user: User):
+def test_process_learn_module():
     learn_module = LearnModuleJsonFactory().json
     xform = get_form_model(form_block=learn_module)
     with patch_multiple(*LEARN_PROCESSOR_PATCHES) as [process_learn_module, process_assessment]:
-        process_learn_form(user, xform, None, None)
+        process_learn_form(None, xform, None, None)
     assert process_learn_module.call_count == 1
     assert process_assessment.call_count == 0
 
 
-def test_process_assessment(user: User):
+def test_process_assessment():
     assessment = AssessmentStubFactory().json
     xform = get_form_model(form_block=assessment)
     with patch_multiple(*LEARN_PROCESSOR_PATCHES) as [process_learn_module, process_assessment]:
-        process_learn_form(user, xform, None, None)
+        process_learn_form(None, xform, None, None)
     assert process_learn_module.call_count == 0
     assert process_assessment.call_count == 1
 
 
-def test_process_deliver_form(user: User):
-    opportunity = OpportunityFactory()
-    deliver_form = opportunity.deliver_form.first()
-    app = deliver_form.app
-    xform = get_form_model(xmlns=deliver_form.xmlns, app_id=app.cc_app_id, domain=app.cc_domain)
-    assert process_deliver_form(user, xform)
-    assert UserVisit.objects.filter(user=user, deliver_form=deliver_form, xform_id=xform.id).count() == 1
+def test_process_deliver_form():
+    deliver_block = DeliverUnitStubFactory().json
+    xform = get_form_model(form_block=deliver_block)
+    with mock.patch("commcare_connect.form_receiver.processor.process_deliver_unit") as process_deliver_unit:
+        process_deliver_form(None, xform, None, None)
+    assert process_deliver_unit.call_count == 1
+
+
+def test_process_deliver_form_no_matches():
+    xform = get_form_model()
+    with mock.patch("commcare_connect.form_receiver.processor.process_deliver_unit") as process_deliver_unit:
+        process_deliver_form(None, xform, None, None)
+    assert process_deliver_unit.call_count == 0
 
 
 @contextmanager

commcare_connect/form_receiver/tests/test_receiver_integration.py

@@ -2,9 +2,14 @@
 from rest_framework.test import APIClient
 
 from commcare_connect.form_receiver.tests.test_receiver_endpoint import add_credentials
-from commcare_connect.form_receiver.tests.xforms import AssessmentStubFactory, LearnModuleJsonFactory, get_form_json
+from commcare_connect.form_receiver.tests.xforms import (
+    AssessmentStubFactory,
+    DeliverUnitStubFactory,
+    LearnModuleJsonFactory,
+    get_form_json,
+)
 from commcare_connect.opportunity.models import Assessment, CompletedModule, LearnModule, Opportunity, UserVisit
-from commcare_connect.opportunity.tests.factories import LearnModuleFactory, OpportunityFactory
+from commcare_connect.opportunity.tests.factories import DeliverUnitFactory, LearnModuleFactory, OpportunityFactory
 from commcare_connect.users.models import ConnectIDUserLink, User
 from commcare_connect.users.tests.factories import MobileUserFactory
 
@@ -100,16 +105,21 @@ def test_form_receiver_assessment(
 
 @pytest.mark.django_db
 def test_receiver_deliver_form(mobile_user_with_connect_link: User, api_client: APIClient, opportunity: Opportunity):
-    deliver_form = opportunity.deliver_form.first()
+    deliver_unit = DeliverUnitFactory(app=opportunity.deliver_app)
+    stub = DeliverUnitStubFactory(id=deliver_unit.slug)
     form_json = get_form_json(
-        xmlns=deliver_form.xmlns,
-        domain=opportunity.deliver_app.cc_domain,
-        app_id=opportunity.deliver_app.cc_app_id,
+        form_block=stub.json,
+        domain=deliver_unit.app.cc_domain,
+        app_id=deliver_unit.app.cc_app_id,
     )
     assert UserVisit.objects.filter(user=mobile_user_with_connect_link).count() == 0
 
     make_request(api_client, form_json, mobile_user_with_connect_link)
     assert UserVisit.objects.filter(user=mobile_user_with_connect_link).count() == 1
+    visit = UserVisit.objects.get(user=mobile_user_with_connect_link)
+    assert visit.deliver_unit == deliver_unit
+    assert visit.entity_id == stub.entity_id
+    assert visit.entity_name == stub.entity_name
 
 
 def _get_form_json(learn_app, module_id, form_block=None):
@@ -124,4 +134,5 @@ def _get_form_json(learn_app, module_id, form_block=None):
 def make_request(api_client, form_json, user, expected_status_code=200):
     add_credentials(api_client, user)
     response = api_client.post("/api/receiver/", data=form_json, format="json")
+    print(response.data)
     assert response.status_code == expected_status_code, response.data

commcare_connect/form_receiver/tests/xforms.py

@@ -53,6 +53,17 @@
     % CCC_LEARN_XMLNS
 )
 
+DELIVER_UNIT_XML_TEMPLATE = (
+    """<data>
+<deliver xmlns="%s" id="{id}">
+    <name>{name}</name>
+    <entity_id>{entity_id}</entity_id>
+    <entity_name>{entity_name}</entity_name>
+</deliver>
+</data>"""
+    % CCC_LEARN_XMLNS
+)
+
 
 def get_form_json(xmlns=DEFAULT_XMLNS, form_block=None, **kwargs):
     form = deepcopy(MOCK_FORM)
@@ -78,15 +89,11 @@ class LearnModuleJsonFactory(factory.StubFactory):
 
     @factory.lazy_attribute
     def json(self):
-        return _get_learn_module_json(self)
-
-
-def _get_learn_module_json(stub):
-    xml = MODULE_XML_TEMPLATE.format(
-        id=stub.id, name=stub.name, description=stub.description, time_estimate=stub.time_estimate
-    )
-    _, module = xml2json(xml)
-    return module
+        xml = MODULE_XML_TEMPLATE.format(
+            id=self.id, name=self.name, description=self.description, time_estimate=self.time_estimate
+        )
+        _, module = xml2json(xml)
+        return module
 
 
 class AssessmentStubFactory(factory.StubFactory):
@@ -95,10 +102,21 @@ class AssessmentStubFactory(factory.StubFactory):
 
     @factory.lazy_attribute
     def json(self):
-        return _get_assessment_json(self)
+        xml = ASSESSMENT_XML_TEMPLATE.format(id=self.id, score=self.score)
+        _, module = xml2json(xml)
+        return module
+
 
+class DeliverUnitStubFactory(factory.StubFactory):
+    id = factory.Faker("slug")
+    name = factory.Faker("name")
+    entity_id = factory.Faker("uuid4")
+    entity_name = factory.Faker("name")
 
-def _get_assessment_json(stub):
-    xml = ASSESSMENT_XML_TEMPLATE.format(id=stub.id, score=stub.score)
-    _, module = xml2json(xml)
-    return module
+    @factory.lazy_attribute
+    def json(self):
+        xml = DELIVER_UNIT_XML_TEMPLATE.format(
+            id=self.id, name=self.name, entity_id=self.entity_id, entity_name=self.entity_name
+        )
+        _, module = xml2json(xml)
+        return module

commcare_connect/opportunity/admin.py

@@ -5,7 +5,7 @@
     Assessment,
     CommCareApp,
     CompletedModule,
-    DeliverForm,
+    DeliverUnit,
     LearnModule,
     Opportunity,
     OpportunityAccess,
@@ -18,7 +18,7 @@
 
 admin.site.register(Opportunity)
 admin.site.register(CommCareApp)
-admin.site.register(DeliverForm)
+admin.site.register(DeliverUnit)
 admin.site.register(LearnModule)
 admin.site.register(CompletedModule)
 admin.site.register(Assessment)

commcare_connect/opportunity/export.py

@@ -18,13 +18,17 @@ def export_user_visit_data(
         user_visits = user_visits.filter(status__in=status)
 
     table = UserVisitTable(user_visits)
-    exclude_columns = ("visit_date",)
+    exclude_columns = ("visit_date", "form_json")
     columns = [
         column
         for column in table.columns.iterall()
         if not (column.column.exclude_from_export or column.name in exclude_columns)
     ]
-    base_data = [[row.get_cell_value(column.name) for column in columns] for row in table.rows]
+    base_data = [
+        # form_json must be the last column in the row
+        [row.get_cell_value(column.name) for column in columns] + [row.get_cell_value("form_json")]
+        for row in table.rows
+    ]
     base_headers = [force_str(column.header, strings_only=True) for column in columns]
     return get_flattened_dataset(base_headers, base_data)
 
@@ -45,7 +49,7 @@ def get_flattened_dataset(headers: list[str], data: list[list]) -> Dataset:
         schema.update(flat_json.keys())
 
     schema = sorted(schema, key=_schema_sort)
-    headers = headers[:-1] + schema
+    headers = headers + schema
     dataset = Dataset(title="Export", headers=headers)
 
     for row, flat_json in zip(data, flat_data):