Merge pull request #713 from hms-dbmi/feature/data-use-reports

Feature/data use reports

app/manage/urls.py

@@ -9,6 +9,7 @@
 from manage.views import team_notification
 from manage.views import UploadSignedAgreementFormView
 from manage.views import UploadSignedAgreementFormFileView
+from manage.views import ProjectDataUseReportParticipants
 
 from manage.api import set_dataproject_details
 from manage.api import set_dataproject_registration_status
@@ -62,6 +63,7 @@
     re_path(r'^remove-view-permission/(?P<project_key>[^/]+)/(?P<user_email>[^/]+)/$', remove_view_permission, name='remove-view-permission'),
     re_path(r'^get-project-participants/(?P<project_key>[^/]+)/$', ProjectParticipants.as_view(), name='get-project-participants'),
     re_path(r'^get-project-pending-participants/(?P<project_key>[^/]+)/$', ProjectPendingParticipants.as_view(), name='get-project-pending-participants'),
+    re_path(r'^get-project-data-use-reporting-participants/(?P<project_key>[^/]+)/$', ProjectDataUseReportParticipants.as_view(), name='get-project-data-use-reporting-participants'),
     re_path(r'^upload-signed-agreement-form/(?P<project_key>[^/]+)/(?P<user_email>[^/]+)/$', UploadSignedAgreementFormView.as_view(), name='upload-signed-agreement-form'),
     re_path(r'^upload-signed-agreement-form-file/(?P<signed_agreement_form_id>[^/]+)/$', UploadSignedAgreementFormFileView.as_view(), name='upload-signed-agreement-form-file'),
     re_path(r'^(?P<project_key>[^/]+)/$', DataProjectManageView.as_view(), name='manage-project'),

app/manage/views.py

@@ -340,8 +340,13 @@ def get(self, request, project_key, *args, **kwargs):
             signed_agreement_forms = []
             signed_accepted_agreement_forms = 0
 
+            # Get all agreement forms
+            agreement_forms = list(project.agreement_forms.all())
+            if project.data_use_report_agreement_form:
+                agreement_forms.append(project.data_use_report_agreement_form)
+
             # For each of the available agreement forms for this project, display only latest version completed by the user
-            for agreement_form in project.agreement_forms.all():
+            for agreement_form in agreement_forms:
 
                 # Check if this project uses shared agreement forms
                 if project.shares_agreement_forms:
@@ -460,6 +465,13 @@ def get(self, request, project_key, *args, **kwargs):
             # Filter
             participants_waiting_access = participants_waiting_access.filter(agreement_form_query)
 
+        # Do not include users whose access was removed due to data use reporting requirements
+        if project.data_use_report_agreement_form:
+
+            # Ensure there exists no data use reporting request for this user
+            data_use_report_query = Q(datausereportrequest__isnull=True)
+            participants_waiting_access = participants_waiting_access.filter(data_use_report_query)
+
         # Secondly, we want Participants with at least one pending SignedAgreementForm
         participants_awaiting_approval = Participant.objects.filter(Q(project=project, permission__isnull=True)).filter(
             Q(
@@ -566,6 +578,128 @@ def get(self, request, project_key, *args, **kwargs):
         return JsonResponse(data=data)
 
 
+@method_decorator(user_auth_and_jwt, name='dispatch')
+class ProjectDataUseReportParticipants(View):
+
+    def get(self, request, project_key, *args, **kwargs):
+
+        # Pull the project
+        try:
+            project = DataProject.objects.get(project_key=project_key)
+        except DataProject.NotFound:
+            logger.exception('DataProject for key "{}" not found'.format(project_key))
+            return HttpResponse(status=404)
+
+        # Get needed params
+        draw = int(request.GET['draw'])
+        start = int(request.GET['start'])
+        length = int(request.GET['length'])
+        order_column = int(request.GET['order[0][column]'])
+        order_direction = request.GET['order[0][dir]']
+
+        # Check for a search value
+        search = request.GET['search[value]']
+
+        # Check what we're sorting by and in what direction
+        if order_column == 0:
+            sort_order = ['email'] if order_direction == 'asc' else ['-email']
+        elif order_column == 3 and not project.has_teams or order_column == 4 and project.has_teams:
+            sort_order = ['modified', '-email'] if order_direction == 'asc' else ['-modified', 'email']
+        else:
+            sort_order = ['modified', '-email'] if order_direction == 'asc' else ['-modified', 'email']
+
+        # Build the query
+
+        # Find users with all access but pending data use report agreement forms
+        participants_waiting_access = Participant.objects.filter(
+            Q(
+                project=project,
+                user__signedagreementform__agreement_form=project.data_use_report_agreement_form,
+                user__signedagreementform__status="P",
+            )
+        )
+
+        # Add search if necessary
+        if search:
+            participants_waiting_access = participants_waiting_access.filter(user__email__icontains=search)
+
+        # We only want distinct Participants belonging to the users query
+        # Django won't sort on a related field after this union so we annotate each queryset with the user's email to sort on
+        query_set = participants_waiting_access.annotate(email=F("user__email")) \
+            .order_by(*sort_order)
+
+        # Setup paginator
+        paginator = Paginator(
+            query_set,
+            length,
+        )
+
+        # Determine page index (1-index) from DT parameters
+        page = start / length + 1
+        participant_page = paginator.page(page)
+
+        participants = []
+        for participant in participant_page:
+
+            signed_agreement_forms = []
+            signed_accepted_agreement_forms = 0
+
+            # Get all agreement forms
+            agreement_forms = list(project.agreement_forms.all()) + [project.data_use_report_agreement_form]
+
+            # Fetch only for this project
+            signed_forms = SignedAgreementForm.objects.filter(
+                user__email=participant.user.email,
+                project=project,
+                agreement_form__in=agreement_forms,
+            )
+            for signed_form in signed_forms:
+                if signed_form is not None:
+                    signed_agreement_forms.append(signed_form)
+
+                # Collect how many forms are approved to craft language for status
+                if signed_form.status == 'A':
+                    signed_accepted_agreement_forms += 1
+
+            # Build the row of the table for this participant
+            participant_row = [
+                participant.user.email.lower(),
+                'Access granted' if participant.permission == 'VIEW' else 'No access',
+                [
+                    {
+                        'status': f.status,
+                        'id': f.id,
+                        'name': f.agreement_form.short_name,
+                        'project': f.project.project_key,
+                    } for f in signed_agreement_forms
+                ],
+                {
+                    'access': True if participant.permission == 'VIEW' else False,
+                    'email': participant.user.email.lower(),
+                    'signed': signed_accepted_agreement_forms,
+                    'team': True if project.has_teams else False,
+                    'required': project.agreement_forms.count()
+                },
+                participant.modified,
+            ]
+
+            # If project has teams, add that
+            if project.has_teams:
+                participant_row.insert(1, participant.team.team_leader.email.lower() if participant.team and participant.team.team_leader else '')
+
+            participants.append(participant_row)
+
+        # Build DataTables response data
+        data = {
+            'draw': draw,
+            'recordsTotal': query_set.count(),
+            'recordsFiltered': paginator.count,
+            'data': participants,
+            'error': None,
+        }
+
+        return JsonResponse(data=data)
+
 @user_auth_and_jwt
 def team_notification(request, project_key=None):
     """

app/projects/admin.py

@@ -17,6 +17,7 @@
 from projects.models import ChallengeTaskSubmissionDownload
 from projects.models import Bucket
 from projects.models import InstitutionalOfficial
+from projects.models import DataUseReportRequest
 
 
 class GroupAdmin(admin.ModelAdmin):
@@ -91,6 +92,9 @@ class ChallengeTaskSubmissionDownloadAdmin(admin.ModelAdmin):
     list_display = ('user', 'submission', 'download_date')
     search_fields = ('user__email', )
 
+class DataUseReportRequestAdmin(admin.ModelAdmin):
+    list_display = ('participant', 'data_project', 'created', 'modified')
+    search_fields = ('participant__user__email', )
 
 admin.site.register(Group, GroupAdmin)
 admin.site.register(Bucket, BucketAdmin)
@@ -107,3 +111,4 @@ class ChallengeTaskSubmissionDownloadAdmin(admin.ModelAdmin):
 admin.site.register(ChallengeTask, ChallengeTaskAdmin)
 admin.site.register(ChallengeTaskSubmission, ChallengeTaskSubmissionAdmin)
 admin.site.register(ChallengeTaskSubmissionDownload, ChallengeTaskSubmissionDownloadAdmin)
+admin.site.register(DataUseReportRequest, DataUseReportRequestAdmin)

app/projects/api.py

@@ -1,6 +1,7 @@
 from copy import copy
 from datetime import datetime
 import json
+import importlib
 import logging
 
 from django.conf import settings
@@ -663,7 +664,7 @@ def save_signed_agreement_form(request):
                     "warning", f"The agreement form contained errors, please review", "warning-sign"
                 )
                 return response
-            
+
             # Use the data from the form
             fields = form.cleaned_data
 
@@ -762,6 +763,20 @@ def save_signed_agreement_form(request):
     # Save the agreement form
     signed_agreement_form.save()
 
+    # Check for a handler
+    if agreement_form.handler:
+        try:
+            # Build function components
+            module_name, handler_name = agreement_form.handler.rsplit('.', 1)
+            module = importlib.import_module(module_name)
+
+            # Call handler
+            getattr(module, handler_name)(signed_agreement_form)
+            logger.debug(f"Handler '{agreement_form.handler}' called for SignedAgreementForm")
+
+        except Exception as e:
+            logger.exception(f"Error calling handler: {e}", exc_info=True)
+
     return HttpResponse(status=200)
 
 @user_auth_and_jwt

app/projects/forms.py

@@ -6,7 +6,10 @@
 from django.utils.translation import gettext_lazy as _
 
 from hypatio.scireg_services import get_current_user_profile
-from projects.models import DataProject, AgreementForm
+from projects.models import DataProject
+from projects.models import AgreementForm
+from projects.models import SignedAgreementForm
+from projects.models import Participant
 
 
 class MultiValueValidationError(ValidationError):
@@ -200,3 +203,27 @@ def set_initial(self, request: HttpRequest, project: DataProject, agreement_form
         }
         '''
         return initial
+
+
+def data_use_report_handler(signed_agreement_form: SignedAgreementForm):
+    """
+    Handler the result of the data use report. This will be determining
+    whether the user's access is ended or paused.
+
+    :param signed_agreement_form: The saved data use report agreement form
+    :type signed_agreement_form: SignedAgreementForm
+    """
+    # The name of the field we are interesed in
+    USING_DATA = "using_data"
+
+    # Check value
+    if signed_agreement_form.fields.get(USING_DATA) in ["No", "no"]:
+
+        # End this user's access immediately
+        participant = Participant.objects.get(project=signed_agreement_form.project, user=signed_agreement_form.user)
+        participant.permission = None
+        participant.save()
+
+        # Auto-approve this signed agreement form since no review is necessary
+        signed_agreement_form.status = "A"
+        signed_agreement_form.save()

app/projects/migrations/0108_dataproject_access_reporting_agreement_form_and_more.py

@@ -0,0 +1,29 @@
+# Generated by Django 4.2.16 on 2024-09-27 16:22
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('projects', '0107_agreementform_form_class'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='dataproject',
+            name='data_use_report_agreement_form',
+            field=models.ForeignKey(blank=True, help_text='The agreement form that will be filled out by a user with access during periodic access and data use reporting', null=True, on_delete=django.db.models.deletion.SET_NULL, to='projects.agreementform'),
+        ),
+        migrations.AddField(
+            model_name='dataproject',
+            name='data_use_report_grace_period',
+            field=models.IntegerField(blank=True, help_text='The number of days in which a user is allotted to complete their access and data use reporting before their access is revoked', null=True),
+        ),
+        migrations.AddField(
+            model_name='dataproject',
+            name='data_use_report_period',
+            field=models.IntegerField(blank=True, help_text='The number of days after access being granted in which emails will be sent prompting users to report on the status of their access and use of data', null=True),
+        ),
+    ]

app/projects/migrations/0109_datausereportrequest.py

@@ -0,0 +1,29 @@
+# Generated by Django 4.2.16 on 2024-10-09 12:26
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('projects', '0108_dataproject_access_reporting_agreement_form_and_more'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='DataUseReportRequest',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('modified', models.DateTimeField(auto_now=True)),
+                ('data_project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='projects.dataproject')),
+                ('participant', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='projects.participant')),
+                ('signed_agreement_form', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='projects.signedagreementform')),
+            ],
+            options={
+                'verbose_name': 'Data Use Report Request',
+                'verbose_name_plural': 'Data Use Report Requests',
+            },
+        ),
+    ]

app/projects/migrations/0110_agreementform_handler.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.16 on 2024-10-10 13:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('projects', '0109_datausereportrequest'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='agreementform',
+            name='handler',
+            field=models.CharField(blank=True, help_text="Set an absolute function's path to be called after the SignedAgreementForm has successfully saved", max_length=512, null=True),
+        ),
+    ]