From 4312439dc619631edb6755bf273e467f4b047ceb Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Mon, 7 Feb 2022 12:03:09 -0500 Subject: [PATCH 001/150] normal users no longer have the option to filter on username in job list view --- coldfront/core/statistics/forms.py | 4 ++++ .../core/statistics/templates/job_list.html | 4 +++- coldfront/core/statistics/views.py | 19 ++++++++++++++++--- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/coldfront/core/statistics/forms.py b/coldfront/core/statistics/forms.py index 832bcdcca..481d936fa 100644 --- a/coldfront/core/statistics/forms.py +++ b/coldfront/core/statistics/forms.py @@ -124,6 +124,7 @@ def clean(self): def __init__(self, *args, **kwargs): user = kwargs.pop('user', None) + is_pi = kwargs.pop('is_pi', None) super(JobSearchForm, self).__init__(*args, **kwargs) if user: @@ -131,6 +132,9 @@ def __init__(self, *args, **kwargs): user.has_perm('statistics.view_job')): self.fields.pop('show_all_jobs') + if not is_pi: + self.fields.pop('username') + self.fields['submitdate'].label = '' self.fields['submit_modifier'].label = '' self.fields['startdate'].label = '' diff --git a/coldfront/core/statistics/templates/job_list.html b/coldfront/core/statistics/templates/job_list.html index 004b69920..7d6ab19bb 100644 --- a/coldfront/core/statistics/templates/job_list.html +++ b/coldfront/core/statistics/templates/job_list.html @@ -62,7 +62,9 @@

Job List

{{ job_search_form.status|as_crispy_field }}
{{ job_search_form.jobslurmid|as_crispy_field }}
{{ job_search_form.project_name|as_crispy_field }}
-
{{ job_search_form.username|as_crispy_field }}
+ {% if is_pi %} +
{{ job_search_form.username|as_crispy_field }}
+ {% endif %}
{{ job_search_form.partition|as_crispy_field }}
Service Units {% include 'service_unit_popover.html' %}
diff --git a/coldfront/core/statistics/views.py b/coldfront/core/statistics/views.py index e25c467d8..7416bf531 100644 --- a/coldfront/core/statistics/views.py +++ b/coldfront/core/statistics/views.py @@ -12,7 +12,7 @@ from django.views.generic import DetailView, ListView, TemplateView from django.utils.html import strip_tags -from coldfront.core.project.models import Project +from coldfront.core.project.models import Project, ProjectUser from coldfront.core.statistics.models import Job from coldfront.core.statistics.forms import JobSearchForm @@ -53,7 +53,12 @@ def get_queryset(self): else: order_by = '-submitdate' - job_search_form = JobSearchForm(self.request.GET, user=self.request.user) + is_pi = ProjectUser.objects.filter( + role__name__in=['Manager', 'Principal Investigator'], + user=self.request.user) + job_search_form = JobSearchForm(self.request.GET, + user=self.request.user, + is_pi=is_pi) if job_search_form.is_valid(): data = job_search_form.cleaned_data @@ -85,7 +90,13 @@ def get_queryset(self): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - job_search_form = JobSearchForm(self.request.GET, user=self.request.user) + is_pi = ProjectUser.objects.filter( + role__name__in=['Manager', 'Principal Investigator'], + user=self.request.user) + job_search_form = JobSearchForm(self.request.GET, + user=self.request.user, + is_pi=is_pi) + if job_search_form.is_valid(): context['job_search_form'] = job_search_form data = job_search_form.cleaned_data @@ -145,6 +156,8 @@ def get_context_data(self, **kwargs): self.request.user.is_superuser or \ self.request.user.has_perm('statistics.view_job') + context['is_pi'] = not (self.request.user.is_superuser or self.request.user.has_perm('statistics.view_job')) and is_pi + return context From 7f39d4b91a5370351103629472d2e592c139c9c5 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Mon, 7 Feb 2022 15:52:53 -0500 Subject: [PATCH 002/150] added select widget to project name filter --- coldfront/core/statistics/forms.py | 26 ++++++++--- .../core/statistics/templates/job_list.html | 45 +++++++++++-------- 2 files changed, 47 insertions(+), 24 deletions(-) diff --git a/coldfront/core/statistics/forms.py b/coldfront/core/statistics/forms.py index 481d936fa..433ce76d5 100644 --- a/coldfront/core/statistics/forms.py +++ b/coldfront/core/statistics/forms.py @@ -1,5 +1,6 @@ from django import forms from django.core.exceptions import ValidationError +from coldfront.core.project.models import Project, ProjectUser class JobSearchForm(forms.Form): @@ -36,8 +37,12 @@ class JobSearchForm(forms.Form): jobslurmid = forms.CharField(label='Slurm ID', max_length=150, required=False) - project_name = forms.CharField(label='Project Name', - max_length=100, required=False) + project_name = forms.CharField( + label='Project Name', + max_length=100, + required=False, + widget=forms.Select()) + username = forms.CharField( label='Username', max_length=100, required=False) @@ -126,15 +131,21 @@ def __init__(self, *args, **kwargs): user = kwargs.pop('user', None) is_pi = kwargs.pop('is_pi', None) super(JobSearchForm, self).__init__(*args, **kwargs) + queryset = Project.objects.all() if user: - if not (user.is_superuser or - user.has_perm('statistics.view_job')): - self.fields.pop('show_all_jobs') - + if not (user.is_superuser or user.has_perm('statistics.view_job')): if not is_pi: self.fields.pop('username') + self.fields.pop('show_all_jobs') + + set = ProjectUser.objects.filter(user=user, + status__name='Active').\ + distinct('project').values_list('project__name') + queryset = \ + Project.objects.filter(name__in=set) + self.fields['submitdate'].label = '' self.fields['submit_modifier'].label = '' self.fields['startdate'].label = '' @@ -143,3 +154,6 @@ def __init__(self, *args, **kwargs): self.fields['end_modifier'].label = '' self.fields['amount'].label = '' self.fields['amount_modifier'].label = '' + + self.fields['project_name'].widget.choices = \ + [('', '-----')] + [(project.name, project.name) for project in queryset] diff --git a/coldfront/core/statistics/templates/job_list.html b/coldfront/core/statistics/templates/job_list.html index 7d6ab19bb..476a52091 100644 --- a/coldfront/core/statistics/templates/job_list.html +++ b/coldfront/core/statistics/templates/job_list.html @@ -246,30 +246,39 @@

Job List

{% endif %} - + }); + + + + + {% endblock %} {% endtimezone %} \ No newline at end of file From 8ab517ed7a235c38d7c0fa8ebc4763fd73323332 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Mon, 7 Feb 2022 16:01:13 -0500 Subject: [PATCH 003/150] moved scripts to top --- coldfront/core/statistics/templates/job_list.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/coldfront/core/statistics/templates/job_list.html b/coldfront/core/statistics/templates/job_list.html index 476a52091..b92b728be 100644 --- a/coldfront/core/statistics/templates/job_list.html +++ b/coldfront/core/statistics/templates/job_list.html @@ -11,6 +11,9 @@ {% block content %} + + +

Job List

@@ -270,8 +273,6 @@

Job List

}); - - - - {% endblock %} {% endtimezone %} \ No newline at end of file From 29a05187ca81dfeb6ddfaa0431550d255e41b04b Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Tue, 8 Feb 2022 09:57:00 -0500 Subject: [PATCH 005/150] added placeholder to service units input --- coldfront/core/statistics/forms.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/coldfront/core/statistics/forms.py b/coldfront/core/statistics/forms.py index 433ce76d5..61e977ae0 100644 --- a/coldfront/core/statistics/forms.py +++ b/coldfront/core/statistics/forms.py @@ -48,7 +48,10 @@ class JobSearchForm(forms.Form): partition = forms.CharField(label='Partition', max_length=100, required=False) - amount = forms.FloatField(label='Service Units', required=False) + amount = forms.FloatField(label='Service Units', + required=False, + widget=forms.NumberInput( + attrs={'placeholder': 'Number of Service Units'})) amount_modifier = forms.ChoiceField( choices=AMOUNT_MODIFIER, From 21ba815d498cfa84829a0e96dfa3282656282f9d Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Tue, 8 Feb 2022 10:00:09 -0500 Subject: [PATCH 006/150] admins can now see username search field --- coldfront/core/statistics/templates/job_list.html | 2 +- coldfront/core/statistics/views.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/coldfront/core/statistics/templates/job_list.html b/coldfront/core/statistics/templates/job_list.html index c7c0a4fd3..2810d93c6 100644 --- a/coldfront/core/statistics/templates/job_list.html +++ b/coldfront/core/statistics/templates/job_list.html @@ -65,7 +65,7 @@

Job List

{{ job_search_form.status|as_crispy_field }}
{{ job_search_form.jobslurmid|as_crispy_field }}
{{ job_search_form.project_name|as_crispy_field }}
- {% if is_pi %} + {% if show_username %}
{{ job_search_form.username|as_crispy_field }}
{% endif %}
{{ job_search_form.partition|as_crispy_field }}
diff --git a/coldfront/core/statistics/views.py b/coldfront/core/statistics/views.py index 7416bf531..91a46775f 100644 --- a/coldfront/core/statistics/views.py +++ b/coldfront/core/statistics/views.py @@ -55,7 +55,7 @@ def get_queryset(self): is_pi = ProjectUser.objects.filter( role__name__in=['Manager', 'Principal Investigator'], - user=self.request.user) + user=self.request.user).exists() job_search_form = JobSearchForm(self.request.GET, user=self.request.user, is_pi=is_pi) @@ -92,7 +92,7 @@ def get_context_data(self, **kwargs): is_pi = ProjectUser.objects.filter( role__name__in=['Manager', 'Principal Investigator'], - user=self.request.user) + user=self.request.user).exists() job_search_form = JobSearchForm(self.request.GET, user=self.request.user, is_pi=is_pi) @@ -156,7 +156,7 @@ def get_context_data(self, **kwargs): self.request.user.is_superuser or \ self.request.user.has_perm('statistics.view_job') - context['is_pi'] = not (self.request.user.is_superuser or self.request.user.has_perm('statistics.view_job')) and is_pi + context['show_username'] = (self.request.user.is_superuser or self.request.user.has_perm('statistics.view_job')) or is_pi return context From 8ae015d3518a14d9eca99609bedb4210c7b8007c Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Tue, 8 Feb 2022 10:03:06 -0500 Subject: [PATCH 007/150] testing if username search field is available --- coldfront/core/statistics/tests/test_job_views.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/coldfront/core/statistics/tests/test_job_views.py b/coldfront/core/statistics/tests/test_job_views.py index 510d641eb..8b2abef5a 100644 --- a/coldfront/core/statistics/tests/test_job_views.py +++ b/coldfront/core/statistics/tests/test_job_views.py @@ -205,12 +205,14 @@ def test_user_list_view_content(self): self.assertContains(response, 'COMPLETED') self.assertNotContains(response, 'COMPLETING') self.assertNotContains(response, 'Show All Jobs') + self.assertNotContains(response, 'div_id_username') # user2 should not be able to see job2 response = self.get_response(self.user2, url) self.assertNotContains(response, self.job1.jobslurmid) self.assertNotContains(response, self.job2.jobslurmid) self.assertNotContains(response, 'Show All Jobs') + self.assertNotContains(response, 'Username') def test_pi_manager_list_view_content(self): """Testing content when users access SlurmJobListView""" @@ -218,11 +220,13 @@ def test_pi_manager_list_view_content(self): response = self.get_response(self.pi, url) self.assertContains(response, self.job1.jobslurmid) + self.assertContains(response, 'div_id_username') self.assertNotContains(response, self.job2.jobslurmid) self.assertNotContains(response, 'Show All Jobs') response = self.get_response(self.manager, url) self.assertContains(response, self.job1.jobslurmid) + self.assertContains(response, 'div_id_username') self.assertNotContains(response, self.job2.jobslurmid) self.assertNotContains(response, 'Show All Jobs') @@ -241,6 +245,7 @@ def test_admin_contents(user): self.assertNotContains(response, self.job2.jobslurmid) self.assertContains(response, 'Show All Jobs') self.assertContains(response, 'Viewing only jobs belonging to') + self.assertContains(response, 'div_id_username') self.assertNotContains(response, 'Viewing all jobs.') self.assertNotContains(response, 'Viewing your jobs and the jobs') @@ -249,6 +254,7 @@ def test_admin_contents(user): self.assertContains(response, self.job2.jobslurmid) self.assertContains(response, 'Show All Jobs') self.assertContains(response, 'Show All Jobs') + self.assertContains(response, 'div_id_username') self.assertNotContains(response, 'Viewing only jobs belonging to') self.assertContains(response, 'Viewing all jobs.') self.assertNotContains(response, 'Viewing your jobs and the jobs') From 8b25bf3bfb437b9b4f212379dfe360f45048a7a2 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Tue, 8 Feb 2022 10:09:45 -0500 Subject: [PATCH 008/150] variable name change. added iterator to widget choices --- coldfront/core/statistics/forms.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/coldfront/core/statistics/forms.py b/coldfront/core/statistics/forms.py index 61e977ae0..90c6ddc68 100644 --- a/coldfront/core/statistics/forms.py +++ b/coldfront/core/statistics/forms.py @@ -143,11 +143,11 @@ def __init__(self, *args, **kwargs): self.fields.pop('show_all_jobs') - set = ProjectUser.objects.filter(user=user, + active_projects = ProjectUser.objects.filter(user=user, status__name='Active').\ distinct('project').values_list('project__name') queryset = \ - Project.objects.filter(name__in=set) + Project.objects.filter(name__in=active_projects) self.fields['submitdate'].label = '' self.fields['submit_modifier'].label = '' @@ -159,4 +159,4 @@ def __init__(self, *args, **kwargs): self.fields['amount_modifier'].label = '' self.fields['project_name'].widget.choices = \ - [('', '-----')] + [(project.name, project.name) for project in queryset] + [('', '-----')] + [(project.name, project.name) for project in queryset.iterator()] From d80b46105a6f8d8a5391fb20e3e186d1fdad712f Mon Sep 17 00:00:00 2001 From: Matthew Li Date: Tue, 8 Feb 2022 14:19:59 -0800 Subject: [PATCH 009/150] Add django-flags as a dependency --- coldfront/config/local_settings.py.sample | 6 ++++++ coldfront/config/settings.py | 1 + requirements.txt | 1 + 3 files changed, 8 insertions(+) diff --git a/coldfront/config/local_settings.py.sample b/coldfront/config/local_settings.py.sample index 72223e2d0..3bb11d658 100644 --- a/coldfront/config/local_settings.py.sample +++ b/coldfront/config/local_settings.py.sample @@ -381,6 +381,12 @@ EMAIL_VERIFICATION_TIMEOUT = 24 * 60 * 60 # The credentials needed to read from Google Sheets. GOOGLE_OAUTH2_KEY_FILE = "/tmp/credentials.json" +#------------------------------------------------------------------------------ +# django-flags settings +#------------------------------------------------------------------------------ + +FLAGS = {} + #------------------------------------------------------------------------------ # Deployment-specific settings #------------------------------------------------------------------------------ diff --git a/coldfront/config/settings.py b/coldfront/config/settings.py index 12948623c..7b9e1d77d 100644 --- a/coldfront/config/settings.py +++ b/coldfront/config/settings.py @@ -58,6 +58,7 @@ # Savio-specific Additional Apps INSTALLED_APPS += [ + 'flags', 'formtools', ] diff --git a/requirements.txt b/requirements.txt index 8f3d5f8e5..7a427a7c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ Django==3.2.5 django-crispy-forms==1.11.1 django-durationwidget==1.0.5 django-filter==2.3.0 +django-flags==5.0.8 django-formtools==2.2 django-model-utils==4.1.1 django-phonenumber-field==5.1.0 From b60c455fa041ed24225bf5ac0c0870113b07e4e3 Mon Sep 17 00:00:00 2001 From: Matthew Li Date: Tue, 25 Jan 2022 15:18:01 -0800 Subject: [PATCH 010/150] Add new billing module with BillingProject and BillingActivity models --- coldfront/config/settings.py | 1 + coldfront/core/billing/__init__.py | 0 .../core/billing/migrations/0001_initial.py | 96 +++++++++++++++++++ coldfront/core/billing/migrations/__init__.py | 0 coldfront/core/billing/models.py | 45 +++++++++ 5 files changed, 142 insertions(+) create mode 100644 coldfront/core/billing/__init__.py create mode 100644 coldfront/core/billing/migrations/0001_initial.py create mode 100644 coldfront/core/billing/migrations/__init__.py create mode 100644 coldfront/core/billing/models.py diff --git a/coldfront/config/settings.py b/coldfront/config/settings.py index 7b9e1d77d..e0c167863 100644 --- a/coldfront/config/settings.py +++ b/coldfront/config/settings.py @@ -54,6 +54,7 @@ # 'coldfront.core.publication', # 'coldfront.core.research_output', 'coldfront.core.statistics', + 'coldfront.core.billing', ] # Savio-specific Additional Apps diff --git a/coldfront/core/billing/__init__.py b/coldfront/core/billing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/coldfront/core/billing/migrations/0001_initial.py b/coldfront/core/billing/migrations/0001_initial.py new file mode 100644 index 000000000..51d5f8d10 --- /dev/null +++ b/coldfront/core/billing/migrations/0001_initial.py @@ -0,0 +1,96 @@ +# Generated by Django 3.2.5 on 2022-01-25 22:56 + +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import model_utils.fields +import simple_history.models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='BillingProject', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('identifier', models.CharField(max_length=6, unique=True, validators=[django.core.validators.RegexValidator('^[0-9]{6}$', message='Identifier must contain 6 numbers.')])), + ('description', models.CharField(max_length=255)), + ], + options={ + 'verbose_name': 'Billing Project', + 'verbose_name_plural': 'Billing Projects', + }, + ), + migrations.CreateModel( + name='HistoricalBillingProject', + fields=[ + ('id', models.IntegerField(auto_created=True, blank=True, db_index=True, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('identifier', models.CharField(db_index=True, max_length=6, validators=[django.core.validators.RegexValidator('^[0-9]{6}$', message='Identifier must contain 6 numbers.')])), + ('description', models.CharField(max_length=255)), + ('history_id', models.AutoField(primary_key=True, serialize=False)), + ('history_date', models.DateTimeField()), + ('history_change_reason', models.CharField(max_length=100, null=True)), + ('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)), + ('history_user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'verbose_name': 'historical Billing Project', + 'ordering': ('-history_date', '-history_id'), + 'get_latest_by': 'history_date', + }, + bases=(simple_history.models.HistoricalChanges, models.Model), + ), + migrations.CreateModel( + name='HistoricalBillingActivity', + fields=[ + ('id', models.IntegerField(auto_created=True, blank=True, db_index=True, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('identifier', models.CharField(max_length=3, validators=[django.core.validators.RegexValidator('^[0-9]{3}$', message='Identifier must contain 3 numbers.')])), + ('description', models.CharField(max_length=255)), + ('is_valid', models.BooleanField(default=False)), + ('history_id', models.AutoField(primary_key=True, serialize=False)), + ('history_date', models.DateTimeField()), + ('history_change_reason', models.CharField(max_length=100, null=True)), + ('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)), + ('billing_project', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='+', to='billing.billingproject')), + ('history_user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'verbose_name': 'historical Billing Activity', + 'ordering': ('-history_date', '-history_id'), + 'get_latest_by': 'history_date', + }, + bases=(simple_history.models.HistoricalChanges, models.Model), + ), + migrations.CreateModel( + name='BillingActivity', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('identifier', models.CharField(max_length=3, validators=[django.core.validators.RegexValidator('^[0-9]{3}$', message='Identifier must contain 3 numbers.')])), + ('description', models.CharField(max_length=255)), + ('is_valid', models.BooleanField(default=False)), + ('billing_project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='billing.billingproject')), + ], + options={ + 'verbose_name': 'Billing Activity', + 'verbose_name_plural': 'Billing Activities', + 'unique_together': {('billing_project', 'identifier')}, + }, + ), + ] diff --git a/coldfront/core/billing/migrations/__init__.py b/coldfront/core/billing/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/coldfront/core/billing/models.py b/coldfront/core/billing/models.py new file mode 100644 index 000000000..e5b38b19d --- /dev/null +++ b/coldfront/core/billing/models.py @@ -0,0 +1,45 @@ +from django.core.validators import RegexValidator +from django.db import models +from model_utils.models import TimeStampedModel +from simple_history.models import HistoricalRecords + + +class BillingProject(TimeStampedModel): + """The prefix of a complete billing ID (i.e., the '123456' of + '123456-789').""" + + identifier = models.CharField( + max_length=6, + unique=True, + validators=[ + RegexValidator( + r'^[0-9]{6}$', message='Identifier must contain 6 numbers.') + ]) + description = models.CharField(max_length=255) + history = HistoricalRecords() + + class Meta: + verbose_name = 'Billing Project' + verbose_name_plural = 'Billing Projects' + + +class BillingActivity(TimeStampedModel): + """The suffix of a complete billing ID (i.e., the '789' of + '123456-789').""" + + billing_project = models.ForeignKey( + BillingProject, on_delete=models.CASCADE) + identifier = models.CharField( + max_length=3, + validators=[ + RegexValidator( + r'^[0-9]{3}$', message='Identifier must contain 3 numbers.') + ]) + description = models.CharField(max_length=255) + is_valid = models.BooleanField(default=False) + history = HistoricalRecords() + + class Meta: + unique_together = ('billing_project', 'identifier') + verbose_name = 'Billing Activity' + verbose_name_plural = 'Billing Activities' From 800a3ba94a67f69b0295294a5acb7051692ae8f2 Mon Sep 17 00:00:00 2001 From: Matthew Li Date: Tue, 25 Jan 2022 15:33:25 -0800 Subject: [PATCH 011/150] Add admin view for displaying billing models --- coldfront/core/billing/admin.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 coldfront/core/billing/admin.py diff --git a/coldfront/core/billing/admin.py b/coldfront/core/billing/admin.py new file mode 100644 index 000000000..3e838645c --- /dev/null +++ b/coldfront/core/billing/admin.py @@ -0,0 +1,33 @@ +from coldfront.core.billing.models import BillingActivity +from coldfront.core.billing.models import BillingProject +from django.contrib import admin + + +@admin.register(BillingActivity) +class BillingActivityAdmin(admin.ModelAdmin): + list_display = ( + 'id', + 'get_project_identifier', + 'get_project_description', + 'identifier', + 'description', + 'is_valid') + + @staticmethod + @admin.display( + description='Project Description', + ordering='billing_project__description') + def get_project_description(obj): + return obj.billing_project.description + + @staticmethod + @admin.display( + description='Project Identifier', + ordering='billing_project__identifier') + def get_project_identifier(obj): + return obj.billing_project.identifier + + +@admin.register(BillingProject) +class BillingProjectAdmin(admin.ModelAdmin): + list_display = ('id', 'identifier', 'description') From 146e4c656441d06789db9ffaab0544a6ad4ff0ee Mon Sep 17 00:00:00 2001 From: Matthew Li Date: Tue, 25 Jan 2022 15:34:01 -0800 Subject: [PATCH 012/150] Add management command for creating/updating billing models from a source file --- .../import_billing_projects_and_activities.py | 74 +++++++++++++++++++ requirements.txt | 2 + 2 files changed, 76 insertions(+) create mode 100644 coldfront/core/billing/management/commands/import_billing_projects_and_activities.py diff --git a/coldfront/core/billing/management/commands/import_billing_projects_and_activities.py b/coldfront/core/billing/management/commands/import_billing_projects_and_activities.py new file mode 100644 index 000000000..89715054e --- /dev/null +++ b/coldfront/core/billing/management/commands/import_billing_projects_and_activities.py @@ -0,0 +1,74 @@ +from coldfront.core.billing.models import BillingActivity +from coldfront.core.billing.models import BillingProject +from django.core.management.base import BaseCommand + +import logging +import openpyxl +import os + +"""An admin command that updates the set of BillingProject and +BillingActivity objects in the database. """ + + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + + help = ( + 'Creates or updates BillingProject and BillingActivity objects, based ' + 'on an Excel spreadsheet containing valid pairs. Invalidate any that ' + 'are not in the file.') + + def add_arguments(self, parser): + parser.add_argument( + 'file', + help=( + 'The path to the file where each row contains a Project ID, ' + 'a Project description, an Activity ID, and an Activity ' + 'description.')) + + def handle(self, *args, **options): + """Iterate over the file, creating and updating BillingProject + and BillingActivity objects. Always update the descriptions + using the first instance seen. Invalidate any BillingActivity + objects not in the file.""" + file_path = options['file'] + if not (os.path.exists(file_path) and os.path.isfile(file_path)): + raise FileNotFoundError(f'File {file_path} does not exist.') + + wb = openpyxl.load_workbook(file_path) + sheet = wb.worksheets[0] + + projects_by_id = {} + activity_id_pairs = set() + valid_activity_pks = set() + + kwargs = { + 'min_row': 3, + 'min_col': 1, + 'max_col': 5, + 'values_only': True, + } + for row in sheet.iter_rows(**kwargs): + project_id, project_desc, _, activity_id, activity_desc = [ + x.strip() for x in row] + + if project_id not in projects_by_id: + project, _ = BillingProject.objects.update_or_create( + identifier=project_id, + defaults={'description': project_desc}) + projects_by_id[project_id] = project + + id_pair = (project_id, activity_id) + if id_pair not in activity_id_pairs: + activity, _ = BillingActivity.objects.update_or_create( + billing_project=projects_by_id[project_id], + identifier=activity_id, + defaults={'description': activity_desc, 'is_valid': True}) + activity_id_pairs.add(id_pair) + valid_activity_pks.add(activity.pk) + + # Invalidate other existing BillingActivities. + BillingActivity.objects.exclude( + pk__in=valid_activity_pks).update(is_valid=False) diff --git a/requirements.txt b/requirements.txt index 7a427a7c6..452fd090b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,6 +30,8 @@ idna==2.8 iso8601==0.1.16 mod-wsgi==4.6.7 oauth2client==4.1.3 +openpyxl==3.0.9 +pandas==1.1.5 phonenumbers==8.12.23 psycopg2-binary==2.8.3 pyparsing==2.4.7 From 8654894dbcf803d612f63bd29deaa15b15190f2d Mon Sep 17 00:00:00 2001 From: Matthew Li Date: Wed, 9 Feb 2022 09:30:52 -0800 Subject: [PATCH 013/150] Add (optional for BRC) billing_activity field to UserProfile to store default PID for monthly fee --- ...09_add_billing_activity_to_user_profile.py | 20 +++++++++++++++++++ coldfront/core/user/models.py | 4 ++++ 2 files changed, 24 insertions(+) create mode 100644 coldfront/core/user/migrations/0009_add_billing_activity_to_user_profile.py diff --git a/coldfront/core/user/migrations/0009_add_billing_activity_to_user_profile.py b/coldfront/core/user/migrations/0009_add_billing_activity_to_user_profile.py new file mode 100644 index 000000000..b48d098c4 --- /dev/null +++ b/coldfront/core/user/migrations/0009_add_billing_activity_to_user_profile.py @@ -0,0 +1,20 @@ +# Generated by Django 3.2.5 on 2022-02-09 17:27 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('billing', '0001_initial'), + ('user', '0008_identitylinkingrequest_identitylinkingrequeststatuschoice'), + ] + + operations = [ + migrations.AddField( + model_name='userprofile', + name='billing_activity', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='billing.billingactivity'), + ), + ] diff --git a/coldfront/core/user/models.py b/coldfront/core/user/models.py index 96cfab8df..1946eed2a 100644 --- a/coldfront/core/user/models.py +++ b/coldfront/core/user/models.py @@ -1,3 +1,4 @@ +from coldfront.core.billing.models import BillingActivity from django.contrib.auth.models import User from django.core.validators import EmailValidator from django.core.validators import MinLengthValidator @@ -32,6 +33,9 @@ class UserProfile(models.Model): access_agreement_signed_date = models.DateTimeField(blank=True, null=True) upgrade_request = models.DateTimeField(blank=True, null=True) + billing_activity = models.ForeignKey( + BillingActivity, blank=True, null=True, on_delete=models.SET_NULL) + class EmailAddress(models.Model): user = models.ForeignKey(User, on_delete=models.CASCADE) From 7a9ff0699a9af0d5df78f5d468b20803f5d2183e Mon Sep 17 00:00:00 2001 From: Matthew Li Date: Wed, 9 Feb 2022 09:32:14 -0800 Subject: [PATCH 014/150] Add as-yet disabled 'LRC_ONLY' flag for LRC-only features; if enabled, create new AllocationAttributeType for storing PID for usage billing --- coldfront/config/local_settings.py.sample | 4 +++- .../commands/add_allocation_defaults.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/coldfront/config/local_settings.py.sample b/coldfront/config/local_settings.py.sample index 3bb11d658..1ed8941c2 100644 --- a/coldfront/config/local_settings.py.sample +++ b/coldfront/config/local_settings.py.sample @@ -385,7 +385,9 @@ GOOGLE_OAUTH2_KEY_FILE = "/tmp/credentials.json" # django-flags settings #------------------------------------------------------------------------------ -FLAGS = {} +FLAGS = { + 'LRC_ONLY': [], +} #------------------------------------------------------------------------------ # Deployment-specific settings diff --git a/coldfront/core/allocation/management/commands/add_allocation_defaults.py b/coldfront/core/allocation/management/commands/add_allocation_defaults.py index 216da6852..2aa52882b 100644 --- a/coldfront/core/allocation/management/commands/add_allocation_defaults.py +++ b/coldfront/core/allocation/management/commands/add_allocation_defaults.py @@ -7,6 +7,8 @@ AllocationStatusChoice, AllocationUserStatusChoice) +from flags.state import flag_enabled + class Command(BaseCommand): help = 'Add default allocation related choices' @@ -55,6 +57,20 @@ def handle(self, *args, **options): AllocationAttributeType.objects.filter( name='Cluster Account Status').update(is_unique=True) + # Create LRC-only AllocationAttributeTypes. + if flag_enabled('LRC_ONLY'): + # The primary key of the BillingActivity object to be treated as + # the default for the Allocation. + AllocationAttributeType.objects.update_or_create( + attribute_type=AttributeType.objects.get(name='Int'), + name='Billing Activity', + defaults={ + 'has_usage': False, + 'is_required': False, + 'is_unique': False, + 'is_private': True, + }) + choices = [ 'Under Review', 'Approved', From 2e357b80ba5352956d14b0381581e16a082ff22c Mon Sep 17 00:00:00 2001 From: Matthew Li Date: Wed, 9 Feb 2022 09:45:46 -0800 Subject: [PATCH 015/150] Set is_unique for 'Billing Activity' to True since each Allocation/AllocationUser only needs one --- .../allocation/management/commands/add_allocation_defaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coldfront/core/allocation/management/commands/add_allocation_defaults.py b/coldfront/core/allocation/management/commands/add_allocation_defaults.py index 2aa52882b..2dfb31f34 100644 --- a/coldfront/core/allocation/management/commands/add_allocation_defaults.py +++ b/coldfront/core/allocation/management/commands/add_allocation_defaults.py @@ -67,7 +67,7 @@ def handle(self, *args, **options): defaults={ 'has_usage': False, 'is_required': False, - 'is_unique': False, + 'is_unique': True, 'is_private': True, }) From d283e86e72d29206df26caa6b940e8a2d40418e7 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Mon, 14 Feb 2022 07:50:44 -0500 Subject: [PATCH 016/150] initial command commit --- .../commands/add_service_units_to_project.py | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 coldfront/core/project/management/commands/add_service_units_to_project.py diff --git a/coldfront/core/project/management/commands/add_service_units_to_project.py b/coldfront/core/project/management/commands/add_service_units_to_project.py new file mode 100644 index 000000000..5d7d946e6 --- /dev/null +++ b/coldfront/core/project/management/commands/add_service_units_to_project.py @@ -0,0 +1,111 @@ +import logging +from decimal import Decimal + +from django.core.management import BaseCommand, CommandError + +from coldfront.core.project.models import Project +from coldfront.core.statistics.models import ProjectTransaction +from coldfront.core.statistics.models import ProjectUserTransaction +from coldfront.core.utils.common import utc_now_offset_aware +from coldfront.api.statistics.utils import get_accounting_allocation_objects +from coldfront.api.statistics.utils import set_project_allocation_value +from coldfront.api.statistics.utils import set_project_user_allocation_value +from coldfront.core.allocation.models import AllocationAttributeType + + +class Command(BaseCommand): + help = 'Command to set a common password for users in the test database.' + logger = logging.getLogger(__name__) + + def add_arguments(self, parser): + parser.add_argument('--project_name', + help='Name of project to add SUs to.', + type=str, + required=True) + parser.add_argument('--amount', + help='Number of SUs to add to a given project.', + type=int, + required=True) + parser.add_argument('--reason', + help='User given reason for adding SUs.', + type=str, + required=True) + + parser.add_argument('--dry_run', + help='Display updates without performing them.', + type=str, + required=False) + + def handle(self, *args, **options): + """ Set user password for all users in test database """ + + project = Project.objects.get(name=options.get('project_name')) + addition = Decimal(options.get('amount')) + reason = options.get('reason') + dry_run = options.get('dry_run', None) + + date_time = utc_now_offset_aware() + + allocation_objects = get_accounting_allocation_objects(project) + + if not allocation_objects.allocation.resources.filter(name='Savio Compute').exists(): + raise CommandError('Can only add SUs to projects that ' + 'have an allocatino to Savio Compute.') + + current_allocation = Decimal(allocation_objects.allocation_attribute.value) + + # Compute the new Service Units value. + allocation = addition + current_allocation + + if dry_run: + f'Would add {addition} additional SUs to project ' \ + f'{project.name}. This would increase {project.name} ' \ + f'SUs from {current_allocation} to {allocation}. ' \ + f'The reason for updating SUs for {project.name}' \ + f'is given as: {reason}' + + else: + # Set the value for the Project. + set_project_allocation_value(project, allocation) + + # Create a transaction to record the change. + ProjectTransaction.objects.create( + project=project, + date_time=date_time, + allocation=allocation) + + # Set the reason for the change in the newly-created historical object. + allocation_objects.allocation_attribute.refresh_from_db() + historical_allocation_attribute = \ + allocation_objects.allocation_attribute.history.latest("id") + historical_allocation_attribute.history_change_reason = reason + historical_allocation_attribute.save() + + # Do the same for each ProjectUser. + allocation_attribute_type = AllocationAttributeType.objects.get( + name="Service Units") + for project_user in project.projectuser_set.all(): + user = project_user.user + # Attempt to set the value for the ProjectUser. The method returns whether + # it succeeded; it may not because not every ProjectUser has a + # corresponding AllocationUser (e.g., PIs). Only proceed with further steps + # if an update occurred. + allocation_updated = set_project_user_allocation_value( + user, project, allocation) + if allocation_updated: + # Create a transaction to record the change. + ProjectUserTransaction.objects.create( + project_user=project_user, + date_time=date_time, + allocation=allocation) + # Set the reason for the change in the newly-created historical object. + allocation_user = \ + allocation_objects.allocation.allocationuser_set.get(user=user) + allocation_user_attribute = \ + allocation_user.allocationuserattribute_set.get( + allocation_attribute_type=allocation_attribute_type, + allocation=allocation_objects.allocation) + historical_allocation_user_attribute = \ + allocation_user_attribute.history.latest("id") + historical_allocation_user_attribute.history_change_reason = reason + historical_allocation_user_attribute.save() From ebd0d9dfb14dc8a7616b718e90b7a53b49b51611 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Mon, 14 Feb 2022 07:53:15 -0500 Subject: [PATCH 017/150] changed help text --- .../project/management/commands/add_service_units_to_project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coldfront/core/project/management/commands/add_service_units_to_project.py b/coldfront/core/project/management/commands/add_service_units_to_project.py index 5d7d946e6..9c897e568 100644 --- a/coldfront/core/project/management/commands/add_service_units_to_project.py +++ b/coldfront/core/project/management/commands/add_service_units_to_project.py @@ -14,7 +14,7 @@ class Command(BaseCommand): - help = 'Command to set a common password for users in the test database.' + help = 'Command to add SUs to a given project.' logger = logging.getLogger(__name__) def add_arguments(self, parser): From 00a8b7cdca5cddc3420924fbac947392b924ca69 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Mon, 14 Feb 2022 09:29:01 -0500 Subject: [PATCH 018/150] fixed checking allocation is in savio compute. altered messages --- .../commands/add_service_units_to_project.py | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/coldfront/core/project/management/commands/add_service_units_to_project.py b/coldfront/core/project/management/commands/add_service_units_to_project.py index 9c897e568..8d786b450 100644 --- a/coldfront/core/project/management/commands/add_service_units_to_project.py +++ b/coldfront/core/project/management/commands/add_service_units_to_project.py @@ -10,7 +10,7 @@ from coldfront.api.statistics.utils import get_accounting_allocation_objects from coldfront.api.statistics.utils import set_project_allocation_value from coldfront.api.statistics.utils import set_project_user_allocation_value -from coldfront.core.allocation.models import AllocationAttributeType +from coldfront.core.allocation.models import AllocationAttributeType, Allocation class Command(BaseCommand): @@ -30,11 +30,9 @@ def add_arguments(self, parser): help='User given reason for adding SUs.', type=str, required=True) - parser.add_argument('--dry_run', help='Display updates without performing them.', - type=str, - required=False) + action='store_true') def handle(self, *args, **options): """ Set user password for all users in test database """ @@ -44,25 +42,25 @@ def handle(self, *args, **options): reason = options.get('reason') dry_run = options.get('dry_run', None) - date_time = utc_now_offset_aware() - - allocation_objects = get_accounting_allocation_objects(project) - - if not allocation_objects.allocation.resources.filter(name='Savio Compute').exists(): + if not Allocation.objects.get(project=project).resources.filter(name='Savio Compute').exists(): raise CommandError('Can only add SUs to projects that ' - 'have an allocatino to Savio Compute.') + 'have an allocation in Savio Compute.') + date_time = utc_now_offset_aware() + allocation_objects = get_accounting_allocation_objects(project) current_allocation = Decimal(allocation_objects.allocation_attribute.value) # Compute the new Service Units value. allocation = addition + current_allocation if dry_run: - f'Would add {addition} additional SUs to project ' \ - f'{project.name}. This would increase {project.name} ' \ - f'SUs from {current_allocation} to {allocation}. ' \ - f'The reason for updating SUs for {project.name}' \ - f'is given as: {reason}' + message = f'Would add {addition} additional SUs to project ' \ + f'{project.name}. This would increase {project.name} ' \ + f'SUs from {current_allocation} to {allocation}. ' \ + f'The reason for updating SUs for {project.name} ' \ + f'would be: {reason}.' + + self.stdout.write(self.style.WARNING(message)) else: # Set the value for the Project. @@ -109,3 +107,9 @@ def handle(self, *args, **options): allocation_user_attribute.history.latest("id") historical_allocation_user_attribute.history_change_reason = reason historical_allocation_user_attribute.save() + + message = f"Successfully added {addition} SUs to {project.name}" \ + f", updating {project.name}'s SUs from " \ + f"{current_allocation} to {allocation}." + + self.stdout.write(self.style.SUCCESS(message)) From 9332f8bb097ee62496b375db9b2a5d7426ba998c Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Mon, 14 Feb 2022 09:30:34 -0500 Subject: [PATCH 019/150] adding tests for add_service_units_to_project command --- .../test_add_service_units_to_project.py | 171 ++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 coldfront/core/project/tests/test_commands/test_add_service_units_to_project.py diff --git a/coldfront/core/project/tests/test_commands/test_add_service_units_to_project.py b/coldfront/core/project/tests/test_commands/test_add_service_units_to_project.py new file mode 100644 index 000000000..29a3ad2ec --- /dev/null +++ b/coldfront/core/project/tests/test_commands/test_add_service_units_to_project.py @@ -0,0 +1,171 @@ +from io import StringIO +import sys + +from django.core.management import call_command, CommandError + +from coldfront.api.allocation.tests.test_allocation_base import TestAllocationBase +from coldfront.api.statistics.utils import get_accounting_allocation_objects +from coldfront.core.allocation.models import Allocation, AllocationAttributeType +from coldfront.core.project.models import Project +from coldfront.core.resource.models import Resource +from coldfront.core.statistics.models import ProjectTransaction, ProjectUserTransaction +from coldfront.core.utils.common import utc_now_offset_aware + + +class TestAddServiceUnitsToProject(TestAllocationBase): + """Class for testing the management command add_service_units_to_project""" + + def setUp(self): + """Set up test data.""" + super().setUp() + + def allocation_values_test(self, project, value, user_value): + allocation_objects = get_accounting_allocation_objects(project) + self.assertEqual(allocation_objects.allocation_attribute.value, value) + + for project_user in project.projectuser_set.all(): + if project_user.role.name != 'User': + continue + allocation_objects = get_accounting_allocation_objects( + project, user=project_user.user) + + self.assertEqual(allocation_objects.allocation_user_attribute.value, + user_value) + + def transactions_created(self, project, pre_time, post_time): + proj_transaction = ProjectTransaction.objects.get(project=project, + allocation=2000.0) + + self.assertTrue(pre_time <= proj_transaction.date_time <= post_time) + + for project_user in project.projectuser_set.all(): + if project_user.role.name != 'User': + continue + + proj_user_transaction = ProjectUserTransaction.objects.get( + project_user=project_user, + allocation=2000.0) + + self.assertTrue(pre_time <= proj_user_transaction.date_time <= post_time) + + def historical_objects_updated(self, project): + allocation_objects = get_accounting_allocation_objects(project) + historical_allocation_attribute = \ + allocation_objects.allocation_attribute.history.latest("id") + historical_reason = historical_allocation_attribute.history_change_reason + + self.assertEqual(historical_reason, 'This is a test') + + allocation_attribute_type = AllocationAttributeType.objects.get( + name="Service Units") + for project_user in project.projectuser_set.all(): + if project_user.role.name != 'User': + continue + + allocation_user = \ + allocation_objects.allocation.allocationuser_set.get( + user=project_user.user) + allocation_user_attribute = \ + allocation_user.allocationuserattribute_set.get( + allocation_attribute_type=allocation_attribute_type, + allocation=allocation_objects.allocation) + historical_allocation_user_attribute = \ + allocation_user_attribute.history.latest("id") + historical_reason = \ + historical_allocation_user_attribute.history_change_reason + self.assertEqual(historical_reason, 'This is a test') + + def test_dry_run(self): + """Testing add_service_units_to_project dry run""" + out, err = StringIO(''), StringIO('') + call_command('add_service_units_to_project', + '--project=project0', + '--amount=1000', + '--reason=This is a test', + '--dry_run', + stdout=out, + stderr=err) + sys.stdout = sys.__stdout__ + out.seek(0) + + dry_run_message = 'Would add 1000 additional SUs to project ' \ + 'project0. This would increase project0 ' \ + 'SUs from 1000.00 to 2000.00. ' \ + 'The reason for updating SUs for project0 ' \ + 'would be: This is a test.' + + self.assertIn(dry_run_message, out.read()) + err.seek(0) + self.assertEqual(err.read(), '') + + def test_command(self): + """Testing add_service_units_to_project dry run""" + + # test allocation values before command + project = Project.objects.get(name='project0') + pre_time = utc_now_offset_aware() + + self.allocation_values_test(project, '1000.00', '500.00') + + # run command + out, err = StringIO(''), StringIO('') + call_command('add_service_units_to_project', + '--project=project0', + '--amount=1000', + '--reason=This is a test', + stdout=out, + stderr=err) + sys.stdout = sys.__stdout__ + out.seek(0) + + message = "Successfully added 1000 SUs to project0" \ + ", updating project0's SUs from " \ + "1000.00 to 2000.00." + + self.assertIn(message, out.read()) + err.seek(0) + self.assertEqual(err.read(), '') + + post_time = utc_now_offset_aware() + + # test allocation values after command + self.allocation_values_test(project, '2000.00', '2000.00') + + # test ProjectTransaction created + self.transactions_created(project, pre_time, post_time) + + # test historical objects updated + self.historical_objects_updated(project) + + def test_non_savio_compute(self): + project = Project.objects.get(name='project1') + allocation = Allocation.objects.get(project=project) + + # clear resources from allocation + allocation.resources.clear() + allocation.refresh_from_db() + self.assertEqual(allocation.resources.all().count(), 0) + + # add vector compute allocation + vector_resource = Resource.objects.get(name='Vector Compute') + allocation.resources.add(vector_resource) + allocation.save() + allocation.refresh_from_db() + self.assertEqual(allocation.resources.all().count(), 1) + self.assertEqual(allocation.resources.first().name, 'Vector Compute') + + # command should throw a CommandError because the allocation is not + # part of Savio Compute + with self.assertRaises(CommandError): + out, err = StringIO(''), StringIO('') + call_command('add_service_units_to_project', + '--project=project1', + '--amount=1000', + '--reason=This is a test', + stdout=out, + stderr=err) + sys.stdout = sys.__stdout__ + out.seek(0) + self.assertEqual(out.read(), '') + err.seek(0) + self.assertEqual(err.read(), '') From 30778ef49d6ab68e5a2d5695ab91b630bf82cae2 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Mon, 14 Feb 2022 12:53:14 -0500 Subject: [PATCH 020/150] fixed issues raised in PR 355 --- .../commands/add_service_units_to_project.py | 63 +++++++-- .../test_add_service_units_to_project.py | 131 ++++++++++++++++-- 2 files changed, 167 insertions(+), 27 deletions(-) diff --git a/coldfront/core/project/management/commands/add_service_units_to_project.py b/coldfront/core/project/management/commands/add_service_units_to_project.py index 8d786b450..077c9f3c2 100644 --- a/coldfront/core/project/management/commands/add_service_units_to_project.py +++ b/coldfront/core/project/management/commands/add_service_units_to_project.py @@ -1,8 +1,10 @@ import logging from decimal import Decimal +from django.core.exceptions import ObjectDoesNotExist from django.core.management import BaseCommand, CommandError +from coldfront.config import settings from coldfront.core.project.models import Project from coldfront.core.statistics.models import ProjectTransaction from coldfront.core.statistics.models import ProjectUserTransaction @@ -34,31 +36,67 @@ def add_arguments(self, parser): help='Display updates without performing them.', action='store_true') + def validate_inputs(self, options): + """Validate inputs to add_service_units_to_project command""" + + # Checking if project exists + if not Project.objects.filter(name=options.get('project_name')).exists(): + error_message = f"Requested project {options.get('project_name')}" \ + f" does not exist." + raise CommandError(error_message) + + # Allocation must be in Savio Compute + project = Project.objects.get(name=options.get('project_name')) + try: + allocation_objects = get_accounting_allocation_objects(project) + except ObjectDoesNotExist as e: + error_message = 'Can only add SUs to projects that have an ' \ + 'allocation in Savio Compute.' + raise CommandError(error_message) + + addition = Decimal(options.get('amount')) + current_allocation = Decimal(allocation_objects.allocation_attribute.value) + + # new service units value + allocation = addition + current_allocation + + # checking SU values + if addition < settings.ALLOCATION_MIN or addition > settings.ALLOCATION_MAX: + error_message = f'Amount of SUs to add must be between ' \ + f'{settings.ALLOCATION_MIN} and ' \ + f'{settings.ALLOCATION_MAX}.' + raise CommandError(error_message) + + if allocation > settings.ALLOCATION_MAX: + error_message = f'Total SUs for allocation {project.name} ' \ + f'cannot be greater than {settings.ALLOCATION_MAX}.' + raise CommandError(error_message) + + if len(options.get('reason')) < 20: + error_message = f'Reason must be at least 20 characters.' + raise CommandError(error_message) + def handle(self, *args, **options): - """ Set user password for all users in test database """ + """ Add SUs to a given project """ + + self.validate_inputs(options) project = Project.objects.get(name=options.get('project_name')) addition = Decimal(options.get('amount')) reason = options.get('reason') dry_run = options.get('dry_run', None) - if not Allocation.objects.get(project=project).resources.filter(name='Savio Compute').exists(): - raise CommandError('Can only add SUs to projects that ' - 'have an allocation in Savio Compute.') - - date_time = utc_now_offset_aware() allocation_objects = get_accounting_allocation_objects(project) current_allocation = Decimal(allocation_objects.allocation_attribute.value) - - # Compute the new Service Units value. allocation = addition + current_allocation + date_time = utc_now_offset_aware() if dry_run: message = f'Would add {addition} additional SUs to project ' \ f'{project.name}. This would increase {project.name} ' \ f'SUs from {current_allocation} to {allocation}. ' \ f'The reason for updating SUs for {project.name} ' \ - f'would be: {reason}.' + f'would be: "{reason}".' self.stdout.write(self.style.WARNING(message)) @@ -108,8 +146,9 @@ def handle(self, *args, **options): historical_allocation_user_attribute.history_change_reason = reason historical_allocation_user_attribute.save() - message = f"Successfully added {addition} SUs to {project.name}" \ - f", updating {project.name}'s SUs from " \ - f"{current_allocation} to {allocation}." + message = f'Successfully added {addition} SUs to {project.name} ' \ + f'and its users, updating {project.name}\'s SUs from ' \ + f'{current_allocation} to {allocation}. The reason ' \ + f'was: "{reason}".' self.stdout.write(self.style.SUCCESS(message)) diff --git a/coldfront/core/project/tests/test_commands/test_add_service_units_to_project.py b/coldfront/core/project/tests/test_commands/test_add_service_units_to_project.py index 29a3ad2ec..7a72bc766 100644 --- a/coldfront/core/project/tests/test_commands/test_add_service_units_to_project.py +++ b/coldfront/core/project/tests/test_commands/test_add_service_units_to_project.py @@ -19,6 +19,37 @@ def setUp(self): """Set up test data.""" super().setUp() + def record_historical_objects_len(self, project): + """ Records the lengths of all relevant historical objects to a dict""" + length_dict = {} + allocation_objects = get_accounting_allocation_objects(project) + historical_allocation_attribute = \ + allocation_objects.allocation_attribute.history.all() + + length_dict['historical_allocation_attribute'] = \ + len(historical_allocation_attribute) + + allocation_attribute_type = AllocationAttributeType.objects.get( + name="Service Units") + for project_user in project.projectuser_set.all(): + if project_user.role.name != 'User': + continue + + allocation_user = \ + allocation_objects.allocation.allocationuser_set.get( + user=project_user.user) + allocation_user_attribute = \ + allocation_user.allocationuserattribute_set.get( + allocation_attribute_type=allocation_attribute_type, + allocation=allocation_objects.allocation) + historical_allocation_user_attribute = \ + allocation_user_attribute.history.all() + + key = 'historical_allocation_user_attribute_' + project_user.user.username + length_dict[key] = len(historical_allocation_user_attribute) + + return length_dict + def allocation_values_test(self, project, value, user_value): allocation_objects = get_accounting_allocation_objects(project) self.assertEqual(allocation_objects.allocation_attribute.value, value) @@ -48,13 +79,18 @@ def transactions_created(self, project, pre_time, post_time): self.assertTrue(pre_time <= proj_user_transaction.date_time <= post_time) + def historical_objects_created(self, pre_length_dict, post_length_dict): + """Test that historical objects were created""" + for k, v in pre_length_dict.items(): + self.assertEqual(v + 1, post_length_dict[k]) + def historical_objects_updated(self, project): allocation_objects = get_accounting_allocation_objects(project) historical_allocation_attribute = \ allocation_objects.allocation_attribute.history.latest("id") historical_reason = historical_allocation_attribute.history_change_reason - self.assertEqual(historical_reason, 'This is a test') + self.assertEqual(historical_reason, 'This is a test for add_service_units command') allocation_attribute_type = AllocationAttributeType.objects.get( name="Service Units") @@ -73,15 +109,16 @@ def historical_objects_updated(self, project): allocation_user_attribute.history.latest("id") historical_reason = \ historical_allocation_user_attribute.history_change_reason - self.assertEqual(historical_reason, 'This is a test') + self.assertEqual(historical_reason, + 'This is a test for add_service_units command') def test_dry_run(self): """Testing add_service_units_to_project dry run""" out, err = StringIO(''), StringIO('') call_command('add_service_units_to_project', - '--project=project0', + '--project_name=project0', '--amount=1000', - '--reason=This is a test', + '--reason=This is a test for add_service_units command', '--dry_run', stdout=out, stderr=err) @@ -92,35 +129,37 @@ def test_dry_run(self): 'project0. This would increase project0 ' \ 'SUs from 1000.00 to 2000.00. ' \ 'The reason for updating SUs for project0 ' \ - 'would be: This is a test.' + 'would be: "This is a test for ' \ + 'add_service_units command".' self.assertIn(dry_run_message, out.read()) err.seek(0) self.assertEqual(err.read(), '') - def test_command(self): + def test_creates_and_updates_objects(self): """Testing add_service_units_to_project dry run""" # test allocation values before command project = Project.objects.get(name='project0') pre_time = utc_now_offset_aware() + pre_length_dict = self.record_historical_objects_len(project) self.allocation_values_test(project, '1000.00', '500.00') # run command out, err = StringIO(''), StringIO('') call_command('add_service_units_to_project', - '--project=project0', + '--project_name=project0', '--amount=1000', - '--reason=This is a test', + '--reason=This is a test for add_service_units command', stdout=out, stderr=err) sys.stdout = sys.__stdout__ out.seek(0) - message = "Successfully added 1000 SUs to project0" \ - ", updating project0's SUs from " \ - "1000.00 to 2000.00." + message = f'Successfully added 1000 SUs to project0 and its users, ' \ + f'updating project0\'s SUs from 1000.00 to 2000.00. The ' \ + f'reason was: "This is a test for add_service_units command".' self.assertIn(message, out.read()) err.seek(0) @@ -134,10 +173,12 @@ def test_command(self): # test ProjectTransaction created self.transactions_created(project, pre_time, post_time) - # test historical objects updated + # test historical objects created and updated + post_length_dict = self.record_historical_objects_len(project) + self.historical_objects_created(pre_length_dict, post_length_dict) self.historical_objects_updated(project) - def test_non_savio_compute(self): + def test_input_validations(self): project = Project.objects.get(name='project1') allocation = Allocation.objects.get(project=project) @@ -159,9 +200,69 @@ def test_non_savio_compute(self): with self.assertRaises(CommandError): out, err = StringIO(''), StringIO('') call_command('add_service_units_to_project', - '--project=project1', + '--project_name=project1', + '--amount=1000', + '--reason=This is a test for add_service_units command', + stdout=out, + stderr=err) + sys.stdout = sys.__stdout__ + out.seek(0) + self.assertEqual(out.read(), '') + err.seek(0) + self.assertEqual(err.read(), '') + + # testing a project that does not exist + with self.assertRaises(CommandError): + out, err = StringIO(''), StringIO('') + call_command('add_service_units_to_project', + '--project_name=project555', + '--amount=1000', + '--reason=This is a test for add_service_units command', + stdout=out, + stderr=err) + sys.stdout = sys.__stdout__ + out.seek(0) + self.assertEqual(out.read(), '') + err.seek(0) + self.assertEqual(err.read(), '') + + # adding service units that are less than settings.ALLOCATION_MIN + with self.assertRaises(CommandError): + out, err = StringIO(''), StringIO('') + call_command('add_service_units_to_project', + '--project_name=project0', + '--amount=-1000', + '--reason=This is a test for add_service_units command', + stdout=out, + stderr=err) + sys.stdout = sys.__stdout__ + out.seek(0) + self.assertEqual(out.read(), '') + err.seek(0) + self.assertEqual(err.read(), '') + + # adding service units that are greater than settings.ALLOCATION_MIN + with self.assertRaises(CommandError): + out, err = StringIO(''), StringIO('') + call_command('add_service_units_to_project', + '--project_name=project0', + '--amount=500000000', + '--reason=This is a test for add_service_units command', + stdout=out, + stderr=err) + sys.stdout = sys.__stdout__ + out.seek(0) + self.assertEqual(out.read(), '') + err.seek(0) + self.assertEqual(err.read(), '') + + # reason is not long enough + with self.assertRaises(CommandError): + out, err = StringIO(''), StringIO('') + call_command('add_service_units_to_project', + '--project_name=project0', '--amount=1000', - '--reason=This is a test', + '--reason=notlong', stdout=out, stderr=err) sys.stdout = sys.__stdout__ From 3ed5512fc90dae856a287869687fda2d7cd3bd25 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Mon, 14 Feb 2022 13:06:36 -0500 Subject: [PATCH 021/150] can now add negative SUs as long as the resulting amount is valid --- .../commands/add_service_units_to_project.py | 12 +++++------ .../test_add_service_units_to_project.py | 21 +++++++++++++++++-- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/coldfront/core/project/management/commands/add_service_units_to_project.py b/coldfront/core/project/management/commands/add_service_units_to_project.py index 077c9f3c2..0146957d0 100644 --- a/coldfront/core/project/management/commands/add_service_units_to_project.py +++ b/coldfront/core/project/management/commands/add_service_units_to_project.py @@ -61,15 +61,15 @@ def validate_inputs(self, options): allocation = addition + current_allocation # checking SU values - if addition < settings.ALLOCATION_MIN or addition > settings.ALLOCATION_MAX: - error_message = f'Amount of SUs to add must be between ' \ - f'{settings.ALLOCATION_MIN} and ' \ - f'{settings.ALLOCATION_MAX}.' + if addition > settings.ALLOCATION_MAX: + error_message = f'Amount of SUs to add must cannot be greater ' \ + f'than {settings.ALLOCATION_MAX}.' raise CommandError(error_message) - if allocation > settings.ALLOCATION_MAX: + if allocation < settings.ALLOCATION_MIN or allocation > settings.ALLOCATION_MAX: error_message = f'Total SUs for allocation {project.name} ' \ - f'cannot be greater than {settings.ALLOCATION_MAX}.' + f'cannot be less than {settings.ALLOCATION_MIN} ' \ + f'or greater than {settings.ALLOCATION_MAX}.' raise CommandError(error_message) if len(options.get('reason')) < 20: diff --git a/coldfront/core/project/tests/test_commands/test_add_service_units_to_project.py b/coldfront/core/project/tests/test_commands/test_add_service_units_to_project.py index 7a72bc766..4ce14d096 100644 --- a/coldfront/core/project/tests/test_commands/test_add_service_units_to_project.py +++ b/coldfront/core/project/tests/test_commands/test_add_service_units_to_project.py @@ -226,12 +226,29 @@ def test_input_validations(self): err.seek(0) self.assertEqual(err.read(), '') - # adding service units that are less than settings.ALLOCATION_MIN + # adding service units that results in allocation having less + # than settings.ALLOCATION_MIN with self.assertRaises(CommandError): out, err = StringIO(''), StringIO('') call_command('add_service_units_to_project', '--project_name=project0', - '--amount=-1000', + '--amount=-100000', + '--reason=This is a test for add_service_units command', + stdout=out, + stderr=err) + sys.stdout = sys.__stdout__ + out.seek(0) + self.assertEqual(out.read(), '') + err.seek(0) + self.assertEqual(err.read(), '') + + # adding service units that results in allocation having more + # than settings.ALLOCATION_MAX + with self.assertRaises(CommandError): + out, err = StringIO(''), StringIO('') + call_command('add_service_units_to_project', + '--project_name=project0', + '--amount=99999500', '--reason=This is a test for add_service_units command', stdout=out, stderr=err) From 2e1cfbb22c7e9f3e4ec44f9c2b09366c645e6a1b Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Mon, 14 Feb 2022 13:07:36 -0500 Subject: [PATCH 022/150] changed try except block in validate_inputs --- .../project/management/commands/add_service_units_to_project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coldfront/core/project/management/commands/add_service_units_to_project.py b/coldfront/core/project/management/commands/add_service_units_to_project.py index 0146957d0..14c36608a 100644 --- a/coldfront/core/project/management/commands/add_service_units_to_project.py +++ b/coldfront/core/project/management/commands/add_service_units_to_project.py @@ -49,7 +49,7 @@ def validate_inputs(self, options): project = Project.objects.get(name=options.get('project_name')) try: allocation_objects = get_accounting_allocation_objects(project) - except ObjectDoesNotExist as e: + except Allocation.DoesNotExist: error_message = 'Can only add SUs to projects that have an ' \ 'allocation in Savio Compute.' raise CommandError(error_message) From ea91bc0504474b0b90b719f7e8ab93b6a26ac0a0 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Mon, 14 Feb 2022 13:23:45 -0500 Subject: [PATCH 023/150] added test for adding negative SUs to project --- .../test_add_service_units_to_project.py | 54 ++++++++++++++++--- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/coldfront/core/project/tests/test_commands/test_add_service_units_to_project.py b/coldfront/core/project/tests/test_commands/test_add_service_units_to_project.py index 4ce14d096..3fc42d633 100644 --- a/coldfront/core/project/tests/test_commands/test_add_service_units_to_project.py +++ b/coldfront/core/project/tests/test_commands/test_add_service_units_to_project.py @@ -63,9 +63,9 @@ def allocation_values_test(self, project, value, user_value): self.assertEqual(allocation_objects.allocation_user_attribute.value, user_value) - def transactions_created(self, project, pre_time, post_time): + def transactions_created(self, project, pre_time, post_time, amount): proj_transaction = ProjectTransaction.objects.get(project=project, - allocation=2000.0) + allocation=amount) self.assertTrue(pre_time <= proj_transaction.date_time <= post_time) @@ -75,7 +75,7 @@ def transactions_created(self, project, pre_time, post_time): proj_user_transaction = ProjectUserTransaction.objects.get( project_user=project_user, - allocation=2000.0) + allocation=amount) self.assertTrue(pre_time <= proj_user_transaction.date_time <= post_time) @@ -136,8 +136,8 @@ def test_dry_run(self): err.seek(0) self.assertEqual(err.read(), '') - def test_creates_and_updates_objects(self): - """Testing add_service_units_to_project dry run""" + def test_creates_and_updates_objects_positive_SU(self): + """Testing add_service_units_to_project with positive SUs""" # test allocation values before command project = Project.objects.get(name='project0') @@ -171,7 +171,49 @@ def test_creates_and_updates_objects(self): self.allocation_values_test(project, '2000.00', '2000.00') # test ProjectTransaction created - self.transactions_created(project, pre_time, post_time) + self.transactions_created(project, pre_time, post_time, 2000.00) + + # test historical objects created and updated + post_length_dict = self.record_historical_objects_len(project) + self.historical_objects_created(pre_length_dict, post_length_dict) + self.historical_objects_updated(project) + + def test_creates_and_updates_objects_negative_SU(self): + """Testing add_service_units_to_project with negative SUs""" + + # test allocation values before command + project = Project.objects.get(name='project0') + pre_time = utc_now_offset_aware() + pre_length_dict = self.record_historical_objects_len(project) + + self.allocation_values_test(project, '1000.00', '500.00') + + # run command + out, err = StringIO(''), StringIO('') + call_command('add_service_units_to_project', + '--project_name=project0', + '--amount=-800', + '--reason=This is a test for add_service_units command', + stdout=out, + stderr=err) + sys.stdout = sys.__stdout__ + out.seek(0) + + message = f'Successfully added -800 SUs to project0 and its users, ' \ + f'updating project0\'s SUs from 1000.00 to 200.00. The ' \ + f'reason was: "This is a test for add_service_units command".' + + self.assertIn(message, out.read()) + err.seek(0) + self.assertEqual(err.read(), '') + + post_time = utc_now_offset_aware() + + # test allocation values after command + self.allocation_values_test(project, '200.00', '200.00') + + # test ProjectTransaction created + self.transactions_created(project, pre_time, post_time, 200.00) # test historical objects created and updated post_length_dict = self.record_historical_objects_len(project) From baf7d8d864c3d2efb0bb976dacacb7851f71feaa Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Thu, 17 Feb 2022 10:52:30 -0500 Subject: [PATCH 024/150] adding management command to deactivate expired ica projects --- .../commands/deactivate_ica_projects.py | 233 ++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 coldfront/core/project/management/commands/deactivate_ica_projects.py diff --git a/coldfront/core/project/management/commands/deactivate_ica_projects.py b/coldfront/core/project/management/commands/deactivate_ica_projects.py new file mode 100644 index 000000000..cc54f1d03 --- /dev/null +++ b/coldfront/core/project/management/commands/deactivate_ica_projects.py @@ -0,0 +1,233 @@ +from decimal import Decimal + +from django.template.loader import render_to_string + +from coldfront.api.statistics.utils import get_accounting_allocation_objects, \ + set_project_allocation_value, set_project_user_allocation_value +from coldfront.config import settings +from coldfront.core.allocation.models import AllocationAttributeType +from coldfront.core.allocation.models import AllocationStatusChoice +from coldfront.core.allocation.utils import get_project_compute_allocation +from coldfront.core.project.models import Project +from coldfront.core.project.models import ProjectStatusChoice +from django.core.management.base import BaseCommand +from django.db.models import Q +import logging + +from coldfront.core.statistics.models import ProjectTransaction, \ + ProjectUserTransaction +from coldfront.core.utils.common import utc_now_offset_aware +from coldfront.core.utils.mail import send_email_template + +"""An admin command that sets expired ICA Projects to 'Inactive' and +their corresponding compute Allocations to 'Expired'.""" + + +class Command(BaseCommand): + + help = ( + 'Set expired ICA Projects to \'Inactive\' and their corresponding ' + 'compute Allocations to \'Expired\'.') + logger = logging.getLogger(__name__) + + def add_arguments(self, parser): + parser.add_argument( + '--dry_run', + action='store_true', + help='Display updates without performing them.') + parser.add_argument( + '--send-emails', + action='store_true', + default=False, + help='Send emails to PIs/managers about project deactivation.') + + def handle(self, *args, **options): + """For each expired ICA Project, set its status to 'Inactive' and its + compute Allocation's status to 'Expired'. Zero out Service Units for + the Allocation and AllocationUsers. + """ + current_date = utc_now_offset_aware() + ica_projects = Project.objects.filter(name__icontains='ic_') + + for project in ica_projects: + allocation = get_project_compute_allocation(project) + expiry_date = allocation.end_date + + if allocation.end_date < current_date: + self.deactivate_project(project, allocation, options['dry_run']) + self.reset_service_units(project, options['dry_run']) + + if options['send_emails']: + self.send_emails(project, expiry_date, options['dry_run']) + + def deactivate_project(self, project, allocation, dry_run): + """ + Sets project status to Inactive and corresponding compute Allocation to + expired. Sets allocation start date to the current date and removes + the end date. + + If dry_run is True, write to stdout without changing object fields. + """ + project_status = ProjectStatusChoice.objects.get(name='Inactive') + allocation_status = AllocationStatusChoice.objects.get(name='Expired') + current_date = utc_now_offset_aware() + + if dry_run: + message = ( + f'Would update Project {project.name} ({project.pk})\'s ' + f'status to {project_status.name} and Allocation ' + f'{allocation.pk}\'s status to {allocation_status.name}.') + + self.stdout.write(self.style.WARNING(message)) + else: + project.status = project_status + project.save() + allocation.status = allocation_status + allocation.start_date = current_date + allocation.end_date = None + allocation.save() + + message = ( + f'Updated Project {project.name} ({project.pk})\'s status to ' + f'{project_status.name} and Allocation {allocation.pk}\'s ' + f'status to {allocation_status.name}.') + + self.logger.info(message) + self.stdout.write(self.style.SUCCESS(message)) + + def reset_service_units(self, project, dry_run): + """ + Resets service units for a project and its users to 0.00. Creates + the relevant transaction objects to record the change. Updates the + relevant historical objects with the reason for the SU change. + + If dry_run is True, write to stdout without changing object fields. + """ + allocation_objects = get_accounting_allocation_objects(project) + current_allocation = Decimal(allocation_objects.allocation_attribute.value) + current_date = utc_now_offset_aware() + reason = 'Resetting SUs while deactivating expired ICA project.' + updated_su = Decimal('0.00') + + if dry_run: + message = f'Would reset {project.name} and its users\'s SUs from ' \ + f'{current_allocation} to {updated_su}. The reason ' \ + f'would be: "Resetting SUs while deactivating expired ' \ + f'ICA project."' + + self.stdout.write(self.style.WARNING(message)) + + else: + # Set the value for the Project. + set_project_allocation_value(project, updated_su) + + # Create a transaction to record the change. + ProjectTransaction.objects.create( + project=project, + date_time=current_date, + allocation=updated_su) + + # Set the reason for the change in the newly-created historical object. + allocation_objects.allocation_attribute.refresh_from_db() + historical_allocation_attribute = \ + allocation_objects.allocation_attribute.history.latest("id") + historical_allocation_attribute.history_change_reason = reason + historical_allocation_attribute.save() + + # Do the same for each ProjectUser. + allocation_attribute_type = AllocationAttributeType.objects.get( + name="Service Units") + for project_user in project.projectuser_set.all(): + user = project_user.user + # Attempt to set the value for the ProjectUser. The method returns whether + # it succeeded; it may not because not every ProjectUser has a + # corresponding AllocationUser (e.g., PIs). Only proceed with further steps + # if an update occurred. + allocation_updated = set_project_user_allocation_value( + user, project, updated_su) + if allocation_updated: + # Create a transaction to record the change. + ProjectUserTransaction.objects.create( + project_user=project_user, + date_time=current_date, + allocation=updated_su) + # Set the reason for the change in the newly-created historical object. + allocation_user = \ + allocation_objects.allocation.allocationuser_set.get(user=user) + allocation_user_attribute = \ + allocation_user.allocationuserattribute_set.get( + allocation_attribute_type=allocation_attribute_type, + allocation=allocation_objects.allocation) + historical_allocation_user_attribute = \ + allocation_user_attribute.history.latest("id") + historical_allocation_user_attribute.history_change_reason = reason + historical_allocation_user_attribute.save() + + message = f'Successfully reset SUs for {project.name} ' \ + f'and its users, updating {project.name}\'s SUs from ' \ + f'{current_allocation} to {updated_su}. The reason ' \ + f'was: "{reason}".' + + self.logger.info(message) + self.stdout.write(self.style.SUCCESS(message)) + + return current_allocation + + def send_emails(self, project, expiry_date, dry_run): + """ + Send emails to managers/PIs of the project that have notifications + enabled about the project deactivation. + """ + + if settings.EMAIL_ENABLED: + context = { + 'project_name': project.name, + 'expiry_date': expiry_date, + 'signature': settings.EMAIL_SIGNATURE, + } + + pi_condition = Q( + role__name='Principal Investigator', status__name='Active', + enable_notifications=True) + manager_condition = Q(role__name='Manager', status__name='Active') + + recipients = list( + project.projectuser_set.filter( + pi_condition | manager_condition + ).values_list( + 'user__email', flat=True + )) + + if dry_run: + msg_plain = \ + render_to_string('email/expired_ica_project.txt', + context) + + message = f'Would send the following email to ' \ + f'{len(recipients)} users:' + self.stdout.write(self.style.WARNING(message)) + self.stdout.write(self.style.WARNING(msg_plain)) + + else: + try: + send_email_template( + 'Expired ICA Project Deactivation', + 'email/expired_ica_project.txt', + context, + settings.EMAIL_SENDER, + recipients) + + message = f'Sent deactivation notification email to ' \ + f'{len(recipients)} users.' + self.stdout.write(self.style.SUCCESS(message)) + + except Exception as e: + message = 'Failed to send notification email. Details:' + self.stderr.write(self.style.ERROR(message)) + self.stderr.write(self.style.ERROR(str(e))) + self.logger.error(message) + self.logger.exception(e) + else: + message = 'settings.EMAIL_ENABLED set to False. ' \ + 'No emails will be sent.' + self.stderr.write(self.style.ERROR(message)) \ No newline at end of file From 57bea671e9803131879bcaf7e3d0ec87e00253aa Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Thu, 17 Feb 2022 10:53:01 -0500 Subject: [PATCH 025/150] adding notification email for expired ica projects --- coldfront/templates/email/expired_ica_project.txt | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 coldfront/templates/email/expired_ica_project.txt diff --git a/coldfront/templates/email/expired_ica_project.txt b/coldfront/templates/email/expired_ica_project.txt new file mode 100644 index 000000000..7400eaf57 --- /dev/null +++ b/coldfront/templates/email/expired_ica_project.txt @@ -0,0 +1,7 @@ +Dear managers of {{ project_name }}, + +This is a notification that the project {{ project_name }} expired on {{ expiry_date }} and has therefore been +deactivated. Accounts under this project will no longer be able to access its compute resources. + +Thank you, +{{ signature }} From 9c3d6219fa16bb60a68b8e37e0469d8d3cac5f07 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Thu, 17 Feb 2022 10:54:42 -0500 Subject: [PATCH 026/150] added support email to notificatin email --- .../core/project/management/commands/deactivate_ica_projects.py | 1 + coldfront/templates/email/expired_ica_project.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/coldfront/core/project/management/commands/deactivate_ica_projects.py b/coldfront/core/project/management/commands/deactivate_ica_projects.py index cc54f1d03..ecb143e05 100644 --- a/coldfront/core/project/management/commands/deactivate_ica_projects.py +++ b/coldfront/core/project/management/commands/deactivate_ica_projects.py @@ -183,6 +183,7 @@ def send_emails(self, project, expiry_date, dry_run): context = { 'project_name': project.name, 'expiry_date': expiry_date, + 'support_email': settings.CENTER_HELP_EMAIL, 'signature': settings.EMAIL_SIGNATURE, } diff --git a/coldfront/templates/email/expired_ica_project.txt b/coldfront/templates/email/expired_ica_project.txt index 7400eaf57..852131fd1 100644 --- a/coldfront/templates/email/expired_ica_project.txt +++ b/coldfront/templates/email/expired_ica_project.txt @@ -3,5 +3,7 @@ Dear managers of {{ project_name }}, This is a notification that the project {{ project_name }} expired on {{ expiry_date }} and has therefore been deactivated. Accounts under this project will no longer be able to access its compute resources. +If you believe this is a mistake, please contact us at {{ support_email }}. + Thank you, {{ signature }} From 8b258c300035424463c8b1527c2e0007a9c2e059 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Thu, 17 Feb 2022 10:55:48 -0500 Subject: [PATCH 027/150] formatted date string --- .../core/project/management/commands/deactivate_ica_projects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coldfront/core/project/management/commands/deactivate_ica_projects.py b/coldfront/core/project/management/commands/deactivate_ica_projects.py index ecb143e05..7027c3d9c 100644 --- a/coldfront/core/project/management/commands/deactivate_ica_projects.py +++ b/coldfront/core/project/management/commands/deactivate_ica_projects.py @@ -182,7 +182,7 @@ def send_emails(self, project, expiry_date, dry_run): if settings.EMAIL_ENABLED: context = { 'project_name': project.name, - 'expiry_date': expiry_date, + 'expiry_date': expiry_date.strftime('%m-%d-%Y'), 'support_email': settings.CENTER_HELP_EMAIL, 'signature': settings.EMAIL_SIGNATURE, } From 864ecbe51a67cb35c0f2723fa250eabc5f534cf8 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Thu, 17 Feb 2022 18:08:56 -0500 Subject: [PATCH 028/150] adding tests for deactivate_ica_projects --- .../test_deactivate_ica_projects.py | 452 ++++++++++++++++++ 1 file changed, 452 insertions(+) create mode 100644 coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py diff --git a/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py b/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py new file mode 100644 index 000000000..84446b429 --- /dev/null +++ b/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py @@ -0,0 +1,452 @@ +import datetime +from decimal import Decimal +from io import StringIO +import sys + +from django.contrib.auth.models import User +from django.core import mail +from django.core.management import call_command +from django.db.models import Q + +from coldfront.api.statistics.utils import create_project_allocation, \ + create_user_project_allocation, AccountingAllocationObjects +from coldfront.config import settings +from coldfront.core.allocation.models import Allocation, \ + AllocationAttributeType, AllocationAttribute, \ + AllocationAttributeUsage, AllocationUserStatusChoice, AllocationUser, \ + AllocationUserAttribute, AllocationUserAttributeUsage +from coldfront.core.project.models import Project, ProjectStatusChoice, \ + ProjectUserStatusChoice, ProjectUserRoleChoice, ProjectUser +from coldfront.core.project.utils import get_project_compute_allocation +from coldfront.core.statistics.models import ProjectTransaction, ProjectUserTransaction +from coldfront.core.user.models import UserProfile +from coldfront.core.utils.common import utc_now_offset_aware +from coldfront.core.utils.tests.test_base import TestBase + + +class TestDeactivateICAProject(TestBase): + """Class for testing the management command deactivate_ica_projects""" + + def setUp(self): + """Set up test data.""" + super().setUp() + + # Create a PI. + self.pi = User.objects.create( + username='pi0', email='pi0@nonexistent.com') + user_profile = UserProfile.objects.get(user=self.pi) + user_profile.is_pi = True + user_profile.save() + + # Create two Users. + for i in range(2): + user = User.objects.create( + username=f'user{i}', email=f'user{i}@nonexistent.com') + user_profile = UserProfile.objects.get(user=user) + user_profile.cluster_uid = f'{i}' + user_profile.save() + setattr(self, f'user{i}', user) + setattr(self, f'user_profile{i}', user_profile) + + # Create Projects and associate Users with them. + project_status = ProjectStatusChoice.objects.get(name='Active') + project_user_status = ProjectUserStatusChoice.objects.get( + name='Active') + user_role = ProjectUserRoleChoice.objects.get(name='User') + manager_role = ProjectUserRoleChoice.objects.get(name='Manager') + current_date = utc_now_offset_aware() + + for i in range(2): + # Create an ICA Project and ProjectUsers. + project = Project.objects.create( + name=f'ic_project{i}', status=project_status) + setattr(self, f'ic_project{i}', project) + for j in range(2): + ProjectUser.objects.create( + user=getattr(self, f'user{j}'), project=project, + role=user_role, status=project_user_status) + ProjectUser.objects.create( + user=self.pi, project=project, role=manager_role, + status=project_user_status) + + # Create a compute allocation for the Project. + allocation = Decimal(f'{i + 1}000.00') + create_project_allocation(project, allocation) + + # Set start and end dates for allocations + allocation_object = get_project_compute_allocation(project) + allocation_object.start_date = \ + current_date - datetime.timedelta(days=(i+1)*5) + allocation_object.end_date = \ + current_date + datetime.timedelta(days=(i+1)*5) + allocation_object.save() + + # Create a compute allocation for each User on the Project. + for j in range(2): + create_user_project_allocation( + getattr(self, f'user{j}'), project, allocation / 2) + + # Clear the mail outbox. + mail.outbox = [] + + def get_accounting_allocation_objects(self, project, user=None): + """Return a namedtuple of database objects related to accounting and + allocation for the given project and optional user. + + Parameters: + - project (Project): an instance of the Project model + - user (User): an instance of the User model + + Returns: + - AccountingAllocationObjects instance + + Raises: + - MultipleObjectsReturned, if a database retrieval returns more + than one object + - ObjectDoesNotExist, if a database retrieval returns less than + one object + - TypeError, if one or more inputs has the wrong type + + NOTE: this function was taken from coldfront.core.statistics.utils. + This version does not check that the allocation status is Active + """ + if not isinstance(project, Project): + raise TypeError(f'Project {project} is not a Project object.') + + objects = AccountingAllocationObjects() + + allocation = Allocation.objects.get( + project=project, resources__name='Savio Compute') + + # Check that the allocation has an attribute for Service Units and + # an associated usage. + allocation_attribute_type = AllocationAttributeType.objects.get( + name='Service Units') + allocation_attribute = AllocationAttribute.objects.get( + allocation_attribute_type=allocation_attribute_type, + allocation=allocation) + allocation_attribute_usage = AllocationAttributeUsage.objects.get( + allocation_attribute=allocation_attribute) + + objects.allocation = allocation + objects.allocation_attribute = allocation_attribute + objects.allocation_attribute_usage = allocation_attribute_usage + + if user is None: + return objects + + if not isinstance(user, User): + raise TypeError(f'User {user} is not a User object.') + + # Check that there is an active association between the user and project. + active_status = ProjectUserStatusChoice.objects.get(name='Active') + ProjectUser.objects.get(user=user, project=project, status=active_status) + + # Check that the user is an active member of the allocation. + active_status = AllocationUserStatusChoice.objects.get(name='Active') + allocation_user = AllocationUser.objects.get( + allocation=allocation, user=user, status=active_status) + + # Check that the allocation user has an attribute for Service Units + # and an associated usage. + allocation_user_attribute = AllocationUserAttribute.objects.get( + allocation_attribute_type=allocation_attribute_type, + allocation=allocation, allocation_user=allocation_user) + allocation_user_attribute_usage = AllocationUserAttributeUsage.objects.get( + allocation_user_attribute=allocation_user_attribute) + + objects.allocation_user = allocation_user + objects.allocation_user_attribute = allocation_user_attribute + objects.allocation_user_attribute_usage = allocation_user_attribute_usage + + return objects + + def create_expired_project(self, project_name): + """Change the end date of ic_project0 to be expired""" + project = Project.objects.get(name=project_name) + allocation = get_project_compute_allocation(project) + current_date = utc_now_offset_aware() + expired_date = (current_date - datetime.timedelta(days=4)).date() + + allocation.end_date = expired_date + allocation.save() + allocation.refresh_from_db() + self.assertEqual(allocation.end_date, expired_date) + + def record_historical_objects_len(self, project): + """ Records the lengths of all relevant historical objects to a dict""" + length_dict = {} + allocation_objects = self.get_accounting_allocation_objects(project) + historical_allocation_attribute = \ + allocation_objects.allocation_attribute.history.all() + + length_dict['historical_allocation_attribute'] = \ + len(historical_allocation_attribute) + + allocation_attribute_type = AllocationAttributeType.objects.get( + name="Service Units") + for project_user in project.projectuser_set.all(): + if project_user.role.name != 'User': + continue + + allocation_user = \ + allocation_objects.allocation.allocationuser_set.get( + user=project_user.user) + allocation_user_attribute = \ + allocation_user.allocationuserattribute_set.get( + allocation_attribute_type=allocation_attribute_type, + allocation=allocation_objects.allocation) + historical_allocation_user_attribute = \ + allocation_user_attribute.history.all() + + key = 'historical_allocation_user_attribute_' + project_user.user.username + length_dict[key] = len(historical_allocation_user_attribute) + + return length_dict + + def allocation_values_test(self, project, value, user_value): + """ + Tests that the allocation user values are correct + """ + allocation_objects = self.get_accounting_allocation_objects(project) + self.assertEqual(allocation_objects.allocation_attribute.value, value) + + for project_user in project.projectuser_set.all(): + if project_user.role.name != 'User': + continue + allocation_objects = self.get_accounting_allocation_objects( + project, user=project_user.user) + + self.assertEqual(allocation_objects.allocation_user_attribute.value, + user_value) + + def transactions_created(self, project, pre_time, post_time, amount): + """ + Tests that transactions were created for the zeroing of SUs + """ + proj_transaction = ProjectTransaction.objects.get(project=project, + allocation=amount) + + self.assertTrue(pre_time <= proj_transaction.date_time <= post_time) + + for project_user in project.projectuser_set.all(): + if project_user.role.name != 'User': + continue + + proj_user_transaction = ProjectUserTransaction.objects.get( + project_user=project_user, + allocation=amount) + + self.assertTrue(pre_time <= proj_user_transaction.date_time <= post_time) + + def historical_objects_created(self, pre_length_dict, post_length_dict): + """Test that historical objects were created""" + for k, v in pre_length_dict.items(): + self.assertEqual(v + 1, post_length_dict[k]) + + def historical_objects_updated(self, project): + """ + Tests that the relevant historical objects have the correct reason + """ + reason = 'Resetting SUs while deactivating expired ICA project.' + allocation_objects = self.get_accounting_allocation_objects(project) + historical_allocation_attribute = \ + allocation_objects.allocation_attribute.history.latest("id") + historical_reason = historical_allocation_attribute.history_change_reason + + self.assertEqual(historical_reason, reason) + + allocation_attribute_type = AllocationAttributeType.objects.get( + name="Service Units") + for project_user in project.projectuser_set.all(): + if project_user.role.name != 'User': + continue + + allocation_user = \ + allocation_objects.allocation.allocationuser_set.get( + user=project_user.user) + allocation_user_attribute = \ + allocation_user.allocationuserattribute_set.get( + allocation_attribute_type=allocation_attribute_type, + allocation=allocation_objects.allocation) + historical_allocation_user_attribute = \ + allocation_user_attribute.history.latest("id") + historical_reason = \ + historical_allocation_user_attribute.history_change_reason + self.assertEqual(historical_reason, reason) + + def project_allocation_updates(self, project, allocation, pre_time, post_time): + """Tests that the project and allocation were correctly updated""" + self.assertEqual(project.status.name, 'Inactive') + self.assertEqual(allocation.status.name, 'Expired') + self.assertTrue(pre_time.date() <= allocation.start_date <= post_time.date()) + self.assertTrue(allocation.end_date is None) + + def test_dry_run_no_expired_projects(self): + """Testing a dry run in which no ICA projects are expired""" + out, err = StringIO(''), StringIO('') + call_command('deactivate_ica_projects', + '--dry_run', + '--send_emails', + stdout=out, + stderr=err) + sys.stdout = sys.__stdout__ + out.seek(0) + + self.assertIn(out.read(), '') + err.seek(0) + self.assertEqual(err.read(), '') + + def test_dry_run_with_expired_projects(self): + """Testing a dry run in which an ICA project is expired""" + self.create_expired_project('ic_project0') + + out, err = StringIO(''), StringIO('') + call_command('deactivate_ica_projects', + '--dry_run', + '--send_emails', + stdout=out, + stderr=err) + sys.stdout = sys.__stdout__ + out.seek(0) + + console_output = out.read() + + project = Project.objects.get(name='ic_project0') + allocation = get_project_compute_allocation(project) + + # Messages that should be output to stdout during a dry run + messages = [f'Would update Project {project.name} ({project.pk})\'s ' + f'status to Inactive and Allocation ' + f'{allocation.pk}\'s status to Expired.', + + f'Would reset {project.name} and its users\'s SUs from ' + f'1000.00 to 0.00. The reason ' + f'would be: "Resetting SUs while deactivating expired ' + f'ICA project."', + + 'Would send the following email to 1 users:', + f'Dear managers of {project.name},', + + f'This is a notification that the project {project.name} ' + f'expired on {allocation.end_date.strftime("%m-%d-%Y")} ' + f'and has therefore been deactivated. ' + f'Accounts under this project will no longer be able ' + f'to access its compute resources.'] + + for message in messages: + self.assertIn(message, console_output) + + err.seek(0) + self.assertEqual(err.read(), '') + + def test_creates_and_updates_objects(self): + """Testing deactivate_ica_projects WITHOUT send_emails flag""" + self.create_expired_project('ic_project0') + + project = Project.objects.get(name='ic_project0') + allocation = get_project_compute_allocation(project) + + pre_time = utc_now_offset_aware() + pre_length_dict = self.record_historical_objects_len(project) + + # test allocation values before command + self.allocation_values_test(project, '1000.00', '500.00') + + # run command + out, err = StringIO(''), StringIO('') + call_command('deactivate_ica_projects', + stdout=out, + stderr=err) + sys.stdout = sys.__stdout__ + out.seek(0) + console_output = out.read() + + messages = [ + f'Updated Project {project.name} ({project.pk})\'s status to ' + f'Inactive and Allocation {allocation.pk}\'s ' + f'status to Expired.', + + f'Successfully reset SUs for {project.name} ' + f'and its users, updating {project.name}\'s SUs from ' + f'1000.00 to 0.00. The reason ' + f'was: "Resetting SUs while deactivating expired ICA ' + f'project.".'] + + for message in messages: + self.assertIn(message, console_output) + err.seek(0) + self.assertEqual(err.read(), '') + + post_time = utc_now_offset_aware() + project.refresh_from_db() + allocation.refresh_from_db() + + # test project and allocation statuses + self.project_allocation_updates(project, allocation, pre_time, post_time) + + # test allocation values after command + self.allocation_values_test(project, '0.00', '0.00') + + # test ProjectTransaction created + self.transactions_created(project, pre_time, post_time, 0.00) + + # test historical objects created and updated + post_length_dict = self.record_historical_objects_len(project) + self.historical_objects_created(pre_length_dict, post_length_dict) + self.historical_objects_updated(project) + + def test_emails_sent(self): + """ + Tests that emails are sent correctly when there is an expired + ICA project + """ + + self.create_expired_project('ic_project0') + + project = Project.objects.get(name='ic_project0') + allocation = get_project_compute_allocation(project) + old_end_date = allocation.end_date + + # run command + out, err = StringIO(''), StringIO('') + call_command('deactivate_ica_projects', + '--send_emails', + stdout=out, + stderr=err) + sys.stdout = sys.__stdout__ + out.seek(0) + + pi_condition = Q( + role__name='Principal Investigator', status__name='Active', + enable_notifications=True) + manager_condition = Q(role__name='Manager', status__name='Active') + + recipients = list( + project.projectuser_set.filter( + pi_condition | manager_condition + ).values_list( + 'user__email', flat=True + )) + + # Testing that the correct text is output to stdout + message = f'Sent deactivation notification email to ' \ + f'{len(recipients)} users.' + self.assertIn(message, out.read()) + + # Testing that the correct number of emails were sent + self.assertEqual(len(mail.outbox), len(recipients)) + + email_body = [f'Dear managers of {project.name},', + + f'This is a notification that the project {project.name} ' + f'expired on {old_end_date.strftime("%m-%d-%Y")} ' + f'and has therefore been deactivated. ' + f'Accounts under this project will no longer be able ' + f'to access its compute resources.'] + + for email in mail.outbox: + for section in email_body: + self.assertIn(section, email.body) + self.assertIn(email.to[0], recipients) + self.assertEqual(settings.EMAIL_SENDER, email.from_email) From 196e371493ace72b4a1254170312af1289299bd9 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Thu, 17 Feb 2022 18:09:31 -0500 Subject: [PATCH 029/150] now zeroes SUs before setting statuses --- .../project/management/commands/deactivate_ica_projects.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/coldfront/core/project/management/commands/deactivate_ica_projects.py b/coldfront/core/project/management/commands/deactivate_ica_projects.py index 7027c3d9c..df70951a4 100644 --- a/coldfront/core/project/management/commands/deactivate_ica_projects.py +++ b/coldfront/core/project/management/commands/deactivate_ica_projects.py @@ -36,7 +36,7 @@ def add_arguments(self, parser): action='store_true', help='Display updates without performing them.') parser.add_argument( - '--send-emails', + '--send_emails', action='store_true', default=False, help='Send emails to PIs/managers about project deactivation.') @@ -53,9 +53,9 @@ def handle(self, *args, **options): allocation = get_project_compute_allocation(project) expiry_date = allocation.end_date - if allocation.end_date < current_date: - self.deactivate_project(project, allocation, options['dry_run']) + if expiry_date < current_date.date(): self.reset_service_units(project, options['dry_run']) + self.deactivate_project(project, allocation, options['dry_run']) if options['send_emails']: self.send_emails(project, expiry_date, options['dry_run']) From 975313c7513217a5db7be52ed3428f360efff68b Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Thu, 17 Feb 2022 18:09:50 -0500 Subject: [PATCH 030/150] removed newline --- coldfront/templates/email/expired_ica_project.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/coldfront/templates/email/expired_ica_project.txt b/coldfront/templates/email/expired_ica_project.txt index 852131fd1..c076b75b9 100644 --- a/coldfront/templates/email/expired_ica_project.txt +++ b/coldfront/templates/email/expired_ica_project.txt @@ -1,7 +1,6 @@ Dear managers of {{ project_name }}, -This is a notification that the project {{ project_name }} expired on {{ expiry_date }} and has therefore been -deactivated. Accounts under this project will no longer be able to access its compute resources. +This is a notification that the project {{ project_name }} expired on {{ expiry_date }} and has therefore been deactivated. Accounts under this project will no longer be able to access its compute resources. If you believe this is a mistake, please contact us at {{ support_email }}. From da0ee16a221ee4a780c974d81f9d1b8b994579dd Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Thu, 17 Feb 2022 18:18:54 -0500 Subject: [PATCH 031/150] changed some comments --- .../project/management/commands/deactivate_ica_projects.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/coldfront/core/project/management/commands/deactivate_ica_projects.py b/coldfront/core/project/management/commands/deactivate_ica_projects.py index df70951a4..b90a5d50b 100644 --- a/coldfront/core/project/management/commands/deactivate_ica_projects.py +++ b/coldfront/core/project/management/commands/deactivate_ica_projects.py @@ -62,8 +62,8 @@ def handle(self, *args, **options): def deactivate_project(self, project, allocation, dry_run): """ - Sets project status to Inactive and corresponding compute Allocation to - expired. Sets allocation start date to the current date and removes + Sets project status to Inactive and corresponding compute allocation to + Expired. Sets allocation start date to the current date and removes the end date. If dry_run is True, write to stdout without changing object fields. @@ -177,6 +177,8 @@ def send_emails(self, project, expiry_date, dry_run): """ Send emails to managers/PIs of the project that have notifications enabled about the project deactivation. + + If dry_run is True, write the emails to stdout instead. """ if settings.EMAIL_ENABLED: From e7a8c0c8515869ccd15dc222534a711f5e6c0880 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Fri, 18 Feb 2022 07:41:26 -0500 Subject: [PATCH 032/150] validate input now returns some database objects --- .../commands/add_service_units_to_project.py | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/coldfront/core/project/management/commands/add_service_units_to_project.py b/coldfront/core/project/management/commands/add_service_units_to_project.py index 14c36608a..656f9a30e 100644 --- a/coldfront/core/project/management/commands/add_service_units_to_project.py +++ b/coldfront/core/project/management/commands/add_service_units_to_project.py @@ -37,16 +37,22 @@ def add_arguments(self, parser): action='store_true') def validate_inputs(self, options): - """Validate inputs to add_service_units_to_project command""" + """ + Validate inputs to add_service_units_to_project command + + Returns a tuple of the project object, allocation objects, current + SU amount, and new SU amount + """ # Checking if project exists - if not Project.objects.filter(name=options.get('project_name')).exists(): + project_query = Project.objects.filter(name=options.get('project_name')) + if not project_query.exists(): error_message = f"Requested project {options.get('project_name')}" \ f" does not exist." raise CommandError(error_message) # Allocation must be in Savio Compute - project = Project.objects.get(name=options.get('project_name')) + project = project_query.first() try: allocation_objects = get_accounting_allocation_objects(project) except Allocation.DoesNotExist: @@ -62,7 +68,7 @@ def validate_inputs(self, options): # checking SU values if addition > settings.ALLOCATION_MAX: - error_message = f'Amount of SUs to add must cannot be greater ' \ + error_message = f'Amount of SUs to add cannot be greater ' \ f'than {settings.ALLOCATION_MAX}.' raise CommandError(error_message) @@ -76,19 +82,18 @@ def validate_inputs(self, options): error_message = f'Reason must be at least 20 characters.' raise CommandError(error_message) + return project, allocation_objects, current_allocation, allocation + def handle(self, *args, **options): """ Add SUs to a given project """ - self.validate_inputs(options) + # validate inputs + project, allocation_objects, current_allocation, allocation = \ + self.validate_inputs(options) - project = Project.objects.get(name=options.get('project_name')) addition = Decimal(options.get('amount')) reason = options.get('reason') dry_run = options.get('dry_run', None) - - allocation_objects = get_accounting_allocation_objects(project) - current_allocation = Decimal(allocation_objects.allocation_attribute.value) - allocation = addition + current_allocation date_time = utc_now_offset_aware() if dry_run: From a8cb628dc1e994d80cadffa90b3bffc524ab4672 Mon Sep 17 00:00:00 2001 From: Matthew Li Date: Fri, 18 Feb 2022 09:59:08 -0800 Subject: [PATCH 033/150] Use 'increase'/'decrease' in dry_run depending on sign of addition; remove unused import and unnecessary comment --- .../management/commands/add_service_units_to_project.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/coldfront/core/project/management/commands/add_service_units_to_project.py b/coldfront/core/project/management/commands/add_service_units_to_project.py index 656f9a30e..5bb33fea8 100644 --- a/coldfront/core/project/management/commands/add_service_units_to_project.py +++ b/coldfront/core/project/management/commands/add_service_units_to_project.py @@ -1,7 +1,6 @@ import logging from decimal import Decimal -from django.core.exceptions import ObjectDoesNotExist from django.core.management import BaseCommand, CommandError from coldfront.config import settings @@ -86,8 +85,6 @@ def validate_inputs(self, options): def handle(self, *args, **options): """ Add SUs to a given project """ - - # validate inputs project, allocation_objects, current_allocation, allocation = \ self.validate_inputs(options) @@ -97,8 +94,9 @@ def handle(self, *args, **options): date_time = utc_now_offset_aware() if dry_run: + verb = 'increase' if addition > 0 else 'decrease' message = f'Would add {addition} additional SUs to project ' \ - f'{project.name}. This would increase {project.name} ' \ + f'{project.name}. This would {verb} {project.name} ' \ f'SUs from {current_allocation} to {allocation}. ' \ f'The reason for updating SUs for {project.name} ' \ f'would be: "{reason}".' From 406be365035ff53499719c6fca22eb90cabeea1e Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Fri, 18 Feb 2022 13:06:52 -0500 Subject: [PATCH 034/150] Allocation attributes, Allocation user attributes, Allocation attribute usages, and Allocation user attribute usages admin pages can now search using project name --- coldfront/core/allocation/admin.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/coldfront/core/allocation/admin.py b/coldfront/core/allocation/admin.py index 786c57813..12e61e5cd 100644 --- a/coldfront/core/allocation/admin.py +++ b/coldfront/core/allocation/admin.py @@ -156,8 +156,8 @@ def queryset(self, request, queryset): @admin.register(AllocationAttribute) class AllocationAttributeAdmin(SimpleHistoryAdmin): readonly_fields_change = ( - 'allocation', 'allocation_attribute_type', 'created', 'modified', 'project_title') - fields_change = ('project_title', 'allocation', + 'allocation', 'allocation_attribute_type', 'created', 'modified', 'project') + fields_change = ('project', 'allocation', 'allocation_attribute_type', 'value', 'created', 'modified',) list_display = ('pk', 'project', 'pis', 'resource', 'allocation_status', 'allocation_attribute_type', 'value', 'usage', 'created', 'modified',) @@ -168,6 +168,7 @@ class AllocationAttributeAdmin(SimpleHistoryAdmin): 'allocation__allocationuser__user__first_name', 'allocation__allocationuser__user__last_name', 'allocation__allocationuser__user__username', + 'allocation__project__name', ) def usage(self, obj): @@ -190,7 +191,8 @@ def pis(self, obj): for pi_user in pi_users]) def project(self, obj): - return textwrap.shorten(obj.allocation.project.title, width=50) + return obj.allocation.project.name + #return textwrap.shorten(obj.allocation.project.title, width=50) def project_title(self, obj): return obj.allocation.project.title @@ -334,12 +336,14 @@ class AllocationAttributeUsageAdmin(SimpleHistoryAdmin): fields = ('allocation_attribute', 'value',) list_filter = ('allocation_attribute__allocation_attribute_type', 'allocation_attribute__allocation__resources', ValueFilter, ) + search_fields = ('allocation_attribute__allocation__resources__name', + 'allocation_attribute__allocation__project__name') def resource(self, obj): return obj.allocation_attribute.allocation.resources.first().name def project(self, obj): - return obj.allocation_attribute.allocation.project.title + return obj.allocation_attribute.allocation.project.name def project_pis(self, obj): project = obj.allocation_attribute.allocation.project @@ -359,8 +363,8 @@ class AllocationUserAttributeUsageInline(admin.TabularInline): @admin.register(AllocationUserAttribute) class AllocationUserAttributeAdmin(SimpleHistoryAdmin): readonly_fields_change = ( - 'allocation_user', 'allocation', 'allocation_attribute_type', 'created', 'modified', 'project_title') - fields_change = ('project_title', 'allocation', 'allocation_user', + 'allocation_user', 'allocation', 'allocation_attribute_type', 'created', 'modified', 'project') + fields_change = ('project', 'allocation', 'allocation_user', 'allocation_attribute_type', 'value', 'created', 'modified',) list_display = ('pk', 'user', 'project', 'resource', 'allocation_attribute_type', 'value', 'created', 'modified',) @@ -370,7 +374,8 @@ class AllocationUserAttributeAdmin(SimpleHistoryAdmin): search_fields = ( 'allocation_user__user__first_name', 'allocation_user__user__last_name', - 'allocation_user__user__username' + 'allocation_user__user__username', + 'allocation_user__allocation__project__name', ) def resource(self, obj): @@ -387,7 +392,8 @@ def pis(self, obj): for pi_user in pi_users]) def project(self, obj): - return textwrap.shorten(obj.allocation.project.title, width=50) + return obj.allocation.project.name + #return textwrap.shorten(obj.allocation.project.title, width=50) def user(self, obj): return textwrap.shorten(obj.allocation_user.user.username, width=50) @@ -416,9 +422,10 @@ class AllocationUserAttributeUsageAdmin(SimpleHistoryAdmin): fields = ('allocation_user_attribute', 'value',) list_filter = ('allocation_user_attribute__allocation_attribute_type', 'allocation_user_attribute__allocation__resources', ValueFilter, ) + search_fields = ('allocation_user_attribute__allocation__project__name',) def resource(self, obj): return obj.allocation_user_attribute.allocation.resources.first().name def project(self, obj): - return obj.allocation_user_attribute.allocation.project.title + return obj.allocation_user_attribute.allocation.project.name From 03d319a3df9e034cc1d6d470e275663b28ac2304 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Fri, 18 Feb 2022 13:10:46 -0500 Subject: [PATCH 035/150] allocation user attribute/usage now searchable with username. username in list display. --- coldfront/core/allocation/admin.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/coldfront/core/allocation/admin.py b/coldfront/core/allocation/admin.py index 12e61e5cd..5aabbde2b 100644 --- a/coldfront/core/allocation/admin.py +++ b/coldfront/core/allocation/admin.py @@ -416,16 +416,20 @@ def get_readonly_fields(self, request, obj): @admin.register(AllocationUserAttributeUsage) class AllocationUserAttributeUsageAdmin(SimpleHistoryAdmin): - list_display = ('allocation_user_attribute', 'project', + list_display = ('allocation_user_attribute', 'user', 'project', 'resource', 'value',) readonly_fields = ('allocation_user_attribute',) fields = ('allocation_user_attribute', 'value',) list_filter = ('allocation_user_attribute__allocation_attribute_type', 'allocation_user_attribute__allocation__resources', ValueFilter, ) - search_fields = ('allocation_user_attribute__allocation__project__name',) + search_fields = ('allocation_user_attribute__allocation__project__name', + 'allocation_user_attribute__allocation_user__user__username') def resource(self, obj): return obj.allocation_user_attribute.allocation.resources.first().name def project(self, obj): return obj.allocation_user_attribute.allocation.project.name + + def user(self, obj): + return obj.allocation_user_attribute.allocation_user.user.username From 7a3a7baf5b3da2351db9a6e007877c93261f26b6 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Fri, 18 Feb 2022 13:18:03 -0500 Subject: [PATCH 036/150] project name now searchable in project(user)transaction admin pages --- coldfront/core/statistics/admin.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/coldfront/core/statistics/admin.py b/coldfront/core/statistics/admin.py index 659bcef05..899ddb9c4 100644 --- a/coldfront/core/statistics/admin.py +++ b/coldfront/core/statistics/admin.py @@ -7,12 +7,17 @@ from coldfront.core.statistics.models import ProjectUserTransaction +@admin.register(ProjectTransaction) class ProjectTransactionAdmin(admin.ModelAdmin): list_display = ('date_time', 'project', 'allocation', ) + search_fields = ('project__name',) + +@admin.register(ProjectUserTransaction) class ProjectUserTransactionAdmin(admin.ModelAdmin): list_display = ('date_time', 'get_project', 'get_user', 'allocation', ) + search_fields = ('project_user__project__name',) def get_project(self, obj): return obj.project_user.project @@ -42,5 +47,3 @@ def has_change_permission(self, request, obj=None): admin.register(CPU) admin.register(Node) -admin.site.register(ProjectTransaction, ProjectTransactionAdmin) -admin.site.register(ProjectUserTransaction, ProjectUserTransactionAdmin) From e6caeb39d8430f587f0048d59d7152c9b42701ba Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Fri, 18 Feb 2022 13:27:26 -0500 Subject: [PATCH 037/150] project admin page now has projecttransaction inlines --- coldfront/core/project/admin.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/coldfront/core/project/admin.py b/coldfront/core/project/admin.py index d5d9ab1d6..42c9dd4dc 100644 --- a/coldfront/core/project/admin.py +++ b/coldfront/core/project/admin.py @@ -8,6 +8,7 @@ ProjectUser, ProjectUserMessage, ProjectUserRoleChoice, ProjectUserStatusChoice) +from coldfront.core.statistics.models import ProjectTransaction @admin.register(ProjectStatusChoice) @@ -84,6 +85,13 @@ class ProjectUserMessageInline(admin.TabularInline): readonly_fields = ('author', 'created') +class ProjectTransactionInline(admin.TabularInline): + model = ProjectTransaction + extra = 0 + fields = ('date_time', 'allocation',), + readonly_fields = ('date_time', 'allocation') + + @admin.register(Project) class ProjectAdmin(SimpleHistoryAdmin): fields_change = ('title', 'description', 'status', 'requires_review', 'force_review', 'created', 'modified', ) @@ -92,7 +100,8 @@ class ProjectAdmin(SimpleHistoryAdmin): search_fields = ['projectuser__user__username', 'projectuser__user__last_name', 'projectuser__user__last_name', 'title'] list_filter = ('status', 'force_review') - inlines = [ProjectUserInline, ProjectAdminCommentInline, ProjectUserMessageInline] + inlines = [ProjectUserInline, ProjectAdminCommentInline, + ProjectUserMessageInline, ProjectTransactionInline] raw_id_fields = [] def PIs(self, obj): From 3dfaeefe6c2561f0df8c12244ae4637b30ff79cd Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Fri, 18 Feb 2022 13:57:32 -0500 Subject: [PATCH 038/150] added inline for projectusertransactions to projectuser admin page --- coldfront/core/project/admin.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/coldfront/core/project/admin.py b/coldfront/core/project/admin.py index 42c9dd4dc..f1209be26 100644 --- a/coldfront/core/project/admin.py +++ b/coldfront/core/project/admin.py @@ -8,7 +8,8 @@ ProjectUser, ProjectUserMessage, ProjectUserRoleChoice, ProjectUserStatusChoice) -from coldfront.core.statistics.models import ProjectTransaction +from coldfront.core.statistics.models import (ProjectTransaction, + ProjectUserTransaction) @admin.register(ProjectStatusChoice) @@ -26,6 +27,13 @@ class ProjectUserStatusChoiceAdmin(admin.ModelAdmin): list_display = ('name',) +class ProjectUserTransactionInline(admin.TabularInline): + model = ProjectUserTransaction + extra = 0 + fields = ('date_time', 'allocation',), + readonly_fields = ('date_time', 'allocation') + + @admin.register(ProjectUser) class ProjectUserAdmin(SimpleHistoryAdmin): fields_change = ('user', 'project', 'role', 'status', 'created', 'modified', ) @@ -34,6 +42,7 @@ class ProjectUserAdmin(SimpleHistoryAdmin): 'modified',) list_filter = ('role', 'status') search_fields = ['user__username', 'user__first_name', 'user__last_name'] + inlines = [ProjectUserTransactionInline] raw_id_fields = ('user', 'project') def project_title(self, obj): @@ -58,10 +67,9 @@ def get_readonly_fields(self, request, obj): def get_inline_instances(self, request, obj=None): if obj is None: # We are adding an object - return super().get_inline_instances(request) - else: return [] - # return [inline(self.model, self.admin_site) for inline in self.inlines_change] + else: + return super().get_inline_instances(request) class ProjectUserInline(admin.TabularInline): From 56f3ee637d4f24b9ecfd300dd5d87af7945de7d3 Mon Sep 17 00:00:00 2001 From: Matthew Li Date: Fri, 18 Feb 2022 13:31:26 -0800 Subject: [PATCH 039/150] Display the correct cluster for the abc project --- coldfront/core/project/templates/project/project_detail.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/coldfront/core/project/templates/project/project_detail.html b/coldfront/core/project/templates/project/project_detail.html index b27fdebd3..16ebc28f0 100644 --- a/coldfront/core/project/templates/project/project_detail.html +++ b/coldfront/core/project/templates/project/project_detail.html @@ -111,7 +111,9 @@

Description: {{ project.description }}

Cluster: - {% if project.name|slice:":1" == "v" %} + {% if project.name == "abc" %} + ABC + {% elif project.name|slice:":1" == "v" %} Vector {% else %} Savio From 343445dfe0ed63541a874ffbaa85af7d7f6bd664 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Fri, 18 Feb 2022 20:49:08 -0500 Subject: [PATCH 040/150] resolving issues in PR 357 --- .../commands/deactivate_ica_projects.py | 70 ++++------ .../test_deactivate_ica_projects.py | 130 +++++++----------- 2 files changed, 77 insertions(+), 123 deletions(-) diff --git a/coldfront/core/project/management/commands/deactivate_ica_projects.py b/coldfront/core/project/management/commands/deactivate_ica_projects.py index b90a5d50b..8e0387ddf 100644 --- a/coldfront/core/project/management/commands/deactivate_ica_projects.py +++ b/coldfront/core/project/management/commands/deactivate_ica_projects.py @@ -3,15 +3,14 @@ from django.template.loader import render_to_string from coldfront.api.statistics.utils import get_accounting_allocation_objects, \ - set_project_allocation_value, set_project_user_allocation_value + set_project_allocation_value, set_project_user_allocation_value, \ + set_project_usage_value, set_project_user_usage_value from coldfront.config import settings -from coldfront.core.allocation.models import AllocationAttributeType from coldfront.core.allocation.models import AllocationStatusChoice from coldfront.core.allocation.utils import get_project_compute_allocation from coldfront.core.project.models import Project from coldfront.core.project.models import ProjectStatusChoice from django.core.management.base import BaseCommand -from django.db.models import Q import logging from coldfront.core.statistics.models import ProjectTransaction, \ @@ -25,9 +24,8 @@ class Command(BaseCommand): - help = ( - 'Set expired ICA Projects to \'Inactive\' and their corresponding ' - 'compute Allocations to \'Expired\'.') + help = ('Expire ICA projects whose end dates have passed and optionally ' + 'notify project owners') logger = logging.getLogger(__name__) def add_arguments(self, parser): @@ -47,13 +45,13 @@ def handle(self, *args, **options): the Allocation and AllocationUsers. """ current_date = utc_now_offset_aware() - ica_projects = Project.objects.filter(name__icontains='ic_') + ica_projects = Project.objects.filter(name__startswith='ic_') for project in ica_projects: allocation = get_project_compute_allocation(project) expiry_date = allocation.end_date - if expiry_date < current_date.date(): + if expiry_date and expiry_date < current_date.date(): self.reset_service_units(project, options['dry_run']) self.deactivate_project(project, allocation, options['dry_run']) @@ -62,9 +60,8 @@ def handle(self, *args, **options): def deactivate_project(self, project, allocation, dry_run): """ - Sets project status to Inactive and corresponding compute allocation to - Expired. Sets allocation start date to the current date and removes - the end date. + Expire ICA projects whose end dates have passed and optionally + notify project owners If dry_run is True, write to stdout without changing object fields. """ @@ -95,6 +92,13 @@ def deactivate_project(self, project, allocation, dry_run): self.logger.info(message) self.stdout.write(self.style.SUCCESS(message)) + def set_historical_reason(self, obj, reason): + """Set the latest historical object reason""" + obj.refresh_from_db() + historical_obj = obj.history.latest('id') + historical_obj.history_change_reason = reason + historical_obj.save() + def reset_service_units(self, project, dry_run): """ Resets service units for a project and its users to 0.00. Creates @@ -120,6 +124,7 @@ def reset_service_units(self, project, dry_run): else: # Set the value for the Project. set_project_allocation_value(project, updated_su) + set_project_usage_value(project, updated_su) # Create a transaction to record the change. ProjectTransaction.objects.create( @@ -128,40 +133,34 @@ def reset_service_units(self, project, dry_run): allocation=updated_su) # Set the reason for the change in the newly-created historical object. - allocation_objects.allocation_attribute.refresh_from_db() - historical_allocation_attribute = \ - allocation_objects.allocation_attribute.history.latest("id") - historical_allocation_attribute.history_change_reason = reason - historical_allocation_attribute.save() + self.set_historical_reason( + allocation_objects.allocation_attribute, reason) # Do the same for each ProjectUser. - allocation_attribute_type = AllocationAttributeType.objects.get( - name="Service Units") for project_user in project.projectuser_set.all(): user = project_user.user # Attempt to set the value for the ProjectUser. The method returns whether # it succeeded; it may not because not every ProjectUser has a # corresponding AllocationUser (e.g., PIs). Only proceed with further steps # if an update occurred. + allocation_updated = set_project_user_allocation_value( user, project, updated_su) - if allocation_updated: + allocation_usage_updated = set_project_user_usage_value( + user, project, updated_su) + + if allocation_updated and allocation_usage_updated: # Create a transaction to record the change. ProjectUserTransaction.objects.create( project_user=project_user, date_time=current_date, allocation=updated_su) # Set the reason for the change in the newly-created historical object. - allocation_user = \ - allocation_objects.allocation.allocationuser_set.get(user=user) - allocation_user_attribute = \ - allocation_user.allocationuserattribute_set.get( - allocation_attribute_type=allocation_attribute_type, - allocation=allocation_objects.allocation) - historical_allocation_user_attribute = \ - allocation_user_attribute.history.latest("id") - historical_allocation_user_attribute.history_change_reason = reason - historical_allocation_user_attribute.save() + + allocation_user_obj = get_accounting_allocation_objects( + project, user=user) + self.set_historical_reason( + allocation_user_obj.allocation_user_attribute, reason) message = f'Successfully reset SUs for {project.name} ' \ f'and its users, updating {project.name}\'s SUs from ' \ @@ -189,17 +188,8 @@ def send_emails(self, project, expiry_date, dry_run): 'signature': settings.EMAIL_SIGNATURE, } - pi_condition = Q( - role__name='Principal Investigator', status__name='Active', - enable_notifications=True) - manager_condition = Q(role__name='Manager', status__name='Active') - - recipients = list( - project.projectuser_set.filter( - pi_condition | manager_condition - ).values_list( - 'user__email', flat=True - )) + recipients = list(project.managers_and_pis_with_notifications() + .values_list('user__email', flat=True)) if dry_run: msg_plain = \ diff --git a/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py b/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py index 84446b429..c21177866 100644 --- a/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py +++ b/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py @@ -9,7 +9,8 @@ from django.db.models import Q from coldfront.api.statistics.utils import create_project_allocation, \ - create_user_project_allocation, AccountingAllocationObjects + create_user_project_allocation, AccountingAllocationObjects, \ + get_accounting_allocation_objects from coldfront.config import settings from coldfront.core.allocation.models import Allocation, \ AllocationAttributeType, AllocationAttribute, \ @@ -24,7 +25,7 @@ from coldfront.core.utils.tests.test_base import TestBase -class TestDeactivateICAProject(TestBase): +class TestDeactivateICAProjects(TestBase): """Class for testing the management command deactivate_ica_projects""" def setUp(self): @@ -89,6 +90,16 @@ def setUp(self): # Clear the mail outbox. mail.outbox = [] + @staticmethod + def call_deactivate_command(*args): + """Call the command with the given arguments, returning the messages + written to stdout and stderr.""" + out, err = StringIO(), StringIO() + args = ['deactivate_ica_projects', *args] + kwargs = {'stdout': out, 'stderr': err} + call_command(*args, **kwargs) + return out.getvalue(), err.getvalue() + def get_accounting_allocation_objects(self, project, user=None): """Return a namedtuple of database objects related to accounting and allocation for the given project and optional user. @@ -189,15 +200,11 @@ def record_historical_objects_len(self, project): if project_user.role.name != 'User': continue - allocation_user = \ - allocation_objects.allocation.allocationuser_set.get( - user=project_user.user) - allocation_user_attribute = \ - allocation_user.allocationuserattribute_set.get( - allocation_attribute_type=allocation_attribute_type, - allocation=allocation_objects.allocation) + allocation_user_obj = self.get_accounting_allocation_objects( + project, user=project_user.user) + historical_allocation_user_attribute = \ - allocation_user_attribute.history.all() + allocation_user_obj.allocation_user_attribute.history.all() key = 'historical_allocation_user_attribute_' + project_user.user.username length_dict[key] = len(historical_allocation_user_attribute) @@ -248,32 +255,26 @@ def historical_objects_updated(self, project): """ Tests that the relevant historical objects have the correct reason """ + reason = 'Resetting SUs while deactivating expired ICA project.' allocation_objects = self.get_accounting_allocation_objects(project) - historical_allocation_attribute = \ - allocation_objects.allocation_attribute.history.latest("id") - historical_reason = historical_allocation_attribute.history_change_reason - self.assertEqual(historical_reason, reason) + alloc_attr_hist_reason = \ + allocation_objects.allocation_attribute.history.\ + latest('id').history_change_reason + self.assertEqual(alloc_attr_hist_reason, reason) - allocation_attribute_type = AllocationAttributeType.objects.get( - name="Service Units") for project_user in project.projectuser_set.all(): if project_user.role.name != 'User': continue - allocation_user = \ - allocation_objects.allocation.allocationuser_set.get( - user=project_user.user) - allocation_user_attribute = \ - allocation_user.allocationuserattribute_set.get( - allocation_attribute_type=allocation_attribute_type, - allocation=allocation_objects.allocation) - historical_allocation_user_attribute = \ - allocation_user_attribute.history.latest("id") - historical_reason = \ - historical_allocation_user_attribute.history_change_reason - self.assertEqual(historical_reason, reason) + allocation_user_obj = self.get_accounting_allocation_objects( + project, user=project_user.user) + + alloc_attr_hist_reason = \ + allocation_user_obj.allocation_user_attribute.history. \ + latest('id').history_change_reason + self.assertEqual(alloc_attr_hist_reason, reason) def project_allocation_updates(self, project, allocation, pre_time, post_time): """Tests that the project and allocation were correctly updated""" @@ -284,33 +285,17 @@ def project_allocation_updates(self, project, allocation, pre_time, post_time): def test_dry_run_no_expired_projects(self): """Testing a dry run in which no ICA projects are expired""" - out, err = StringIO(''), StringIO('') - call_command('deactivate_ica_projects', - '--dry_run', - '--send_emails', - stdout=out, - stderr=err) - sys.stdout = sys.__stdout__ - out.seek(0) - - self.assertIn(out.read(), '') - err.seek(0) - self.assertEqual(err.read(), '') + output, error = self.call_deactivate_command('--dry_run', + '--send_emails') + self.assertIn(output, '') + self.assertEqual(error, '') def test_dry_run_with_expired_projects(self): """Testing a dry run in which an ICA project is expired""" self.create_expired_project('ic_project0') - out, err = StringIO(''), StringIO('') - call_command('deactivate_ica_projects', - '--dry_run', - '--send_emails', - stdout=out, - stderr=err) - sys.stdout = sys.__stdout__ - out.seek(0) - - console_output = out.read() + output, error = self.call_deactivate_command('--dry_run', + '--send_emails') project = Project.objects.get(name='ic_project0') allocation = get_project_compute_allocation(project) @@ -335,10 +320,9 @@ def test_dry_run_with_expired_projects(self): f'to access its compute resources.'] for message in messages: - self.assertIn(message, console_output) + self.assertIn(message, output) - err.seek(0) - self.assertEqual(err.read(), '') + self.assertEqual(error, '') def test_creates_and_updates_objects(self): """Testing deactivate_ica_projects WITHOUT send_emails flag""" @@ -354,13 +338,7 @@ def test_creates_and_updates_objects(self): self.allocation_values_test(project, '1000.00', '500.00') # run command - out, err = StringIO(''), StringIO('') - call_command('deactivate_ica_projects', - stdout=out, - stderr=err) - sys.stdout = sys.__stdout__ - out.seek(0) - console_output = out.read() + output, error = self.call_deactivate_command() messages = [ f'Updated Project {project.name} ({project.pk})\'s status to ' @@ -374,9 +352,8 @@ def test_creates_and_updates_objects(self): f'project.".'] for message in messages: - self.assertIn(message, console_output) - err.seek(0) - self.assertEqual(err.read(), '') + self.assertIn(message, output) + self.assertEqual(error, '') post_time = utc_now_offset_aware() project.refresh_from_db() @@ -409,30 +386,17 @@ def test_emails_sent(self): old_end_date = allocation.end_date # run command - out, err = StringIO(''), StringIO('') - call_command('deactivate_ica_projects', - '--send_emails', - stdout=out, - stderr=err) - sys.stdout = sys.__stdout__ - out.seek(0) - - pi_condition = Q( - role__name='Principal Investigator', status__name='Active', - enable_notifications=True) - manager_condition = Q(role__name='Manager', status__name='Active') - - recipients = list( - project.projectuser_set.filter( - pi_condition | manager_condition - ).values_list( - 'user__email', flat=True - )) + output, error = self.call_deactivate_command('--send_emails') + + recipients = list(project.managers_and_pis_with_notifications() + .values_list('user__email', flat=True)) # Testing that the correct text is output to stdout message = f'Sent deactivation notification email to ' \ f'{len(recipients)} users.' - self.assertIn(message, out.read()) + + self.assertIn(message, output) + self.assertEqual(error, '') # Testing that the correct number of emails were sent self.assertEqual(len(mail.outbox), len(recipients)) From 99252515abdc0e770f60bde3fccbe931d0fd6057 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Fri, 18 Feb 2022 20:49:41 -0500 Subject: [PATCH 041/150] added helper managers_and_pis_with_notifications to project class --- coldfront/core/project/models.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/coldfront/core/project/models.py b/coldfront/core/project/models.py index 7b3e9a788..a7ec0b3e8 100644 --- a/coldfront/core/project/models.py +++ b/coldfront/core/project/models.py @@ -5,6 +5,7 @@ from django.core.exceptions import ValidationError from django.core.validators import MinLengthValidator from django.db import models +from django.db.models import Q from model_utils.models import TimeStampedModel from simple_history.models import HistoricalRecords @@ -177,6 +178,15 @@ def is_pooled(self): name='Principal Investigator') return self.projectuser_set.filter(role=pi_role).count() > 1 + def managers_and_pis_with_notifications(self): + """Return a queryset of ProjectUsers that are active managers and PIs + with enable_notifications=True.""" + pi_condition = Q( + role__name='Principal Investigator', status__name='Active', + enable_notifications=True) + manager_condition = Q(role__name='Manager', status__name='Active') + return self.projectuser_set.filter(pi_condition | manager_condition) + def __str__(self): return self.name From dcaaea1877cbfa8933d95e18c78182d87c24a1c8 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Fri, 18 Feb 2022 20:54:58 -0500 Subject: [PATCH 042/150] changed helper in project class to return list of emails --- .../management/commands/deactivate_ica_projects.py | 3 +-- coldfront/core/project/models.py | 12 ++++++++---- .../test_commands/test_deactivate_ica_projects.py | 3 +-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/coldfront/core/project/management/commands/deactivate_ica_projects.py b/coldfront/core/project/management/commands/deactivate_ica_projects.py index 8e0387ddf..4c47e326a 100644 --- a/coldfront/core/project/management/commands/deactivate_ica_projects.py +++ b/coldfront/core/project/management/commands/deactivate_ica_projects.py @@ -188,8 +188,7 @@ def send_emails(self, project, expiry_date, dry_run): 'signature': settings.EMAIL_SIGNATURE, } - recipients = list(project.managers_and_pis_with_notifications() - .values_list('user__email', flat=True)) + recipients = project.managers_and_pis_emails() if dry_run: msg_plain = \ diff --git a/coldfront/core/project/models.py b/coldfront/core/project/models.py index a7ec0b3e8..244e06c1b 100644 --- a/coldfront/core/project/models.py +++ b/coldfront/core/project/models.py @@ -178,14 +178,18 @@ def is_pooled(self): name='Principal Investigator') return self.projectuser_set.filter(role=pi_role).count() > 1 - def managers_and_pis_with_notifications(self): - """Return a queryset of ProjectUsers that are active managers and PIs - with enable_notifications=True.""" + def managers_and_pis_emails(self): + """Return a list of emails belonging to active managers and PIs that + have enable_notifications=True.""" pi_condition = Q( role__name='Principal Investigator', status__name='Active', enable_notifications=True) manager_condition = Q(role__name='Manager', status__name='Active') - return self.projectuser_set.filter(pi_condition | manager_condition) + + return list( + self.projectuser_set.filter( + pi_condition | manager_condition + ).distinct().values_list('user__email', flat=True)) def __str__(self): return self.name diff --git a/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py b/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py index c21177866..0343b54f5 100644 --- a/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py +++ b/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py @@ -388,8 +388,7 @@ def test_emails_sent(self): # run command output, error = self.call_deactivate_command('--send_emails') - recipients = list(project.managers_and_pis_with_notifications() - .values_list('user__email', flat=True)) + recipients = project.managers_and_pis_emails() # Testing that the correct text is output to stdout message = f'Sent deactivation notification email to ' \ From 89a3bbc3fd33a8ed73e010d0a160296139b29129 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Fri, 18 Feb 2022 21:07:23 -0500 Subject: [PATCH 043/150] utilized new project method managers_and_pis_emails --- .../commands/pending_join_request_reminder.py | 12 +---- .../test_pending_auto_request_reminder.py | 51 ++----------------- coldfront/core/project/utils.py | 24 +-------- coldfront/core/project/utils_/email_utils.py | 9 +--- .../core/project/utils_/renewal_utils.py | 11 +--- 5 files changed, 10 insertions(+), 97 deletions(-) diff --git a/coldfront/core/project/management/commands/pending_join_request_reminder.py b/coldfront/core/project/management/commands/pending_join_request_reminder.py index be56ae034..0d399f61f 100644 --- a/coldfront/core/project/management/commands/pending_join_request_reminder.py +++ b/coldfront/core/project/management/commands/pending_join_request_reminder.py @@ -60,16 +60,8 @@ def handle(self, *args, **options): 'review_url': review_project_join_requests_url(project), 'signature': settings.EMAIL_SIGNATURE, } - pi_condition = Q( - role__name='Principal Investigator', status__name='Active', - enable_notifications=True) - manager_condition = Q(role__name='Manager', status__name='Active') - recipients = list( - project.projectuser_set.filter( - pi_condition | manager_condition - ).values_list( - 'user__email', flat=True - )) + + recipients = project.managers_and_pis_emails() try: msg_plain = \ render_to_string('email/project_join_request/pending_project_join_requests.txt', diff --git a/coldfront/core/project/tests/test_commands/test_pending_auto_request_reminder.py b/coldfront/core/project/tests/test_commands/test_pending_auto_request_reminder.py index a14e522cb..07d24f8c5 100644 --- a/coldfront/core/project/tests/test_commands/test_pending_auto_request_reminder.py +++ b/coldfront/core/project/tests/test_commands/test_pending_auto_request_reminder.py @@ -149,16 +149,7 @@ def test_command_single_proj_multiple_requests(self): call_command('pending_join_request_reminder', stdout=out, stderr=err) sys.stdout = sys.__stdout__ - pi_condition = Q( - role__name='Principal Investigator', status__name='Active', - enable_notifications=True) - manager_condition = Q(role__name='Manager', status__name='Active') - manager_emails = list( - self.project1.projectuser_set.filter( - pi_condition | manager_condition - ).values_list( - 'user__email', flat=True - )) + manager_emails = self.project1.managers_and_pis_emails() for email in mail.outbox: for addr in email.to: @@ -208,16 +199,7 @@ def test_command_single_proj_multiple_requests_pi_no_notifications(self): call_command('pending_join_request_reminder', stdout=out, stderr=err) sys.stdout = sys.__stdout__ - pi_condition = Q( - role__name='Principal Investigator', status__name='Active', - enable_notifications=True) - manager_condition = Q(role__name='Manager', status__name='Active') - manager_emails = list( - self.project1.projectuser_set.filter( - pi_condition | manager_condition - ).values_list( - 'user__email', flat=True - )) + manager_emails = self.project1.managers_and_pis_emails() for email in mail.outbox: for addr in email.to: @@ -297,23 +279,7 @@ def test_command_two_proj_multiple_requests(self): call_command('pending_join_request_reminder', stdout=out, stderr=err) sys.stdout = sys.__stdout__ - pi_condition = Q( - role__name='Principal Investigator', status__name='Active', - enable_notifications=True) - manager_condition = Q(role__name='Manager', status__name='Active') - manager_emails1 = list( - self.project1.projectuser_set.filter( - pi_condition | manager_condition - ).values_list( - 'user__email', flat=True - )) - - manager_emails2 = list( - project2.projectuser_set.filter( - pi_condition | manager_condition - ).values_list( - 'user__email', flat=True - )) + manager_emails1 = self.project1.managers_and_pis_emails() for email in mail.outbox: for addr in email.to: @@ -381,16 +347,7 @@ def test_command_previously_denied_requests(self): call_command('pending_join_request_reminder', stdout=out, stderr=err) sys.stdout = sys.__stdout__ - pi_condition = Q( - role__name='Principal Investigator', status__name='Active', - enable_notifications=True) - manager_condition = Q(role__name='Manager', status__name='Active') - manager_emails = list( - self.project1.projectuser_set.filter( - pi_condition | manager_condition - ).values_list( - 'user__email', flat=True - )) + manager_emails = self.project1.managers_and_pis_emails() for email in mail.outbox: for addr in email.to: diff --git a/coldfront/core/project/utils.py b/coldfront/core/project/utils.py index 0f9f066ce..e3e3b11f4 100644 --- a/coldfront/core/project/utils.py +++ b/coldfront/core/project/utils.py @@ -128,16 +128,7 @@ def send_project_join_notification_email(project, project_user): 'review_url': review_project_join_requests_url(project), 'url': __project_detail_url(project)} - pi_condition = Q( - role__name='Principal Investigator', status__name='Active', - enable_notifications=True) - manager_condition = Q(role__name='Manager', status__name='Active') - receiver_list = list( - project.projectuser_set.filter( - pi_condition | manager_condition - ).values_list( - 'user__email', flat=True - )) + receiver_list = project.managers_and_pis_emails() msg_plain = \ render_to_string('email/new_project_join_request.txt', @@ -438,18 +429,7 @@ def send_project_request_pooling_email(request): } sender = settings.EMAIL_SENDER - - pi_condition = Q( - role__name='Principal Investigator', status__name='Active', - enable_notifications=True) - manager_condition = Q(role__name='Manager', status__name='Active') - receiver_list = list( - request.project.projectuser_set.filter( - pi_condition | manager_condition - ).values_list( - 'user__email', flat=True - )) - + receiver_list = request.project.managers_and_pis_emails() send_email_template(subject, template_name, context, sender, receiver_list) diff --git a/coldfront/core/project/utils_/email_utils.py b/coldfront/core/project/utils_/email_utils.py index 3924e8de4..52ecf6ac5 100644 --- a/coldfront/core/project/utils_/email_utils.py +++ b/coldfront/core/project/utils_/email_utils.py @@ -22,11 +22,4 @@ def project_email_receiver_list(project): """ if not isinstance(project, Project): raise TypeError(f'{project} is not a Project object.') - pi_condition = Q( - role__name='Principal Investigator', status__name='Active', - enable_notifications=True) - manager_condition = Q(role__name='Manager', status__name='Active') - return list( - project.projectuser_set.filter( - pi_condition | manager_condition - ).distinct().values_list('user__email', flat=True)) + return project.managers_and_pis_emails() diff --git a/coldfront/core/project/utils_/renewal_utils.py b/coldfront/core/project/utils_/renewal_utils.py index c3b73338d..f47542e1c 100644 --- a/coldfront/core/project/utils_/renewal_utils.py +++ b/coldfront/core/project/utils_/renewal_utils.py @@ -334,16 +334,7 @@ def send_new_allocation_renewal_request_pooling_notification_email(request): } sender = settings.EMAIL_SENDER - pi_condition = Q( - role__name='Principal Investigator', status__name='Active', - enable_notifications=True) - manager_condition = Q(role__name='Manager', status__name='Active') - receiver_list = list( - request.post_project.projectuser_set.filter( - pi_condition | manager_condition - ).values_list( - 'user__email', flat=True - )) + receiver_list = request.post_project.managers_and_pis_emails() send_email_template(subject, template_name, context, sender, receiver_list) From 72354d112cdde83b67470003b898e4135d2616b6 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Tue, 22 Feb 2022 23:06:39 -0500 Subject: [PATCH 044/150] added user inline to groups admin --- coldfront/core/utils/admin.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/coldfront/core/utils/admin.py b/coldfront/core/utils/admin.py index 8c38f3f3d..ba4575e5b 100644 --- a/coldfront/core/utils/admin.py +++ b/coldfront/core/utils/admin.py @@ -1,3 +1,19 @@ from django.contrib import admin +from django.contrib.auth.admin import GroupAdmin +from django.contrib.auth.models import Group, User -# Register your models here. + +class UserInLine(admin.TabularInline): + model = User.groups.through + readonly_fields = ('user',) + extra = 0 + can_delete = False + + +class GroupsAdmin(GroupAdmin): + list_display = ["name", "pk"] + inlines = [UserInLine] + + +admin.site.unregister(Group) +admin.site.register(Group, GroupsAdmin) From d55162e5e55c025ad2507facd323aa5d2295c432 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Tue, 22 Feb 2022 23:16:56 -0500 Subject: [PATCH 045/150] added EmailAddress inline to User admin page --- coldfront/core/utils/admin.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/coldfront/core/utils/admin.py b/coldfront/core/utils/admin.py index ba4575e5b..cfe42b080 100644 --- a/coldfront/core/utils/admin.py +++ b/coldfront/core/utils/admin.py @@ -1,7 +1,9 @@ from django.contrib import admin -from django.contrib.auth.admin import GroupAdmin +from django.contrib.auth.admin import GroupAdmin, UserAdmin from django.contrib.auth.models import Group, User +from coldfront.core.user.models import EmailAddress + class UserInLine(admin.TabularInline): model = User.groups.through @@ -15,5 +17,18 @@ class GroupsAdmin(GroupAdmin): inlines = [UserInLine] +class EmailAddressInLine(admin.TabularInline): + model = EmailAddress + extra = 0 + + +class UsersAdmin(UserAdmin): + list_display = ["username", "email", "first_name", "last_name", "is_staff"] + inlines = [EmailAddressInLine] + + admin.site.unregister(Group) admin.site.register(Group, GroupsAdmin) + +admin.site.unregister(User) +admin.site.register(User, UsersAdmin) \ No newline at end of file From 9c7a067873548265e0ec1960e645fcd0db5d5ee9 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Thu, 24 Feb 2022 09:03:12 -0500 Subject: [PATCH 046/150] removed return value for reset_service_units --- .../project/management/commands/deactivate_ica_projects.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/coldfront/core/project/management/commands/deactivate_ica_projects.py b/coldfront/core/project/management/commands/deactivate_ica_projects.py index 4c47e326a..001a1a030 100644 --- a/coldfront/core/project/management/commands/deactivate_ica_projects.py +++ b/coldfront/core/project/management/commands/deactivate_ica_projects.py @@ -155,8 +155,8 @@ def reset_service_units(self, project, dry_run): project_user=project_user, date_time=current_date, allocation=updated_su) - # Set the reason for the change in the newly-created historical object. + # Set the reason for the change in the newly-created historical object. allocation_user_obj = get_accounting_allocation_objects( project, user=user) self.set_historical_reason( @@ -170,8 +170,6 @@ def reset_service_units(self, project, dry_run): self.logger.info(message) self.stdout.write(self.style.SUCCESS(message)) - return current_allocation - def send_emails(self, project, expiry_date, dry_run): """ Send emails to managers/PIs of the project that have notifications From 7d6c9b7a966f14026eeda9ce8042da7fbd786f81 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Thu, 24 Feb 2022 09:03:42 -0500 Subject: [PATCH 047/150] added helper function to set historical reasons --- .../commands/add_service_units_to_project.py | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/coldfront/core/project/management/commands/add_service_units_to_project.py b/coldfront/core/project/management/commands/add_service_units_to_project.py index 5bb33fea8..31b60c5eb 100644 --- a/coldfront/core/project/management/commands/add_service_units_to_project.py +++ b/coldfront/core/project/management/commands/add_service_units_to_project.py @@ -83,6 +83,13 @@ def validate_inputs(self, options): return project, allocation_objects, current_allocation, allocation + def set_historical_reason(self, obj, reason): + """Set the latest historical object reason""" + obj.refresh_from_db() + historical_obj = obj.history.latest('id') + historical_obj.history_change_reason = reason + historical_obj.save() + def handle(self, *args, **options): """ Add SUs to a given project """ project, allocation_objects, current_allocation, allocation = \ @@ -114,11 +121,8 @@ def handle(self, *args, **options): allocation=allocation) # Set the reason for the change in the newly-created historical object. - allocation_objects.allocation_attribute.refresh_from_db() - historical_allocation_attribute = \ - allocation_objects.allocation_attribute.history.latest("id") - historical_allocation_attribute.history_change_reason = reason - historical_allocation_attribute.save() + self.set_historical_reason( + allocation_objects.allocation_attribute, reason) # Do the same for each ProjectUser. allocation_attribute_type = AllocationAttributeType.objects.get( @@ -137,21 +141,17 @@ def handle(self, *args, **options): project_user=project_user, date_time=date_time, allocation=allocation) + # Set the reason for the change in the newly-created historical object. - allocation_user = \ - allocation_objects.allocation.allocationuser_set.get(user=user) - allocation_user_attribute = \ - allocation_user.allocationuserattribute_set.get( - allocation_attribute_type=allocation_attribute_type, - allocation=allocation_objects.allocation) - historical_allocation_user_attribute = \ - allocation_user_attribute.history.latest("id") - historical_allocation_user_attribute.history_change_reason = reason - historical_allocation_user_attribute.save() + allocation_user_obj = get_accounting_allocation_objects( + project, user=user) + self.set_historical_reason( + allocation_user_obj.allocation_user_attribute, reason) message = f'Successfully added {addition} SUs to {project.name} ' \ f'and its users, updating {project.name}\'s SUs from ' \ f'{current_allocation} to {allocation}. The reason ' \ f'was: "{reason}".' + self.logger.info(message) self.stdout.write(self.style.SUCCESS(message)) From de0cf3f0ecd7257ff138cca2871ae550af6cac3b Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Thu, 24 Feb 2022 09:04:08 -0500 Subject: [PATCH 048/150] adding base testing class for commands that alter service unit values --- .../test_commands/test_service_units_base.py | 220 ++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 coldfront/core/project/tests/test_commands/test_service_units_base.py diff --git a/coldfront/core/project/tests/test_commands/test_service_units_base.py b/coldfront/core/project/tests/test_commands/test_service_units_base.py new file mode 100644 index 000000000..5c1fb4394 --- /dev/null +++ b/coldfront/core/project/tests/test_commands/test_service_units_base.py @@ -0,0 +1,220 @@ +from io import StringIO + +from django.contrib.auth.models import User +from django.core import mail +from django.core.management import call_command + +from coldfront.api.statistics.utils import AccountingAllocationObjects +from coldfront.core.allocation.models import Allocation, \ + AllocationAttributeType, AllocationAttribute, \ + AllocationAttributeUsage, AllocationUserStatusChoice, AllocationUser, \ + AllocationUserAttribute, AllocationUserAttributeUsage +from coldfront.core.project.models import Project, \ + ProjectUserStatusChoice, ProjectUser +from coldfront.core.statistics.models import ProjectTransaction, \ + ProjectUserTransaction +from coldfront.core.user.models import UserProfile +from coldfront.core.utils.tests.test_base import TestBase + + +class TestSUBase(TestBase): + """Base class for testing the management commands that alter project SUs""" + + def setUp(self): + """Set up test data.""" + super().setUp() + + # Create a PI. + self.pi = User.objects.create( + username='pi0', email='pi0@nonexistent.com') + user_profile = UserProfile.objects.get(user=self.pi) + user_profile.is_pi = True + user_profile.save() + + # Create two Users. + for i in range(2): + user = User.objects.create( + username=f'user{i}', email=f'user{i}@nonexistent.com') + user_profile = UserProfile.objects.get(user=user) + user_profile.cluster_uid = f'{i}' + user_profile.save() + setattr(self, f'user{i}', user) + setattr(self, f'user_profile{i}', user_profile) + + # Clear the mail outbox. + mail.outbox = [] + + @staticmethod + def call_deactivate_command(*args): + """ + Call the command with the given arguments, returning the messages + written to stdout and stderr. + """ + out, err = StringIO(), StringIO() + kwargs = {'stdout': out, 'stderr': err} + call_command(*args, **kwargs) + return out.getvalue(), err.getvalue() + + def get_accounting_allocation_objects(self, project, user=None): + """Return a namedtuple of database objects related to accounting and + allocation for the given project and optional user. + + Parameters: + - project (Project): an instance of the Project model + - user (User): an instance of the User model + + Returns: + - AccountingAllocationObjects instance + + Raises: + - MultipleObjectsReturned, if a database retrieval returns more + than one object + - ObjectDoesNotExist, if a database retrieval returns less than + one object + - TypeError, if one or more inputs has the wrong type + + NOTE: this function was taken from coldfront.core.statistics.utils. + This version does not check that the allocation status is Active + """ + if not isinstance(project, Project): + raise TypeError(f'Project {project} is not a Project object.') + + objects = AccountingAllocationObjects() + + allocation = Allocation.objects.get( + project=project, resources__name='Savio Compute') + + # Check that the allocation has an attribute for Service Units and + # an associated usage. + allocation_attribute_type = AllocationAttributeType.objects.get( + name='Service Units') + allocation_attribute = AllocationAttribute.objects.get( + allocation_attribute_type=allocation_attribute_type, + allocation=allocation) + allocation_attribute_usage = AllocationAttributeUsage.objects.get( + allocation_attribute=allocation_attribute) + + objects.allocation = allocation + objects.allocation_attribute = allocation_attribute + objects.allocation_attribute_usage = allocation_attribute_usage + + if user is None: + return objects + + if not isinstance(user, User): + raise TypeError(f'User {user} is not a User object.') + + # Check that there is an active association between the user and project. + active_status = ProjectUserStatusChoice.objects.get(name='Active') + ProjectUser.objects.get(user=user, project=project, status=active_status) + + # Check that the user is an active member of the allocation. + active_status = AllocationUserStatusChoice.objects.get(name='Active') + allocation_user = AllocationUser.objects.get( + allocation=allocation, user=user, status=active_status) + + # Check that the allocation user has an attribute for Service Units + # and an associated usage. + allocation_user_attribute = AllocationUserAttribute.objects.get( + allocation_attribute_type=allocation_attribute_type, + allocation=allocation, allocation_user=allocation_user) + allocation_user_attribute_usage = AllocationUserAttributeUsage.objects.get( + allocation_user_attribute=allocation_user_attribute) + + objects.allocation_user = allocation_user + objects.allocation_user_attribute = allocation_user_attribute + objects.allocation_user_attribute_usage = allocation_user_attribute_usage + + return objects + + def record_historical_objects_len(self, project): + """ + Records the lengths of all relevant historical objects to a dict + """ + length_dict = {} + allocation_objects = self.get_accounting_allocation_objects(project) + historical_allocation_attribute = \ + allocation_objects.allocation_attribute.history.all() + + length_dict['historical_allocation_attribute'] = \ + len(historical_allocation_attribute) + + for project_user in project.projectuser_set.all(): + if project_user.role.name != 'User': + continue + + allocation_user_obj = self.get_accounting_allocation_objects( + project, user=project_user.user) + + historical_allocation_user_attribute = \ + allocation_user_obj.allocation_user_attribute.history.all() + + key = 'historical_allocation_user_attribute_' + project_user.user.username + length_dict[key] = len(historical_allocation_user_attribute) + + return length_dict + + def allocation_values_test(self, project, value, user_value): + """ + Tests that the allocation user values are correct + """ + allocation_objects = self.get_accounting_allocation_objects(project) + self.assertEqual(allocation_objects.allocation_attribute.value, value) + + for project_user in project.projectuser_set.all(): + if project_user.role.name != 'User': + continue + allocation_objects = self.get_accounting_allocation_objects( + project, user=project_user.user) + + self.assertEqual(allocation_objects.allocation_user_attribute.value, + user_value) + + def transactions_created(self, project, pre_time, post_time, amount): + """ + Tests that transactions were created for the zeroing of SUs + """ + proj_transaction = ProjectTransaction.objects.get(project=project, + allocation=amount) + + self.assertTrue(pre_time <= proj_transaction.date_time <= post_time) + + for project_user in project.projectuser_set.all(): + if project_user.role.name != 'User': + continue + + proj_user_transaction = ProjectUserTransaction.objects.get( + project_user=project_user, + allocation=amount) + + self.assertTrue(pre_time <= proj_user_transaction.date_time <= post_time) + + def historical_objects_created(self, pre_length_dict, post_length_dict): + """ + Test that historical objects were created + """ + for k, v in pre_length_dict.items(): + self.assertEqual(v + 1, post_length_dict[k]) + + def historical_objects_updated(self, project, reason): + """ + Tests that the relevant historical objects have the correct reason + """ + allocation_objects = self.get_accounting_allocation_objects(project) + + alloc_attr_hist_reason = \ + allocation_objects.allocation_attribute.history. \ + latest('id').history_change_reason + self.assertEqual(alloc_attr_hist_reason, reason) + + for project_user in project.projectuser_set.all(): + if project_user.role.name != 'User': + continue + + allocation_user_obj = self.get_accounting_allocation_objects( + project, user=project_user.user) + + alloc_attr_hist_reason = \ + allocation_user_obj.allocation_user_attribute.history. \ + latest('id').history_change_reason + self.assertEqual(alloc_attr_hist_reason, reason) \ No newline at end of file From e05711c64927f7e2307c8395a470e2e17b92a5a7 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Thu, 24 Feb 2022 09:04:32 -0500 Subject: [PATCH 049/150] now inherits from TestSUBase --- .../test_add_service_units_to_project.py | 324 ++++++------------ 1 file changed, 108 insertions(+), 216 deletions(-) diff --git a/coldfront/core/project/tests/test_commands/test_add_service_units_to_project.py b/coldfront/core/project/tests/test_commands/test_add_service_units_to_project.py index 3fc42d633..2703c4bb2 100644 --- a/coldfront/core/project/tests/test_commands/test_add_service_units_to_project.py +++ b/coldfront/core/project/tests/test_commands/test_add_service_units_to_project.py @@ -1,129 +1,63 @@ -from io import StringIO -import sys +from decimal import Decimal -from django.core.management import call_command, CommandError +from django.core.management import CommandError -from coldfront.api.allocation.tests.test_allocation_base import TestAllocationBase -from coldfront.api.statistics.utils import get_accounting_allocation_objects -from coldfront.core.allocation.models import Allocation, AllocationAttributeType -from coldfront.core.project.models import Project +from coldfront.api.statistics.utils import create_project_allocation, \ + create_user_project_allocation +from coldfront.core.allocation.models import Allocation +from coldfront.core.project.models import Project, ProjectStatusChoice, \ + ProjectUserStatusChoice, ProjectUserRoleChoice, ProjectUser +from coldfront.core.project.tests.test_commands.test_service_units_base import \ + TestSUBase from coldfront.core.resource.models import Resource -from coldfront.core.statistics.models import ProjectTransaction, ProjectUserTransaction from coldfront.core.utils.common import utc_now_offset_aware -class TestAddServiceUnitsToProject(TestAllocationBase): +class TestAddServiceUnitsToProject(TestSUBase): """Class for testing the management command add_service_units_to_project""" def setUp(self): """Set up test data.""" super().setUp() - def record_historical_objects_len(self, project): - """ Records the lengths of all relevant historical objects to a dict""" - length_dict = {} - allocation_objects = get_accounting_allocation_objects(project) - historical_allocation_attribute = \ - allocation_objects.allocation_attribute.history.all() - - length_dict['historical_allocation_attribute'] = \ - len(historical_allocation_attribute) - - allocation_attribute_type = AllocationAttributeType.objects.get( - name="Service Units") - for project_user in project.projectuser_set.all(): - if project_user.role.name != 'User': - continue - - allocation_user = \ - allocation_objects.allocation.allocationuser_set.get( - user=project_user.user) - allocation_user_attribute = \ - allocation_user.allocationuserattribute_set.get( - allocation_attribute_type=allocation_attribute_type, - allocation=allocation_objects.allocation) - historical_allocation_user_attribute = \ - allocation_user_attribute.history.all() - - key = 'historical_allocation_user_attribute_' + project_user.user.username - length_dict[key] = len(historical_allocation_user_attribute) - - return length_dict - - def allocation_values_test(self, project, value, user_value): - allocation_objects = get_accounting_allocation_objects(project) - self.assertEqual(allocation_objects.allocation_attribute.value, value) - - for project_user in project.projectuser_set.all(): - if project_user.role.name != 'User': - continue - allocation_objects = get_accounting_allocation_objects( - project, user=project_user.user) - - self.assertEqual(allocation_objects.allocation_user_attribute.value, - user_value) - - def transactions_created(self, project, pre_time, post_time, amount): - proj_transaction = ProjectTransaction.objects.get(project=project, - allocation=amount) - - self.assertTrue(pre_time <= proj_transaction.date_time <= post_time) - - for project_user in project.projectuser_set.all(): - if project_user.role.name != 'User': - continue - - proj_user_transaction = ProjectUserTransaction.objects.get( - project_user=project_user, - allocation=amount) - - self.assertTrue(pre_time <= proj_user_transaction.date_time <= post_time) - - def historical_objects_created(self, pre_length_dict, post_length_dict): - """Test that historical objects were created""" - for k, v in pre_length_dict.items(): - self.assertEqual(v + 1, post_length_dict[k]) - - def historical_objects_updated(self, project): - allocation_objects = get_accounting_allocation_objects(project) - historical_allocation_attribute = \ - allocation_objects.allocation_attribute.history.latest("id") - historical_reason = historical_allocation_attribute.history_change_reason - - self.assertEqual(historical_reason, 'This is a test for add_service_units command') - - allocation_attribute_type = AllocationAttributeType.objects.get( - name="Service Units") - for project_user in project.projectuser_set.all(): - if project_user.role.name != 'User': - continue - - allocation_user = \ - allocation_objects.allocation.allocationuser_set.get( - user=project_user.user) - allocation_user_attribute = \ - allocation_user.allocationuserattribute_set.get( - allocation_attribute_type=allocation_attribute_type, - allocation=allocation_objects.allocation) - historical_allocation_user_attribute = \ - allocation_user_attribute.history.latest("id") - historical_reason = \ - historical_allocation_user_attribute.history_change_reason - self.assertEqual(historical_reason, - 'This is a test for add_service_units command') + self.reason = 'This is a test for add_service_units command' + + # Create Projects and associate Users with them. + project_status = ProjectStatusChoice.objects.get(name='Active') + project_user_status = ProjectUserStatusChoice.objects.get( + name='Active') + user_role = ProjectUserRoleChoice.objects.get(name='User') + manager_role = ProjectUserRoleChoice.objects.get(name='Manager') + + for i in range(2): + # Create an ICA Project and ProjectUsers. + project = Project.objects.create( + name=f'project{i}', status=project_status) + setattr(self, f'project{i}', project) + for j in range(2): + ProjectUser.objects.create( + user=getattr(self, f'user{j}'), project=project, + role=user_role, status=project_user_status) + ProjectUser.objects.create( + user=self.pi, project=project, role=manager_role, + status=project_user_status) + + # Create a compute allocation for the Project. + allocation = Decimal(f'{i + 1}000.00') + create_project_allocation(project, allocation) + + # Create a compute allocation for each User on the Project. + for j in range(2): + create_user_project_allocation( + getattr(self, f'user{j}'), project, allocation / 2) def test_dry_run(self): """Testing add_service_units_to_project dry run""" - out, err = StringIO(''), StringIO('') - call_command('add_service_units_to_project', - '--project_name=project0', - '--amount=1000', - '--reason=This is a test for add_service_units command', - '--dry_run', - stdout=out, - stderr=err) - sys.stdout = sys.__stdout__ - out.seek(0) + output, error = self.call_deactivate_command('add_service_units_to_project', + '--project_name=project0', + '--amount=1000', + f'--reason={self.reason}', + '--dry_run') dry_run_message = 'Would add 1000 additional SUs to project ' \ 'project0. This would increase project0 ' \ @@ -132,13 +66,11 @@ def test_dry_run(self): 'would be: "This is a test for ' \ 'add_service_units command".' - self.assertIn(dry_run_message, out.read()) - err.seek(0) - self.assertEqual(err.read(), '') + self.assertIn(dry_run_message, output) + self.assertEqual(error, '') def test_creates_and_updates_objects_positive_SU(self): """Testing add_service_units_to_project with positive SUs""" - # test allocation values before command project = Project.objects.get(name='project0') pre_time = utc_now_offset_aware() @@ -147,23 +79,17 @@ def test_creates_and_updates_objects_positive_SU(self): self.allocation_values_test(project, '1000.00', '500.00') # run command - out, err = StringIO(''), StringIO('') - call_command('add_service_units_to_project', - '--project_name=project0', - '--amount=1000', - '--reason=This is a test for add_service_units command', - stdout=out, - stderr=err) - sys.stdout = sys.__stdout__ - out.seek(0) + output, error = self.call_deactivate_command('add_service_units_to_project', + '--project_name=project0', + '--amount=1000', + f'--reason={self.reason}') message = f'Successfully added 1000 SUs to project0 and its users, ' \ f'updating project0\'s SUs from 1000.00 to 2000.00. The ' \ f'reason was: "This is a test for add_service_units command".' - self.assertIn(message, out.read()) - err.seek(0) - self.assertEqual(err.read(), '') + self.assertIn(message, output) + self.assertEqual(error, '') post_time = utc_now_offset_aware() @@ -176,11 +102,10 @@ def test_creates_and_updates_objects_positive_SU(self): # test historical objects created and updated post_length_dict = self.record_historical_objects_len(project) self.historical_objects_created(pre_length_dict, post_length_dict) - self.historical_objects_updated(project) + self.historical_objects_updated(project, self.reason) def test_creates_and_updates_objects_negative_SU(self): """Testing add_service_units_to_project with negative SUs""" - # test allocation values before command project = Project.objects.get(name='project0') pre_time = utc_now_offset_aware() @@ -189,23 +114,17 @@ def test_creates_and_updates_objects_negative_SU(self): self.allocation_values_test(project, '1000.00', '500.00') # run command - out, err = StringIO(''), StringIO('') - call_command('add_service_units_to_project', - '--project_name=project0', - '--amount=-800', - '--reason=This is a test for add_service_units command', - stdout=out, - stderr=err) - sys.stdout = sys.__stdout__ - out.seek(0) + output, error = self.call_deactivate_command('add_service_units_to_project', + '--project_name=project0', + '--amount=-800', + f'--reason={self.reason}') message = f'Successfully added -800 SUs to project0 and its users, ' \ f'updating project0\'s SUs from 1000.00 to 200.00. The ' \ f'reason was: "This is a test for add_service_units command".' - self.assertIn(message, out.read()) - err.seek(0) - self.assertEqual(err.read(), '') + self.assertIn(message, output) + self.assertEqual(error, '') post_time = utc_now_offset_aware() @@ -218,9 +137,12 @@ def test_creates_and_updates_objects_negative_SU(self): # test historical objects created and updated post_length_dict = self.record_historical_objects_len(project) self.historical_objects_created(pre_length_dict, post_length_dict) - self.historical_objects_updated(project) + self.historical_objects_updated(project, self.reason) def test_input_validations(self): + """ + Tests that validate_inputs throws errors in the correct situations + """ project = Project.objects.get(name='project1') allocation = Allocation.objects.get(project=project) @@ -240,92 +162,62 @@ def test_input_validations(self): # command should throw a CommandError because the allocation is not # part of Savio Compute with self.assertRaises(CommandError): - out, err = StringIO(''), StringIO('') - call_command('add_service_units_to_project', - '--project_name=project1', - '--amount=1000', - '--reason=This is a test for add_service_units command', - stdout=out, - stderr=err) - sys.stdout = sys.__stdout__ - out.seek(0) - self.assertEqual(out.read(), '') - err.seek(0) - self.assertEqual(err.read(), '') + output, error = \ + self.call_deactivate_command('add_service_units_to_project', + '--project_name=project1', + '--amount=1000', + f'--reason={self.reason}') + self.assertEqual(output, '') + self.assertEqual(error, '') # testing a project that does not exist with self.assertRaises(CommandError): - out, err = StringIO(''), StringIO('') - call_command('add_service_units_to_project', - '--project_name=project555', - '--amount=1000', - '--reason=This is a test for add_service_units command', - stdout=out, - stderr=err) - sys.stdout = sys.__stdout__ - out.seek(0) - self.assertEqual(out.read(), '') - err.seek(0) - self.assertEqual(err.read(), '') + output, error = \ + self.call_deactivate_command('add_service_units_to_project', + '--project_name=project555', + '--amount=1000', + f'--reason={self.reason}') + self.assertEqual(output, '') + self.assertEqual(error, '') # adding service units that results in allocation having less # than settings.ALLOCATION_MIN with self.assertRaises(CommandError): - out, err = StringIO(''), StringIO('') - call_command('add_service_units_to_project', - '--project_name=project0', - '--amount=-100000', - '--reason=This is a test for add_service_units command', - stdout=out, - stderr=err) - sys.stdout = sys.__stdout__ - out.seek(0) - self.assertEqual(out.read(), '') - err.seek(0) - self.assertEqual(err.read(), '') + output, error = \ + self.call_deactivate_command('add_service_units_to_project', + '--project_name=project0', + '--amount=-100000', + f'--reason={self.reason}') + self.assertEqual(output, '') + self.assertEqual(error, '') # adding service units that results in allocation having more # than settings.ALLOCATION_MAX with self.assertRaises(CommandError): - out, err = StringIO(''), StringIO('') - call_command('add_service_units_to_project', - '--project_name=project0', - '--amount=99999500', - '--reason=This is a test for add_service_units command', - stdout=out, - stderr=err) - sys.stdout = sys.__stdout__ - out.seek(0) - self.assertEqual(out.read(), '') - err.seek(0) - self.assertEqual(err.read(), '') - - # adding service units that are greater than settings.ALLOCATION_MIN + output, error = \ + self.call_deactivate_command('add_service_units_to_project', + '--project_name=project0', + '--amount=99999500', + f'--reason={self.reason}') + self.assertEqual(output, '') + self.assertEqual(error, '') + + # adding service units that are greater than settings.ALLOCATION_MAX with self.assertRaises(CommandError): - out, err = StringIO(''), StringIO('') - call_command('add_service_units_to_project', - '--project_name=project0', - '--amount=500000000', - '--reason=This is a test for add_service_units command', - stdout=out, - stderr=err) - sys.stdout = sys.__stdout__ - out.seek(0) - self.assertEqual(out.read(), '') - err.seek(0) - self.assertEqual(err.read(), '') + output, error = \ + self.call_deactivate_command('add_service_units_to_project', + '--project_name=project0', + '--amount=500000000', + f'--reason={self.reason}') + self.assertEqual(output, '') + self.assertEqual(error, '') # reason is not long enough with self.assertRaises(CommandError): - out, err = StringIO(''), StringIO('') - call_command('add_service_units_to_project', - '--project_name=project0', - '--amount=1000', - '--reason=notlong', - stdout=out, - stderr=err) - sys.stdout = sys.__stdout__ - out.seek(0) - self.assertEqual(out.read(), '') - err.seek(0) - self.assertEqual(err.read(), '') + output, error = \ + self.call_deactivate_command('add_service_units_to_project', + '--project_name=project0', + '--amount=1000', + '--reason=notlong') + self.assertEqual(output, '') + self.assertEqual(error, '') From fecc3388bb11e6342905e2e37607988e6921c598 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Thu, 24 Feb 2022 09:04:56 -0500 Subject: [PATCH 050/150] now inherits from TestSUBase. Added test for allocation and user usages --- .../test_deactivate_ica_projects.py | 247 +++--------------- 1 file changed, 35 insertions(+), 212 deletions(-) diff --git a/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py b/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py index 0343b54f5..acdd2f931 100644 --- a/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py +++ b/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py @@ -1,54 +1,27 @@ import datetime from decimal import Decimal -from io import StringIO -import sys -from django.contrib.auth.models import User from django.core import mail -from django.core.management import call_command -from django.db.models import Q from coldfront.api.statistics.utils import create_project_allocation, \ - create_user_project_allocation, AccountingAllocationObjects, \ - get_accounting_allocation_objects + create_user_project_allocation from coldfront.config import settings -from coldfront.core.allocation.models import Allocation, \ - AllocationAttributeType, AllocationAttribute, \ - AllocationAttributeUsage, AllocationUserStatusChoice, AllocationUser, \ - AllocationUserAttribute, AllocationUserAttributeUsage +from coldfront.core.allocation.models import AllocationAttributeUsage, \ + AllocationUserAttributeUsage from coldfront.core.project.models import Project, ProjectStatusChoice, \ ProjectUserStatusChoice, ProjectUserRoleChoice, ProjectUser from coldfront.core.project.utils import get_project_compute_allocation -from coldfront.core.statistics.models import ProjectTransaction, ProjectUserTransaction -from coldfront.core.user.models import UserProfile from coldfront.core.utils.common import utc_now_offset_aware -from coldfront.core.utils.tests.test_base import TestBase +from coldfront.core.project.tests.test_commands.test_service_units_base import TestSUBase -class TestDeactivateICAProjects(TestBase): +class TestDeactivateICAProjects(TestSUBase): """Class for testing the management command deactivate_ica_projects""" def setUp(self): """Set up test data.""" super().setUp() - # Create a PI. - self.pi = User.objects.create( - username='pi0', email='pi0@nonexistent.com') - user_profile = UserProfile.objects.get(user=self.pi) - user_profile.is_pi = True - user_profile.save() - - # Create two Users. - for i in range(2): - user = User.objects.create( - username=f'user{i}', email=f'user{i}@nonexistent.com') - user_profile = UserProfile.objects.get(user=user) - user_profile.cluster_uid = f'{i}' - user_profile.save() - setattr(self, f'user{i}', user) - setattr(self, f'user_profile{i}', user_profile) - # Create Projects and associate Users with them. project_status = ProjectStatusChoice.objects.get(name='Active') project_user_status = ProjectUserStatusChoice.objects.get( @@ -87,91 +60,6 @@ def setUp(self): create_user_project_allocation( getattr(self, f'user{j}'), project, allocation / 2) - # Clear the mail outbox. - mail.outbox = [] - - @staticmethod - def call_deactivate_command(*args): - """Call the command with the given arguments, returning the messages - written to stdout and stderr.""" - out, err = StringIO(), StringIO() - args = ['deactivate_ica_projects', *args] - kwargs = {'stdout': out, 'stderr': err} - call_command(*args, **kwargs) - return out.getvalue(), err.getvalue() - - def get_accounting_allocation_objects(self, project, user=None): - """Return a namedtuple of database objects related to accounting and - allocation for the given project and optional user. - - Parameters: - - project (Project): an instance of the Project model - - user (User): an instance of the User model - - Returns: - - AccountingAllocationObjects instance - - Raises: - - MultipleObjectsReturned, if a database retrieval returns more - than one object - - ObjectDoesNotExist, if a database retrieval returns less than - one object - - TypeError, if one or more inputs has the wrong type - - NOTE: this function was taken from coldfront.core.statistics.utils. - This version does not check that the allocation status is Active - """ - if not isinstance(project, Project): - raise TypeError(f'Project {project} is not a Project object.') - - objects = AccountingAllocationObjects() - - allocation = Allocation.objects.get( - project=project, resources__name='Savio Compute') - - # Check that the allocation has an attribute for Service Units and - # an associated usage. - allocation_attribute_type = AllocationAttributeType.objects.get( - name='Service Units') - allocation_attribute = AllocationAttribute.objects.get( - allocation_attribute_type=allocation_attribute_type, - allocation=allocation) - allocation_attribute_usage = AllocationAttributeUsage.objects.get( - allocation_attribute=allocation_attribute) - - objects.allocation = allocation - objects.allocation_attribute = allocation_attribute - objects.allocation_attribute_usage = allocation_attribute_usage - - if user is None: - return objects - - if not isinstance(user, User): - raise TypeError(f'User {user} is not a User object.') - - # Check that there is an active association between the user and project. - active_status = ProjectUserStatusChoice.objects.get(name='Active') - ProjectUser.objects.get(user=user, project=project, status=active_status) - - # Check that the user is an active member of the allocation. - active_status = AllocationUserStatusChoice.objects.get(name='Active') - allocation_user = AllocationUser.objects.get( - allocation=allocation, user=user, status=active_status) - - # Check that the allocation user has an attribute for Service Units - # and an associated usage. - allocation_user_attribute = AllocationUserAttribute.objects.get( - allocation_attribute_type=allocation_attribute_type, - allocation=allocation, allocation_user=allocation_user) - allocation_user_attribute_usage = AllocationUserAttributeUsage.objects.get( - allocation_user_attribute=allocation_user_attribute) - - objects.allocation_user = allocation_user - objects.allocation_user_attribute = allocation_user_attribute - objects.allocation_user_attribute_usage = allocation_user_attribute_usage - - return objects - def create_expired_project(self, project_name): """Change the end date of ic_project0 to be expired""" project = Project.objects.get(name=project_name) @@ -184,108 +72,37 @@ def create_expired_project(self, project_name): allocation.refresh_from_db() self.assertEqual(allocation.end_date, expired_date) - def record_historical_objects_len(self, project): - """ Records the lengths of all relevant historical objects to a dict""" - length_dict = {} - allocation_objects = self.get_accounting_allocation_objects(project) - historical_allocation_attribute = \ - allocation_objects.allocation_attribute.history.all() - - length_dict['historical_allocation_attribute'] = \ - len(historical_allocation_attribute) - - allocation_attribute_type = AllocationAttributeType.objects.get( - name="Service Units") - for project_user in project.projectuser_set.all(): - if project_user.role.name != 'User': - continue - - allocation_user_obj = self.get_accounting_allocation_objects( - project, user=project_user.user) - - historical_allocation_user_attribute = \ - allocation_user_obj.allocation_user_attribute.history.all() - - key = 'historical_allocation_user_attribute_' + project_user.user.username - length_dict[key] = len(historical_allocation_user_attribute) - - return length_dict + def project_allocation_updates(self, project, allocation, pre_time, post_time): + """Tests that the project and allocation were correctly updated""" + self.assertEqual(project.status.name, 'Inactive') + self.assertEqual(allocation.status.name, 'Expired') + self.assertTrue(pre_time.date() <= allocation.start_date <= post_time.date()) + self.assertTrue(allocation.end_date is None) - def allocation_values_test(self, project, value, user_value): - """ - Tests that the allocation user values are correct - """ + def usage_values_updated(self, project, updated_value): + """Tests that allocation and allocation user usages are reset""" + updated_value = Decimal(updated_value) allocation_objects = self.get_accounting_allocation_objects(project) - self.assertEqual(allocation_objects.allocation_attribute.value, value) + project_usage = \ + AllocationAttributeUsage.objects.get( + pk=allocation_objects.allocation_attribute_usage.pk) + self.assertEqual(project_usage.value, updated_value) for project_user in project.projectuser_set.all(): if project_user.role.name != 'User': continue allocation_objects = self.get_accounting_allocation_objects( project, user=project_user.user) - - self.assertEqual(allocation_objects.allocation_user_attribute.value, - user_value) - - def transactions_created(self, project, pre_time, post_time, amount): - """ - Tests that transactions were created for the zeroing of SUs - """ - proj_transaction = ProjectTransaction.objects.get(project=project, - allocation=amount) - - self.assertTrue(pre_time <= proj_transaction.date_time <= post_time) - - for project_user in project.projectuser_set.all(): - if project_user.role.name != 'User': - continue - - proj_user_transaction = ProjectUserTransaction.objects.get( - project_user=project_user, - allocation=amount) - - self.assertTrue(pre_time <= proj_user_transaction.date_time <= post_time) - - def historical_objects_created(self, pre_length_dict, post_length_dict): - """Test that historical objects were created""" - for k, v in pre_length_dict.items(): - self.assertEqual(v + 1, post_length_dict[k]) - - def historical_objects_updated(self, project): - """ - Tests that the relevant historical objects have the correct reason - """ - - reason = 'Resetting SUs while deactivating expired ICA project.' - allocation_objects = self.get_accounting_allocation_objects(project) - - alloc_attr_hist_reason = \ - allocation_objects.allocation_attribute.history.\ - latest('id').history_change_reason - self.assertEqual(alloc_attr_hist_reason, reason) - - for project_user in project.projectuser_set.all(): - if project_user.role.name != 'User': - continue - - allocation_user_obj = self.get_accounting_allocation_objects( - project, user=project_user.user) - - alloc_attr_hist_reason = \ - allocation_user_obj.allocation_user_attribute.history. \ - latest('id').history_change_reason - self.assertEqual(alloc_attr_hist_reason, reason) - - def project_allocation_updates(self, project, allocation, pre_time, post_time): - """Tests that the project and allocation were correctly updated""" - self.assertEqual(project.status.name, 'Inactive') - self.assertEqual(allocation.status.name, 'Expired') - self.assertTrue(pre_time.date() <= allocation.start_date <= post_time.date()) - self.assertTrue(allocation.end_date is None) + user_project_usage = \ + AllocationUserAttributeUsage.objects.get( + pk=allocation_objects.allocation_user_attribute_usage.pk) + self.assertEqual(user_project_usage.value, + updated_value) def test_dry_run_no_expired_projects(self): """Testing a dry run in which no ICA projects are expired""" - output, error = self.call_deactivate_command('--dry_run', + output, error = self.call_deactivate_command('deactivate_ica_projects', + '--dry_run', '--send_emails') self.assertIn(output, '') self.assertEqual(error, '') @@ -294,7 +111,8 @@ def test_dry_run_with_expired_projects(self): """Testing a dry run in which an ICA project is expired""" self.create_expired_project('ic_project0') - output, error = self.call_deactivate_command('--dry_run', + output, error = self.call_deactivate_command('deactivate_ica_projects', + '--dry_run', '--send_emails') project = Project.objects.get(name='ic_project0') @@ -338,7 +156,7 @@ def test_creates_and_updates_objects(self): self.allocation_values_test(project, '1000.00', '500.00') # run command - output, error = self.call_deactivate_command() + output, error = self.call_deactivate_command('deactivate_ica_projects') messages = [ f'Updated Project {project.name} ({project.pk})\'s status to ' @@ -362,6 +180,9 @@ def test_creates_and_updates_objects(self): # test project and allocation statuses self.project_allocation_updates(project, allocation, pre_time, post_time) + # test usages are updated + self.usage_values_updated(project, '0.00') + # test allocation values after command self.allocation_values_test(project, '0.00', '0.00') @@ -369,9 +190,10 @@ def test_creates_and_updates_objects(self): self.transactions_created(project, pre_time, post_time, 0.00) # test historical objects created and updated + reason = 'Resetting SUs while deactivating expired ICA project.' post_length_dict = self.record_historical_objects_len(project) self.historical_objects_created(pre_length_dict, post_length_dict) - self.historical_objects_updated(project) + self.historical_objects_updated(project, reason) def test_emails_sent(self): """ @@ -386,7 +208,8 @@ def test_emails_sent(self): old_end_date = allocation.end_date # run command - output, error = self.call_deactivate_command('--send_emails') + output, error = self.call_deactivate_command('deactivate_ica_projects', + '--send_emails') recipients = project.managers_and_pis_emails() From 88083eab5162831a6418f813d0b33febff3e0d38 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Thu, 24 Feb 2022 09:22:55 -0500 Subject: [PATCH 051/150] moved logic about setting SUs to helper --- .../commands/add_service_units_to_project.py | 43 +++------------ .../commands/deactivate_ica_projects.py | 53 +++---------------- 2 files changed, 12 insertions(+), 84 deletions(-) diff --git a/coldfront/core/project/management/commands/add_service_units_to_project.py b/coldfront/core/project/management/commands/add_service_units_to_project.py index 31b60c5eb..6f48ed758 100644 --- a/coldfront/core/project/management/commands/add_service_units_to_project.py +++ b/coldfront/core/project/management/commands/add_service_units_to_project.py @@ -5,6 +5,7 @@ from coldfront.config import settings from coldfront.core.project.models import Project +from coldfront.core.project.utils_.addition_utils import set_service_units from coldfront.core.statistics.models import ProjectTransaction from coldfront.core.statistics.models import ProjectUserTransaction from coldfront.core.utils.common import utc_now_offset_aware @@ -98,7 +99,6 @@ def handle(self, *args, **options): addition = Decimal(options.get('amount')) reason = options.get('reason') dry_run = options.get('dry_run', None) - date_time = utc_now_offset_aware() if dry_run: verb = 'increase' if addition > 0 else 'decrease' @@ -111,42 +111,11 @@ def handle(self, *args, **options): self.stdout.write(self.style.WARNING(message)) else: - # Set the value for the Project. - set_project_allocation_value(project, allocation) - - # Create a transaction to record the change. - ProjectTransaction.objects.create( - project=project, - date_time=date_time, - allocation=allocation) - - # Set the reason for the change in the newly-created historical object. - self.set_historical_reason( - allocation_objects.allocation_attribute, reason) - - # Do the same for each ProjectUser. - allocation_attribute_type = AllocationAttributeType.objects.get( - name="Service Units") - for project_user in project.projectuser_set.all(): - user = project_user.user - # Attempt to set the value for the ProjectUser. The method returns whether - # it succeeded; it may not because not every ProjectUser has a - # corresponding AllocationUser (e.g., PIs). Only proceed with further steps - # if an update occurred. - allocation_updated = set_project_user_allocation_value( - user, project, allocation) - if allocation_updated: - # Create a transaction to record the change. - ProjectUserTransaction.objects.create( - project_user=project_user, - date_time=date_time, - allocation=allocation) - - # Set the reason for the change in the newly-created historical object. - allocation_user_obj = get_accounting_allocation_objects( - project, user=user) - self.set_historical_reason( - allocation_user_obj.allocation_user_attribute, reason) + set_service_units(project, + allocation_objects, + allocation, + reason, + False) message = f'Successfully added {addition} SUs to {project.name} ' \ f'and its users, updating {project.name}\'s SUs from ' \ diff --git a/coldfront/core/project/management/commands/deactivate_ica_projects.py b/coldfront/core/project/management/commands/deactivate_ica_projects.py index 001a1a030..aca67e9d0 100644 --- a/coldfront/core/project/management/commands/deactivate_ica_projects.py +++ b/coldfront/core/project/management/commands/deactivate_ica_projects.py @@ -13,6 +13,7 @@ from django.core.management.base import BaseCommand import logging +from coldfront.core.project.utils_.addition_utils import set_service_units from coldfront.core.statistics.models import ProjectTransaction, \ ProjectUserTransaction from coldfront.core.utils.common import utc_now_offset_aware @@ -92,13 +93,6 @@ def deactivate_project(self, project, allocation, dry_run): self.logger.info(message) self.stdout.write(self.style.SUCCESS(message)) - def set_historical_reason(self, obj, reason): - """Set the latest historical object reason""" - obj.refresh_from_db() - historical_obj = obj.history.latest('id') - historical_obj.history_change_reason = reason - historical_obj.save() - def reset_service_units(self, project, dry_run): """ Resets service units for a project and its users to 0.00. Creates @@ -109,7 +103,6 @@ def reset_service_units(self, project, dry_run): """ allocation_objects = get_accounting_allocation_objects(project) current_allocation = Decimal(allocation_objects.allocation_attribute.value) - current_date = utc_now_offset_aware() reason = 'Resetting SUs while deactivating expired ICA project.' updated_su = Decimal('0.00') @@ -122,45 +115,11 @@ def reset_service_units(self, project, dry_run): self.stdout.write(self.style.WARNING(message)) else: - # Set the value for the Project. - set_project_allocation_value(project, updated_su) - set_project_usage_value(project, updated_su) - - # Create a transaction to record the change. - ProjectTransaction.objects.create( - project=project, - date_time=current_date, - allocation=updated_su) - - # Set the reason for the change in the newly-created historical object. - self.set_historical_reason( - allocation_objects.allocation_attribute, reason) - - # Do the same for each ProjectUser. - for project_user in project.projectuser_set.all(): - user = project_user.user - # Attempt to set the value for the ProjectUser. The method returns whether - # it succeeded; it may not because not every ProjectUser has a - # corresponding AllocationUser (e.g., PIs). Only proceed with further steps - # if an update occurred. - - allocation_updated = set_project_user_allocation_value( - user, project, updated_su) - allocation_usage_updated = set_project_user_usage_value( - user, project, updated_su) - - if allocation_updated and allocation_usage_updated: - # Create a transaction to record the change. - ProjectUserTransaction.objects.create( - project_user=project_user, - date_time=current_date, - allocation=updated_su) - - # Set the reason for the change in the newly-created historical object. - allocation_user_obj = get_accounting_allocation_objects( - project, user=user) - self.set_historical_reason( - allocation_user_obj.allocation_user_attribute, reason) + set_service_units(project, + allocation_objects, + updated_su, + reason, + True) message = f'Successfully reset SUs for {project.name} ' \ f'and its users, updating {project.name}\'s SUs from ' \ From 8ce8b9a25256a9b1784a209b69ed2a5cc9e45869 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Thu, 24 Feb 2022 09:23:29 -0500 Subject: [PATCH 052/150] added helper to perform all necessary actions when setting SUs --- .../core/project/utils_/addition_utils.py | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/coldfront/core/project/utils_/addition_utils.py b/coldfront/core/project/utils_/addition_utils.py index 8d2a3fa9e..6f742f1cb 100644 --- a/coldfront/core/project/utils_/addition_utils.py +++ b/coldfront/core/project/utils_/addition_utils.py @@ -223,3 +223,69 @@ def has_pending_allocation_addition_request(project): name='Under Review') return AllocationAdditionRequest.objects.filter( project=project, status=under_review_status).exists() + + +def set_service_units(project, allocation_objects, updated_su, reason, update_usage): + """ + Sets allocation and allocation_user service units to updated_su. Creates + the relevant transaction objects to record the change. Updates the + relevant historical objects with the reason for the SU change. If + update_usage is True, allocation and allocation_user usage values are + updated. + """ + + def set_historical_reason(obj): + """Set the latest historical object reason""" + obj.refresh_from_db() + historical_obj = obj.history.latest('id') + historical_obj.history_change_reason = reason + historical_obj.save() + + current_date = utc_now_offset_aware() + + # Set the value for the Project. + set_project_allocation_value(project, updated_su) + + if update_usage: + set_project_usage_value(project, updated_su) + + # Create a transaction to record the change. + ProjectTransaction.objects.create( + project=project, + date_time=current_date, + allocation=updated_su) + + # Set the reason for the change in the newly-created historical object. + set_historical_reason( + allocation_objects.allocation_attribute) + + # Do the same for each ProjectUser. + for project_user in project.projectuser_set.all(): + user = project_user.user + # Attempt to set the value for the ProjectUser. The method returns whether + # it succeeded; it may not because not every ProjectUser has a + # corresponding AllocationUser (e.g., PIs). Only proceed with further steps + # if an update occurred. + + allocation_updated = set_project_user_allocation_value( + user, project, updated_su) + success_flag = allocation_updated + + if update_usage: + allocation_usage_updated = set_project_user_usage_value( + user, project, updated_su) + + success_flag = allocation_updated and allocation_usage_updated + + if success_flag: + # Create a transaction to record the change. + ProjectUserTransaction.objects.create( + project_user=project_user, + date_time=current_date, + allocation=updated_su) + + # Set the reason for the change in the newly-created historical object. + allocation_user_obj = get_accounting_allocation_objects( + project, user=user) + set_historical_reason( + allocation_user_obj.allocation_user_attribute) \ No newline at end of file From f33d9d9f9194e97e1868622cdca259f1a0d1cd1a Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Thu, 24 Feb 2022 10:43:13 -0500 Subject: [PATCH 053/150] moved EmailAddressInLine to user/admin.py --- coldfront/core/user/admin.py | 5 +++++ coldfront/core/utils/admin.py | 9 ++------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/coldfront/core/user/admin.py b/coldfront/core/user/admin.py index 52567e0d1..be0ae38a6 100644 --- a/coldfront/core/user/admin.py +++ b/coldfront/core/user/admin.py @@ -99,3 +99,8 @@ def delete_queryset(self, request, queryset): error_message = ( f'Skipped deleting {num_primary} primary EmailAddresses.') messages.error(request, error_message) + + +class EmailAddressInline(admin.TabularInline): + model = EmailAddress + extra = 0 diff --git a/coldfront/core/utils/admin.py b/coldfront/core/utils/admin.py index cfe42b080..2f9840a80 100644 --- a/coldfront/core/utils/admin.py +++ b/coldfront/core/utils/admin.py @@ -2,7 +2,7 @@ from django.contrib.auth.admin import GroupAdmin, UserAdmin from django.contrib.auth.models import Group, User -from coldfront.core.user.models import EmailAddress +from coldfront.core.user.admin import EmailAddressInline class UserInLine(admin.TabularInline): @@ -17,14 +17,9 @@ class GroupsAdmin(GroupAdmin): inlines = [UserInLine] -class EmailAddressInLine(admin.TabularInline): - model = EmailAddress - extra = 0 - - class UsersAdmin(UserAdmin): list_display = ["username", "email", "first_name", "last_name", "is_staff"] - inlines = [EmailAddressInLine] + inlines = [EmailAddressInline] admin.site.unregister(Group) From e962059aac91d8f976ad5cbd8e266221607c6cdc Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Thu, 24 Feb 2022 12:36:52 -0500 Subject: [PATCH 054/150] projects now listed in admin pages as project names --- coldfront/core/allocation/admin.py | 10 ++++------ coldfront/core/project/admin.py | 18 ++++++++++++------ coldfront/core/statistics/admin.py | 8 +++++++- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/coldfront/core/allocation/admin.py b/coldfront/core/allocation/admin.py index 5aabbde2b..cbbb19380 100644 --- a/coldfront/core/allocation/admin.py +++ b/coldfront/core/allocation/admin.py @@ -56,7 +56,7 @@ class AllocationAdmin(SimpleHistoryAdmin): 'project', 'justification', 'created', 'modified',) fields_change = ('project', 'resources', 'quantity', 'justification', 'status', 'start_date', 'end_date', 'description', 'created', 'modified', 'is_locked') - list_display = ('pk', 'project_title', 'project_pis', 'resource', 'quantity', + list_display = ('pk', 'project', 'project_pis', 'resource', 'quantity', 'justification', 'start_date', 'end_date', 'status', 'created', 'modified', ) inlines = [AllocationUserInline, AllocationAttributeInline, @@ -76,8 +76,8 @@ def resource(self, obj): def project_pis(self, obj): return ', '.join(obj.project.pis().values_list('username', flat=True)) - def project_title(self, obj): - return textwrap.shorten(obj.project.title, width=50) + def project(self, obj): + return obj.project.name def get_fields(self, request, obj): if obj is None: @@ -192,7 +192,6 @@ def pis(self, obj): def project(self, obj): return obj.allocation.project.name - #return textwrap.shorten(obj.allocation.project.title, width=50) def project_title(self, obj): return obj.allocation.project.title @@ -252,7 +251,7 @@ def project_pis(self, obj): return ', '.join(project.pis().values_list('username', flat=True)) def project(self, obj): - return textwrap.shorten(obj.allocation.project.title, width=50) + return obj.allocation.project.name def get_fields(self, request, obj): if obj is None: @@ -393,7 +392,6 @@ def pis(self, obj): def project(self, obj): return obj.allocation.project.name - #return textwrap.shorten(obj.allocation.project.title, width=50) def user(self, obj): return textwrap.shorten(obj.allocation_user.user.username, width=50) diff --git a/coldfront/core/project/admin.py b/coldfront/core/project/admin.py index f1209be26..e5b006f38 100644 --- a/coldfront/core/project/admin.py +++ b/coldfront/core/project/admin.py @@ -38,15 +38,15 @@ class ProjectUserTransactionInline(admin.TabularInline): class ProjectUserAdmin(SimpleHistoryAdmin): fields_change = ('user', 'project', 'role', 'status', 'created', 'modified', ) readonly_fields_change = ('user', 'project', 'created', 'modified', ) - list_display = ('pk', 'project_title', 'User', 'role', 'status', 'created', + list_display = ('pk', 'project', 'User', 'role', 'status', 'created', 'modified',) list_filter = ('role', 'status') search_fields = ['user__username', 'user__first_name', 'user__last_name'] inlines = [ProjectUserTransactionInline] raw_id_fields = ('user', 'project') - def project_title(self, obj): - return textwrap.shorten(obj.project.title, width=50) + def project(self, obj): + return obj.project.name def User(self, obj): return '{} {} ({})'.format(obj.user.first_name, obj.user.last_name, obj.user.username) @@ -102,11 +102,14 @@ class ProjectTransactionInline(admin.TabularInline): @admin.register(Project) class ProjectAdmin(SimpleHistoryAdmin): - fields_change = ('title', 'description', 'status', 'requires_review', 'force_review', 'created', 'modified', ) + fields_change = ('name', 'description', 'status', 'requires_review', 'force_review', 'created', 'modified', ) readonly_fields_change = ('created', 'modified', ) - list_display = ('pk', 'title', 'PIs', 'created', 'modified', 'status') + list_display = ('pk', 'name', 'PIs', 'created', 'modified', 'status') search_fields = ['projectuser__user__username', - 'projectuser__user__last_name', 'projectuser__user__last_name', 'title'] + 'projectuser__user__last_name', + 'projectuser__user__last_name', + 'title', + 'name'] list_filter = ('status', 'force_review') inlines = [ProjectUserInline, ProjectAdminCommentInline, ProjectUserMessageInline, ProjectTransactionInline] @@ -160,3 +163,6 @@ def PIs(self, obj): '{} {} ({})'.format( pi_user.first_name, pi_user.last_name, pi_user.username) for pi_user in pi_users]) + + def project(self, obj): + return obj.project.name diff --git a/coldfront/core/statistics/admin.py b/coldfront/core/statistics/admin.py index 899ddb9c4..0b21bb4fb 100644 --- a/coldfront/core/statistics/admin.py +++ b/coldfront/core/statistics/admin.py @@ -32,7 +32,7 @@ def get_user(self, obj): @admin.register(Job) class JobAdmin(admin.ModelAdmin): - list_display = ('jobslurmid', 'userid', 'accountid', 'jobstatus', 'partition') + list_display = ('jobslurmid', 'user', 'project', 'jobstatus', 'partition') search_fields = ['jobslurmid', 'userid__username', 'accountid__name', 'jobstatus', 'partition'] def has_add_permission(self, request, obj=None): @@ -44,6 +44,12 @@ def has_delete_permission(self, request, obj=None): def has_change_permission(self, request, obj=None): return False + def user(self, obj): + return obj.userid.username + + def project(self, obj): + return obj.accountid.name + admin.register(CPU) admin.register(Node) From 30ecb10dee51a024ffe7fc3191b98d007e2fd14e Mon Sep 17 00:00:00 2001 From: Matthew Li Date: Fri, 25 Feb 2022 10:22:21 -0800 Subject: [PATCH 055/150] Move set_service_units; remove unused imports --- .../project/management/commands/__init__.py | 0 .../commands/add_service_units_to_project.py | 7 +- .../commands/deactivate_ica_projects.py | 6 +- .../core/project/management/commands/utils.py | 75 +++++++++++++++++++ .../test_deactivate_ica_projects.py | 2 +- .../core/project/utils_/addition_utils.py | 68 ----------------- 6 files changed, 79 insertions(+), 79 deletions(-) create mode 100644 coldfront/core/project/management/commands/__init__.py create mode 100644 coldfront/core/project/management/commands/utils.py diff --git a/coldfront/core/project/management/commands/__init__.py b/coldfront/core/project/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/coldfront/core/project/management/commands/add_service_units_to_project.py b/coldfront/core/project/management/commands/add_service_units_to_project.py index 6f48ed758..ee9c72f37 100644 --- a/coldfront/core/project/management/commands/add_service_units_to_project.py +++ b/coldfront/core/project/management/commands/add_service_units_to_project.py @@ -5,13 +5,8 @@ from coldfront.config import settings from coldfront.core.project.models import Project -from coldfront.core.project.utils_.addition_utils import set_service_units -from coldfront.core.statistics.models import ProjectTransaction -from coldfront.core.statistics.models import ProjectUserTransaction -from coldfront.core.utils.common import utc_now_offset_aware +from coldfront.core.project.management.commands.utils import set_service_units from coldfront.api.statistics.utils import get_accounting_allocation_objects -from coldfront.api.statistics.utils import set_project_allocation_value -from coldfront.api.statistics.utils import set_project_user_allocation_value from coldfront.core.allocation.models import AllocationAttributeType, Allocation diff --git a/coldfront/core/project/management/commands/deactivate_ica_projects.py b/coldfront/core/project/management/commands/deactivate_ica_projects.py index aca67e9d0..bab60e85b 100644 --- a/coldfront/core/project/management/commands/deactivate_ica_projects.py +++ b/coldfront/core/project/management/commands/deactivate_ica_projects.py @@ -13,9 +13,7 @@ from django.core.management.base import BaseCommand import logging -from coldfront.core.project.utils_.addition_utils import set_service_units -from coldfront.core.statistics.models import ProjectTransaction, \ - ProjectUserTransaction +from coldfront.core.project.management.commands.utils import set_service_units from coldfront.core.utils.common import utc_now_offset_aware from coldfront.core.utils.mail import send_email_template @@ -107,7 +105,7 @@ def reset_service_units(self, project, dry_run): updated_su = Decimal('0.00') if dry_run: - message = f'Would reset {project.name} and its users\'s SUs from ' \ + message = f'Would reset {project.name} and its users\' SUs from ' \ f'{current_allocation} to {updated_su}. The reason ' \ f'would be: "Resetting SUs while deactivating expired ' \ f'ICA project."' diff --git a/coldfront/core/project/management/commands/utils.py b/coldfront/core/project/management/commands/utils.py new file mode 100644 index 000000000..6be3409a5 --- /dev/null +++ b/coldfront/core/project/management/commands/utils.py @@ -0,0 +1,75 @@ +from coldfront.api.statistics.utils import get_accounting_allocation_objects +from coldfront.api.statistics.utils import set_project_allocation_value +from coldfront.api.statistics.utils import set_project_usage_value +from coldfront.api.statistics.utils import set_project_user_allocation_value +from coldfront.api.statistics.utils import set_project_user_usage_value +from coldfront.core.statistics.models import ProjectTransaction +from coldfront.core.statistics.models import ProjectUserTransaction +from coldfront.core.utils.common import utc_now_offset_aware + + +def set_service_units(project, allocation_objects, updated_su, reason, + update_usage): + """ + Sets allocation and allocation_user service units to updated_su. Creates + the relevant transaction objects to record the change. Updates the + relevant historical objects with the reason for the SU change. If + update_usage is True, allocation and allocation_user usage values are + updated. + """ + + def set_historical_reason(obj): + """Set the latest historical object reason""" + obj.refresh_from_db() + historical_obj = obj.history.latest('id') + historical_obj.history_change_reason = reason + historical_obj.save() + + current_date = utc_now_offset_aware() + + # Set the value for the Project. + set_project_allocation_value(project, updated_su) + + if update_usage: + set_project_usage_value(project, updated_su) + + # Create a transaction to record the change. + ProjectTransaction.objects.create( + project=project, + date_time=current_date, + allocation=updated_su) + + # Set the reason for the change in the newly-created historical object. + set_historical_reason( + allocation_objects.allocation_attribute) + + # Do the same for each ProjectUser. + for project_user in project.projectuser_set.all(): + user = project_user.user + # Attempt to set the value for the ProjectUser. The method returns + # whether it succeeded; it may not because not every ProjectUser has a + # corresponding AllocationUser (e.g., PIs). Only proceed with further + # steps if an update occurred. + allocation_updated = set_project_user_allocation_value( + user, project, updated_su) + success_flag = allocation_updated + + if update_usage: + allocation_usage_updated = set_project_user_usage_value( + user, project, updated_su) + + success_flag = allocation_updated and allocation_usage_updated + + if success_flag: + # Create a transaction to record the change. + ProjectUserTransaction.objects.create( + project_user=project_user, + date_time=current_date, + allocation=updated_su) + + # Set the reason for the change in the newly-created historical + # object. + allocation_user_obj = get_accounting_allocation_objects( + project, user=user) + set_historical_reason( + allocation_user_obj.allocation_user_attribute) diff --git a/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py b/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py index acdd2f931..810faf4a4 100644 --- a/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py +++ b/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py @@ -123,7 +123,7 @@ def test_dry_run_with_expired_projects(self): f'status to Inactive and Allocation ' f'{allocation.pk}\'s status to Expired.', - f'Would reset {project.name} and its users\'s SUs from ' + f'Would reset {project.name} and its users\' SUs from ' f'1000.00 to 0.00. The reason ' f'would be: "Resetting SUs while deactivating expired ' f'ICA project."', diff --git a/coldfront/core/project/utils_/addition_utils.py b/coldfront/core/project/utils_/addition_utils.py index 6f742f1cb..21ed70af2 100644 --- a/coldfront/core/project/utils_/addition_utils.py +++ b/coldfront/core/project/utils_/addition_utils.py @@ -11,10 +11,8 @@ from coldfront.core.statistics.models import ProjectUserTransaction from coldfront.core.utils.common import project_detail_url from coldfront.core.utils.common import utc_now_offset_aware -from coldfront.core.utils.common import validate_num_service_units from coldfront.core.utils.mail import send_email_template -from collections import namedtuple from decimal import Decimal from django.conf import settings @@ -223,69 +221,3 @@ def has_pending_allocation_addition_request(project): name='Under Review') return AllocationAdditionRequest.objects.filter( project=project, status=under_review_status).exists() - - -def set_service_units(project, allocation_objects, updated_su, reason, update_usage): - """ - Sets allocation and allocation_user service units to updated_su. Creates - the relevant transaction objects to record the change. Updates the - relevant historical objects with the reason for the SU change. If - update_usage is True, allocation and allocation_user usage values are - updated. - """ - - def set_historical_reason(obj): - """Set the latest historical object reason""" - obj.refresh_from_db() - historical_obj = obj.history.latest('id') - historical_obj.history_change_reason = reason - historical_obj.save() - - current_date = utc_now_offset_aware() - - # Set the value for the Project. - set_project_allocation_value(project, updated_su) - - if update_usage: - set_project_usage_value(project, updated_su) - - # Create a transaction to record the change. - ProjectTransaction.objects.create( - project=project, - date_time=current_date, - allocation=updated_su) - - # Set the reason for the change in the newly-created historical object. - set_historical_reason( - allocation_objects.allocation_attribute) - - # Do the same for each ProjectUser. - for project_user in project.projectuser_set.all(): - user = project_user.user - # Attempt to set the value for the ProjectUser. The method returns whether - # it succeeded; it may not because not every ProjectUser has a - # corresponding AllocationUser (e.g., PIs). Only proceed with further steps - # if an update occurred. - - allocation_updated = set_project_user_allocation_value( - user, project, updated_su) - success_flag = allocation_updated - - if update_usage: - allocation_usage_updated = set_project_user_usage_value( - user, project, updated_su) - - success_flag = allocation_updated and allocation_usage_updated - - if success_flag: - # Create a transaction to record the change. - ProjectUserTransaction.objects.create( - project_user=project_user, - date_time=current_date, - allocation=updated_su) - - # Set the reason for the change in the newly-created historical object. - allocation_user_obj = get_accounting_allocation_objects( - project, user=user) - set_historical_reason( - allocation_user_obj.allocation_user_attribute) \ No newline at end of file From 315450bd30e977da63396ff109d3f1156b2f2b6e Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Fri, 25 Feb 2022 13:36:52 -0500 Subject: [PATCH 056/150] projectusertransaction admin searchable via username --- coldfront/core/statistics/admin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/coldfront/core/statistics/admin.py b/coldfront/core/statistics/admin.py index 0b21bb4fb..1ea873da3 100644 --- a/coldfront/core/statistics/admin.py +++ b/coldfront/core/statistics/admin.py @@ -17,7 +17,8 @@ class ProjectTransactionAdmin(admin.ModelAdmin): @admin.register(ProjectUserTransaction) class ProjectUserTransactionAdmin(admin.ModelAdmin): list_display = ('date_time', 'get_project', 'get_user', 'allocation', ) - search_fields = ('project_user__project__name',) + search_fields = ('project_user__project__name', + 'project_user__user__username') def get_project(self, obj): return obj.project_user.project From 5e40e889c8f88bd5a639a48bec1c251833c36073 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Fri, 25 Feb 2022 13:45:24 -0500 Subject: [PATCH 057/150] resolved issues in PR 365 --- .../commands/deactivate_ica_projects.py | 12 +-- .../test_add_service_units_to_project.py | 74 +++++++++---------- .../test_deactivate_ica_projects.py | 27 +++---- .../test_commands/test_service_units_base.py | 2 +- 4 files changed, 52 insertions(+), 63 deletions(-) diff --git a/coldfront/core/project/management/commands/deactivate_ica_projects.py b/coldfront/core/project/management/commands/deactivate_ica_projects.py index bab60e85b..8dfa076cd 100644 --- a/coldfront/core/project/management/commands/deactivate_ica_projects.py +++ b/coldfront/core/project/management/commands/deactivate_ica_projects.py @@ -144,16 +144,12 @@ def send_emails(self, project, expiry_date, dry_run): } recipients = project.managers_and_pis_emails() + plural = '' if len(recipients) == 1 else 's' if dry_run: - msg_plain = \ - render_to_string('email/expired_ica_project.txt', - context) - - message = f'Would send the following email to ' \ - f'{len(recipients)} users:' + message = f'Would send a notification email to ' \ + f'{len(recipients)} user{plural}.' self.stdout.write(self.style.WARNING(message)) - self.stdout.write(self.style.WARNING(msg_plain)) else: try: @@ -165,7 +161,7 @@ def send_emails(self, project, expiry_date, dry_run): recipients) message = f'Sent deactivation notification email to ' \ - f'{len(recipients)} users.' + f'{len(recipients)} user{plural}.' self.stdout.write(self.style.SUCCESS(message)) except Exception as e: diff --git a/coldfront/core/project/tests/test_commands/test_add_service_units_to_project.py b/coldfront/core/project/tests/test_commands/test_add_service_units_to_project.py index 2703c4bb2..7941cb8a3 100644 --- a/coldfront/core/project/tests/test_commands/test_add_service_units_to_project.py +++ b/coldfront/core/project/tests/test_commands/test_add_service_units_to_project.py @@ -53,11 +53,11 @@ def setUp(self): def test_dry_run(self): """Testing add_service_units_to_project dry run""" - output, error = self.call_deactivate_command('add_service_units_to_project', - '--project_name=project0', - '--amount=1000', - f'--reason={self.reason}', - '--dry_run') + output, error = self.call_command('add_service_units_to_project', + '--project_name=project0', + '--amount=1000', + f'--reason={self.reason}', + '--dry_run') dry_run_message = 'Would add 1000 additional SUs to project ' \ 'project0. This would increase project0 ' \ @@ -79,10 +79,10 @@ def test_creates_and_updates_objects_positive_SU(self): self.allocation_values_test(project, '1000.00', '500.00') # run command - output, error = self.call_deactivate_command('add_service_units_to_project', - '--project_name=project0', - '--amount=1000', - f'--reason={self.reason}') + output, error = self.call_command('add_service_units_to_project', + '--project_name=project0', + '--amount=1000', + f'--reason={self.reason}') message = f'Successfully added 1000 SUs to project0 and its users, ' \ f'updating project0\'s SUs from 1000.00 to 2000.00. The ' \ @@ -114,10 +114,10 @@ def test_creates_and_updates_objects_negative_SU(self): self.allocation_values_test(project, '1000.00', '500.00') # run command - output, error = self.call_deactivate_command('add_service_units_to_project', - '--project_name=project0', - '--amount=-800', - f'--reason={self.reason}') + output, error = self.call_command('add_service_units_to_project', + '--project_name=project0', + '--amount=-800', + f'--reason={self.reason}') message = f'Successfully added -800 SUs to project0 and its users, ' \ f'updating project0\'s SUs from 1000.00 to 200.00. The ' \ @@ -163,20 +163,20 @@ def test_input_validations(self): # part of Savio Compute with self.assertRaises(CommandError): output, error = \ - self.call_deactivate_command('add_service_units_to_project', - '--project_name=project1', - '--amount=1000', - f'--reason={self.reason}') + self.call_command('add_service_units_to_project', + '--project_name=project1', + '--amount=1000', + f'--reason={self.reason}') self.assertEqual(output, '') self.assertEqual(error, '') # testing a project that does not exist with self.assertRaises(CommandError): output, error = \ - self.call_deactivate_command('add_service_units_to_project', - '--project_name=project555', - '--amount=1000', - f'--reason={self.reason}') + self.call_command('add_service_units_to_project', + '--project_name=project555', + '--amount=1000', + f'--reason={self.reason}') self.assertEqual(output, '') self.assertEqual(error, '') @@ -184,10 +184,10 @@ def test_input_validations(self): # than settings.ALLOCATION_MIN with self.assertRaises(CommandError): output, error = \ - self.call_deactivate_command('add_service_units_to_project', - '--project_name=project0', - '--amount=-100000', - f'--reason={self.reason}') + self.call_command('add_service_units_to_project', + '--project_name=project0', + '--amount=-100000', + f'--reason={self.reason}') self.assertEqual(output, '') self.assertEqual(error, '') @@ -195,29 +195,29 @@ def test_input_validations(self): # than settings.ALLOCATION_MAX with self.assertRaises(CommandError): output, error = \ - self.call_deactivate_command('add_service_units_to_project', - '--project_name=project0', - '--amount=99999500', - f'--reason={self.reason}') + self.call_command('add_service_units_to_project', + '--project_name=project0', + '--amount=99999500', + f'--reason={self.reason}') self.assertEqual(output, '') self.assertEqual(error, '') # adding service units that are greater than settings.ALLOCATION_MAX with self.assertRaises(CommandError): output, error = \ - self.call_deactivate_command('add_service_units_to_project', - '--project_name=project0', - '--amount=500000000', - f'--reason={self.reason}') + self.call_command('add_service_units_to_project', + '--project_name=project0', + '--amount=500000000', + f'--reason={self.reason}') self.assertEqual(output, '') self.assertEqual(error, '') # reason is not long enough with self.assertRaises(CommandError): output, error = \ - self.call_deactivate_command('add_service_units_to_project', - '--project_name=project0', - '--amount=1000', - '--reason=notlong') + self.call_command('add_service_units_to_project', + '--project_name=project0', + '--amount=1000', + '--reason=notlong') self.assertEqual(output, '') self.assertEqual(error, '') diff --git a/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py b/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py index 810faf4a4..89d4edb66 100644 --- a/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py +++ b/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py @@ -101,9 +101,9 @@ def usage_values_updated(self, project, updated_value): def test_dry_run_no_expired_projects(self): """Testing a dry run in which no ICA projects are expired""" - output, error = self.call_deactivate_command('deactivate_ica_projects', - '--dry_run', - '--send_emails') + output, error = self.call_command('deactivate_ica_projects', + '--dry_run', + '--send_emails') self.assertIn(output, '') self.assertEqual(error, '') @@ -111,9 +111,9 @@ def test_dry_run_with_expired_projects(self): """Testing a dry run in which an ICA project is expired""" self.create_expired_project('ic_project0') - output, error = self.call_deactivate_command('deactivate_ica_projects', - '--dry_run', - '--send_emails') + output, error = self.call_command('deactivate_ica_projects', + '--dry_run', + '--send_emails') project = Project.objects.get(name='ic_project0') allocation = get_project_compute_allocation(project) @@ -128,14 +128,7 @@ def test_dry_run_with_expired_projects(self): f'would be: "Resetting SUs while deactivating expired ' f'ICA project."', - 'Would send the following email to 1 users:', - f'Dear managers of {project.name},', - - f'This is a notification that the project {project.name} ' - f'expired on {allocation.end_date.strftime("%m-%d-%Y")} ' - f'and has therefore been deactivated. ' - f'Accounts under this project will no longer be able ' - f'to access its compute resources.'] + 'Would send a notification email to 1 user.'] for message in messages: self.assertIn(message, output) @@ -156,7 +149,7 @@ def test_creates_and_updates_objects(self): self.allocation_values_test(project, '1000.00', '500.00') # run command - output, error = self.call_deactivate_command('deactivate_ica_projects') + output, error = self.call_command('deactivate_ica_projects') messages = [ f'Updated Project {project.name} ({project.pk})\'s status to ' @@ -208,8 +201,8 @@ def test_emails_sent(self): old_end_date = allocation.end_date # run command - output, error = self.call_deactivate_command('deactivate_ica_projects', - '--send_emails') + output, error = self.call_command('deactivate_ica_projects', + '--send_emails') recipients = project.managers_and_pis_emails() diff --git a/coldfront/core/project/tests/test_commands/test_service_units_base.py b/coldfront/core/project/tests/test_commands/test_service_units_base.py index 5c1fb4394..8210a4fb2 100644 --- a/coldfront/core/project/tests/test_commands/test_service_units_base.py +++ b/coldfront/core/project/tests/test_commands/test_service_units_base.py @@ -45,7 +45,7 @@ def setUp(self): mail.outbox = [] @staticmethod - def call_deactivate_command(*args): + def call_command(*args): """ Call the command with the given arguments, returning the messages written to stdout and stderr. From 58bab756ca0430da04b9838e63028475ee5b25bf Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Fri, 25 Feb 2022 17:03:29 -0500 Subject: [PATCH 058/150] initial request hub commit --- .../templates/request_hub/request_hub.html | 261 ++++++++++++++++++ .../core/user/views_/request_hub_views.py | 115 ++++++++ 2 files changed, 376 insertions(+) create mode 100644 coldfront/core/user/templates/request_hub/request_hub.html create mode 100644 coldfront/core/user/views_/request_hub_views.py diff --git a/coldfront/core/user/templates/request_hub/request_hub.html b/coldfront/core/user/templates/request_hub/request_hub.html new file mode 100644 index 000000000..46a5afe3f --- /dev/null +++ b/coldfront/core/user/templates/request_hub/request_hub.html @@ -0,0 +1,261 @@ +{% extends "common/base.html" %} {% load common_tags %} {% block content %} + +

+
+

Request Hub

+
+ +

Below are all your active requests. To view your completed requests, click to view completed requests in the appropriate section

+
+
+ + +
+
+
+ + +
+ + Cluster Account Requests + {% if num_active_cluster_account_requests %} + {{ num_active_cluster_account_requests }} active requests + {% endif %} + +
+ + +
+
+
+ +
+
+ {% if cluster_account_list_active %} + +
+
+
Active Cluster Account Requests
+
+ + + + + + + + + + + + + + {% for cluster_account in cluster_account_list_active %} + + + + + + + + + + {% endfor %} + +
+ # + + + + Date Requested/
Last Modified + + + 		+ User Email + + + + Cluster Username + + + + Project + + + + Allocation + + Status +
{{ cluster_account.pk }}{{ cluster_account.modified|date:"M. d, Y" }}{{ cluster_account.allocation_user.user.email }} + {% if cluster_account.allocation_user.user.userprofile.cluster_uid %} + {{ cluster_account.allocation_user.user.username }} + {% else %} + + No cluster account. + + {% endif %} + + + {{ cluster_account.allocation.project.name }} + + + + Allocation {{ allocation.pk }} + {{ cluster_account.allocation.pk }} + + + {% with status=cluster_account.value %} + {% if status == "Pending - Add" %} + Pending + {% elif status == "Processing" %} + {{ status }} + {% elif status == "Active" %} + {{ status }} + {% elif status == "Denied" %} + {{ status }} + {% endif %} + {% endwith %} +
+ + Page {{ cluster_account_list_active.number }} of {{ cluster_account_list_active.paginator.num_pages }} +
    + {% if cluster_account_list_active.has_previous %} +
  • Previous
    • + {% else %} +
  • Previous
    • + {% endif %} + {% if cluster_account_list_active.has_next %} +
  • Next
    • + {% else %} +
  • Next
    • + {% endif %} +
+
+
+
+ + {% else %} +
+ No active cluster account requests! +
+ {% endif %} +
+
+

+
+
+ {% if cluster_account_list_complete %} +
+
+
Completed Cluster Account Requests
+
+ + + + + + + + + + + + + + {% for cluster_account in cluster_account_list_complete %} + + + + + + + + + + {% endfor %} + +
+ # + + + + Date Requested/
Last Modified + + + 		+ User Email + + + + Cluster Username + + + + Project + + + + Allocation + + Status +
{{ cluster_account.pk }}{{ cluster_account.modified|date:"M. d, Y" }}{{ cluster_account.allocation_user.user.email }} + {% if cluster_account.allocation_user.user.userprofile.cluster_uid %} + {{ cluster_account.allocation_user.user.username }} + {% else %} + + No cluster account. + + {% endif %} + + + {{ cluster_account.allocation.project.name }} + + + + Allocation {{ allocation.pk }} + {{ cluster_account.allocation.pk }} + + + {% with status=cluster_account.value %} + {% if status == "Pending - Add" %} + Pending + {% elif status == "Processing" %} + {{ status }} + {% elif status == "Active" %} + {{ status }} + {% elif status == "Denied" %} + {{ status }} + {% endif %} + {% endwith %} +
+ + Page {{ cluster_account_list_complete.number }} of {{ cluster_account_list_complete.paginator.num_pages }} +
    + {% if cluster_account_list_complete.has_previous %} +
  • Previous
    • + {% else %} +
  • Previous
    • + {% endif %} + {% if cluster_account_list_complete.has_next %} +
  • Next
    • + {% else %} +
  • Next
    • + {% endif %} +
+
+
+
+ + {% else %} +
+ No new or pending cluster account requests! +
+ {% endif %} +
+
+
+
+
+
+ +{% endblock %} diff --git a/coldfront/core/user/views_/request_hub_views.py b/coldfront/core/user/views_/request_hub_views.py new file mode 100644 index 000000000..ba0049cff --- /dev/null +++ b/coldfront/core/user/views_/request_hub_views.py @@ -0,0 +1,115 @@ +from itertools import chain + +from django.contrib import messages +from django.contrib.auth.mixins import LoginRequiredMixin, UserPassesTestMixin +from django.contrib.auth.models import User +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator +from django.db.models import Q +from django.http import HttpResponseRedirect +from django.shortcuts import get_object_or_404, render +from django.urls import reverse +from django.views.generic import ListView +from django.views.generic.base import TemplateView, View +from django.views.generic.edit import FormView + +from coldfront.core.allocation.models import (Allocation, + AllocationAttributeType, + AllocationUserStatusChoice, + AllocationUserAttribute) + +from coldfront.core.project.forms_.removal_forms import \ + (ProjectRemovalRequestSearchForm, + ProjectRemovalRequestUpdateStatusForm, + ProjectRemovalRequestCompletionForm) +from coldfront.core.project.models import (Project, + ProjectUserStatusChoice, + ProjectUserRemovalRequest, + ProjectUserRemovalRequestStatusChoice) +from coldfront.core.project.utils_.removal_utils import ProjectRemovalRequestRunner +from coldfront.core.utils.common import (import_from_settings, + utc_now_offset_aware) +from coldfront.core.utils.mail import send_email_template + +import logging + +EMAIL_ENABLED = import_from_settings('EMAIL_ENABLED', False) + +if EMAIL_ENABLED: + EMAIL_SENDER = import_from_settings('EMAIL_SENDER') + EMAIL_SIGNATURE = import_from_settings('EMAIL_SIGNATURE') + SUPPORT_EMAIL = import_from_settings('CENTER_HELP_EMAIL') + +logger = logging.getLogger(__name__) + + +class RequestHub(LoginRequiredMixin, + UserPassesTestMixin, + TemplateView): + template_name = 'request_hub/request_hub.html' + paginate_by = 2 + paginators = 0 + + def get_cluster_account_requests(self): + user = self.request.user + + cluster_account_status = AllocationAttributeType.objects.get( + name='Cluster Account Status') + + cluster_account_list_complete = AllocationUserAttribute.objects.filter( + allocation_attribute_type=cluster_account_status, + value__in=['Denied', 'Active'], + allocation_user__user=user) + + cluster_account_list_active = AllocationUserAttribute.objects.filter( + allocation_attribute_type=cluster_account_status, + value__in=['Pending - Add', 'Processing'], + allocation_user__user=user) + + return cluster_account_list_active, cluster_account_list_complete + + def test_func(self): + """UserPassesTestMixin tests.""" + if self.request.user.is_superuser: + return True + + if self.request.user.has_perm('project.view_projectuserremovalrequest'): + return True + # + # message = ( + # 'You do not have permission to review project removal requests.') + # messages.error(self.request, message) + + return True + + def get_context_data(self, **kwargs): + def create_paginator(queryset, context_name): + """ + Creates a paginator object for the given queryset + and updates the context with the created object. + """ + paginator = Paginator(queryset, self.paginate_by) + page = self.request.GET.get(f'page{self.paginators}') + try: + queryset = paginator.page(page) + except PageNotAnInteger: + queryset = paginator.page(1) + except EmptyPage: + queryset = paginator.page(paginator.num_pages) + + context[context_name] = queryset + + self.paginators += 1 + + context = super().get_context_data(**kwargs) + + cluster_account_list_active, cluster_account_list_complete = \ + self.get_cluster_account_requests() + + create_paginator(cluster_account_list_active, + 'cluster_account_list_active') + context['num_active_cluster_account_requests'] = len(cluster_account_list_active) + + create_paginator(cluster_account_list_complete, + 'cluster_account_list_complete') + + return context \ No newline at end of file From fe3871a73af9a6bf94bd727b20fc75513a60acc2 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Fri, 25 Feb 2022 17:03:48 -0500 Subject: [PATCH 059/150] added path for request hub --- coldfront/core/user/urls.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/coldfront/core/user/urls.py b/coldfront/core/user/urls.py index c48520c12..a50278965 100644 --- a/coldfront/core/user/urls.py +++ b/coldfront/core/user/urls.py @@ -7,6 +7,7 @@ from django.urls import path, reverse_lazy import coldfront.core.user.views as user_views +import coldfront.core.user.views_.request_hub_views as request_hub_views from coldfront.core.user.forms import VerifiedEmailAddressPasswordResetForm from coldfront.core.user.forms import UserLoginForm @@ -113,4 +114,9 @@ path('identity-linking-request', user_views.IdentityLinkingRequestView.as_view(), name='identity-linking-request'), + + # Request Hub + path('request-hub', + request_hub_views.RequestHub.as_view(), + name='request-hub'), ] From a87f65a14dc3e0f89db1037e2711eb3eb7edf60e Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Fri, 25 Feb 2022 17:04:11 -0500 Subject: [PATCH 060/150] added request hub to navbar --- coldfront/templates/common/authorized_navbar.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/coldfront/templates/common/authorized_navbar.html b/coldfront/templates/common/authorized_navbar.html index 854ffca5e..066c6dad7 100644 --- a/coldfront/templates/common/authorized_navbar.html +++ b/coldfront/templates/common/authorized_navbar.html @@ -30,6 +30,9 @@

+
  • + Request Hub +
    • {% if request.user.is_superuser %} {% include 'common/navbar_admin.html' %} {% elif request.user.is_staff %} From 1e282ec7dc3e844c4019dbe24b483dcd1c9fee63 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Fri, 25 Feb 2022 17:14:46 -0500 Subject: [PATCH 061/150] changed icons and spacing --- .../user/templates/request_hub/request_hub.html | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/coldfront/core/user/templates/request_hub/request_hub.html b/coldfront/core/user/templates/request_hub/request_hub.html index 46a5afe3f..4046a5e9a 100644 --- a/coldfront/core/user/templates/request_hub/request_hub.html +++ b/coldfront/core/user/templates/request_hub/request_hub.html @@ -19,7 +19,10 @@

    Request Hub

    Cluster Account Requests {% if num_active_cluster_account_requests %} - {{ num_active_cluster_account_requests }} active requests + + + + {{ num_active_cluster_account_requests }} active requests {% endif %} @@ -141,7 +144,7 @@
    Active Cluster Account Requests
    {% endif %} -

    +
    {% if cluster_account_list_complete %} @@ -248,9 +251,17 @@
    Completed Cluster Account Requests
    {% else %}
    - No new or pending cluster account requests! + No completed cluster account requests!
    {% endif %} + + {% if user.is_superuser %} +

    + + + Go To Cluster Account Requests Main Page + + {% endif %}
    From d06800ffe4386c81473a7ec22fd07ff79288fa67 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Fri, 25 Feb 2022 18:20:59 -0500 Subject: [PATCH 062/150] moved cluster request table to separate html file --- .../core/user/views_/request_hub_views.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/coldfront/core/user/views_/request_hub_views.py b/coldfront/core/user/views_/request_hub_views.py index ba0049cff..3a22cf551 100644 --- a/coldfront/core/user/views_/request_hub_views.py +++ b/coldfront/core/user/views_/request_hub_views.py @@ -67,6 +67,22 @@ def get_cluster_account_requests(self): return cluster_account_list_active, cluster_account_list_complete + def get_removal_requests(self): + user = self.request.user + + project_user_cond = Q(project_user__user=self.request.user) + requester_cond = Q(requester=self.request.user) + + removal_request_active = ProjectUserRemovalRequest.objects.filter( + status__name__in=['Pending', 'Processing']).\ + filter(project_user_cond | requester_cond) + + removal_request_complete = ProjectUserRemovalRequest.objects.filter( + status__name='Complete').\ + filter(project_user_cond | requester_cond) + + return removal_request_active, removal_request_complete + def test_func(self): """UserPassesTestMixin tests.""" if self.request.user.is_superuser: @@ -112,4 +128,14 @@ def create_paginator(queryset, context_name): create_paginator(cluster_account_list_complete, 'cluster_account_list_complete') + removal_request_active, removal_request_complete = \ + self.get_removal_requests() + + create_paginator(removal_request_active, + 'removal_request_active') + context['num_removal_request_active'] = len(removal_request_active) + + create_paginator(removal_request_complete, + 'removal_request_complete') + return context \ No newline at end of file From 2b8805ac0ce3b0f8e8b3849ef797e78599446415 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Fri, 25 Feb 2022 18:23:53 -0500 Subject: [PATCH 063/150] moved cluster request table to separate html file --- .../request_hub/cluster_account_list.html | 112 +++++++++ .../templates/request_hub/request_hub.html | 232 +----------------- 2 files changed, 120 insertions(+), 224 deletions(-) create mode 100644 coldfront/core/user/templates/request_hub/cluster_account_list.html diff --git a/coldfront/core/user/templates/request_hub/cluster_account_list.html b/coldfront/core/user/templates/request_hub/cluster_account_list.html new file mode 100644 index 000000000..c7c699c51 --- /dev/null +++ b/coldfront/core/user/templates/request_hub/cluster_account_list.html @@ -0,0 +1,112 @@ +
    +
    + {% if queryset %} + +
    +
    +
    {{ adj }} Cluster Account Requests
    +
    + + + + + + + + + + + + + + {% for cluster_account in queryset %} + + + + + + + + + + {% endfor %} + +
    + # + + + + Date Requested/
    Last Modified + + +     		+ User Email + + + + Cluster Username + + + + Project + + + + Allocation + + Status +
    {{ cluster_account.pk }}{{ cluster_account.modified|date:"M. d, Y" }}{{ cluster_account.allocation_user.user.email }} + {% if cluster_account.allocation_user.user.userprofile.cluster_uid %} + {{ cluster_account.allocation_user.user.username }} + {% else %} + + No cluster account. + + {% endif %} + + + {{ cluster_account.allocation.project.name }} + + + + Allocation {{ allocation.pk }} + {{ cluster_account.allocation.pk }} + + + {% with status=cluster_account.value %} + {% if status == "Pending - Add" %} + Pending + {% elif status == "Processing" %} + {{ status }} + {% elif status == "Active" %} + {{ status }} + {% elif status == "Denied" %} + {{ status }} + {% endif %} + {% endwith %} +
    + + Page {{ queryset.number }} of {{ queryset.paginator.num_pages }} +
      + {% if queryset.has_previous %} +
    • Previous
      • + {% else %} +
    • Previous
      • + {% endif %} + {% if queryset.has_next %} +
    • Next
      • + {% else %} +
    • Next
      • + {% endif %} +
    +
    +
    +
    + + {% else %} +
    + No {{ adj }} cluster account requests! +
    + {% endif %} +
    +
    diff --git a/coldfront/core/user/templates/request_hub/request_hub.html b/coldfront/core/user/templates/request_hub/request_hub.html index 4046a5e9a..f64077267 100644 --- a/coldfront/core/user/templates/request_hub/request_hub.html +++ b/coldfront/core/user/templates/request_hub/request_hub.html @@ -32,241 +32,25 @@

    Request Hub

    -
    -
    - {% if cluster_account_list_active %} + {% with queryset=cluster_account_list_active adj='active' page_num='0' %} + {% include 'request_hub/cluster_account_list.html' %} + {% endwith %} -
    -
    -
    Active Cluster Account Requests
    -
    - - - - - - - - - - - - - - {% for cluster_account in cluster_account_list_active %} - - - - - - - - - - {% endfor %} - -
    - # - - - - Date Requested/
    Last Modified - - -     		- User Email - - - - Cluster Username - - - - Project - - - - Allocation - - Status -
    {{ cluster_account.pk }}{{ cluster_account.modified|date:"M. d, Y" }}{{ cluster_account.allocation_user.user.email }} - {% if cluster_account.allocation_user.user.userprofile.cluster_uid %} - {{ cluster_account.allocation_user.user.username }} - {% else %} - - No cluster account. - - {% endif %} - - - {{ cluster_account.allocation.project.name }} - - - - Allocation {{ allocation.pk }} - {{ cluster_account.allocation.pk }} - - - {% with status=cluster_account.value %} - {% if status == "Pending - Add" %} - Pending - {% elif status == "Processing" %} - {{ status }} - {% elif status == "Active" %} - {{ status }} - {% elif status == "Denied" %} - {{ status }} - {% endif %} - {% endwith %} -
    - - Page {{ cluster_account_list_active.number }} of {{ cluster_account_list_active.paginator.num_pages }} -
      - {% if cluster_account_list_active.has_previous %} -
    • Previous
      • - {% else %} -
    • Previous
      • - {% endif %} - {% if cluster_account_list_active.has_next %} -
    • Next
      • - {% else %} -
    • Next
      • - {% endif %} -
    -
    -
    -
    - - {% else %} -
    - No active cluster account requests! -
    - {% endif %} -
    -
    - -
    -
    - {% if cluster_account_list_complete %} -
    -
    -
    Completed Cluster Account Requests
    -
    - - - - - - - - - - - - - - {% for cluster_account in cluster_account_list_complete %} - - - - - - - - - - {% endfor %} - -
    - # - - - - Date Requested/
    Last Modified - - -     		- User Email - - - - Cluster Username - - - - Project - - - - Allocation - - Status -
    {{ cluster_account.pk }}{{ cluster_account.modified|date:"M. d, Y" }}{{ cluster_account.allocation_user.user.email }} - {% if cluster_account.allocation_user.user.userprofile.cluster_uid %} - {{ cluster_account.allocation_user.user.username }} - {% else %} - - No cluster account. - - {% endif %} - - - {{ cluster_account.allocation.project.name }} - - - - Allocation {{ allocation.pk }} - {{ cluster_account.allocation.pk }} - - - {% with status=cluster_account.value %} - {% if status == "Pending - Add" %} - Pending - {% elif status == "Processing" %} - {{ status }} - {% elif status == "Active" %} - {{ status }} - {% elif status == "Denied" %} - {{ status }} - {% endif %} - {% endwith %} -
    - - Page {{ cluster_account_list_complete.number }} of {{ cluster_account_list_complete.paginator.num_pages }} -
      - {% if cluster_account_list_complete.has_previous %} -
    • Previous
      • - {% else %} -
    • Previous
      • - {% endif %} - {% if cluster_account_list_complete.has_next %} -
    • Next
      • - {% else %} -
    • Next
      • - {% endif %} -
    -
    -
    -
    - - {% else %} -
    - No completed cluster account requests! -
    - {% endif %} + {% with queryset=cluster_account_list_complete adj='complete' page_num='1' %} + {% include 'request_hub/cluster_account_list.html' %} + {% endwith %} {% if user.is_superuser %} -

    +
    Go To Cluster Account Requests Main Page +
    {% endif %}
    - - {% endblock %} From 4d5d00dedba563a68e72d5bddba4cc2359da042d Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Fri, 25 Feb 2022 20:41:58 -0500 Subject: [PATCH 064/150] added removal requests --- coldfront/core/user/views_/request_hub_views.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/coldfront/core/user/views_/request_hub_views.py b/coldfront/core/user/views_/request_hub_views.py index 3a22cf551..00c8ac387 100644 --- a/coldfront/core/user/views_/request_hub_views.py +++ b/coldfront/core/user/views_/request_hub_views.py @@ -48,6 +48,7 @@ class RequestHub(LoginRequiredMixin, template_name = 'request_hub/request_hub.html' paginate_by = 2 paginators = 0 + show_all_requests = False def get_cluster_account_requests(self): user = self.request.user @@ -138,4 +139,8 @@ def create_paginator(queryset, context_name): create_paginator(removal_request_complete, 'removal_request_complete') + context['show_all'] = (self.request.user.is_superuser or + self.request.user.is_staff) and \ + self.show_all_requests + return context \ No newline at end of file From 07299c6eeef84f3e55f9c6f465730eb45d86aaff Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Fri, 25 Feb 2022 20:42:24 -0500 Subject: [PATCH 065/150] added removal requests --- .../request_hub/removal_request_list.html | 107 ++++++++++++++++++ .../templates/request_hub/request_hub.html | 51 ++++++++- 2 files changed, 154 insertions(+), 4 deletions(-) create mode 100644 coldfront/core/user/templates/request_hub/removal_request_list.html diff --git a/coldfront/core/user/templates/request_hub/removal_request_list.html b/coldfront/core/user/templates/request_hub/removal_request_list.html new file mode 100644 index 000000000..e031b1111 --- /dev/null +++ b/coldfront/core/user/templates/request_hub/removal_request_list.html @@ -0,0 +1,107 @@ +
    +
    + {% if queryset %} +
    +
    +
    {{ adj }} Project Removal Requests
    +
    + + + + + + + + + + + + + + {% for removal_request in queryset %} + + {% if show_all %} + + {% else %} + + {% endif %} + {% if adj == 'active' %} + + {% else %} + + {% endif %} + + + + + + + + {% endfor %} + +
    + # + + + + {% if adj == 'active' %} + Date Requested + {% else %} + Date Completed + {% endif %} + + + + User Email + + + + User + + + + Requester + + + + Project + + + + Status +
    {{ removal_request.pk }}{{ forloop.counter }}{{ removal_request.request_time|date:"M. d, Y" }}{{ removal_request.completion_time|date:"M. d, Y" }}{{ removal_request.project_user.user.email }}{{ removal_request.project_user.user.username }}{{ removal_request.requester.username }} + + {{ removal_request.project_user.project.name }} + + + {% with status=removal_request.status.name %} + {% if status == "Pending" %} + {{ status }} + {% elif status == "Processing" %} + {{ status }} + {% elif status == "Complete" %} + {{ status }} + {% endif %} + {% endwith %} +
    + + {% with pag_obj=queryset %} + {% include 'common/pagination.html' %} + {% endwith %} + +
    +
    +
    + {% else %} + {% if adj == 'active' %} +
    + No active project removal requests! +
    + {% else %} +
    + No completed project removal requests! +
    + {% endif %} + {% endif %} +
    +
    \ No newline at end of file diff --git a/coldfront/core/user/templates/request_hub/request_hub.html b/coldfront/core/user/templates/request_hub/request_hub.html index f64077267..b63ead754 100644 --- a/coldfront/core/user/templates/request_hub/request_hub.html +++ b/coldfront/core/user/templates/request_hub/request_hub.html @@ -10,10 +10,10 @@

    Request Hub

    -
    +
    - + -
    +
    {% with queryset=cluster_account_list_active adj='active' page_num='0' %} {% include 'request_hub/cluster_account_list.html' %} {% endwith %} - {% with queryset=cluster_account_list_complete adj='complete' page_num='1' %} + {% with queryset=cluster_account_list_complete adj='completed' page_num='1' %} {% include 'request_hub/cluster_account_list.html' %} {% endwith %} @@ -53,4 +53,47 @@

    Request Hub

    +
    +
    +
    + + +
    + + Project Removal Requests + {% if num_removal_request_active %} + + + + {{ num_active_cluster_account_requests }} active requests + {% endif %} + +
    + +     +
    +
    +
    + + {% with queryset=removal_request_active adj='active' page_num='2'%} + {% include 'request_hub/removal_request_list.html' %} + {% endwith %} + + {% with queryset=removal_request_complete adj='completed' page_num='3' %} + {% include 'request_hub/removal_request_list.html' %} + {% endwith %} + + {% if user.is_superuser %} +
    + + + Go To Project Removal Requests Main Page + +
    + {% endif %} +
    +
    +
    +
    + {% endblock %} From a60bec5925788874e516aa44caf6a11e3bf10137 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Fri, 25 Feb 2022 20:42:42 -0500 Subject: [PATCH 066/150] exported pagination code to separate html --- coldfront/templates/common/pagination.html | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 coldfront/templates/common/pagination.html diff --git a/coldfront/templates/common/pagination.html b/coldfront/templates/common/pagination.html new file mode 100644 index 000000000..8dc73231a --- /dev/null +++ b/coldfront/templates/common/pagination.html @@ -0,0 +1,13 @@ +Page {{ pag_obj.number }} of {{ pag_obj.paginator.num_pages }} +
      + {% if pag_obj.has_previous %} +
    • Previous
      • + {% else %} +
    • Previous
      • + {% endif %} + {% if pag_obj.has_next %} +
    • Next
      • + {% else %} +
    • Next
      • + {% endif %} +
    From 64302ad236627e202c64db77d62d0f5e25816033 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Fri, 25 Feb 2022 20:43:17 -0500 Subject: [PATCH 067/150] show forloop counter instead of pk if not superuser --- .../request_hub/cluster_account_list.html | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/coldfront/core/user/templates/request_hub/cluster_account_list.html b/coldfront/core/user/templates/request_hub/cluster_account_list.html index c7c699c51..434de22b1 100644 --- a/coldfront/core/user/templates/request_hub/cluster_account_list.html +++ b/coldfront/core/user/templates/request_hub/cluster_account_list.html @@ -45,7 +45,11 @@
    {{ adj }} Cluste {% for cluster_account in queryset %} - {{ cluster_account.pk }} + {% if show_all %} + {{ cluster_account.pk }} + {% else %} + {{ forloop.counter }} + {% endif %} {{ cluster_account.modified|date:"M. d, Y" }} {{ cluster_account.allocation_user.user.email }} @@ -85,20 +89,9 @@
    {{ adj }} Cluste {% endfor %} - - Page {{ queryset.number }} of {{ queryset.paginator.num_pages }} -
      - {% if queryset.has_previous %} -
    • Previous
      • - {% else %} -
    • Previous
      • - {% endif %} - {% if queryset.has_next %} -
    • Next
      • - {% else %} -
    • Next
      • - {% endif %} -
    + {% with pag_obj=queryset %} + {% include 'common/pagination.html' %} + {% endwith %}
    From 910dbc4128d349c35f9ff927b419300a629a37fd Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Sat, 26 Feb 2022 00:08:19 -0500 Subject: [PATCH 068/150] added script to maintain scroll position --- .../templates/request_hub/request_hub.html | 104 ++++-------------- 1 file changed, 19 insertions(+), 85 deletions(-) diff --git a/coldfront/core/user/templates/request_hub/request_hub.html b/coldfront/core/user/templates/request_hub/request_hub.html index b63ead754..9c6c163a0 100644 --- a/coldfront/core/user/templates/request_hub/request_hub.html +++ b/coldfront/core/user/templates/request_hub/request_hub.html @@ -10,90 +10,24 @@

    Request Hub

    -
    -
    -
    - - -
    - - Cluster Account Requests - {% if num_active_cluster_account_requests %} - - - - {{ num_active_cluster_account_requests }} active requests - {% endif %} - -
    - -     -
    -
    -
    - - {% with queryset=cluster_account_list_active adj='active' page_num='0' %} - {% include 'request_hub/cluster_account_list.html' %} - {% endwith %} - - {% with queryset=cluster_account_list_complete adj='completed' page_num='1' %} - {% include 'request_hub/cluster_account_list.html' %} - {% endwith %} - - {% if user.is_superuser %} -
    - - - Go To Cluster Account Requests Main Page - -
    - {% endif %} -
    -
    -
    -
    - -
    -
    -
    - - -
    - - Project Removal Requests - {% if num_removal_request_active %} - - - - {{ num_active_cluster_account_requests }} active requests - {% endif %} - -
    - -     -
    -
    -
    - - {% with queryset=removal_request_active adj='active' page_num='2'%} - {% include 'request_hub/removal_request_list.html' %} - {% endwith %} - - {% with queryset=removal_request_complete adj='completed' page_num='3' %} - {% include 'request_hub/removal_request_list.html' %} - {% endwith %} - - {% if user.is_superuser %} -
    - - - Go To Project Removal Requests Main Page - -
    - {% endif %} -
    -
    -
    -
    + {% with num=0 title='Cluster Account Requests' num_active=num_active_cluster_account_requests list_template='request_hub/cluster_account_list.html' active_queryset=cluster_account_list_active complete_queryset=cluster_account_list_complete button_path='allocation-cluster-account-request-list' button_text='Go To Cluster Account Requests Main Page' %} + {% include 'request_hub/request_section.html' %} + {% endwith %} + + {% with num=2 title='Project Removal Requests' num_active=num_removal_request_active list_template='request_hub/removal_request_list.html' active_queryset=removal_request_active complete_queryset=removal_request_complete button_path='project-removal-request-list' button_text='Go To Project Removal Requests Main Page' %} + {% include 'request_hub/request_section.html' %} + {% endwith %} + + {% endblock %} From d8fcac4520e840208994a2cf229df095bbcacb77 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Sat, 26 Feb 2022 00:08:45 -0500 Subject: [PATCH 069/150] initial commit --- .../request_hub/request_section.html | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 coldfront/core/user/templates/request_hub/request_section.html diff --git a/coldfront/core/user/templates/request_hub/request_section.html b/coldfront/core/user/templates/request_hub/request_section.html new file mode 100644 index 000000000..87d8161b0 --- /dev/null +++ b/coldfront/core/user/templates/request_hub/request_section.html @@ -0,0 +1,64 @@ +
    +
    +
    + + +
    + + {{ title}} + {% if num_active %} + + + + {{ num_active }} active requests + {% endif %} + +
    + +     +
    +
    +
    + + {% with queryset=active_queryset adj='active' page_num=num %} + {% include list_template %} + {% endwith %} + + {% with queryset=complete_queryset adj='completed' page_num=num|add:1 %} + {% include list_template %} + {% endwith %} + + {% if user.is_superuser %} +
    + + + {{ button_text }} + +
    + {% endif %} +
    +
    +
    +
    + + From a6944e5a841aa09c76f2fd297b712baafc97a15b Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Sat, 26 Feb 2022 10:32:39 -0500 Subject: [PATCH 070/150] created obj to store values used in templates --- .../templates/request_hub/request_hub.html | 8 +- .../request_hub/request_section.html | 24 +-- .../core/user/views_/request_hub_views.py | 154 ++++++++++-------- 3 files changed, 99 insertions(+), 87 deletions(-) diff --git a/coldfront/core/user/templates/request_hub/request_hub.html b/coldfront/core/user/templates/request_hub/request_hub.html index 9c6c163a0..7437dd74a 100644 --- a/coldfront/core/user/templates/request_hub/request_hub.html +++ b/coldfront/core/user/templates/request_hub/request_hub.html @@ -10,13 +10,13 @@

    Request Hub

    - {% with num=0 title='Cluster Account Requests' num_active=num_active_cluster_account_requests list_template='request_hub/cluster_account_list.html' active_queryset=cluster_account_list_active complete_queryset=cluster_account_list_complete button_path='allocation-cluster-account-request-list' button_text='Go To Cluster Account Requests Main Page' %} + {% with request_obj=cluster_account_request_obj %} {% include 'request_hub/request_section.html' %} {% endwith %} - {% with num=2 title='Project Removal Requests' num_active=num_removal_request_active list_template='request_hub/removal_request_list.html' active_queryset=removal_request_active complete_queryset=removal_request_complete button_path='project-removal-request-list' button_text='Go To Project Removal Requests Main Page' %} - {% include 'request_hub/request_section.html' %} - {% endwith %} +{# {% with num=2 title='Project Removal Requests' num_active=num_active_removal_request list_template='request_hub/removal_request_list.html' active_queryset=removal_request_active complete_queryset=removal_request_complete button_path='project-removal-request-list' button_text='Go To Project Removal Requests Main Page' %}#} +{# {% include 'request_hub/request_section.html' %}#} +{# {% endwith %}#} {% endblock %} diff --git a/coldfront/templates/common/authorized_navbar.html b/coldfront/templates/common/authorized_navbar.html index fb18dafea..3a486beb8 100644 --- a/coldfront/templates/common/authorized_navbar.html +++ b/coldfront/templates/common/authorized_navbar.html @@ -30,7 +30,7 @@ -
  • +
    • {% if request.user.is_superuser %} From f2da3322c6f1adb51fdd74f7bd088b7183cd3dff Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Sat, 26 Feb 2022 11:19:28 -0500 Subject: [PATCH 074/150] removed sorting options, added title to list template include --- .../request_hub/cluster_account_list.html | 10 ------- .../request_hub/removal_request_list.html | 26 +++---------------- 2 files changed, 4 insertions(+), 32 deletions(-) diff --git a/coldfront/core/user/templates/request_hub/cluster_account_list.html b/coldfront/core/user/templates/request_hub/cluster_account_list.html index 434de22b1..9761e2700 100644 --- a/coldfront/core/user/templates/request_hub/cluster_account_list.html +++ b/coldfront/core/user/templates/request_hub/cluster_account_list.html @@ -11,28 +11,18 @@
    {{ adj }} Cluste # - - Date Requested/
    Last Modified - - User Email - - Cluster Username - - Project - - Allocation diff --git a/coldfront/core/user/templates/request_hub/removal_request_list.html b/coldfront/core/user/templates/request_hub/removal_request_list.html index e031b1111..aefd47bed 100644 --- a/coldfront/core/user/templates/request_hub/removal_request_list.html +++ b/coldfront/core/user/templates/request_hub/removal_request_list.html @@ -3,15 +3,13 @@ {% if queryset %}
    -
    {{ adj }} Project Removal Requests
    +
    {{ adj }} {{ title }}
    - - -
    # - - {% if adj == 'active' %} @@ -19,28 +17,18 @@
    {{ adj }} Projec {% else %} Date Completed {% endif %} - -
    		User Email - - User - - Requester - - Project - - Status @@ -93,15 +81,9 @@
    {{ adj }} Projec {% else %} - {% if adj == 'active' %} -
    - No active project removal requests! -
    - {% else %} -
    - No completed project removal requests! -
    - {% endif %} +
    + No {{ adj }} project removal requests! +
    {% endif %} \ No newline at end of file From 3f7b881f2756fcbbdf063149f932f3cdb8d7433a Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Sat, 26 Feb 2022 11:19:50 -0500 Subject: [PATCH 075/150] adding savio project requests --- .../savio_project_request_list.html | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 coldfront/core/user/templates/request_hub/savio_project_request_list.html diff --git a/coldfront/core/user/templates/request_hub/savio_project_request_list.html b/coldfront/core/user/templates/request_hub/savio_project_request_list.html new file mode 100644 index 000000000..87b4527c8 --- /dev/null +++ b/coldfront/core/user/templates/request_hub/savio_project_request_list.html @@ -0,0 +1,78 @@ +
    +
    + {% if queryset %} +
    +
    +
    {{ adj }} Savio Project Requests
    +
    + + + + + + + + + + + + + + + {% for savio_project_request in queryset %} + + + + + + + + + + + {% endfor %} + +
    + # + + Date Requested/
    Last Modified +     		+ Requester Email + + Project + + Allocation Type + Pooling? + PI + Status
    + + Project {{ savio_project_request.pk }} + {{ savio_project_request.pk }} + + {{ savio_project_request.modified|date:"M. d, Y" }}{{ savio_project_request.requester.email }}{{ savio_project_request.project.name }}{{ savio_project_request.allocation_type }}{{ savio_project_request.pool }}{{ savio_project_request.pi.email }} + {% with status=savio_project_request.status.name %} + {% if status == 'Under Review' %} + {{ status }} + {% elif status == 'Approved - Processing' %} + {{ status }} + {% elif status == 'Approved - Complete' %} + {{ status }} + {% else %} + {{ status }} + {% endif %} + {% endwith %} +
    + + {% with pag_obj=queryset %} + {% include 'common/pagination.html' %} + {% endwith %} +
    +
    +
    + {% else %} +
    + No {{ adj }} Savio project requests! +
    + {% endif %} +
    +
    \ No newline at end of file From 8608b62331a4b0ee76f4fa0f665d5e6b48745c26 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Sat, 26 Feb 2022 11:20:06 -0500 Subject: [PATCH 076/150] adding savio project requests --- .../core/user/views_/request_hub_views.py | 64 +++++++++++++------ 1 file changed, 45 insertions(+), 19 deletions(-) diff --git a/coldfront/core/user/views_/request_hub_views.py b/coldfront/core/user/views_/request_hub_views.py index a143bdd4b..ef291d2e0 100644 --- a/coldfront/core/user/views_/request_hub_views.py +++ b/coldfront/core/user/views_/request_hub_views.py @@ -24,7 +24,8 @@ from coldfront.core.project.models import (Project, ProjectUserStatusChoice, ProjectUserRemovalRequest, - ProjectUserRemovalRequestStatusChoice) + ProjectUserRemovalRequestStatusChoice, + SavioProjectAllocationRequest) from coldfront.core.project.utils_.removal_utils import ProjectRemovalRequestRunner from coldfront.core.utils.common import (import_from_settings, utc_now_offset_aware) @@ -60,7 +61,6 @@ class RequestHub(LoginRequiredMixin, paginate_by = 5 paginators = 0 show_all_requests = False - cur_num = 0 def create_paginator(self, queryset): """ @@ -88,17 +88,16 @@ def get_cluster_account_request(self): cluster_account_status = AllocationAttributeType.objects.get( name='Cluster Account Status') + kwargs = {'allocation_user__user': user, + 'allocation_attribute_type': cluster_account_status} cluster_account_list_complete = AllocationUserAttribute.objects.filter( - allocation_attribute_type=cluster_account_status, - value__in=['Denied', 'Active'], - allocation_user__user=user) + value__in=['Denied', 'Active'], **kwargs) cluster_account_list_active = AllocationUserAttribute.objects.filter( - allocation_attribute_type=cluster_account_status, - value__in=['Pending - Add', 'Processing'], - allocation_user__user=user) + value__in=['Pending - Add', 'Processing'], **kwargs) + cluster_request_object.num = self.paginators cluster_request_object.active_queryset = \ self.create_paginator(cluster_account_list_active) @@ -114,8 +113,6 @@ def get_cluster_account_request(self): 'allocation-cluster-account-request-list' cluster_request_object.button_text = \ 'Go To Cluster Account Requests Main Page' - cluster_request_object.num = self.cur_num - self.cur_num += 2 return cluster_request_object @@ -124,17 +121,15 @@ def get_project_removal_request(self): removal_request_object = RequestListItem() user = self.request.user - project_user_cond = Q(project_user__user=user) - requester_cond = Q(requester=user) + args = [Q(project_user__user=user) | Q(requester=user)] removal_request_active = ProjectUserRemovalRequest.objects.filter( - status__name__in=['Pending', 'Processing']).\ - filter(project_user_cond | requester_cond) + status__name__in=['Pending', 'Processing'], *args) removal_request_complete = ProjectUserRemovalRequest.objects.filter( - status__name='Complete').\ - filter(project_user_cond | requester_cond) + status__name='Complete', *args) + removal_request_object.num = self.paginators removal_request_object.active_queryset = \ self.create_paginator(removal_request_active) @@ -150,16 +145,47 @@ def get_project_removal_request(self): 'project-removal-request-list' removal_request_object.button_text = \ 'Go To Project Removal Requests Main Page' - removal_request_object.num = self.cur_num - self.cur_num += 2 return removal_request_object + def get_savio_project_request(self): + """Populates a RequestListItem with data for savio project requests""" + savio_proj_request_object = RequestListItem() + user = self.request.user + + args = [Q(pi=user) | Q(requester=user)] + + project_request_active = SavioProjectAllocationRequest.objects.filter( + status__name__in=['Under Review', 'Approved - Processing'], *args) + + project_request_complete = SavioProjectAllocationRequest.objects.filter( + status__name__in=['Approved - Complete', 'Denied'], *args) + + savio_proj_request_object.num = self.paginators + savio_proj_request_object.active_queryset = \ + self.create_paginator(project_request_active) + + savio_proj_request_object.complete_queryset = \ + self.create_paginator(project_request_complete) + + savio_proj_request_object.num_active = project_request_active.count() + + savio_proj_request_object.title = 'Savio Project Requests' + savio_proj_request_object.list_template = \ + 'request_hub/savio_project_request_list.html' + savio_proj_request_object.button_path = \ + 'savio-project-pending-request-list' + savio_proj_request_object.button_text = \ + 'Go To Savio Project Requests Main Page' + + return savio_proj_request_object + def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) requests = ['cluster_account_request', - 'project_removal_request'] + 'project_removal_request', + 'savio_project_request'] for request in requests: context[f'{request}_obj'] = eval(f'self.get_{request}()') From 81893d05f6eacce5d5c6101b6c23628f24e245d2 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Sat, 26 Feb 2022 11:21:39 -0500 Subject: [PATCH 077/150] added title vairable to list template --- .../core/user/templates/request_hub/request_section.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/coldfront/core/user/templates/request_hub/request_section.html b/coldfront/core/user/templates/request_hub/request_section.html index bb087c8e1..ec2576957 100644 --- a/coldfront/core/user/templates/request_hub/request_section.html +++ b/coldfront/core/user/templates/request_hub/request_section.html @@ -17,14 +17,14 @@ -
    +
    - {% with queryset=request_obj.active_queryset adj='active' page_num=request_obj.num %} + {% with queryset=request_obj.active_queryset adj='active' page_num=request_obj.num title=request_obj.title %} {% include request_obj.list_template %} {% endwith %} - {% with queryset=request_obj.complete_queryset adj='completed' page_num=request_obj.num|add:1 %} + {% with queryset=request_obj.complete_queryset adj='completed' page_num=request_obj.num|add:1 title=request_obj.title %} {% include request_obj.list_template %} {% endwith %} From 5734fba7d1d3b7721450b2bbcb90a70a4a40ef2b Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Sat, 26 Feb 2022 11:22:02 -0500 Subject: [PATCH 078/150] adding savio project requests --- coldfront/core/user/templates/request_hub/request_hub.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/coldfront/core/user/templates/request_hub/request_hub.html b/coldfront/core/user/templates/request_hub/request_hub.html index 59aafcef0..bac2c11f9 100644 --- a/coldfront/core/user/templates/request_hub/request_hub.html +++ b/coldfront/core/user/templates/request_hub/request_hub.html @@ -18,6 +18,10 @@

    Request Hub

    {% include 'request_hub/request_section.html' %} {% endwith %} + {% with request_obj=savio_project_request_obj %} + {% include 'request_hub/request_section.html' %} + {% endwith %} + {% endblock %} From f833049bb72ff9a5e3438fd7ac1e9cf54d3f3e5e Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Sat, 26 Feb 2022 16:32:16 -0500 Subject: [PATCH 082/150] moved collapse script to request_hub.html --- .../request_hub/request_section.html | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/coldfront/core/user/templates/request_hub/request_section.html b/coldfront/core/user/templates/request_hub/request_section.html index ec2576957..ea85484c4 100644 --- a/coldfront/core/user/templates/request_hub/request_section.html +++ b/coldfront/core/user/templates/request_hub/request_section.html @@ -41,24 +41,3 @@
    - From d27c75a0941588069ea0407ad502f46881952bff Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Sat, 26 Feb 2022 16:32:52 -0500 Subject: [PATCH 083/150] added request hub to admin dropdown --- coldfront/templates/common/navbar_admin.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/coldfront/templates/common/navbar_admin.html b/coldfront/templates/common/navbar_admin.html index 114185bea..a8b729388 100644 --- a/coldfront/templates/common/navbar_admin.html +++ b/coldfront/templates/common/navbar_admin.html @@ -13,6 +13,9 @@
  • Requests
      +
    • + View All Requests +
    • Cluster Account Requests
      • From 58ef0450a466ef33087efab974f7e1078c4f79f8 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Sat, 26 Feb 2022 17:20:40 -0500 Subject: [PATCH 084/150] added support for vector proj requests --- .../templates/request_hub/request_hub.html | 24 ++++++++- .../core/user/views_/request_hub_views.py | 51 ++++++++++++++++--- 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/coldfront/core/user/templates/request_hub/request_hub.html b/coldfront/core/user/templates/request_hub/request_hub.html index f0499cefb..f0847e6a6 100644 --- a/coldfront/core/user/templates/request_hub/request_hub.html +++ b/coldfront/core/user/templates/request_hub/request_hub.html @@ -2,10 +2,26 @@
      -

      Request Hub

      +

      Request Hub

      + {% if show_all %} +

      + Below are all of the requests in the myBRC Portal. Click on a request + section to view requests of that type. To perform actions on a specific request, + click the button to go to the request's main page and perform the actions there. + Click here to view only your requests. +

      + {% else %} +

      + Below are all of your requests in the myBRC Portal. Click on a request + section to view your requests of that type. -

      Below are all your active requests. To view your completed requests, click to view completed requests in the appropriate section

      + {% if admin_staff %} + Click here to view all requests in the myBRC Portal. + {% endif %} +

      + {% endif %} +
      @@ -22,6 +38,10 @@

      Request Hub

      {% include 'request_hub/request_section.html' %} {% endwith %} + {% with request_obj=vector_project_request_obj %} + {% include 'request_hub/request_section.html' %} + {% endwith %} + diff --git a/coldfront/core/user/templates/request_hub/request_section.html b/coldfront/core/user/templates/request_hub/request_section.html index 7e63503d9..9fca67180 100644 --- a/coldfront/core/user/templates/request_hub/request_section.html +++ b/coldfront/core/user/templates/request_hub/request_section.html @@ -17,7 +17,7 @@
    • -
    +
    {% with queryset=request_obj.active_queryset adj='active' page_num=request_obj.num title=request_obj.title %} From 3a586a6ea57c150159f5ebdb3bd9cc927e6bf803 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Sat, 26 Feb 2022 23:21:41 -0500 Subject: [PATCH 105/150] added order_by to all queries --- .../core/user/views_/request_hub_views.py | 107 +++++++++++------- 1 file changed, 67 insertions(+), 40 deletions(-) diff --git a/coldfront/core/user/views_/request_hub_views.py b/coldfront/core/user/views_/request_hub_views.py index dcbdd2d16..0442a3d11 100644 --- a/coldfront/core/user/views_/request_hub_views.py +++ b/coldfront/core/user/views_/request_hub_views.py @@ -69,11 +69,15 @@ def get_cluster_account_request(self): if not self.show_all_requests: kwargs['allocation_user__user'] = user - cluster_account_list_complete = AllocationUserAttribute.objects.filter( - value__in=['Denied', 'Active'], **kwargs) + cluster_account_list_complete = \ + AllocationUserAttribute.objects.filter( + value__in=['Denied', 'Active'], **kwargs).order_by( + 'modified') - cluster_account_list_active = AllocationUserAttribute.objects.filter( - value__in=['Pending - Add', 'Processing'], **kwargs) + cluster_account_list_active = \ + AllocationUserAttribute.objects.filter( + value__in=['Pending - Add', 'Processing'], **kwargs).order_by( + 'modified') cluster_request_object.num = self.paginators cluster_request_object.active_queryset = \ @@ -103,11 +107,15 @@ def get_project_removal_request(self): if not self.show_all_requests: args.append(Q(project_user__user=user) | Q(requester=user)) - removal_request_active = ProjectUserRemovalRequest.objects.filter( - status__name__in=['Pending', 'Processing'], *args) + removal_request_active = \ + ProjectUserRemovalRequest.objects.filter( + status__name__in=['Pending', 'Processing'], *args).order_by( + 'modified') - removal_request_complete = ProjectUserRemovalRequest.objects.filter( - status__name='Complete', *args) + removal_request_complete = \ + ProjectUserRemovalRequest.objects.filter( + status__name='Complete', *args).order_by( + 'modified') removal_request_object.num = self.paginators removal_request_object.active_queryset = \ @@ -137,11 +145,17 @@ def get_savio_project_request(self): if not self.show_all_requests: args.append(Q(pi=user) | Q(requester=user)) - project_request_active = SavioProjectAllocationRequest.objects.filter( - status__name__in=['Under Review', 'Approved - Processing'], *args) + project_request_active = \ + SavioProjectAllocationRequest.objects.filter( + status__name__in=['Under Review', 'Approved - Processing'], + *args).order_by( + 'modified') - project_request_complete = SavioProjectAllocationRequest.objects.filter( - status__name__in=['Approved - Complete', 'Denied'], *args) + project_request_complete = \ + SavioProjectAllocationRequest.objects.filter( + status__name__in=['Approved - Complete', 'Denied'], + *args).order_by( + 'modified') savio_proj_request_object.num = self.paginators savio_proj_request_object.active_queryset = \ @@ -171,11 +185,17 @@ def get_vector_project_request(self): if not self.show_all_requests: args.append(Q(pi=user) | Q(requester=user)) - project_request_active = VectorProjectAllocationRequest.objects.filter( - status__name__in=['Under Review', 'Approved - Processing'], *args) + project_request_active = \ + VectorProjectAllocationRequest.objects.filter( + status__name__in=['Under Review', 'Approved - Processing'], + *args).order_by( + 'modified') - project_request_complete = VectorProjectAllocationRequest.objects.filter( - status__name__in=['Approved - Complete', 'Denied'], *args) + project_request_complete = \ + VectorProjectAllocationRequest.objects.filter( + status__name__in=['Approved - Complete', 'Denied'], + *args).order_by( + 'modified') vector_proj_request_object.num = self.paginators vector_proj_request_object.active_queryset = \ @@ -205,11 +225,16 @@ def get_project_join_request(self): if not self.show_all_requests: args.append(Q(project_user__user=user)) - project_join_request_active = ProjectUserJoinRequest.objects.filter( - project_user__status__name='Pending - Add', *args) + project_join_request_active = \ + ProjectUserJoinRequest.objects.filter( + project_user__status__name='Pending - Add', *args).order_by( + 'modified') - project_join_request_complete = ProjectUserJoinRequest.objects.filter( - project_user__status__name__in=['Active', 'Denied'], *args) + project_join_request_complete = \ + ProjectUserJoinRequest.objects.filter( + project_user__status__name__in=['Active', 'Denied'], + *args).order_by( + 'modified') proj_join_request_object.num = self.paginators proj_join_request_object.active_queryset = \ @@ -239,11 +264,15 @@ def get_project_renewal_request(self): if not self.show_all_requests: args.append(Q(requester=user) | Q(pi=user)) - project_renewal_request_active = AllocationRenewalRequest.objects.filter( - status__name__in=['Approved', 'Under Review'], *args) + project_renewal_request_active = \ + AllocationRenewalRequest.objects.filter( + status__name__in=['Approved', 'Under Review'], *args).order_by( + 'modified') - project_renewal_request_complete = AllocationRenewalRequest.objects.filter( - status__name__in=['Complete', 'Denied'], *args) + project_renewal_request_complete = \ + AllocationRenewalRequest.objects.filter( + status__name__in=['Complete', 'Denied'], *args).order_by( + 'modified') proj_renewal_request_object.num = self.paginators proj_renewal_request_object.active_queryset = \ @@ -269,33 +298,33 @@ def get_su_purchase_request(self): su_pruchase_request_object = RequestListItem() user = self.request.user - su_pruchase_request_active = AllocationAdditionRequest.objects.filter( - status__name__in=['Under Review']) + su_purchase_request_active = AllocationAdditionRequest.objects.filter( + status__name__in=['Under Review']).order_by('modified') - su_pruchase_request_complete = AllocationAdditionRequest.objects.filter( - status__name__in=['Complete', 'Denied']) + su_purchase_request_complete = AllocationAdditionRequest.objects.filter( + status__name__in=['Complete', 'Denied']).order_by('modified') if not self.show_all_requests: request_ids = [ - r.id for r in su_pruchase_request_active + r.id for r in su_purchase_request_active if is_user_manager_or_pi_of_project(user, r.project)] - su_pruchase_request_active = \ - su_pruchase_request_active.filter(id__in=request_ids) + su_purchase_request_active = \ + su_purchase_request_active.filter(id__in=request_ids) request_ids = [ - r.id for r in su_pruchase_request_complete + r.id for r in su_purchase_request_complete if is_user_manager_or_pi_of_project(user, r.project)] - su_pruchase_request_complete = \ - su_pruchase_request_complete.filter(id__in=request_ids) + su_purchase_request_complete = \ + su_purchase_request_complete.filter(id__in=request_ids) su_pruchase_request_object.num = self.paginators su_pruchase_request_object.active_queryset = \ - self.create_paginator(su_pruchase_request_active) + self.create_paginator(su_purchase_request_active) su_pruchase_request_object.complete_queryset = \ - self.create_paginator(su_pruchase_request_complete) + self.create_paginator(su_purchase_request_complete) - su_pruchase_request_object.num_active = su_pruchase_request_active.count() + su_pruchase_request_object.num_active = su_purchase_request_active.count() su_pruchase_request_object.title = 'Service Unit Purchase Requests' su_pruchase_request_object.table = \ @@ -322,12 +351,10 @@ def get_context_data(self, **kwargs): context[f'{request}_obj'] = eval(f'self.get_{request}()') context['show_all'] = ((self.request.user.is_superuser or - self.request.user.is_staff) and + self.request.user.is_staff) and self.show_all_requests) context['admin_staff'] = (self.request.user.is_superuser or self.request.user.is_staff) - context['pi_manager'] = None - return context From 23341e670a0ad886c656e576c4a2145949969bc0 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Sat, 26 Feb 2022 23:35:25 -0500 Subject: [PATCH 106/150] moved buttons --- .../templates/request_hub/request_hub.html | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/coldfront/core/user/templates/request_hub/request_hub.html b/coldfront/core/user/templates/request_hub/request_hub.html index 80abcca40..474d856d1 100644 --- a/coldfront/core/user/templates/request_hub/request_hub.html +++ b/coldfront/core/user/templates/request_hub/request_hub.html @@ -23,25 +23,26 @@

    Request Hub

    -
    - - -
    -
    +
    +
    +
    + + +
    +
    +
    + {% with request_obj=cluster_account_request_obj %} {% include 'request_hub/request_section.html' %} From 91609724d451348a21ba3c38cb88e0cbe47e0755 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Sat, 26 Feb 2022 23:35:46 -0500 Subject: [PATCH 107/150] fixed wrong href in button --- coldfront/core/user/templates/request_hub/request_section.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coldfront/core/user/templates/request_hub/request_section.html b/coldfront/core/user/templates/request_hub/request_section.html index 9fca67180..b62aa99cd 100644 --- a/coldfront/core/user/templates/request_hub/request_section.html +++ b/coldfront/core/user/templates/request_hub/request_section.html @@ -1,7 +1,7 @@
    - +
    From 749ce698d9620d7445d07bbae632994b287a2437 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Sat, 26 Feb 2022 23:38:25 -0500 Subject: [PATCH 108/150] changed name of request hub --- coldfront/templates/common/navbar_admin.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coldfront/templates/common/navbar_admin.html b/coldfront/templates/common/navbar_admin.html index a8b729388..4b3f539ac 100644 --- a/coldfront/templates/common/navbar_admin.html +++ b/coldfront/templates/common/navbar_admin.html @@ -14,7 +14,7 @@
  • Requests
    • - From 65210ae97767cbce4a4194417e5096815e4fd09a Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Sat, 26 Feb 2022 23:47:13 -0500 Subject: [PATCH 113/150] fixed newlines --- coldfront/core/user/templates/request_hub/request_hub.html | 3 --- 1 file changed, 3 deletions(-) diff --git a/coldfront/core/user/templates/request_hub/request_hub.html b/coldfront/core/user/templates/request_hub/request_hub.html index 474d856d1..f7c055696 100644 --- a/coldfront/core/user/templates/request_hub/request_hub.html +++ b/coldfront/core/user/templates/request_hub/request_hub.html @@ -20,14 +20,12 @@

    Request Hub

    Click     here to view all requests in the myBRC Portal. {% endif %} {% endif %} -
    -
    @@ -43,7 +41,6 @@

    Request Hub

    - {% with request_obj=cluster_account_request_obj %} {% include 'request_hub/request_section.html' %} {% endwith %} From 34729f896003a010c9d9b206eef48fdbee5e0d7c Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Sun, 27 Feb 2022 01:10:46 -0500 Subject: [PATCH 114/150] changed pag_obj to page_obj. added conditional to pagination.html --- .../user/templates/request_hub/request_section.html | 4 ++-- coldfront/templates/common/pagination.html | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/coldfront/core/user/templates/request_hub/request_section.html b/coldfront/core/user/templates/request_hub/request_section.html index d59d37147..781ff4f4b 100644 --- a/coldfront/core/user/templates/request_hub/request_section.html +++ b/coldfront/core/user/templates/request_hub/request_section.html @@ -25,7 +25,7 @@
    {{ adj }} {{ title }}
    {% include request_obj.table %} - {% with pag_obj=queryset %} + {% with page_obj=queryset %} {% include 'common/pagination.html' %} {% endwith %}
    @@ -48,7 +48,7 @@
    {{ adj }} {{ tit
    {{ adj }} {{ title }}
    {% include request_obj.table %} - {% with pag_obj=queryset %} + {% with page_obj=queryset %} {% include 'common/pagination.html' %} {% endwith %}
    diff --git a/coldfront/templates/common/pagination.html b/coldfront/templates/common/pagination.html index 8dc73231a..00d7cbc83 100644 --- a/coldfront/templates/common/pagination.html +++ b/coldfront/templates/common/pagination.html @@ -1,13 +1,14 @@ -Page {{ pag_obj.number }} of {{ pag_obj.paginator.num_pages }} +{% if is_paginated %} Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }}
      - {% if pag_obj.has_previous %} -
    • Previous
      • + {% if page_obj.has_previous %} +
    • Previous
      • {% else %}
    • Previous
      • {% endif %} - {% if pag_obj.has_next %} -
    • Next
      • + {% if page_obj.has_next %} +
    • Next
      • {% else %}
    • Next
      • {% endif %}
    +{% endif %} From 2202c0efe9d402cce7734acf0c793d6411861f72 Mon Sep 17 00:00:00 2001 From: JoJo Feinstein Date: Mon, 28 Feb 2022 10:00:32 -0500 Subject: [PATCH 115/150] removed if paginated as it didnt work in request hub --- coldfront/templates/common/pagination.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/coldfront/templates/common/pagination.html b/coldfront/templates/common/pagination.html index 00d7cbc83..39294c611 100644 --- a/coldfront/templates/common/pagination.html +++ b/coldfront/templates/common/pagination.html @@ -1,4 +1,4 @@ -{% if is_paginated %} Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }} +Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }}
      {% if page_obj.has_previous %}
    • Previous
      • @@ -11,4 +11,3 @@
    • Next
      • {% endif %}
    -{% endif %} From 4fc20b1fb6131b6a1c4f824b01344aa2d4c7ed53 Mon Sep 17 00:00:00 2001 From: JoJo Feinstein Date: Mon, 28 Feb 2022 10:00:56 -0500 Subject: [PATCH 116/150] refactored expand/collapse all scripts --- .../core/user/templates/request_hub/request_hub.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/coldfront/core/user/templates/request_hub/request_hub.html b/coldfront/core/user/templates/request_hub/request_hub.html index f7c055696..ef17e6cfc 100644 --- a/coldfront/core/user/templates/request_hub/request_hub.html +++ b/coldfront/core/user/templates/request_hub/request_hub.html @@ -1,4 +1,6 @@ -{% extends "common/base.html" %} {% load common_tags %} {% block content %} +{% extends "common/base.html" %} +{% load common_tags %} +{% block content %}

    Request Hub

    @@ -104,15 +106,13 @@

    Request Hub

    if (scrollpos && resetpos) window.scrollTo(0, scrollpos); $(document).on('click', '#expand_all_button', function() { - var matches = document.querySelectorAll("[id^='collapse-']"); - matches.forEach(function (item, index) { + document.querySelectorAll("[id^='collapse-']").forEach(function (item, index) { $(item).collapse("show"); }); }); $(document).on('click', '#collapse_all_button', function() { - var matches = document.querySelectorAll("[id^='collapse-']"); - matches.forEach(function (item, index) { + document.querySelectorAll("[id^='collapse-']").forEach(function (item, index) { $(item).collapse("hide"); }); }); From e9132a697cd355b6709337e2eb92426a0353f6ab Mon Sep 17 00:00:00 2001 From: Matthew Li Date: Mon, 28 Feb 2022 08:13:51 -0800 Subject: [PATCH 117/150] Update method signatures of SessionWizardView methods to match parent's --- coldfront/core/project/views.py | 6 +++--- .../core/project/views_/renewal_views/request_views.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/coldfront/core/project/views.py b/coldfront/core/project/views.py index bc9a2b1f4..1856430f5 100644 --- a/coldfront/core/project/views.py +++ b/coldfront/core/project/views.py @@ -1976,7 +1976,7 @@ def get_context_data(self, form, **kwargs): self.__set_data_from_previous_steps(current_step, context) return context - def get_form_kwargs(self, step): + def get_form_kwargs(self, step=None): kwargs = {} step = int(step) # The names of steps that require the past data. @@ -1995,7 +1995,7 @@ def get_form_kwargs(self, step): def get_template_names(self): return [self.TEMPLATES[self.FORMS[int(self.steps.current)][0]]] - def done(self, form_list, form_dict, **kwargs): + def done(self, form_list, **kwargs): """Perform processing and store information in a request object.""" redirect_url = '/' @@ -2003,7 +2003,7 @@ def done(self, form_list, form_dict, **kwargs): # Retrieve form data; include empty dictionaries for skipped steps. data = iter([form.cleaned_data for form in form_list]) form_data = [{} for _ in range(len(self.form_list))] - for step in sorted(form_dict.keys()): + for step in sorted(kwargs['form_dict'].keys()): form_data[int(step)] = next(data) request_kwargs = { diff --git a/coldfront/core/project/views_/renewal_views/request_views.py b/coldfront/core/project/views_/renewal_views/request_views.py index ce26653e3..12a17739a 100644 --- a/coldfront/core/project/views_/renewal_views/request_views.py +++ b/coldfront/core/project/views_/renewal_views/request_views.py @@ -180,7 +180,7 @@ def get_context_data(self, form, **kwargs): self.__set_data_from_previous_steps(current_step, context) return context - def get_form_kwargs(self, step): + def get_form_kwargs(self, step=None): kwargs = {} step = int(step) if step == self.step_numbers_by_form_name['pi_selection']: @@ -220,12 +220,12 @@ def get_form_kwargs(self, step): def get_template_names(self): return [self.TEMPLATES[self.FORMS[int(self.steps.current)][0]]] - def done(self, form_list, form_dict, **kwargs): + def done(self, form_list, **kwargs): """Perform processing and store information in a request object.""" redirect_url = '/' try: - form_data = self.__get_form_data(form_list, form_dict) + form_data = self.__get_form_data(form_list, kwargs['form_dict']) tmp = {} self.__set_data_from_previous_steps(len(self.FORMS), tmp) pi = tmp['PI'].user From a2078d3aa72315944c1535ed975eb5b36ba79649 Mon Sep 17 00:00:00 2001 From: Matthew Li Date: Mon, 28 Feb 2022 08:41:14 -0800 Subject: [PATCH 118/150] Update session settings to use a secure cookie for HTTPS and to increase session time to an hour --- bootstrap/ansible/cf_mybrc_settings_template.tmpl | 7 +++++++ coldfront/config/local_settings.py.sample | 3 +++ 2 files changed, 10 insertions(+) diff --git a/bootstrap/ansible/cf_mybrc_settings_template.tmpl b/bootstrap/ansible/cf_mybrc_settings_template.tmpl index de8a050af..8c3fbd13c 100644 --- a/bootstrap/ansible/cf_mybrc_settings_template.tmpl +++ b/bootstrap/ansible/cf_mybrc_settings_template.tmpl @@ -44,3 +44,10 @@ API_LOG_PATH = '{{ log_path }}/{{ api_log_file }}' # A list of admin email addresses to CC when certain requests are approved. REQUEST_APPROVAL_CC_LIST = {{ request_approval_cc_list }} + +# Use a secure cookie for the session cookie (HTTPS only). +{% if ssl_enabled | bool %} +SESSION_COOKIE_SECURE = True +{% else %} +SESSION_COOKIE_SECURE = False +{% endif %} diff --git a/coldfront/config/local_settings.py.sample b/coldfront/config/local_settings.py.sample index 1ed8941c2..a61bfc579 100644 --- a/coldfront/config/local_settings.py.sample +++ b/coldfront/config/local_settings.py.sample @@ -26,6 +26,9 @@ if DEBUG is False and SECRET_KEY == 'vtri&lztlbinerr4+yg1yzm23ez@+ub6=4*63z1%d!) # This should be set to True in production when using HTTPS SESSION_COOKIE_SECURE = False +# Sessions should last for one hour. +SESSION_COOKIE_AGE = 60 * 60 + # ------------------------------------------------------------------------------ # myBRC settings # ------------------------------------------------------------------------------ From dbe07749c686b93bc4b9223811a26ac757e34381 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Mon, 28 Feb 2022 13:18:27 -0500 Subject: [PATCH 119/150] changed active to pending --- .../request_hub/request_section.html | 6 +- .../core/user/views_/request_hub_views.py | 78 +++++++++---------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/coldfront/core/user/templates/request_hub/request_section.html b/coldfront/core/user/templates/request_hub/request_section.html index 781ff4f4b..77442d756 100644 --- a/coldfront/core/user/templates/request_hub/request_section.html +++ b/coldfront/core/user/templates/request_hub/request_section.html @@ -5,10 +5,10 @@
    {{ request_obj.title}} - {% if request_obj.num_active %} + {% if request_obj.num_pending %} - {{ request_obj.num_active }} active requests + {{ request_obj.num_pending }} pending requests {% endif %}
    @@ -16,7 +16,7 @@
    - {% with queryset=request_obj.active_queryset adj='active' page_num=request_obj.num title=request_obj.title %} + {% with queryset=request_obj.pending_queryset adj='pending' page_num=request_obj.num title=request_obj.title %}
    {% if queryset %} diff --git a/coldfront/core/user/views_/request_hub_views.py b/coldfront/core/user/views_/request_hub_views.py index 021f54ebb..5c050f412 100644 --- a/coldfront/core/user/views_/request_hub_views.py +++ b/coldfront/core/user/views_/request_hub_views.py @@ -21,16 +21,16 @@ class RequestListItem: in the request hub """ - __slots__ = ['num', 'title', 'num_active', 'table', - 'active_queryset', 'complete_queryset', + __slots__ = ['num', 'title', 'num_pending', 'table', + 'pending_queryset', 'complete_queryset', 'button_path', 'button_text'] def __init__(self): num = None title = None - num_active = None + num_pending = None table = None - active_queryset = None + pending_queryset = None complete_queryset = None button_path = None button_text = None @@ -79,19 +79,19 @@ def get_cluster_account_request(self): value__in=['Denied', 'Active'], **kwargs).order_by( 'modified') - cluster_account_list_active = \ + cluster_account_list_pending = \ AllocationUserAttribute.objects.filter( value__in=['Pending - Add', 'Processing'], **kwargs).order_by( 'modified') cluster_request_object.num = self.paginators - cluster_request_object.active_queryset = \ - self.create_paginator(cluster_account_list_active) + cluster_request_object.pending_queryset = \ + self.create_paginator(cluster_account_list_pending) cluster_request_object.complete_queryset = \ self.create_paginator(cluster_account_list_complete) - cluster_request_object.num_active = cluster_account_list_active.count() + cluster_request_object.num_pending = cluster_account_list_pending.count() cluster_request_object.title = 'Cluster Account Requests' cluster_request_object.table = \ @@ -112,7 +112,7 @@ def get_project_removal_request(self): if not self.show_all_requests: args.append(Q(project_user__user=user) | Q(requester=user)) - removal_request_active = \ + removal_request_pending = \ ProjectUserRemovalRequest.objects.filter( status__name__in=['Pending', 'Processing'], *args).order_by( 'modified') @@ -123,13 +123,13 @@ def get_project_removal_request(self): 'modified') removal_request_object.num = self.paginators - removal_request_object.active_queryset = \ - self.create_paginator(removal_request_active) + removal_request_object.pending_queryset = \ + self.create_paginator(removal_request_pending) removal_request_object.complete_queryset = \ self.create_paginator(removal_request_complete) - removal_request_object.num_active = removal_request_active.count() + removal_request_object.num_pending = removal_request_pending.count() removal_request_object.title = 'Project Removal Requests' removal_request_object.table = \ @@ -150,7 +150,7 @@ def get_savio_project_request(self): if not self.show_all_requests: args.append(Q(pi=user) | Q(requester=user)) - project_request_active = \ + project_request_pending = \ SavioProjectAllocationRequest.objects.filter( status__name__in=['Under Review', 'Approved - Processing'], *args).order_by( @@ -163,13 +163,13 @@ def get_savio_project_request(self): 'modified') savio_proj_request_object.num = self.paginators - savio_proj_request_object.active_queryset = \ - self.create_paginator(project_request_active) + savio_proj_request_object.pending_queryset = \ + self.create_paginator(project_request_pending) savio_proj_request_object.complete_queryset = \ self.create_paginator(project_request_complete) - savio_proj_request_object.num_active = project_request_active.count() + savio_proj_request_object.num_pending = project_request_pending.count() savio_proj_request_object.title = 'Savio Project Requests' savio_proj_request_object.table = \ @@ -190,7 +190,7 @@ def get_vector_project_request(self): if not self.show_all_requests: args.append(Q(pi=user) | Q(requester=user)) - project_request_active = \ + project_request_pending = \ VectorProjectAllocationRequest.objects.filter( status__name__in=['Under Review', 'Approved - Processing'], *args).order_by( @@ -203,13 +203,13 @@ def get_vector_project_request(self): 'modified') vector_proj_request_object.num = self.paginators - vector_proj_request_object.active_queryset = \ - self.create_paginator(project_request_active) + vector_proj_request_object.pending_queryset = \ + self.create_paginator(project_request_pending) vector_proj_request_object.complete_queryset = \ self.create_paginator(project_request_complete) - vector_proj_request_object.num_active = project_request_active.count() + vector_proj_request_object.num_pending = project_request_pending.count() vector_proj_request_object.title = 'Vector Project Requests' vector_proj_request_object.table = \ @@ -230,26 +230,26 @@ def get_project_join_request(self): if not self.show_all_requests: args.append(Q(project_user__user=user)) - project_join_request_active = \ + project_join_request_pending = \ ProjectUserJoinRequest.objects.filter( project_user__status__name='Pending - Add', *args).order_by( 'modified') project_join_request_complete = \ ProjectUserJoinRequest.objects.filter( - project_user__status__name__in=['Active', 'Denied'], + project_user__status__name__in=['pending', 'Denied'], *args).order_by( 'modified') proj_join_request_object.num = self.paginators - proj_join_request_object.active_queryset = \ - self.create_paginator(project_join_request_active) + proj_join_request_object.pending_queryset = \ + self.create_paginator(project_join_request_pending) proj_join_request_object.complete_queryset = \ self.create_paginator(project_join_request_complete) - proj_join_request_object.num_active = \ - project_join_request_active.count() + proj_join_request_object.num_pending = \ + project_join_request_pending.count() proj_join_request_object.title = 'Project Join Requests' proj_join_request_object.table = \ @@ -270,7 +270,7 @@ def get_project_renewal_request(self): if not self.show_all_requests: args.append(Q(requester=user) | Q(pi=user)) - project_renewal_request_active = \ + project_renewal_request_pending = \ AllocationRenewalRequest.objects.filter( status__name__in=['Approved', 'Under Review'], *args).order_by( 'modified') @@ -281,14 +281,14 @@ def get_project_renewal_request(self): 'modified') proj_renewal_request_object.num = self.paginators - proj_renewal_request_object.active_queryset = \ - self.create_paginator(project_renewal_request_active) + proj_renewal_request_object.pending_queryset = \ + self.create_paginator(project_renewal_request_pending) proj_renewal_request_object.complete_queryset = \ self.create_paginator(project_renewal_request_complete) - proj_renewal_request_object.num_active = \ - project_renewal_request_active.count() + proj_renewal_request_object.num_pending = \ + project_renewal_request_pending.count() proj_renewal_request_object.title = 'Project Renewal Requests' proj_renewal_request_object.table = \ @@ -305,7 +305,7 @@ def get_su_purchase_request(self): su_purchase_request_object = RequestListItem() user = self.request.user - su_purchase_request_active = AllocationAdditionRequest.objects.filter( + su_purchase_request_pending = AllocationAdditionRequest.objects.filter( status__name__in=['Under Review']).order_by('modified') su_purchase_request_complete = AllocationAdditionRequest.objects.filter( @@ -313,10 +313,10 @@ def get_su_purchase_request(self): if not self.show_all_requests: request_ids = [ - r.id for r in su_purchase_request_active + r.id for r in su_purchase_request_pending if is_user_manager_or_pi_of_project(user, r.project)] - su_purchase_request_active = \ - su_purchase_request_active.filter(id__in=request_ids) + su_purchase_request_pending = \ + su_purchase_request_pending.filter(id__in=request_ids) request_ids = [ r.id for r in su_purchase_request_complete @@ -325,14 +325,14 @@ def get_su_purchase_request(self): su_purchase_request_complete.filter(id__in=request_ids) su_purchase_request_object.num = self.paginators - su_purchase_request_object.active_queryset = \ - self.create_paginator(su_purchase_request_active) + su_purchase_request_object.pending_queryset = \ + self.create_paginator(su_purchase_request_pending) su_purchase_request_object.complete_queryset = \ self.create_paginator(su_purchase_request_complete) - su_purchase_request_object.num_active = \ - su_purchase_request_active.count() + su_purchase_request_object.num_pending = \ + su_purchase_request_pending.count() su_purchase_request_object.title = 'Service Unit Purchase Requests' su_purchase_request_object.table = \ From b91d81dc0901f14cd5c1d743063823dc440564d2 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Mon, 28 Feb 2022 13:24:48 -0500 Subject: [PATCH 120/150] removed invoice section. moved jobs to its own tab --- coldfront/templates/common/authorized_navbar.html | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/coldfront/templates/common/authorized_navbar.html b/coldfront/templates/common/authorized_navbar.html index 854ffca5e..42bcd05e5 100644 --- a/coldfront/templates/common/authorized_navbar.html +++ b/coldfront/templates/common/authorized_navbar.html @@ -17,7 +17,6 @@ + {% if request.user.is_superuser %} {% include 'common/navbar_admin.html' %} {% elif request.user.is_staff %} @@ -37,9 +39,6 @@ {% elif perms.project.can_review_pending_project_reviews or perms.grant.can_view_all_grants %} {% include 'common/navbar_director.html' %} {% endif %} - {% if request.user.is_superuser or perms.allocation.can_manage_invoice %} - {% include 'common/navbar_invoice.html' %} - {% endif %} From 2b2cec30e24416f3ce52fc1a3640d07a3af9c69d Mon Sep 17 00:00:00 2001 From: Matthew Li Date: Mon, 28 Feb 2022 14:57:32 -0800 Subject: [PATCH 121/150] Add middleware that logs exceptions raised by views --- coldfront/config/local_settings.py.sample | 11 +++++++++-- coldfront/core/utils/middleware.py | 24 +++++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 coldfront/core/utils/middleware.py diff --git a/coldfront/config/local_settings.py.sample b/coldfront/config/local_settings.py.sample index a61bfc579..8865504ef 100644 --- a/coldfront/config/local_settings.py.sample +++ b/coldfront/config/local_settings.py.sample @@ -338,9 +338,9 @@ LOCAL_SETTINGS_EXPORT += [ # XDMOD_API_URL = 'http://localhost' -# ------------------------------------------------------------------------------ +# ----------------------------------------------------------------------------- # Enable myBRC REST API -# ------------------------------------------------------------------------------ +# ----------------------------------------------------------------------------- EXTRA_APPS += [ 'rest_framework', 'django_filters', @@ -392,6 +392,13 @@ FLAGS = { 'LRC_ONLY': [], } +# ----------------------------------------------------------------------------- +# Miscellaneous settings +# ----------------------------------------------------------------------------- +EXTRA_MIDDLEWARE += [ + 'coldfront.core.utils.middleware.ExceptionMiddleware', +] + #------------------------------------------------------------------------------ # Deployment-specific settings #------------------------------------------------------------------------------ diff --git a/coldfront/core/utils/middleware.py b/coldfront/core/utils/middleware.py new file mode 100644 index 000000000..7ae4d2942 --- /dev/null +++ b/coldfront/core/utils/middleware.py @@ -0,0 +1,24 @@ +import logging + + +logger = logging.getLogger(__name__) + + +class ExceptionMiddleware: + """Log exceptions raised by views.""" + + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + response = self.get_response(request) + return response + + @staticmethod + def process_exception(request, exception): + message = ( + f'{request.user} encountered an uncaught exception at ' + f'{request.path}. Details:') + logger.error(message) + logger.exception(exception) + return None From a18101c0ea18c11b4b8a5688570e57c7c82dcdda Mon Sep 17 00:00:00 2001 From: Matthew Li Date: Mon, 28 Feb 2022 14:59:44 -0800 Subject: [PATCH 122/150] Install Sentry; send errors to it, except for some noisy logs --- .../ansible/cf_mybrc_settings_template.tmpl | 21 +++++++++++++++++++ bootstrap/ansible/main.copyme | 11 ++++++---- bootstrap/development/main.copyme | 3 +++ requirements.txt | 1 + 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/bootstrap/ansible/cf_mybrc_settings_template.tmpl b/bootstrap/ansible/cf_mybrc_settings_template.tmpl index 8c3fbd13c..0a6f4b78a 100644 --- a/bootstrap/ansible/cf_mybrc_settings_template.tmpl +++ b/bootstrap/ansible/cf_mybrc_settings_template.tmpl @@ -51,3 +51,24 @@ SESSION_COOKIE_SECURE = True {% else %} SESSION_COOKIE_SECURE = False {% endif %} + +# Configure Sentry. +import sentry_sdk +from sentry_sdk.integrations.django import DjangoIntegration +from sentry_sdk.integrations.logging import ignore_logger + + +{% if sentry_dsn|length > 0 %} +SENTRY_DSN = '{{ sentry_dsn.strip() }}' +{% else %} +SENTRY_DSN = '' +{% endif %} +if SENTRY_DSN: + sentry_sdk.init( + dsn=SENTRY_DSN, + integrations=[DjangoIntegration()], + traces_sample_rate=0.01, + send_default_pii=True) + # Ignore noisy loggers. + ignore_logger('coldfront.api') + ignore_logger('coldfront.core.utils.middleware') diff --git a/bootstrap/ansible/main.copyme b/bootstrap/ansible/main.copyme index 1b655bf31..72e1266dd 100644 --- a/bootstrap/ansible/main.copyme +++ b/bootstrap/ansible/main.copyme @@ -33,6 +33,9 @@ email_subject_prefix: '[MyBRC-User-Portal]' google_sheets_credentials: '/tmp/credentials.json' +# The URL of the Sentry instance to send errors to. +sentry_dsn: "" + ##################################### #STAGING SETTINGS ##################################### @@ -51,9 +54,9 @@ google_sheets_credentials: '/tmp/credentials.json' #ssl_certificate_key_file: /etc/ssl/ssl_certificate_key.file #ssl_certificate_chain_file: /etc/ssl/ssl_certification_chain.file -ip_range_with_api_access: 0.0.0.0/24 +#ip_range_with_api_access: 0.0.0.0/24 -request_approval_cc_list: [] +#request_approval_cc_list: [] ##################################### #PROD SETTINGS @@ -73,6 +76,6 @@ request_approval_cc_list: [] #ssl_certificate_key_file: /etc/ssl/ssl_certificate_key.file #ssl_certificate_chain_file: /etc/ssl/ssl_certification_chain.file -ip_range_with_api_access: 10.0.0.0/8 +#ip_range_with_api_access: 10.0.0.0/8 -request_approval_cc_list: ['wfeinstein@lbl.gov'] +#request_approval_cc_list: ['wfeinstein@lbl.gov'] diff --git a/bootstrap/development/main.copyme b/bootstrap/development/main.copyme index c1afe9f4b..aaa572368 100644 --- a/bootstrap/development/main.copyme +++ b/bootstrap/development/main.copyme @@ -29,6 +29,9 @@ email_subject_prefix: "[MyBRC-User-Portal]" google_sheets_credentials: /tmp/credentials.json +# The URL of the Sentry instance to send errors to. +sentry_dsn: "" + ############################################################################### # dev_settings ############################################################################### diff --git a/requirements.txt b/requirements.txt index 452fd090b..ac49c38af 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,6 +40,7 @@ python-memcached==1.59 pytz==2018.9 redis==3.2.1 requests==2.22.0 +sentry-sdk==1.5.6 six==1.12.0 sqlparse==0.3.0 text-unidecode==1.3 From 9bd4beafb7d1e54c5b946fba91d465539f957b6a Mon Sep 17 00:00:00 2001 From: Matthew Li Date: Mon, 28 Feb 2022 15:51:35 -0800 Subject: [PATCH 123/150] Add smoke test that form for creating new Savio project succeeds --- .../test_savio_project_request_wizard.py | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 coldfront/core/project/tests/test_views/test_savio_project_request_wizard.py diff --git a/coldfront/core/project/tests/test_views/test_savio_project_request_wizard.py b/coldfront/core/project/tests/test_views/test_savio_project_request_wizard.py new file mode 100644 index 000000000..012cbac9d --- /dev/null +++ b/coldfront/core/project/tests/test_views/test_savio_project_request_wizard.py @@ -0,0 +1,96 @@ +from coldfront.core.project.models import Project +from coldfront.core.project.models import SavioProjectAllocationRequest +from coldfront.core.utils.tests.test_base import TestBase +from django.urls import reverse +from http import HTTPStatus + + +class TestSavioProjectRequestWizard(TestBase): + """A class for testing SavioProjectRequestWizard.""" + + def setUp(self): + """Set up test data.""" + super().setUp() + self.create_test_user() + self.sign_user_access_agreement(self.user) + self.client.login(username=self.user.username, password=self.password) + + @staticmethod + def request_url(): + """Return the URL for requesting to create a new Savio + project.""" + return reverse('savio-project-request') + + def test_post_creates_request(self): + """Test that a POST request creates a + SavioProjectAllocationRequest.""" + self.assertEqual(SavioProjectAllocationRequest.objects.count(), 0) + self.assertEqual(Project.objects.count(), 0) + + view_name = 'savio_project_request_wizard' + current_step_key = f'{view_name}-current_step' + allocation_type_form_data = { + '0-allocation_type': 'FCA', + current_step_key: '0', + } + existing_pi_form_data = { + '1-PI': self.user.pk, + current_step_key: '1', + } + pool_allocations_data = { + '5-pool': False, + current_step_key: '5', + } + details_data = { + '7-name': 'name', + '7-title': 'title', + '7-description': 'a' * 20, + current_step_key: '7', + } + survey_data = { + '8-scope_and_intent': 'b' * 20, + '8-computational_aspects': 'c' * 20, + current_step_key: '8', + } + form_data = [ + allocation_type_form_data, + existing_pi_form_data, + pool_allocations_data, + details_data, + survey_data, + ] + + url = self.request_url() + for i, data in enumerate(form_data): + response = self.client.post(url, data) + if i == len(form_data) - 1: + self.assertRedirects(response, reverse('home')) + else: + self.assertEqual(response.status_code, HTTPStatus.OK) + + requests = SavioProjectAllocationRequest.objects.all() + self.assertEqual(requests.count(), 1) + projects = Project.objects.all() + self.assertEqual(projects.count(), 1) + + request = requests.first() + project = projects.first() + self.assertEqual(request.requester, self.user) + self.assertEqual( + request.allocation_type, + allocation_type_form_data['0-allocation_type']) + self.assertEqual(request.pi, self.user) + self.assertEqual(request.project, project) + self.assertEqual(project.name, f'fc_{details_data["7-name"]}') + self.assertEqual(project.title, details_data['7-title']) + self.assertEqual(project.description, details_data['7-description']) + self.assertFalse(request.pool) + self.assertEqual( + request.survey_answers['scope_and_intent'], + survey_data['8-scope_and_intent']) + self.assertEqual( + request.survey_answers['computational_aspects'], + survey_data['8-computational_aspects']) + self.assertEqual(request.status.name, 'Under Review') + + # TODO From 29010f6045e4cb3af462c6beb7758a0d8932a42b Mon Sep 17 00:00:00 2001 From: Matthew Li Date: Mon, 28 Feb 2022 15:59:35 -0800 Subject: [PATCH 124/150] Uncomment Ansible settings --- bootstrap/ansible/main.copyme | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bootstrap/ansible/main.copyme b/bootstrap/ansible/main.copyme index 72e1266dd..e0b179097 100644 --- a/bootstrap/ansible/main.copyme +++ b/bootstrap/ansible/main.copyme @@ -54,9 +54,9 @@ sentry_dsn: "" #ssl_certificate_key_file: /etc/ssl/ssl_certificate_key.file #ssl_certificate_chain_file: /etc/ssl/ssl_certification_chain.file -#ip_range_with_api_access: 0.0.0.0/24 +ip_range_with_api_access: 0.0.0.0/24 -#request_approval_cc_list: [] +request_approval_cc_list: [] ##################################### #PROD SETTINGS @@ -76,6 +76,6 @@ sentry_dsn: "" #ssl_certificate_key_file: /etc/ssl/ssl_certificate_key.file #ssl_certificate_chain_file: /etc/ssl/ssl_certification_chain.file -#ip_range_with_api_access: 10.0.0.0/8 +ip_range_with_api_access: 10.0.0.0/8 -#request_approval_cc_list: ['wfeinstein@lbl.gov'] +request_approval_cc_list: ['wfeinstein@lbl.gov'] From dc427c94c2beaf8bf9e7834a621d53396e55d7ac Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Wed, 2 Mar 2022 09:58:33 -0500 Subject: [PATCH 125/150] added userpassestestmixin --- coldfront/core/user/views_/request_hub_views.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/coldfront/core/user/views_/request_hub_views.py b/coldfront/core/user/views_/request_hub_views.py index 5c050f412..a6bda4713 100644 --- a/coldfront/core/user/views_/request_hub_views.py +++ b/coldfront/core/user/views_/request_hub_views.py @@ -1,4 +1,5 @@ -from django.contrib.auth.mixins import LoginRequiredMixin +from django.contrib import messages +from django.contrib.auth.mixins import LoginRequiredMixin, UserPassesTestMixin from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q from django.views.generic.base import TemplateView @@ -37,12 +38,23 @@ def __init__(self): class RequestHub(LoginRequiredMixin, + UserPassesTestMixin, TemplateView): template_name = 'request_hub/request_hub.html' paginate_by = 10 paginators = 0 show_all_requests = False + def test_func(self): + """ UserPassesTestMixin Tests""" + if self.show_all_requests: + if self.request.user.is_superuser or self.request.user.is_staff: + return True + else: + return True + + return False + def create_paginator(self, queryset): """ Creates a paginator object for the given queryset From 5de44c1dbafde1c246d15fc208c8694f912d1ee3 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Wed, 2 Mar 2022 10:01:54 -0500 Subject: [PATCH 126/150] changed name of view --- coldfront/core/user/urls.py | 4 ++-- coldfront/core/user/views_/request_hub_views.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/coldfront/core/user/urls.py b/coldfront/core/user/urls.py index d03f298f7..6d852cbbf 100644 --- a/coldfront/core/user/urls.py +++ b/coldfront/core/user/urls.py @@ -117,10 +117,10 @@ # Request Hub path('request-hub', - request_hub_views.RequestHub.as_view(show_all_requests=False), + request_hub_views.RequestHubView.as_view(show_all_requests=False), name='request-hub'), path('request-hub-admin', - request_hub_views.RequestHub.as_view(show_all_requests=True), + request_hub_views.RequestHubView.as_view(show_all_requests=True), name='request-hub-admin'), ] diff --git a/coldfront/core/user/views_/request_hub_views.py b/coldfront/core/user/views_/request_hub_views.py index a6bda4713..8595eef9a 100644 --- a/coldfront/core/user/views_/request_hub_views.py +++ b/coldfront/core/user/views_/request_hub_views.py @@ -37,7 +37,7 @@ def __init__(self): button_text = None -class RequestHub(LoginRequiredMixin, +class RequestHubView(LoginRequiredMixin, UserPassesTestMixin, TemplateView): template_name = 'request_hub/request_hub.html' From cb7384360ac334ec6063b41893cf34086b9f80bf Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Wed, 2 Mar 2022 10:02:24 -0500 Subject: [PATCH 127/150] initial commit --- .../tests/test_views/test_request_hub_view.py | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 coldfront/core/user/tests/test_views/test_request_hub_view.py diff --git a/coldfront/core/user/tests/test_views/test_request_hub_view.py b/coldfront/core/user/tests/test_views/test_request_hub_view.py new file mode 100644 index 000000000..4087e5fc4 --- /dev/null +++ b/coldfront/core/user/tests/test_views/test_request_hub_view.py @@ -0,0 +1,118 @@ +import os +import sys +from decimal import Decimal +from http import HTTPStatus +from io import StringIO + +from django.core.management import call_command +from django.test import TestCase + +from coldfront.api.statistics.utils import create_project_allocation, \ + create_user_project_allocation +from coldfront.core.project.models import ProjectStatusChoice, \ + ProjectUserStatusChoice, ProjectUserRoleChoice, Project, ProjectUser +from coldfront.core.user.models import EmailAddress, UserProfile +from coldfront.core.user.tests.utils import TestUserBase +from coldfront.core.user.utils import account_activation_url +from django.contrib.auth.models import User +from django.contrib.messages import get_messages +from django.test import Client +from django.urls import reverse + + +class TestRequestHubView(TestCase): + """A class for testing the view for activating a user's account.""" + + password = 'test1234' + + def setUp(self): + """Set up test data.""" + out, err = StringIO(), StringIO() + commands = [ + 'add_resource_defaults', + 'add_allocation_defaults', + 'import_field_of_science_data', + 'add_default_project_choices', + 'create_staff_group', + ] + sys.stdout = open(os.devnull, 'w') + for command in commands: + call_command(command, stdout=out, stderr=err) + sys.stdout = sys.__stdout__ + + self.password = 'password' + + self.pi = User.objects.create( + username='pi0', email='pi0@nonexistent.com') + user_profile = UserProfile.objects.get(user=self.pi) + user_profile.is_pi = True + user_profile.save() + + self.admin = User.objects.create( + username='admin', email='admin@nonexistent.com') + user_profile = UserProfile.objects.get(user=self.admin) + user_profile.is_superuser = True + user_profile.save() + + self.staff = User.objects.create( + username='staff', email='staff@nonexistent.com') + user_profile = UserProfile.objects.get(user=self.staff) + user_profile.is_staff = True + user_profile.save() + + # Create two Users. + for i in range(2): + user = User.objects.create( + username=f'user{i}', email=f'user{i}@nonexistent.com') + user_profile = UserProfile.objects.get(user=user) + user_profile.cluster_uid = f'{i}' + user_profile.save() + setattr(self, f'user{i}', user) + setattr(self, f'user_profile{i}', user_profile) + + # Create Projects and associate Users with them. + project_status = ProjectStatusChoice.objects.get(name='Active') + project_user_status = ProjectUserStatusChoice.objects.get( + name='Active') + user_role = ProjectUserRoleChoice.objects.get(name='User') + manager_role = ProjectUserRoleChoice.objects.get(name='Manager') + for i in range(2): + # Create a Project and ProjectUsers. + project = Project.objects.create( + name=f'project{i}', status=project_status) + setattr(self, f'project{i}', project) + ProjectUser.objects.create( + user=getattr(self, 'user0'), project=project, + role=user_role, status=project_user_status) + ProjectUser.objects.create( + user=self.pi, project=project, role=manager_role, + status=project_user_status) + + # Create a compute allocation for the Project. + allocation = Decimal(f'{i + 1}000.00') + create_project_allocation(project, allocation) + + # Create a compute allocation for each User on the Project. + for j in range(2): + create_user_project_allocation( + getattr(self, f'user{j}'), project, allocation / 2) + + for user in User.objects.all(): + user.set_password(self.password) + + self.url = reverse('request-hub') + self.admin_url = reverse('request-hub-admin') + + def assert_has_access(self, user, url, has_access): + self.client.login(username=user.username, password=self.password) + response = self.client.get(url) + status_code = HTTPStatus.OK if has_access else HTTPStatus.FORBIDDEN + self.assertEqual(response.status_code, status_code) + self.client.logout() + + def get_response(self, user, url): + self.client.login(username=user.username, password=self.password) + return self.client.get(url) + + def test_access(self): + """Testing access to RequestHubView""" From 3550f288893ae5244c9548f6ba4000f2630adaba Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Wed, 2 Mar 2022 17:58:19 -0500 Subject: [PATCH 128/150] added tests for request_hub views --- .../tests/test_views/test_request_hub_view.py | 359 ++++++++++++++++-- 1 file changed, 330 insertions(+), 29 deletions(-) diff --git a/coldfront/core/user/tests/test_views/test_request_hub_view.py b/coldfront/core/user/tests/test_views/test_request_hub_view.py index 4087e5fc4..9c69edf5d 100644 --- a/coldfront/core/user/tests/test_views/test_request_hub_view.py +++ b/coldfront/core/user/tests/test_views/test_request_hub_view.py @@ -1,16 +1,25 @@ import os import sys +import datetime from decimal import Decimal from http import HTTPStatus from io import StringIO +from bs4 import BeautifulSoup + from django.core.management import call_command from django.test import TestCase from coldfront.api.statistics.utils import create_project_allocation, \ - create_user_project_allocation + create_user_project_allocation, get_accounting_allocation_objects +from coldfront.core.allocation.models import AllocationUserAttribute, \ + AllocationAttributeType, Allocation from coldfront.core.project.models import ProjectStatusChoice, \ - ProjectUserStatusChoice, ProjectUserRoleChoice, Project, ProjectUser + ProjectUserStatusChoice, ProjectUserRoleChoice, Project, ProjectUser, \ + ProjectUserRemovalRequestStatusChoice, ProjectUserRemovalRequest, \ + SavioProjectAllocationRequest, ProjectAllocationRequestStatusChoice, \ + savio_project_request_state_schema, vector_project_request_state_schema, \ + VectorProjectAllocationRequest, ProjectUserJoinRequest from coldfront.core.user.models import EmailAddress, UserProfile from coldfront.core.user.tests.utils import TestUserBase from coldfront.core.user.utils import account_activation_url @@ -21,23 +30,17 @@ class TestRequestHubView(TestCase): - """A class for testing the view for activating a user's account.""" - - password = 'test1234' + """A class for testing RequestHubView.""" def setUp(self): """Set up test data.""" - out, err = StringIO(), StringIO() - commands = [ - 'add_resource_defaults', - 'add_allocation_defaults', - 'import_field_of_science_data', - 'add_default_project_choices', - 'create_staff_group', - ] sys.stdout = open(os.devnull, 'w') - for command in commands: - call_command(command, stdout=out, stderr=err) + call_command('import_field_of_science_data') + call_command('add_default_project_choices') + call_command('add_resource_defaults') + call_command('add_allocation_defaults') + call_command('add_brc_accounting_defaults') + call_command('create_staff_group') sys.stdout = sys.__stdout__ self.password = 'password' @@ -48,17 +51,12 @@ def setUp(self): user_profile.is_pi = True user_profile.save() + # create staff and admin users self.admin = User.objects.create( - username='admin', email='admin@nonexistent.com') - user_profile = UserProfile.objects.get(user=self.admin) - user_profile.is_superuser = True - user_profile.save() + username='admin', email='admin@nonexistent.com', is_superuser=True) self.staff = User.objects.create( - username='staff', email='staff@nonexistent.com') - user_profile = UserProfile.objects.get(user=self.staff) - user_profile.is_staff = True - user_profile.save() + username='staff', email='staff@nonexistent.com', is_staff=True) # Create two Users. for i in range(2): @@ -81,28 +79,39 @@ def setUp(self): project = Project.objects.create( name=f'project{i}', status=project_status) setattr(self, f'project{i}', project) - ProjectUser.objects.create( - user=getattr(self, 'user0'), project=project, + proj_user = ProjectUser.objects.create( + user=self.user0, project=project, role=user_role, status=project_user_status) - ProjectUser.objects.create( + setattr(self, f'project{i}_user0', proj_user) + pi_proj_user = ProjectUser.objects.create( user=self.pi, project=project, role=manager_role, status=project_user_status) + setattr(self, f'project{i}_pi', pi_proj_user) # Create a compute allocation for the Project. allocation = Decimal(f'{i + 1}000.00') create_project_allocation(project, allocation) # Create a compute allocation for each User on the Project. - for j in range(2): - create_user_project_allocation( - getattr(self, f'user{j}'), project, allocation / 2) + create_user_project_allocation( + self.user0, project, allocation / 2) + # set passwords for user in User.objects.all(): user.set_password(self.password) + user.save() self.url = reverse('request-hub') self.admin_url = reverse('request-hub-admin') + self.requests = ['cluster account request', + 'project removal request', + 'savio project request', + 'vector project request', + 'project join request', + 'project renewal request', + 'service unit purchase request'] + def assert_has_access(self, user, url, has_access): self.client.login(username=user.username, password=self.password) response = self.client.get(url) @@ -114,5 +123,297 @@ def get_response(self, user, url): self.client.login(username=user.username, password=self.password) return self.client.get(url) + def assert_no_requests(self, user, url): + response = self.get_response(user, url) + soup = BeautifulSoup(response.content, 'html.parser') + for request in self.requests: + divs = soup.find(id=f'{request.replace(" ", "_")}_section'). \ + find_all('div', {'class': 'alert alert-info'}) + self.assertEqual(len(divs), 2) + for i, div in enumerate(divs): + self.assertIn(request.title(), str(div)) + self.assertIn('No pending' if i == 0 else 'No completed', + str(div)) + def test_access(self): """Testing access to RequestHubView""" + + # all users should have access to the normal view + for user in User.objects.all(): + self.assert_has_access(user, self.url, True) + + # only staff/admin should have access to request-hub-admin + self.assert_has_access(self.admin, self.admin_url, True) + self.assert_has_access(self.staff, self.admin_url, True) + self.assert_has_access(self.pi, self.admin_url, False) + self.assert_has_access(self.user0, self.admin_url, False) + self.assert_has_access(self.user1, self.admin_url, False) + + def test_admin_buttons(self): + """Test that 'Go to main request page' buttons only appear for + admin/staff""" + + def assert_button_displayed(user, displayed): + """Assert that the relevant button appears if the + given boolean is True; otherwise, assert that they do not + appear.""" + button_list = [f'Go To {request.title()}s Main Page' + for request in self.requests] + response = self.get_response(user, self.url) + html = response.content.decode('utf-8') + func = self.assertIn if displayed else self.assertNotIn + + for button in button_list: + func(button, html) + + assert_button_displayed(self.user0, False) + assert_button_displayed(self.admin, True) + assert_button_displayed(self.staff, True) + + def test_no_requests(self): + """Testing that the correct message is displayed when + there are no requests""" + + self.assert_no_requests(self.user0, self.url) + self.assert_no_requests(self.user1, self.url) + self.assert_no_requests(self.pi, self.url) + self.assert_no_requests(self.admin, self.url) + self.assert_no_requests(self.staff, self.url) + self.assert_no_requests(self.admin, self.admin_url) + self.assert_no_requests(self.staff, self.admin_url) + + def test_cluster_account_requests(self): + """Testing that cluster account requests appear""" + + def assert_request_shown(user, url): + response = self.get_response(user, url) + soup = BeautifulSoup(response.content, 'html.parser') + + pending_div = str( + soup.find(id=f'cluster_account_request_section_pending')) + self.assertIn(str(pending_req.pk), pending_div) + self.assertIn(pending_req.allocation_user.user.email, pending_div) + self.assertIn(pending_req.value, pending_div) + + completed_div = str( + soup.find(id=f'cluster_account_request_section_completed')) + self.assertIn(str(completed_req.pk), completed_div) + self.assertIn(completed_req.allocation_user.user.email, + completed_div) + self.assertIn(completed_req.value, completed_div) + + # creating two cluster account requests for user0 + allocation_obj = \ + get_accounting_allocation_objects(self.project0) + allocation_user_obj = \ + get_accounting_allocation_objects(self.project0, self.user0) + + cluster_account_status = AllocationAttributeType.objects.get( + name='Cluster Account Status') + + kwargs = { + 'allocation_attribute_type': cluster_account_status, + 'allocation': allocation_obj.allocation, + 'allocation_user': allocation_user_obj.allocation_user, + } + + pending_req = \ + AllocationUserAttribute.objects.create(value='Processing', **kwargs) + + completed_req = \ + AllocationUserAttribute.objects.create(value='Denied', **kwargs) + + assert_request_shown(self.user0, self.url) + assert_request_shown(self.admin, self.admin_url) + assert_request_shown(self.staff, self.admin_url) + + def test_project_removal_requests(self): + """Testing that project removal requests appear""" + def assert_request_shown(user, url): + response = self.get_response(user, url) + soup = BeautifulSoup(response.content, 'html.parser') + + pending_div = str( + soup.find(id=f'project_removal_request_section_pending')) + self.assertIn(str(pending_req.pk), pending_div) + self.assertIn(pending_req.project_user.user.username, pending_div) + self.assertIn(pending_req.requester.username, pending_div) + self.assertIn(pending_req.request_time.strftime("%b. %d, %Y"), pending_div) + self.assertIn(pending_req.status.name, pending_div) + + completed_div = str( + soup.find(id=f'project_removal_request_section_completed')) + self.assertIn(str(completed_req.pk), pending_div) + self.assertIn(completed_req.project_user.user.username, completed_div) + self.assertIn(completed_req.requester.username, completed_div) + self.assertIn(completed_req.completion_time.strftime("%b. %d, %Y"), completed_div) + self.assertIn(completed_req.status.name, completed_div) + + processing_status = \ + ProjectUserRemovalRequestStatusChoice.objects.get(name='Processing') + complete_status = \ + ProjectUserRemovalRequestStatusChoice.objects.get(name='Complete') + current_time = datetime.datetime.now() + + kwargs = { + 'project_user': self.project0_user0, + 'requester': self.project0_pi.user, + 'request_time': current_time, + } + pending_req = ProjectUserRemovalRequest.objects.create( + status=processing_status, **kwargs) + completed_req = ProjectUserRemovalRequest.objects.create( + status=complete_status, + completion_time=current_time + datetime.timedelta(days=4), + **kwargs + ) + + assert_request_shown(self.user0, self.url) + assert_request_shown(self.pi, self.url) + assert_request_shown(self.admin, self.admin_url) + assert_request_shown(self.staff, self.admin_url) + + def test_savio_project_requests(self): + """Testing that savio project requests appear""" + + def assert_request_shown(user, url): + response = self.get_response(user, url) + soup = BeautifulSoup(response.content, 'html.parser') + + pending_div = str( + soup.find(id=f'savio_project_request_section_pending')) + self.assertIn(str(pending_req.pk), pending_div) + self.assertIn(pending_req.requester.email, pending_div) + self.assertIn(pending_req.pi.email, pending_div) + self.assertIn(pending_req.modified.strftime("%b. %d, %Y"), pending_div) + self.assertIn(pending_req.status.name, pending_div) + + completed_div = str( + soup.find(id=f'savio_project_request_section_completed')) + self.assertIn(str(completed_req.pk), completed_div) + self.assertIn(completed_req.requester.email, completed_div) + self.assertIn(completed_req.pi.email, completed_div) + self.assertIn(completed_req.modified.strftime("%b. %d, %Y"), completed_div) + self.assertIn(completed_req.status.name, completed_div) + + kwargs = { + 'requester': self.user0, + 'allocation_type': 'FCA', + 'pi': self.pi, + 'project': self.project0, + 'pool': False, + 'survey_answers': savio_project_request_state_schema() + } + + processing_status = \ + ProjectAllocationRequestStatusChoice.objects.get( + name='Approved - Processing') + complete_status = \ + ProjectAllocationRequestStatusChoice.objects.get( + name='Approved - Complete') + + pending_req = SavioProjectAllocationRequest.objects.create( + status=processing_status, **kwargs) + completed_req = SavioProjectAllocationRequest.objects.create( + status=complete_status, **kwargs) + + assert_request_shown(self.user0, self.url) + assert_request_shown(self.pi, self.url) + assert_request_shown(self.admin, self.admin_url) + assert_request_shown(self.staff, self.admin_url) + + def test_vector_project_requests(self): + """Testing that vector project requests appear""" + def assert_request_shown(user, url): + response = self.get_response(user, url) + soup = BeautifulSoup(response.content, 'html.parser') + + pending_div = str( + soup.find(id=f'vector_project_request_section_pending')) + self.assertIn(str(pending_req.pk), pending_div) + self.assertIn(pending_req.requester.email, pending_div) + self.assertIn(pending_req.pi.email, pending_div) + self.assertIn(pending_req.modified.strftime("%b. %d, %Y"), pending_div) + self.assertIn(pending_req.status.name, pending_div) + + completed_div = str( + soup.find(id=f'vector_project_request_section_completed')) + self.assertIn(str(completed_req.pk), completed_div) + self.assertIn(completed_req.requester.email, completed_div) + self.assertIn(completed_req.pi.email, completed_div) + self.assertIn(completed_req.modified.strftime("%b. %d, %Y"), completed_div) + self.assertIn(completed_req.status.name, completed_div) + + kwargs = { + 'requester': self.user0, + 'pi': self.pi, + 'project': self.project0, + 'state': vector_project_request_state_schema() + } + + processing_status = \ + ProjectAllocationRequestStatusChoice.objects.get( + name='Approved - Processing') + complete_status = \ + ProjectAllocationRequestStatusChoice.objects.get( + name='Approved - Complete') + + pending_req = VectorProjectAllocationRequest.objects.create( + status=processing_status, **kwargs) + completed_req = VectorProjectAllocationRequest.objects.create( + status=complete_status, **kwargs) + + assert_request_shown(self.user0, self.url) + assert_request_shown(self.pi, self.url) + assert_request_shown(self.admin, self.admin_url) + assert_request_shown(self.staff, self.admin_url) + + def test_project_join_requests(self): + """Testing that project join requests appear correctly""" + + def assert_request_shown(user, url, section): + response = self.get_response(user, url) + soup = BeautifulSoup(response.content, 'html.parser') + + if section == 'both' or section == 'pending': + pending_div = str( + soup.find(id=f'project_join_request_section_pending')) + self.assertIn(str(pending_req.pk), pending_div) + self.assertIn(pending_req.project_user.user.username, pending_div) + self.assertIn(pending_req.project_user.project.name, pending_div) + self.assertIn(pending_req.created.strftime("%b. %d, %Y"), pending_div) + self.assertIn(pending_req.reason, pending_div) + + if section == 'both' or section == 'completed': + completed_div = str( + soup.find(id=f'project_join_request_section_completed')) + self.assertIn(str(completed_req.pk), completed_div) + self.assertIn(completed_req.project_user.user.username, completed_div) + self.assertIn(completed_req.project_user.project.name, completed_div) + self.assertIn(completed_req.created.strftime("%b. %d, %Y"), completed_div) + self.assertIn(completed_req.reason, completed_div) + + project_user_status = ProjectUserStatusChoice.objects.get( + name='Pending - Add') + user_role = ProjectUserRoleChoice.objects.get(name='User') + pending_proj_user = ProjectUser.objects.create( + user=self.user1, project=self.project0, + role=user_role, status=project_user_status) + + pending_req = ProjectUserJoinRequest.objects.create( + project_user=pending_proj_user, + reason='Request hub testing.') + completed_req = ProjectUserJoinRequest.objects.create( + project_user=self.project0_user0, + reason='Request hub testing.') + + assert_request_shown(self.user0, self.url, 'completed') + assert_request_shown(self.user1, self.url, 'pending') + assert_request_shown(self.pi, self.url, 'both') + assert_request_shown(self.admin, self.admin_url, 'both') + assert_request_shown(self.staff, self.admin_url, 'both') + + # def test_project_renewal_requests(self): + # """Testing that project renewal requests appear correctly""" + # + # AllocationRenewalRequest \ No newline at end of file From 04d890cdafd18e62b6b16ee7feaa6b4a701c2f4a Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Wed, 2 Mar 2022 17:59:26 -0500 Subject: [PATCH 129/150] added ids to div sections --- .../core/user/templates/request_hub/request_section.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/coldfront/core/user/templates/request_hub/request_section.html b/coldfront/core/user/templates/request_hub/request_section.html index 77442d756..bd11f5d51 100644 --- a/coldfront/core/user/templates/request_hub/request_section.html +++ b/coldfront/core/user/templates/request_hub/request_section.html @@ -1,5 +1,5 @@
    -
    +
    @@ -17,7 +17,7 @@
    {% with queryset=request_obj.pending_queryset adj='pending' page_num=request_obj.num title=request_obj.title %} -
    +
    {% if queryset %}
    @@ -40,7 +40,7 @@
    {{ adj }} {{ tit
    {% endwith %} {% with queryset=request_obj.complete_queryset adj='completed' page_num=request_obj.num|add:1 title=request_obj.title %} -
    +
    {% if queryset %}
    @@ -62,7 +62,7 @@
    {{ adj }} {{ tit
    {% endwith %} - {% if user.is_superuser %} + {% if admin_staff %}
    From 873a349f59ebc78860822f60a5ad22294b5e3f2e Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Wed, 2 Mar 2022 18:00:01 -0500 Subject: [PATCH 130/150] added adj variable to make it compatible with request hub --- .../project_removal/project_removal_request_list_table.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/coldfront/core/project/templates/project/project_removal/project_removal_request_list_table.html b/coldfront/core/project/templates/project/project_removal/project_removal_request_list_table.html index db4b230ec..927f0b57e 100644 --- a/coldfront/core/project/templates/project/project_removal/project_removal_request_list_table.html +++ b/coldfront/core/project/templates/project/project_removal/project_removal_request_list_table.html @@ -7,7 +7,7 @@ - {% if request_filter == 'pending' %} + {% if request_filter == 'pending' or adj == 'pending' %} Date Requested {% else %} Date Completed @@ -38,7 +38,7 @@ Status - {% if user.is_superuser and request_filter == 'pending' %} + {% if user.is_superuser and request_filter == 'pending'%} Actions @@ -49,7 +49,7 @@ {% for removal_request in queryset %} {{ removal_request.pk }} - {% if request_filter == 'pending' or adj == 'active' %} + {% if request_filter == 'pending' or adj == 'pending' %} {{ removal_request.request_time|date:"M. d, Y" }} {% else %} {{ removal_request.completion_time|date:"M. d, Y" }} From aa454c4695ebd8b35630f38e2688a53b25b3cabc Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Wed, 2 Mar 2022 18:00:54 -0500 Subject: [PATCH 131/150] added id field to RequestListItem. PI/managers can now see proj join requests for their projects --- .../core/user/views_/request_hub_views.py | 40 +++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/coldfront/core/user/views_/request_hub_views.py b/coldfront/core/user/views_/request_hub_views.py index 8595eef9a..2d7f206a2 100644 --- a/coldfront/core/user/views_/request_hub_views.py +++ b/coldfront/core/user/views_/request_hub_views.py @@ -11,7 +11,7 @@ from coldfront.core.project.models import (ProjectUserRemovalRequest, SavioProjectAllocationRequest, VectorProjectAllocationRequest, - ProjectUserJoinRequest) + ProjectUserJoinRequest, Project) from coldfront.core.project.utils_.permissions_utils import \ is_user_manager_or_pi_of_project @@ -24,7 +24,7 @@ class RequestListItem: __slots__ = ['num', 'title', 'num_pending', 'table', 'pending_queryset', 'complete_queryset', - 'button_path', 'button_text'] + 'button_path', 'button_text', 'id'] def __init__(self): num = None @@ -35,11 +35,12 @@ def __init__(self): complete_queryset = None button_path = None button_text = None + id = None class RequestHubView(LoginRequiredMixin, - UserPassesTestMixin, - TemplateView): + UserPassesTestMixin, + TemplateView): template_name = 'request_hub/request_hub.html' paginate_by = 10 paginators = 0 @@ -53,8 +54,6 @@ def test_func(self): else: return True - return False - def create_paginator(self, queryset): """ Creates a paginator object for the given queryset @@ -112,6 +111,7 @@ def get_cluster_account_request(self): 'allocation-cluster-account-request-list' cluster_request_object.button_text = \ 'Go To Cluster Account Requests Main Page' + cluster_request_object.id = 'cluster_account_request_section' return cluster_request_object @@ -150,6 +150,7 @@ def get_project_removal_request(self): 'project-removal-request-list' removal_request_object.button_text = \ 'Go To Project Removal Requests Main Page' + removal_request_object.id = 'project_removal_request_section' return removal_request_object @@ -190,6 +191,7 @@ def get_savio_project_request(self): 'savio-project-pending-request-list' savio_proj_request_object.button_text = \ 'Go To Savio Project Requests Main Page' + savio_proj_request_object.id = 'savio_project_request_section' return savio_proj_request_object @@ -230,6 +232,7 @@ def get_vector_project_request(self): 'vector-project-pending-request-list' vector_proj_request_object.button_text = \ 'Go To Vector Project Requests Main Page' + vector_proj_request_object.id = 'vector_project_request_section' return vector_proj_request_object @@ -240,18 +243,28 @@ def get_project_join_request(self): args = [] if not self.show_all_requests: - args.append(Q(project_user__user=user)) + project_set = set() + for project in Project.objects.all(): + pi_condition = Q( + role__name='Principal Investigator') + manager_condition = Q(role__name='Manager') + + if project.projectuser_set.filter(user=user, + status__name='Active').filter( + Q(pi_condition | manager_condition)).count() > 0: + project_set.add(project) + + args.append(Q(project_user__user=user) | Q(project_user__project__in=project_set)) project_join_request_pending = \ ProjectUserJoinRequest.objects.filter( - project_user__status__name='Pending - Add', *args).order_by( - 'modified') + project_user__status__name='Pending - Add', + *args).order_by('modified') project_join_request_complete = \ ProjectUserJoinRequest.objects.filter( - project_user__status__name__in=['pending', 'Denied'], - *args).order_by( - 'modified') + project_user__status__name__in=['Active', 'Denied'], + *args).order_by('modified') proj_join_request_object.num = self.paginators proj_join_request_object.pending_queryset = \ @@ -270,6 +283,7 @@ def get_project_join_request(self): 'project-join-request-list' proj_join_request_object.button_text = \ 'Go To Project Join Requests Main Page' + proj_join_request_object.id = 'project_join_request_section' return proj_join_request_object @@ -309,6 +323,7 @@ def get_project_renewal_request(self): 'pi-allocation-renewal-pending-request-list' proj_renewal_request_object.button_text = \ 'Go To Project Renewal Requests Main Page' + proj_renewal_request_object.id = 'project_renewal_request_section' return proj_renewal_request_object @@ -353,6 +368,7 @@ def get_su_purchase_request(self): 'service-units-purchase-pending-request-list' su_purchase_request_object.button_text = \ 'Go To Service Unit Purchase Requests Main Page' + su_purchase_request_object.id = 'service_unit_purchase_request_section' return su_purchase_request_object From fc822548c7eb279ed5e69c32d270696dbe29a693 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Thu, 3 Mar 2022 09:22:27 -0500 Subject: [PATCH 132/150] removed imports --- coldfront/core/user/views_/request_hub_views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/coldfront/core/user/views_/request_hub_views.py b/coldfront/core/user/views_/request_hub_views.py index 2d7f206a2..74fb0f0f2 100644 --- a/coldfront/core/user/views_/request_hub_views.py +++ b/coldfront/core/user/views_/request_hub_views.py @@ -1,4 +1,3 @@ -from django.contrib import messages from django.contrib.auth.mixins import LoginRequiredMixin, UserPassesTestMixin from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q From 814143ac97a10ab9e4cc684f71094e6e684d184c Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Thu, 3 Mar 2022 09:22:44 -0500 Subject: [PATCH 133/150] added tests for each request type --- .../tests/test_views/test_request_hub_view.py | 211 ++++++++++++++++-- 1 file changed, 195 insertions(+), 16 deletions(-) diff --git a/coldfront/core/user/tests/test_views/test_request_hub_view.py b/coldfront/core/user/tests/test_views/test_request_hub_view.py index 9c69edf5d..2df4c1673 100644 --- a/coldfront/core/user/tests/test_views/test_request_hub_view.py +++ b/coldfront/core/user/tests/test_views/test_request_hub_view.py @@ -3,30 +3,29 @@ import datetime from decimal import Decimal from http import HTTPStatus -from io import StringIO - from bs4 import BeautifulSoup from django.core.management import call_command from django.test import TestCase +from django.contrib.auth.models import User +from django.urls import reverse from coldfront.api.statistics.utils import create_project_allocation, \ create_user_project_allocation, get_accounting_allocation_objects from coldfront.core.allocation.models import AllocationUserAttribute, \ - AllocationAttributeType, Allocation + AllocationAttributeType, \ + allocation_renewal_request_state_schema, AllocationPeriod, \ + AllocationRenewalRequestStatusChoice, AllocationRenewalRequest, \ + allocation_addition_request_state_schema, \ + AllocationAdditionRequestStatusChoice, AllocationAdditionRequest from coldfront.core.project.models import ProjectStatusChoice, \ ProjectUserStatusChoice, ProjectUserRoleChoice, Project, ProjectUser, \ ProjectUserRemovalRequestStatusChoice, ProjectUserRemovalRequest, \ SavioProjectAllocationRequest, ProjectAllocationRequestStatusChoice, \ savio_project_request_state_schema, vector_project_request_state_schema, \ VectorProjectAllocationRequest, ProjectUserJoinRequest -from coldfront.core.user.models import EmailAddress, UserProfile -from coldfront.core.user.tests.utils import TestUserBase -from coldfront.core.user.utils import account_activation_url -from django.contrib.auth.models import User -from django.contrib.messages import get_messages -from django.test import Client -from django.urls import reverse +from coldfront.core.user.models import UserProfile +from coldfront.core.utils.common import utc_now_offset_aware class TestRequestHubView(TestCase): @@ -41,6 +40,7 @@ def setUp(self): call_command('add_allocation_defaults') call_command('add_brc_accounting_defaults') call_command('create_staff_group') + call_command('create_allocation_periods') sys.stdout = sys.__stdout__ self.password = 'password' @@ -123,10 +123,12 @@ def get_response(self, user, url): self.client.login(username=user.username, password=self.password) return self.client.get(url) - def assert_no_requests(self, user, url): + def assert_no_requests(self, user, url, exclude=None): response = self.get_response(user, url) soup = BeautifulSoup(response.content, 'html.parser') for request in self.requests: + if request == exclude: + continue divs = soup.find(id=f'{request.replace(" ", "_")}_section'). \ find_all('div', {'class': 'alert alert-info'}) self.assertEqual(len(divs), 2) @@ -223,10 +225,22 @@ def assert_request_shown(user, url): completed_req = \ AllocationUserAttribute.objects.create(value='Denied', **kwargs) + # assert the correct requests are shown assert_request_shown(self.user0, self.url) assert_request_shown(self.admin, self.admin_url) assert_request_shown(self.staff, self.admin_url) + # other request sections should be empty + self.assert_no_requests(self.user0, self.url, exclude='cluster account request') + self.assert_no_requests(self.admin, self.admin_url, exclude='cluster account request') + self.assert_no_requests(self.staff, self.admin_url, exclude='cluster account request') + + # should not see any requests + self.assert_no_requests(self.user1, self.url) + self.assert_no_requests(self.pi, self.url) + self.assert_no_requests(self.staff, self.url) + self.assert_no_requests(self.admin, self.url) + def test_project_removal_requests(self): """Testing that project removal requests appear""" def assert_request_shown(user, url): @@ -253,7 +267,7 @@ def assert_request_shown(user, url): ProjectUserRemovalRequestStatusChoice.objects.get(name='Processing') complete_status = \ ProjectUserRemovalRequestStatusChoice.objects.get(name='Complete') - current_time = datetime.datetime.now() + current_time = utc_now_offset_aware() kwargs = { 'project_user': self.project0_user0, @@ -268,11 +282,23 @@ def assert_request_shown(user, url): **kwargs ) + # assert the correct requests are shown assert_request_shown(self.user0, self.url) assert_request_shown(self.pi, self.url) assert_request_shown(self.admin, self.admin_url) assert_request_shown(self.staff, self.admin_url) + # other request sections should be empty + self.assert_no_requests(self.user0, self.url, exclude='project removal request') + self.assert_no_requests(self.pi, self.url, exclude='project removal request') + self.assert_no_requests(self.admin, self.admin_url, exclude='project removal request') + self.assert_no_requests(self.staff, self.admin_url, exclude='project removal request') + + # should not see any requests + self.assert_no_requests(self.user1, self.url) + self.assert_no_requests(self.staff, self.url) + self.assert_no_requests(self.admin, self.url) + def test_savio_project_requests(self): """Testing that savio project requests appear""" @@ -317,11 +343,23 @@ def assert_request_shown(user, url): completed_req = SavioProjectAllocationRequest.objects.create( status=complete_status, **kwargs) + # assert the correct requests are shown assert_request_shown(self.user0, self.url) assert_request_shown(self.pi, self.url) assert_request_shown(self.admin, self.admin_url) assert_request_shown(self.staff, self.admin_url) + # other request sections should be empty + self.assert_no_requests(self.user0, self.url, exclude='savio project request') + self.assert_no_requests(self.pi, self.url, exclude='savio project request') + self.assert_no_requests(self.admin, self.admin_url, exclude='savio project request') + self.assert_no_requests(self.staff, self.admin_url, exclude='savio project request') + + # should not see any requests + self.assert_no_requests(self.user1, self.url) + self.assert_no_requests(self.staff, self.url) + self.assert_no_requests(self.admin, self.url) + def test_vector_project_requests(self): """Testing that vector project requests appear""" def assert_request_shown(user, url): @@ -363,11 +401,23 @@ def assert_request_shown(user, url): completed_req = VectorProjectAllocationRequest.objects.create( status=complete_status, **kwargs) + # assert the correct requests are shown assert_request_shown(self.user0, self.url) assert_request_shown(self.pi, self.url) assert_request_shown(self.admin, self.admin_url) assert_request_shown(self.staff, self.admin_url) + # other request sections should be empty + self.assert_no_requests(self.user0, self.url, exclude='vector project request') + self.assert_no_requests(self.pi, self.url, exclude='vector project request') + self.assert_no_requests(self.admin, self.admin_url, exclude='vector project request') + self.assert_no_requests(self.staff, self.admin_url, exclude='vector project request') + + # should not see any requests + self.assert_no_requests(self.user1, self.url) + self.assert_no_requests(self.staff, self.url) + self.assert_no_requests(self.admin, self.url) + def test_project_join_requests(self): """Testing that project join requests appear correctly""" @@ -407,13 +457,142 @@ def assert_request_shown(user, url, section): project_user=self.project0_user0, reason='Request hub testing.') + # assert the correct requests are shown assert_request_shown(self.user0, self.url, 'completed') assert_request_shown(self.user1, self.url, 'pending') assert_request_shown(self.pi, self.url, 'both') assert_request_shown(self.admin, self.admin_url, 'both') assert_request_shown(self.staff, self.admin_url, 'both') - # def test_project_renewal_requests(self): - # """Testing that project renewal requests appear correctly""" - # - # AllocationRenewalRequest \ No newline at end of file + # other request sections should be empty + self.assert_no_requests(self.user0, self.url, exclude='project join request') + self.assert_no_requests(self.user1, self.url, exclude='project join request') + self.assert_no_requests(self.pi, self.url, exclude='project join request') + self.assert_no_requests(self.admin, self.admin_url, exclude='project join request') + self.assert_no_requests(self.staff, self.admin_url, exclude='project join request') + + # should not see any requests + self.assert_no_requests(self.staff, self.url) + self.assert_no_requests(self.admin, self.url) + + def test_project_renewal_requests(self): + """Testing that project renewal requests appear correctly""" + + def assert_request_shown(user, url): + response = self.get_response(user, url) + soup = BeautifulSoup(response.content, 'html.parser') + + pending_div = str( + soup.find(id=f'project_renewal_request_section_pending')) + self.assertIn(str(pending_req.pk), pending_div) + self.assertIn(pending_req.requester.email, pending_div) + self.assertIn(pending_req.pre_project.name, pending_div) + self.assertIn(pending_req.post_project.name, pending_div) + self.assertIn(pending_req.modified.strftime("%b. %d, %Y"), pending_div) + self.assertIn(pending_req.status.name, pending_div) + + completed_div = str( + soup.find(id=f'project_renewal_request_section_completed')) + self.assertIn(str(completed_req.pk), completed_div) + self.assertIn(completed_req.requester.email, completed_div) + self.assertIn(completed_req.pre_project.name, completed_div) + self.assertIn(completed_req.post_project.name, completed_div) + self.assertIn(completed_req.modified.strftime("%b. %d, %Y"), completed_div) + self.assertIn(completed_req.status.name, completed_div) + + kwargs = { + 'pi': self.pi, + 'requester': self.user0, + 'allocation_period': AllocationPeriod.objects.get(name='AY21-22'), + 'pre_project': self.project0, + 'post_project': self.project0, + 'num_service_units': 1000, + 'request_time': utc_now_offset_aware(), + 'state': allocation_renewal_request_state_schema() + } + pending_status = AllocationRenewalRequestStatusChoice.objects.get( + name='Approved') + complete_status = AllocationRenewalRequestStatusChoice.objects.get( + name='Complete') + pending_req = AllocationRenewalRequest.objects.create( + status=pending_status, **kwargs) + completed_req = AllocationRenewalRequest.objects.create( + status=complete_status, **kwargs) + + # assert the correct requests are shown + assert_request_shown(self.user0, self.url) + assert_request_shown(self.pi, self.url) + assert_request_shown(self.admin, self.admin_url) + assert_request_shown(self.staff, self.admin_url) + + # other request sections should be empty + self.assert_no_requests(self.user0, self.url, exclude='project renewal request') + self.assert_no_requests(self.pi, self.url, exclude='project renewal request') + self.assert_no_requests(self.admin, self.admin_url, exclude='project renewal request') + self.assert_no_requests(self.staff, self.admin_url, exclude='project renewal request') + + # should not see any requests + self.assert_no_requests(self.user1, self.url) + self.assert_no_requests(self.staff, self.url) + self.assert_no_requests(self.admin, self.url) + + def test_su_purchase_requests(self): + """Testing that su purchase requests appear correctly""" + + def assert_request_shown(user, url): + response = self.get_response(user, url) + soup = BeautifulSoup(response.content, 'html.parser') + + pending_div = str( + soup.find(id=f'service_unit_purchase_request_section_pending')) + self.assertIn(str(pending_req.pk), pending_div) + self.assertIn(pending_req.requester.email, pending_div) + self.assertIn(pending_req.project.name, pending_div) + self.assertIn(str(pending_req.num_service_units), pending_div) + self.assertIn(pending_req.modified.strftime("%b. %d, %Y"), pending_div) + self.assertIn(pending_req.status.name, pending_div) + + completed_div = str( + soup.find(id=f'service_unit_purchase_request_section_completed')) + self.assertIn(str(completed_req.pk), completed_div) + self.assertIn(completed_req.requester.email, completed_div) + self.assertIn(completed_req.project.name, completed_div) + self.assertIn(str(completed_req.num_service_units), completed_div) + self.assertIn(completed_req.modified.strftime("%b. %d, %Y"), completed_div) + self.assertIn(completed_req.status.name, completed_div) + + current_time = utc_now_offset_aware() + kwargs = { + 'requester': self.pi, + 'project': self.project0, + 'num_service_units': 1000, + 'request_time': current_time, + 'state': allocation_addition_request_state_schema() + } + + pending_status = AllocationAdditionRequestStatusChoice.objects.get( + name='Under Review') + complete_status = AllocationAdditionRequestStatusChoice.objects.get( + name='Complete') + pending_req = AllocationAdditionRequest.objects.create( + status=pending_status, **kwargs) + completed_req = AllocationAdditionRequest.objects.create( + status=complete_status, + completion_time=current_time + datetime.timedelta(days=4), + **kwargs) + + # assert the correct requests are shown + assert_request_shown(self.pi, self.url) + assert_request_shown(self.admin, self.admin_url) + assert_request_shown(self.staff, self.admin_url) + + # other request sections should be empty + self.assert_no_requests(self.pi, self.url, exclude='service unit purchase request') + self.assert_no_requests(self.admin, self.admin_url, exclude='service unit purchase request') + self.assert_no_requests(self.staff, self.admin_url, exclude='service unit purchase request') + + # should not see any requests + self.assert_no_requests(self.user0, self.url) + self.assert_no_requests(self.user1, self.url) + self.assert_no_requests(self.staff, self.url) + self.assert_no_requests(self.admin, self.url) From 8ab2933be7c5cbbe46edebe4c22e1c6269f195e8 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Thu, 3 Mar 2022 09:41:35 -0500 Subject: [PATCH 134/150] added tests for pending notification badge --- .../tests/test_views/test_request_hub_view.py | 89 +++++++++++++++---- 1 file changed, 72 insertions(+), 17 deletions(-) diff --git a/coldfront/core/user/tests/test_views/test_request_hub_view.py b/coldfront/core/user/tests/test_views/test_request_hub_view.py index 2df4c1673..6388d9b3c 100644 --- a/coldfront/core/user/tests/test_views/test_request_hub_view.py +++ b/coldfront/core/user/tests/test_views/test_request_hub_view.py @@ -137,6 +137,13 @@ def assert_no_requests(self, user, url, exclude=None): self.assertIn('No pending' if i == 0 else 'No completed', str(div)) + def assert_pending_request_badge_shown(self, section, response, num_requests): + soup = BeautifulSoup(response.content, 'html.parser') + divs = soup.find(id=section) + notification = f'{num_requests} pending request' \ + f'{"s" if num_requests > 1 else ""}' + self.assertIn(notification, str(divs)) + def test_access(self): """Testing access to RequestHubView""" @@ -188,17 +195,24 @@ def test_cluster_account_requests(self): """Testing that cluster account requests appear""" def assert_request_shown(user, url): + section = 'cluster_account_request_section' response = self.get_response(user, url) soup = BeautifulSoup(response.content, 'html.parser') + # notification badge is shown + self.assert_pending_request_badge_shown( + section, response, 1) + + # pending request is shown pending_div = str( - soup.find(id=f'cluster_account_request_section_pending')) + soup.find(id=f'{section}_pending')) self.assertIn(str(pending_req.pk), pending_div) self.assertIn(pending_req.allocation_user.user.email, pending_div) self.assertIn(pending_req.value, pending_div) + # completed request is shown completed_div = str( - soup.find(id=f'cluster_account_request_section_completed')) + soup.find(id=f'{section}_completed')) self.assertIn(str(completed_req.pk), completed_div) self.assertIn(completed_req.allocation_user.user.email, completed_div) @@ -246,17 +260,24 @@ def test_project_removal_requests(self): def assert_request_shown(user, url): response = self.get_response(user, url) soup = BeautifulSoup(response.content, 'html.parser') + section = 'project_removal_request_section' + # notification badge is shown + self.assert_pending_request_badge_shown( + section, response, 1) + + # pending request is shown pending_div = str( - soup.find(id=f'project_removal_request_section_pending')) + soup.find(id=f'{section}_pending')) self.assertIn(str(pending_req.pk), pending_div) self.assertIn(pending_req.project_user.user.username, pending_div) self.assertIn(pending_req.requester.username, pending_div) self.assertIn(pending_req.request_time.strftime("%b. %d, %Y"), pending_div) self.assertIn(pending_req.status.name, pending_div) + # completed request is shown completed_div = str( - soup.find(id=f'project_removal_request_section_completed')) + soup.find(id=f'{section}_completed')) self.assertIn(str(completed_req.pk), pending_div) self.assertIn(completed_req.project_user.user.username, completed_div) self.assertIn(completed_req.requester.username, completed_div) @@ -305,17 +326,24 @@ def test_savio_project_requests(self): def assert_request_shown(user, url): response = self.get_response(user, url) soup = BeautifulSoup(response.content, 'html.parser') + section = 'savio_project_request_section' + + # notification badge is shown + self.assert_pending_request_badge_shown( + section, response, 1) + # pending request is shown pending_div = str( - soup.find(id=f'savio_project_request_section_pending')) + soup.find(id=f'{section}_pending')) self.assertIn(str(pending_req.pk), pending_div) self.assertIn(pending_req.requester.email, pending_div) self.assertIn(pending_req.pi.email, pending_div) self.assertIn(pending_req.modified.strftime("%b. %d, %Y"), pending_div) self.assertIn(pending_req.status.name, pending_div) + # completed request is shown completed_div = str( - soup.find(id=f'savio_project_request_section_completed')) + soup.find(id=f'{section}_completed')) self.assertIn(str(completed_req.pk), completed_div) self.assertIn(completed_req.requester.email, completed_div) self.assertIn(completed_req.pi.email, completed_div) @@ -365,17 +393,24 @@ def test_vector_project_requests(self): def assert_request_shown(user, url): response = self.get_response(user, url) soup = BeautifulSoup(response.content, 'html.parser') + section = 'vector_project_request_section' + # notification badge is shown + self.assert_pending_request_badge_shown( + section, response, 1) + + # pending request is shown pending_div = str( - soup.find(id=f'vector_project_request_section_pending')) + soup.find(id=f'{section}_pending')) self.assertIn(str(pending_req.pk), pending_div) self.assertIn(pending_req.requester.email, pending_div) self.assertIn(pending_req.pi.email, pending_div) self.assertIn(pending_req.modified.strftime("%b. %d, %Y"), pending_div) self.assertIn(pending_req.status.name, pending_div) + # completed request is shown completed_div = str( - soup.find(id=f'vector_project_request_section_completed')) + soup.find(id=f'{section}_completed')) self.assertIn(str(completed_req.pk), completed_div) self.assertIn(completed_req.requester.email, completed_div) self.assertIn(completed_req.pi.email, completed_div) @@ -421,22 +456,29 @@ def assert_request_shown(user, url): def test_project_join_requests(self): """Testing that project join requests appear correctly""" - def assert_request_shown(user, url, section): + def assert_request_shown(user, url, status): response = self.get_response(user, url) soup = BeautifulSoup(response.content, 'html.parser') + section = 'project_join_request_section' + + # pending request is shown + if status == 'both' or status == 'pending': + # notification badge is shown + self.assert_pending_request_badge_shown( + section, response, 1) - if section == 'both' or section == 'pending': pending_div = str( - soup.find(id=f'project_join_request_section_pending')) + soup.find(id=f'{section}_pending')) self.assertIn(str(pending_req.pk), pending_div) self.assertIn(pending_req.project_user.user.username, pending_div) self.assertIn(pending_req.project_user.project.name, pending_div) self.assertIn(pending_req.created.strftime("%b. %d, %Y"), pending_div) self.assertIn(pending_req.reason, pending_div) - if section == 'both' or section == 'completed': + # completed request is shown + if status == 'both' or status == 'completed': completed_div = str( - soup.find(id=f'project_join_request_section_completed')) + soup.find(id=f'{section}_completed')) self.assertIn(str(completed_req.pk), completed_div) self.assertIn(completed_req.project_user.user.username, completed_div) self.assertIn(completed_req.project_user.project.name, completed_div) @@ -481,9 +523,15 @@ def test_project_renewal_requests(self): def assert_request_shown(user, url): response = self.get_response(user, url) soup = BeautifulSoup(response.content, 'html.parser') + section = 'project_renewal_request_section' + # notification badge is shown + self.assert_pending_request_badge_shown( + section, response, 1) + + # pending request is shown pending_div = str( - soup.find(id=f'project_renewal_request_section_pending')) + soup.find(id=f'{section}_pending')) self.assertIn(str(pending_req.pk), pending_div) self.assertIn(pending_req.requester.email, pending_div) self.assertIn(pending_req.pre_project.name, pending_div) @@ -491,8 +539,9 @@ def assert_request_shown(user, url): self.assertIn(pending_req.modified.strftime("%b. %d, %Y"), pending_div) self.assertIn(pending_req.status.name, pending_div) + # completed request is shown completed_div = str( - soup.find(id=f'project_renewal_request_section_completed')) + soup.find(id=f'{section}_completed')) self.assertIn(str(completed_req.pk), completed_div) self.assertIn(completed_req.requester.email, completed_div) self.assertIn(completed_req.pre_project.name, completed_div) @@ -542,9 +591,15 @@ def test_su_purchase_requests(self): def assert_request_shown(user, url): response = self.get_response(user, url) soup = BeautifulSoup(response.content, 'html.parser') + section = 'service_unit_purchase_request_section' + + # notification badge is shown + self.assert_pending_request_badge_shown( + section, response, 1) + # pending request is shown pending_div = str( - soup.find(id=f'service_unit_purchase_request_section_pending')) + soup.find(id=f'{section}_pending')) self.assertIn(str(pending_req.pk), pending_div) self.assertIn(pending_req.requester.email, pending_div) self.assertIn(pending_req.project.name, pending_div) @@ -553,7 +608,7 @@ def assert_request_shown(user, url): self.assertIn(pending_req.status.name, pending_div) completed_div = str( - soup.find(id=f'service_unit_purchase_request_section_completed')) + soup.find(id=f'{section}_completed')) self.assertIn(str(completed_req.pk), completed_div) self.assertIn(completed_req.requester.email, completed_div) self.assertIn(completed_req.project.name, completed_div) From 2b2776abc44988861fc2c666d7f3b18af3900e90 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Thu, 3 Mar 2022 09:42:04 -0500 Subject: [PATCH 135/150] pending notification badge is plural when num_requests > 1 --- coldfront/core/user/templates/request_hub/request_section.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coldfront/core/user/templates/request_hub/request_section.html b/coldfront/core/user/templates/request_hub/request_section.html index bd11f5d51..6fa24bcc0 100644 --- a/coldfront/core/user/templates/request_hub/request_section.html +++ b/coldfront/core/user/templates/request_hub/request_section.html @@ -8,7 +8,7 @@ {% if request_obj.num_pending %} - {{ request_obj.num_pending }} pending requests + {{ request_obj.num_pending }} pending request{% if request_obj.num_pending > 1 %}s{% endif %} {% endif %}
    From 98c85831f35f956743eea6b8807bf0885d367b0a Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Thu, 3 Mar 2022 09:47:16 -0500 Subject: [PATCH 136/150] TestRequestHubView now inherits TestBase --- .../tests/test_views/test_request_hub_view.py | 37 +++++-------------- 1 file changed, 9 insertions(+), 28 deletions(-) diff --git a/coldfront/core/user/tests/test_views/test_request_hub_view.py b/coldfront/core/user/tests/test_views/test_request_hub_view.py index 6388d9b3c..86dbe1da5 100644 --- a/coldfront/core/user/tests/test_views/test_request_hub_view.py +++ b/coldfront/core/user/tests/test_views/test_request_hub_view.py @@ -1,12 +1,7 @@ -import os -import sys import datetime from decimal import Decimal -from http import HTTPStatus from bs4 import BeautifulSoup -from django.core.management import call_command -from django.test import TestCase from django.contrib.auth.models import User from django.urls import reverse @@ -26,22 +21,15 @@ VectorProjectAllocationRequest, ProjectUserJoinRequest from coldfront.core.user.models import UserProfile from coldfront.core.utils.common import utc_now_offset_aware +from coldfront.core.utils.tests.test_base import TestBase -class TestRequestHubView(TestCase): +class TestRequestHubView(TestBase): """A class for testing RequestHubView.""" def setUp(self): """Set up test data.""" - sys.stdout = open(os.devnull, 'w') - call_command('import_field_of_science_data') - call_command('add_default_project_choices') - call_command('add_resource_defaults') - call_command('add_allocation_defaults') - call_command('add_brc_accounting_defaults') - call_command('create_staff_group') - call_command('create_allocation_periods') - sys.stdout = sys.__stdout__ + super().setUp() self.password = 'password' @@ -112,13 +100,6 @@ def setUp(self): 'project renewal request', 'service unit purchase request'] - def assert_has_access(self, user, url, has_access): - self.client.login(username=user.username, password=self.password) - response = self.client.get(url) - status_code = HTTPStatus.OK if has_access else HTTPStatus.FORBIDDEN - self.assertEqual(response.status_code, status_code) - self.client.logout() - def get_response(self, user, url): self.client.login(username=user.username, password=self.password) return self.client.get(url) @@ -149,14 +130,14 @@ def test_access(self): # all users should have access to the normal view for user in User.objects.all(): - self.assert_has_access(user, self.url, True) + self.assert_has_access(self.url, user, True) # only staff/admin should have access to request-hub-admin - self.assert_has_access(self.admin, self.admin_url, True) - self.assert_has_access(self.staff, self.admin_url, True) - self.assert_has_access(self.pi, self.admin_url, False) - self.assert_has_access(self.user0, self.admin_url, False) - self.assert_has_access(self.user1, self.admin_url, False) + self.assert_has_access(self.admin_url, self.admin, True) + self.assert_has_access(self.admin_url, self.staff, True) + self.assert_has_access(self.admin_url, self.pi, False) + self.assert_has_access(self.admin_url, self.user0, False) + self.assert_has_access(self.admin_url, self.user1, False) def test_admin_buttons(self): """Test that 'Go to main request page' buttons only appear for From d268777e8279ce8d8ce6b27b0ce1a9c737eea0e9 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Fri, 4 Mar 2022 18:34:04 -0500 Subject: [PATCH 137/150] adding template for help text popover --- .../user/templates/request_hub/help_text_popover.html | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 coldfront/core/user/templates/request_hub/help_text_popover.html diff --git a/coldfront/core/user/templates/request_hub/help_text_popover.html b/coldfront/core/user/templates/request_hub/help_text_popover.html new file mode 100644 index 000000000..22a3f31ab --- /dev/null +++ b/coldfront/core/user/templates/request_hub/help_text_popover.html @@ -0,0 +1,11 @@ +     + Help Text + + + + \ No newline at end of file From 4f9ea1d6cbfc575a6a8a459320561ca137e496d5 Mon Sep 17 00:00:00 2001 From: jofeinstein Date: Fri, 4 Mar 2022 18:35:33 -0500 Subject: [PATCH 138/150] changed alignment of buttons. changed help text --- coldfront/core/user/templates/request_hub/request_hub.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/coldfront/core/user/templates/request_hub/request_hub.html b/coldfront/core/user/templates/request_hub/request_hub.html index ef17e6cfc..e4b615923 100644 --- a/coldfront/core/user/templates/request_hub/request_hub.html +++ b/coldfront/core/user/templates/request_hub/request_hub.html @@ -10,12 +10,12 @@

    Request Hub

    {% if show_all %} - Below are all of the requests in the myBRC Portal. Click on a request + Below are all of the requests in MyBRC. Click on a request section to view requests of that type. To perform actions on a specific request, click the button to go to the request's main page and perform the actions there. Click here to view only your requests. {% else %} - Below are all of your requests in the myBRC Portal. Click on a request + Below are all of your requests in MyBRC. Click on a request section to view your requests of that type. {% if admin_staff %} @@ -30,7 +30,7 @@

    Request Hub

    -
    +