Merge pull request #416 from ucb-rit/develop

Update secure directory, accounting, email logic; fix bugs

bootstrap/ansible/playbook.yml

@@ -217,6 +217,15 @@
         virtualenv: "{{ git_prefix }}/venv"
       become_user: "{{ djangooperator }}"
 
+    - name: Run Django management command - create_allocation_periods
+      django_manage:
+        command: create_allocation_periods
+        app_path: "{{ git_prefix }}/{{ reponame }}"
+        settings: "config.settings"
+        pythonpath: "{{ git_prefix }}/{{ reponame }}/{{ djangoprojname }}"
+        virtualenv: "{{ git_prefix }}/venv"
+      become_user: "{{ djangooperator }}"
+
     - name: Run Django management command - create_staff_group
       django_manage:
         command: create_staff_group

bootstrap/development/playbook.yml

@@ -333,6 +333,15 @@
         virtualenv: "{{ git_prefix }}/venv"
       become_user: "{{ djangooperator }}"
 
+    - name: Run Django management command - create_allocation_periods
+      django_manage:
+        command: create_allocation_periods
+        app_path: "{{ git_prefix }}/{{ reponame }}"
+        settings: "config.settings"
+        pythonpath: "{{ git_prefix }}/{{ reponame }}/{{ djangoprojname }}"
+        virtualenv: "{{ git_prefix }}/venv"
+      become_user: "{{ djangooperator }}"
+
     - name: Run Django management command - create_staff_group
       django_manage:
         command: create_staff_group

coldfront/api/statistics/tests/test_can_submit_job.py

@@ -85,7 +85,7 @@ def test_no_user_account_association(self):
         fail."""
         self.project_user.delete()
         message = f'User user0 is not a member of account test_project.'
-        self.assert_result('1.00', '0', 'test_project', 400, False, message)
+        self.assert_result('1.00', '0', 'test_project', 200, False, message)
 
     def test_no_active_compute_allocation(self):
         """Test that requests wherein the account has no active compute
@@ -95,17 +95,17 @@ def test_no_active_compute_allocation(self):
         self.allocation.status = AllocationStatusChoice.objects.get(
             name='Expired')
         self.allocation.save()
-        self.assert_result('1.00', '0', 'test_project', 400, False, message)
+        self.assert_result('1.00', '0', 'test_project', 200, False, message)
         # The allocation is active, but does not have Savio Compute as a
         # resource.
         self.allocation.status = AllocationStatusChoice.objects.get(
             name='Active')
         self.allocation.resources.all().delete()
         self.allocation.save()
-        self.assert_result('1.00', '0', 'test_project', 400, False, message)
+        self.assert_result('1.00', '0', 'test_project', 200, False, message)
         # The allocation does not exist.
         self.allocation.delete()
-        self.assert_result('1.00', '0', 'test_project', 400, False, message)
+        self.assert_result('1.00', '0', 'test_project', 200, False, message)
 
     def test_user_not_member_of_compute_allocation(self):
         """Test that requests wherein the user is not an active member
@@ -117,10 +117,10 @@ def test_user_not_member_of_compute_allocation(self):
         self.allocation_user.status = AllocationUserStatusChoice.objects.get(
             name='Removed')
         self.allocation_user.save()
-        self.assert_result('1.00', '0', 'test_project', 400, False, message)
+        self.assert_result('1.00', '0', 'test_project', 200, False, message)
         # The allocation user does not exist.
         self.allocation_user.delete()
-        self.assert_result('1.00', '0', 'test_project', 400, False, message)
+        self.assert_result('1.00', '0', 'test_project', 200, False, message)
 
     def test_bad_database_state_causes_server_error(self):
         """Test that requests fails if there are too few or too many of

coldfront/api/statistics/tests/test_job_base.py

@@ -12,6 +12,7 @@
 from coldfront.core.user.models import UserProfile
 from coldfront.core.utils.common import display_time_zone_date_to_utc_datetime
 from datetime import datetime
+from datetime import timedelta
 from decimal import Decimal
 from django.contrib.auth.models import User
 
@@ -65,7 +66,7 @@ def setUp(self):
         self.default_start = display_time_zone_date_to_utc_datetime(
             period_start_date)
         self.default_end = display_time_zone_date_to_utc_datetime(
-            period_end_date)
+            period_end_date) + timedelta(hours=24) - timedelta(microseconds=1)
 
         # Create a "CLUSTER_NAME Compute" Allocation for the Project.
         allocation_objects = create_project_allocation(

coldfront/api/statistics/tests/test_job_view_set.py

@@ -23,6 +23,7 @@
 from coldfront.core.statistics.models import Job
 from coldfront.core.user.models import ExpiringToken
 from coldfront.core.user.models import UserProfile
+from coldfront.core.utils.common import utc_datetime_to_display_time_zone_date
 
 from django.contrib.auth.models import User
 from django.db.models import Sum
@@ -69,8 +70,10 @@ def setUp(self):
                 name=f'PROJECT_{i}', status=project_status)
             allocation_objects = create_project_allocation(
                 project, allocation_amount)
-            allocation_objects.allocation.start_date = self.default_start
-            allocation_objects.allocation.end_date = self.default_end
+            allocation_objects.allocation.start_date = \
+                utc_datetime_to_display_time_zone_date(self.default_start)
+            allocation_objects.allocation.end_date = \
+                utc_datetime_to_display_time_zone_date(self.default_end)
             allocation_objects.allocation.save()
             allocation_pks[project.pk] = allocation_objects.allocation.pk
 

coldfront/api/statistics/views.py

@@ -5,6 +5,7 @@
 from datetime import date
 from datetime import datetime
 from datetime import MAXYEAR
+from datetime import timedelta
 from decimal import Decimal, InvalidOperation
 
 from django.core.exceptions import MultipleObjectsReturned
@@ -180,8 +181,11 @@ def get_queryset(self):
                     get_current_allowance_year_period()
                 default_start = display_time_zone_date_to_utc_datetime(
                     current_allowance_year_period.start_date)
-                default_end = display_time_zone_date_to_utc_datetime(
-                    current_allowance_year_period.end_date)
+                default_end = (
+                    display_time_zone_date_to_utc_datetime(
+                        current_allowance_year_period.end_date) +
+                    timedelta(hours=24) -
+                    timedelta(microseconds=1))
             except Exception as e:
                 raise serializers.ValidationError(
                     f'Failed to retrieve default start and end times. '
@@ -496,14 +500,16 @@ def validate_job_dates(job_data, allocation, end_date_expected=False):
                     f'Allocation {allocation.pk} (Project {account_name}) '
                     f'does not have an end date.')
                 return False
+            allocation_end_dt_utc = (
+                display_time_zone_date_to_utc_datetime(allocation_end_date) +
+                timedelta(hours=24) -
+                timedelta(microseconds=1))
         else:
-            allocation_end_date = date(MAXYEAR, 12, 31)
+            allocation_end_dt_utc = datetime.max.replace(tzinfo=pytz.utc)
 
-        # The Job should not have ended after its corresponding Allocation's
-        # end date.
+        # The Job should not have ended after the last microsecond of its
+        # corresponding Allocation's end date.
         job_end_dt_utc = expected_dates['enddate']
-        allocation_end_dt_utc = display_time_zone_date_to_utc_datetime(
-            allocation_end_date)
         if job_end_dt_utc > allocation_end_dt_utc:
             logger.warning(
                 f'Job {jobslurmid} end date '
@@ -698,11 +704,11 @@ def client_error(data_message):
         message = (
             f'User {user.username} is not a member of account {account.name}.')
         logger.error(message)
-        return client_error(message)
+        return non_affirmative(message)
     except Allocation.DoesNotExist:
         message = f'Account {account.name} has no active compute allocation.'
         logger.error(message)
-        return client_error(message)
+        return non_affirmative(message)
     except Allocation.MultipleObjectsReturned:
         logger.error(
             f'Account {account.name} has more than one active compute '
@@ -713,7 +719,7 @@ def client_error(data_message):
             f'User {user.username} is not an active member of the compute '
             f'allocation for account {account.name}.')
         logger.error(message)
-        return client_error(message)
+        return non_affirmative(message)
     except (MultipleObjectsReturned, ObjectDoesNotExist) as e:
         logger.error(
             f'Failed to retrieve a required database object. Details: {e}')

coldfront/core/allocation/management/commands/add_directory_defaults.py

@@ -24,7 +24,7 @@ def handle(self, *args, **options):
         """
         from coldfront.core.resource.models import AttributeType
         attribute_type, _ = AttributeType.objects.get_or_create(name='Text')
-        path, _ = ResourceAttributeType.objects.update_or_create(
+        path, _ = ResourceAttributeType.objects.get_or_create(
             attribute_type=AttributeType.objects.get(name='Text'),
             name='path',
             defaults={
@@ -33,60 +33,60 @@ def handle(self, *args, **options):
                 'is_value_unique': True,
             })
 
-        cluster_directory, _ = ResourceType.objects.update_or_create(
+        cluster_directory, _ = ResourceType.objects.get_or_create(
             name='Cluster Directory',
             defaults={
                 'description': 'Directory on a cluster.',
             })
 
-        groups_directory, _ = Resource.objects.update_or_create(
+        groups_directory, _ = Resource.objects.get_or_create(
             resource_type=cluster_directory,
             name='Groups Directory',
             description='The parent directory containing shared group data.')
 
-        groups_path, _ = ResourceAttribute.objects.update_or_create(
+        groups_path, _ = ResourceAttribute.objects.get_or_create(
             resource_attribute_type=path,
             resource=groups_directory,
             value='/global/home/groups/')
 
-        scratch2_directory, _ = Resource.objects.update_or_create(
+        scratch_directory, _ = Resource.objects.get_or_create(
             resource_type=cluster_directory,
-            name='Scratch2 Directory',
-            description='The parent directory containing scratch2 data.')
+            name='Scratch Directory',
+            description='The parent directory containing scratch data.')
 
-        scratch2_path, _ = ResourceAttribute.objects.update_or_create(
+        scratch_path, _ = ResourceAttribute.objects.get_or_create(
             resource_attribute_type=path,
-            resource=scratch2_directory,
-            value='/global/scratch2/')
+            resource=scratch_directory,
+            value='/global/scratch/')
 
-        groups_p2p3_directory, _ = Resource.objects.update_or_create(
+        groups_p2p3_directory, _ = Resource.objects.get_or_create(
             parent_resource=groups_directory,
             resource_type=cluster_directory,
             name='Groups P2/P3 Directory',
             description='The parent directory containing P2/P3 data '
                         'in the groups directory.')
 
-        groups_p2p3_path, _ = ResourceAttribute.objects.update_or_create(
+        groups_p2p3_path, _ = ResourceAttribute.objects.get_or_create(
             resource_attribute_type=path,
             resource=groups_p2p3_directory,
             value=os.path.join(groups_path.value, 'pl1data'))
 
-        scratch2_p2p3_directory, _ = Resource.objects.update_or_create(
-            parent_resource=scratch2_directory,
+        scratch_p2p3_directory, _ = Resource.objects.get_or_create(
+            parent_resource=scratch_directory,
             resource_type=cluster_directory,
-            name='Scratch2 P2/P3 Directory',
+            name='Scratch P2/P3 Directory',
             description='The parent directory containing P2/P3 data in the '
-                        'scratch2 directory.')
+                        'scratch directory.')
 
-        scratch2_p2p3_path, _ = ResourceAttribute.objects.update_or_create(
+        scratch_p2p3_path, _ = ResourceAttribute.objects.get_or_create(
             resource_attribute_type=path,
-            resource=scratch2_p2p3_directory,
-            value=os.path.join(scratch2_path.value, 'pl1data'))
+            resource=scratch_p2p3_directory,
+            value=os.path.join(scratch_path.value, 'p2p3'))
 
         from coldfront.core.allocation.models import AttributeType
         attribute_type, _ = AttributeType.objects.get_or_create(name='Text')
         allocation_attr_type, _ = \
-            AllocationAttributeType.objects.update_or_create(
+            AllocationAttributeType.objects.get_or_create(
                 attribute_type=attribute_type,
                 name='Cluster Directory Access',
                 defaults={

coldfront/core/allocation/templates/secure_dir/secure_dir_manage_users.html

@@ -10,7 +10,6 @@ <h1>{{ action|title }} users {{ preposition }}: {{ directory }}</h1>
   {% if formset %}
     <div class="card border-light">
       <div class="card-body">
-
         <form action="{% url url allocation.pk action %}" method="post">
           {% csrf_token %}
           <div class="table-responsive">
@@ -35,11 +34,20 @@ <h1>{{ action|title }} users {{ preposition }}: {{ directory }}</h1>
                   {% if can_manage_users %}
                     <td>{{ form.selected }}</td>
                   {% endif %}
-                  <td>{{ forloop.counter }}</td>
-                  <td>{{ form.username.value }}</td>
-                  <td>{{ form.first_name.value }}</td>
-                  <td>{{ form.last_name.value }}</td>
-                  <td>{{ form.email.value }}</td>
+
+                  {% if form.selected %}
+                    <td>{{ forloop.counter }}</td>
+                    <td>{{ form.username.value }}</td>
+                    <td>{{ form.first_name.value }}</td>
+                    <td>{{ form.last_name.value }}</td>
+                    <td>{{ form.email.value }}</td>
+                  {% else %}
+                    <td><div class="text-muted">{{ forloop.counter }}</div></td>
+                    <td><div class="text-muted">{{ form.username.value }}</div></td>
+                    <td><div class="text-muted">{{ form.first_name.value }}</div></td>
+                    <td><div class="text-muted">{{ form.last_name.value }}</div></td>
+                    <td><div class="text-muted">{{ form.email.value }}</div></td>
+                  {% endif %}
                 </tr>
               {% endfor %}
               </tbody>

coldfront/core/allocation/tests/test_utils/test_secure_dir_utils.py

@@ -35,56 +35,62 @@ def setUp(self):
 
         self.project1 = self.create_active_project_with_pi('project1', self.pi)
 
-        self.subdirectory_name = 'test_dir'
+        self.groups_subdirectory_name = 'project1/test_groups'
+        self.scratch_subdirectory_name = 'test_scratch'
         call_command('add_directory_defaults')
-        create_secure_dirs(self.project1, self.subdirectory_name)
+        create_secure_dirs(self.project1,
+                           self.groups_subdirectory_name,
+                           'groups')
+        create_secure_dirs(self.project1,
+                           self.scratch_subdirectory_name,
+                           'scratch')
 
     def test_allocation_objects_created(self):
         """Testing that allocation objects are created"""
-        scratch2_p2p3_directory = Resource.objects.get(
-            name='Scratch2 P2/P3 Directory')
+        scratch_p2p3_directory = Resource.objects.get(
+            name='Scratch P2/P3 Directory')
         groups_p2p3_directory = Resource.objects.get(
             name='Groups P2/P3 Directory')
 
         groups_p2p3_path = \
             groups_p2p3_directory.resourceattribute_set.get(
                 resource_attribute_type__name='path')
-        scratch2_p2p3_path = \
-            scratch2_p2p3_directory.resourceattribute_set.get(
+        scratch_p2p3_path = \
+            scratch_p2p3_directory.resourceattribute_set.get(
                 resource_attribute_type__name='path')
 
         groups_allocation = Allocation.objects.filter(
             project=self.project1,
             status=AllocationStatusChoice.objects.get(name='Active'),
             resources=groups_p2p3_directory)
 
-        scratch2_allocation = Allocation.objects.filter(
+        scratch_allocation = Allocation.objects.filter(
             project=self.project1,
             status=AllocationStatusChoice.objects.get(name='Active'),
-            resources=scratch2_p2p3_directory)
+            resources=scratch_p2p3_directory)
 
         self.assertTrue(groups_allocation.exists())
-        self.assertTrue(scratch2_allocation.exists())
+        self.assertTrue(scratch_allocation.exists())
 
         groups_allocation = groups_allocation.first()
-        scratch2_allocation = scratch2_allocation.first()
+        scratch_allocation = scratch_allocation.first()
 
         allocation_attribute_type = AllocationAttributeType.objects.get(
             name='Cluster Directory Access')
         groups_p2p3_subdirectory = AllocationAttribute.objects.filter(
             allocation_attribute_type=allocation_attribute_type,
             allocation=groups_allocation,
             value=os.path.join(groups_p2p3_path.value,
-                               self.subdirectory_name))
+                               self.groups_subdirectory_name))
 
-        scratch2_p2p3_subdirectory = AllocationAttribute.objects.filter(
+        scratch_p2p3_subdirectory = AllocationAttribute.objects.filter(
             allocation_attribute_type=allocation_attribute_type,
-            allocation=scratch2_allocation,
-            value=os.path.join(scratch2_p2p3_path.value,
-                               self.subdirectory_name))
+            allocation=scratch_allocation,
+            value=os.path.join(scratch_p2p3_path.value,
+                               self.scratch_subdirectory_name))
 
         self.assertTrue(groups_p2p3_subdirectory.exists())
-        self.assertTrue(scratch2_p2p3_subdirectory.exists())
+        self.assertTrue(scratch_p2p3_subdirectory.exists())
 
 
 class TestGetSecureDirManageUserRequestObjects(TestBase):