Merge pull request #804 from CDLUC3/develop

Merge develop to main

dev/js/main2.js

@@ -6,18 +6,20 @@ $(document).ready(function(){
 
   // Before toggling menu, change default header menu class to non-selected state and change default aria attribute:
 
+  // According to ChatGPT using aria-hidden for visual appearance isn't recommended and items should not be hidden from
+  // screen readers. Instead, use aria-expanded to indicate the state of the menu.
   $('#js-header__nav').attr('class', 'header__nav');
-  $('#js-header__nav').attr('aria-hidden', 'true');
+  $('#js-header__nav').attr('aria-expanded', 'false');
 
   // Toggle classes and attributes:
   $('#js-header__nav-button').click(function(){
 
     $('#js-header__nav').toggleClass('header__nav header__nav--selected', 300, 'easeInOutCubic');
 
-    if($('#js-header__nav').attr('aria-hidden') == 'true') {
-      $('#js-header__nav').attr('aria-hidden', 'false');
+    if($('#js-header__nav').attr('aria-expanded') == 'false') {
+      $('#js-header__nav').attr('aria-expanded', 'true');
     } else {
-      $('#js-header__nav').attr('aria-hidden', 'true');
+      $('#js-header__nav').attr('aria-expanded', 'false');
     }
 
   });
@@ -62,18 +64,25 @@ $(document).ready(function(){
   // Toggle open and closed from login button
 
   $('#js-header__loginout-button').click(function(){
+    if ($('#js-login-modal').attr('aria-hidden') == 'true') {
+      $('#js-login-modal').attr('aria-hidden', 'false');
+    }else {
+      $('#js-login-modal').attr('aria-hidden', 'true');
+    }
     $('#js-login-modal').fadeToggle(200);
   });
 
   // Close when close icon is clicked
 
   $('#js-login-modal__close').click(function(){
+    $('#js-login-modal').attr('aria-hidden', 'true');
     $('#js-login-modal').fadeToggle(200);
   });
 
   // Close when form is submitted
 
   $('#js-login-modal__form').submit(function(){
+    $('#js-login-modal').attr('aria-hidden', 'true');
     $('#js-login-modal').fadeToggle(200);
   });
 

dev/legacy-scss/variables.scss

@@ -16,30 +16,30 @@ $breadcrumb-separator:          "/" !default; // now missing from bootstrap
 
 $text-link-color: rgb(176, 189, 202); // muted blue #B0BDCA
 
-$text-link-over-beige: rgb(0, 86, 149); // #005695 blue link over beige backgrounds
+$text-link-over-beige: rgb(0, 57, 97); // #003961 blue link over beige backgrounds
 
-$design-primary-color: rgb(0, 119, 138); // Teal Blue #00778b
-$design-primary-gradient-top: rgb(6, 80, 89); // Lighter Teal Blue #168b99 was #17A2AC
+$design-primary-color: rgb(0, 95, 112); // Teal Blue #005F70
+$design-primary-gradient-top: rgb(6, 80, 89); // Lighter #11616A
 $design-primary-gradient-bottom: rgb(1, 64, 71); // Slightly darker Teal Blue #056a7b
 
 // Brown colors
-$design-secondary-gradient-top: rgb(198, 176, 130); // #C6B082  Used for subnav
-$design-secondary-color: rgb(180, 151, 90); // Metallic Gold  #B4975A
-$design-secondary-lighter: rgb(199, 177, 131); // #C7B183   Used for table headers
+$design-secondary-gradient-top: rgb(213, 197, 165); // #D5C5A5
+$design-secondary-color: rgb(208, 190, 149); // #D0BE95
+$design-secondary-lighter: rgb(214, 200, 164); // #D6C8A4
 
 $design-secondary-footer-newsfeed: rgb(221, 208, 181); // #DDD0B5
 $design-secondary-footer-icon-bg: rgb(229, 219, 197); // #E5DBC5
 $design-secondary-lightest: rgb(236, 229, 214); // Beige  #ECE5D6
 
 // Orange colors
-$design-pop-color: rgb(255, 127, 34); // Bright Orange  #FF7F22
-$design-pop-gradient-top: rgb(255,142,40); // Slightly Lighter Orange  #FF8E28
-$design-pop-gradient-bottom: rgb(255,112,28); // Slightly Darker Orange  #FF701C
-$design-pop-dark: rgb(216,64,47); // Reddish Orange  #D8402F
+$design-pop-color: rgb(124, 24, 44); // # 7C182C
+$design-pop-gradient-top: rgb(138, 25, 48); // Slightly Lighter #8A1930
+$design-pop-gradient-bottom: rgb(116, 22, 41); // Slightly Darker #741629
+$design-pop-dark: rgb(106, 20, 37); // Deep Burgundy  #6A1425
 
 // These were missing so I'm guessing at the values and required to compile scss
-$gray-light: rgb(190, 182, 175); // #BEB6AF
-$gray-lighter: rgb(237, 234, 229); // #EDEAE5
+$gray-light: rgb(230, 230, 230); // #E6E6E6
+$gray-lighter: rgb(240, 240, 240); // #F0F0F0
 
 $design-light-gray-border-color: rgb(190, 182, 175);
 $design-light-gray-bg-color: rgb(249, 248, 247);  // #F9F8F7

dev/scss/_customize-table.scss

@@ -9,6 +9,7 @@
 	margin: 0 auto 10px;
 	padding: 10px;
 	background: linear-gradient(to bottom, $design-pop-gradient-top, $design-pop-gradient-bottom);
+	color: $design-white-color;
 
 	@include bp(screen1) {
 		flex-flow: row nowrap;

dev/scss/_header.scss

@@ -8,7 +8,7 @@
 		justify-content: flex-end;
 		max-width: 1000px;
 		margin: 10px auto;
-		color: $design-pop-color;
+		color: $design-primary-color;
 	}
 
 }

dev/scss/_login-menu.scss

@@ -51,7 +51,7 @@
 
 	&:hover,
 	&:focus {
-		color: $design-white-color;
+		color: $text-link-over-beige;
 	}
 
 }
@@ -62,7 +62,7 @@
 
 .login-menu__link--selected {
 	@extend %login-menu__link;
-	color: $design-white-color;
+	color: $text-link-over-beige;
 
 	&::before {
 		@include bp(screen1) {

dev/scss/_login-modal.scss

@@ -33,6 +33,7 @@
 	padding: 2px;
 	background-color: $design-white-color;
 	cursor: pointer;
+	object-fit: contain; // Or "cover" depending on the fit you need
 }
 
 .login-modal__form {

ezidapp/management/commands/opensearch-update.py

@@ -17,7 +17,8 @@
 urllib3.disable_warnings(InsecureRequestWarning)
 # end suppression of urllib3 InsecureRequestWarning
 
-SPLIT_SIZE = 100
+SPLIT_SIZE = 5
+DB_PAGE_SIZE = 100
 
 # run: python manage.py opensearch-update
 # optional parameters: --starting_id 1234 --updated_since 2023-10-10T00:00:00Z
@@ -37,7 +38,7 @@
 
 class Command(BaseCommand):
     def handle(self, *args, **options):
-        # Get all items from Identifier table 100 at a time manually since
+        # Get all items from Identifier table DB_PAGE_SIZE at a time manually since
         # I had lockup issues with the ORM, even with constructs that were
         # supposed to be lazy and handle large datasets. :shrug:
         #
@@ -68,7 +69,7 @@ def handle(self, *args, **options):
 
         while True:
             iden_arr = (SearchIdentifier.objects.filter(id__gt=start_after_id)
-                        .filter(additional_filter).order_by('id')[:100])
+                        .filter(additional_filter).order_by('id')[:DB_PAGE_SIZE])
 
             # break when we run out of items
             if not iden_arr:

ezidapp/management/commands/proc-cleanup-async-queues.py

@@ -15,19 +15,22 @@
 
 import logging
 import time
+from datetime import datetime
+from dateutil.parser import parse
 
-import django.conf
 import django.conf
 import django.db
-import django.db.transaction
+from django.db import transaction
+from django.db.models import Q
 
 import ezidapp.management.commands.proc_base
 import ezidapp.models.identifier
 import ezidapp.models.shoulder
-from django.db.models import Q
+
 
 log = logging.getLogger(__name__)
 
+
 class Command(ezidapp.management.commands.proc_base.AsyncProcessingCommand):
     help = __doc__
     name = __name__
@@ -45,6 +48,29 @@ class Command(ezidapp.management.commands.proc_base.AsyncProcessingCommand):
     def __init__(self):
         super().__init__()
 
+    def add_arguments(self, parser):
+        super().add_arguments(parser)
+        parser.add_argument(
+            '--pagesize', help='Rows in each batch select.', type=int)
+
+        parser.add_argument(
+            '--updated_range_from', type=str,
+            help = (
+                'Updated date range from - local date/time in ISO 8601 format without timezone \n'
+                'YYYYMMDD, YYYYMMDDTHHMMSS, YYYY-MM-DD, YYYY-MM-DDTHH:MM:SS. \n'
+                'Examples: 20241001, 20241001T131001, 2024-10-01, 2024-10-01T13:10:01 or 2024-10-01'
+            )
+        )
+
+        parser.add_argument(
+            '--updated_range_to', type=str,
+            help = (
+                'Updated date range to - local date/time in ISO 8601 format without timezone \n'
+                'YYYYMMDD, YYYYMMDDTHHMMSS, YYYY-MM-DD, YYYY-MM-DDTHH:MM:SS. \n'
+                'Examples: 20241001, 20241001T131001, 2024-10-01, 2024-10-01T13:10:01 or 2024-10-01'
+            )
+        )
+
 
     def run(self):
         """
@@ -53,21 +79,54 @@ def run(self):
             Args:
                 None
         """
+        ASYNC_CLEANUP_SLEEP = 60 * 60
+
+        BATCH_SIZE = self.opt.pagesize
+        if BATCH_SIZE is None:
+            BATCH_SIZE = 10000
+
+        updated_from = None
+        updated_to = None
+        updated_from_str = self.opt.updated_range_from
+        updated_to_str = self.opt.updated_range_to
+        if updated_from_str is not None:
+            try:
+                updated_from = self.date_to_seconds(updated_from_str)
+            except Exception as ex:
+                log.error(f"Input date/time error: {ex}")
+                exit()
+        if updated_to_str is not None:
+            try:
+                updated_to = self.date_to_seconds(updated_to_str)
+            except Exception as ex:
+                log.error(f"Input date/time error: {ex}")
+                exit()
+
+        if updated_from is not None and updated_to is not None:
+            time_range = Q(updateTime__gte=updated_from) & Q(updateTime__lte=updated_to)
+            time_range_str = f"updated between: {updated_from_str} and {updated_to_str}"
+        elif updated_to is not None:
+            time_range = Q(updateTime__lte=updated_to)
+            time_range_str = f"updated before: {updated_to_str}"
+        else:
+            max_age_ts = int(time.time()) - django.conf.settings.DAEMONS_EXPUNGE_MAX_AGE_SEC
+            min_age_ts = max_age_ts - django.conf.settings.DAEMONS_EXPUNGE_MAX_AGE_SEC
+            time_range = Q(updateTime__gte=min_age_ts) & Q(updateTime__lte=max_age_ts)
+            time_range_str = f"updated between: {self.seconds_to_date(min_age_ts)} and {self.seconds_to_date(max_age_ts)}"
+
+        last_id = 0
         # keep running until terminated
         while not self.terminated():
-            currentTime=int(time.time())
-            timeDelta=django.conf.settings.DAEMONS_CHECK_IDENTIFIER_ASYNC_STATUS_TIMESTAMP
+            # retrieve identifiers with update timestamp within a date range
+            filter = time_range & Q(id__gt=last_id)
+            refIdsQS = self.refIdentifier.objects.filter(filter).order_by("pk")[: BATCH_SIZE]
 
-            # retrieve identifiers with update timestamp within a set range
-            refIdsQS = self.refIdentifier.objects.filter(
-                updateTime__lte=currentTime,
-                updateTime__gte=currentTime - timeDelta
-            ).order_by("-pk")[: django.conf.settings.DAEMONS_MAX_BATCH_SIZE]
-
-            log.info("Checking ref Ids in the range: " + str(currentTime) + " - " + str(currentTime - timeDelta))
+            log.info(f"Checking ref Ids: {time_range_str}, filter: {filter}")
+            log.info(f"Checking ref Ids returned: {len(refIdsQS)} records")
 
             # iterate over query set to check each identifier status
             for refId in refIdsQS:
+                last_id = refId.pk
 
                 # set status for each handle system
                 identifierStatus = {
@@ -109,7 +168,20 @@ def run(self):
                         "Delete identifier: " + refId.identifier + " from refIdentifier table.")
                     self.deleteRecord(self.refIdentifier, refId.pk, record_type='refId', identifier=refId.identifier)
 
-            self.sleep(django.conf.settings.DAEMONS_BATCH_SLEEP)
+            if len(refIdsQS) < BATCH_SIZE:
+                if updated_from is not None or updated_to is not None:
+                    log.info(f"Finished - Checking ref Ids: {time_range_str}, filter: {filter}")
+                    exit()
+                else:
+                    log.info(f"Sleep {ASYNC_CLEANUP_SLEEP} seconds before processing next time range.")
+                    self.sleep(ASYNC_CLEANUP_SLEEP)
+                    last_id = 0
+                    min_age_ts = max_age_ts
+                    max_age_ts = int(time.time()) - django.conf.settings.DAEMONS_EXPUNGE_MAX_AGE_SEC
+                    time_range = Q(updateTime__gte=min_age_ts) & Q(updateTime__lte=max_age_ts)
+                    time_range_str = f"updated between: {self.seconds_to_date(min_age_ts)} and {self.seconds_to_date(max_age_ts)}"
+            else:
+                self.sleep(django.conf.settings.DAEMONS_BATCH_SLEEP)
 
     def deleteRecord(self, queue, primary_key, record_type=None, identifier=None):
         """
@@ -125,13 +197,49 @@ def deleteRecord(self, queue, primary_key, record_type=None, identifier=None):
         try:
             # check if the record to be deleted is a refIdentifier record
             if (record_type is not None and record_type == 'refId'):
-                log.info(type(queue))
-                log.info("Delete refId: " + str(primary_key))
-                queue.objects.filter(id=primary_key).delete()
+                log.info(f"Delete from {queue.__name__} refId: " + str(primary_key))
+                with transaction.atomic():
+                    obj = queue.objects.select_for_update().get(id=primary_key)
+                    obj.delete()
             else:
-                log.info("Delete async entry: " + str(primary_key))
-                queue.objects.filter(seq=primary_key).delete()
+                log.info(f"Delete async queue {queue.__name__} entry: " + str(primary_key))
+                with transaction.atomic():
+                    obj = queue.objects.select_for_update().get(seq=primary_key)
+                    obj.delete()
         except Exception as e:
             log.error("Exception occured while processing identifier '" + identifier + "' for '" +
                         record_type + "' table")
             log.error(e)
+
+
+    def date_to_seconds(self, date_time_str: str) -> int:
+        """
+        Convert date/time string to seconds since the Epotch.
+        For example:
+        2024-01-01 00:00:00 => 1704096000
+        2024-10-10 00:00:00 => 1728543600
+
+        Parameter:
+        date_time_str: A date/time string in in ISO 8601 format without timezone.
+        For example: 'YYYYMMDD, YYYYMMDDTHHMMSS, YYYY-MM-DD, YYYY-MM-DDTHH:MM:SS.
+
+        Returns:
+        int: seconds since the Epotch
+
+        """
+
+        # Parse the date and time string to a datetime object
+        dt_object = parse(date_time_str)
+
+        # Convert the datetime object to seconds since the Epoch
+        seconds_since_epoch = int(dt_object.timestamp())
+
+        return seconds_since_epoch
+
+
+    def seconds_to_date(self, seconds_since_epoch: int) -> str:
+        dt_object = datetime.fromtimestamp(seconds_since_epoch)
+
+        # Format the datetime object to a string in the desired format
+        formatted_time = dt_object.strftime("%Y-%m-%dT%H:%M:%S")
+        return formatted_time