From 4b7f3130abc295a9dbb8ac2ab9b0bb37b30193cb Mon Sep 17 00:00:00 2001 From: sfisher Date: Thu, 7 Nov 2024 16:02:25 -0800 Subject: [PATCH 01/27] This gets the homepage "try" creation for identifiers working, but something has messed up the layout in "create id" now. --- impl/ui_home.py | 18 +++++++++ settings/urls.py | 1 + static_src/javascripts/simple_create_ajax.js | 29 ++++++++++++++ templates/create/_home_demo_form.html | 40 +++++++++++++++++++ templates/index.html | 42 +++----------------- 5 files changed, 93 insertions(+), 37 deletions(-) create mode 100644 static_src/javascripts/simple_create_ajax.js create mode 100644 templates/create/_home_demo_form.html diff --git a/impl/ui_home.py b/impl/ui_home.py index aa31a6672..12d08f505 100644 --- a/impl/ui_home.py +++ b/impl/ui_home.py @@ -35,6 +35,24 @@ def index(request): "/id/" + urllib.parse.quote(result.split()[1], ":/") ) # ID Details page +def ajax_index_form(request): + if request.method not in ["GET"]: + return impl.ui_common.methodNotAllowed(request) + d = {'menu_item': 'ui_home.index'} + d['prefixes'] = sorted( + django.conf.settings.TEST_SHOULDER_DICT, key=lambda p: p['namespace'].lower() + ) + d['form_placeholder'] = True # is this necessary? + d = impl.ui_create.simple_form(request, d) + result = d['id_gen_result'] + if result == 'edit_page': + # noinspection PyUnresolvedReferences + # return impl.ui_common.render(request, 'index', d) # ID Creation page + return impl.ui_common.render(request, 'create/_home_demo_form', d) + # return render(request, 'create/home_demo_form.html', d) + elif result == 'bad_request': + return impl.ui_common.badRequest(request) + def learn(request): if request.method != "GET": diff --git a/settings/urls.py b/settings/urls.py index 5d7c2917d..f3763377e 100644 --- a/settings/urls.py +++ b/settings/urls.py @@ -33,6 +33,7 @@ urlpatterns = [ # UI - RENDERED FROM TEMPLATES IN INFO REPOSITORY django.urls.re_path("^$", impl.ui_home.index, name="ui_home.index"), + django.urls.re_path("^home/ajax_index_form$", impl.ui_home.ajax_index_form, name="ui_home.ajax_index_form"), django.urls.re_path("^learn/$", impl.ui_home.learn, name="ui_home.learn"), django.urls.re_path("^learn/ark_open_faq$", impl.ui_home.ark_open_faq, name="ui_home.ark_open_faq"), django.urls.re_path("^learn/crossref_faq$", impl.ui_home.crossref_faq, name="ui_home.crossref_faq"), diff --git a/static_src/javascripts/simple_create_ajax.js b/static_src/javascripts/simple_create_ajax.js new file mode 100644 index 000000000..f371dc0b9 --- /dev/null +++ b/static_src/javascripts/simple_create_ajax.js @@ -0,0 +1,29 @@ +document.getElementById('form-shoulder').value = document.querySelectorAll('input[name=selshoulder]')[0].value; + +document.querySelectorAll('input[name="selshoulder"]').forEach(radio => { + radio.addEventListener('change', function () { + var profile; + if(this.value.startsWith('ark')) { + profile = 'erc'; + } else { + profile = 'datacite'; + } + + const form = document.querySelector('#create_form'); + const formData = new FormData(form); + formData.set('current_profile', profile); + + // Convert FormData to a query string + const queryString = new URLSearchParams(formData).toString(); + fetch(`/home/ajax_index_form?${queryString}`, { + headers: { + 'X-CSRFToken': document.querySelector('input[name="csrfmiddlewaretoken"]').value, + }, + }) + .then(response => response.text()) + .then(data => { + document.getElementById('form-container').innerHTML = data; // Replace form container HTML + document.getElementById('form-shoulder').value = this.value; // needs to happen after the form is replaced + }); + }); +}); \ No newline at end of file diff --git a/templates/create/_home_demo_form.html b/templates/create/_home_demo_form.html new file mode 100644 index 000000000..575c87cb6 --- /dev/null +++ b/templates/create/_home_demo_form.html @@ -0,0 +1,40 @@ +{% load i18n %} +{% load layout_extras %} +
+ + +{% csrf_token %} + +
+ +
+ {% trans "Describe the identified object" %} +
+ +
+ {{ form.non_field_errors }} + {% for field in form %} +
+
+ {% if field|fieldtype == "TextInput" %} + + {{ field|add_attributes:"fcontrol__text-field-stacked" }} + {% else %} + + {{ field|add_attributes:"fcontrol__select" }} + {% endif %} + {% if field.errors %} + {% for error in field.errors %}{{ error|escape }}{% endfor %} + {% endif %} +
+
+ {% endfor %} + +
+ +
+ +
+
+
\ No newline at end of file diff --git a/templates/index.html b/templates/index.html index df3f11e6e..227aedf7c 100644 --- a/templates/index.html +++ b/templates/index.html @@ -19,6 +19,7 @@ } {% endblock %} {% block content %} +{% csrf_token %}

EZID: {% trans "Identifiers made easy" %}

@@ -52,53 +53,20 @@

EZID: {% trans "Identifiers made easy" %}

-
-
{% trans "Choose an identifier type" %} {% for p in prefixes %} {% endfor %} {% help_icon "choose_id_demo" _("on choosing the type of identifier") %}
- -
- -
- {% trans "Describe the identified object" %} -
- -
- {{ form.non_field_errors }} - {% for field in form %} -
-
- {% if field|fieldtype == "TextInput" %} - - {{ field|add_attributes:"fcontrol__text-field-stacked" }} - {% else %} - - {{ field|add_attributes:"fcontrol__select" }} - {% endif %} - {% if field.errors %} - {% for error in field.errors %}{{ error|escape }}{% endfor %} - {% endif %} -
-
- {% endfor %} - -
- -
- -
+
+ {% include "create/_home_demo_form.html" %}
-
- + {% include "info/popup_help.html" %} From 7cb0ca165537f33b08ee7d529e321464f93c51ce Mon Sep 17 00:00:00 2001 From: sfisher Date: Fri, 8 Nov 2024 16:34:38 -0800 Subject: [PATCH 02/27] Need to change this fieldset back to a div since otherwise it destroys the whole layout of the form. --- templates/includes/simple_id_type.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/includes/simple_id_type.html b/templates/includes/simple_id_type.html index c49da366b..d7cc2876d 100644 --- a/templates/includes/simple_id_type.html +++ b/templates/includes/simple_id_type.html @@ -1,14 +1,14 @@ {% load layout_extras %} {% load i18n %} -
+

{% trans "Choose an identifier type" %}

{% if calling_page == 'demo' %} {% help_icon "choose_id_demo" _("on choosing the type of identifier") %} {% else %} {% help_icon "choose_id" _("on choosing the type of identifier") %} {% endif %} -
+ {% if prefixes|duplicate_id_types %} {% comment %} class 'ays-ignore' is used by 'are-you-sure.js' which prevents users from accidentally leaving From fac439fa005666b0f457ebbb8304921f0bedaedb Mon Sep 17 00:00:00 2001 From: sfisher Date: Tue, 12 Nov 2024 17:34:25 -0800 Subject: [PATCH 03/27] Change this to a heading, I guess I missed this particular place. --- templates/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/index.html b/templates/index.html index 227aedf7c..d2127354a 100644 --- a/templates/index.html +++ b/templates/index.html @@ -34,7 +34,7 @@

EZID: {% trans "Identifiers made easy" %}

-

{% trans "See how easy it is" %}:

+

{% trans "See how easy it is" %}:

{% comment %}Translators: Copy HTML tags over and only translate words outside of these tags i.e.:

TRANSLATE TEXT WRAPPED BY HTML TAGS

From a7ada1403f30cb88e3d1fa013ba04c0759f80988 Mon Sep 17 00:00:00 2001 From: jsjiang Date: Wed, 13 Nov 2024 08:56:09 -0800 Subject: [PATCH 04/27] backup proc-cleanup-async-queues.py to _v1 --- .../commands/proc-cleanup-async-queues_v1.py | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 ezidapp/management/commands/proc-cleanup-async-queues_v1.py diff --git a/ezidapp/management/commands/proc-cleanup-async-queues_v1.py b/ezidapp/management/commands/proc-cleanup-async-queues_v1.py new file mode 100644 index 000000000..d9f578ff1 --- /dev/null +++ b/ezidapp/management/commands/proc-cleanup-async-queues_v1.py @@ -0,0 +1,137 @@ +#! /usr/bin/env python + +# Copyright©2021, Regents of the University of California +# http://creativecommons.org/licenses/BSD + +""" + +Clean up entries that are successfully completed or are a 'no-op' + +Identifier operation entries are retrieved by querying the database; +operations that successfully completed or are a no-op are deleted based on +pre-set interval. + +""" + +import logging +import time + +import django.conf +import django.conf +import django.db +import django.db.transaction + +import ezidapp.management.commands.proc_base +import ezidapp.models.identifier +import ezidapp.models.shoulder +from django.db.models import Q + +log = logging.getLogger(__name__) + +class Command(ezidapp.management.commands.proc_base.AsyncProcessingCommand): + help = __doc__ + name = __name__ + + setting = 'DAEMONS_QUEUE_CLEANUP_ENABLED' + + queueType = { + 'crossref': ezidapp.models.async_queue.CrossrefQueue, + 'datacite': ezidapp.models.async_queue.DataciteQueue, + 'search': ezidapp.models.async_queue.SearchIndexerQueue + } + + refIdentifier = ezidapp.models.identifier.RefIdentifier + + def __init__(self): + super().__init__() + + + def run(self): + """ + Checks for the successfully processed identifier + + Args: + None + """ + # keep running until terminated + while not self.terminated(): + currentTime=int(time.time()) + timeDelta=django.conf.settings.DAEMONS_CHECK_IDENTIFIER_ASYNC_STATUS_TIMESTAMP + + # retrieve identifiers with update timestamp within a set range + refIdsQS = self.refIdentifier.objects.filter( + updateTime__lte=currentTime, + updateTime__gte=currentTime - timeDelta + ).order_by("-pk")[: django.conf.settings.DAEMONS_MAX_BATCH_SIZE] + + log.info("Checking ref Ids in the range: " + str(currentTime) + " - " + str(currentTime - timeDelta)) + + # iterate over query set to check each identifier status + for refId in refIdsQS: + + # set status for each handle system + identifierStatus = { + 'crossref' : False, + 'datacite' : False, + 'search' : False + } + + # check if the identifier is processed for each background job + for key, value in self.queueType.items(): + queue = value + + qs = queue.objects.filter( + Q(refIdentifier_id=refId.pk) + ) + + # if the identifier does not exist in the table + # mark as 'OK' to delete from the refIdentifier + if not qs: + identifierStatus[key] = True + continue + + for task_model in qs: + log.info('-' * 10) + log.info("Running job for identifier: " + refId.identifier + " in " + key + " queue") + + # delete identifier if the status is successfully synced or + # not applicable for this handle system + if (task_model.status==queue.SUCCESS or task_model.status==queue.IGNORED): + log.info( + "Delete identifier: " + refId.identifier + " in " + key + " queue") + identifierStatus[key] = True + self.deleteRecord(queue, task_model.pk, record_type=key, identifier=refId.identifier) + + # if the identifier is successfully processed for all the handle system + # delete it from the refIdentifier table + if all(i for i in identifierStatus.values()): + log.info( + "Delete identifier: " + refId.identifier + " from refIdentifier table.") + self.deleteRecord(self.refIdentifier, refId.pk, record_type='refId', identifier=refId.identifier) + + self.sleep(django.conf.settings.DAEMONS_BATCH_SLEEP) + + def deleteRecord(self, queue, primary_key, record_type=None, identifier=None): + """ + Deletes the identifier record that has been successfully completed + based on the record's primary key provided + + Args: + queue : async handle queue + primary_key (str): primary key of the record to be deleted. + record_type (str): . Defaults to None. + identifier (str): . Defaults to None. + """ + try: + # check if the record to be deleted is a refIdentifier record + if (record_type is not None and record_type == 'refId'): + log.info(type(queue)) + log.info("Delete refId: " + str(primary_key)) + queue.objects.filter(id=primary_key).delete() + else: + log.info("Delete async entry: " + str(primary_key)) + queue.objects.filter(seq=primary_key).delete() + except Exception as e: + log.error("Exception occured while processing identifier '" + identifier + "' for '" + + record_type + "' table") + log.error(e) From 728a861cd95ee69213a93d67470935d1699dfea9 Mon Sep 17 00:00:00 2001 From: jsjiang Date: Wed, 13 Nov 2024 08:57:05 -0800 Subject: [PATCH 05/27] replace proc-cleanup-async-queues with _v2 --- .../commands/proc-cleanup-async-queues.py | 144 +++++++++++++++--- 1 file changed, 126 insertions(+), 18 deletions(-) diff --git a/ezidapp/management/commands/proc-cleanup-async-queues.py b/ezidapp/management/commands/proc-cleanup-async-queues.py index d9f578ff1..85640407f 100644 --- a/ezidapp/management/commands/proc-cleanup-async-queues.py +++ b/ezidapp/management/commands/proc-cleanup-async-queues.py @@ -15,19 +15,22 @@ import logging import time +from datetime import datetime +from dateutil.parser import parse -import django.conf import django.conf import django.db -import django.db.transaction +from django.db import transaction +from django.db.models import Q import ezidapp.management.commands.proc_base import ezidapp.models.identifier import ezidapp.models.shoulder -from django.db.models import Q + log = logging.getLogger(__name__) + class Command(ezidapp.management.commands.proc_base.AsyncProcessingCommand): help = __doc__ name = __name__ @@ -45,6 +48,29 @@ class Command(ezidapp.management.commands.proc_base.AsyncProcessingCommand): def __init__(self): super().__init__() + def add_arguments(self, parser): + super().add_arguments(parser) + parser.add_argument( + '--pagesize', help='Rows in each batch select.', type=int) + + parser.add_argument( + '--updated_range_from', type=str, + help = ( + 'Updated date range from - local date/time in ISO 8601 format without timezone \n' + 'YYYYMMDD, YYYYMMDDTHHMMSS, YYYY-MM-DD, YYYY-MM-DDTHH:MM:SS. \n' + 'Examples: 20241001, 20241001T131001, 2024-10-01, 2024-10-01T13:10:01 or 2024-10-01' + ) + ) + + parser.add_argument( + '--updated_range_to', type=str, + help = ( + 'Updated date range to - local date/time in ISO 8601 format without timezone \n' + 'YYYYMMDD, YYYYMMDDTHHMMSS, YYYY-MM-DD, YYYY-MM-DDTHH:MM:SS. \n' + 'Examples: 20241001, 20241001T131001, 2024-10-01, 2024-10-01T13:10:01 or 2024-10-01' + ) + ) + def run(self): """ @@ -53,21 +79,54 @@ def run(self): Args: None """ + ASYNC_CLEANUP_SLEEP = 60 * 10 + + BATCH_SIZE = self.opt.pagesize + if BATCH_SIZE is None: + BATCH_SIZE = 10000 + + updated_from = None + updated_to = None + updated_from_str = self.opt.updated_range_from + updated_to_str = self.opt.updated_range_to + if updated_from_str is not None: + try: + updated_from = self.date_to_seconds(updated_from_str) + except Exception as ex: + log.error(f"Input date/time error: {ex}") + exit() + if updated_to_str is not None: + try: + updated_to = self.date_to_seconds(updated_to_str) + except Exception as ex: + log.error(f"Input date/time error: {ex}") + exit() + + if updated_from is not None and updated_to is not None: + time_range = Q(updateTime__gte=updated_from) & Q(updateTime__lte=updated_to) + time_range_str = f"updated between: {updated_from_str} and {updated_to_str}" + elif updated_to is not None: + time_range = Q(updateTime__lte=updated_to) + time_range_str = f"updated before: {updated_to_str}" + else: + max_age_ts = int(time.time()) - django.conf.settings.DAEMONS_EXPUNGE_MAX_AGE_SEC + min_age_ts = max_age_ts - django.conf.settings.DAEMONS_EXPUNGE_MAX_AGE_SEC + time_range = Q(updateTime__gte=min_age_ts) & Q(updateTime__lte=max_age_ts) + time_range_str = f"updated between: {self.seconds_to_date(min_age_ts)} and {self.seconds_to_date(max_age_ts)}" + + last_id = 0 # keep running until terminated while not self.terminated(): - currentTime=int(time.time()) - timeDelta=django.conf.settings.DAEMONS_CHECK_IDENTIFIER_ASYNC_STATUS_TIMESTAMP + # retrieve identifiers with update timestamp within a date range + filter = time_range & Q(id__gt=last_id) + refIdsQS = self.refIdentifier.objects.filter(filter).order_by("pk")[: BATCH_SIZE] - # retrieve identifiers with update timestamp within a set range - refIdsQS = self.refIdentifier.objects.filter( - updateTime__lte=currentTime, - updateTime__gte=currentTime - timeDelta - ).order_by("-pk")[: django.conf.settings.DAEMONS_MAX_BATCH_SIZE] - - log.info("Checking ref Ids in the range: " + str(currentTime) + " - " + str(currentTime - timeDelta)) + log.info(f"Checking ref Ids: {time_range_str}") + log.info(f"Checking ref Ids returned: {len(refIdsQS)} records") # iterate over query set to check each identifier status for refId in refIdsQS: + last_id = refId.pk # set status for each handle system identifierStatus = { @@ -109,7 +168,20 @@ def run(self): "Delete identifier: " + refId.identifier + " from refIdentifier table.") self.deleteRecord(self.refIdentifier, refId.pk, record_type='refId', identifier=refId.identifier) - self.sleep(django.conf.settings.DAEMONS_BATCH_SLEEP) + if len(refIdsQS) < BATCH_SIZE: + if updated_from is not None or updated_to is not None: + log.info(f"Finished - Checking ref Ids: {time_range_str}") + exit() + else: + log.info(f"Sleep {ASYNC_CLEANUP_SLEEP} seconds before processing next time range.") + self.sleep(ASYNC_CLEANUP_SLEEP) + last_id = 0 + min_age_ts = max_age_ts + max_age_ts = int(time.time()) - django.conf.settings.DAEMONS_EXPUNGE_MAX_AGE_SEC + time_range = Q(updateTime__gte=min_age_ts) & Q(updateTime__lte=max_age_ts) + time_range_str = f"updated between: {self.seconds_to_date(min_age_ts)} and {self.seconds_to_date(max_age_ts)}" + else: + self.sleep(django.conf.settings.DAEMONS_BATCH_SLEEP) def deleteRecord(self, queue, primary_key, record_type=None, identifier=None): """ @@ -125,13 +197,49 @@ def deleteRecord(self, queue, primary_key, record_type=None, identifier=None): try: # check if the record to be deleted is a refIdentifier record if (record_type is not None and record_type == 'refId'): - log.info(type(queue)) - log.info("Delete refId: " + str(primary_key)) - queue.objects.filter(id=primary_key).delete() + log.info(f"Delete from {queue.__name__} refId: " + str(primary_key)) + with transaction.atomic(): + obj = queue.objects.select_for_update().get(id=primary_key) + obj.delete() else: - log.info("Delete async entry: " + str(primary_key)) - queue.objects.filter(seq=primary_key).delete() + log.info(f"Delete async queue {queue.__name__} entry: " + str(primary_key)) + with transaction.atomic(): + obj = queue.objects.select_for_update().get(seq=primary_key) + obj.delete() except Exception as e: log.error("Exception occured while processing identifier '" + identifier + "' for '" + record_type + "' table") log.error(e) + + + def date_to_seconds(self, date_time_str: str) -> int: + """ + Convert date/time string to seconds since the Epotch. + For example: + 2024-01-01 00:00:00 => 1704096000 + 2024-10-10 00:00:00 => 1728543600 + + Parameter: + date_time_str: A date/time string in in ISO 8601 format without timezone. + For example: 'YYYYMMDD, YYYYMMDDTHHMMSS, YYYY-MM-DD, YYYY-MM-DDTHH:MM:SS. + + Returns: + int: seconds since the Epotch + + """ + + # Parse the date and time string to a datetime object + dt_object = parse(date_time_str) + + # Convert the datetime object to seconds since the Epoch + seconds_since_epoch = int(dt_object.timestamp()) + + return seconds_since_epoch + + + def seconds_to_date(self, seconds_since_epoch: int) -> str: + dt_object = datetime.fromtimestamp(seconds_since_epoch) + + # Format the datetime object to a string in the desired format + formatted_time = dt_object.strftime("%Y-%m-%dT%H:%M:%S") + return formatted_time \ No newline at end of file From 4fc55b6feed6b9a29c03b7324431a1dc2824b3e6 Mon Sep 17 00:00:00 2001 From: sfisher Date: Wed, 13 Nov 2024 15:07:45 -0800 Subject: [PATCH 06/27] The X button was not showing up at all in Chrome. This fixes that problem by adding to styles and re-compiling the SVG. --- dev/images/icon_cross.svg | 17 +++-------------- dev/scss/_login-modal.scss | 1 + static_src/images/icon_cross.svg | 22 +++------------------- static_src/stylesheets/main2.min.css | 4 +++- 4 files changed, 10 insertions(+), 34 deletions(-) mode change 100755 => 100644 dev/images/icon_cross.svg diff --git a/dev/images/icon_cross.svg b/dev/images/icon_cross.svg old mode 100755 new mode 100644 index ba2b3cc1f..a037959bd --- a/dev/images/icon_cross.svg +++ b/dev/images/icon_cross.svg @@ -1,14 +1,3 @@ - - - - - - - - + + + diff --git a/dev/scss/_login-modal.scss b/dev/scss/_login-modal.scss index 245e7808e..7c204d6b3 100644 --- a/dev/scss/_login-modal.scss +++ b/dev/scss/_login-modal.scss @@ -33,6 +33,7 @@ padding: 2px; background-color: $design-white-color; cursor: pointer; + object-fit: contain; // Or "cover" depending on the fit you need } .login-modal__form { diff --git a/static_src/images/icon_cross.svg b/static_src/images/icon_cross.svg index 6de30c8bb..a037959bd 100644 --- a/static_src/images/icon_cross.svg +++ b/static_src/images/icon_cross.svg @@ -1,19 +1,3 @@ - - - - - - - - - - + + + diff --git a/static_src/stylesheets/main2.min.css b/static_src/stylesheets/main2.min.css index 11c7a4c53..2d946b0ed 100644 --- a/static_src/stylesheets/main2.min.css +++ b/static_src/stylesheets/main2.min.css @@ -1521,7 +1521,9 @@ Selector pattern using above mixin: height: 15px; padding: 2px; background-color: white; - cursor: pointer; } + cursor: pointer; + -o-object-fit: contain; + object-fit: contain; } .login-modal__form { display: -webkit-box; From 0f04f5993532f91b009313068f7918e70c3765d4 Mon Sep 17 00:00:00 2001 From: sfisher Date: Wed, 13 Nov 2024 15:13:16 -0800 Subject: [PATCH 07/27] Give the close image alt text to show "close" for screen readers --- templates/includes/login-modal.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/templates/includes/login-modal.html b/templates/includes/login-modal.html index a3fa07491..4329458e8 100644 --- a/templates/includes/login-modal.html +++ b/templates/includes/login-modal.html @@ -2,7 +2,9 @@