From 2c5196f9a273e6a617df33b58b64e9e3cb0b1a2d Mon Sep 17 00:00:00 2001 From: Norman Hooper Date: Fri, 10 May 2024 10:46:15 +0100 Subject: [PATCH 01/63] Catch missing SQLLocation --- corehq/apps/app_manager/util.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/corehq/apps/app_manager/util.py b/corehq/apps/app_manager/util.py index 5d3c7e63daab..36c80ba31a79 100644 --- a/corehq/apps/app_manager/util.py +++ b/corehq/apps/app_manager/util.py @@ -672,7 +672,11 @@ def get_latest_app_release_by_location(domain, location_id, app_id): Child location's setting takes precedence over parent """ from corehq.apps.app_manager.models import AppReleaseByLocation - location = SQLLocation.active_objects.get(location_id=location_id) + + try: + location = SQLLocation.active_objects.get(location_id=location_id) + except SQLLocation.DoesNotExist: + return None location_and_ancestor_ids = location.get_ancestors(include_self=True).values_list( 'location_id', flat=True).reverse() # get all active enabled releases and order by version desc to get one with the highest version in the end From c84f1a25b378fe05ad6fa29b0b457c0cb2d796ae Mon Sep 17 00:00:00 2001 From: Ethan Soergel Date: Fri, 10 May 2024 14:56:23 -0400 Subject: [PATCH 02/63] override query class instead of using custom `run` This should hopefully make the profiling code a bit less obtrusive It also means the ancestor queries will be profiled now too --- corehq/apps/case_search/utils.py | 78 +++++++++---------- .../xpath_functions/ancestor_functions.py | 6 +- .../xpath_functions/subcase_functions.py | 8 +- 3 files changed, 42 insertions(+), 50 deletions(-) diff --git a/corehq/apps/case_search/utils.py b/corehq/apps/case_search/utils.py index 595adf202f45..8a0a97e072c1 100644 --- a/corehq/apps/case_search/utils.py +++ b/corehq/apps/case_search/utils.py @@ -65,36 +65,32 @@ class CaseSearchProfiler: queries: list = field(default_factory=list) _query_number: int = 0 - def run_query(self, slug, es_query): - self._query_number += 1 - if self.debug_mode: - es_query = es_query.enable_profiling() - - tc = self.timing_context(f'run query #{self._query_number}: {slug}') - timer = tc.peek() - with tc: - results = es_query.run() - - if self.debug_mode: - self.queries.append({ - 'slug': slug, - 'query_number': self._query_number, - 'query': es_query.raw_query, - 'duration': timer.duration, - 'profile_url': self._get_profile_url(slug, self._query_number, results.raw.get('profile')), - }) - return results - - def add_query(self, slug, es_query): - self._query_number += 1 - if self.debug_mode: - self.queries.append({ - 'slug': slug, - 'query_number': self._query_number, - 'query': es_query.raw_query, - 'duration': None, - 'profile_url': None, - }) + def get_case_search_class(self, slug=None): + profiler = self + + class ProfiledCaseSearchES(CaseSearchES): + def run(self): + profiler._query_number += 1 + if profiler.debug_mode: + self.es_query['profile'] = True + + tc = profiler.timing_context(f'run query #{profiler._query_number}: {slug}') + timer = tc.peek() + with tc: + results = super().run() + + if profiler.debug_mode: + profiler.queries.append({ + 'slug': slug, + 'query_number': profiler._query_number, + 'query': self.raw_query, + 'duration': timer.duration, + 'profile_url': profiler._get_profile_url( + slug, profiler._query_number, results.raw.get('profile')), + }) + return results + + return ProfiledCaseSearchES @staticmethod def _get_profile_url(slug, query_number, profile_json): @@ -171,7 +167,7 @@ def get_primary_case_search_results(helper, case_types, criteria, commcare_sort= notify_exception(None, str(e), details={'exception_type': type(e)}) raise CaseSearchUserError(str(e)) - results = helper.profiler.run_query('main', search_es) + results = search_es.run() with helper.profiler.timing_context('wrap_cases'): cases = [helper.wrap_case(hit, include_score=True) for hit in results.raw_hits] return cases @@ -195,7 +191,9 @@ def __init__(self, domain): self.domain = domain self.profiler = CaseSearchProfiler() - def get_base_queryset(self): + def get_base_queryset(self, slug=None): + # slug is only informational, used for profiling + CaseSearchES = self.profiler.get_case_search_class(slug) return CaseSearchES(index=self.config.index_name or None).domain(self.domain) def wrap_case(self, es_hit, include_score=False): @@ -222,7 +220,8 @@ def __init__(self, domain, couch_user, registry_helper): self._couch_user = couch_user self._registry_helper = registry_helper - def get_base_queryset(self): + def get_base_queryset(self, slug=None): + CaseSearchES = self.profiler.get_case_search_class(slug) return CaseSearchES().domain(self._registry_helper.visible_domains) def wrap_case(self, es_hit, include_score=False): @@ -256,7 +255,7 @@ def _get_initial_search_es(self): max_results = CASE_SEARCH_MAX_RESULTS if toggles.INCREASED_MAX_SEARCH_RESULTS.enabled(self.request_domain): max_results = 1500 - return (self.helper.get_base_queryset() + return (self.helper.get_base_queryset('main') .case_type(self.case_types) .is_closed(False) .size(max_results)) @@ -522,12 +521,10 @@ def get_child_case_types(app): @time_function() def get_child_case_results(helper, parent_case_ids, child_case_types=None): - query = helper.get_base_queryset().get_child_cases(parent_case_ids, "parent") + query = helper.get_base_queryset('get_child_case_results').get_child_cases(parent_case_ids, "parent") if child_case_types: query = query.case_type(child_case_types) - - results = helper.profiler.run_query('get_child_case_results', query) - return [helper.wrap_case(result) for result in results.hits] + return [helper.wrap_case(result) for result in query.run().hits] @time_function() @@ -541,9 +538,8 @@ def get_expanded_case_results(helper, custom_related_case_property, cases): @time_function() def _get_case_search_cases(helper, case_ids): - query = helper.get_base_queryset().case_ids(case_ids) - results = helper.profiler.run_query('_get_case_search_cases', query) - return [helper.wrap_case(result) for result in results.hits] + query = helper.get_base_queryset('_get_case_search_cases').case_ids(case_ids) + return [helper.wrap_case(result) for result in query.run().hits] # Warning: '_tag_is_related_case' may cause the relevant user-defined properties to be overwritten. diff --git a/corehq/apps/case_search/xpath_functions/ancestor_functions.py b/corehq/apps/case_search/xpath_functions/ancestor_functions.py index 042b3f3fa587..985ed1f7804d 100644 --- a/corehq/apps/case_search/xpath_functions/ancestor_functions.py +++ b/corehq/apps/case_search/xpath_functions/ancestor_functions.py @@ -90,8 +90,7 @@ def _is_ancestor_path_expression(node): def _child_case_lookup(context, case_ids, identifier): """returns a list of all case_ids who have parents `case_id` with the relationship `identifier` """ - es_query = context.helper.get_base_queryset().get_child_cases(case_ids, identifier) - context.profiler.add_query('_child_case_lookup', es_query) + es_query = context.helper.get_base_queryset('_child_case_lookup').get_child_cases(case_ids, identifier) return es_query.get_ids() @@ -144,8 +143,7 @@ def _get_case_ids_from_ast_filter(context, filter_node): else: from corehq.apps.case_search.filter_dsl import build_filter_from_ast es_filter = build_filter_from_ast(filter_node, context) - es_query = context.helper.get_base_queryset().filter(es_filter) - context.profiler.add_query('_get_case_ids_from_ast_filter', es_query) + es_query = context.helper.get_base_queryset('_get_case_ids_from_ast_filter').filter(es_filter) if es_query.count() > MAX_RELATED_CASES: new_query = serialize(filter_node) raise TooManyRelatedCasesError( diff --git a/corehq/apps/case_search/xpath_functions/subcase_functions.py b/corehq/apps/case_search/xpath_functions/subcase_functions.py index b890a21465cf..b8f94b5bf5f2 100644 --- a/corehq/apps/case_search/xpath_functions/subcase_functions.py +++ b/corehq/apps/case_search/xpath_functions/subcase_functions.py @@ -99,8 +99,8 @@ def _run_subcase_query(subcase_query, context): else: subcase_filter = filters.match_all() - es_query = ( - context.helper.get_base_queryset() + return ( + context.helper.get_base_queryset('subcase_query') .nested( 'indices', queries.filtered( @@ -113,11 +113,9 @@ def _run_subcase_query(subcase_query, context): ) .filter(subcase_filter) .source(['indices.referenced_id', 'indices.identifier']) + .run().hits ) - results = context.profiler.run_query('subcase_query', es_query) - return results.hits - def _parse_normalize_subcase_query(node) -> SubCaseQuery: """Parse the subcase query and normalize it to the form 'subcase-count > N' or 'subcase-count = N' From c93dde89ab694093e076cb0de07f5f5933cc097a Mon Sep 17 00:00:00 2001 From: Ethan Soergel Date: Fri, 10 May 2024 15:14:59 -0400 Subject: [PATCH 03/63] Move profile processing outside timing context To avoid messing up the numbers of the actual query --- corehq/apps/case_search/utils.py | 15 +-------------- corehq/apps/case_search/views.py | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/corehq/apps/case_search/utils.py b/corehq/apps/case_search/utils.py index 8a0a97e072c1..3cdcc6085435 100644 --- a/corehq/apps/case_search/utils.py +++ b/corehq/apps/case_search/utils.py @@ -3,7 +3,6 @@ from collections import defaultdict from dataclasses import dataclass, field from functools import wraps -from io import BytesIO from django.utils.functional import cached_property from django.utils.translation import gettext as _ @@ -44,13 +43,11 @@ reverse_index_case_query, wrap_case_search_hit, ) -from corehq.apps.hqadmin.utils import get_download_url from corehq.apps.registry.exceptions import ( RegistryAccessException, RegistryNotFound, ) from corehq.apps.registry.helper import DataRegistryHelper -from corehq.util.dates import get_timestamp_for_filename from corehq.util.quickcache import quickcache from corehq.util.timer import TimingContext @@ -85,22 +82,12 @@ def run(self): 'query_number': profiler._query_number, 'query': self.raw_query, 'duration': timer.duration, - 'profile_url': profiler._get_profile_url( - slug, profiler._query_number, results.raw.get('profile')), + 'profile_json': results.raw.pop('profile'), }) return results return ProfiledCaseSearchES - @staticmethod - def _get_profile_url(slug, query_number, profile_json): - timestamp = get_timestamp_for_filename() - name = f'es_profile_{query_number}_{slug}_{timestamp}.json' - io = BytesIO() - io.write(json.dumps(profile_json).encode('utf-8')) - io.seek(0) - return get_download_url(io, name, content_type='application/json') - def time_function(): """Decorator to get timing information on a case search function that has `helper` as the first arg""" diff --git a/corehq/apps/case_search/views.py b/corehq/apps/case_search/views.py index f27170d9aa12..b0ce5fa60213 100644 --- a/corehq/apps/case_search/views.py +++ b/corehq/apps/case_search/views.py @@ -1,18 +1,21 @@ import json import re +from io import BytesIO from django.http import Http404 from django.urls import reverse +from django.utils.decorators import method_decorator from django.utils.translation import gettext_lazy -from django.utils.decorators import method_decorator from dimagi.utils.web import json_response from corehq.apps.case_search.models import case_search_enabled_for_domain from corehq.apps.case_search.utils import get_case_search_results_from_request from corehq.apps.domain.decorators import cls_require_superuser_or_contractor from corehq.apps.domain.views.base import BaseDomainView +from corehq.apps.hqadmin.utils import get_download_url from corehq.apps.hqwebapp.decorators import use_bootstrap5 +from corehq.util.dates import get_timestamp_for_filename from corehq.util.view_utils import BadRequest, json_error @@ -115,5 +118,16 @@ def post(self, request, *args, **kwargs): 'primary_count': profiler.primary_count, 'related_count': profiler.related_count, 'timing_data': profiler.timing_context.to_dict(), - 'queries': profiler.queries, + 'queries': [self._make_profile_downloadable(q) for q in profiler.queries], }) + + @staticmethod + def _make_profile_downloadable(query): + profile_json = query.pop('profile_json') + timestamp = get_timestamp_for_filename() + name = f"es_profile_{query['query_number']}_{query['slug']}_{timestamp}.json" + io = BytesIO() + io.write(json.dumps(profile_json).encode('utf-8')) + io.seek(0) + query['profile_url'] = get_download_url(io, name, content_type='application/json') + return query From b1ee6aaa8436d4b67f779330909e014acfc02f5d Mon Sep 17 00:00:00 2001 From: Jonathan Tang Date: Tue, 21 May 2024 17:30:24 -0700 Subject: [PATCH 04/63] prefactor: Separate out base Tableau User Form from the form specific to editing Web User --- corehq/apps/users/forms.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/corehq/apps/users/forms.py b/corehq/apps/users/forms.py index 88b09e72c135..d942ef1213c4 100644 --- a/corehq/apps/users/forms.py +++ b/corehq/apps/users/forms.py @@ -1762,7 +1762,7 @@ def clean(self): return data -class TableauUserForm(forms.Form): +class BaseTableauUserForm(forms.Form): role = forms.ChoiceField( label=gettext_noop("Role"), choices=TableauUser.Roles.choices, @@ -1776,27 +1776,36 @@ class TableauUserForm(forms.Form): ) def __init__(self, *args, **kwargs): - readonly = kwargs.pop('readonly', True) - self.request = kwargs.pop('request') self.domain = kwargs.pop('domain', None) - self.username = kwargs.pop('username', None) - super(TableauUserForm, self).__init__(*args, **kwargs) + super(BaseTableauUserForm, self).__init__(*args, **kwargs) self.allowed_tableau_groups = [ TableauGroupTuple(group.name, group.id) for group in get_all_tableau_groups(self.domain) if group.name in get_allowed_tableau_groups_for_domain(self.domain)] - user_group_names = [group.name for group in get_tableau_groups_for_user(self.domain, self.username)] self.fields['groups'].choices = [] self.fields['groups'].initial = [] for i, group in enumerate(self.allowed_tableau_groups): # Add a choice for each tableau group on the server self.fields['groups'].choices.append((i, group.name)) - if group.name in user_group_names: - # Pre-choose groups that the user already belongs to - self.fields['groups'].initial.append(i) if not self.fields['groups'].choices: del self.fields['groups'] + +class TableauUserForm(BaseTableauUserForm): + + def __init__(self, *args, **kwargs): + self.request = kwargs.pop('request') + readonly = kwargs.pop('readonly', True) + self.username = kwargs.pop('username', None) + super(TableauUserForm, self).__init__(*args, **kwargs) + + if 'groups' in self.fields: + user_group_names = [group.name for group in get_tableau_groups_for_user(self.domain, self.username)] + for i, group_name in self.fields['groups'].choices: + if group_name in user_group_names: + # Pre-choose groups that the user already belongs to + self.fields['groups'].initial.append(i) + if readonly: self.fields['role'].widget.attrs['readonly'] = True self.fields['groups'].widget.attrs['disabled'] = True From 9b2ee89216dfa414621be03cfd9b1dee39630484 Mon Sep 17 00:00:00 2001 From: Jonathan Tang Date: Tue, 21 May 2024 17:37:42 -0700 Subject: [PATCH 05/63] bug: save depends on 'groups' field existing. Hide field insted of deleting if choices is empty --- corehq/apps/users/forms.py | 21 ++++++++++++------- .../users/templates/users/edit_web_user.html | 1 - 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/corehq/apps/users/forms.py b/corehq/apps/users/forms.py index d942ef1213c4..de5f2ab59dee 100644 --- a/corehq/apps/users/forms.py +++ b/corehq/apps/users/forms.py @@ -1787,8 +1787,6 @@ def __init__(self, *args, **kwargs): for i, group in enumerate(self.allowed_tableau_groups): # Add a choice for each tableau group on the server self.fields['groups'].choices.append((i, group.name)) - if not self.fields['groups'].choices: - del self.fields['groups'] class TableauUserForm(BaseTableauUserForm): @@ -1799,12 +1797,11 @@ def __init__(self, *args, **kwargs): self.username = kwargs.pop('username', None) super(TableauUserForm, self).__init__(*args, **kwargs) - if 'groups' in self.fields: - user_group_names = [group.name for group in get_tableau_groups_for_user(self.domain, self.username)] - for i, group_name in self.fields['groups'].choices: - if group_name in user_group_names: - # Pre-choose groups that the user already belongs to - self.fields['groups'].initial.append(i) + user_group_names = [group.name for group in get_tableau_groups_for_user(self.domain, self.username)] + for i, group_name in self.fields['groups'].choices: + if group_name in user_group_names: + # Pre-choose groups that the user already belongs to + self.fields['groups'].initial.append(i) if readonly: self.fields['role'].widget.attrs['readonly'] = True @@ -1819,6 +1816,14 @@ def __init__(self, *args, **kwargs): self.helper.label_class = 'col-sm-3 col-md-2' self.helper.field_class = 'col-sm-9 col-md-8 col-lg-6' + self.helper.layout = crispy.Layout( + crispy.Fieldset( + _('Tableau Configuration'), + 'role', + 'groups' if len(self.fields['groups'].choices) > 0 else None + ) + ) + def save(self, username, commit=True): if not self.request.couch_user.has_permission(self.domain, 'edit_user_tableau_config'): raise forms.ValidationError(_("You do not have permission to edit Tableau Configuraion.")) diff --git a/corehq/apps/users/templates/users/edit_web_user.html b/corehq/apps/users/templates/users/edit_web_user.html index 6123b9cab1b5..ef61c98a1893 100644 --- a/corehq/apps/users/templates/users/edit_web_user.html +++ b/corehq/apps/users/templates/users/edit_web_user.html @@ -108,7 +108,6 @@ {% csrf_token %}
- {% trans 'Tableau Configuration' %} {% crispy tableau_form %}
{% if not request.is_view_only and edit_user_tableau_config%} From a6732e7d2e206a03fc5e20247a6a8a707ccbdfb3 Mon Sep 17 00:00:00 2001 From: Ethan Soergel Date: Wed, 22 May 2024 17:09:16 -0400 Subject: [PATCH 06/63] Add case search query language docs Copied and reformatted from here: https://dimagi.atlassian.net/wiki/spaces/GS/pages/2146611711/Case+Search+Query+Language+CSQL --- docs/case_search_query_language.rst | 449 ++++++++++++++++++++++++++++ docs/index.rst | 1 + 2 files changed, 450 insertions(+) create mode 100644 docs/case_search_query_language.rst diff --git a/docs/case_search_query_language.rst b/docs/case_search_query_language.rst new file mode 100644 index 000000000000..0e6386dd2507 --- /dev/null +++ b/docs/case_search_query_language.rst @@ -0,0 +1,449 @@ +================================= +Case Search Query Language (CSQL) +================================= + +Underpinning some of the advanced search capabilities of `Case Search`_ and `Case List Explorer`_ is +the Case Search Query Language. This page describes the syntax and capabilities of the language as +well as it's limitations. + +.. _Case Search: https://dimagi.atlassian.net/wiki/spaces/GS/pages/2146606528 +.. _Case List Explorer: https://dimagi.atlassian.net/wiki/x/KTXKfw + +.. contents:: + :local: + +Syntax +====== + +* **Available operators**: ``and``, ``or`` , ``=``, ``!=``, ``<``, ``<=``, ``>``, ``>=``, ``(``, + ``)`` +* **Brackets**: can be used to group logic and make complex expressions +* **Dates**: dates can be filtered with format ``YYYY-MM-DD``. This must include the apostrophes + around the date. The ``date()`` function may be used to validate a date value: + ``date('2021-08-20')`` + +Examples:: + + name = 'john' and age > 10 + + region = 'london' and registration_date < '2019-12-01' + + (sex = 'm' and weight < 23) or (sex = 'f' and weight < 20) + + status = 'negative' and subcase-exists('host', @case_type = 'lab_result' and result = 'positive') + + +Supported Functions +=================== + +The following functions are supported: + + +``date`` +-------- + +* **Behavior**: Will convert a string or a number value into an equivalent date. Will throw an error + if the format of the string is wrong or an invalid date is passed. +* **Return**: Returns a date +* **Arguments**: The value to be converted (either a string in the format ``YYYY-MM-DD`` or a + number). Numbers are interpreted as days since Jan 01 1970. +* **Usage**: ``date(value_to_convert)`` + +``today`` +--------- +* **Return**: Returns the current date according in the timezone of the project space. +* **Arguments**: None +* **Usage**: ``today()`` +* **Note**: When comparing the output of this function to a value with a date and a time by using + the ``=`` operator, this function returns the current date at midnight. For example, + ``last_modified=today()`` will look for an exact match between the date and time in + ``last_modified`` and the current date exactly at midnight. + +``not`` +------- +* **Behavior**: Invert a boolean search expression +* **Arguments**: The expression to invert +* **Usage**: ``not(is_active = 0 and stock_level < 15)`` + +``starts-with`` +--------------- +* **Behavior**: Match cases where a multi-select case property value begins with the given argument + value. +* **Arguments**: Two arguments, the multi-select case property and the value to check. +* **Notes**: + * This filter is case sensitive + * Using this filter may impact the performance of your query +* **Usage**: + * ``starts-with(social_security_num, "123")`` + * ``starts-with(timezone, "Africa/")`` + +``selected`` +------------ +* **Behavior**: Match cases where a multi-select case property contains the given value. The + behavior of this function matches that of the ``selected-any`` function. +* **Return**: True if that particular value is present in the case property. Otherwise False. +* **Arguments**: Two arguments, the multi-select case property and the value to check. +* **Notes**: + * If the 'value to check' contains spaces, each word will be considered independently as in + ``select-any`` + * `See notes on how case properties are segmented `_ +* **Usage**: + * ``selected(tests_performed, "testA")`` + * ``selected(tests_performed, "testA testB")`` + * This works the same as the 'selected-any' function. Best practice would be to use + ``selected-any`` in this instance to make the intention clear. + +``selected-any`` +---------------- +* **Behavior**: Match cases where a multi-select case property contains ANY of the values in the + input provided +* **Arguments**: Two arguments, the multi-select case property and the values to check represented + as a space separated string. +* **Notes**: `See notes on how case properties are segmented `_ +* **Usage**: ``selected-any(tests_performed, "testA testB testC")`` + +.. list-table:: Outcomes table for ``selected-any`` + :header-rows: 1 + + * - Search term + - Case Property Value + - Search Result + - Note + * - value1 + - value2 **value1** value3 + - Match + - Property contains all of the search terms + * - value1 value2 + - **value2** value5 **value1** value3 + - Match + - Property contains all of the search terms + * - value1 value2 + - **value1** value3 + - Match + - Property contains at least one of the search terms + * - value1 value2 + - value3 value4 + - No Match + - Property does not contain any of the search terms + +``selected-all`` +---------------- + +* **Behavior**: Match cases where a multi-select case property contains ALL of the values in the + input provided +* **Arguments**: Two arguments, the multi-select case property and the values to check represented + as a space separated string. +* **Notes**: + * `See notes on how case properties are segmented `_ +* **Usage**: ``selected-all(tests_performed, "testA testB testC")`` + +.. list-table:: Outcomes table for ``selected-all`` + :header-rows: 1 + + * - Search term + - Case Property Value + - Search Result + - Note + * - value1 + - value2 **value1** value3 + - Match + - Property contains all of the search terms + * - value1 value2 + - **value2** value5 **value1** value3 + - Match + - Property contains all of the search terms + * - value1 value2 + - **value1** value3 + - No match + - Property does not contain ALL of the search terms + +``within-distance`` +------------------- +* **Requirements**: GPS case properties set up as described in this page: `Storing GPS Case + Properties in Elasticsearch as GeoPoints `_ +* **Behavior**: Match cases within a certain geographic distance (as the crow flies) of the provided + point +* **Return**: True if that case is within range, otherwise false +* **Arguments**: + * ``property_name``: The GPS case property on the cases being searched + * ``coordinates``: This can be the output of a "geopoint" receiver from a geocoder question as + described in `Address Geocoding in Web Apps `_ + * ``distance``: The distance from ``coordinates`` to search + * ``unit``: The units for that distance. Options are: miles, yards, feet, inch, kilometers, + meters, centimeters, millimeters, nauticalmiles +* **Usage**: ``within-distance(location, '42.4402967 -71.1453275', 30, 'miles')`` + +.. _geopoints: https://dimagi.atlassian.net/wiki/x/dgf4fw +.. _address_geocoding: https://dimagi.atlassian.net/wiki/x/dALKfw + +``fuzzy-match`` +--------------- +* **Behavior**: Determines if a given value is a fuzzy match for a given case property. This ignores + the `domain-level case search fuzziness settings `_. +* **Return**: True if that particular value matches the case property. Otherwise False. +* **Arguments**: Two arguments: the case property and the value to check. +* **Usage**: ``fuzzy-match(first_name, "Sara")`` + +.. note:: + ``fuzzy-match`` is backed by Elasticsearch's `Fuzzy query`_, which uses `Levenshtein distance`_ + to gauge similarity. To consider something a match, it requires an exact prefix match and an edit + distance based on the length of the string (longer strings can have more edits). + +.. _fuzziness_settings: https://dimagi.atlassian.net/wiki/spaces/GS/pages/2146606704/Project+Case+Search+Configuration#Configure-Fuzzy-Properties-(optional) +.. _Fuzzy Query: https://www.elastic.co/guide/en/elasticsearch/reference/8.11/query-dsl-fuzzy-query.html +.. _Levenshtein distance: https://en.wikipedia.org/wiki/Levenshtein_distance + +``phonetic-match`` +------------------ +* **Behavior**: Match cases if a given value "sounds like" (using `Soundex`_) the value of a given + case property. (e.g. "Joolea" will match "Julia") +* **Return**: True if that particular value matches the case property. Otherwise False. +* **Arguments**: Two arguments: the case property and the value to check. +* **Usage**: ``phonetic-match(first_name, "Julia")`` + +.. _Soundex: https://en.wikipedia.org/wiki/Soundex#American_Soundex + +``match-all`` +------------- +* **Behavior**: Matches ALL cases +* **Arguments**: No arguments +* **Usage**: ``match-all()`` +* **Example**: ``match-all() and first_name = "Julia"`` + * Matches cases that have a property ``first_name`` equal to ``"Julia"`` + +``match-none`` +-------------- +* **Behavior**: Matches no cases at all +* **Arguments**: No arguments +* **Usage**: ``match-none()`` +* **Example**: ``match-none() or first_name = "Julia"`` + * Matches cases that have a property ``first_name`` equal to ``"Julia"`` + + +Filtering on parent (ancestor) cases +==================================== + +Searches may be performed against ancestor cases (e.g. parent cases) using the ``/`` operator + +.. code-block:: + + # search for cases that have a 'parent' case that matches the filter 'age > 55' + parent/age > 55 + + # successive steps can be added to navigate further up the case hierarchy + parent/parent/dod = '' + +``ancestor-exists`` +------------------- +* **Behavior**: Match cases that have an ancestor with the given relation that matches the ancestor + filter expression. +* **Arguments**: Two arguments, the ancestor relationship (usually one of parent or host) and the + ancestor filter expression. +* **Usage**: + * ``ancestor-exists(parent/parent, city = 'SF')`` + * ``ancestor-exists(parent, food_included = 'yes' and ancestor-exists(parent, city!='' and + selected(city, 'Boston')))`` +* **Limitation**: + * The arguments can't be a standalone function and must be a binary expression + * This will *not* work: ``ancestor-exists(parent, selected(city, 'SF'))`` + * This will work: ``ancestor-exists(parent, city != '' and selected(city, 'SF'))`` + * The ancestor filter expression may not include ``subcase-exists`` or ``subcase-count`` + * **Best Practices**: + * Limit multiple uses of this function in your query to avoid performance implications + * Add as many arguments to ``ancestor-exists()`` as possible to help narrow down results (i.e., + by ``@case_type`` or ``@status``) + + +Filtering on child cases (subcases) +=================================== + +Special functions are provided to support filtering based on the properties of subcases. These are: + +``subcase-exists`` +------------------ +* **Behavior**: Match cases that have a subcase with the given relation that matches the subcase + filter expression. +* **Arguments**: Two arguments, the subcase relationship (usually one of 'parent' or 'host') and the + subcase filter expression. +* **Usage**: ``subcase-exists('parent', lab_type = 'blood' and result = 1)`` +* **Best Practices**: + * Limit multiple uses of this function in your query to avoid performance implications + * Add as many arguments to ``subcase-exists()`` as possible to help narrow down results (i.e., + by ``@case_type`` or ``@status``) + +``subcase-count`` +----------------- +* **Behavior**: Match cases where the number of subcases matches the given expression. +* **Arguments**: Two arguments, the subcase relationship (usually one of 'parent' or 'host') and the + subcase filter expression. +* **Usage**: ``subcase-count('parent', lab_type = 'blood' and result = 1) > 3`` + * The count function must be used in conjunction with a comparison operator. All operators are + supported (``=``, ``!=``, ``<``, ``<=``, ``>``, ``>=``) +* **Best Practices**: + * Limit multiple uses of this function in your query to avoid performance implications + +.. warning:: + When utilizing the special subcase function, be mindful that the *quantity of search results* + and the *number of subcase functions* in a single search are important factors. As the number of + subcase functions and search results increases, the time required to perform the search will + also increase. + + **Performance testing is required** when using this function in your app. + * `Link to performance testing in HQ's case search (not Web Apps) `_ + * `Link to performance testing in HQ's case search (WebApps - BHA) `_ + + Keep in mind that a higher number of search results will lead to longer execution times for the + search query. The threshold is around 400K to 500K search results, after which a timeout error + may occur. It is recommended to keep your search results well below this number for optimal + performance. + + To manage the number of search results when incorporating subcase functions in your search + query, you can apply required fields in the search form. For instance, requiring users to search + by both first and last name is more effective than just using the first name. Including more + required fields in the search form is likely to reduce the number of search results returned. + + If you have any questions regarding the limitation usage of this subcase function, please reach + out to the AE team. + + +.. _perf_testing: https://docs.google.com/spreadsheets/d/1T4tX1tbFaiTBFWpJoi_4p4_UtnRL3_SRoXJgADraXo8/edit#gid=1095465676 +.. _bha_perf_testing: https://docs.google.com/spreadsheets/d/1B9ySwSahf4qUKWEmX0FfdK0uArOBnJ6pXWh7eYvnaiA/edit#gid=671814666 + +**Examples** + +A very common implementation of subcase-exists search queries involves utilizing the user's +'search-input'. Please see an example of this configuration below. + +.. code-block:: + + if(count(instance("search-input:results")/input/field[@name = "clinic"]), + concat('subcase-exists("parent", @case_type = "service" and current_status = "active" and central_registry = "yes" and clinic_case_id = "', + instance("search-input:results")/input/field[@name = "clinic"], + '")'), + '@case_id != "c"') + + +.. _multiselect: + +Multi-select case property searches +=================================== +As shown above, the ``selected`` , ``selected-any`` and ``selected-all`` functions can be used to filter cases based on multi-select case properties. + +A multi-select case property is a case property whose value contains multiple 'terms'. Each 'term' in the case property value is typically separated by a space. +tests_completed = 'math english physics' + +The following table illustrates how a case property value is split up into component terms. Note that some characters are removed and other are used as separators. + +.. list-table:: + :header-rows: 1 + + * - Case property value + - Searchable terms + - Note + * - Case property value + - Searchable terms + - Note + * - word1 word2 word3 + - [word1, word2, word3] + - Split on white space + * - word1 word-two 9-8 + - [word1, word, two, 9, 8] + - Split on '-' + * - word1 word_2 + - [word1, word_2] + - Not split on '_' + * - word1 5.9 word.2 + - [word1, 5.9, word, 2] + - Split on 'period' between 'letters' but not between + * - 'word1' "word2" word3?! + - [word1, word2, word3] + - Quotes and punctuation are ignored + * - 你好 + - [你, 好] + - Supports unicode characters + * - word1 🧀 🍌 word2 + - [word1, word2] + - Emoji are ignored + * - word's + - [words, words] + - Apostrophe are removed + * - word"s + - [word, s] + - Split on double quote between letters + * - word1\\nword2 + - [word1, word2] + - Split on white space ("\n" is a newline) + * - 12/2=6x1 4*5 98% 3^2 + - [12, 2, 6x1, 4, 5, 98, 3, 2] + - Split on 'non-word' characters + * - start Date: Wed, 22 May 2024 13:28:02 -0700 Subject: [PATCH 07/63] feat: add tableau fields to invite form --- corehq/apps/registration/forms.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/corehq/apps/registration/forms.py b/corehq/apps/registration/forms.py index 2a110e8ca991..7a5eea497c24 100644 --- a/corehq/apps/registration/forms.py +++ b/corehq/apps/registration/forms.py @@ -24,7 +24,7 @@ from corehq.apps.hqwebapp import crispy as hqcrispy from corehq.apps.hqwebapp.utils.translation import mark_safe_lazy from corehq.apps.programs.models import Program -from corehq.apps.users.forms import BaseLocationForm +from corehq.apps.users.forms import BaseLocationForm, BaseTableauUserForm from corehq.apps.users.models import CouchUser @@ -512,6 +512,9 @@ def __init__(self, data=None, excluded_emails=None, is_add_user=None, choices = [('', '')] + list((prog.get_id, prog.name) for prog in programs) self.fields['program'].choices = choices self.excluded_emails = excluded_emails or [] + + self._initialize_tableau_fields(data, domain) + self.helper = FormHelper() self.helper.form_method = 'POST' self.helper.form_class = 'form-horizontal form-ko-validation' @@ -530,6 +533,8 @@ def __init__(self, data=None, excluded_emails=None, is_add_user=None, 'profile' if ('profile' in self.fields and len(self.fields['profile'].choices) > 0) else None, 'assigned_locations' if should_show_location else None, 'primary_location' if should_show_location else None, + 'tableau_role', + 'tableau_group_ids' ), crispy.HTML( render_to_string( @@ -569,3 +574,10 @@ def clean(self): if isinstance(cleaned_data[field], str): cleaned_data[field] = cleaned_data[field].strip() return cleaned_data + + def _initialize_tableau_fields(self, data, domain): + tableau_form = BaseTableauUserForm(data, domain=domain) + self.fields['tableau_group_ids'] = tableau_form.fields["groups"] + self.fields['tableau_group_ids'].label = _('Tableau Groups') + self.fields['tableau_role'] = tableau_form.fields['role'] + self.fields['tableau_role'].label = _('Tableau Role') From d3bf0ca33ff72a7d4d928e5b388dcce147158618 Mon Sep 17 00:00:00 2001 From: Jonathan Tang Date: Wed, 22 May 2024 16:43:48 -0700 Subject: [PATCH 08/63] validate for edit access and store ids of tableau groups On post, the data is passed directly to create Invitation which expects the key to be "tableau_group_ids" --- corehq/apps/registration/forms.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/corehq/apps/registration/forms.py b/corehq/apps/registration/forms.py index 7a5eea497c24..f2d90f5f12a3 100644 --- a/corehq/apps/registration/forms.py +++ b/corehq/apps/registration/forms.py @@ -493,6 +493,7 @@ class AdminInvitesUserForm(BaseLocationForm): def __init__(self, data=None, excluded_emails=None, is_add_user=None, role_choices=(), should_show_location=False, *, domain, **kwargs): super(AdminInvitesUserForm, self).__init__(domain=domain, data=data, **kwargs) + self.request = kwargs.get('request') domain_obj = Domain.get_by_name(domain) self.fields['role'].choices = role_choices if domain_obj: @@ -513,7 +514,8 @@ def __init__(self, data=None, excluded_emails=None, is_add_user=None, self.fields['program'].choices = choices self.excluded_emails = excluded_emails or [] - self._initialize_tableau_fields(data, domain) + if self.request.couch_user.has_permission(self.domain, 'edit_user_tableau_config'): + self._initialize_tableau_fields(data, domain) self.helper = FormHelper() self.helper.form_method = 'POST' @@ -533,8 +535,8 @@ def __init__(self, data=None, excluded_emails=None, is_add_user=None, 'profile' if ('profile' in self.fields and len(self.fields['profile'].choices) > 0) else None, 'assigned_locations' if should_show_location else None, 'primary_location' if should_show_location else None, - 'tableau_role', - 'tableau_group_ids' + 'tableau_role' if 'tableau_role' in self.fields else None, + 'tableau_group_indices' if 'tableau_group_indices' in self.fields else None ), crispy.HTML( render_to_string( @@ -570,14 +572,26 @@ def clean_email(self): def clean(self): cleaned_data = super(AdminInvitesUserForm, self).clean() + + if (('tableau_role' in cleaned_data or 'tableau_group_indices' in cleaned_data) + and not self.request.couch_user.has_permission(self.domain, 'edit_user_tableau_config')): + raise forms.ValidationError(_("You do not have permission to edit Tableau Configuraion.")) + + if 'tableau_group_indices' in cleaned_data: + cleaned_data['tableau_group_ids'] = [ + self.tableau_form.allowed_tableau_groups[int(i)].id + for i in cleaned_data['tableau_group_indices'] + ] + del cleaned_data['tableau_group_indices'] + for field in cleaned_data: if isinstance(cleaned_data[field], str): cleaned_data[field] = cleaned_data[field].strip() return cleaned_data def _initialize_tableau_fields(self, data, domain): - tableau_form = BaseTableauUserForm(data, domain=domain) - self.fields['tableau_group_ids'] = tableau_form.fields["groups"] - self.fields['tableau_group_ids'].label = _('Tableau Groups') - self.fields['tableau_role'] = tableau_form.fields['role'] + self.tableau_form = BaseTableauUserForm(data, domain=domain) + self.fields['tableau_group_indices'] = self.tableau_form.fields["groups"] + self.fields['tableau_group_indices'].label = _('Tableau Groups') + self.fields['tableau_role'] = self.tableau_form.fields['role'] self.fields['tableau_role'].label = _('Tableau Role') From 0a2acd750f6d297d97521b18d6baf5e4ee92363e Mon Sep 17 00:00:00 2001 From: Jonathan Tang Date: Wed, 22 May 2024 17:07:41 -0700 Subject: [PATCH 09/63] bug: profile needs to be passed for creation of webuser this logic flow is for users who already requested access and the submitted invitation form is for that user --- corehq/apps/users/views/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/corehq/apps/users/views/__init__.py b/corehq/apps/users/views/__init__.py index 8dac664b721c..1a3e402f2f25 100644 --- a/corehq/apps/users/views/__init__.py +++ b/corehq/apps/users/views/__init__.py @@ -1153,6 +1153,10 @@ def post(self, request, *args, **kwargs): create_invitation = True data = self.invite_web_user_form.cleaned_data domain_request = DomainRequest.by_email(self.domain, data["email"]) + profile_id = data.get("profile", None) + profile = CustomDataFieldsProfile.objects.get( + id=profile_id, + definition__domain=self.domain) if profile_id else None if domain_request is not None: domain_request.is_approved = True domain_request.save() @@ -1164,6 +1168,7 @@ def post(self, request, *args, **kwargs): primary_location_id=data.get("primary_location", None), program_id=data.get("program", None), assigned_location_ids=data.get("assigned_locations", None), + profile=profile ) messages.success(request, "%s added." % data["email"]) else: @@ -1184,10 +1189,7 @@ def post(self, request, *args, **kwargs): if primary_location_id: assert primary_location_id in assigned_location_ids self._assert_user_has_permission_to_access_locations(assigned_location_ids) - profile_id = data.get("profile", None) - data["profile"] = CustomDataFieldsProfile.objects.get( - id=profile_id, - definition__domain=self.domain) if profile_id else None + data["profile"] = profile invite = Invitation(**data) invite.save() From 5accbcb183a7e550d3475150d9cc42eaaa0b9509 Mon Sep 17 00:00:00 2001 From: Jonathan Tang Date: Wed, 22 May 2024 17:10:02 -0700 Subject: [PATCH 10/63] feat: uses tableau role and group when adding web user if already requested access --- corehq/apps/users/views/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/corehq/apps/users/views/__init__.py b/corehq/apps/users/views/__init__.py index 1a3e402f2f25..bde158ad5c83 100644 --- a/corehq/apps/users/views/__init__.py +++ b/corehq/apps/users/views/__init__.py @@ -1168,7 +1168,9 @@ def post(self, request, *args, **kwargs): primary_location_id=data.get("primary_location", None), program_id=data.get("program", None), assigned_location_ids=data.get("assigned_locations", None), - profile=profile + profile=profile, + tableau_role=data.get("tableau_role", None), + tableau_group_ids=data.get("tableau_group_ids", None) ) messages.success(request, "%s added." % data["email"]) else: From c95a2792ef42db00570615bab8b472d5faa83a52 Mon Sep 17 00:00:00 2001 From: Jonathan Tang Date: Wed, 22 May 2024 17:13:26 -0700 Subject: [PATCH 11/63] use tableau role and group ids from invitation for accepting invite via management command --- corehq/apps/users/management/commands/accept_invite.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/corehq/apps/users/management/commands/accept_invite.py b/corehq/apps/users/management/commands/accept_invite.py index c36b32ba3d10..3a44f191d5be 100644 --- a/corehq/apps/users/management/commands/accept_invite.py +++ b/corehq/apps/users/management/commands/accept_invite.py @@ -29,7 +29,9 @@ def handle(self, username, domain, **options): assigned_location_ids=list( invitation.assigned_locations.all().values_list('location_id', flat=True)), program_id=invitation.program, - profile=invitation.profile) + profile=invitation.profile, + tableau_role=invitation.tableau_role, + tableau_group_ids=invitation.tableau_group_ids) invitation.is_accepted = True invitation.save() print("Operation completed") From 654781338d52c0ea3a094ad460767a9eb2324498 Mon Sep 17 00:00:00 2001 From: Jonathan Tang Date: Thu, 23 May 2024 11:43:42 -0700 Subject: [PATCH 12/63] bug: need to consider FF when displaying tableau cofig field --- corehq/apps/registration/forms.py | 9 +++++---- corehq/apps/users/views/__init__.py | 4 ++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/corehq/apps/registration/forms.py b/corehq/apps/registration/forms.py index f2d90f5f12a3..5b0a6bc1d5f4 100644 --- a/corehq/apps/registration/forms.py +++ b/corehq/apps/registration/forms.py @@ -491,9 +491,10 @@ class AdminInvitesUserForm(BaseLocationForm): role = forms.ChoiceField(choices=(), label="Project Role") def __init__(self, data=None, excluded_emails=None, is_add_user=None, - role_choices=(), should_show_location=False, *, domain, **kwargs): + role_choices=(), should_show_location=False, can_edit_tableau_config=False, + *, domain, **kwargs): super(AdminInvitesUserForm, self).__init__(domain=domain, data=data, **kwargs) - self.request = kwargs.get('request') + self.can_edit_tableau_config = can_edit_tableau_config domain_obj = Domain.get_by_name(domain) self.fields['role'].choices = role_choices if domain_obj: @@ -514,7 +515,7 @@ def __init__(self, data=None, excluded_emails=None, is_add_user=None, self.fields['program'].choices = choices self.excluded_emails = excluded_emails or [] - if self.request.couch_user.has_permission(self.domain, 'edit_user_tableau_config'): + if self.can_edit_tableau_config: self._initialize_tableau_fields(data, domain) self.helper = FormHelper() @@ -574,7 +575,7 @@ def clean(self): cleaned_data = super(AdminInvitesUserForm, self).clean() if (('tableau_role' in cleaned_data or 'tableau_group_indices' in cleaned_data) - and not self.request.couch_user.has_permission(self.domain, 'edit_user_tableau_config')): + and self.can_edit_tableau_config): raise forms.ValidationError(_("You do not have permission to edit Tableau Configuraion.")) if 'tableau_group_indices' in cleaned_data: diff --git a/corehq/apps/users/views/__init__.py b/corehq/apps/users/views/__init__.py index bde158ad5c83..a24a7e73fcf7 100644 --- a/corehq/apps/users/views/__init__.py +++ b/corehq/apps/users/views/__init__.py @@ -1107,6 +1107,8 @@ def invite_web_user_form(self): initial = { 'email': domain_request.email if domain_request else None, } + can_edit_tableau_config = (self.request.couch_user.has_permission(self.domain, 'edit_user_tableau_config') + and toggles.TABLEAU_USER_SYNCING.enabled(self.domain)) if self.request.method == 'POST': current_users = [user.username for user in WebUser.by_domain(self.domain)] pending_invites = [di.email for di in Invitation.by_domain(self.domain)] @@ -1117,6 +1119,7 @@ def invite_web_user_form(self): domain=self.domain, is_add_user=is_add_user, should_show_location=self.request.project.uses_locations, + can_edit_tableau_config=can_edit_tableau_config, request=self.request ) return AdminInvitesUserForm( @@ -1125,6 +1128,7 @@ def invite_web_user_form(self): domain=self.domain, is_add_user=is_add_user, should_show_location=self.request.project.uses_locations, + can_edit_tableau_config=can_edit_tableau_config, request=self.request ) From 33cee1c5b17c9cd21065922b3a83f60afcbe0a37 Mon Sep 17 00:00:00 2001 From: Jonathan Tang Date: Thu, 23 May 2024 11:58:53 -0700 Subject: [PATCH 13/63] add header labels for fields --- corehq/apps/registration/forms.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/corehq/apps/registration/forms.py b/corehq/apps/registration/forms.py index 5b0a6bc1d5f4..de68b11b57fc 100644 --- a/corehq/apps/registration/forms.py +++ b/corehq/apps/registration/forms.py @@ -524,7 +524,7 @@ def __init__(self, data=None, excluded_emails=None, is_add_user=None, self.helper.label_class = 'col-sm-3 col-md-2' self.helper.field_class = 'col-sm-9 col-md-8 col-lg-6' - self.helper.layout = crispy.Layout( + fields = [ crispy.Fieldset( gettext("Information for new Web User"), crispy.Field( @@ -534,11 +534,26 @@ def __init__(self, data=None, excluded_emails=None, is_add_user=None, ), 'role', 'profile' if ('profile' in self.fields and len(self.fields['profile'].choices) > 0) else None, - 'assigned_locations' if should_show_location else None, - 'primary_location' if should_show_location else None, - 'tableau_role' if 'tableau_role' in self.fields else None, - 'tableau_group_indices' if 'tableau_group_indices' in self.fields else None - ), + ) + ] + if should_show_location: + fields.append( + crispy.Fieldset( + gettext("Location Settings"), + 'assigned_locations' if should_show_location else None, + 'primary_location' if should_show_location else None, + ) + ) + if self.can_edit_tableau_config: + fields.append( + crispy.Fieldset( + gettext("Tableau Configuration"), + 'tableau_role' if 'tableau_role' in self.fields else None, + 'tableau_group_indices' if 'tableau_group_indices' in self.fields else None + ), + ) + self.helper.layout = crispy.Layout( + *fields, crispy.HTML( render_to_string( 'users/partials/confirm_trust_identity_provider_message.html', From 1ee0e50b52c7952b3beca5fed1336bbc1ac02217 Mon Sep 17 00:00:00 2001 From: Jonathan Tang Date: Thu, 23 May 2024 12:01:56 -0700 Subject: [PATCH 14/63] control visibility of views via fieldset --- corehq/apps/registration/forms.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/corehq/apps/registration/forms.py b/corehq/apps/registration/forms.py index de68b11b57fc..ca37472b4ea6 100644 --- a/corehq/apps/registration/forms.py +++ b/corehq/apps/registration/forms.py @@ -540,16 +540,16 @@ def __init__(self, data=None, excluded_emails=None, is_add_user=None, fields.append( crispy.Fieldset( gettext("Location Settings"), - 'assigned_locations' if should_show_location else None, - 'primary_location' if should_show_location else None, + 'assigned_locations', + 'primary_location', ) ) if self.can_edit_tableau_config: fields.append( crispy.Fieldset( gettext("Tableau Configuration"), - 'tableau_role' if 'tableau_role' in self.fields else None, - 'tableau_group_indices' if 'tableau_group_indices' in self.fields else None + 'tableau_role', + 'tableau_group_indices' if len(self.fields['tableau_group_indices'].choices) > 0 else None ), ) self.helper.layout = crispy.Layout( From 109725fb2e9a4a099685e5b4a63ab38e3d342846 Mon Sep 17 00:00:00 2001 From: Ethan Soergel Date: Thu, 23 May 2024 15:53:17 -0400 Subject: [PATCH 15/63] Remove links to internal-only wiki --- docs/case_search_query_language.rst | 35 ++++++----------------------- 1 file changed, 7 insertions(+), 28 deletions(-) diff --git a/docs/case_search_query_language.rst b/docs/case_search_query_language.rst index 0e6386dd2507..e9d66f4d8c04 100644 --- a/docs/case_search_query_language.rst +++ b/docs/case_search_query_language.rst @@ -2,12 +2,9 @@ Case Search Query Language (CSQL) ================================= -Underpinning some of the advanced search capabilities of `Case Search`_ and `Case List Explorer`_ is -the Case Search Query Language. This page describes the syntax and capabilities of the language as -well as it's limitations. - -.. _Case Search: https://dimagi.atlassian.net/wiki/spaces/GS/pages/2146606528 -.. _Case List Explorer: https://dimagi.atlassian.net/wiki/x/KTXKfw +Underpinning some of the advanced search capabilities of Case Search and Case List Explorer is the +Case Search Query Language. This page describes the syntax and capabilities of the language as well +as it's limitations. .. contents:: :local: @@ -159,27 +156,21 @@ The following functions are supported: ``within-distance`` ------------------- -* **Requirements**: GPS case properties set up as described in this page: `Storing GPS Case - Properties in Elasticsearch as GeoPoints `_ +* **Requirements**: GPS case properties * **Behavior**: Match cases within a certain geographic distance (as the crow flies) of the provided point * **Return**: True if that case is within range, otherwise false * **Arguments**: * ``property_name``: The GPS case property on the cases being searched - * ``coordinates``: This can be the output of a "geopoint" receiver from a geocoder question as - described in `Address Geocoding in Web Apps `_ + * ``coordinates``: This can be the output of a "geopoint" receiver from a geocoder question * ``distance``: The distance from ``coordinates`` to search * ``unit``: The units for that distance. Options are: miles, yards, feet, inch, kilometers, meters, centimeters, millimeters, nauticalmiles * **Usage**: ``within-distance(location, '42.4402967 -71.1453275', 30, 'miles')`` -.. _geopoints: https://dimagi.atlassian.net/wiki/x/dgf4fw -.. _address_geocoding: https://dimagi.atlassian.net/wiki/x/dALKfw - ``fuzzy-match`` --------------- -* **Behavior**: Determines if a given value is a fuzzy match for a given case property. This ignores - the `domain-level case search fuzziness settings `_. +* **Behavior**: Determines if a given value is a fuzzy match for a given case property. * **Return**: True if that particular value matches the case property. Otherwise False. * **Arguments**: Two arguments: the case property and the value to check. * **Usage**: ``fuzzy-match(first_name, "Sara")`` @@ -189,7 +180,6 @@ The following functions are supported: to gauge similarity. To consider something a match, it requires an exact prefix match and an edit distance based on the length of the string (longer strings can have more edits). -.. _fuzziness_settings: https://dimagi.atlassian.net/wiki/spaces/GS/pages/2146606704/Project+Case+Search+Configuration#Configure-Fuzzy-Properties-(optional) .. _Fuzzy Query: https://www.elastic.co/guide/en/elasticsearch/reference/8.11/query-dsl-fuzzy-query.html .. _Levenshtein distance: https://en.wikipedia.org/wiki/Levenshtein_distance @@ -288,10 +278,6 @@ Special functions are provided to support filtering based on the properties of s subcase functions and search results increases, the time required to perform the search will also increase. - **Performance testing is required** when using this function in your app. - * `Link to performance testing in HQ's case search (not Web Apps) `_ - * `Link to performance testing in HQ's case search (WebApps - BHA) `_ - Keep in mind that a higher number of search results will lead to longer execution times for the search query. The threshold is around 400K to 500K search results, after which a timeout error may occur. It is recommended to keep your search results well below this number for optimal @@ -302,16 +288,9 @@ Special functions are provided to support filtering based on the properties of s by both first and last name is more effective than just using the first name. Including more required fields in the search form is likely to reduce the number of search results returned. - If you have any questions regarding the limitation usage of this subcase function, please reach - out to the AE team. - - -.. _perf_testing: https://docs.google.com/spreadsheets/d/1T4tX1tbFaiTBFWpJoi_4p4_UtnRL3_SRoXJgADraXo8/edit#gid=1095465676 -.. _bha_perf_testing: https://docs.google.com/spreadsheets/d/1B9ySwSahf4qUKWEmX0FfdK0uArOBnJ6pXWh7eYvnaiA/edit#gid=671814666 - **Examples** -A very common implementation of subcase-exists search queries involves utilizing the user's +A very common implementation of ``subcase-exists`` search queries involves utilizing the user's 'search-input'. Please see an example of this configuration below. .. code-block:: From c98633e223c61b4eaf5f4767ebe091e4c97e2912 Mon Sep 17 00:00:00 2001 From: Jonathan Tang Date: Thu, 23 May 2024 14:41:46 -0700 Subject: [PATCH 16/63] bug: should check if user does not have permission to edit tableau config --- corehq/apps/registration/forms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corehq/apps/registration/forms.py b/corehq/apps/registration/forms.py index ca37472b4ea6..7ddbe3ed0ba6 100644 --- a/corehq/apps/registration/forms.py +++ b/corehq/apps/registration/forms.py @@ -590,7 +590,7 @@ def clean(self): cleaned_data = super(AdminInvitesUserForm, self).clean() if (('tableau_role' in cleaned_data or 'tableau_group_indices' in cleaned_data) - and self.can_edit_tableau_config): + and not self.can_edit_tableau_config): raise forms.ValidationError(_("You do not have permission to edit Tableau Configuraion.")) if 'tableau_group_indices' in cleaned_data: From 09c2d32dedb9592c14b755e661caebf9a862d234 Mon Sep 17 00:00:00 2001 From: Ethan Soergel Date: Thu, 23 May 2024 21:21:14 -0400 Subject: [PATCH 17/63] Merge related case query sections and move warning to top --- docs/case_search_query_language.rst | 55 +++++++++++------------------ 1 file changed, 21 insertions(+), 34 deletions(-) diff --git a/docs/case_search_query_language.rst b/docs/case_search_query_language.rst index e9d66f4d8c04..0c0b51fb7363 100644 --- a/docs/case_search_query_language.rst +++ b/docs/case_search_query_language.rst @@ -210,8 +210,27 @@ The following functions are supported: * Matches cases that have a property ``first_name`` equal to ``"Julia"`` -Filtering on parent (ancestor) cases -==================================== +Filtering on related cases +========================== + +CSQL includes utilities for searching against ancestor cases (such as parents) and subcases (such as children) + +.. warning:: + When utilizing related cases function, be mindful that the *quantity of search results* and the + *number of subcase or ancestor functions* in a single search are important factors. As the + number of related case functions and search results increases, the time required to perform the + search will also increase. + + Keep in mind that a higher number of search results will lead to longer execution times for the + search query. The threshold is around 400K to 500K search results, after which a timeout error + may occur. It is recommended to keep your search results well below this number for optimal + performance. + + To manage the number of search results when incorporating subcase or ancestor functions in your + search query, you can apply required fields in the search form. For instance, requiring users to + search by both first and last name is more effective than just using the first name. Including + more required fields in the search form is likely to reduce the number of search results + returned. Searches may be performed against ancestor cases (e.g. parent cases) using the ``/`` operator @@ -238,16 +257,6 @@ Searches may be performed against ancestor cases (e.g. parent cases) using the ` * This will *not* work: ``ancestor-exists(parent, selected(city, 'SF'))`` * This will work: ``ancestor-exists(parent, city != '' and selected(city, 'SF'))`` * The ancestor filter expression may not include ``subcase-exists`` or ``subcase-count`` - * **Best Practices**: - * Limit multiple uses of this function in your query to avoid performance implications - * Add as many arguments to ``ancestor-exists()`` as possible to help narrow down results (i.e., - by ``@case_type`` or ``@status``) - - -Filtering on child cases (subcases) -=================================== - -Special functions are provided to support filtering based on the properties of subcases. These are: ``subcase-exists`` ------------------ @@ -256,10 +265,6 @@ Special functions are provided to support filtering based on the properties of s * **Arguments**: Two arguments, the subcase relationship (usually one of 'parent' or 'host') and the subcase filter expression. * **Usage**: ``subcase-exists('parent', lab_type = 'blood' and result = 1)`` -* **Best Practices**: - * Limit multiple uses of this function in your query to avoid performance implications - * Add as many arguments to ``subcase-exists()`` as possible to help narrow down results (i.e., - by ``@case_type`` or ``@status``) ``subcase-count`` ----------------- @@ -269,24 +274,6 @@ Special functions are provided to support filtering based on the properties of s * **Usage**: ``subcase-count('parent', lab_type = 'blood' and result = 1) > 3`` * The count function must be used in conjunction with a comparison operator. All operators are supported (``=``, ``!=``, ``<``, ``<=``, ``>``, ``>=``) -* **Best Practices**: - * Limit multiple uses of this function in your query to avoid performance implications - -.. warning:: - When utilizing the special subcase function, be mindful that the *quantity of search results* - and the *number of subcase functions* in a single search are important factors. As the number of - subcase functions and search results increases, the time required to perform the search will - also increase. - - Keep in mind that a higher number of search results will lead to longer execution times for the - search query. The threshold is around 400K to 500K search results, after which a timeout error - may occur. It is recommended to keep your search results well below this number for optimal - performance. - - To manage the number of search results when incorporating subcase functions in your search - query, you can apply required fields in the search form. For instance, requiring users to search - by both first and last name is more effective than just using the first name. Including more - required fields in the search form is likely to reduce the number of search results returned. **Examples** From 8fade9fbf8aa8afa5c29c713c307b327f35a1a70 Mon Sep 17 00:00:00 2001 From: Simon Kelly Date: Tue, 28 May 2024 16:14:25 +0200 Subject: [PATCH 18/63] add status chart --- corehq/apps/app_execution/db_accessors.py | 46 +++++++++++++++- ...low_timing_chart.js => workflow_charts.js} | 55 +++++++++++++++++-- .../static/app_execution/js/workflow_logs.js | 2 +- .../app_execution/workflow_list.html | 19 +++++-- .../app_execution/workflow_log_list.html | 17 ++++-- corehq/apps/app_execution/views.py | 14 ++--- 6 files changed, 127 insertions(+), 26 deletions(-) rename corehq/apps/app_execution/static/app_execution/js/{workflow_timing_chart.js => workflow_charts.js} (54%) diff --git a/corehq/apps/app_execution/db_accessors.py b/corehq/apps/app_execution/db_accessors.py index de9ea05a9a5b..7099d8692b40 100644 --- a/corehq/apps/app_execution/db_accessors.py +++ b/corehq/apps/app_execution/db_accessors.py @@ -1,4 +1,6 @@ -from django.db.models import Avg, DateTimeField, DurationField, ExpressionWrapper, F, Max +from datetime import timedelta + +from django.db.models import Avg, Count, DateTimeField, DurationField, ExpressionWrapper, F, Max from django.db.models.functions import Trunc from corehq.apps.app_execution.models import AppExecutionLog, AppWorkflowConfig @@ -41,3 +43,45 @@ def get_avg_duration_data(domain, start, end, workflow_id=None): for workflow_data in data: workflow_data["label"] = workflow_names[workflow_data["key"]] return data + + +def get_status_data(domain, start, end, workflow_id=None): + query = AppExecutionLog.objects.filter(workflow__domain=domain, started__gte=start, started__lt=end) + if workflow_id: + query = query.filter(workflow_id=workflow_id) + + chart_logs = ( + query.annotate(date=Trunc("started", "hour", output_field=DateTimeField())) + .values("date", "success") + .annotate(count=Count("success")) + ) + + success = [] + error = [] + seen_success_dates = set() + seen_error_dates = set() + for row in chart_logs: + item = { + "date": row["date"].isoformat(), + "count": row["count"], + } + if row["success"]: + success.append(item) + seen_success_dates.add(row["date"]) + else: + error.append(item) + seen_error_dates.add(row["date"]) + + start = start.replace(minute=0, second=0, microsecond=0) + current = start + while current < end: + if current not in seen_error_dates: + error.append({"date": current.isoformat(), "count": 0}) + if current not in seen_success_dates: + success.append({"date": current.isoformat(), "count": 0}) + current += timedelta(hours=1) + + return [ + {"key": "Success", "values": sorted(success, key=lambda x: x["date"])}, + {"key": "Error", "values": sorted(error, key=lambda x: x["date"])}, + ] diff --git a/corehq/apps/app_execution/static/app_execution/js/workflow_timing_chart.js b/corehq/apps/app_execution/static/app_execution/js/workflow_charts.js similarity index 54% rename from corehq/apps/app_execution/static/app_execution/js/workflow_timing_chart.js rename to corehq/apps/app_execution/static/app_execution/js/workflow_charts.js index 63fa7b9e7f91..0348fee4062d 100644 --- a/corehq/apps/app_execution/static/app_execution/js/workflow_timing_chart.js +++ b/corehq/apps/app_execution/static/app_execution/js/workflow_charts.js @@ -1,5 +1,5 @@ // /* globals moment */ -hqDefine("app_execution/js/workflow_timing_chart", [ +hqDefine("app_execution/js/workflow_charts", [ 'jquery', 'moment/moment', 'd3/d3.min', @@ -12,7 +12,6 @@ hqDefine("app_execution/js/workflow_timing_chart", [ return includeSeries.map((seriesMeta) => { return { // include key in the label to differentiate between series with the same label - label: `${data.label}${seriesMeta.label}`, key: `${data.label}${seriesMeta.label}[${data.key}]`, values: data.values.map((item) => { return { @@ -24,7 +23,7 @@ hqDefine("app_execution/js/workflow_timing_chart", [ }); } - function setupLineChart(data, includeSeries) { + function setupTimingChart(data, includeSeries) { const timingSeries = data.flatMap((series) => getSeries(series, includeSeries)); nv.addGraph(function () { let chart = nv.models.lineChart() @@ -58,9 +57,53 @@ hqDefine("app_execution/js/workflow_timing_chart", [ } + function setupStatusChart(data) { + const colors = { + "Success": "#6dcc66", + "Error": "#f44", + }; + data = data.map((series) => { + return { + key: series.key, + values: series.values.map((item) => { + return { + x: moment(item.date), + y: item.count, + }; + }), + color: colors[series.key], + }; + }); + + nv.addGraph(function () { + let chart = nv.models.lineChart() + .showYAxis(true) + .showXAxis(true); + + chart.yAxis + .axisLabel('Count'); + chart.forceY(0); + chart.xScale(d3.time.scale()); + chart.margin({left: 80, bottom: 100}); + chart.xAxis.rotateLabels(-45) + .tickFormat(function (d) { + return moment(d).format("MMM DD [@] HH"); + }); + + d3.select('#status_barchart svg') + .datum(data) + .call(chart); + + nv.utils.windowResize(chart.update); + return chart; + }); + } + $(document).ready(function () { - const chartData = JSON.parse(document.getElementById('chart_data').textContent); - const includeSeries = JSON.parse(document.getElementById('includeSeries').textContent); - setupLineChart(chartData, includeSeries); + const timingData = JSON.parse(document.getElementById('timing_chart_data').textContent); + const statusData = JSON.parse(document.getElementById('status_chart_data').textContent); + const includeSeries = JSON.parse(document.getElementById('timingSeries').textContent); + setupTimingChart(timingData, includeSeries); + setupStatusChart(statusData); }); }); diff --git a/corehq/apps/app_execution/static/app_execution/js/workflow_logs.js b/corehq/apps/app_execution/static/app_execution/js/workflow_logs.js index 0bcca816ac71..0e6cb6dcf28b 100644 --- a/corehq/apps/app_execution/static/app_execution/js/workflow_logs.js +++ b/corehq/apps/app_execution/static/app_execution/js/workflow_logs.js @@ -2,7 +2,7 @@ hqDefine("app_execution/js/workflow_logs", [ 'jquery', 'knockout', 'hqwebapp/js/initial_page_data', - 'app_execution/js/workflow_timing_chart', + 'app_execution/js/workflow_charts', 'hqwebapp/js/bootstrap5/components.ko', ], function ($, ko, initialPageData) { let logsModel = function () { diff --git a/corehq/apps/app_execution/templates/app_execution/workflow_list.html b/corehq/apps/app_execution/templates/app_execution/workflow_list.html index 014329d47f9f..64fea39c4475 100644 --- a/corehq/apps/app_execution/templates/app_execution/workflow_list.html +++ b/corehq/apps/app_execution/templates/app_execution/workflow_list.html @@ -12,15 +12,21 @@ {% endcompress %} {% endblock %} -{% requirejs_main_b5 'app_execution/js/workflow_timing_chart' %} +{% requirejs_main_b5 'app_execution/js/workflow_charts' %} {% block page_content %} -
-

{% trans "Average Timings Per Hour" %}

-
+
+
+

{% trans "Average Timings" %}

+ +
+
+

{% trans "Status" %}

+ +
@@ -65,8 +71,9 @@

{% trans "Average Timings Per Hour" %}

-{{ chart_data|json_script:"chart_data" }} - {% endblock %} diff --git a/corehq/apps/app_execution/templates/app_execution/workflow_log_list.html b/corehq/apps/app_execution/templates/app_execution/workflow_log_list.html index 7ed76f988007..693d10041122 100644 --- a/corehq/apps/app_execution/templates/app_execution/workflow_log_list.html +++ b/corehq/apps/app_execution/templates/app_execution/workflow_log_list.html @@ -19,9 +19,15 @@ {% registerurl "app_execution:logs_json" request.domain workflow.id %}

Logs for workflow "{{ workflow.name }}"

-
-

{% trans "Average Timings Per Hour" %}

-
+
+
+

{% trans "Average Timings" %}

+ +
+
+

{% trans "Log Status" %}

+ +
@@ -50,8 +56,9 @@

{% trans "Average Timings Per Hour" %}

-{{ chart_data|json_script:"chart_data" }} - {% endblock %} diff --git a/corehq/apps/app_execution/views.py b/corehq/apps/app_execution/views.py index 5d3f424d6f3b..b13a6dc49dfb 100644 --- a/corehq/apps/app_execution/views.py +++ b/corehq/apps/app_execution/views.py @@ -9,7 +9,7 @@ from corehq.apps.app_execution import const from corehq.apps.app_execution.api import execute_workflow from corehq.apps.app_execution.data_model import EXAMPLE_WORKFLOW -from corehq.apps.app_execution.db_accessors import get_avg_duration_data +from corehq.apps.app_execution.db_accessors import get_avg_duration_data, get_status_data from corehq.apps.app_execution.exceptions import AppExecutionError, FormplayerException from corehq.apps.app_execution.forms import AppWorkflowConfigForm from corehq.apps.app_execution.har_parser import parse_har_from_string @@ -24,13 +24,14 @@ def workflow_list(request, domain): workflows = AppWorkflowConfig.objects.filter(domain=domain) _augment_with_logs(workflows) utcnow = datetime.utcnow() - chart_data = get_avg_duration_data(domain, start=utcnow - relativedelta(months=1), end=utcnow) + start = utcnow - relativedelta(months=1) context = _get_context( request, "Automatically Executed App Workflows", reverse("app_execution:workflow_list", args=[domain]), workflows=workflows, - chart_data=chart_data + timing_chart_data=get_avg_duration_data(domain, start=start, end=utcnow), + status_chart_data=get_status_data(domain, start=start, end=utcnow) ) return render(request, "app_execution/workflow_list.html", context) @@ -165,9 +166,7 @@ def _get_context(request, title, url, add_parent=False, **kwargs): @use_bootstrap5 def workflow_log_list(request, domain, pk): utcnow = datetime.utcnow() - chart_data = get_avg_duration_data( - domain, start=utcnow - relativedelta(months=1), end=utcnow, workflow_id=pk - ) + start = utcnow - relativedelta(months=1) context = _get_context( request, "Automatically Executed App Workflow Logs", @@ -175,7 +174,8 @@ def workflow_log_list(request, domain, pk): add_parent=True, workflow=AppWorkflowConfig.objects.get(id=pk), total=AppExecutionLog.objects.filter(workflow__domain=domain, workflow_id=pk).count(), - chart_data=chart_data + timing_chart_data=get_avg_duration_data(domain, start=start, end=utcnow, workflow_id=pk), + status_chart_data=get_status_data(domain, start=start, end=utcnow, workflow_id=pk), ) return render(request, "app_execution/workflow_log_list.html", context) From 3cd4e781006c82c4102a08f3adb0f81bd4175211 Mon Sep 17 00:00:00 2001 From: Simon Kelly Date: Tue, 28 May 2024 16:40:07 +0200 Subject: [PATCH 19/63] pad timing data --- corehq/apps/app_execution/db_accessors.py | 36 +++++++++++-------- .../app_execution/js/workflow_charts.js | 13 ++++--- .../app_execution/workflow_list.html | 3 +- .../app_execution/workflow_log_list.html | 3 +- corehq/apps/app_execution/views.py | 12 ++++--- 5 files changed, 38 insertions(+), 29 deletions(-) diff --git a/corehq/apps/app_execution/db_accessors.py b/corehq/apps/app_execution/db_accessors.py index 7099d8692b40..b23e281868ee 100644 --- a/corehq/apps/app_execution/db_accessors.py +++ b/corehq/apps/app_execution/db_accessors.py @@ -1,3 +1,4 @@ +from collections import defaultdict from datetime import timedelta from django.db.models import Avg, Count, DateTimeField, DurationField, ExpressionWrapper, F, Max @@ -18,31 +19,38 @@ def get_avg_duration_data(domain, start, end, workflow_id=None): ).values("date", "workflow_id") .annotate(avg_duration=Avg('duration')) .annotate(max_duration=Max('duration')) - .order_by("workflow_id", "date") ) - data = [] - seen_workflows = set() + data = defaultdict(list) + seen_dates = defaultdict(set) for row in chart_logs: - if row["workflow_id"] not in seen_workflows: - seen_workflows.add(row["workflow_id"]) - data.append({ - "key": row["workflow_id"], - "values": [] - }) - data[-1]["values"].append({ + data[row["workflow_id"]].append({ "date": row["date"].isoformat(), "avg_duration": row["avg_duration"].total_seconds(), "max_duration": row["max_duration"].total_seconds(), }) + seen_dates[row["workflow_id"]].add(row["date"]) + + start = start.replace(minute=0, second=0, microsecond=0) + current = start + while current < end: + for workflow_id, dates in seen_dates.items(): + if current not in dates: + data[workflow_id].append({"date": current.isoformat(), "avg_duration": None, "max_duration": None}) + current += timedelta(hours=1) workflow_names = { workflow["id"]: workflow["name"] - for workflow in AppWorkflowConfig.objects.filter(id__in=seen_workflows).values("id", "name") + for workflow in AppWorkflowConfig.objects.filter(id__in=list(data)).values("id", "name") } - for workflow_data in data: - workflow_data["label"] = workflow_names[workflow_data["key"]] - return data + return [ + { + "key": workflow_id, + "label": workflow_names[workflow_id], + "values": sorted(data, key=lambda x: x["date"]) + } + for workflow_id, data in data.items() + ] def get_status_data(domain, start, end, workflow_id=None): diff --git a/corehq/apps/app_execution/static/app_execution/js/workflow_charts.js b/corehq/apps/app_execution/static/app_execution/js/workflow_charts.js index 0348fee4062d..54fd33b2b3eb 100644 --- a/corehq/apps/app_execution/static/app_execution/js/workflow_charts.js +++ b/corehq/apps/app_execution/static/app_execution/js/workflow_charts.js @@ -24,7 +24,7 @@ hqDefine("app_execution/js/workflow_charts", [ } function setupTimingChart(data, includeSeries) { - const timingSeries = data.flatMap((series) => getSeries(series, includeSeries)); + const timingSeries = data.timing.flatMap((series) => getSeries(series, includeSeries)); nv.addGraph(function () { let chart = nv.models.lineChart() .showYAxis(true) @@ -62,7 +62,7 @@ hqDefine("app_execution/js/workflow_charts", [ "Success": "#6dcc66", "Error": "#f44", }; - data = data.map((series) => { + let seriesData = data.status.map((series) => { return { key: series.key, values: series.values.map((item) => { @@ -91,7 +91,7 @@ hqDefine("app_execution/js/workflow_charts", [ }); d3.select('#status_barchart svg') - .datum(data) + .datum(seriesData) .call(chart); nv.utils.windowResize(chart.update); @@ -100,10 +100,9 @@ hqDefine("app_execution/js/workflow_charts", [ } $(document).ready(function () { - const timingData = JSON.parse(document.getElementById('timing_chart_data').textContent); - const statusData = JSON.parse(document.getElementById('status_chart_data').textContent); + const data = JSON.parse(document.getElementById('chart_data').textContent); const includeSeries = JSON.parse(document.getElementById('timingSeries').textContent); - setupTimingChart(timingData, includeSeries); - setupStatusChart(statusData); + setupTimingChart(data, includeSeries); + setupStatusChart(data); }); }); diff --git a/corehq/apps/app_execution/templates/app_execution/workflow_list.html b/corehq/apps/app_execution/templates/app_execution/workflow_list.html index 64fea39c4475..52585f63f37c 100644 --- a/corehq/apps/app_execution/templates/app_execution/workflow_list.html +++ b/corehq/apps/app_execution/templates/app_execution/workflow_list.html @@ -71,8 +71,7 @@

{% trans "Status" %}

-{{ timing_chart_data|json_script:"timing_chart_data" }} -{{ status_chart_data|json_script:"status_chart_data" }} +{{ chart_data|json_script:"chart_data" }} diff --git a/corehq/apps/app_execution/templates/app_execution/workflow_log_list.html b/corehq/apps/app_execution/templates/app_execution/workflow_log_list.html index 693d10041122..40888a38bdc0 100644 --- a/corehq/apps/app_execution/templates/app_execution/workflow_log_list.html +++ b/corehq/apps/app_execution/templates/app_execution/workflow_log_list.html @@ -56,8 +56,7 @@

{% trans "Log Status" %}

-{{ timing_chart_data|json_script:"timing_chart_data" }} -{{ status_chart_data|json_script:"status_chart_data" }} +{{ chart_data|json_script:"chart_data" }} diff --git a/corehq/apps/app_execution/views.py b/corehq/apps/app_execution/views.py index b13a6dc49dfb..73f42877683a 100644 --- a/corehq/apps/app_execution/views.py +++ b/corehq/apps/app_execution/views.py @@ -30,8 +30,10 @@ def workflow_list(request, domain): "Automatically Executed App Workflows", reverse("app_execution:workflow_list", args=[domain]), workflows=workflows, - timing_chart_data=get_avg_duration_data(domain, start=start, end=utcnow), - status_chart_data=get_status_data(domain, start=start, end=utcnow) + chart_data={ + "timing": get_avg_duration_data(domain, start=start, end=utcnow), + "status": get_status_data(domain, start=start, end=utcnow), + } ) return render(request, "app_execution/workflow_list.html", context) @@ -174,8 +176,10 @@ def workflow_log_list(request, domain, pk): add_parent=True, workflow=AppWorkflowConfig.objects.get(id=pk), total=AppExecutionLog.objects.filter(workflow__domain=domain, workflow_id=pk).count(), - timing_chart_data=get_avg_duration_data(domain, start=start, end=utcnow, workflow_id=pk), - status_chart_data=get_status_data(domain, start=start, end=utcnow, workflow_id=pk), + chart_data={ + "timing": get_avg_duration_data(domain, start=start, end=utcnow, workflow_id=pk), + "status": get_status_data(domain, start=start, end=utcnow, workflow_id=pk), + } ) return render(request, "app_execution/workflow_log_list.html", context) From 746883239be66aac5661980194d2af6cc7b5d8cd Mon Sep 17 00:00:00 2001 From: Simon Kelly Date: Tue, 28 May 2024 17:31:55 +0200 Subject: [PATCH 20/63] add filters to log view --- .../static/app_execution/js/workflow_logs.js | 24 +++++++++++++++++- .../app_execution/workflow_log_list.html | 17 +++++++++++++ corehq/apps/app_execution/views.py | 25 ++++++++++++++++--- corehq/apps/registry/views.py | 18 +++---------- corehq/util/view_utils.py | 12 +++++++++ 5 files changed, 77 insertions(+), 19 deletions(-) diff --git a/corehq/apps/app_execution/static/app_execution/js/workflow_logs.js b/corehq/apps/app_execution/static/app_execution/js/workflow_logs.js index 0e6cb6dcf28b..8488c8c5dc51 100644 --- a/corehq/apps/app_execution/static/app_execution/js/workflow_logs.js +++ b/corehq/apps/app_execution/static/app_execution/js/workflow_logs.js @@ -2,27 +2,49 @@ hqDefine("app_execution/js/workflow_logs", [ 'jquery', 'knockout', 'hqwebapp/js/initial_page_data', + 'hqwebapp/js/tempus_dominus',, 'app_execution/js/workflow_charts', 'hqwebapp/js/bootstrap5/components.ko', -], function ($, ko, initialPageData) { +], function ($, ko, initialPageData, hqTempusDominus) { let logsModel = function () { let self = {}; + self.statusFilter = ko.observable(""); + let allDatesText = gettext("Show All Dates"); + self.dateRange = ko.observable(allDatesText); self.items = ko.observableArray(); self.totalItems = ko.observable(initialPageData.get('total_items')); self.perPage = ko.observable(25); self.goToPage = function (page) { let params = {page: page, per_page: self.perPage()}; const url = initialPageData.reverse('app_execution:logs_json'); + if (self.statusFilter()) { + params.status = self.statusFilter(); + } + if (self.dateRange() && self.dateRange() !== allDatesText) { + const separator = hqTempusDominus.getDateRangeSeparator(), + dates = self.dateRange().split(separator); + params.startDate = dates[0]; + params.endDate = dates[1] || dates[0]; + } $.getJSON(url, params, function (data) { self.items(data.logs); }); }; + self.filter = ko.computed(() => { + self.statusFilter(); + if (self.dateRange().includes(hqTempusDominus.getDateRangeSeparator())) { + self.goToPage(1); + } + }).extend({throttle: 500}); + self.onLoad = function () { self.goToPage(1); }; + hqTempusDominus.createDefaultDateRangePicker(document.getElementById('id_date_range')); + return self; }; diff --git a/corehq/apps/app_execution/templates/app_execution/workflow_log_list.html b/corehq/apps/app_execution/templates/app_execution/workflow_log_list.html index 40888a38bdc0..6552e6ec4bfc 100644 --- a/corehq/apps/app_execution/templates/app_execution/workflow_log_list.html +++ b/corehq/apps/app_execution/templates/app_execution/workflow_log_list.html @@ -9,6 +9,10 @@ rel="stylesheet" media="all" href="{% static 'nvd3-1.8.6/build/nv.d3.css' %}" /> + {% endcompress %} {% endblock %} @@ -30,6 +34,19 @@

{% trans "Log Status" %}

+
+ +
+ +
+
+ +
+
diff --git a/corehq/apps/app_execution/views.py b/corehq/apps/app_execution/views.py index 73f42877683a..b3b33fa87d16 100644 --- a/corehq/apps/app_execution/views.py +++ b/corehq/apps/app_execution/views.py @@ -16,6 +16,8 @@ from corehq.apps.app_execution.models import AppExecutionLog, AppWorkflowConfig from corehq.apps.domain.decorators import require_superuser_or_contractor from corehq.apps.hqwebapp.decorators import use_bootstrap5 +from corehq.util.timezones.utils import get_timezone_for_user +from corehq.util.view_utils import get_date_param @require_superuser_or_contractor @@ -179,7 +181,7 @@ def workflow_log_list(request, domain, pk): chart_data={ "timing": get_avg_duration_data(domain, start=start, end=utcnow, workflow_id=pk), "status": get_status_data(domain, start=start, end=utcnow, workflow_id=pk), - } + }, ) return render(request, "app_execution/workflow_log_list.html", context) @@ -187,12 +189,27 @@ def workflow_log_list(request, domain, pk): @require_superuser_or_contractor @use_bootstrap5 def workflow_logs_json(request, domain, pk): + status = request.GET.get('status', None) limit = int(request.GET.get('per_page', 10)) page = int(request.GET.get('page', 1)) skip = limit * (page - 1) - logs = AppExecutionLog.objects.filter( - workflow__domain=domain, workflow_id=pk - ).order_by("-started")[skip:skip + limit] + + timezone = get_timezone_for_user(request.couch_user, domain) + try: + start_date = get_date_param(request, 'startDate', timezone=timezone) + end_date = get_date_param(request, 'endDate', timezone=timezone) + except ValueError: + return JsonResponse({"error": "Invalid date parameter"}) + + query = AppExecutionLog.objects.filter(workflow__domain=domain, workflow_id=pk) + if status: + query = query.filter(success=status == "success") + if start_date: + query = query.filter(started__gte=start_date) + if end_date: + query = query.filter(started__lte=datetime.combine(end_date, datetime.max.time())) + + logs = query.order_by("-started")[skip:skip + limit] return JsonResponse({ "logs": [ { diff --git a/corehq/apps/registry/views.py b/corehq/apps/registry/views.py index 42e9ae05849e..321a18c3750d 100644 --- a/corehq/apps/registry/views.py +++ b/corehq/apps/registry/views.py @@ -1,6 +1,5 @@ import json from collections import Counter -from datetime import datetime from django.contrib import messages from django.db.models import Q @@ -24,9 +23,9 @@ manage_all_registries_required, RegistryPermissionCheck, ) -from corehq.util.timezones.conversions import ServerTime, UserTime +from corehq.util.timezones.conversions import ServerTime from corehq.util.timezones.utils import get_timezone_for_user -from dimagi.utils.parsing import ISO_DATE_FORMAT +from corehq.util.view_utils import get_date_param @manage_some_registries_required @@ -393,8 +392,8 @@ def registry_audit_logs(request, domain, registry_slug): timezone = get_timezone_for_user(request.couch_user, domain) try: - start_date = _get_date_param(request, 'startDate', timezone=timezone) - end_date = _get_date_param(request, 'endDate', timezone=timezone) + start_date = get_date_param(request, 'startDate', timezone=timezone) + end_date = get_date_param(request, 'endDate', timezone=timezone) except ValueError: return JsonResponse({"error": "Invalid date parameter"}) @@ -411,12 +410,3 @@ def registry_audit_logs(request, domain, registry_slug): "total": helper.get_total(), "logs": logs }) - - -def _get_date_param(request, param_name, timezone=None): - param = request.GET.get(param_name) or None - if param: - value = datetime.strptime(param, ISO_DATE_FORMAT) - if timezone: - value = UserTime(value, tzinfo=timezone).server_time().done() - return value diff --git a/corehq/util/view_utils.py b/corehq/util/view_utils.py index eb4229ef906c..dbe42cee3bed 100644 --- a/corehq/util/view_utils.py +++ b/corehq/util/view_utils.py @@ -1,6 +1,7 @@ import json import logging import traceback +from datetime import datetime from functools import wraps from django import http @@ -10,7 +11,9 @@ from django.urls import reverse as _reverse from django.utils.http import urlencode +from corehq.util.timezones.conversions import UserTime from dimagi.utils.logging import notify_exception +from dimagi.utils.parsing import ISO_DATE_FORMAT from dimagi.utils.web import get_url_base from corehq.util import global_request @@ -193,3 +196,12 @@ def is_ajax(request): https://docs.djangoproject.com/en/4.0/releases/3.1/#id2 """ return request.headers.get('x-requested-with') == 'XMLHttpRequest' + + +def get_date_param(request, param_name, timezone=None): + param = request.GET.get(param_name) or None + if param: + value = datetime.strptime(param, ISO_DATE_FORMAT) + if timezone: + value = UserTime(value, tzinfo=timezone).server_time().done() + return value From d700ce117b8e08cc4214996af9d5000e2804576e Mon Sep 17 00:00:00 2001 From: Simon Kelly Date: Thu, 30 May 2024 11:58:20 +0200 Subject: [PATCH 21/63] extract function --- .../app_execution/js/workflow_charts.js | 56 ++++++++----------- 1 file changed, 24 insertions(+), 32 deletions(-) diff --git a/corehq/apps/app_execution/static/app_execution/js/workflow_charts.js b/corehq/apps/app_execution/static/app_execution/js/workflow_charts.js index 54fd33b2b3eb..a834cdfdbc49 100644 --- a/corehq/apps/app_execution/static/app_execution/js/workflow_charts.js +++ b/corehq/apps/app_execution/static/app_execution/js/workflow_charts.js @@ -23,24 +23,31 @@ hqDefine("app_execution/js/workflow_charts", [ }); } + function buildChart(yLabel) { + let chart = nv.models.lineChart() + .showYAxis(true) + .showXAxis(true); + + chart.yAxis + .axisLabel(yLabel); + chart.forceY(0); + chart.xScale(d3.time.scale()); + chart.margin({left: 80, bottom: 100}); + chart.xAxis.rotateLabels(-45) + .tickFormat(function (d) { + return moment(d).format("MMM DD [@] HH"); + }); + + nv.utils.windowResize(chart.update); + return chart; + } + function setupTimingChart(data, includeSeries) { const timingSeries = data.timing.flatMap((series) => getSeries(series, includeSeries)); - nv.addGraph(function () { - let chart = nv.models.lineChart() - .showYAxis(true) - .showXAxis(true); - - chart.yAxis - .axisLabel('Seconds') - .tickFormat(d3.format(".1f")); - chart.forceY(0); - chart.xScale(d3.time.scale()); - chart.margin({left: 80, bottom: 100}); - chart.xAxis.rotateLabels(-45) - .tickFormat(function (d) { - return moment(d).format("MMM DD [@] HH"); - }); + nv.addGraph(() => { + let chart = buildChart(gettext("Seconds")); + chart.yAxis.tickFormat(d3.format(".1f")); // remove the key from the label chart.legend.key((d) => d.key.split("[")[0]); chart.tooltip.keyFormatter((d) => { @@ -51,10 +58,8 @@ hqDefine("app_execution/js/workflow_charts", [ .datum(timingSeries) .call(chart); - nv.utils.windowResize(chart.update); return chart; }); - } function setupStatusChart(data) { @@ -75,26 +80,13 @@ hqDefine("app_execution/js/workflow_charts", [ }; }); - nv.addGraph(function () { - let chart = nv.models.lineChart() - .showYAxis(true) - .showXAxis(true); - - chart.yAxis - .axisLabel('Count'); - chart.forceY(0); - chart.xScale(d3.time.scale()); - chart.margin({left: 80, bottom: 100}); - chart.xAxis.rotateLabels(-45) - .tickFormat(function (d) { - return moment(d).format("MMM DD [@] HH"); - }); + nv.addGraph(() => { + let chart = buildChart(gettext("Chart")); d3.select('#status_barchart svg') .datum(seriesData) .call(chart); - nv.utils.windowResize(chart.update); return chart; }); } From c9962c2db6615b757cfdf718b04773cc7f5bd036 Mon Sep 17 00:00:00 2001 From: Simon Kelly Date: Thu, 30 May 2024 12:20:47 +0200 Subject: [PATCH 22/63] translations --- corehq/apps/app_execution/forms.py | 22 ++++++++------- corehq/apps/app_execution/models.py | 14 +++++----- .../app_execution/components/logs.html | 7 ++--- .../app_execution/workflow_list.html | 18 ++++++------- .../templates/app_execution/workflow_log.html | 18 +++++++------ .../app_execution/workflow_log_list.html | 27 ++++++++++--------- .../app_execution/workflow_test.html | 11 +++++--- corehq/apps/app_execution/views.py | 23 +++++++++------- 8 files changed, 79 insertions(+), 61 deletions(-) diff --git a/corehq/apps/app_execution/forms.py b/corehq/apps/app_execution/forms.py index 84bfc94258d2..4d5e7c491682 100644 --- a/corehq/apps/app_execution/forms.py +++ b/corehq/apps/app_execution/forms.py @@ -13,10 +13,10 @@ class AppWorkflowConfigForm(forms.ModelForm): - run_every = forms.IntegerField(min_value=1, required=False, label="Run Every (minutes)") - username = forms.CharField(max_length=255, label="Username", - help_text="Username of the user to run the workflow") - har_file = forms.FileField(label="HAR File", required=False) + run_every = forms.IntegerField(min_value=1, required=False, label=_("Run Every (minutes)")) + username = forms.CharField(max_length=255, label=_("Username"), + help_text=_("Username of the user to run the workflow")) + har_file = forms.FileField(label=_("HAR File"), required=False) class Meta: model = AppWorkflowConfig @@ -52,6 +52,7 @@ def __init__(self, request, *args, **kwargs): if request.user.is_superuser: fields += ["run_every", "notification_emails"] + har_help = _("HAR file recording should start with the selection of the app (navigate_menu_start).") self.helper.layout = crispy.Layout( crispy.Div( crispy.Div( @@ -59,23 +60,22 @@ def __init__(self, request, *args, **kwargs): css_class="col", ), crispy.Div( - crispy.HTML("

HAR file recording should start with the " - "selection of the app (navigate_menu_start).

"), + crispy.HTML(f"

{har_help}

"), "har_file", twbscrispy.StrictButton( - "Populate workflow from HAR file", + _("Populate workflow from HAR file"), type='submit', css_class='btn-secondary', name="import_har", value="1", formnovalidate=True, ), crispy.HTML("

 

"), - crispy.HTML("

Workflow:

"), + crispy.HTML(f"

{_('Workflow:')}

"), InlineField("workflow"), css_class="col" ), css_class="row mb-3" ), hqcrispy.FormActions( - twbscrispy.StrictButton("Save", type='submit', css_class='btn-primary') + twbscrispy.StrictButton(_("Save"), type='submit', css_class='btn-primary') ), ) @@ -98,7 +98,9 @@ def clean_app_id(self): try: get_brief_app(domain, app_id) except NoResultFound: - raise forms.ValidationError(f"App not found in domain: {domain}:{app_id}") + raise forms.ValidationError(_("App not found in domain: {domain}:{app_id}").format( + domain=domain, app_id=app_id + )) return app_id diff --git a/corehq/apps/app_execution/models.py b/corehq/apps/app_execution/models.py index 12f716cae364..0a75457267b9 100644 --- a/corehq/apps/app_execution/models.py +++ b/corehq/apps/app_execution/models.py @@ -11,6 +11,7 @@ from corehq.apps.app_manager.dbaccessors import get_brief_app from corehq.sql_db.functions import MakeInterval from corehq.util.jsonattrs import AttrsObject +from django.utils.translation import gettext_lazy class AppWorkflowManager(models.Manager): @@ -23,9 +24,9 @@ def get_due(self): class AppWorkflowConfig(models.Model): FORM_MODE_CHOICES = [ - (const.FORM_MODE_HUMAN, "Human: Answer each question individually and submit form"), - (const.FORM_MODE_NO_SUBMIT, "No Submit: Answer all questions but don't submit the form"), - (const.FORM_MODE_IGNORE, "Ignore: Do not complete or submit forms"), + (const.FORM_MODE_HUMAN, gettext_lazy("Human: Answer each question individually and submit form")), + (const.FORM_MODE_NO_SUBMIT, gettext_lazy("No Submit: Answer all questions but don't submit the form")), + (const.FORM_MODE_IGNORE, gettext_lazy("Ignore: Do not complete or submit forms")), ] name = models.CharField(max_length=255) domain = models.CharField(max_length=255) @@ -34,11 +35,12 @@ class AppWorkflowConfig(models.Model): django_user = models.ForeignKey(User, on_delete=models.CASCADE) workflow = AttrsObject(AppWorkflow) form_mode = models.CharField(max_length=255, choices=FORM_MODE_CHOICES) - sync_before_run = models.BooleanField(default=False, help_text="Sync user data before running") - run_every = models.IntegerField(help_text="Number of minutes between runs", null=True, blank=True) + sync_before_run = models.BooleanField(default=False, help_text=gettext_lazy("Sync user data before running")) + run_every = models.IntegerField( + help_text=gettext_lazy("Number of minutes between runs"), null=True, blank=True) last_run = models.DateTimeField(null=True, blank=True) notification_emails = ArrayField( - models.EmailField(), default=list, help_text="Emails to notify on failure", blank=True + models.EmailField(), default=list, help_text=gettext_lazy("Emails to notify on failure"), blank=True ) objects = AppWorkflowManager() diff --git a/corehq/apps/app_execution/templates/app_execution/components/logs.html b/corehq/apps/app_execution/templates/app_execution/components/logs.html index 254a40b02d93..a0d174af09ca 100644 --- a/corehq/apps/app_execution/templates/app_execution/components/logs.html +++ b/corehq/apps/app_execution/templates/app_execution/components/logs.html @@ -1,19 +1,20 @@ +{% load i18n %}

Logs

{{ output }}
{% if success %}
- Success + {% trans "Success" %}
{% else %}
- Error: {{ error }} + {% trans "Error:" %} {{ error }}
{% endif %}
-

Workflow

+

{% trans "Workflow" %}

{{ workflow_json }}
diff --git a/corehq/apps/app_execution/templates/app_execution/workflow_list.html b/corehq/apps/app_execution/templates/app_execution/workflow_list.html index 52585f63f37c..6bf5333247fa 100644 --- a/corehq/apps/app_execution/templates/app_execution/workflow_list.html +++ b/corehq/apps/app_execution/templates/app_execution/workflow_list.html @@ -31,11 +31,11 @@

{% trans "Status" %}

- - - - - + + + + + @@ -60,10 +60,10 @@

{% trans "Status" %}

diff --git a/corehq/apps/app_execution/templates/app_execution/workflow_log.html b/corehq/apps/app_execution/templates/app_execution/workflow_log.html index 9dd7e9d676c9..0114ee840793 100644 --- a/corehq/apps/app_execution/templates/app_execution/workflow_log.html +++ b/corehq/apps/app_execution/templates/app_execution/workflow_log.html @@ -2,17 +2,19 @@ {% load i18n %} {% block page_content %}
-

Workflow Execution Log for {{ log.config.name }}: {{ log.started|date }}

- Edit Workflow +

{% blocktrans with workflow_name=log.config.name date=log.started|date %} + Workflow Execution Log for {{ workflow_name }}: {{ date }} + {% endblocktrans %}

+ {% translate 'Edit Workflow' %}
-

Details

+

{% translate 'Details' %}

    -
  • Started: {{ log.started|date:"DATETIME_FORMAT" }}
  • -
  • Ended: {{ log.completed|date:"DATETIME_FORMAT" }}
  • -
  • Duration: {{ log.duration }}
  • -
  • Status: - {% if log.success %}Success{% else %}Error{% endif %} +
  • {% blocktrans with date_started=log.started|date:"DATETIME_FORMAT" %}Started: {{ date_started }}{% endblocktrans %}
  • +
  • {% blocktrans with date_ended=log.completed|date:"DATETIME_FORMAT" %}Ended: {{ date_ended }}{% endblocktrans %}
  • +
  • {% blocktrans with duration=log.duration %}Duration: {{ duration }}{% endblocktrans %}
  • +
  • {% translate 'Status:' %} + {% if log.success %}{% translate 'Success' %}{% else %}{% translate 'Error' %}{% endif %}

diff --git a/corehq/apps/app_execution/templates/app_execution/workflow_log_list.html b/corehq/apps/app_execution/templates/app_execution/workflow_log_list.html index 6552e6ec4bfc..9cb34e8c8041 100644 --- a/corehq/apps/app_execution/templates/app_execution/workflow_log_list.html +++ b/corehq/apps/app_execution/templates/app_execution/workflow_log_list.html @@ -22,25 +22,25 @@ {% initial_page_data 'total_items' total %} {% registerurl "app_execution:logs_json" request.domain workflow.id %} -

Logs for workflow "{{ workflow.name }}"

+

{% translate 'Logs for workflow' %} "{{ workflow.name }}"

-

{% trans "Average Timings" %}

+

{% translate "Average Timings" %}

-

{% trans "Log Status" %}

+

{% translate "Log Status" %}

- +
@@ -50,9 +50,9 @@

{% trans "Log Status" %}

NameApp NameUserLast RunLast 10 Runs{% translate 'Name' %}{% translate 'App Name' %}{% translate 'User' %}{% translate 'Last Run' %}{% translate 'Last 10 Runs' %}
-
- - - + + + @@ -66,7 +66,7 @@

{% trans "Log Status" %}

- +
StatusStartedDuration{% translate 'Status' %}{% translate 'Started' %}{% translate 'Duration' %}
Details{% translate 'Details' %}
@@ -75,6 +75,9 @@

{% trans "Log Status" %}

{{ chart_data|json_script:"chart_data" }} {% endblock %} diff --git a/corehq/apps/app_execution/templates/app_execution/workflow_test.html b/corehq/apps/app_execution/templates/app_execution/workflow_test.html index 727dcb7be21c..3ac915e0568c 100644 --- a/corehq/apps/app_execution/templates/app_execution/workflow_test.html +++ b/corehq/apps/app_execution/templates/app_execution/workflow_test.html @@ -14,15 +14,18 @@ {% endblock %} {% block page_content %}
-

Testing {{ workflow.name }}

- Edit +

{% blocktrans with name=workflow.name %}Testing {{ name }}{% endblocktrans %}

+ {% translate 'Edit' %}
{% if result %} {% include "app_execution/components/logs.html" %} {% endif %}
{% csrf_token %} - +
{% endblock %} diff --git a/corehq/apps/app_execution/views.py b/corehq/apps/app_execution/views.py index b3b33fa87d16..ff8e8c9aa96d 100644 --- a/corehq/apps/app_execution/views.py +++ b/corehq/apps/app_execution/views.py @@ -18,6 +18,7 @@ from corehq.apps.hqwebapp.decorators import use_bootstrap5 from corehq.util.timezones.utils import get_timezone_for_user from corehq.util.view_utils import get_date_param +from django.utils.translation import gettext as _ @require_superuser_or_contractor @@ -29,7 +30,7 @@ def workflow_list(request, domain): start = utcnow - relativedelta(months=1) context = _get_context( request, - "Automatically Executed App Workflows", + _("Automatically Executed App Workflows"), reverse("app_execution:workflow_list", args=[domain]), workflows=workflows, chart_data={ @@ -77,8 +78,11 @@ def new_workflow(request, domain): return redirect("app_execution:workflow_list", domain) context = _get_context( - request, "New App Workflow", reverse("app_execution:new_workflow", args=[domain]), - add_parent=True, form=form + request, + _("New App Workflow"), + reverse("app_execution:new_workflow", args=[domain]), + add_parent=True, + form=form ) return render(request, "app_execution/workflow_form.html", context) @@ -95,14 +99,15 @@ def edit_workflow(request, domain, pk): if import_har and har_file: form = _get_form_from_har(har_file.read(), request, instance=config) elif har_file: - messages.error(request, "You must use the 'Import HAR' button to upload a HAR file.") + messages.error(request, _("You must use the 'Import HAR' button to upload a HAR file.")) else: if form.is_valid(): form.save() return redirect("app_execution:workflow_list", domain) context = _get_context( - request, f"Edit App Workflow: {config.name}", reverse("app_execution:edit_workflow", args=[domain, pk]), + request, _("Edit App Workflow: {name}").format(name=config.name), + reverse("app_execution:edit_workflow", args=[domain, pk]), add_parent=True, form=form ) return render(request, "app_execution/workflow_form.html", context) @@ -116,7 +121,7 @@ def _get_form_from_har(har_data_string, request, instance=None): post_data["app_id"] = config.app_id post_data["workflow"] = AppWorkflowConfig.workflow_object_to_json_string(config.workflow) except Exception as e: - messages.error(request, "Unable to process HAR file: " + str(e)) + messages.error(request, _("Unable to process HAR file: {error}").format(error=str(e))) return AppWorkflowConfigForm(request, post_data, instance=instance) @@ -173,7 +178,7 @@ def workflow_log_list(request, domain, pk): start = utcnow - relativedelta(months=1) context = _get_context( request, - "Automatically Executed App Workflow Logs", + _("Automatically Executed App Workflow Logs"), reverse("app_execution:workflow_logs", args=[domain, pk]), add_parent=True, workflow=AppWorkflowConfig.objects.get(id=pk), @@ -199,7 +204,7 @@ def workflow_logs_json(request, domain, pk): start_date = get_date_param(request, 'startDate', timezone=timezone) end_date = get_date_param(request, 'endDate', timezone=timezone) except ValueError: - return JsonResponse({"error": "Invalid date parameter"}) + return JsonResponse({"error": _("Invalid date parameter")}) query = AppExecutionLog.objects.filter(workflow__domain=domain, workflow_id=pk) if status: @@ -233,7 +238,7 @@ def workflow_log(request, domain, pk): request, "app_execution/workflow_log.html", _get_context( request, - f"Workflow Log: {log.workflow.name}", + _("Workflow Log: {name}").format(name=log.workflow.name), reverse("app_execution:workflow_log", args=[domain, pk]), add_parent=True, log=log, From f776a2cb429595976d05dd01d29dc1cb4dab1c95 Mon Sep 17 00:00:00 2001 From: Simon Kelly Date: Thu, 30 May 2024 13:15:33 +0200 Subject: [PATCH 23/63] sidebar navigation --- corehq/tabs/tabclasses.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/corehq/tabs/tabclasses.py b/corehq/tabs/tabclasses.py index e4a93fa12b3a..46fe52702606 100644 --- a/corehq/tabs/tabclasses.py +++ b/corehq/tabs/tabclasses.py @@ -1134,6 +1134,40 @@ def dropdown_items(self): return submenu_context + @property + @memoized + def sidebar_items(self): + return [ + (_("Application Test Flows"), [ + { + 'title': "Workflow List", + 'url': reverse("app_execution:workflow_list", args=[self.domain]), + 'subpages': [ + { + 'title': _("New"), + 'urlname': "new_workflow", + }, + { + 'title': _("Edit"), + 'urlname': "edit_workflow", + }, + { + 'title': _("Run"), + 'urlname': "test_workflow", + }, + { + 'title': _("Logs"), + 'urlname': "workflow_logs", + }, + { + 'title': _("Log Details"), + 'urlname': "workflow_log", + }, + ], + }, + ]), + ] + @property def _is_viewable(self): couch_user = self.couch_user From 935455fe0b58095405c88dac2700f3ca29be705b Mon Sep 17 00:00:00 2001 From: Simon Kelly Date: Thu, 30 May 2024 13:37:43 +0200 Subject: [PATCH 24/63] responsive graph layout --- .../app_execution/templates/app_execution/workflow_list.html | 2 +- .../templates/app_execution/workflow_log_list.html | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/corehq/apps/app_execution/templates/app_execution/workflow_list.html b/corehq/apps/app_execution/templates/app_execution/workflow_list.html index 6bf5333247fa..a8a9f7555658 100644 --- a/corehq/apps/app_execution/templates/app_execution/workflow_list.html +++ b/corehq/apps/app_execution/templates/app_execution/workflow_list.html @@ -18,7 +18,7 @@ -
+

{% trans "Average Timings" %}

diff --git a/corehq/apps/app_execution/templates/app_execution/workflow_log_list.html b/corehq/apps/app_execution/templates/app_execution/workflow_log_list.html index 9cb34e8c8041..e36a2100ab8d 100644 --- a/corehq/apps/app_execution/templates/app_execution/workflow_log_list.html +++ b/corehq/apps/app_execution/templates/app_execution/workflow_log_list.html @@ -23,7 +23,7 @@ {% registerurl "app_execution:logs_json" request.domain workflow.id %}

{% translate 'Logs for workflow' %} "{{ workflow.name }}"

-
+

{% translate "Average Timings" %}

@@ -34,7 +34,7 @@

{% translate "Log Status" %}

-
+
- - - + {% if location_case_sync_restriction_enabled %} + + + + {% endif %}