Merge pull request #367 from dimagi/QA-6752-regular-bha-specific-perf…

…ormance-testing-on-prod

QA-6752 changes for outgoing referrals

LocustScripts/update-scripts/case_search/README.md

@@ -0,0 +1,80 @@
+# Case Search Locust Scripts
+
+This load test script performs continuous case search request per user with a constant wait time of 1s between requests.
+
+The query data for each request is selected by randomizing the data in the query YAML configuration file.
+
+Run from `LocustScripts` directory:
+
+```shell
+locust -f case_search/locustfile.py --headless -u 1 -r 1 \
+  --host http://localhost:8000 \
+  --domain skelly-1 \
+  --app-id b62974969e57051ad70160a798ed79e8  \
+  --queries case_search/co_carecoordination_queries.yml \
+  --user-details case_search/user_credentials_example.json
+```
+
+## Options
+
+- `--host` - URL of the application
+- `--domain` - Domain of the application
+- `--app-id` - Application ID
+- `--queries` - Path to the queries file
+- `--user-details` - Path to the user credentials file
+
+## Request generation
+
+Each request is generated as follows:
+
+1. Select a random query from the list of queries provided.
+2. Select random value sets for the query using the `query.value_set_types` field.
+3. Fill in the query parameters with values from the value set.
+4. Send the request.
+
+## Queries file
+
+This YAML file contains request data that will be used by the locust scripts to perform case search requests.
+
+### Queries ("queries" key)
+A list of parameterized queries. Each request will select one query at random and fill in the
+parameters with values from a random value set.
+
+Query fields:
+- `name`: The name of the query (used for logging)
+- `case_types`: A list of case types to pass via the 'case_type' query parameter (required)
+- `value_set_types`: (optional) The value set types to use to select a value set data for the query at runtime. If
+  multiple types are provided, the data will be merged into a single dictionary before filling in the query parameters.
+- `query_params`: A dictionary of query parameters to pass in the request. They query values may contain
+  variable references using the '{name}' syntax which will be filled by values from the value set. The values
+  of the dictionary may be a string or a list.
+
+```yaml
+queries:
+  - name: test
+    case_types: [client]
+    value_set_types: client
+    query_params:
+      case_name: {case_name}
+      _xpath_query:
+        - first_name='{first_name}' and last_name='{last_name}'
+        - subcase-exists('parent', @case_type = 'alias' and first_name='{first_name}' and last_name='{last_name}')
+```
+
+### Value Sets ("value_sets" key)
+A list of parameter values which are used to format the queries.
+
+Value set fields:
+- `type`: A string which indicates which type of value set this is. A value set with
+  a type matching `query.value_set_types` will be selected at runtime.
+- `values`: A dictionary of values which are used to format the query at runtime. The keys in this dictionary are used
+  to replace the variable references in the query parameters.
+
+```yaml
+value_sets:
+  - type: client
+    values:
+      first_name: bob
+      last_name: smith
+      alias: bobby
+```

LocustScripts/update-scripts/case_search/co_carecoordination_queries.yml

@@ -0,0 +1,144 @@
+queries:
+  - name: search_admit_client
+    case_types: [client]
+    value_set_types: client
+    query_params:
+      _xpath_query:
+        - central_registry = "yes" and current_status != "pending"
+        - consent_collected = "yes"
+        - (
+            (
+              (
+                (
+                  fuzzy-match(first_name, "{first_name}") and fuzzy-match(last_name, "{last_name}"))
+                  or (
+                    phonetic-match(first_name, "{first_name}") and phonetic-match(last_name, "{last_name}")
+                  )
+                ) and fuzzy-match(dob, "{dob}")
+              )
+            ) or subcase-exists("parent", @case_type = "alias" and @status = "open" and (
+              (
+                (fuzzy-match(first_name, "{first_name}") and fuzzy-match(last_name, "{last_name}"))
+                or (phonetic-match(first_name, "{first_name}") and phonetic-match(last_name, "{last_name}"))
+              ) and fuzzy-match(dob, "{dob}")
+            )
+          )
+  - name: search_my_clients
+    case_types: [client]
+    value_set_types: [client, clinic_case_ids]
+    query_params:
+      _xpath_query:
+        - central_registry = "yes" and subcase-exists(
+            "parent", @case_type = "service" and @status != "closed" and central_registry = "yes"
+            and selected(clinic_case_id,"{clinic_case_ids}")
+          )
+        - match-none() or match-none() or match-none() or match-none() or (
+            (
+              (
+                (fuzzy-match(first_name, "{first_name}") or phonetic-match(first_name, "{first_name}"))
+                and (fuzzy-match(last_name, "{last_name}") or phonetic-match(last_name, "{last_name}"))
+              )
+              or subcase-exists(
+                "parent", @case_type = "alias" and @status != "closed" and (
+                  (fuzzy-match(first_name, "{first_name}") or phonetic-match(first_name, "{first_name}"))
+                  and (fuzzy-match(last_name, "{last_name}") or phonetic-match(last_name, "{last_name}"))
+                  )
+                )
+              ) and (match-all())
+            )
+        - current_status = "admitted" and selected(active_admission_clinic_id, "{clinic_case_ids}")
+        - match-all()
+        - match-all()
+  - name: search_incoming_requests
+    case_types: [client]
+    value_set_types: [gender_params, age_range_adult, clinic_case_ids]
+    query_params:
+      x_commcare_include_all_related_cases: "true"
+      _xpath_query:
+        - selected(destination_clinic_case_id, "{clinic_case_ids}")
+        - "ancestor-exists(parent,  @status = 'open' and @case_type = 'client' and current_status != 'closed' and central_registry = 'no')"
+        - "ancestor-exists(parent, @status = 'open' and @case_type = 'client' and current_status != 'closed' and selected(gender, '{gender}') or selected(gender, '{gender_restrictions}'))"
+        - "ancestor-exists(parent, @status = 'open' and @case_type = 'client' and current_status != 'closed' and age > '{age_range_low}' and age < '{age_range_high}')"
+        - selected(current_status, 'open info_requested')
+        - match-all()
+        - match-all()
+        - match-all()
+  - name: potential_duplicate_clients
+    case_types: [ client ]
+    value_set_types: [ client ]
+    query_params:
+      _xpath_query:
+        - central_registry = "yes"
+        - consent_collected = "yes"
+        - >
+          match-none() or match-none() or match-none() or match-none() or (
+            (
+              (
+                (fuzzy-match(first_name, "{first_name}") or phonetic-match(first_name, "{first_name}"))
+                and (fuzzy-match(last_name, "{last_name}") or phonetic-match(last_name, "{last_name}"))
+              )
+              or subcase-exists("parent", 
+                @case_type = "alias" and @status != "closed"
+                and (
+                  (fuzzy-match(first_name, "{first_name}") or phonetic-match(first_name, "{first_name}"))
+                  and (fuzzy-match(last_name, "{last_name}") or phonetic-match(last_name, "{last_name}"))
+                )
+              )
+            ) and (
+              fuzzy-match(dob, "{dob}")
+              or subcase-exists("parent", @case_type = "alias" and @status != "closed" and fuzzy-match(dob, "{dob}"))
+            )
+          )
+        - '@case_id != "{case_id}"'
+        - 'not(selected(@case_id, "{case_id}"))'
+  - name: find_new_facilities
+    case_types: [unit]
+    query_params:
+      commcare_sort: +clinic_display_name:exact
+      _xpath_query:
+        - current_status != "closed"
+        - match-all()
+        - match-all()
+        - match-all()
+        - match-all()
+        - match-all()
+        - match-all()
+        - match-all()
+        - match-all()
+        - match-all()
+  - name: search_outgoing_referrals
+    case_types: [ referral ]
+    value_set_types: [clinic_case_ids, age_range_adult]
+    query_params:
+      x_commcare_include_all_related_cases: "true"
+      _xpath_query:
+        - selected(referring_clinic_case_id, "{clinic_case_ids}")
+        - ancestor-exists(parent,  @status = "open" and @case_type = "client" and central_registry = "no" and (
+            current_status = "open" or current_status = "withdrawn" or current_status = "escalated"
+          ))
+        - match-all()
+        - match-all()
+        - ancestor-exists(parent, @status = 'open' and @case_type = 'client' and current_status != 'closed' and age > '{age_range_low}' and age < '{age_range_high}')
+        - match-all()
+        - match-all()
+        - match-all()
+        - match-all()
+value_sets:
+  - path: ush_envelop_testing_clients.csv
+    type: client
+    format: csv
+  - type: clinic_case_ids
+    values:
+      clinic_case_ids: 123 abc 456
+  - type: gender_params
+    values:
+      gender: men
+      gender_restrictions: no_gender_restrictions
+  - type: gender_params
+    values:
+      gender: women
+      gender_restrictions: no_gender_restrictions
+  - type: age_range_adult
+    values:
+      age_range_low: 17
+      age_range_high: 65

LocustScripts/update-scripts/case_search/loader.py

@@ -0,0 +1,54 @@
+import pathlib
+
+from case_search.models import QueryData
+from common.utils import load_csv_data, load_data, load_yaml_data
+
+
+def load_query_data(path):
+    data = load_yaml_data(path)
+    value_sets = []
+    for value_set in data["value_sets"]:
+        if "path" in value_set:
+            value_sets.extend(load_value_set_from_reference(path, value_set))
+        else:
+            value_sets.append(value_set)
+
+    return QueryData.model_validate({
+        "value_sets": value_sets,
+        "queries": data["queries"],
+    })
+
+
+def load_value_set_from_reference(source_path, reference):
+    path_str = reference.pop("path")
+    path = _get_reference_path(path_str, source_path)
+    _format = reference.pop("format")
+    return {
+        "csv": load_value_set_from_csv,
+    }[_format](path, **reference)
+
+
+def _get_reference_path(path_str, source_path):
+    path = pathlib.Path(path_str)
+    if not path.is_absolute():
+        path = source_path.resolve().parent / path
+    if not path.exists():
+        raise ValueError(f"Value set file not found: {path}")
+    return path
+
+
+def load_value_set_from_csv(path, type):
+    """Load value sets from a CSV file.
+
+    Args:
+        path (Path): Path to the CSV file.
+        type (str): Keys for the value set.
+    """
+    data = load_csv_data(path)
+    return [
+        {
+            "type": type,
+            "values": row,
+        }
+        for row in data
+    ]

LocustScripts/update-scripts/case_search/locustfile.py

@@ -0,0 +1,87 @@
+
+import logging
+
+from locust import HttpUser, constant, events, task
+from locust.exception import InterruptTaskSet, StopUser
+
+from case_search.loader import load_query_data
+from user.models import UserDetails
+from common.args import file_path
+from common.utils import RandomItems, load_json_data
+
+
+@events.init_command_line_parser.add_listener
+def _(parser):
+    parser.add_argument("--domain", help="CommCare domain", required=True, env_var="COMMCARE_DOMAIN")
+    parser.add_argument("--app-id", help="CommCare app id", required=True, env_var="COMMCARE_APP_ID")
+    parser.add_argument("--queries", help="Path to queries YAML file", required=True)
+    parser.add_argument("--user-details", help="Path to user details file", required=True)
+
+
+QUERY_DATA = []
+USERS = RandomItems()
+
+
+def get_random_query():
+    return QUERY_DATA[-1].get_random_query()
+
+
+@events.init.add_listener
+def _(environment, **kw):
+    try:
+        queries = file_path(environment.parsed_options.queries)
+        QUERY_DATA.append(load_query_data(queries))
+        logging.info("Loaded %s queries and %s value sets", len(QUERY_DATA[0].queries), len(QUERY_DATA[0].value_sets))
+    except Exception as e:
+        logging.error("Error loading queries: %s", e)
+        raise InterruptTaskSet from e
+
+    try:
+        user_path = file_path(environment.parsed_options.user_details)
+        user_data = load_json_data(user_path)["user"]
+        USERS.set([UserDetails(**user) for user in user_data])
+        logging.info("Loaded %s users", len(USERS.items))
+    except Exception as e:
+        logging.error("Error loading users: %s", e)
+        raise InterruptTaskSet from e
+
+
+class CaseSearchUser(HttpUser):
+    wait_time = constant(1)
+
+    def on_start(self):
+        self.user_details = USERS.get()
+        self.login()
+
+    def login(self):
+        login_url = f"/a/{self.environment.parsed_options.domain}/login/"
+        self.client.get(login_url)  # get CSRF token
+        response = self.client.post(
+            login_url,
+            {
+                "auth-username": self.user_details.username,
+                "auth-password": self.user_details.password,
+                "cloud_care_login_view-current_step": ['auth'],  # fake out two_factor ManagementForm
+            },
+            headers={
+                "X-CSRFToken": self.client.cookies.get('csrftoken'),
+                "REFERER": f"{self.environment.parsed_options.host}{login_url}",  # csrf requires this
+            },
+        )
+        if not response.status_code == 200:
+            raise StopUser(f"Login failed for user {self.user_details.username}: {response.status_code}")
+        if 'Sign In' in response.text:
+            raise StopUser(f"Login failed for user {self.user_details.username}: Sign In failed")
+
+    @task
+    def search_case(self):
+        url = f"/a/{self.environment.parsed_options.domain}/phone/search/{self.environment.parsed_options.app_id}/"
+        name, query = get_random_query()
+        with self.client.post(
+            url,
+            data=query,
+            name=f"Search cases: {name}",
+            catch_response=True,
+        ) as resp:
+            if resp.status_code == 400:
+                logging.error("Bad request for query '%s': %s", name, resp.text)