From 0b515bfc840b8461e5e24110be3d43ccb954bda6 Mon Sep 17 00:00:00 2001 From: Zandre Engelbrecht Date: Wed, 30 Oct 2024 16:09:50 +0200 Subject: [PATCH] require refresh on saving polygon --- corehq/mobile_ucr_v2_update_script.py | 186 ++++++++++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 corehq/mobile_ucr_v2_update_script.py diff --git a/corehq/mobile_ucr_v2_update_script.py b/corehq/mobile_ucr_v2_update_script.py new file mode 100644 index 0000000000000..ebe42bb855d17 --- /dev/null +++ b/corehq/mobile_ucr_v2_update_script.py @@ -0,0 +1,186 @@ +# This script updates the latest versions of all apps across domains that are using v1 with no manual references +# Steps followed +# 1. Get All domains with mobile ucr flag enabled +# 2. Get all apps for domain with latest released versions and mobile ucr versions that are not v2 +# 3. For each app, if it contains no V1 UCR references, update the version to 2 + +# How to run +# Can be run in django shell. Paste the script and execute the function process() +# File is stored in home directory of cchq user. + +# V1 Examples +# https://staging.commcarehq.org/a/test-commcare-superset/apps/view/f940fcc83bae44b8a0adaf719673fd1e/form/a0f3c5b483c645e78b6f89ee0b3b3c03/source/#form/table_child_count + + +import json +import re +import resource +import traceback +import os + +from corehq.apps.app_manager.dbaccessors import get_apps_by_id +from corehq.apps.app_manager.const import MOBILE_UCR_VERSION_2 +from corehq.apps.app_manager.models import Application +from corehq.toggles import MOBILE_UCR +from corehq.toggles.shortcuts import find_domains_with_toggle_enabled +from corehq.util.log import with_progress_bar + + +PROCESSED_DOMAINS_PATH = os.path.expanduser("~/zeng/mobile_ucr_migration/updated_domains.ndjson") +LOG_FILE = os.path.expanduser("~/zeng/mobile_ucr_migration/update_to_v2_ucr_script.log") + +V1_FIXTURE_IDENTIFIER = 'src="jr://fixture/commcare:reports' +V1_FIXTURE_PATTERN = r'<.*src="jr://fixture/commcare:reports.*>' +V1_REFERENCES_PATTERN = r"<.*instance\('reports'\)/reports/.*>" +RE_V1_ALL_REFERENCES = re.compile(f"{V1_FIXTURE_PATTERN}|{V1_REFERENCES_PATTERN}") + +# Pattern for references in case list/detail. +# The below instance needs to be referred in a fixture setup and +# is then later used in Case List/Detail filters and columns. +V1_CASE_LIST_REFERENCES_PATTERN = r"instance\('commcare:reports'\)/reports/report\[@id='.*'\]/rows/row" +RE_V1_CASE_LIST_REFERENCES_PATTERN = re.compile(V1_CASE_LIST_REFERENCES_PATTERN) + +skip_domains = set() + + +def process(dry_run=True, max_memory_size=None, single_app_retrieve=False): + """ + - single_app_retrieve: Only retrieve and process one app at a time from DB. Results + in more DB hits, but reduces memory usage + """ + if max_memory_size: + set_max_memory(max_memory_size) + + try: + processed_domains = read_ndjson_file(PROCESSED_DOMAINS_PATH) + except FileNotFoundError: + processed_domains = set() + + mobile_ucr_domains = find_domains_with_toggle_enabled(MOBILE_UCR) + + save_in_log(f"Number of domains with mobile ucr flag enabled: {len(mobile_ucr_domains)} ") + + for domain in with_progress_bar(mobile_ucr_domains): + if domain in processed_domains: + save_in_log(f"Already processed domain: {domain}") + continue + if domain in skip_domains: + save_in_log(f"Skipped domain: {domain}") + continue + + save_in_log(f"Processing domain: {domain} ...") + try: + app_ids = list(get_latest_app_ids_and_versions(domain)) + if single_app_retrieve: + apps_or_ids = app_ids + else: + apps_or_ids = get_apps_by_id(domain, app_ids) + except Exception as error: + save_in_log(f'Failed to get apps for {domain} with error: {str(error)}') + continue + + for app_or_id in apps_or_ids: + try: + if single_app_retrieve: + app = get_apps_by_id(domain, [app_or_id])[0] + else: + app = app_or_id + process_app(domain, app, dry_run) + except Exception as e: + save_in_log(f"Error occurred for {domain}: {str(e)}") + save_in_log(traceback.format_exc()) + continue + save_as_ndjson(PROCESSED_DOMAINS_PATH, domain) + + +def save_in_log(data): + print(data) + with open(LOG_FILE, 'a') as file: + file.write(data + '\n') + + +def save_as_ndjson(path, data): + with open(path, 'a') as file: + print(json.dumps(data, separators=(',', ':')), file=file) + + +def read_ndjson_file(path): + with open(path, 'r') as file: + return set(json.loads(line) for line in file.readlines()) + + +def has_non_v2_reference(domain, app): + try: + suite = app.create_suite() + except Exception as error: + save_in_log(f"Error occured while fetching reports details for: {domain}: {app.name}: {app.id}:" + f" {app.mobile_ucr_restore_version} with error: {str(error)}") + raise + if re.search(RE_V1_CASE_LIST_REFERENCES_PATTERN, suite.decode()): + save_in_log(f"App Case List Contains V1 Refs: {domain}: {app.name}") + return True + + for form in app.get_forms(): + save_in_log(f"Processing Form: {domain}: {form.name}") + # The second condition should always be False if the first one is + # but just as a precaution we check for it + if V1_FIXTURE_IDENTIFIER in form.source or RE_V1_ALL_REFERENCES.search(form.source): + save_in_log(f"App Contains V1 Refs: {domain}: {app.name}") + return True + return False + + +def process_app(domain, app, dry_run): + if not list(app.get_report_modules()): + return + + # Don't look at app.is_released since the latest version might not be released yet + if app.mobile_ucr_restore_version == '2.0': + return + + save_in_log(f"Processing App: {domain}: {app.name}: {app.id}") + if not has_non_v2_reference(domain, app): + update_app(domain, app, dry_run) + else: + save_in_log( + f"App contains V1 references and couldn't updated: {domain}: {app.name}: {app.id}", + ) + + +def update_app(domain, app, dry_run): + save_in_log(f"Updating App: {domain}: {app.name}: {app.id}") + if not dry_run: + app.mobile_ucr_restore_version = MOBILE_UCR_VERSION_2 + app.save() + + +def set_max_memory(size): + """ + Can be used to set max memory for the process (used for low memory machines) + Example: 800 MB set_max_memory(1024 * 1000 * 800) + + - size: in bytes + """ + soft, hard = resource.getrlimit(resource.RLIMIT_AS) + resource.setrlimit(resource.RLIMIT_AS, (size, hard)) + + +def get_latest_app_ids_and_versions(domain): + key = [domain] + results = Application.get_db().view( + 'app_manager/applications_brief', + startkey=key + [{}], + endkey=key, + descending=True, + reduce=False, + include_docs=False, + ) + latest_ids_and_versions = {} + for result in results: + app_id = result['value']['_id'] + version = result['value']['version'] + + # Since we have sorted, we know the first instance is the latest version + if app_id not in latest_ids_and_versions: + latest_ids_and_versions[app_id] = version + return latest_ids_and_versions