diff --git a/bin/_help b/bin/_help
index 7e9cd10..123dd59 100755
--- a/bin/_help
+++ b/bin/_help
@@ -34,6 +34,7 @@ function help() {
echo "postprocess | Post-process (and index) a result with run-result-dir argument immediately following 'index'"
echo "opensearch init | Completely delete and init OpenSearch, leaving OpenSearch with only empty indices"
echo "opensearch rebuild | Completely delete and init OpenSearch, then post-process and index all runs found in ${var_run_crucible}"
+ echo "opensearch repair | Clear read-only bits applied to OpenSearch indices when the disk gets too full. Only do this after freeing up disk space."
echo "extract | Extract data from a run directory:"
echo " run-id
| - Such as the run ID"
echo " primary-periods | - Or a list of the primary period IDs"
diff --git a/bin/_main b/bin/_main
index af5ef07..c4cd723 100755
--- a/bin/_main
+++ b/bin/_main
@@ -126,6 +126,8 @@ elif [ "${1}" == "opensearch" ]; then
init_opensearch
elif [ "${1}" == "rebuild" ]; then
reinit_opensearch
+ elif [ "${1}" == "repair" ]; then
+ repair_opensearch
fi
if [ ${EXIT_VAL} == 0 -a "${1}" == "rebuild" ]; then
if pushd ${var_run_crucible} >/dev/null; then
diff --git a/bin/base b/bin/base
index 24e4e98..3d9bdf6 100644
--- a/bin/base
+++ b/bin/base
@@ -782,6 +782,31 @@ function reinit_opensearch() {
return ${RC}
}
+function repair_opensearch() {
+ local RC
+ RC=0
+
+ opensearch_put_cmd="curl --silent --show-error --stderr - -X PUT localhost:9200/_all/_settings"
+
+ echo "Clearing read-only bit:"
+ pod_name="crucible-clear-opensearch-read-only"
+ ${podman_run} --name ${pod_name} "${container_common_args[@]}" ${CRUCIBLE_CONTAINER_IMAGE} ${opensearch_put_cmd} -H 'Content-Type: application/json' --data '{ "index": { "blocks": { "read_only": false } } }'
+ RC=$?
+ if [ ${RC} != 0 ]; then
+ echo "ERROR: Failed to clear read-only bit on OpenSearch indices"
+ else
+ echo "Clearing read-only-allow-delete bit:"
+ pod_name="crucible-clear-opensearch-read-only-allow-delete"
+ ${podman_run} --name ${pod_name} "${container_common_args[@]}" ${CRUCIBLE_CONTAINER_IMAGE} ${opensearch_put_cmd} -H 'Content-Type: application/json' --data '{ "index": { "blocks": { "read_only_allow_delete": false } } }'
+ RC=$?
+ if [ ${RC} != 0 ]; then
+ echo "ERROR: Failed to clear read-only-allow-delete bit on OpenSearch indices"
+ fi
+ fi
+
+ return ${RC}
+}
+
function confirm_continue() {
local ref
local action_message