Merge pull request #342 from perftool-incubator/dev-kmr

allow OpenSearch read-only bits to be cleared from indices

bin/_help

@@ -34,6 +34,7 @@ function help() {
     echo "postprocess                  |  Post-process (and index) a result with run-result-dir argument immediately following 'index'"
     echo "opensearch init              |  Completely delete and init OpenSearch, leaving OpenSearch with only empty indices"
     echo "opensearch rebuild           |  Completely delete and init OpenSearch, then post-process and index all runs found in ${var_run_crucible}"
+    echo "opensearch repair            |  Clear read-only bits applied to OpenSearch indices when the disk gets too full.  Only do this after freeing up disk space."
     echo "extract                      |  Extract data from a run directory:"
     echo "  run-id <dir>               |    - Such as the run ID"
     echo "  primary-periods <dir>      |    - Or a list of the primary period IDs"

bin/_main

@@ -126,6 +126,8 @@ elif [ "${1}" == "opensearch" ]; then
             init_opensearch
         elif [ "${1}" == "rebuild" ]; then
             reinit_opensearch
+        elif [ "${1}" == "repair" ]; then
+            repair_opensearch
         fi
         if [ ${EXIT_VAL} == 0 -a "${1}" == "rebuild" ]; then
             if pushd ${var_run_crucible} >/dev/null; then

bin/base

@@ -782,6 +782,31 @@ function reinit_opensearch() {
     return ${RC}
 }
 
+function repair_opensearch() {
+    local RC
+    RC=0
+
+    opensearch_put_cmd="curl --silent --show-error --stderr - -X PUT localhost:9200/_all/_settings"
+
+    echo "Clearing read-only bit:"
+    pod_name="crucible-clear-opensearch-read-only"
+    ${podman_run} --name ${pod_name} "${container_common_args[@]}" ${CRUCIBLE_CONTAINER_IMAGE} ${opensearch_put_cmd}  -H 'Content-Type: application/json' --data '{ "index": { "blocks": { "read_only": false } } }'
+    RC=$?
+    if [ ${RC} != 0 ]; then
+        echo "ERROR: Failed to clear read-only bit on OpenSearch indices"
+    else
+        echo "Clearing read-only-allow-delete bit:"
+        pod_name="crucible-clear-opensearch-read-only-allow-delete"
+        ${podman_run} --name ${pod_name} "${container_common_args[@]}" ${CRUCIBLE_CONTAINER_IMAGE} ${opensearch_put_cmd}  -H 'Content-Type: application/json' --data '{ "index": { "blocks": { "read_only_allow_delete": false } } }'
+        RC=$?
+        if [ ${RC} != 0 ]; then
+            echo "ERROR: Failed to clear read-only-allow-delete bit on OpenSearch indices"
+        fi
+    fi
+
+    return ${RC}
+}
+
 function confirm_continue() {
     local ref
     local action_message