diff --git a/config/es_template.json b/config/es_template.json new file mode 100644 index 0000000..09e80dd --- /dev/null +++ b/config/es_template.json @@ -0,0 +1,53 @@ +{ + "order" : 0, + "template" : "{{ index }}", + "settings" : { + "index.refresh_interval" : "5s", + "analysis": { + "analyzer": { + "default": { + "filter": [ + "standard", + "lowercase", + "word_delimiter" + ], + "tokenizer": "keyword" + } + } + } + }, + "mappings" : { + "_default_" : { + "dynamic_templates" : [ { + "string_fields" : { + "mapping" : { + "index" : "analyzed", + "omit_norms" : true, + "type" : "string", + "fields" : { + "raw" : { + "index" : "not_analyzed", + "ignore_above" : 256, + "type" : "string" + } + } + }, + "match_mapping_type" : "string", + "match" : "*" + } + } ], + "_id": { + "type": "string", + "store": true, + "index": "not_analyzed" + }, + "_timestamp": { + "enabled": true, + "store": true + }, + "_all" : { + "enabled" : true + } + } + } +} diff --git a/scripts/install_es_template.py b/scripts/install_es_template.py new file mode 100755 index 0000000..d0b524b --- /dev/null +++ b/scripts/install_es_template.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +import os, sys, json, requests +from jinja2 import Template + +from hysds.celery import app + + +def write_template(es_url, index, tmpl_file): + """Write template to ES.""" + + with open(tmpl_file) as f: + tmpl = Template(f.read()).render(index=index) + tmpl_url = "%s/_template/%s" % (es_url, index) + r = requests.delete(tmpl_url) + r = requests.put(tmpl_url, data=tmpl) + r.raise_for_status() + print r.json() + print "Successfully installed template %s at %s." % (index, tmpl_url) + + +if __name__ == "__main__": + node = sys.argv[1] + if node == "mozart": + es_url = app.conf['JOBS_ES_URL'] + indices = [ "containers", "job_specs", "hysds_ios" ] + elif node == "grq": + es_url = app.conf['GRQ_ES_URL'] + indices = [ "hysds_ios" ] + else: + raise RuntimeError("Invalid node: %s" % node) + tmpl_file = os.path.normpath(os.path.abspath(os.path.join( + os.path.dirname(__file__), '..', 'config', 'es_template.json' + ))) + for index in indices: + write_template(es_url, index, tmpl_file) diff --git a/scripts/install_es_template.sh b/scripts/install_es_template.sh new file mode 100755 index 0000000..b64d1c0 --- /dev/null +++ b/scripts/install_es_template.sh @@ -0,0 +1,7 @@ +#!/bin/bash +BASE_PATH=$(dirname "${BASH_SOURCE}") +BASE_PATH=$(cd "${BASE_PATH}"; pwd) +export PYTHONPATH=$BASE_PATH/..:$PYTHONPATH + +# install templates for containers, job_specs, and hysds_ios +$BASE_PATH/install_es_template.py $*