Skip to content
This repository has been archived by the owner. It is now read-only.

Commit

Permalink
Merge pull request #33 from HubbleStack/develop
Browse files Browse the repository at this point in the history 
Merge to master (prep v2016.10.1)
  • Loading branch information
@basepi
basepi authored Oct 18, 2016
2 parents 29b3d37 + 1333558 commit 8e6c239
Show file tree
Hide file tree
Showing 6 changed files with 202 additions and 83 deletions.
2 changes: 1 addition & 1 deletion FORMULA
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name: hubblestack_quasar
os: RedHat, CentOS, Debian, Ubuntu
os_family: RedHat, Debian
version: 2016.7.1
version: 2016.9.0
release: 1
summary: HubbleStack Quasar
description: HubbleStack Quasar
4 changes: 2 additions & 2 deletions README.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,8 @@ repo for updates and bugfixes!)

.. code-block:: shell
wget https://spm.hubblestack.io/2016.7.1/hubblestack_quasar-2016.7.1-1.spm
spm local install hubblestack_quasar-2016.7.1-1.spm
wget https://spm.hubblestack.io/quasar/hubblestack_quasar-2016.9.0-1.spm
spm local install hubblestack_quasar-2016.9.0-1.spm
You should now be able to sync the new modules to your minion(s) using the
``sync_returners`` Salt utility:
Expand Down
17 changes: 14 additions & 3 deletions _returners/slack_pulsar_returner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@
# Import Salt Libs
import salt.returners

__version__ = 'v2016.10.1'

log = logging.getLogger(__name__)

__virtualname__ = 'slack_pulsar'
Expand Down Expand Up @@ -276,9 +278,18 @@ def returner(ret):
log.error('slack_pulsar.api_key not defined in salt config')
return

message = ('id: {0}\r\n'
'return: {1}\r\n').format(__opts__['id'],
pprint.pformat(ret.get('return')))
if isinstance(ret, dict):
message = ('id: {0}\r\n'
'return: {1}\r\n').format(__opts__['id'],
pprint.pformat(ret.get('return')))
elif isinstance(ret, list):
message = 'id: {0}\r\n'
for r in ret:
message += pprint.pformat(r.get('return'))
message += '\r\n'
else:
log.error('Data sent to slack_pulsar formatted incorrectly')
return

slack = _post_message(channel,
message,
Expand Down
63 changes: 44 additions & 19 deletions _returners/splunk_nebula_return.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@

import logging

__version__ = 'v2016.10.1'

_max_content_bytes = 100000
http_event_collector_SSL_verify = False
http_event_collector_debug = True
Expand All @@ -46,29 +48,36 @@ def returner(ret):
logging.info("Options: %s" % json.dumps(opts))
http_event_collector_key = opts['token']
http_event_collector_host = opts['indexer']
hec_ssl = opts['http_event_server_ssl']
proxy = opts['proxy']
timeout = opts['timeout']
# Set up the collector
hec = http_event_collector(http_event_collector_key, http_event_collector_host)
hec = http_event_collector(http_event_collector_key, http_event_collector_host, http_event_server_ssl=hec_ssl, proxy=proxy, timeout=timeout)

# st = "salt:hubble:nova"
data = ret['return']
minion_id = ret['id']
jid = ret['jid']
master = __grains__['master']

for query in data:
for query_name, query_results in query.iteritems():
for query_result in query_results['data']:
event = {}
payload = {}
event.update(query_result)
event.update({'query': query_name})
event.update({"master": master})
event.update({"minion_id": minion_id})
event.update({"job_id": jid})
payload.update({"host": minion_id})
payload.update({"index": opts['index']})
payload.update({"sourcetype": opts['sourcetype']})
payload.update({'event': event})
hec.batchEvent(payload)
if not data:
return
else:
for query in data:
for query_name, query_results in query.iteritems():
for query_result in query_results['data']:
event = {}
payload = {}
event.update(query_result)
event.update({'query': query_name})
event.update({"master": master})
event.update({"minion_id": minion_id})
event.update({"job_id": jid})
payload.update({"host": minion_id})
payload.update({"index": opts['index']})
payload.update({"sourcetype": opts['sourcetype']})
payload.update({'event': event})
hec.batchEvent(payload)

hec.flushBatch()
return
Expand All @@ -83,6 +92,12 @@ def _get_options():
except:
return None
splunk_opts = {"token": token, "indexer": indexer, "sourcetype": sourcetype, "index": index}

hec_ssl = __salt__['config.get']('hubblestack:nebula:returner:splunk:hec_ssl', True)
splunk_opts["http_event_server_ssl"] = hec_ssl
splunk_opts["proxy"] = __salt__['config.get']('hubblestack:nebula:returner:splunk:proxy', {})
splunk_opts['timeout'] = __salt__['config.get']('hubblestack:nebula:returner:splunk:timeout', 9.05)

return splunk_opts


Expand Down Expand Up @@ -118,11 +133,18 @@ def send_splunk(event, index_override=None, sourcetype_override=None):

class http_event_collector:

def __init__(self, token, http_event_server, host="", http_event_port='8088', http_event_server_ssl=True, max_bytes=_max_content_bytes):
def __init__(self, token, http_event_server, host="", http_event_port='8088', http_event_server_ssl=True, max_bytes=_max_content_bytes, proxy=None, timeout=9.05):
self.timeout = timeout
self.token = token
self.batchEvents = []
self.maxByteLength = max_bytes
self.currentByteLength = 0
if proxy and http_event_server_ssl:
self.proxy = {'https': 'https://{0}'.format(proxy)}
elif proxy:
self.proxy = {'http': 'http://{0}'.format(proxy)}
else:
self.proxy = {}

# Set host to specified value or default to localhostname if no value provided
if host:
Expand Down Expand Up @@ -164,7 +186,7 @@ def sendEvent(self, payload, eventtime=""):
data.update(payload)

# send event to http event collector
r = requests.post(self.server_uri, data=json.dumps(data), headers=headers, verify=http_event_collector_SSL_verify)
r = requests.post(self.server_uri, data=json.dumps(data), headers=headers, verify=http_event_collector_SSL_verify, proxies=self.proxy)

# Print debug info if flag set
if http_event_collector_debug:
Expand Down Expand Up @@ -203,6 +225,9 @@ def flushBatch(self):

if len(self.batchEvents) > 0:
headers = {'Authorization': 'Splunk ' + self.token}
r = requests.post(self.server_uri, data=" ".join(self.batchEvents), headers=headers, verify=http_event_collector_SSL_verify)
try:
r = requests.post(self.server_uri, data=" ".join(self.batchEvents), headers=headers, verify=http_event_collector_SSL_verify, proxies=self.proxy, timeout=self.timeout)
except requests.exceptions.Timeout:
log.error('Request to splunk timed out. Not retrying.')
self.batchEvents = []
self.currentByteLength = 0
31 changes: 26 additions & 5 deletions _returners/splunk_nova_return.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@

import logging

__version__ = 'v2016.10.1'

_max_content_bytes = 100000
http_event_collector_SSL_verify = False
http_event_collector_debug = True
Expand All @@ -45,8 +47,11 @@ def returner(ret):
logging.info("Options: %s" % json.dumps(opts))
http_event_collector_key = opts['token']
http_event_collector_host = opts['indexer']
hec_ssl = opts['http_event_server_ssl']
proxy = opts['proxy']
timeout = opts['timeout']
# Set up the collector
hec = http_event_collector(http_event_collector_key, http_event_collector_host)
hec = http_event_collector(http_event_collector_key, http_event_collector_host, http_event_server_ssl=hec_ssl, proxy=proxy, timeout=timeout)
# st = "salt:hubble:nova"
data = ret['return']
minion_id = ret['id']
Expand All @@ -55,7 +60,7 @@ def returner(ret):
if __grains__['master']:
master = __grains__['master']
else:
master = socket.gethostname() # We *are* the master, so use our hostname
master = socket.gethostname() # We *are* the master, so use our hostname

for fai in data.get('Failure', []):
check_id = fai.keys()[0]
Expand Down Expand Up @@ -144,6 +149,12 @@ def _get_options():
except:
return None
splunk_opts = {"token": token, "indexer": indexer, "sourcetype": sourcetype, "index": index}

hec_ssl = __salt__['config.get']('hubblestack:nova:returner:splunk:hec_ssl', True)
splunk_opts["http_event_server_ssl"] = hec_ssl
splunk_opts["proxy"] = __salt__['config.get']('hubblestack:nova:returner:splunk:proxy', {})
splunk_opts['timeout'] = __salt__['config.get']('hubblestack:nova:returner:splunk:timeout', 9.05)

return splunk_opts


Expand Down Expand Up @@ -179,11 +190,18 @@ def send_splunk(event, index_override=None, sourcetype_override=None):

class http_event_collector:

def __init__(self, token, http_event_server, host="", http_event_port='8088', http_event_server_ssl=True, max_bytes=_max_content_bytes):
def __init__(self, token, http_event_server, host="", http_event_port='8088', http_event_server_ssl=True, max_bytes=_max_content_bytes, proxy=None, timeout=9.05):
self.timeout = timeout
self.token = token
self.batchEvents = []
self.maxByteLength = max_bytes
self.currentByteLength = 0
if proxy and http_event_server_ssl:
self.proxy = {'https': 'https://{0}'.format(proxy)}
elif proxy:
self.proxy = {'http': 'http://{0}'.format(proxy)}
else:
self.proxy = {}

# Set host to specified value or default to localhostname if no value provided
if host:
Expand Down Expand Up @@ -225,7 +243,7 @@ def sendEvent(self, payload, eventtime=""):
data.update(payload)

# send event to http event collector
r = requests.post(self.server_uri, data=json.dumps(data), headers=headers, verify=http_event_collector_SSL_verify)
r = requests.post(self.server_uri, data=json.dumps(data), headers=headers, verify=http_event_collector_SSL_verify, proxies=self.proxy)

# Print debug info if flag set
if http_event_collector_debug:
Expand Down Expand Up @@ -264,6 +282,9 @@ def flushBatch(self):

if len(self.batchEvents) > 0:
headers = {'Authorization': 'Splunk ' + self.token}
r = requests.post(self.server_uri, data=" ".join(self.batchEvents), headers=headers, verify=http_event_collector_SSL_verify)
try:
r = requests.post(self.server_uri, data=" ".join(self.batchEvents), headers=headers, verify=http_event_collector_SSL_verify, proxies=self.proxy, timeout=self.timeout)
except requests.exceptions.Timeout:
log.error('Request to splunk timed out. Not retrying.')
self.batchEvents = []
self.currentByteLength = 0
Loading

0 comments on commit 8e6c239

Please sign in to comment.