From 0ce85d8ff7e869aa6d30be99e2c02735d4226b72 Mon Sep 17 00:00:00 2001 From: don sizemore Date: Mon, 29 Feb 2016 16:23:51 -0500 Subject: [PATCH] #15 import from cyverse --- ansible/playbooks/support-filebeat.yaml | 113 ++++++++++++++++++ ansible/playbooks/support-logstash-cfg.yaml | 27 +++++ ansible/roles/support-filebeat/.travis.yml | 29 +++++ ansible/roles/support-filebeat/README.md | 45 +++++++ .../roles/support-filebeat/defaults/main.yaml | 6 + .../roles/support-filebeat/files/beats.repo | 7 ++ .../roles/support-filebeat/handlers/main.yaml | 6 + ansible/roles/support-filebeat/meta/main.yaml | 37 ++++++ .../roles/support-filebeat/tasks/CentOS.yaml | 19 +++ .../roles/support-filebeat/tasks/Ubuntu.yaml | 22 ++++ .../roles/support-filebeat/tasks/check.yaml | 17 +++ .../roles/support-filebeat/tasks/main.yaml | 29 +++++ .../templates/filebeat.yml.j2 | 3 + .../roles/support-filebeat/tests/inventory | 1 + .../roles/support-filebeat/tests/test.yaml | 5 + .../roles/support-logstash-cfg/.travis.yml | 29 +++++ ansible/roles/support-logstash-cfg/README.md | 51 ++++++++ .../support-logstash-cfg/files/10-de-log.conf | 36 ++++++ .../files/12-timestamp.conf | 24 ++++ .../files/13-embed-requests.conf | 61 ++++++++++ .../files/14-embed-responses.conf | 58 +++++++++ .../files/15-embed-user.conf | 10 ++ .../files/17-embed-exception.conf | 10 ++ .../files/31-embed-analysis.conf | 22 ++++ ...rge-requests-and-responses.conf.DO_NOT_USE | 11 ++ .../20-clone-to-error.conf.DO_NOT_USE | 6 + .../21-clone-to-metrics.conf.DO_NOT_USE | 16 +++ .../22-clone-by-metric-type.conf.DO_NOT_USE | 21 ++++ .../30-embed-app.conf.DO_NOT_USE | 20 ++++ .../roles/support-logstash-cfg/meta/main.yaml | 23 ++++ .../templates/01-lumberjack-input.conf | 8 ++ .../templates/50-lumberjack-output.conf | 19 +++ .../support-logstash-cfg/tests/inventory | 1 + .../support-logstash-cfg/tests/test.yaml | 5 + ansible/roles/support-logstash/.travis.yml | 29 +++++ ansible/roles/support-logstash/README.md | 41 +++++++ .../roles/support-logstash/defaults/main.yaml | 11 ++ .../support-logstash/files/logstash.repo | 7 ++ .../roles/support-logstash/handlers/main.yaml | 7 ++ ansible/roles/support-logstash/meta/main.yaml | 19 +++ .../roles/support-logstash/tasks/CentOS.yaml | 19 +++ .../roles/support-logstash/tasks/Ubuntu.yaml | 21 ++++ .../roles/support-logstash/tasks/check.yaml | 7 ++ .../roles/support-logstash/tasks/geoIP.yaml | 21 ++++ .../roles/support-logstash/tasks/main.yaml | 56 +++++++++ .../roles/support-logstash/tests/inventory | 1 + .../roles/support-logstash/tests/test.yaml | 5 + 47 files changed, 1041 insertions(+) create mode 100644 ansible/playbooks/support-filebeat.yaml create mode 100644 ansible/playbooks/support-logstash-cfg.yaml create mode 100644 ansible/roles/support-filebeat/.travis.yml create mode 100644 ansible/roles/support-filebeat/README.md create mode 100644 ansible/roles/support-filebeat/defaults/main.yaml create mode 100644 ansible/roles/support-filebeat/files/beats.repo create mode 100644 ansible/roles/support-filebeat/handlers/main.yaml create mode 100644 ansible/roles/support-filebeat/meta/main.yaml create mode 100644 ansible/roles/support-filebeat/tasks/CentOS.yaml create mode 100644 ansible/roles/support-filebeat/tasks/Ubuntu.yaml create mode 100644 ansible/roles/support-filebeat/tasks/check.yaml create mode 100644 ansible/roles/support-filebeat/tasks/main.yaml create mode 100644 ansible/roles/support-filebeat/templates/filebeat.yml.j2 create mode 100644 ansible/roles/support-filebeat/tests/inventory create mode 100644 ansible/roles/support-filebeat/tests/test.yaml create mode 100644 ansible/roles/support-logstash-cfg/.travis.yml create mode 100644 ansible/roles/support-logstash-cfg/README.md create mode 100644 ansible/roles/support-logstash-cfg/files/10-de-log.conf create mode 100644 ansible/roles/support-logstash-cfg/files/12-timestamp.conf create mode 100644 ansible/roles/support-logstash-cfg/files/13-embed-requests.conf create mode 100644 ansible/roles/support-logstash-cfg/files/14-embed-responses.conf create mode 100644 ansible/roles/support-logstash-cfg/files/15-embed-user.conf create mode 100644 ansible/roles/support-logstash-cfg/files/17-embed-exception.conf create mode 100644 ansible/roles/support-logstash-cfg/files/31-embed-analysis.conf create mode 100644 ansible/roles/support-logstash-cfg/filter-sandbox/16-merge-requests-and-responses.conf.DO_NOT_USE create mode 100644 ansible/roles/support-logstash-cfg/filter-sandbox/20-clone-to-error.conf.DO_NOT_USE create mode 100644 ansible/roles/support-logstash-cfg/filter-sandbox/21-clone-to-metrics.conf.DO_NOT_USE create mode 100644 ansible/roles/support-logstash-cfg/filter-sandbox/22-clone-by-metric-type.conf.DO_NOT_USE create mode 100644 ansible/roles/support-logstash-cfg/filter-sandbox/30-embed-app.conf.DO_NOT_USE create mode 100644 ansible/roles/support-logstash-cfg/meta/main.yaml create mode 100644 ansible/roles/support-logstash-cfg/templates/01-lumberjack-input.conf create mode 100644 ansible/roles/support-logstash-cfg/templates/50-lumberjack-output.conf create mode 100644 ansible/roles/support-logstash-cfg/tests/inventory create mode 100644 ansible/roles/support-logstash-cfg/tests/test.yaml create mode 100644 ansible/roles/support-logstash/.travis.yml create mode 100644 ansible/roles/support-logstash/README.md create mode 100644 ansible/roles/support-logstash/defaults/main.yaml create mode 100644 ansible/roles/support-logstash/files/logstash.repo create mode 100644 ansible/roles/support-logstash/handlers/main.yaml create mode 100644 ansible/roles/support-logstash/meta/main.yaml create mode 100644 ansible/roles/support-logstash/tasks/CentOS.yaml create mode 100644 ansible/roles/support-logstash/tasks/Ubuntu.yaml create mode 100644 ansible/roles/support-logstash/tasks/check.yaml create mode 100644 ansible/roles/support-logstash/tasks/geoIP.yaml create mode 100644 ansible/roles/support-logstash/tasks/main.yaml create mode 100644 ansible/roles/support-logstash/tests/inventory create mode 100644 ansible/roles/support-logstash/tests/test.yaml diff --git a/ansible/playbooks/support-filebeat.yaml b/ansible/playbooks/support-filebeat.yaml new file mode 100644 index 000000000..1cc4b214d --- /dev/null +++ b/ansible/playbooks/support-filebeat.yaml @@ -0,0 +1,113 @@ +# Installs filebeat to all managed systems in inventory +# +--- +- hosts: services:ui:&systems + become: true + vars: + ssl_docker: + image: "{{docker.registry.base}}/{{data_container.image_name}}" + container: "{{data_container.container_name}}" + cert_path: "{{logstash.ssl.cert}}" + roles: + - role: infra-import-trusted-ca-cert + +- hosts: services:ui:&systems + become: true + vars: + filebeat_logstash_host: "{{groups['elasticsearch'][0]}}:{{logstash.port}}" + filebeat_index_pattern: "de-filebeat" + filebeat_config: + filebeat: + prospectors: + - paths: + - "/var/log/de/anon-files-docker.log" + document_type: "anon-files-log" + - paths: + - "/var/log/de/clockwork-docker.log" + document_type: "clockwork-log" + - paths: + - "/var/log/de/condor-log-monitor-docker.log" + document_type: "condor-log-monitor-log" + - paths: + - "/var/log/de/data-info-docker.log" + document_type: "data-info-log" + - paths: + - "/var/log/de/nginx-de-ui.log" + document_type: "de-ui-nginx-log" + - paths: + - "/var/log/de/de-ui.log" + document_type: "ui-log" + - paths: + - "/var/log/de/dewey-docker.log" + document_type: "dewey-log" + - paths: + - "/var/log/de/terrain-docker.log" + document_type: "terrain-log" + - paths: + - "/var/log/de/exim-docker.log" + document_type: "exim-sender-log" + - paths: + - "/var/log/de/infosquito-docker.log" + document_type: "infosquito-log" + - paths: + - "/var/log/de/info-typer-docker.log" + document_type: "info-typer-log" + - paths: + - "/var/log/de/iplant-email-docker.log" + document_type: "iplant-email-log" + - paths: + - "/var/log/de/iplant-groups-docker.log" + document_type: "iplant-groups-log" + - paths: + - "/var/log/de/jex/jex.log" + document_type: "jex-log" + - paths: + - "/var/log/de/jex-events-docker.log" + document_type: "jexevents-log" + - paths: + - "/var/log/de/kifshare-docker.log" + document_type: "kifshare-log" + - paths: + - "/var/log/de/apps-docker.log" + document_type: "apps-log" + - paths: + - "/var/log/de/metadata-docker.log" + document_type: "metadata-log" + - paths: + - "/var/log/de/monkey-docker.log" + document_type: "monkey-log" + - paths: + - "/var/log/de/notificationagent-docker.log" + document_type: "notificationagent-log" + - paths: + - "/var/log/de/saved-searches-docker.log" + document_type: "saved-searches-log" + - paths: + - "/var/log/de/tree-urls-docker.log" + document_type: "tree-urls-log" + - paths: + - "/var/log/de/user-preferences-docker.log" + document_type: "user-preferences-log" + - paths: + - "/var/log/de/user-sessions-docker.log" + document_type: "user-sessions-log" + output: + logstash: + hosts: + - "{{filebeat_logstash_host}}" + tls: + certificate_authorities: [] + index: "{{filebeat_index_pattern}}" + shipper: + name: "{{ansible_hostname}}" + tags: + - "{{ inventory_file.split('/')[-1].split('.')[0] }}" + logging: + to_files: true + level: info + files: + path: /var/log/filebeat + name: filebeat + rotateeverybytes: 10485760 + roles: + - role: support-filebeat diff --git a/ansible/playbooks/support-logstash-cfg.yaml b/ansible/playbooks/support-logstash-cfg.yaml new file mode 100644 index 000000000..4a62d4b48 --- /dev/null +++ b/ansible/playbooks/support-logstash-cfg.yaml @@ -0,0 +1,27 @@ +# Installs logstash to the elasticsearch client node +# +--- +- hosts: elasticsearch[0]:&systems + become: true + vars: + ssl_docker: + image: "{{docker.registry.base}}/{{data_container.image_name}}" + container: "{{data_container.container_name}}" + cert_path: "{{logstash.ssl.cert}}" + key_path: "{{logstash.ssl.key}}" + ssl_dest: + cert_path: "{{logstash.ssl.cert}}" + key_path: "{{logstash.ssl.key}}" + owner: "logstash" + group: "logstash" + roles: + - role: infra-place-ssl-keys + +- hosts: elasticsearch[0]:&systems + become: true + vars: + logstash_elasticsearch_host: "{{groups['elasticsearch'][0]}}:{{elasticsearch.network_http_port}}" + logstash_plugins: + - plugin: logstash-input-beats + roles: + - role: support-logstash-cfg diff --git a/ansible/roles/support-filebeat/.travis.yml b/ansible/roles/support-filebeat/.travis.yml new file mode 100644 index 000000000..36bbf6208 --- /dev/null +++ b/ansible/roles/support-filebeat/.travis.yml @@ -0,0 +1,29 @@ +--- +language: python +python: "2.7" + +# Use the new container infrastructure +sudo: false + +# Install ansible +addons: + apt: + packages: + - python-pip + +install: + # Install ansible + - pip install ansible + + # Check ansible version + - ansible --version + + # Create ansible.cfg with correct roles_path + - printf '[defaults]\nroles_path=../' >ansible.cfg + +script: + # Basic role syntax check + - ansible-playbook tests/test.yml -i tests/inventory --syntax-check + +notifications: + webhooks: https://galaxy.ansible.com/api/v1/notifications/ \ No newline at end of file diff --git a/ansible/roles/support-filebeat/README.md b/ansible/roles/support-filebeat/README.md new file mode 100644 index 000000000..ad5300502 --- /dev/null +++ b/ansible/roles/support-filebeat/README.md @@ -0,0 +1,45 @@ +support-filebeat +================ + +Installs and configures a Filebeat instance. + +https://www.elastic.co/guide/en/beats/filebeat/current/index.html + +Requirements +------------ + +Ansible 2.x +Requires sudo. + +Role Variables +-------------- + +| Variable | required | default | choices | comments | +|------------------------------------|----------|-------------------------|---------|--------------------------------------------------------| +| filebeat_version | no | | | When defined, will install the specified version of Filebeat, otherwise, the most current will be installed. | +| filebeat_install | no | true | | A flag used to control whether the role should perform installation steps. | +| filebeat_config | no | | | If defined, is used to populate the filebeat config file. If undefined, the config file is unchanged. | + + +Dependencies +------------ + +N/A + +Example Playbook +---------------- + + - hosts: systems + vars: + roles: + - { role: username.rolename, x: 42 } + +License +------- + +BSD + +Author Information +------------------ + +Jonathan Strootman "jstroot@cyverse.org" diff --git a/ansible/roles/support-filebeat/defaults/main.yaml b/ansible/roles/support-filebeat/defaults/main.yaml new file mode 100644 index 000000000..b15a1d97e --- /dev/null +++ b/ansible/roles/support-filebeat/defaults/main.yaml @@ -0,0 +1,6 @@ +--- +# defaults file for support-filebeat +filebeat_install: true +filebeat_apt_repo: + gpgkey: "http://packages.elastic.co/GPG-KEY-elasticsearch" + repo: "deb https://packages.elastic.co/beats/apt stable main" diff --git a/ansible/roles/support-filebeat/files/beats.repo b/ansible/roles/support-filebeat/files/beats.repo new file mode 100644 index 000000000..d9c029e61 --- /dev/null +++ b/ansible/roles/support-filebeat/files/beats.repo @@ -0,0 +1,7 @@ +[beats] +name=Elastic Beats Repository +baseurl=https://packages.elastic.co/beats/yum/el/$basearch +enabled=1 +gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch +gpgcheck=1 + diff --git a/ansible/roles/support-filebeat/handlers/main.yaml b/ansible/roles/support-filebeat/handlers/main.yaml new file mode 100644 index 000000000..472089712 --- /dev/null +++ b/ansible/roles/support-filebeat/handlers/main.yaml @@ -0,0 +1,6 @@ +--- +# handlers file for support-filebeat +- name: restart filebeat + service: + name: filebeat + state: restarted diff --git a/ansible/roles/support-filebeat/meta/main.yaml b/ansible/roles/support-filebeat/meta/main.yaml new file mode 100644 index 000000000..b66c44e12 --- /dev/null +++ b/ansible/roles/support-filebeat/meta/main.yaml @@ -0,0 +1,37 @@ +galaxy_info: + author: Jonathan Strootman + description: Installs filebeat instance + company: CyVerse + # Some suggested licenses: + # - BSD (default) + # - MIT + # - GPLv2 + # - GPLv3 + # - Apache + # - CC-BY + license: license (GPLv2, CC-BY, etc) + min_ansible_version: 1.9 + platforms: + - name: EL + versions: + - 6 + - 7 + - name: Ubuntu + versions: + - trusty + - utopic + - vivid + + galaxy_tags: [] + # List tags for your role here, one per line. A tag is + # a keyword that describes and categorizes the role. + # Users find roles by searching for tags. Be sure to + # remove the '[]' above if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of + # alphanumeric characters. Maximum 20 tags per role. + +dependencies: [] + # List your role dependencies here, one per line. + # Be sure to remove the '[]' above if you add dependencies + # to this list. diff --git a/ansible/roles/support-filebeat/tasks/CentOS.yaml b/ansible/roles/support-filebeat/tasks/CentOS.yaml new file mode 100644 index 000000000..50aca8f5e --- /dev/null +++ b/ansible/roles/support-filebeat/tasks/CentOS.yaml @@ -0,0 +1,19 @@ +--- +- name: Add Elastic repo public signing key + shell: rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch + tags: + - install + +- name: Add Beats repo + copy: src=beats.repo dest=/etc/yum.repos.d/ mode=0644 + tags: + - install + +- name: Install Filebeat + yum: + name: "filebeat{% if filebeat_version is defined %}-{{filebeat_version}}{% endif %}" + update_cache: yes + state: present + tags: + - install + diff --git a/ansible/roles/support-filebeat/tasks/Ubuntu.yaml b/ansible/roles/support-filebeat/tasks/Ubuntu.yaml new file mode 100644 index 000000000..741d420f0 --- /dev/null +++ b/ansible/roles/support-filebeat/tasks/Ubuntu.yaml @@ -0,0 +1,22 @@ +--- +- name: Add Logstash repo key + when: + apt_key: "url={{filebeat_apt_repo.gpgkey}} state=present" + tags: + - install + +- name: Add Beats repo + apt_repository: + repo: "{{filebeat_apt_repo.repo}}" + state: present + tags: + - install + +- name: Install filebeat + apt: + name: "filebeat{% if filebeat_version is defined %}-{{filebeat_version}}{% endif %}" + update_cache: yes + state: present + tags: + - install + diff --git a/ansible/roles/support-filebeat/tasks/check.yaml b/ansible/roles/support-filebeat/tasks/check.yaml new file mode 100644 index 000000000..28353c0ac --- /dev/null +++ b/ansible/roles/support-filebeat/tasks/check.yaml @@ -0,0 +1,17 @@ +--- +- debug: var=ansible_distribution + when: (ansible_distribution == "CentOS" and + ansible_distribution_major_version | version_compare('6', '<')) or + (ansible_distribution == "Ubuntu" and + ansible_distribution_major_version | version_compare('14', '<')) +- debug: var=ansible_distribution_major_version + when: (ansible_distribution == "CentOS" and + ansible_distribution_major_version | version_compare('6', '<')) or + (ansible_distribution == "Ubuntu" and + ansible_distribution_major_version | version_compare('14', '<')) +- name: Verify OS + fail: msg="This role only supported on Ubuntu vivid or CentOS 7" + failed_when: (ansible_distribution == "CentOS" and + ansible_distribution_major_version | version_compare('6', '<')) or + (ansible_distribution == "Ubuntu" and + ansible_distribution_major_version | version_compare('14', '<')) diff --git a/ansible/roles/support-filebeat/tasks/main.yaml b/ansible/roles/support-filebeat/tasks/main.yaml new file mode 100644 index 000000000..160a22d3b --- /dev/null +++ b/ansible/roles/support-filebeat/tasks/main.yaml @@ -0,0 +1,29 @@ +--- +# tasks file for support-filebeat +- include: check.yaml + +- when: filebeat_install + block: + - include: Ubuntu.yaml + when: ansible_distribution == 'Ubuntu' + - include: CentOS.yaml + when: ansible_distribution == "CentOS" + - name: Enable Filebeat at boot + service: + name: filebeat + enabled: yes + notify: restart filebeat + tags: + - install + +- debug: var=filebeat_config +- name: Configure Filebeat + when: filebeat_config is defined + template: + src: "filebeat.yml.j2" + dest: "/etc/filebeat/filebeat.yml" + notify: restart filebeat + tags: + - config + + diff --git a/ansible/roles/support-filebeat/templates/filebeat.yml.j2 b/ansible/roles/support-filebeat/templates/filebeat.yml.j2 new file mode 100644 index 000000000..e1a1fa4e8 --- /dev/null +++ b/ansible/roles/support-filebeat/templates/filebeat.yml.j2 @@ -0,0 +1,3 @@ +## {{ ansible_managed }} +--- +{{filebeat_config | to_nice_yaml }} diff --git a/ansible/roles/support-filebeat/tests/inventory b/ansible/roles/support-filebeat/tests/inventory new file mode 100644 index 000000000..d18580b3c --- /dev/null +++ b/ansible/roles/support-filebeat/tests/inventory @@ -0,0 +1 @@ +localhost \ No newline at end of file diff --git a/ansible/roles/support-filebeat/tests/test.yaml b/ansible/roles/support-filebeat/tests/test.yaml new file mode 100644 index 000000000..2b7e3a0dd --- /dev/null +++ b/ansible/roles/support-filebeat/tests/test.yaml @@ -0,0 +1,5 @@ +--- +- hosts: localhost + remote_user: root + roles: + - support-filebeat \ No newline at end of file diff --git a/ansible/roles/support-logstash-cfg/.travis.yml b/ansible/roles/support-logstash-cfg/.travis.yml new file mode 100644 index 000000000..9ec2ce58e --- /dev/null +++ b/ansible/roles/support-logstash-cfg/.travis.yml @@ -0,0 +1,29 @@ +--- +language: python +python: "2.7" + +# Use the new container infrastructure +sudo: false + +# Install ansible +addons: + apt: + packages: + - python-pip + +install: + # Install ansible + - pip install ansible + + # Check ansible version + - ansible --version + + # Create ansible.cfg with correct roles_path + - printf '[defaults]\nroles_path=../' >ansible.cfg + +script: + # Basic role syntax check + - ansible-playbook tests/test.yaml -i tests/inventory --syntax-check + +notifications: + webhooks: https://galaxy.ansible.com/api/v1/notifications/ diff --git a/ansible/roles/support-logstash-cfg/README.md b/ansible/roles/support-logstash-cfg/README.md new file mode 100644 index 000000000..821c2a05d --- /dev/null +++ b/ansible/roles/support-logstash-cfg/README.md @@ -0,0 +1,51 @@ +support-logstash-cfg +==================== + +Provides configuration files for deploying logstash and installs logstash on a remote host (using the support-logstash +role). + +Requirements +------------ + +This role uses the support-logstash role. + +Role Variables +-------------- + +| Variable | required | default | choices | comments | +|------------------------------------|----------|-------------------------|---------|--------------------------------------------------------| +| logstash_version | no | "2.2" | | The version of Logstash to install | +| logstash_install | no | true | | A flag used to control whether the role should perform installation steps. | +| logstash_base_dir | no | "/opt/logstash" | | Logstash's install location. | +| logstash_cfg_dir | no | "/etc/logstash/conf.d" | | Logstash's config directory. | +| logstash_clean_cfg_dir | no | true | | Determines whether the cfg dir will be cleaned prior to uploading new ones.| +| logstash_elasticsearch_host | no | "localhost:9200" | | The host name and port for the elasticsearch instance to forward messages to. | +| logstash_plugins | no | | | A list of objects representing logstash plugins to be installed. The `plugin` key is required, while the `version` key is optional. | + + +Dependencies +------------ + +This role requires the support-logstash role. + +Example Playbook +---------------- + +- hosts: logstash + become: true + vars: + logstash_elasticsearch_host: "localhost:9200" + logstash_plugins: + - plugin: logstash-input-beats + roles: + - role: support-logstash-cfg + +License +------- + +BSD + +Author Information +------------------ + +Jonathan Strootman diff --git a/ansible/roles/support-logstash-cfg/files/10-de-log.conf b/ansible/roles/support-logstash-cfg/files/10-de-log.conf new file mode 100644 index 000000000..5a0a96c9c --- /dev/null +++ b/ansible/roles/support-logstash-cfg/files/10-de-log.conf @@ -0,0 +1,36 @@ +filter { + # When events get here from the forwarders, they should be typed as "[service/component]-log". + # It is also assumed that these events will fit the syslog format, and the syslog message will + # be a JSON object. + if [type] == "docker-log" { + # Do nothing for now + } else if [type] =~ ".*-log" { + # If any part of the message contains error, tag it + if [message] =~ "[Ee][Rr]{2}[Oo][Rr]" { + mutate { add_tag => [ "error" ] } + } + grok { + match => { "message" => "%{SYSLOGBASE} %{GREEDYDATA:message.syslog}" } + } + if "_grokparsefailure" not in [tags] { + json { + source => "message.syslog" + } + } + + mutate { + # Not needed after it is parsed out + remove_field => ["message.syslog"] + remove_field => ["desc"] + # The next two always accompany the 'service' field, unnecessary duplicates. + remove_field => ["app-name"] + remove_field => ["art-id"] + remove_field => ["group-id"] + # This is not useful information. + remove_field => ["loggerFqcn"] + # This value is always blank or false, not helpful. + remove_field => ["endOfBatch"] + } + } +} + diff --git a/ansible/roles/support-logstash-cfg/files/12-timestamp.conf b/ansible/roles/support-logstash-cfg/files/12-timestamp.conf new file mode 100644 index 000000000..d2321ef20 --- /dev/null +++ b/ansible/roles/support-logstash-cfg/files/12-timestamp.conf @@ -0,0 +1,24 @@ +# TODO: Use the mappings in an ES index template to do this instead +filter { + # If the 'timeMillis' field exists, use it as the primary log timestamp + if [timeMillis] { + date { + match => ["timeMillis", "UNIX_MS"] + } + + # Remove the 'timestamp' field if it exists + mutate { + remove_field => ["timestamp"] + } + } else if [timestamp] { + date { + match => ["timestamp", "MMM dd HH:mm:ss", + "MMM d HH:mm:ss"] + } + # Remove the 'timestamp' field + mutate { + remove_field => ["timestamp"] + } + } + +} diff --git a/ansible/roles/support-logstash-cfg/files/13-embed-requests.conf b/ansible/roles/support-logstash-cfg/files/13-embed-requests.conf new file mode 100644 index 000000000..572fcc0e5 --- /dev/null +++ b/ansible/roles/support-logstash-cfg/files/13-embed-requests.conf @@ -0,0 +1,61 @@ +filter { + ## This filter is meant to clean up any existing 'request' fields, and turn + ## them into nested fields + if [request] { + json { + source => "request" + target => "request" + } + + mutate { + remove_field => ["[request][path-info]"] + } + mutate { + add_field => {"[request][path-info]" => "%{[request][uri]}"} + } + ## Extract UUID + grok { + keep_empty_captures => true + match => {"[request][path-info]" => "(?(^/(?:(?[A-Za-z0-9$.+!*'(){},~:;=#%_-]*)/)+))(?=%{UUID})(%{UUID:uuid})(%{GREEDYDATA:end})?" } + tag_on_failure => ["extract_uuid_failure"] + } + if "extract_uuid_failure" not in [tags] { + if [end] { + mutate { + replace => {"[request][path-info]" => "%{start}[UUID]%{end}"} + } + } else { + mutate { + replace => {"[request][path-info]" => "%{start}[UUID]"} + } + } + mutate { + remove_field => ["[request][route-params][%{resource}-id]"] + } + mutate { + add_field => {"[request][route-params][%{resource}-id]" => "%{uuid}"} + } + mutate { + remove_field => ["start", + "resource", + "uuid", + "end"] + } + } else { + ## Remove failure tag + mutate { + remove_tag => ["extract_uuid_failure"] + } + } + + + ## Copy Request ID to dedicated field + if [request][headers][X-DE-request-id] { + mutate { + add_field => {"[request][id]" => "%{[request][headers][X-DE-request-id]}"} + } + } + } + +} + diff --git a/ansible/roles/support-logstash-cfg/files/14-embed-responses.conf b/ansible/roles/support-logstash-cfg/files/14-embed-responses.conf new file mode 100644 index 000000000..a3f3ddcd6 --- /dev/null +++ b/ansible/roles/support-logstash-cfg/files/14-embed-responses.conf @@ -0,0 +1,58 @@ +filter { + # If there are any responses, it is expected that their bodies can be json encoded. + if [response] { + json { + source => "response" + target => "response" + } + + mutate { + remove_field => ["[response][path-info]"] + } + mutate { + add_field => {"[response][path-info]" => "%{[response][uri]}"} + } + ## Extract UUID + grok { + keep_empty_captures => true + match => {"[response][path-info]" => "(?(^/(?:(?[A-Za-z0-9$.+!*'(){},~:;=#%_-]*)/)+))(?=%{UUID})(%{UUID:uuid})(%{GREEDYDATA:end})?" } + tag_on_failure => ["extract_uuid_failure"] + } + if "extract_uuid_failure" not in [tags] { + if [end] { + mutate { + replace => {"[response][path-info]" => "%{start}[UUID]%{end}"} + } + } else { + mutate { + replace => {"[response][path-info]" => "%{start}[UUID]"} + } + } + mutate { + remove_field => ["[response][route-params][%{resource}-id]"] + } + mutate { + add_field => {"[response][route-params][%{resource}-id]" => "%{uuid}"} + } + mutate { + remove_field => ["start", + "resource", + "uuid", + "end"] + } + } else { + ## Remove failure tag + mutate { + remove_tag => ["extract_uuid_failure"] + } + } + + ## Copy Request ID to dedicated field + if [response][headers][X-DE-request-id] { + mutate { + add_field => {"[request][id]" => "%{[response][headers][X-DE-request-id]}"} + } + } + } +} + diff --git a/ansible/roles/support-logstash-cfg/files/15-embed-user.conf b/ansible/roles/support-logstash-cfg/files/15-embed-user.conf new file mode 100644 index 000000000..830fcc415 --- /dev/null +++ b/ansible/roles/support-logstash-cfg/files/15-embed-user.conf @@ -0,0 +1,10 @@ +filter { + # If there is a 'user-info' key, it is expected that it is an encodable JSON body. + if [user-info] { + json { + source => "user-info" + target => "user-info" + } + } +} + diff --git a/ansible/roles/support-logstash-cfg/files/17-embed-exception.conf b/ansible/roles/support-logstash-cfg/files/17-embed-exception.conf new file mode 100644 index 000000000..3f6b27cc5 --- /dev/null +++ b/ansible/roles/support-logstash-cfg/files/17-embed-exception.conf @@ -0,0 +1,10 @@ +filter { + # If there is an 'exception' key, it is expected that it is an encodable JSON body. + if [exception] { + json { + source => "exception" + target => "exception" + } + } +} + diff --git a/ansible/roles/support-logstash-cfg/files/31-embed-analysis.conf b/ansible/roles/support-logstash-cfg/files/31-embed-analysis.conf new file mode 100644 index 000000000..b029f7aa1 --- /dev/null +++ b/ansible/roles/support-logstash-cfg/files/31-embed-analysis.conf @@ -0,0 +1,22 @@ +filter { + if [analysis.id] { + mutate { + add_field => { "[analysis][id]" => "%{[analysis.id]}" } + remove_field => ["analysis.id"] + } + } + if [analysis.name] { + mutate { + add_field => { "[analysis][name]" => "%{[analysis.name]}" } + remove_field => ["analysis.name"] + } + } + if [analysis.output_dir] { + mutate { + add_field => { "[analysis][output_dir]" => "%{[analysis.output_dir]}" } + remove_field => ["analysis.output_dir"] + } + } + +} + diff --git a/ansible/roles/support-logstash-cfg/filter-sandbox/16-merge-requests-and-responses.conf.DO_NOT_USE b/ansible/roles/support-logstash-cfg/filter-sandbox/16-merge-requests-and-responses.conf.DO_NOT_USE new file mode 100644 index 000000000..cc8c1f164 --- /dev/null +++ b/ansible/roles/support-logstash-cfg/filter-sandbox/16-merge-requests-and-responses.conf.DO_NOT_USE @@ -0,0 +1,11 @@ +## Merges request/response pairs into single event +filter { + if [request][id] { + multiline { + pattern => "." + stream_identity => "%{[request][id]}" + what => "previous" + } + } +} + diff --git a/ansible/roles/support-logstash-cfg/filter-sandbox/20-clone-to-error.conf.DO_NOT_USE b/ansible/roles/support-logstash-cfg/filter-sandbox/20-clone-to-error.conf.DO_NOT_USE new file mode 100644 index 000000000..d24b7dd95 --- /dev/null +++ b/ansible/roles/support-logstash-cfg/filter-sandbox/20-clone-to-error.conf.DO_NOT_USE @@ -0,0 +1,6 @@ +# If something is tagged as "error", clone it and cleanup undesirable fields +filter { + if "error" in [tags] { + clone { clones => ["de-error"] } + } +} diff --git a/ansible/roles/support-logstash-cfg/filter-sandbox/21-clone-to-metrics.conf.DO_NOT_USE b/ansible/roles/support-logstash-cfg/filter-sandbox/21-clone-to-metrics.conf.DO_NOT_USE new file mode 100644 index 000000000..36f4b4e93 --- /dev/null +++ b/ansible/roles/support-logstash-cfg/filter-sandbox/21-clone-to-metrics.conf.DO_NOT_USE @@ -0,0 +1,16 @@ +# If something is tagged as "metrics", clone it and cleanup undesirable fields +filter { + if "metrics" in [tags] { + clone { clones => ["metrics"] } + mutate { + remove_field => ["file", + "host", + "loggerName", + "offset", + "pid", + "program"] + } + + } + +} diff --git a/ansible/roles/support-logstash-cfg/filter-sandbox/22-clone-by-metric-type.conf.DO_NOT_USE b/ansible/roles/support-logstash-cfg/filter-sandbox/22-clone-by-metric-type.conf.DO_NOT_USE new file mode 100644 index 000000000..22256d827 --- /dev/null +++ b/ansible/roles/support-logstash-cfg/filter-sandbox/22-clone-by-metric-type.conf.DO_NOT_USE @@ -0,0 +1,21 @@ +# If there is a "metric_type" field, use it to clone the event to the specified type, +# then cleanup undesirable fields. +filter { + if [metric_type] { + if [metric_type] == "app_event" { + clone { clones => ["app_event"] } + } else if [metric_type] == "share_event" { + clone { clones => ["share_event"] } + } + mutate { + remove_field => ["metric_type", + "file", + "host", + "loggerName", + "offset", + "pid", + "program"] + } + } +} + diff --git a/ansible/roles/support-logstash-cfg/filter-sandbox/30-embed-app.conf.DO_NOT_USE b/ansible/roles/support-logstash-cfg/filter-sandbox/30-embed-app.conf.DO_NOT_USE new file mode 100644 index 000000000..3a11d0dfc --- /dev/null +++ b/ansible/roles/support-logstash-cfg/filter-sandbox/30-embed-app.conf.DO_NOT_USE @@ -0,0 +1,20 @@ +filter { + # Lookup App info by id + if [app.id] { + elasticsearch { + hosts => ["elasticsearch"] + sort => "name:desc" + query => "_type:app AND _id:%{[app.id]}" + + fields => ["name", "[app][name]"] + fields => ["description", "[app][description]"] + fields => ["integrator_name", "[app][integrator_name]"] + fields => ["step_count", "[app][step_count]"] + add_field => { + "[app][id]" => "%{[app.id]}" + } + remove_field => ["app.id"] + } + } +} + diff --git a/ansible/roles/support-logstash-cfg/meta/main.yaml b/ansible/roles/support-logstash-cfg/meta/main.yaml new file mode 100644 index 000000000..27d72e669 --- /dev/null +++ b/ansible/roles/support-logstash-cfg/meta/main.yaml @@ -0,0 +1,23 @@ +--- +galaxy_info: + author: Jonathan Strootman + description: + company: CyVerse + license: BSD + + min_ansible_version: 2.0 + platforms: + - name: EL + versions: + - 7 + - name: Ubuntu + versions: + - trusty + - utopic + - vivid + galaxy_tags: [] + +dependencies: + - role: support-logstash + logstash_cfg_file_glob: "../support-logstash-cfg/files/*.conf" + logstash_cfg_template_glob: "../support-logstash-cfg/templates/*.conf" diff --git a/ansible/roles/support-logstash-cfg/templates/01-lumberjack-input.conf b/ansible/roles/support-logstash-cfg/templates/01-lumberjack-input.conf new file mode 100644 index 000000000..391ef1df9 --- /dev/null +++ b/ansible/roles/support-logstash-cfg/templates/01-lumberjack-input.conf @@ -0,0 +1,8 @@ +input { + beats { + port => {{logstash.port}} + ssl => true + ssl_certificate => "{{ logstash.ssl.cert }}" + ssl_key => "{{ logstash.ssl.key }}" + } +} diff --git a/ansible/roles/support-logstash-cfg/templates/50-lumberjack-output.conf b/ansible/roles/support-logstash-cfg/templates/50-lumberjack-output.conf new file mode 100644 index 000000000..cc82d699f --- /dev/null +++ b/ansible/roles/support-logstash-cfg/templates/50-lumberjack-output.conf @@ -0,0 +1,19 @@ +output { + if [service] { + # If the service field exists, use it to designate the document_type + # and index + elasticsearch { + hosts => ["{{logstash_elasticsearch_host}}"] + index => "de-%{service}-logs-%{+YYYY.MM.dd}" + document_type => "%{service}-log" + } + } else { + # If the service field doesn't exist, use the beat metadata to designate + # the document_type and index + elasticsearch { + hosts => ["{{logstash_elasticsearch_host}}"] + index => "de-%{[@metadata][type]}-%{+YYYY.MM.dd}" + document_type => "{[@metadata][type]}" + } + } +} diff --git a/ansible/roles/support-logstash-cfg/tests/inventory b/ansible/roles/support-logstash-cfg/tests/inventory new file mode 100644 index 000000000..d18580b3c --- /dev/null +++ b/ansible/roles/support-logstash-cfg/tests/inventory @@ -0,0 +1 @@ +localhost \ No newline at end of file diff --git a/ansible/roles/support-logstash-cfg/tests/test.yaml b/ansible/roles/support-logstash-cfg/tests/test.yaml new file mode 100644 index 000000000..d0588f89a --- /dev/null +++ b/ansible/roles/support-logstash-cfg/tests/test.yaml @@ -0,0 +1,5 @@ +--- +- hosts: localhost + remote_user: root + roles: + - support-logstash-cfg \ No newline at end of file diff --git a/ansible/roles/support-logstash/.travis.yml b/ansible/roles/support-logstash/.travis.yml new file mode 100644 index 000000000..9ec2ce58e --- /dev/null +++ b/ansible/roles/support-logstash/.travis.yml @@ -0,0 +1,29 @@ +--- +language: python +python: "2.7" + +# Use the new container infrastructure +sudo: false + +# Install ansible +addons: + apt: + packages: + - python-pip + +install: + # Install ansible + - pip install ansible + + # Check ansible version + - ansible --version + + # Create ansible.cfg with correct roles_path + - printf '[defaults]\nroles_path=../' >ansible.cfg + +script: + # Basic role syntax check + - ansible-playbook tests/test.yaml -i tests/inventory --syntax-check + +notifications: + webhooks: https://galaxy.ansible.com/api/v1/notifications/ diff --git a/ansible/roles/support-logstash/README.md b/ansible/roles/support-logstash/README.md new file mode 100644 index 000000000..120f7e684 --- /dev/null +++ b/ansible/roles/support-logstash/README.md @@ -0,0 +1,41 @@ +support-logstash +================ + +Installs logstash on a remote host. + +Requirements +------------ + +Requires become. + +Role Variables +-------------- + +| Variable | required | default | choices | comments | +|------------------------------------|----------|-------------------------|---------|--------------------------------------------------------| +| logstash_version | no | "2.2" | | The version of Logstash to install | +| logstash_install | no | true | | A flag used to control whether the role should perform installation steps. | +| logstash_base_dir | no | "/opt/logstash" | | Logstash's install location. | +| logstash_cfg_dir | no | "/etc/logstash/conf.d" | | Logstash's config directory. | +| logstash_clean_cfg_dir | no | true | | Determines whether the cfg dir will be cleaned prior to uploading new ones.| +| logstash_elasticsearch_host | no | "localhost:9200" | | The host name and port for the elasticsearch instance to forward messages to. | +| logstash_cfg_template_glob | no | | | Optionally specify a glob pattern to a directory containing template config files.[1] | +| logstash_cfg_file_glob | no | | | Optionally specify a glob pattern to a directory containing static config files.[1] | +| logstash_plugins | no | | | A list of objects representing logstash plugins to be installed. The `plugin` key is required, while the `version` key is optional. | + +[1] - http://docs.ansible.com/ansible/playbooks_loops.html#id4 + +Dependencies +------------ + +There are no external dependencies at this time. + +License +------- + +BSD + +Author Information +------------------ + +Jonathan Strootman diff --git a/ansible/roles/support-logstash/defaults/main.yaml b/ansible/roles/support-logstash/defaults/main.yaml new file mode 100644 index 000000000..da279b2d5 --- /dev/null +++ b/ansible/roles/support-logstash/defaults/main.yaml @@ -0,0 +1,11 @@ +--- +# defaults file for support-logstash +logstash_version: "2.2" +logstash_install: true +logstash_elasticsearch_host: "localhost:9200" +logstash_base_dir: "/opt/logstash" +logstash_cfg_dir: "/etc/logstash/conf.d" +logstash_clean_cfg_dir: true +logstash_apt_repo: + gpgkey: "http://packages.elastic.co/GPG-KEY-elasticsearch" + repo: "deb http://packages.elastic.co/logstash/2.2/debian stable main" diff --git a/ansible/roles/support-logstash/files/logstash.repo b/ansible/roles/support-logstash/files/logstash.repo new file mode 100644 index 000000000..44f44d02a --- /dev/null +++ b/ansible/roles/support-logstash/files/logstash.repo @@ -0,0 +1,7 @@ +[logstash-2.2] +name=Logstash repository for 2.2.x packages +baseurl=http://packages.elastic.co/logstash/2.2/centos +gpgcheck=1 +gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch +enabled=1 + diff --git a/ansible/roles/support-logstash/handlers/main.yaml b/ansible/roles/support-logstash/handlers/main.yaml new file mode 100644 index 000000000..0f815436a --- /dev/null +++ b/ansible/roles/support-logstash/handlers/main.yaml @@ -0,0 +1,7 @@ +--- +# handlers file for support-logstash +- name: restart logstash + service: + name: logstash + state: restarted + diff --git a/ansible/roles/support-logstash/meta/main.yaml b/ansible/roles/support-logstash/meta/main.yaml new file mode 100644 index 000000000..332c9e77f --- /dev/null +++ b/ansible/roles/support-logstash/meta/main.yaml @@ -0,0 +1,19 @@ +galaxy_info: + author: Jonathan Strootman + description: + company: CyVerse + license: BSD + + min_ansible_version: 2.0 + platforms: + - name: EL + versions: + - 7 + - name: Ubuntu + versions: + - trusty + - utopic + - vivid + galaxy_tags: [] + +dependencies: [] diff --git a/ansible/roles/support-logstash/tasks/CentOS.yaml b/ansible/roles/support-logstash/tasks/CentOS.yaml new file mode 100644 index 000000000..52c10be2e --- /dev/null +++ b/ansible/roles/support-logstash/tasks/CentOS.yaml @@ -0,0 +1,19 @@ +--- +- name: Add Elastic repo public signing key + rpm_key: state=present key=https://packages.elastic.co/GPG-KEY-elasticsearch + tags: + - install + +- name: Add Logstash repo + copy: src=logstash.repo dest=/etc/yum.repos.d/ mode=0644 + tags: + - install + +- name: Install logstash + yum: + name: logstash + update_cache: yes + state: present + tags: + - install + diff --git a/ansible/roles/support-logstash/tasks/Ubuntu.yaml b/ansible/roles/support-logstash/tasks/Ubuntu.yaml new file mode 100644 index 000000000..9d3d8f791 --- /dev/null +++ b/ansible/roles/support-logstash/tasks/Ubuntu.yaml @@ -0,0 +1,21 @@ +--- +- name: Add Logstash repo key + when: + apt_key: "url={{logstash_apt_repo.gpgkey}} state=present" + tags: + - install + +- name: Add Logstash repo + apt_repository: + repo: "{{logstash_apt_repo.repo}}" + state: present + tags: + - install + +- name: Install logstash + apt: + name: logstash + state: present + tags: + - install + diff --git a/ansible/roles/support-logstash/tasks/check.yaml b/ansible/roles/support-logstash/tasks/check.yaml new file mode 100644 index 000000000..8825fcf5e --- /dev/null +++ b/ansible/roles/support-logstash/tasks/check.yaml @@ -0,0 +1,7 @@ +--- +- name: Config directory exists + file: + path: "{{ logstash_cfg_dir }}" + state: directory + + diff --git a/ansible/roles/support-logstash/tasks/geoIP.yaml b/ansible/roles/support-logstash/tasks/geoIP.yaml new file mode 100644 index 000000000..6b37a0a0a --- /dev/null +++ b/ansible/roles/support-logstash/tasks/geoIP.yaml @@ -0,0 +1,21 @@ +# Install geoip database +# TODO: Change this to its own role. Re-use. +--- +- name: Download GeoLite City database + get_url: url=http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz dest=/tmp + register: download_geolite + tags: + - geoip + +- name: Ensure that GeoIP folder exists + file: path=/usr/share/GeoIP state=directory mode=0755 + tags: + - geoip + +- name: Unpack GeoLite database + shell: "gunzip {{download_geolite.dest}} && mv /tmp/GeoLiteCity.dat /usr/share/GeoIP/" + args: + creates: /usr/share/GeoIP/GeoLiteCity.dat + tags: + - geoip + diff --git a/ansible/roles/support-logstash/tasks/main.yaml b/ansible/roles/support-logstash/tasks/main.yaml new file mode 100644 index 000000000..8ffba19fd --- /dev/null +++ b/ansible/roles/support-logstash/tasks/main.yaml @@ -0,0 +1,56 @@ +--- +# tasks file for support-logstash +- include: check.yaml + +- when: logstash_install + block: + - include: Ubuntu.yaml + when: ansible_distribution == 'Ubuntu' + - include: CentOS.yaml + when: ansible_distribution == "CentOS" + - include: geoIP.yaml + +- name: Install logstash plugins + when: logstash_plugins is defined + command: "{{logstash_base_dir}}/bin/plugin install {{ item.plugin }}{% if item.version is defined and item.version != '' %}/{{ item.version }}{% endif %}" + register: plugin_installed + failed_when: "'Failed to install' in plugin_installed.stderr" + changed_when: plugin_installed.rc == 0 + with_items: logstash_plugins | default([]) + tags: + - plugins + notify: + - restart logstash + +- when: logstash_clean_cfg_dir + block: + - name: Clean configuration directory + file: + path: "{{ logstash_cfg_dir }}" + state: absent + - name: Create configuration directory + file: + path: "{{ logstash_cfg_dir }}" + state: directory + owner: logstash + group: logstash + mode: 0700 + +- name: Add configuration files + when: logstash_cfg_file_glob is defined + copy: src={{ item }} dest={{ logstash_cfg_dir }} owner=logstash group=logstash mode=0600 + with_fileglob: logstash_cfg_file_glob | default("") + tags: + - config + notify: + - restart logstash + +- name: Add configuration template files + when: logstash_cfg_template_glob is defined + template: src={{ item }} dest={{ logstash_cfg_dir }} owner=logstash group=logstash mode=0600 + with_fileglob: logstash_cfg_template_glob | default("") + tags: + - config + notify: + - restart logstash + diff --git a/ansible/roles/support-logstash/tests/inventory b/ansible/roles/support-logstash/tests/inventory new file mode 100644 index 000000000..d18580b3c --- /dev/null +++ b/ansible/roles/support-logstash/tests/inventory @@ -0,0 +1 @@ +localhost \ No newline at end of file diff --git a/ansible/roles/support-logstash/tests/test.yaml b/ansible/roles/support-logstash/tests/test.yaml new file mode 100644 index 000000000..639b0b805 --- /dev/null +++ b/ansible/roles/support-logstash/tests/test.yaml @@ -0,0 +1,5 @@ +--- +- hosts: localhost + remote_user: root + roles: + - support-logstash \ No newline at end of file