From 28673fdf7a1fac16395361d6c6c98dc6e93ab094 Mon Sep 17 00:00:00 2001 From: Joseph McCullough Date: Mon, 7 Mar 2016 13:52:43 -0600 Subject: [PATCH 1/8] Add datadog role --- group_vars/all/vault | 31 +++++++++++++++------------ group_vars/dd-agent-nodes | 1 + replacements | 6 +++++- roles/dd-agent/tasks/main.yml | 40 +++++++++++++++++++++++++++++++++++ site-infrastructure.yml | 7 ++++++ 5 files changed, 70 insertions(+), 15 deletions(-) create mode 100644 group_vars/dd-agent-nodes create mode 100644 roles/dd-agent/tasks/main.yml diff --git a/group_vars/all/vault b/group_vars/all/vault index eaf9b34..81d064d 100644 --- a/group_vars/all/vault +++ b/group_vars/all/vault @@ -1,15 +1,18 @@ $ANSIBLE_VAULT;1.1;AES256 -38383233633833303633626534333435336339663534316534646262633039376435663531393332 -6166383764356639396438323439623139333930386463640a313166346335643630326664336664 -36363932616632366433303833666362636637316265386466633931626633313632646533346564 -3765383731393231350a636539343866363130313136646564326461383032373238373035393031 -33623935376139653763363735326130376663306636316262646362343065353639633664346263 -66346230633439616464653535653063643733373566386463626366393631613534386630336430 -66356332636238643438333332386261363364613761616233353533653335653836333565333736 -33323961666564386433363961326330346230636162346566643336386666333436643232393333 -30396334616238393037323865656437356336353263633762386533306238626661666637306463 -35356238323834326439653037666632643139663739343661396666363561666530303139626433 -62373461623035623065386438636234663766396431373333653865613533353632333536666531 -36396162646139363031363837653336333733623365303965333361616239393662313536356539 -35373636633930653030613463363965396261353066353462633039333338373138613630333432 -3130613836636639316266323230383037643738353739626361 +62373630323933386334356366386230663337343766336466316138636636373937383932336434 +3131653734626235333637653932326632633633636334650a386437666535376432333935313564 +39316433643734616666303538313437613534663432373764376334363839323633323364623539 +6236663538383864660a393936386561313633303037366439616432623730643665336362653931 +62616563353931323538323161316535656262643534366437376364633737366637653332623437 +33363265613162363239343139326530626339623333613336386638333932643330313062306339 +61653836373237383736303861323465613335313065313733616535316365633834333132393263 +30326366636532306161383131613930323330333862313663613435396362393464393062313862 +65613235303163386663346137326432346662373864333338666332623932306161656465356230 +34323561643937616166653333643264663435303131333139623962343937366566643161626230 +36373061643334643862346134396232353465646339326531333062363537653637396333363131 +62336430613364386136643038303033303639316238613764646537646332323064646438663730 +37383236343338396336333938366235363437653862363761396230333563333832616165643234 +66656636323530373964376266633534653266653034343837396162303035633963313632666136 +66313131613031373636363131343764623235633463376365306362383466363238386639663130 +36643361626265633166303530616363623464623036316130373061306436663239346661383531 +31303734653933373564383930336261653664306535653662386432626364393436 diff --git a/group_vars/dd-agent-nodes b/group_vars/dd-agent-nodes new file mode 100644 index 0000000..297601f --- /dev/null +++ b/group_vars/dd-agent-nodes @@ -0,0 +1 @@ +datadog_api_key: "{{ vault_datadog_api_key }}" diff --git a/replacements b/replacements index d36177a..3de573a 100644 --- a/replacements +++ b/replacements @@ -8,6 +8,11 @@ [all:vars] repository_host=repo.istresearch.com +[dd-agent-nodes:children] +internap +aws +other + # # Memex infrastructure and software topology # @@ -193,7 +198,6 @@ memex-crawler06.istresearch.com traptor-follow-nodes traptor-track-nodes - [other] in-namenode01.nj.istresearch.com memexdev.istresearch.com diff --git a/roles/dd-agent/tasks/main.yml b/roles/dd-agent/tasks/main.yml new file mode 100644 index 0000000..0884100 --- /dev/null +++ b/roles/dd-agent/tasks/main.yml @@ -0,0 +1,40 @@ +--- +# file: roles/dd-agent/tasks/main.yml +# dd-agent install: https://app.datadoghq.com/account/settings#agent/ubuntu + +- name: Install dependencies for dd-agent + apt: + name: apt-transport-https + state: present + update_cache: yes + +- name: Add dd-agent repository to source list + apt_repository: + repo: 'deb http://apt.datadoghq.com/ stable main' + state: present + +- name: Import dd-agent apt key + apt_key: + keyserver: keyserver.ubuntu.com + id: C7A7DA52 + +- name: Install dd-agent + apt: + name: datadog-agent + state: present + update_cache: yes + +- name: Copy default dd-agent config + command: cp -n /etc/dd-agent/datadog.conf.example /etc/dd-agent/datadog.conf + +- name: Alter dd-agent config to include API Key + lineinfile: + dest: /etc/dd-agent/datadog.conf + state: present + regexp: "^api_key:" + line: "api_key: {{ datadog_api_key }}" + +- name: Start dd-agent + service: + name: datadog-agent + state: started diff --git a/site-infrastructure.yml b/site-infrastructure.yml index 8589713..96cac08 100644 --- a/site-infrastructure.yml +++ b/site-infrastructure.yml @@ -381,3 +381,10 @@ roles: [ tangelo ] tags: - site-tangelo + +# ---------------------------------- DataDog -------------------------------- + +- hosts: dd-agent-nodes + roles: [ dd-agent ] + tags: + - site-dd-agent From 2cf5d292985fb8b8341753dd72446a1982537030 Mon Sep 17 00:00:00 2001 From: Joseph McCullough Date: Fri, 25 Mar 2016 14:30:45 -0500 Subject: [PATCH 2/8] Move ddagent under supervisor --- roles/dd-agent/defaults/main.yml | 4 ++++ roles/dd-agent/tasks/main.yml | 18 ++++++++++++++---- .../templates/dd-agent-supervisord.conf.j2 | 6 ++++++ 3 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 roles/dd-agent/defaults/main.yml create mode 100644 roles/dd-agent/templates/dd-agent-supervisord.conf.j2 diff --git a/roles/dd-agent/defaults/main.yml b/roles/dd-agent/defaults/main.yml new file mode 100644 index 0000000..15a6188 --- /dev/null +++ b/roles/dd-agent/defaults/main.yml @@ -0,0 +1,4 @@ +--- + +ddagent_install_dir: /usr/bin +ddagent_conf_path: /etc/dd-agent/datadog.conf diff --git a/roles/dd-agent/tasks/main.yml b/roles/dd-agent/tasks/main.yml index 0884100..518a57d 100644 --- a/roles/dd-agent/tasks/main.yml +++ b/roles/dd-agent/tasks/main.yml @@ -25,16 +25,26 @@ update_cache: yes - name: Copy default dd-agent config - command: cp -n /etc/dd-agent/datadog.conf.example /etc/dd-agent/datadog.conf + command: cp -n {{ ddagent_conf_path }}.example {{ ddagent_conf_path }} - name: Alter dd-agent config to include API Key lineinfile: - dest: /etc/dd-agent/datadog.conf + dest: {{ ddagent_conf_path }} state: present regexp: "^api_key:" line: "api_key: {{ datadog_api_key }}" -- name: Start dd-agent +# dd-agent was previously just run as a service before being run under +# supervisor. +- name: Stop service based dd-agent service: name: datadog-agent - state: started + state: stopped + +- name: Copy ddagent supervisord config + template: + src=dd-agent-supervisord.conf.j2 + dest={{ supervisord_programs_dir }}/ddagent-supervisord.conf + mode=0644 + notify: + - reread supervisord diff --git a/roles/dd-agent/templates/dd-agent-supervisord.conf.j2 b/roles/dd-agent/templates/dd-agent-supervisord.conf.j2 new file mode 100644 index 0000000..d560d3a --- /dev/null +++ b/roles/dd-agent/templates/dd-agent-supervisord.conf.j2 @@ -0,0 +1,6 @@ +[program:dd-agent] +command={{ ddagent_install_dir }}/dd-agent foreground +autostart=true +autorestart=true +startsecs=5 +stopsignal=KILL From 9e21546484444ebf41823a305fa9b0964d9a2674 Mon Sep 17 00:00:00 2001 From: Joseph McCullough Date: Fri, 25 Mar 2016 14:45:09 -0500 Subject: [PATCH 3/8] Fix task variable error --- roles/dd-agent/tasks/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/dd-agent/tasks/main.yml b/roles/dd-agent/tasks/main.yml index 518a57d..180a018 100644 --- a/roles/dd-agent/tasks/main.yml +++ b/roles/dd-agent/tasks/main.yml @@ -25,11 +25,11 @@ update_cache: yes - name: Copy default dd-agent config - command: cp -n {{ ddagent_conf_path }}.example {{ ddagent_conf_path }} + command: cp -n "{{ ddagent_conf_path }}.example {{ ddagent_conf_path }}" - name: Alter dd-agent config to include API Key lineinfile: - dest: {{ ddagent_conf_path }} + dest: "{{ ddagent_conf_path }}" state: present regexp: "^api_key:" line: "api_key: {{ datadog_api_key }}" @@ -44,7 +44,7 @@ - name: Copy ddagent supervisord config template: src=dd-agent-supervisord.conf.j2 - dest={{ supervisord_programs_dir }}/ddagent-supervisord.conf + dest="{{ supervisord_programs_dir }}/ddagent-supervisord.conf" mode=0644 notify: - reread supervisord From 3e186cb0773ca6eedbd379f9f0c42db82cf82498 Mon Sep 17 00:00:00 2001 From: Joseph McCullough Date: Fri, 25 Mar 2016 16:18:05 -0500 Subject: [PATCH 4/8] Progress on converting dd-agent to supervisor --- roles/dd-agent/defaults/main.yml | 1 + roles/dd-agent/handlers/main.yml | 5 +++++ roles/dd-agent/meta/main.yml | 2 ++ roles/dd-agent/tasks/main.yml | 5 +++-- roles/dd-agent/templates/datadog.conf.j2 | 10 ++++++++++ 5 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 roles/dd-agent/handlers/main.yml create mode 100644 roles/dd-agent/meta/main.yml create mode 100644 roles/dd-agent/templates/datadog.conf.j2 diff --git a/roles/dd-agent/defaults/main.yml b/roles/dd-agent/defaults/main.yml index 15a6188..b256884 100644 --- a/roles/dd-agent/defaults/main.yml +++ b/roles/dd-agent/defaults/main.yml @@ -2,3 +2,4 @@ ddagent_install_dir: /usr/bin ddagent_conf_path: /etc/dd-agent/datadog.conf +additional_checksd_dir: /etc/dd-agent/checks-enabled/ diff --git a/roles/dd-agent/handlers/main.yml b/roles/dd-agent/handlers/main.yml new file mode 100644 index 0000000..272d6bd --- /dev/null +++ b/roles/dd-agent/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: restart dd-agent + supervisorctl: + name=dd-agent + state=restarted diff --git a/roles/dd-agent/meta/main.yml b/roles/dd-agent/meta/main.yml new file mode 100644 index 0000000..ed68af2 --- /dev/null +++ b/roles/dd-agent/meta/main.yml @@ -0,0 +1,2 @@ +dependencies: + - { role: supervisord, when: "supervisord_has_run is not defined"} \ No newline at end of file diff --git a/roles/dd-agent/tasks/main.yml b/roles/dd-agent/tasks/main.yml index 180a018..9255c99 100644 --- a/roles/dd-agent/tasks/main.yml +++ b/roles/dd-agent/tasks/main.yml @@ -25,7 +25,7 @@ update_cache: yes - name: Copy default dd-agent config - command: cp -n "{{ ddagent_conf_path }}.example {{ ddagent_conf_path }}" + command: cp -n "{{ ddagent_conf_path }}.example" "{{ ddagent_conf_path }}" - name: Alter dd-agent config to include API Key lineinfile: @@ -44,7 +44,8 @@ - name: Copy ddagent supervisord config template: src=dd-agent-supervisord.conf.j2 - dest="{{ supervisord_programs_dir }}/ddagent-supervisord.conf" + dest="{{ supervisord_programs_dir }}/dd-agent-supervisord.conf" mode=0644 notify: - reread supervisord + - restart dd-agent diff --git a/roles/dd-agent/templates/datadog.conf.j2 b/roles/dd-agent/templates/datadog.conf.j2 new file mode 100644 index 0000000..85a1c07 --- /dev/null +++ b/roles/dd-agent/templates/datadog.conf.j2 @@ -0,0 +1,10 @@ +# Managed by Ansible + +[Main] + +dd_url: https://app.datadoghq.com +api_key: {{ datadog_api_key }} +use_mount: no + +# Additional directory to look for Datadog checks +additional_checksd: {{ additional_checksd_dir }} From 001e39965821eeb21e0247cd806d197db1f450e4 Mon Sep 17 00:00:00 2001 From: Joseph McCullough Date: Mon, 28 Mar 2016 09:50:13 -0500 Subject: [PATCH 5/8] Complete moving datadog-agent under supervisor --- roles/dd-agent/tasks/main.yml | 25 +++++++++++++----------- roles/dd-agent/templates/datadog.conf.j2 | 2 ++ 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/roles/dd-agent/tasks/main.yml b/roles/dd-agent/tasks/main.yml index 9255c99..74f805a 100644 --- a/roles/dd-agent/tasks/main.yml +++ b/roles/dd-agent/tasks/main.yml @@ -24,16 +24,6 @@ state: present update_cache: yes -- name: Copy default dd-agent config - command: cp -n "{{ ddagent_conf_path }}.example" "{{ ddagent_conf_path }}" - -- name: Alter dd-agent config to include API Key - lineinfile: - dest: "{{ ddagent_conf_path }}" - state: present - regexp: "^api_key:" - line: "api_key: {{ datadog_api_key }}" - # dd-agent was previously just run as a service before being run under # supervisor. - name: Stop service based dd-agent @@ -41,7 +31,7 @@ name: datadog-agent state: stopped -- name: Copy ddagent supervisord config +- name: Copy dd-agent supervisord config template: src=dd-agent-supervisord.conf.j2 dest="{{ supervisord_programs_dir }}/dd-agent-supervisord.conf" @@ -49,3 +39,16 @@ notify: - reread supervisord - restart dd-agent + +- name: Create additional_checksd directory + file: + path: "{{ additional_checksd_dir }}" + state: directory + +- name: Copy dd-agent datadog config + template: + src=datadog.conf.j2 + dest="{{ ddagent_conf_path }}" + mode=0644 + notify: + - restart dd-agent diff --git a/roles/dd-agent/templates/datadog.conf.j2 b/roles/dd-agent/templates/datadog.conf.j2 index 85a1c07..7540fe7 100644 --- a/roles/dd-agent/templates/datadog.conf.j2 +++ b/roles/dd-agent/templates/datadog.conf.j2 @@ -1,4 +1,6 @@ +########################## # Managed by Ansible +########################## [Main] From ecbf58e1316fdff5a866a0b8fb701c89a895e1b1 Mon Sep 17 00:00:00 2001 From: Joseph McCullough Date: Mon, 28 Mar 2016 12:51:21 -0500 Subject: [PATCH 6/8] Add datadog to inventory --- cdr-es-cluster.inventory | 3 +++ dig-es-cluster.inventory | 3 +++ ist-crawlers.inventory | 3 +++ ist-es-cluster.inventory | 3 +++ production.inventory | 5 +++++ pulse-internap.inventory | 3 +++ 6 files changed, 20 insertions(+) diff --git a/cdr-es-cluster.inventory b/cdr-es-cluster.inventory index d3ba0d2..b6308d9 100644 --- a/cdr-es-cluster.inventory +++ b/cdr-es-cluster.inventory @@ -7,6 +7,9 @@ [all:vars] repository_host=repo.istresearch.com +[dd-agent-nodes:children] +servers + [servers] #127.0.0.1 host_alias=localhost diff --git a/dig-es-cluster.inventory b/dig-es-cluster.inventory index c8657b4..d488996 100644 --- a/dig-es-cluster.inventory +++ b/dig-es-cluster.inventory @@ -7,6 +7,9 @@ [all:vars] repository_host=repo.istresearch.com +[dd-agent-nodes:children] +servers + [servers] #127.0.0.1 host_alias=localhost # This group is diff --git a/ist-crawlers.inventory b/ist-crawlers.inventory index 0846cc0..d7a52f9 100644 --- a/ist-crawlers.inventory +++ b/ist-crawlers.inventory @@ -7,6 +7,9 @@ [all:vars] repository_host=repo.istresearch.com +[dd-agent-nodes:children] +servers + [servers] #127.0.0.1 host_alias=localhost # This group is diff --git a/ist-es-cluster.inventory b/ist-es-cluster.inventory index 1d0c58d..e4b69bb 100644 --- a/ist-es-cluster.inventory +++ b/ist-es-cluster.inventory @@ -8,6 +8,9 @@ repository_host=repo.istresearch.com elasticsearch_heap_size=16g +[dd-agent-nodes:children] +servers + [servers] #127.0.0.1 host_alias=localhost # This group is diff --git a/production.inventory b/production.inventory index a8161c7..5efac67 100644 --- a/production.inventory +++ b/production.inventory @@ -8,6 +8,11 @@ [all:vars] repository_host=repo.istresearch.com +[dd-agent-nodes:children] +internap +aws +other + # # Memex infrastructure and software topology # diff --git a/pulse-internap.inventory b/pulse-internap.inventory index eb81cca..cb85349 100644 --- a/pulse-internap.inventory +++ b/pulse-internap.inventory @@ -8,6 +8,9 @@ [all:vars] repository_host=repo.istresearch.com +[dd-agent-nodes:children] +internap + [internap:children] zookeeper-nodes kafka-nodes From 588cc627be0478bcae75af8ed5f9d67e23ed47cd Mon Sep 17 00:00:00 2001 From: Jason Haas Date: Thu, 31 Mar 2016 17:56:13 -0500 Subject: [PATCH 7/8] Don't start DD service on system boot --- roles/dd-agent/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/dd-agent/tasks/main.yml b/roles/dd-agent/tasks/main.yml index 74f805a..662163c 100644 --- a/roles/dd-agent/tasks/main.yml +++ b/roles/dd-agent/tasks/main.yml @@ -30,6 +30,7 @@ service: name: datadog-agent state: stopped + enabled: no - name: Copy dd-agent supervisord config template: From f3d02e542d4648e9054104e53629cb821df5d1e4 Mon Sep 17 00:00:00 2001 From: Jason Haas Date: Thu, 31 Mar 2016 17:56:40 -0500 Subject: [PATCH 8/8] redirect_stderr to see log output in single log file --- roles/dd-agent/templates/dd-agent-supervisord.conf.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/dd-agent/templates/dd-agent-supervisord.conf.j2 b/roles/dd-agent/templates/dd-agent-supervisord.conf.j2 index d560d3a..2437636 100644 --- a/roles/dd-agent/templates/dd-agent-supervisord.conf.j2 +++ b/roles/dd-agent/templates/dd-agent-supervisord.conf.j2 @@ -2,5 +2,6 @@ command={{ ddagent_install_dir }}/dd-agent foreground autostart=true autorestart=true +redirect_stderr=true startsecs=5 stopsignal=KILL