diff --git a/environments/production/vars.yml b/environments/production/vars.yml index 4961a43..f757532 100644 --- a/environments/production/vars.yml +++ b/environments/production/vars.yml @@ -13,6 +13,8 @@ superset_db_name: superset_meta oauth2_db_name: superset_oauth2 cchq_db_name: superset_cchq_data +rabbitmq_version: 3.10.13-1 + datadog: enabled: true hostname: production diff --git a/environments/production/vault.yml b/environments/production/vault.yml index d1a41d7..c8ca7c6 100644 --- a/environments/production/vault.yml +++ b/environments/production/vault.yml @@ -1,51 +1,55 @@ $ANSIBLE_VAULT;1.1;AES256 -62323666386235316366363337326430333133373832323366616162376530613631303534343137 -6633336137323038653635366364623330663863646631330a623633653732373331386635633364 -64383239356232356266303132653863323736623232373332663838646134363139353637666464 -3035316533303339350a646339383261636565633364323964613462346130363663633135366137 -61336539643866616431323936306232306635303138616233643438616133353037366463356536 -30383566316330636564326461633763386632646532646436663935333565303338633832306238 -32663436353530613132663039353662376563626637643464363463616639363934663831333533 -63643635303037376639356663653434376239616461666239393336393764336366636463636663 -62313733383930303639653637616465306132616361353337656563663337316565323830666137 -37313966373033633533366334643965343566633061393666656639613030616132326535366561 -31626466633966623830336436316238343163643133633738363164303435626365613263633261 -38316363663234343666303332373263303262626439396638323964633233663131636365303738 -64383166333933666434366265333134353935363863353563333861616532663033376461313334 -38663963663466386235333639316466326436643734353931653662613939633138373337613730 -62333766616639303962643863316636316437316339643336356335666538373162363333303962 -38343339373764623163316638663235303362643231383162653864613161346632346137376166 -61623830353930643036356165393766313665353363666439316562393937663566633431313963 -64396662626339303036313665663662636633613230653564383134363765356431373830383265 -30396131643733343638653330613136653631396630333532333031363937653234636334386466 -64373739623530326166656233643233353866663765306236383663653334386562366663636133 -31663563306434613164656630363962643264653039613836663130636230393632323439316631 -31313536333665633964646436333136363866313865333765373164633230616634303438353461 -31306663313331666562363231663662313934313662366165346565333265663136376533346466 -30333065383431633935636232313639646631326136616234393837326566313732396332656162 -63353338303035326263346131383265663535643433306631313666613038663735383831303661 -62313130613130333735613332643231333166626665656362623935393938343865646533343563 -63313562306462663637366562373035626532313361643237326362396233653535393538626439 -32393034303930343138613033386366323464323333643031383933323038303862336639613464 -32636637343933363232623836656533626664643166363031343763633133313539373130616663 -36646265373433653534333061383832343330363966663730363162363732366663633865626639 -61343265313536383738623061343766663336633665393961363431356534663237343331333364 -37333039393461613532663566376430633464313364613864616134616163303539303839326334 -37653736383731313637336264646635346661386264393037336361643566613863333563323133 -65333466376239343137643266323738663039336163346638626439613831663863363834326463 -65396466323436353666383638363935316266663237613131653336643161346233656237316331 -32383265343935336531613837353333666134353064383430626535393932333264393037333034 -37383831326463303162336639356566323132613432326363306264353966316231373633366363 -38663938383364396534376134646337373263633763633361626666613862363930653630613736 -37346133383734623839306262383665333061393762616133623566383362326666643964363161 -66393434623564356639356638353564353730353731323137656333633866323864623537643937 -63313533306337336336653031303462383764386139336637656235396637336137343037303164 -32663561646165326339346236353863346335393038346336616665666234363837313932633738 -32356332663064353364326333363335393165316635316363376562376361653934373234336533 -35636236343462626363643637643335313261653031653736393636333237333632393335626632 -31623730613039356665336263666366353665343535626531363263313532636365376532386230 -66653439383135376339363939313361393966306537633562363463393632336630346232663431 -33346231343462613963386663353066623032383836343032323534306562353763303832373236 -35636265323032303233663835366366633236313931666366643935396464636363336137613864 -31353364653266323266333162646538333765336466373937343132366434613831653339633438 -3139353038303235383936633734376333613539626361313362 +34376162313939646634643964346539383930623061623264646635376162363466316138303637 +3737373166653364646239393937316665663835383530650a316265346536393137373266386537 +39363862356564356532626435623239373632383133356535636165336532633130376137316237 +6464373531633631640a363436333633336264356565396561313566613362373530643333646333 +33313236333133303535336331626339386136323266663966643064623235666534303437366266 +62376538616366323064356337666563623231353530393934633137666265323066313363376164 +64643464343531336130346466376339623265636530656661353861663530343533316535373264 +37653063323936616366376664643931393830613464313533343331333730353530383239323034 +37656237303939343164343865636166663362326339383133653637306336363835646562333365 +31653835666135346366313566396336306632653438616563396565346635636562363664346336 +34626265663266366166346438326238663866396536626561663630336635343336333038353666 +32663431323933376634343432353833346138333061363665383438646664636438346433336164 +33353862623534363766393936343432623436623163656137376631346561393166653531613033 +30363739626439376363393438346230653765663131383237383861613233646237623332623063 +32613939353031323031633333376239626562616135626437633330363664653035623237393963 +32613866613934646336333435393861363835656632613961386334633863313236323063623465 +32376637356262366435623534653064646261373632303962373531366231653832333539323830 +66396233396130373137346566646235363339626132666235333965323065326463633735393063 +63626239656565623238633765333866666532626232343435316334316462323864396639613530 +39326139343061386666366361613731376635653135623531343934313735633961393734323539 +37356361643237656339343938303431663834316261353164616333303536316137353839393161 +35316239663466346139386334333834343638346433363563326136373365353037313530653635 +30643132373965353730363534663363623236333033643661373336333963636533663737323535 +64313666366362616661376533613530313330303364343135353538666437383363616430626535 +63643465626466643066663363306338333935316166333038323734623338656466383930626234 +33653961663031326561303465396566393836346431633135383938653063626364386363343431 +61616664646364663761336133323134336437623762353966396231343564663737316263323666 +34666531303261623664643732303162356162323261643765356636396334386264653139306238 +38626266313833333332343539376164663136383932666364333039616563633664313038373339 +37646665666438346266353466623363636561383262343333303563633633653032336138633130 +38366632326131363936313938643030643066613765353066363038303264633130323032386634 +31633839333333323665306563616162396133633363643263396338663363666336316162333765 +36343966363234346165396663343262613236663934613935643866353363333430613064633131 +33363235373861323864343631636330356332383064393532353734356637646564303434656438 +64353033336365396464633033633332643833343636633866393463326662636335363336663236 +65613965316631396438383062396131323236343261656261343931663339623233633466333536 +65343464323033393065303235303166343831373938393164653734323035633631346332343137 +38386236376264356138666232633232623662383337663030353537343037376533626231323062 +36316232303564653333616630306232633431386664646265343534633362323433653761643830 +39343637373966346134363137393930656137333431393434313130343235303565386236346539 +37613439383366636536313366636430346666373636613666336536383066626461666432613766 +38373462653536613831376166376132356131383434373832666236376661356138366133373861 +62303161653562356663366436306530316366343930636430653165653931633362323137383936 +31306136623239653961306564666466333266396563666161643930616139383135643366333664 +36313163396437323237666564653532646531363536333232666564646338393630333235313030 +32663665303333363538303235316565663961303336353835636232396238356131363865646438 +63323361356532623533616463626464316339643265646461363432306561373139353266366466 +36303535323431393866663432343666363130336239323663313639623962636165393564306632 +32353737343531336230663234313438393465646233346338333531316237353835646163353332 +32343631356137656431303361326635626464376565383531383265396539303038363834613562 +39316233333137343962653630636362343962663135626434316462623237383430303638616331 +65326334666263343631326136373832616636643331656133336238326562666363343839393834 +30636532666461333465386130653639393032633933636337636132356133613130663834636438 +6365366338383264343566396432326532303366353562396633 diff --git a/environments/staging/vars.yml b/environments/staging/vars.yml index 178f9a4..d13deb3 100644 --- a/environments/staging/vars.yml +++ b/environments/staging/vars.yml @@ -12,6 +12,8 @@ superset_db_name: superset_meta oauth2_db_name: superset_oauth2 cchq_db_name: superset_cchq_data +rabbitmq_version: 3.10.13-1 + datadog: enabled: true hostname: staging diff --git a/environments/staging/vault.yml b/environments/staging/vault.yml index 0a79d90..ed2f54b 100644 --- a/environments/staging/vault.yml +++ b/environments/staging/vault.yml @@ -1,46 +1,50 @@ $ANSIBLE_VAULT;1.1;AES256 -36383163336537383366323966353632303235383433623538636464303362626433653730663064 -6530396438376133646330643766343635653639333334650a643032666538333466656230633961 -35316337363263396531323130386564363536643734313138303863356336623936366435393731 -3630633636633737660a313835316562643036313164376562613632386363336237653930306630 -32346138356234336336363964353232343462663630646631316233653133613063343230376661 -37316634656265396164623933333362336363323932346433363831326334336435666538333632 -64376631636134333533316561326465303534373838636136393830363537373364623661323331 -37313238333164666432663538666635653534393361393031373866333266636636343865313539 -64366664653538336634633338666335633563316136616234613063353133333933613230643635 -32646434633638643664643837626162313833316333373231316438653033383231303535343931 -65313837653739616237626138613261323237346565623863363539316265656362376566333365 -35363465636135616462363566323166356533383164366233316533363731613237356439663830 -34333861393333376435353734666130653965373638656133383865346632313239323136626465 -34633962303535363236373864356634356538313333306535323130653332653638303432333932 -66373639393231663737323030666238663363393966363936396531326364323961343239313936 -31356331363731366234376438633735623634386430376239363631616463353431636365626238 -30363037353436313935643337633630643039653564626139653766616561343465316630313736 -65303433356164386533303362336135333939636265366539636161653036326263353763393366 -31616133656338656238383162363939663962653333316265616436383563303233666463363163 -31633430613036393939623031353066326233643337363338333161363032633335666639663733 -37313662643463303266646533636233373433333866613737633435626166643036363361613465 -65336630666533663432646638363439316135336664613965663164643832656261613339346666 -61336664666137653639366332353264326562363431306236643263346531336566346133383334 -30303638353364303139393763363839636331313263313734343939666465356465343064653439 -63373563323937663738333037316366376538353033643466316139653764363866633839353562 -65383661386330383130336664313066343633643362306538363661626337616139396436316335 -36656362333165643838376438363237626333633234316538666636636366623461656535623432 -39333066313036306333636163643833333539626264306664363137643434636637666533386435 -64653562393631393066303137373634303362613232396233393134353363366163316437373436 -36343834313063633636396265343430353639616464663334626132353765303139326635303765 -32303033396161663437366262383638666331303132303066373434366131346532656365333864 -64616136303739323037643564313265383331623563636536663133333434386162356236316463 -35323633383766373738313739323236616238613564346662663363373465656164373834373163 -39663364336638333330393962643135636331323764613538663362313966323263373933626365 -61656534663138616432333861633737623163303833633165343733336334333831613238393334 -65626162633233656532326464326438336563383265636533373361643331353463396166626462 -39333333373661663061366131306535313432386531636636333636656532346262396636353535 -35346561616634643932646436306236353534623334613830353064646161376335306130303461 -33346163613830363331323763633038663862343136373461653233663761303664323139363262 -34633061363138393763643765376231326236326131363034393236316530346539626131313531 -30333863373638336131626161373339363131343233333462326265343562336266313665623633 -39643365623963663937386134386361373233393331303131363864336362636339633436363932 -37646239663530373831316230356239336532636462393264646535303734303136623033373863 -39393735663663303061643230633439623763323335663537656530306333333834643362383135 -3330 +33303630383036356237633431393536383230346662366262386166393330636232396663333438 +6266313838363237363936303033643839636565323437660a333162613033323061356435306536 +39636334333065363166616532333966333830303333323561626333393336353461366362396535 +3331313063643035650a303937636232303561383931643163373363383438303934633338396561 +64613966386536636331633861616637393362306534653434323639396633323138376636346531 +66666466353566666237336261643037313234653763633362653830353863316566396337643330 +36633434353635386437653764623232326635313537306362633164613934386664663962303635 +36323562616264353532663863376632396534376231396465303635643863636337303830313566 +33656665353335613838656533303638616334323962353532633864393032393838343033383963 +33316435663738376134353738303166376335326635663565656432353862626536333963323362 +61333531633739306163636538616466666264643139306235343034373263666438643663313438 +61633630323865366565346535643562353461653331646165393534393935653363633533383862 +63363561303433353937363565333434363732353530326531386661623661396434643762386233 +62303633376263316637393939613033646230363339376230316333646164666334643661363064 +38343132633339636530373130663134303066333837323737353137306638336239343962333734 +31336561666333373264343936656138623631326536623938373766313764656436343063336336 +34373164343633383661346234396331386132343634336161306330346433623364663861376639 +66333739656532396564343231366533353065353162663365313431386366653530363631356163 +35363437333432656231356664303163653966643932396166643834336364323165373331643266 +32623165373838646337653336373139383033303030393462356235353162356565656563333263 +63646665333731613661303336303436363235616134303063366463616266666161653664326164 +35366139363834363461653236663839363962366535656136336538373063306330353266326434 +33363862613762616136346161383236373034373933366537373332363739666137353662623866 +33343838643631663432303131323062393064646565313238323030666563376538633434306432 +34386133623331363538633065393932656461653531313066636130393235353664363765313839 +39613233313930633737313532316235613737373032396363646232633732646330613335346639 +31306235303863383762386264313838353564353065346430363763633265616537666665386138 +61643661376463323635366239636230313534323632643164343830663238646636376264303165 +62646233363465363465356539643463353064396430636138303933623832623834346631343431 +61636563386331646365366435386439323934306238316362396339633865616331303362353465 +61643762663432306632643238323761366332636565303334366430326364643630396437333034 +37613233353366386465373363613831623439343262663362373833313065656530393538366663 +66623834303765623935376131323639383039373232663337396361393661386333643362623836 +64643030313462626534633733373061636234333430376633303730346336383631373730326332 +61646434323362306661356263653861396139636535303266313962353932643036363032633033 +31356162316532663266633562656464363630376263346262313839646132633634306137323666 +35326363666336323563613363303338303234356232313034313462323038353964396439363965 +62383638313462653535393739613965623332353530646362313337633737656631353862643636 +30326333646632323038376135356466393932386266363164656365343266633635353535393235 +66313538613062646661326337376139323162663736366164346662396535613539393463333061 +63653937386164386335646465623463376539613464356334336531636461396239613236646366 +38383363313332333831336238383635643936366366373531386531353433393262623730333138 +36303865643836336161633931373637626539613337356161386633626462373261326633643636 +36386631306338663161393739333664656636396463323734383162633861333766353531313263 +61613562633834303739653766393038336364386262303131376539653664343465623835306133 +65386635646337643761613631343034623938336165363764336235333835366663326638336163 +39313765383066333064623535643963316661333666313731363366373938383732636135386432 +33363162386338313664346561383965633131663035366133666262303231616332623638346361 +3638 diff --git a/roles/commcare_analytics/handlers/main.yml b/roles/commcare_analytics/handlers/main.yml index 0e73fff..f5b7c3e 100644 --- a/roles/commcare_analytics/handlers/main.yml +++ b/roles/commcare_analytics/handlers/main.yml @@ -7,4 +7,8 @@ enabled: yes - name: reload systemctl - shell: systemctl daemon-reload \ No newline at end of file + shell: systemctl daemon-reload + +- name: restart rabbitmq + become: yes + service: name=rabbitmq-server state=restarted diff --git a/roles/commcare_analytics/tasks/main.yml b/roles/commcare_analytics/tasks/main.yml index 509e659..6eb0561 100644 --- a/roles/commcare_analytics/tasks/main.yml +++ b/roles/commcare_analytics/tasks/main.yml @@ -93,11 +93,15 @@ - include_tasks: postgres.yml tags: postgres +- include_tasks: rabbitmq.yml + tags: rabbitmq + - include_tasks: superset.yml tags: - superset - superset-admin - celery + - superset_config - include_tasks: logrotate.yml tags: logging diff --git a/roles/commcare_analytics/tasks/rabbitmq.yml b/roles/commcare_analytics/tasks/rabbitmq.yml new file mode 100644 index 0000000..b76ceaf --- /dev/null +++ b/roles/commcare_analytics/tasks/rabbitmq.yml @@ -0,0 +1,96 @@ +--- +- name: Install Erlang dependency + apt: + name: erlang + state: present + tags: rabbitmq + +- name: Add trusted key + apt_key: + url: https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey + state: present + tags: rabbitmq + +- name: Add rabbitmq official apt repository + apt_repository: + repo: 'deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu {{ ansible_distribution_release|lower }} main' + state: present + update_cache: "{{ apt_update_cache }}" + tags: rabbitmq + +- name: Install package + apt: + name: "rabbitmq-server={{ rabbitmq_version }}" + update_cache: "{{ apt_update_cache }}" + state: present + ignore_errors: '{{ ansible_check_mode }}' + tags: rabbitmq + +- name: Place config file + template: + src: rabbitmq/rabbitmq.conf.j2 + dest: /etc/rabbitmq/rabbitmq.conf + group: root + owner: rabbitmq + mode: 0644 + tags: rabbitmq + +- name: Create directory for ulimt + file: + path: /etc/systemd/system/rabbitmq-server.service.d + state: directory + mode: '0755' + tags: rabbitmq + +- name: setting Ulimit for rabbitmq + raw: echo "[Service]\nLimitNOFILE=65536" > /etc/systemd/system/rabbitmq-server.service.d/limits.conf + tags: rabbitmq + +- name: just force systemd to reread configs + systemd: + daemon_reload: yes + tags: rabbitmq + +- name: Make sure RabbitMQ service is started before running config tasks + become: yes + service: name=rabbitmq-server state=started + ignore_errors: '{{ ansible_check_mode }}' + tags: rabbitmq + +- name: Ensure vhost commcare_analytics is present + become: true + rabbitmq_vhost: + name: commcare_analytics + state: present + notify: + - restart rabbitmq + tags: rabbitmq + +- name: Add users + become: true + rabbitmq_user: > + user={{ rabbitmq_user }} + password={{ rabbitmq_password }} + tags='administrator' + vhost=commcare_analytics + configure_priv=.* + write_priv=.* + read_priv=.* + state=present + notify: + - restart rabbitmq + tags: rabbitmq + +- name: Remove default guest user + become: true + rabbitmq_user: + user: guest + state: absent + notify: + - restart rabbitmq + tags: rabbitmq + +- name: Enable and start RabbitMQ service + become: yes + service: name=rabbitmq-server state=started + tags: rabbitmq diff --git a/roles/commcare_analytics/tasks/superset.yml b/roles/commcare_analytics/tasks/superset.yml index 5201001..8343bb4 100644 --- a/roles/commcare_analytics/tasks/superset.yml +++ b/roles/commcare_analytics/tasks/superset.yml @@ -89,7 +89,9 @@ group: "{{ app_group }}" mode: 0640 backup: yes - tags: superset + tags: + - superset + - superset_config - name: Run Superset database migrations. diff --git a/roles/commcare_analytics/templates/rabbitmq/rabbitmq.conf.j2 b/roles/commcare_analytics/templates/rabbitmq/rabbitmq.conf.j2 new file mode 100644 index 0000000..b3ace38 --- /dev/null +++ b/roles/commcare_analytics/templates/rabbitmq/rabbitmq.conf.j2 @@ -0,0 +1,9 @@ +# +# {{ ansible_managed }} +# +log.channel.level = warning +log.connection.level = warning +log.federation.level = warning +log.mirroring.level = warning +consumer_timeout = 86400001 +heartbeat = 600 \ No newline at end of file diff --git a/roles/commcare_analytics/templates/superset/superset_config.py.j2 b/roles/commcare_analytics/templates/superset/superset_config.py.j2 index 7cd95cc..b23b44c 100644 --- a/roles/commcare_analytics/templates/superset/superset_config.py.j2 +++ b/roles/commcare_analytics/templates/superset/superset_config.py.j2 @@ -61,6 +61,7 @@ FEATURE_FLAGS = { SHARED_DIR = '{{ superset_shared_dir }}' _REDIS_URL = 'redis://localhost:6379/0' +_RABBITMQ_URL = 'amqp://{{ rabbitmq_user }}:{{ rabbitmq_password }}@localhost:5672/commcare_analytics' CACHE_CONFIG = { 'CACHE_TYPE': 'RedisCache', @@ -76,7 +77,7 @@ RESULTS_BACKEND = RedisCache( class CeleryConfig(object): accept_content = ['pickle', 'application/json'] - broker_url = _REDIS_URL + broker_url = _RABBITMQ_URL imports = ( 'superset.sql_lab', 'superset.tasks',