Update ansible, use EC2 plugin, update deployment

ansible.cfg

@@ -1,5 +1,5 @@
 [defaults]
-inventory = ./hosts
+inventory = servers.aws_ec2.yml
 vault_password_file=.vault_pass.txt
 host_key_checking = False
 [ssh_connection]

crontab.yml

@@ -1,5 +1,5 @@
 ---
-- hosts: servers
+- hosts: ynr_cron
   vars_files:
     - vars.yml
     # - @vault.yml
@@ -26,19 +26,18 @@
       minute: "30,04,19"
       job: "nice -n 19 ionice -c 3 output-on-error {{project_root}}/env/bin/python {{project_root}}/code/manage.py moderation_queue_detect_faces_in_queued_images"
       disabled: no
-    when: deploy_env == "production"
 
   - cron:
       name: "Import elections from EE"
       minute: "06"
       job: "output-on-error {{project_root}}/env/bin/python {{project_root}}/code/manage.py uk_create_elections_from_every_election"
+      disabled: yes
 
   - cron:
       name: "Update Twitter usernames"
       minute: "06"
       hour: "01"
       job: "output-on-error {{project_root}}/env/bin/python {{project_root}}/code/manage.py twitterbot_update_usernames"
-    when: deploy_env == "production"
 
   - cron:
       name: "Update parties from EC"
@@ -51,16 +50,24 @@
       minute: "06"
       hour: "03"
       job: "output-on-error {{project_root}}/env/bin/python {{project_root}}/code/manage.py twitterbot_add_images_to_queue"
-    when: deploy_env == "production"
 
   - cron:
       name: "Build static person pages"
       minute: "10"
-      hour: 1 # For non-election times
+      hour: "1" # For non-election times
       job: "output-on-error {{project_root}}/env/bin/python {{project_root}}/code/manage.py candidates_cache_api_to_directory --url-prefix https://static-candidates.democracyclub.org.uk/media/cached-api --prune"
-    when: deploy_env == "production"
+
   - cron:
       name: "Update current candidates for party"
       minute: "10"
-      hour: 2 # For non-election times
+      hour: "2" # For non-election times
       job: "output-on-error {{project_root}}/env/bin/python {{project_root}}/code/manage.py parties_update_current_candidates"
+
+  - cron:
+      name: "Update 2022 results CSV"
+      minute: "*/15"
+      job: "output-on-error /var/www/ynr/env/bin/python /var/www/ynr/code/manage.py uk_results_create_file --election-date=2022-05-05 --format=csv"
+  - cron:
+      name: "Look for recent changes in EE"
+      minute: "*/5"
+      job: "output-on-error /var/www/ynr/env/bin/python /var/www/ynr/code/manage.py uk_create_elections_from_every_election --recently-updated"

deploy.yml

@@ -1,27 +1,28 @@
 ---
-- hosts: servers
+- hosts: ynr
   vars_files:
     - vars.yml
     - vault.yml
   gather_facts: true
   become: true
   become_user: "{{ project_name }}"
+  remote_user: ubuntu
   roles:
     - common
   tasks:
     - name: Pull sources from the repository.
       git:
         repo: "{{ project_repo }}"
         dest: "{{ project_root }}/code/"
-        version: "{{ app_branch }}"
+        version: "master"
         accept_hostkey: true
       notify:
         - restart web frontend
       register: application_git_repo
 
     - name: Upload configuration.
       template:
-        src: "webapp_settings/{{ deploy_env }}.py"
+        src: "webapp_settings/production.py"
         dest: "{{ project_root }}/code/ynr/settings/local.py"
       notify:
         - restart web frontend
@@ -60,14 +61,6 @@
         virtualenv: "{{ project_root }}/env"
       when: (application_git_repo.changed) or (application_config.changed)
 
-    - name: Install rsyslog configuration files
-      template:
-        src: files/conf/rsyslog.conf
-        dest: /etc/rsyslog.conf
-      become_user: root
-      notify:
-        - restart rsyslog
-
     - name: Check django is working
       django_manage:
         command: check

group_vars/ynr.yml

@@ -0,0 +1,27 @@
+$ANSIBLE_VAULT;1.1;AES256
+37396665643662353662383536653736663361386161666630633439656430623665623230383931
+3164333264366564363763363162306238663762343931300a323835396562383333306437333331
+30336165656531323636633533313939376134376139346463346462343531653533366465393061
+3235346134613961650a386466343335303335373131346436623363316631623038343665313334
+32643064396631336330646138373165306538346463353564666133653738313065633736626139
+61636235653833333231323637356362383631396130326363353766343032613362613461616339
+36326330646339646638356430393133336334313538663462633530656166643664373939666463
+35646434613462343337653338346538646231306461326163616439633664336437306637633864
+31633535303830343235393236623438313965326131333839643661616165623031316666613632
+32346365323263393764313762663165323230393164653631333166376437613234376161383930
+64613366656230326530393462316561396262663239643332353031313137313738323164373139
+62333930626433366439316630356261656331393938356335653835613637326461626362326630
+30646566343737396164633634393362306439323766636338633265383162653464373065663430
+32386265613166396635643135643934653462393937306431326631363361313130346439323432
+36323234373861383837616466646438353334623336336633613663633137333332663665313666
+31373232353132653162663463326164386333376338336130616165373430613834353663356434
+63336239386339353561333832336233616230376434393133666263313432373765656664656634
+61323765303933353639333039306666343939366430333564653330316564366437656231613365
+39366236663565613439343034353262336533666432613530646434616666663731313631616131
+31393039376631333662386433663165373838386635326434386131383334376161303331343734
+31326662383436303163326530396432376233646366376137396365353838613135613232663264
+66333034363037616339653037333066633030386537663063663634346565303264353038383062
+65383031616235613037636136636539383666646633386539386333373364323661356633613036
+34333063366261343563353138316661376336656234373431653162383166656434373036623064
+62393963386535636565636630366538363763333863643964346165626530666631303535376438
+3766353339653363333864306536633030353763626436376134

requirements.txt

@@ -1,4 +1,4 @@
-ansible==2.10.1
+ansible==3.2.0
 appdirs==1.4.3
 asn1crypto==0.22.0
 enum34==1.1.6

servers.aws_ec2.yml

@@ -0,0 +1,18 @@
+plugin: amazon.aws.aws_ec2
+regions:
+  - eu-west-2
+filters:
+  instance-state-name: running
+keyed_groups:
+  # add hosts to tag_Name_value groups for each aws_ec2 host's tags.Name variable
+  - key: tags.product
+    separator: ""
+  - key: tags.worker_type
+    separator: ""
+groups:
+  # add hosts to the group development if any of the dictionary's keys or values is the word 'devel'
+  prod: "'ynr' in (tags|list)"
+  prod_cron: "'ynr_cron' in (tags|list)"
+compose:
+  # set the ansible_host variable to connect with the private IP address without changing the hostname
+  ansible_host: public_ip_address

webapp_settings/production.py

@@ -123,18 +123,17 @@
     'level': 'WARNING',
     'class': 'raven.contrib.django.raven_compat.handlers.SentryHandler',
 }
-LOGGING["loggers"]["account_adapter"]: {
-    'level': 'WARNING',
-    'handlers': ['sentry'],
-    'propagate': False,
-}
+# LOGGING["loggers"]["account_adapter"]: {
+#     'level': 'WARNING',
+#     'handlers': ['sentry'],
+#     'propagate': False,
+# }
 
 
 SLACK_TOKEN = "{{slack_token}}"
 
 CELERY_IMPORTS = ["ynr.apps.sopn_parsing.tasks",]
 ALWAYS_ALLOW_RESULT_RECORDING = True
-
 FF_COOKIE_PATH = "/var/www/ynr/cookies.sqlite"
 FACEBOOK_TOKEN = "EAAHFkAGrkL8BAEDkR3PMZBYBhNKTcxrejQQm3cpfkpApT9BhenJmqkzgTeCYDNSkY2nZBDoPo1ztaOKsf8EKCHpsel8dasjJxua1dS0XIlZBBlIgpovpwY6S1hGXStM6tlK78OF6hr4owcZAZAcZA3WS2dHG6CGNIEuEEqFpLCHkB9WaT56HWt"