diff --git a/ansible.cfg b/ansible.cfg index e20aa68..7a643b2 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -1,6 +1,6 @@ [defaults] -hostfile = hosts +inventory = ./hosts vault_password_file=.vault_pass.txt +host_key_checking = False [ssh_connection] pipelining = True - diff --git a/deploy.yml b/deploy.yml index 5665c36..bf55d42 100644 --- a/deploy.yml +++ b/deploy.yml @@ -26,6 +26,8 @@ notify: - restart web frontend register: application_config + become_user: root + become: true - name: Upgrade the virtualenv. pip: @@ -42,6 +44,15 @@ virtualenv: "{{ project_root }}/env" when: (application_git_repo.changed) or (application_config.changed) + - name: Install packages based on package.json. + npm: + path: "{{ project_root }}/code" + + - name: Build NPM assets + command: npm run build + args: + chdir: "{{ project_root }}/code" + - name: Generate Django media. django_manage: command: collectstatic --noinput @@ -49,15 +60,6 @@ virtualenv: "{{ project_root }}/env" when: (application_git_repo.changed) or (application_config.changed) - - name: Compile all translations from .po files into .mo files - django_manage: - command: compilemessages - app_path: "{{ project_root }}/code" - virtualenv: "{{ project_root }}/env" - notify: - - restart web frontend - when: (application_git_repo.changed) or (application_config.changed) - - name: Install rsyslog configuration files template: src: files/conf/rsyslog.conf @@ -66,5 +68,18 @@ notify: - restart rsyslog + - name: Check django is working + django_manage: + command: check + app_path: "{{ project_root }}/code" + virtualenv: "{{ project_root }}/env" + + - name: Flush handlers + meta: flush_handlers + + - name: Check gunicorn is up + command: service ynr_gunicorn status + args: + warn: false -- include: crontab.yml +- import_playbook: crontab.yml diff --git a/files/conf/nginx.conf b/files/conf/nginx_ynr.conf similarity index 100% rename from files/conf/nginx.conf rename to files/conf/nginx_ynr.conf diff --git a/hosts b/hosts index ff11e40..b1d69d3 100644 --- a/hosts +++ b/hosts @@ -1,13 +1,8 @@ [remote:children] production -staging [servers:children] production -staging - -[staging] -DC_prod deploy_env=staging domain=stage.candidates.democracyclub.org.uk app_branch=staging [production] -YNR_Prod deploy_env=production domain=candidates.democracyclub.org.uk app_branch=master +YNR_Prod_2020 deploy_env=production domain=candidates.democracyclub.org.uk app_branch=master diff --git a/provision.yml b/provision.yml index e11dec1..55e46bd 100644 --- a/provision.yml +++ b/provision.yml @@ -5,20 +5,12 @@ - provision - s3_cmd - harden - - opencv - users # - { role: backup_ynr, when: "deploy_env == 'production'" } # - { role: restore_from_ynr_backup, when: "deploy_env == 'staging'" } vars_files: - vars.yml - vault.yml - gather_facts: false + gather_facts: true become: true - pre_tasks: - - name: Install python for Ansible - raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal) - register: output - changed_when: 'output.stdout != ""' - - name: setup - setup: filter=ansible_* -- include: deploy.yml +- import_playbook: deploy.yml diff --git a/requirements.txt b/requirements.txt index ad60c7b..d211503 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,10 @@ -ansible==2.2.2.0 +ansible==2.10.1 appdirs==1.4.3 asn1crypto==0.22.0 -cffi==1.10.0 -cryptography==1.8.1 enum34==1.1.6 idna==2.5 ipaddress==1.0.18 Jinja2==2.9.6 -MarkupSafe==1.0 packaging==16.8 paramiko>=2.1.6 pyasn1==0.2.3 diff --git a/roles/common/handlers/main.yml b/roles/common/handlers/main.yml index 140652c..64b285b 100644 --- a/roles/common/handlers/main.yml +++ b/roles/common/handlers/main.yml @@ -8,15 +8,18 @@ become: true - name: restart web frontend - action: service name={{ project_name }}_gunicorn state=restarted + systemd: + state: restarted + daemon_reload: yes + name: "{{ project_name }}_gunicorn" become: true become_user: root - name: reload systemd - sudo: yes + become: true command: systemctl daemon-reload - name: restart rsyslog service: name=rsyslog state=restarted - become: True + become: true become_user: root diff --git a/roles/harden/tasks/main.yml b/roles/harden/tasks/main.yml index 4b67406..3423e9b 100644 --- a/roles/harden/tasks/main.yml +++ b/roles/harden/tasks/main.yml @@ -1,7 +1,7 @@ --- # fail2ban - name: install fail2ban - apt: pkg=fail2ban state=installed + apt: pkg=fail2ban state=latest tags: package - name: configure fail2ban local jails @@ -15,7 +15,7 @@ # UFW - name: install UFW - apt: pkg=ufw state=installed + apt: pkg=ufw state=latest tags: package - ufw: diff --git a/roles/opencv/tasks/main.yml b/roles/opencv/tasks/main.yml deleted file mode 100644 index 0bbe285..0000000 --- a/roles/opencv/tasks/main.yml +++ /dev/null @@ -1,64 +0,0 @@ ---- -- command: opencv_version - register: opencv_version_result - ignore_errors: True - - -- name: Create temporary download directory - file: name={{ download_dir }} state=directory - when: opencv_version_result|failed - -- name: Download and unzip opencv - unarchive: - src: "https://github.com/Itseez/opencv/archive/{{ opencv_version }}.zip" - dest: "{{ download_dir }}" - copy: no - creates: "{{ download_dir }}/opencv-{{ opencv_version }}" - when: opencv_version_result|failed - -- name: Download and unzip opencv_contrib - unarchive: - src: "https://github.com/Itseez/opencv_contrib/archive/{{ opencv_version }}.zip" - dest: "{{ download_dir }}" - copy: no - creates: "{{ download_dir }}/opencv_contrib-{{ opencv_version }}" - when: opencv_version_result|failed - -- name: Create build directory - file: name={{ download_dir }}/opencv-{{ opencv_version }}/build state=directory - when: opencv_version_result|failed - -- name: Make the opencv build artifacts - shell: | - cmake -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local -D INSTALL_C_EXAMPLES=OFF -D INSTALL_PYTHON_EXAMPLES=OFF -D BUILD_EXAMPLES=OFF .. - args: - chdir: "{{ download_dir }}/opencv-{{ opencv_version }}/build" - executable: /bin/bash - when: opencv_version_result|failed - -- shell: | - cat /proc/cpuinfo | grep processor | wc -l - register: num_processors - when: opencv_version_result|failed - -- name: Compile opencv - shell: | - make -j{{num_processors.stdout.0}} - args: - chdir: "{{ download_dir }}/opencv-{{ opencv_version }}/build" - executable: /bin/bash - when: opencv_version_result|failed - - -- name: Install opencv - become: true - shell: | - make install - ldconfig - args: - chdir: "{{ download_dir }}/opencv-{{ opencv_version }}/build" - executable: /bin/bash - when: opencv_version_result|failed - -# - name: Symlink opencv to virtualenv -# file: path={{ virtualenv_home }}/{{ virtualenv_name }}/lib/python3.4/site-packages/cv2.so src=/usr/local/lib/python3.4/site-packages/cv2.cpython-34m.so state=link diff --git a/roles/provision/tasks/main.yml b/roles/provision/tasks/main.yml index cc47c48..3906a0a 100644 --- a/roles/provision/tasks/main.yml +++ b/roles/provision/tasks/main.yml @@ -4,7 +4,7 @@ - name: Install required system packages. apt: pkg: "{{ item }}" - state: installed + state: latest update-cache: yes cache_valid_time: 43200 with_items: "{{ system_packages }}" @@ -60,7 +60,6 @@ pip: name: "gunicorn" virtualenv: "{{ project_root }}/env/" - virtualenv_python: "python3.5" become_user: "{{ project_name }}" - name: Install python2 psycopg2 @@ -80,14 +79,28 @@ dest: "{{ project_root }}/.htpasswd" notify: restart nginx -- name: Install nginx configuration file. - template: src=files/conf/nginx.conf dest=/etc/nginx/sites-enabled/{{ project_name }} +# - name: Install nginx configuration file. +# template: src=files/conf/nginx.conf dest=/etc/nginx/nginx.conf +# notify: restart nginx + +- name: Install nginx app configuration file. + template: src=files/conf/nginx_ynr.conf dest=/etc/nginx/sites-enabled/{{ project_name }} notify: restart nginx - name: Install nginx refirects file. template: src=files/conf/nginx_redirects.conf dest=/etc/nginx/{{ project_name }}_redirects.conf notify: restart nginx +- name: Increase map_hash_bucket_size + lineinfile: + dest: /etc/nginx/nginx.conf + insertafter: "sendfile on;" + line: "\tmap_hash_bucket_size 512;" + state: present + backup: yes + notify: restart nginx + + - name: Remove default nginx config file: state: absent @@ -97,12 +110,16 @@ template: src=files/conf/mime.types dest=/etc/nginx/mime.types notify: "restart nginx" +- name: Validate Nginx config + become: yes + command: nginx -t + changed_when: false + - name: Install systemd init scripts. template: src: files/systemd/{{ item }}.service dest: /etc/systemd/system/{{ project_name }}_{{ item }}.service with_items: "{{ initfiles }}" - when: ansible_distribution_version == "16.04" notify: - reload systemd - "restart web frontend" diff --git a/roles/s3_cmd/tasks/main.yml b/roles/s3_cmd/tasks/main.yml index 1d13d42..519a59e 100644 --- a/roles/s3_cmd/tasks/main.yml +++ b/roles/s3_cmd/tasks/main.yml @@ -2,13 +2,11 @@ - name: install s3cmd apt: pkg: s3cmd - state: installed + state: latest tags: package - name: configure s3cmd - sudo: true + become: true template: > src=s3cmd.conf dest={{project_root}}/home/.s3cfg - - diff --git a/vars.yml b/vars.yml index 519c68c..59e29d1 100644 --- a/vars.yml +++ b/vars.yml @@ -37,15 +37,15 @@ system_packages: - libxslt-dev - libyaml-dev - memcached + - npm - nginx - opencv-data - postfix - postgis + - python-is-python3 - python3-dev - - python3-pip - python3-psycopg2 - python3-setuptools - - python3-virtualenv - redis-server - s3cmd - yui-compressor diff --git a/vault.yml b/vault.yml index 9aa9ca7..bc5e014 100644 --- a/vault.yml +++ b/vault.yml @@ -1,38 +1,42 @@ $ANSIBLE_VAULT;1.1;AES256 -61363363663861646264323833323161396332313663306466633364643839663730373531633263 -3962653734366362663861316365366134386362383961360a393565666439343035323539306165 -36333530383030343037333766666532303538346362313261623339666564306463386133363865 -3836633034353761640a633963303830316336363865623961356336333834353139643632373034 -38326133626431353534316631343530356364366564306133616432303563316632393630653261 -61366531656562373332613132373234646238343935336162643535353636326135373865653631 -65353630313566353437643961333638303162376465396366353230666661323637643537336534 -39353062303533623662626531373165383131363565616537386534636236363830333763393934 -62396664353637366539633437613538343766393531306230373866353633323634643163616539 -33363563336537333936653432336537396362663238616131363863316264613135646664633066 -34313334633537353033343962383530643361656330663861373931343932613033663933626464 -31383837616337353837393661626133313439643763333564643130306234386465656333306230 -37373566336164366363303436626339643863393233656161333438366364383133313034633935 -65353661303165363735373036623939323832663533336163653030663232306431323934323733 -64363136393936643235313037633834616131653539356434666562386334336438633465383139 -35616533623335313138633730636163393335633461306430653832613331306537663337333034 -30623463323030303437623964353338656233636135666562396635396636663539643138623539 -62396264663362303737396339353930386534393436373961343365383632643365663464326339 -31646238353338313665666633663934306536393830376131636631363433623135336161373963 -64363730393231666235323535313832613862646337633334356437663164323731346465333535 -64306137633066303734303963363633643131373839356631633639643234303662666163653138 -39663161313961326138306534313264616539373337343562336239356562326430333633633261 -33336435366334323734383466343462303066633530373330393237383139343538356435356236 -63353834613661616237326166336533376532653530333666326532626132613165666434396137 -62363364653939393162393531613862626139376332376166616339303237376662663261623737 -61656538333637303532336630373035366464333763363965643063396134316531346161316564 -39636566663264636466613763313163323866633163323963356430353965313933666431313930 -39323131646561643132663032373165306239626534346234323234623330643664333835366131 -31333736346433636639613932313463373830306561633835393736323730363235366561306461 -35643338383566643539333438653033383630626531313831303763623266346264343233326639 -34363465316634633837636562326431356435363939313364623038373161323033383031353166 -61346430326133366331646137323862333834633661343139393132636237343066313861303764 -63643965616561663835616564653933356362343331326537323338616132373233653031313564 -61313439383862646132646662633434623163303063373139663733623536363532636166643631 -63306538366234643761643731323234323635303935643861616335393738623237386465613962 -64346333653034313861393563393962626664626161626461623634303439633637366166663262 -3635 +35326239663139313464316435393739333964356238633064313863326332613163363134313131 +6435633363386165353239303339306666636366333335300a366262333035643564626662303838 +62653232363434386462386530363163383539353938326165643135643539383661646139393430 +3739343239666239610a353963376562353263353361386231646665646334326634353635623930 +37363437303261323037616132356661343931393535616532346166346633306539313639646133 +65633839626133343037663866326561616263336164383662303839333733306330643139613834 +64633231643136303533326237313137326137356639633861353831626566333161303535623662 +32623964303038636461323533343333356566366663383935656131656630663763633633613861 +35613233396531353365313530646266336564333032616134343336653234386663663664663636 +31643438326365636636663935623033346166623466396132323462343230626264663536323834 +38336261343864316334366663313632653063366365306161353736623436656137663632323630 +32626339653334656538346261643961386463343662623830616136326432666235656130343333 +30386237663961653864633734663361373839643031613461376335383336653262333635343064 +30643062633463353033663762643961643932343138343365633634303333313136393833316662 +35343039346535653866643364386632646331323038653266376438386461616664613436313637 +30353762303637353833653437373865376435333236313539313436383237326632636331643731 +65653737323661353663636634313638353837353232613864613662633263663063646364633834 +66333131643961393765633332386264383063643337666664386332346432303030366664376136 +64636132336134333063313863336435326235623734663335313936353063616231616161333962 +31373036313931303135633664633438636466306234633837363265666366333634323064313466 +39333639623238626631316539623765393337356461306664343831343635363135366338616537 +63613161623035623135393362316639386266316362623265383266346637383838663538386432 +65653963656531313636633131656564303830386565343335616138346430383637656463616437 +36396362346662346636356634373562353633653561656138626536636662646535353336663866 +31376238616632623065316330303166316265313431666266653233386438666236353930366438 +38346432633362313930626464393238363662356231386638616539356433323662323732353832 +32316632323137366531626432356161353733376638646439306635656638356361323266633838 +65373333366366663537333733316430333033383662326437396263656530363934353038306166 +35623365346233333363366231613766303761393237613139646361373163363163653634666533 +62343334343132623765646336356565393438336165373535313461396665343166333436303261 +63663237343965616663366432653962373235653961616664633033616630373030623933336633 +32633830326662356465373937326138626463626237393465343966313838393061656331623138 +38666561633731656165616533316161373737313431323236353436663035646434306634326262 +31393062383531373735663165323934303337336130316232616536653333656231386162366230 +63316265643934396538653130356230353033333035373466623834643637376638643861383762 +62656461316463666533376135616365373564373762376531363837633332656334626130623736 +39396266306261613530623636653937643831616436633330653136323038626161326436323764 +33303762656338376330323836663465306462616333636331633664666639386238633536636136 +65656635623531393234643637346263643364353961653537306338636139313336313732386631 +66636166343833666132393939323034363330313931366232663361626264623135643530623537 +35383739386639396638643265323133386464663736343637393036353033393630 diff --git a/webapp_settings/production.py b/webapp_settings/production.py index bf26665..5344686 100644 --- a/webapp_settings/production.py +++ b/webapp_settings/production.py @@ -1,4 +1,5 @@ # Only set this to True in development environments +from .base import * import certifi DEBUG = False CAN_EDIT_ELECTIONS = False @@ -8,7 +9,6 @@ # https://www.miniwebtool.com/django-secret-key-generator/ SECRET_KEY = "{{ production_django_secret_key }}" MEDIA_ROOT = "{{ django_media_root }}" -STATICFILES_DIRS = () DATABASES = { 'default': { @@ -79,7 +79,7 @@ # SOPN_TRACKER # RESULTS_PROGRESS # BY_ELECTIONS -FRONT_PAGE_CTA = 'SOPN_TRACKER' +FRONT_PAGE_CTA = 'BY_ELECTIONS' SOPN_TRACKER_INFO = { # Will be used as "{} nomination papers (SOPNs)" # and "Help us find all the nomination papers for the {}s" @@ -89,17 +89,20 @@ } SCHEDULED_ELECTION_DATES = [ - "2019-05-02" + "2021-05-06" ] -STATICFILES_STORAGE = 'ynr.s3_storage.StaticStorage' -DEFAULT_FILE_STORAGE= 'ynr.s3_storage.MediaStorage' +STATICFILES_STORAGE = 'ynr.storages.StaticStorage' +DEFAULT_FILE_STORAGE= 'ynr.storages.MediaStorage' AWS_S3_REGION_NAME = 'eu-west-2' AWS_STORAGE_BUCKET_NAME = "static-candidates.democracyclub.org.uk" AWS_S3_CUSTOM_DOMAIN = "static-candidates.democracyclub.org.uk" STATICFILES_LOCATION = 'static' MEDIAFILES_LOCATION = 'media' +AWS_DEFAULT_ACL = "public-read" +AWS_BUCKET_ACL = AWS_DEFAULT_ACL + CSRF_TRUSTED_ORIGINS = [ "{{ domain }}", @@ -128,3 +131,12 @@ SLACK_TOKEN = "{{slack_token}}" + +CELERY_IMPORTS = ["ynr.apps.sopn_parsing.tasks",] +ALWAYS_ALLOW_RESULT_RECORDING = True + +FF_COOKIE_PATH = "/var/www/ynr/cookies.sqlite" +FACEBOOK_TOKEN = "EAAHFkAGrkL8BAEDkR3PMZBYBhNKTcxrejQQm3cpfkpApT9BhenJmqkzgTeCYDNSkY2nZBDoPo1ztaOKsf8EKCHpsel8dasjJxua1dS0XIlZBBlIgpovpwY6S1hGXStM6tlK78OF6hr4owcZAZAcZA3WS2dHG6CGNIEuEEqFpLCHkB9WaT56HWt" + + +EDITS_ALLOWED = True