Merge branch 'release/1.33.0'

README.md

@@ -2,7 +2,7 @@
 
 ## Local Development
 
-A combination of Vagrant 2.2+ and Ansible 2.5+ is used to setup the development environment for this project. The project consists of the following virtual machines:
+A combination of Vagrant 2.2+ and Ansible 2.8 is used to setup the development environment for this project. The project consists of the following virtual machines:
 
 - `app`
 - `services`
@@ -32,10 +32,16 @@ First, ensure that you have a set of Amazon Web Services (AWS) credentials with
 $ aws configure --profile mmw-stg
 ```
 
+Ensure you have the [vagrant-disksize](https://github.com/sprotheroe/vagrant-disksize) plugin installed:
+
+```bash
+$ vagrant plugin install vagrant-disksize
+```
+
 Next, use the following command to bring up a local development environment:
 
 ```bash
-$ MMW_ITSI_SECRET_KEY="***" vagrant up
+$ vagrant up
 ```
 
 The application will now be running at [http://localhost:8000](http://localhost:8000).
@@ -130,6 +136,12 @@ $ vagrant ssh worker -c 'sudo service celeryd restart'
 
 To enable the geoprocessing cache simply set it to `1` and restart the `celeryd` service.
 
+In some cases, it may be necessary to remove all cached values. This can be done with:
+
+```bash
+$ vagrant ssh services -c 'redis-cli -n 1 --raw KEYS ":1:geop_*" | xargs redis-cli -n 1 DEL'
+```
+
 ### Test Mode
 
 In order to run the app in test mode, which simulates the production static asset bundle, reprovision with `VAGRANT_ENV=TEST vagrant provision`.

Vagrantfile

@@ -3,6 +3,16 @@
 
 Vagrant.require_version ">= 2.2"
 
+# Install vagrant-disksize to allow resizing the services VM.
+unless Vagrant.has_plugin?("vagrant-disksize")
+  raise  Vagrant::Errors::VagrantError.new, "vagrant-disksize plugin is missing. Please install it using 'vagrant plugin install vagrant-disksize' and rerun 'vagrant up'"
+end
+
+# We need to stay on Ansible 2.8 because the version_compare filter was removed
+# in 2.9.
+# https://github.com/ansible/ansible/issues/64174#issuecomment-548639160
+ANSIBLE_VERSION = "2.8.*"
+
 if ["up", "provision", "status"].include?(ARGV.first)
   require_relative "vagrant/ansible_galaxy_helper"
 
@@ -32,59 +42,88 @@ else
   VAGRANT_NETWORK_OPTIONS = { auto_correct: false }
 end
 
+MMW_EXTRA_VARS = {
+  django_test_database: ENV["MMW_TEST_DB_NAME"] || "test_mmw",
+  services_ip: ENV["MMW_SERVICES_IP"] || "33.33.34.30",
+  tiler_host: ENV["MMW_TILER_IP"] || "33.33.34.35",
+  itsi_secret_key: ENV["MMW_ITSI_SECRET_KEY"],
+  concord_secret_key: ENV["MMW_CONCORD_SECRET_KEY"],
+  hydroshare_secret_key: ENV["MMW_HYDROSHARE_SECRET_KEY"],
+  srat_catchment_api_key: ENV["MMW_SRAT_CATCHMENT_API_KEY"],
+  tilecache_bucket_name: ENV["MMW_TILECACHE_BUCKET"] || "",
+}
+
 Vagrant.configure("2") do |config|
-  config.vm.box = "bento/ubuntu-16.04"
+  config.vm.box = "bento/ubuntu-20.04"
 
   config.vm.define "services" do |services|
     services.vm.hostname = "services"
-    services.vm.network "private_network", ip: ENV.fetch("MMW_SERVICES_IP", "33.33.34.30")
+    services.vm.network "private_network", ip: ENV["MMW_SERVICES_IP"] || "33.33.34.30"
+    services.disksize.size = '64GB'
 
     # PostgreSQL
-    services.vm.network "forwarded_port", {
+    services.vm.network "forwarded_port", **{
       guest: 5432,
       host: 5432
     }.merge(VAGRANT_NETWORK_OPTIONS)
     # Redis
-    services.vm.network "forwarded_port", {
+    services.vm.network "forwarded_port", **{
       guest: 6379,
       host: 6379
     }.merge(VAGRANT_NETWORK_OPTIONS)
 
     services.vm.provider "virtualbox" do |v|
       v.customize ["guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-set-threshold", 10000 ]
-      v.memory = 2048
+      v.memory = 4096
+      v.cpus = 4
     end
 
-    services.vm.provision "ansible" do |ansible|
+    services.vm.provision "ansible_local" do |ansible|
       ansible.compatibility_mode = "2.0"
+      ansible.install_mode = "pip_args_only"
+      ansible.pip_install_cmd = "sudo apt-get install -y python3-distutils && curl https://bootstrap.pypa.io/get-pip.py | sudo python3"
+      ansible.pip_args = "ansible==#{ANSIBLE_VERSION}"
       ansible.playbook = "deployment/ansible/services.yml"
       ansible.groups = ANSIBLE_GROUPS.merge(ANSIBLE_ENV_GROUPS)
       ansible.raw_arguments = ["--timeout=60"]
+      ansible.extra_vars = MMW_EXTRA_VARS
+    end
+
+    services.vm.provision "shell" do |s|
+      # The base box we use comes with a ~30GB disk. The physical disk is
+      # expanded to 64GB above using vagrant-disksize. The logical disk and
+      # the file system need to be expanded as well to make full use of the
+      # space. `lvextend` expands the logical disk, and `resize2fs` expands
+      # the files system.
+      s.inline = <<-SHELL
+        sudo lvextend -l +100%FREE /dev/ubuntu-vg/ubuntu-lv
+        sudo resize2fs /dev/mapper/ubuntu--vg-ubuntu--lv
+      SHELL
     end
   end
 
   config.vm.define "worker" do |worker|
     worker.vm.hostname = "worker"
-    worker.vm.network "private_network", ip: ENV.fetch("MMW_WORKER_IP", "33.33.34.20")
+    worker.vm.network "private_network", ip: ENV["MMW_WORKER_IP"] || "33.33.34.20"
 
     worker.vm.synced_folder "src/mmw", "/opt/app"
     # Facilitates the sharing of Django media root directories across virtual machines.
     worker.vm.synced_folder ".vagrant/machines/app/virtualbox/media", "/tmp/media",
       create: true
 
     # Path to RWD data (ex. /media/passport/rwd-nhd)
-    worker.vm.synced_folder ENV.fetch("RWD_DATA", "/tmp"), "/opt/rwd-data"
+    worker.vm.synced_folder ENV["RWD_DATA"] || "/tmp", "/opt/rwd-data"
 
     # AWS
     worker.vm.synced_folder "~/.aws", "/var/lib/mmw/.aws"
 
     # Docker
-    worker.vm.network "forwarded_port", {
+    worker.vm.network "forwarded_port", **{
       guest: 2375,
       host: 2375
     }.merge(VAGRANT_NETWORK_OPTIONS)
     # Geoprocessing Service
-    worker.vm.network "forwarded_port", {
+    worker.vm.network "forwarded_port", **{
       guest: 8090,
       host: 8090
     }.merge(VAGRANT_NETWORK_OPTIONS)
@@ -95,35 +134,39 @@ Vagrant.configure("2") do |config|
       v.cpus = 2
     end
 
-    worker.vm.provision "ansible" do |ansible|
+    worker.vm.provision "ansible_local" do |ansible|
       ansible.compatibility_mode = "2.0"
+      ansible.install_mode = "pip_args_only"
+      ansible.pip_install_cmd = "sudo apt-get install -y python3-distutils && curl https://bootstrap.pypa.io/get-pip.py | sudo python3"
+      ansible.pip_args = "ansible==#{ANSIBLE_VERSION}"
       ansible.playbook = "deployment/ansible/workers.yml"
       ansible.groups = ANSIBLE_GROUPS.merge(ANSIBLE_ENV_GROUPS)
       ansible.raw_arguments = ["--timeout=60"]
+      ansible.extra_vars = MMW_EXTRA_VARS
     end
   end
 
   config.vm.define "app" do |app|
     app.vm.hostname = "app"
-    app.vm.network "private_network", ip: ENV.fetch("MMW_APP_IP", "33.33.34.10")
+    app.vm.network "private_network", ip: ENV["MMW_APP_IP"] || "33.33.34.10"
 
     app.vm.synced_folder "src/mmw", "/opt/app"
     # Facilitates the sharing of Django media root directories across virtual machines.
     app.vm.synced_folder ".vagrant/machines/app/virtualbox/media", "/var/www/mmw/media",
       create: true, mount_options: ["dmode=777"]
 
     # Django via Nginx/Gunicorn
-    app.vm.network "forwarded_port", {
+    app.vm.network "forwarded_port", **{
       guest: 80,
       host: 8000
     }.merge(VAGRANT_NETWORK_OPTIONS)
     # Livereload server
-    app.vm.network "forwarded_port", {
+    app.vm.network "forwarded_port", **{
       guest: 35729,
       host: 35729,
     }.merge(VAGRANT_NETWORK_OPTIONS)
     # Testem server
-    app.vm.network "forwarded_port", {
+    app.vm.network "forwarded_port", **{
       guest: 7357,
       host: 7357
     }.merge(VAGRANT_NETWORK_OPTIONS)
@@ -135,22 +178,26 @@ Vagrant.configure("2") do |config|
       v.memory = 2048
     end
 
-    app.vm.provision "ansible" do |ansible|
+    app.vm.provision "ansible_local" do |ansible|
       ansible.compatibility_mode = "2.0"
+      ansible.install_mode = "pip_args_only"
+      ansible.pip_install_cmd = "sudo apt-get install -y python3-distutils && curl https://bootstrap.pypa.io/get-pip.py | sudo python3"
+      ansible.pip_args = "ansible==#{ANSIBLE_VERSION}"
       ansible.playbook = "deployment/ansible/app-servers.yml"
       ansible.groups = ANSIBLE_GROUPS.merge(ANSIBLE_ENV_GROUPS)
       ansible.raw_arguments = ["--timeout=60"]
+      ansible.extra_vars = MMW_EXTRA_VARS
     end
   end
 
   config.vm.define "tiler" do |tiler|
     tiler.vm.hostname = "tiler"
-    tiler.vm.network "private_network", ip: ENV.fetch("MMW_TILER_IP", "33.33.34.35")
+    tiler.vm.network "private_network", ip: ENV["MMW_TILER_IP"] || "33.33.34.35"
 
     tiler.vm.synced_folder "src/tiler", "/opt/tiler"
 
     # Expose the tiler. Tiler is served by Nginx.
-    tiler.vm.network "forwarded_port", {
+    tiler.vm.network "forwarded_port", **{
       guest: 80,
       host: 4000
     }.merge(VAGRANT_NETWORK_OPTIONS)
@@ -159,11 +206,15 @@ Vagrant.configure("2") do |config|
       v.memory = 1024
     end
 
-    tiler.vm.provision "ansible" do |ansible|
+    tiler.vm.provision "ansible_local" do |ansible|
       ansible.compatibility_mode = "2.0"
+      ansible.install_mode = "pip_args_only"
+      ansible.pip_install_cmd = "sudo apt-get install -y python3-distutils && curl https://bootstrap.pypa.io/get-pip.py | sudo python3"
+      ansible.pip_args = "ansible==#{ANSIBLE_VERSION}"
       ansible.playbook = "deployment/ansible/tile-servers.yml"
       ansible.groups = ANSIBLE_GROUPS.merge(ANSIBLE_ENV_GROUPS)
       ansible.raw_arguments = ["--timeout=60"]
+      ansible.extra_vars = MMW_EXTRA_VARS
     end
   end
 end

deployment/ansible/group_vars/all

@@ -1,9 +1,4 @@
 ---
-pip_get_pip_version: "2.7"
-pip_version: "20.3.*"
-
-django_test_database: "{{ lookup('env', 'MMW_TEST_DB_NAME') | default('test_mmw', true) }}"
-
 redis_port: 6379
 postgresql_port: 5432
 
@@ -19,43 +14,46 @@ postgresql_username: mmw
 postgresql_password: mmw
 postgresql_database: mmw
 
-postgresql_version: "9.6"
-postgresql_package_version: "9.6.*.pgdg16.04+1"
+postgresql_version: "13"
+postgresql_package_version: "13.*.pgdg20.04+1"
 postgresql_support_repository_channel: "main"
-postgresql_support_libpq_version: "13.*.pgdg16.04+1"
-postgresql_support_psycopg2_version: "2.7"
-postgis_version: "2.3"
-postgis_package_version: "2.3.*.pgdg16.04+1"
+postgresql_support_libpq_version: "13.*.pgdg20.04+1"
+postgresql_support_psycopg2_version: "2.8.*"
+postgis_version: "3"
+postgis_package_version: "3.2*pgdg20.04+1"
 
-daemontools_version: "1:0.76-6ubuntu1"
+daemontools_version: "1:0.76-7"
 
-python_version: "2.7.12-1~16.04"
+python_version: "3.8.*"
+ansible_python_interpreter: "/usr/bin/python3"
 
 yarn_version: "1.19.*"
 app_nodejs_version: "12.11.1"
 app_nodejs_npm_version: "6.9.0"
 tiler_nodejs_version: "10.16.0"
-tiler_nodejs_npm_version: "6.9.0"
+tiler_nodejs_npm_version: "7.20.6"
 
 java_version: "8u*"
 java_major_version: "8"
 java_flavor: "openjdk"
 
-docker_version: "5:18.*"
-docker_compose_version: "1.23.*"
+docker_version: "5:19.*"
+docker_python_version: "4.4.*"
+docker_compose_version: "1.26.*"
 
 geop_host: "localhost"
 geop_port: 8090
 
-geop_version: "4.0.3"
+geop_version: "5.2.0"
 geop_cache_enabled: 1
+geop_timeout: 200
 
 nginx_cache_dir: "/var/cache/nginx"
 
 enabled_features: ''
 
-llvmlite_version: "0.31.0"
-numba_version: "0.38.1"
+llvmlite_version: "0.37.0"
+numba_version: "0.54.0"
 phantomjs_version: "2.1.*"
 
-redis_version: "2:3.0.6-1ubuntu0.*"
+redis_version: "5:5.0.*"

deployment/ansible/group_vars/development

@@ -9,28 +9,18 @@ postgresql_hba_mapping:
   - { type: "host", database: "all", user: "all", address: "33.33.34.1/24", method: "md5" }
   - { type: "host", database: "all", user: "all", address: "10.0.2.0/24", method: "md5" }
 
-services_ip: "{{ lookup('env', 'MMW_SERVICES_IP') | default('33.33.34.30', true) }}"
-
 redis_host: "{{ services_ip }}"
 postgresql_host: "{{ services_ip }}"
-tiler_host: "{{ lookup('env', 'MMW_TILER_IP') | default('33.33.34.35', true) }}"
 
 celery_log_level: "DEBUG"
 celery_number_of_workers: 2
 celery_processes_per_worker: 1
 
 itsi_base_url: "https://learn.staging.concord.org/"
-itsi_secret_key: "{{ lookup('env', 'MMW_ITSI_SECRET_KEY') }}"
-
-concord_secret_key: "{{ lookup('env', 'MMW_CONCORD_SECRET_KEY') }}"
 
 hydroshare_base_url: "https://beta.hydroshare.org/"
-hydroshare_secret_key: "{{ lookup('env', 'MMW_HYDROSHARE_SECRET_KEY') }}"
 
 srat_catchment_api_url: "https://802or9kkk2.execute-api.us-east-2.amazonaws.com/prod/SratRunModel_DEV"
-srat_catchment_api_key: "{{ lookup('env', 'MMW_SRAT_CATCHMENT_API_KEY') }}"
-
-tilecache_bucket_name: "{{ lookup('env', 'MMW_TILECACHE_BUCKET') | default('', true) }}"
 
 docker_options: "-H tcp://0.0.0.0:2375 -H unix:///var/run/docker.sock"
 

deployment/ansible/group_vars/test

@@ -7,25 +7,17 @@ postgresql_listen_addresses: "*"
 postgresql_hba_mapping:
   - { type: "host", database: "all", user: "all", address: "33.33.34.1/24", method: "md5" }
 
-services_ip: "{{ lookup('env', 'MMW_SERVICES_IP') | default('33.33.34.30', true) }}"
-
 redis_host: "{{ services_ip }}"
 postgresql_host: "{{ services_ip }}"
-tiler_host: "{{ lookup('env', 'MMW_TILER_IP') | default('33.33.34.35', true) }}"
 
 celery_number_of_workers: 2
 celery_processes_per_worker: 1
 
 itsi_base_url: "https://learn.staging.concord.org/"
-itsi_secret_key: "{{ lookup('env', 'MMW_ITSI_SECRET_KEY') }}"
-
-concord_secret_key: "{{ lookup('env', 'MMW_CONCORD_SECRET_KEY') }}"
 
 hydroshare_base_url: "https://beta.hydroshare.org/"
-hydroshare_secret_key: "{{ lookup('env', 'MMW_HYDROSHARE_SECRET_KEY') }}"
 
 srat_catchment_api_url: "https://802or9kkk2.execute-api.us-east-2.amazonaws.com/prod/SratRunModel_DEV"
-srat_catchment_api_key: "{{ lookup('env', 'MMW_SRAT_CATCHMENT_API_KEY') }}"
 
 tilecache_bucket_name: "tile-cache.staging.app.wikiwatershed.org"