Merge pull request #339 from CityOfNewYork/feature/OP-1419

Feature/OP-1419: Create OpenRecords v2.2 Release

.env.example

@@ -10,9 +10,16 @@ APP_VERSION_STRING=<STRING OF APP VERSION>
 LOGFILE_DIRECTORY=<PATH TO LOG DIRECTORY>
 
 # Data
-AGENCY_DATA=<FULL PATH TO "agencies.csv">
+AGENCY_DATA=<FULL PATH TO "agencies.json">
+LETTER_TEMPLATES_DATA=<FULL PATH TO "letter_templates.csv">
 REASON_DATA=<FULL PATH TO "reasons.csv">
 STAFF_DATA=<FULL PATH TO "staff.csv">
+ENVELOPE_TEMPLATES_DATA=<FULL PATH TO "envelope_templates.csv">
+LATEX_TEMPLATE_DIRECTORY=<FULL PATH TO "latex" directory>
+
+
+# JSON Schema
+JSON_SCHEMA_DIRECTORY=<FULL PATH TO "schemas directory">
 
 # Database
 DATABASE_URL=postgresql://<DATABASE USERNAME>:<DATABASE PASSWORD>@<DATABASE HOSTNAME>:<DATABASE PORT>/<DATABASE NAME>
@@ -80,3 +87,7 @@ LDAP_BASE_DN=<LDAP SEARCH BASE>
 # ReCaptcha
 RECAPTCHA_SITE_KEY=<SITE KEY>
 RECAPTCHA_SECRET_KEY=<SECRET KEY>
+
+# Sentry
+SENTRY_DSN=<SENTRY DSN>
+USE_SENTRY=False

.gitignore

@@ -9,7 +9,8 @@ logs/
 received-emails/
 ldap/
 Vagrantfile
-.vagrant/
+Vagrantfile.aws
+.vagrant*
 *.pem
 *.crt
 *.key

README.md

@@ -1,4 +1,4 @@
-[![Codacy Badge](https://api.codacy.com/project/badge/Grade/64dc52651e994c6d995b68be84823cb0)](https://www.codacy.com/app/joel-castillo/openrecords_v2_0?utm_source=github.com&utm_medium=referral&utm_content=joelbcastillo/openrecords_v2_0&utm_campaign=Badge_Grade) [![Codacy Badge](https://api.codacy.com/project/badge/Coverage/64dc52651e994c6d995b68be84823cb0)](https://www.codacy.com/app/joel-castillo/openrecords_v2_0?utm_source=github.com&utm_medium=referral&utm_content=joelbcastillo/openrecords_v2_0&utm_campaign=Badge_Coverage) [![Requirements Status](https://requires.io/github/joelbcastillo/openrecords_v2_0/requirements.svg?branch=develop)](https://requires.io/github/joelbcastillo/openrecords_v2_0/requirements/?branch=develop) 
+[![Codacy Badge](https://api.codacy.com/project/badge/Grade/2b97fe8319344d699a4bbba48827637b)](https://www.codacy.com/app/NYCRecords/NYCOpenRecords?utm_source=github.com&utm_medium=referral&utm_content=CityOfNewYork/NYCOpenRecords&utm_campaign=Badge_Grade) [![Requirements Status](https://requires.io/github/joelbcastillo/openrecords_v2_0/requirements.svg?branch=develop)](https://requires.io/github/joelbcastillo/openrecords_v2_0/requirements/?branch=develop) 
 
 
 # NYC OpenRecords

Vagrantfile.example

@@ -1,28 +1,34 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 
-$redhat_username = <REDHAT USERNAME>
-$redhat_password = <REDHAT PASSWORD>
+$default_system_name = "OpenRecords_" + Time.now.strftime("%m-%d-%Y_%H-%M")
+
+$redhat_username = ENV['RH_USER'] || "<INSERT_USERNAME_HERE>"
+$redhat_password = ENV['RH_PASSWORD'].to_s.empty? ? "<INSERT_PASSWORD_HERE>" : ENV['RH_PASSWORD']
+$rhsn_system_name = ENV['RHSN_SYSTEM_NAME'].to_s.empty? ? $default_system_name : ENV['RHSN_SYSTEM_NAME']
 
 Vagrant.configure("2") do |config|
-  config.vm.box = "rhel-6.8"
+  config.vm.box = "rhel-6.8_vb-5.1.30"
 
   config.vm.provider "virtualbox" do |vb|
     # Customize the amount of memory on the VM:
     vb.memory = "4096"
+    vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
   end
 
-  # Proxy Configuration
-  config.vm.provision "shell" do |s|
-    s.inline = 'sudo cp /vagrant/build_scripts/proxy.sh /etc/profile.d/'
-  end
+  if ENV['http_proxy']
+    # Proxy Configuration
+    config.vm.provision "shell" do |s|
+      s.inline = 'sudo cp /vagrant/build_scripts/proxy.sh /etc/profile.d/'
+    end
 
-  config.vm.provision :reload
+    config.vm.provision :reload
+  end
 
   # RedHat Subscription Management
   config.vm.provision "shell" do |s|
     s.path = "build_scripts/subscription_manager.sh"
-    s.args = [$redhat_username, $redhat_password]
+    s.args = [$redhat_username, $redhat_password, $rhsn_system_name]
   end
 
   config.vm.provision :reload
@@ -41,44 +47,59 @@ Vagrant.configure("2") do |config|
     default.vm.network "private_network", ip: "10.0.0.2"
     default.vm.provision "shell", path: "build_scripts/web_setup/web_setup.sh", args: 'single_server'
     default.vm.provision "shell", path: "build_scripts/app_setup/app_setup.sh"
+    default.vm.provision "shell", path: "build_scripts/filebeat_setup/app_nginx/install.sh"
     default.vm.provision "shell", path: "build_scripts/es_setup/es_setup.sh", args: 'single_server'
-    default.vm.provision "shell", path: "build_scripts/db_setup/db_setup.sh"
-  end
-
-
-  config.vm.define "web_1", autostart: false do |web|
-    # Web Server 1 Configuration
+    default.vm.provision "shell", path: "build_scripts/db_setup/db_setup.sh", args: 'single_server'
   end
 
-  config.vm.define "web_2", autostart: false do |web|
-    # Web Server 1 Configuration
+  # VM for Sentry
+  config.vm.define "sentry" do |sentry|
+    sentry.vm.provider "virtualbox" do |vb|
+      vb.memory = "2048"
+    end
+
+    sentry.vm.network "forwarded_port", guest: 9000,  host: 9090
+    sentry.vm.network "private_network", ip: "10.0.0.3"
+    sentry.vm.provision "shell", path: "build_scripts/sentry_setup/nginx_setup.sh"
+    sentry.vm.provision "shell", path: "build_scripts/sentry_setup/db_setup_install.sh"
+    sentry.vm.provision "shell", path: "build_scripts/sentry_setup/db_setup_config.sh"
+    sentry.vm.provision "shell", path: "build_scripts/sentry_setup/setup.sh"
   end
 
-  config.vm.define "app_1", autostart: false do |app|
-    # App Server 1 Configuration
+  # VM for ELK
+  config.vm.define "elk" do |elk|
+    elk.vm.provider "virtualbox" do |vb|
+      vb.memory = "4096"
+    end
+
+    elk.vm.network "forwarded_port", guest: 9200, host: 9201
+    elk.vm.network "forwarded_port", guest: 5601, host: 5602
+    elk.vm.network "forwarded_port", guest: 5044, host: 5044
+    elk.vm.network "private_network", ip: "10.0.0.4"
+    elk.vm.provision "shell", path: "build_scripts/elk_setup/elasticsearch/setup.sh"
+    elk.vm.provision "shell", path: "build_scripts/elk_setup/kibana/setup.sh"
+    elk.vm.provision "shell", path: "build_scripts/elk_setup/logstash/setup.sh"
+    elk.vm.provision "shell", path: "build_scripts/elk_setup/nginx/setup.sh"
   end
 
-  config.vm.define "app_2", autostart: false do |app|
-    # App Server 2 Configuration
+  config.trigger.before :destroy, :force => true, :vm => ["default"] do  # add more VMs if necessary
+    run_remote "subscription-manager unsubscribe --all"
+    run_remote "subscription-manager unregister"
   end
 
-  config.vm.define "db_1", autostart: false do |db|
-    # Database Server 1 Configuration
+  # VM for Consul/Vault
+  config.vm.define "consul" do |consul|
+    consul.vm.provider "virtualbox" do |vb|
+      vb.memory = "2048"
+    end
+
+    consul.vm.network "forwarded_port", guest: 443, host: 8443
+    consul.vm.network "forwarded_port", guest: 8500, host: 8501
+    consul.vm.network "forwarded_port", guest: 8200, host: 8201
+    consul.vm.network "private_network", ip: "10.0.0.5"
+    consul.vm.provision "shell", path: "build_scripts/consul-vault_setup/consul_setup.sh"
+    consul.vm.provision "shell", path: "build_scripts/consul-vault_setup/vault_setup.sh"
+    consul.vm.provision "shell", path: "build_scripts/consul-vault_setup/nginx/nginx_setup.sh"
   end
 
-  config.vm.define "db_2", autostart: false do |db|
-    # Database Server 2 Configuration
-  end
-
-  config.vm.define "es_1", autostart: false do |es|
-    # Elasticsearch Server 1 Configuration
-  end
-
-  config.vm.define "es_2", autostart: false do |es|
-    # Elasticsearch Server 1 Configuration
-  end
-
-  config.vm.define "es_3", autostart: false do |es|
-    # Elasticsearch Server 1 Configuration
-  end
 end

app/__init__.py

@@ -33,6 +33,7 @@
 from app.constants import OPENRECORDS_DL_EMAIL
 
 from config import config, Config
+from raven.contrib.flask import Sentry
 
 recaptcha = ReCaptcha()
 bootstrap = Bootstrap()
@@ -45,8 +46,9 @@
 login_manager = LoginManager()
 scheduler = APScheduler()
 store = RedisStore(redis.StrictRedis(db=Config.SESSION_REDIS_DB, host=Config.REDIS_HOST, port=Config.REDIS_PORT))
-prefixed_store = PrefixDecorator('session_', store)
+session_redis = PrefixDecorator('session_', store)
 celery = Celery(__name__, broker=Config.CELERY_BROKER_URL)
+sentry = Sentry()
 
 upload_redis = redis.StrictRedis(db=Config.UPLOAD_REDIS_DB, host=Config.REDIS_HOST, port=Config.REDIS_PORT)
 email_redis = redis.StrictRedis(db=Config.EMAIL_REDIS_DB, host=Config.REDIS_HOST, port=Config.REDIS_PORT)
@@ -117,14 +119,16 @@ def create_app(config_name, jobs_enabled=True):
     login_manager.init_app(app)
     mail.init_app(app)
     celery.conf.update(app.config)
+    sentry.init_app(app, logging=app.config["USE_SENTRY"], level=logging.INFO)
+
     if jobs_enabled:
         scheduler.init_app(app)
 
     with app.app_context():
         from app.models import Anonymous
         login_manager.login_view = 'auth.login'
         login_manager.anonymous_user = Anonymous
-        KVSessionExtension(prefixed_store, app)
+        KVSessionExtension(session_redis, app)
 
     # schedule jobs
     if jobs_enabled:
@@ -225,6 +229,9 @@ def add_session_config():
     from .agency import agency
     app.register_blueprint(agency, url_prefix="/agency")
 
+    from .agency.api import agency_api_blueprint
+    app.register_blueprint(agency_api_blueprint, url_prefix="/agency/api/v1.0")
+
     from .search import search
     app.register_blueprint(search, url_prefix="/search")
 

app/admin/forms.py

@@ -38,20 +38,19 @@ def __init__(self, agency_ein=None):
             ]
         else:
             # Multi-Agency Admin Users will only see the agencies that they administer in the dropdown.
-            self.agencies.choices = [
-                (
-                    agency.ein,
-                    '({status}) {agency_name}'.format(
-                        status='ACTIVE',
-                        agency_name=agency.name
-                    ) if agency.is_active else
-                    '{agency_name}'.format(
-                        agency_name=agency.name
-                    )
-                )
-                for agency in current_user.agencies.order_by(Agencies.is_active.desc(),
-                                                             Agencies._name.asc()).all()
-            ]
+            agency_choices = []
+            for agency in current_user.agencies.order_by(Agencies.is_active.desc(), Agencies._name.asc()).all():
+                if current_user.is_agency_admin(agency.ein):
+                    agency_tuple = (agency.ein,
+                                    '({status}) {agency_name}'.format(
+                                        status='ACTIVE',
+                                        agency_name=agency.name
+                                    ) if agency.is_active else
+                                    '{agency_name}'.format(
+                                        agency_name=agency.name)
+                                    )
+                    agency_choices.append(agency_tuple)
+            self.agencies.choices = agency_choices
         if agency_ein:
             for agency in self.agencies.choices:
                 if agency[0] == agency_ein:

app/admin/views.py

@@ -11,19 +11,18 @@
 )
 from flask import render_template, abort
 from flask_login import current_user
-from app.constants import user_type_auth
 from app.admin.utils import get_agency_active_users
 
 
-# TODO: View function to handle updates to agency wide settings (see models.py:183
+# TODO: View function to handle updates to agency wide settings (see models.py:183)
 
 
 @admin.route('/')
 @admin.route('/<agency_ein>')
 def main(agency_ein=None):
     if not current_user.is_anonymous:
         if agency_ein is None:
-            agency_ein = current_user.default_agency_ein
+            agency_ein = current_user.find_admin_agency_ein
         if current_user.is_super:
             agency_form = SelectAgencyForm(agency_ein)
             agency_ein = agency_ein or agency_form.agencies.choices[0][0]

app/agency/api/__init__.py

@@ -0,0 +1,5 @@
+from flask import Blueprint
+
+agency_api_blueprint = Blueprint('agency_api_blueprint', __name__)
+
+from . import views

app/agency/api/utils.py

@@ -0,0 +1,72 @@
+from itertools import groupby
+from operator import itemgetter
+
+from app.models import (
+    Agencies,
+    LetterTemplates
+)
+
+
+def get_active_users_as_choices(agency_ein):
+    """
+    Retrieve a list of users that are active for a given agency
+    :param agency_ein: Agency EIN (String)
+    :return: A list of user tuples (id, name)
+    """
+    active_users = sorted(
+        [(user.get_id(), user.name)
+         for user in Agencies.query.filter_by(ein=agency_ein).one().active_users],
+        key=lambda x: x[1])
+    active_users.insert(0, ('', 'All'))
+    return active_users
+
+
+def get_letter_templates(agency_ein, template_type=None):
+    """
+    Retrieve letter templates for the specified agency as a JSON object. If template type is provided, only get
+    templates of that type.
+
+    :param agency_ein: Agency EIN (String)
+    :param template_type: One of "acknowledgment", "denial", "closing", "letter", "extension", "re-opening" (String)
+    :return: Dictionary
+
+        {
+            'type_': [(template_id, template_name),...]
+        }
+    """
+    if template_type is not None:
+        templates = LetterTemplates.query.with_entities(LetterTemplates.id, LetterTemplates.title,
+                                                        LetterTemplates.type_).filter(
+            LetterTemplates.type_ == template_type).all()
+    else:
+        templates = LetterTemplates.query.with_entities(LetterTemplates.id, LetterTemplates.title,
+                                                        LetterTemplates.type_).filter_by(agency_ein=agency_ein).all()
+
+    templates = list(_group_templates(templates))
+
+    template_dict = {}
+
+    for i in templates:
+        type_ = i[0]
+        vals = i[1]
+
+        template_dict[type_] = []
+
+        for i in vals:
+            template_dict[type_].append((i[0], i[1]))
+
+    if template_type is not None:
+        return template_dict[template_type]
+    return template_dict
+
+
+def _group_templates(templates):
+    """
+    Group a list of templates by their type
+    :param templates: List of templates (template.id, template.title, template.type_)
+    :return: a generator containing each grouped template type
+    """
+    grouped = groupby(templates, itemgetter(2))
+
+    for key, sub_iter in grouped:
+        yield key, list(sub_iter)