Bulk upload facilities V1

* Handle both .csv & .xlsx file uploads
* Fixes bs-custom-file-input UI filename issue as per:
bootstrap-ruby/bootstrap_form#528
* Add validation to ensure file contains at least one facility

Gemfile

@@ -39,6 +39,7 @@ gem 'whenever', require: false
 gem 'redis'
 gem 'redis-rails'
 gem 'activerecord-import'
+gem "roo", "~> 2.8.0"
 
 group :development, :test do
   gem 'byebug', platforms: [:mri, :mingw, :x64_mingw]

Gemfile.lock

@@ -287,6 +287,9 @@ GEM
     rgeo (1.1.2)
     rgeo-geojson (2.1.1)
       rgeo (>= 1.0.0)
+    roo (2.8.2)
+      nokogiri (~> 1)
+      rubyzip (>= 1.2.1, < 2.0.0)
     rspec-core (3.7.1)
       rspec-support (~> 3.7.0)
     rspec-expectations (3.7.0)
@@ -318,6 +321,7 @@ GEM
       actionpack (>= 3.1, < 6.0)
       railties (>= 3.1, < 6.0)
     ruby_dep (1.5.0)
+    rubyzip (1.2.3)
     safe_yaml (1.0.5)
     sassc (2.0.0)
       ffi (~> 1.9.6)
@@ -436,6 +440,7 @@ DEPENDENCIES
   rails-controller-testing
   redis
   redis-rails
+  roo (~> 2.8.0)
   rspec-rails (~> 3.7)
   rswag (~> 1.6.0)
   sassc-rails
@@ -454,4 +459,4 @@ DEPENDENCIES
   whenever
 
 BUNDLED WITH
-   1.17.1
+   1.17.3

app/assets/javascripts/application.js

@@ -14,4 +14,6 @@
 //= require jquery3
 //= require popper
 //= require bootstrap-sprockets
+//= require bs-custom-file-input.js
+//= require bs-file-input-init.js
 //= require_tree .

app/assets/javascripts/bs-file-input-init.js

@@ -0,0 +1,3 @@
+$(document).ready(function () {
+    bsCustomFileInput.init();
+});

app/controllers/admin/facilities_controller.rb

@@ -1,6 +1,10 @@
 class Admin::FacilitiesController < AdminController
+  include FileUploadable
   before_action :set_facility, only: [:show, :edit, :update, :destroy]
   before_action :set_facility_group, only: [:show, :new, :create, :edit, :update, :destroy]
+  before_action :initialize_upload, :validate_file_type, :validate_file_size, :parse_file,
+                :validate_at_least_one_facility, :validate_duplicate_rows, :validate_facilities,
+                :if => :file_exists?, only: [:upload]
 
   def index
     authorize Facility
@@ -42,6 +46,17 @@ def destroy
     redirect_to admin_facilities_url, notice: 'Facility was successfully deleted.'
   end
 
+  def upload
+    authorize Facility
+    return render :upload, :status => :bad_request if @errors.present?
+
+    if @facilities.present?
+      ImportFacilitiesJob.perform_later(@facilities)
+      flash.now[:notice] = 'File upload successful, your facilities will be created shortly.'
+    end
+    render :upload
+  end
+
   private
 
   def set_facility
@@ -67,4 +82,47 @@ def facility_params
       :longitude
     )
   end
+
+  def initialize_upload
+    @errors = []
+    @file = params.require(:upload_facilities_file)
+  end
+
+  def parse_file
+    return render :upload, :status => :bad_request if @errors.present?
+
+    @file_contents = read_xlsx_or_csv_file(@file)
+    @facilities = Facility.parse_facilities(@file_contents)
+  end
+
+  def validate_at_least_one_facility
+    @errors << "Uploaded file doesn't contain any valid facilities" if @facilities.blank?
+  end
+
+  def validate_duplicate_rows
+    facilities_slice = @facilities.map { |facility|
+      facility.slice(:organization_name, :facility_group_name, :name) }
+    @errors << 'Uploaded file has duplicate facilities' if
+        facilities_slice.count != facilities_slice.uniq.count
+  end
+
+  def validate_facilities
+    row_num = 2
+    @facilities.each do |facility|
+      import_facility = Facility.new(facility)
+      if import_facility.invalid?
+        row_errors = import_facility.errors.full_messages.to_sentence
+        @errors << "Row #{row_num}: #{row_errors}" if row_errors.present?
+      end
+      row_num += 1
+    end
+  end
+
+  def file_exists?
+    params[:upload_facilities_file].present?
+  end
+
+  def file_valid?
+    params[:upload_facilities_file].present? && @errors.blank?
+  end
 end

app/controllers/concerns/file_uploadable.rb

@@ -0,0 +1,29 @@
+module FileUploadable
+  extend ActiveSupport::Concern
+
+  VALID_MIME_TYPES = %w[text/csv application/vnd.openxmlformats-officedocument.spreadsheetml.sheet].freeze
+
+  def read_xlsx_or_csv_file(file)
+    file_contents = ''
+    if file.content_type == 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
+      xlsx = Roo::Spreadsheet.open(file.path)
+      file_contents = xlsx.to_csv
+    elsif file.content_type == 'text/csv'
+      file_contents = file.read
+    end
+  end
+
+  def initialize_upload
+    @errors = []
+    @file = params.require(:file)
+  end
+
+  def validate_file_type
+    @errors << 'File type not supported, please upload a csv or xlsx file instead' if
+        VALID_MIME_TYPES.exclude?(@file.content_type)
+  end
+
+  def validate_file_size
+    @errors << 'File is too big, must be smaller than 5MB' if @file.size > 5.megabytes
+  end
+end

app/jobs/import_facilities_job.rb

@@ -0,0 +1,17 @@
+class ImportFacilitiesJob < ApplicationJob
+  queue_as :default
+  self.queue_adapter = :sidekiq
+
+  def perform(facilities)
+
+    import_facilities = []
+    facilities.each do |facility|
+      organization = Organization.find_by(name: facility[:organization_name])
+      facility_group = FacilityGroup.find_by(name: facility[:facility_group_name],
+                                             organization_id: organization.id)
+      import_facility = Facility.new(facility.merge!(facility_group_id: facility_group.id))
+      import_facilities << import_facility
+    end
+    Facility.import!(import_facilities, validate: true)
+  end
+end

app/models/facility.rb

@@ -1,8 +1,14 @@
+require 'roo'
+
 class Facility < ApplicationRecord
   include Mergeable
   include PatientSetAnalyticsReportable
   extend FriendlyId
 
+  attribute :import, :boolean, default: false
+  attribute :organization_name, :string
+  attribute :facility_group_name, :string
+
   belongs_to :facility_group, optional: true
 
   has_many :users, foreign_key: 'registration_facility_id'
@@ -20,6 +26,14 @@ class Facility < ApplicationRecord
   validates :country, presence: true
   validates :pin, numericality: true, allow_blank: true
 
+  with_options if: :import do |facility|
+    facility.validates :organization_name, presence: true
+    facility.validates :facility_group_name, presence: true
+    facility.validate  :organization_exists
+    facility.validate  :facility_group_exists
+    facility.validate  :facility_is_unique
+  end
+
   delegate :protocol, to: :facility_group, allow_nil: true
   delegate :organization, to: :facility_group, allow_nil: true
 
@@ -28,4 +42,54 @@ class Facility < ApplicationRecord
   def report_on_patients
     registered_patients
   end
+
+  def self.parse_facilities(file_contents)
+    facilities = []
+    CSV.parse(file_contents, headers: true, converters: :strip_whitespace) do |row|
+      facility = {organization_name: row['organization'],
+                  facility_group_name: row['facility_group'],
+                  name: row['facility_name'],
+                  facility_type: row['facility_type'],
+                  street_address: row['street_address (optional)'],
+                  village_or_colony: row['village_or_colony (optional)'],
+                  district: row['district'],
+                  state: row['state'],
+                  country: row['country'],
+                  pin: row['pin (optional)'],
+                  latitude: row['latitude (optional)'],
+                  longitude: row['longitude (optional)'],
+                  import: true}
+      next if facility.except(:import).values.all?(&:blank?)
+
+      facilities << facility
+    end
+    facilities
+  end
+
+  def organization_exists
+    organization = Organization.find_by(name: organization_name)
+    errors.add(:organization, "doesn't exist") if
+        organization_name.present? && organization.blank?
+  end
+
+  def facility_group_exists
+    organization = Organization.find_by(name: organization_name)
+    facility_group = FacilityGroup.find_by(name: facility_group_name, organization_id: organization.id) if
+        organization.present?
+    errors.add(:facility_group, "doesn't exist for the organization") if
+        organization.present? && facility_group_name.present? && facility_group.blank?
+
+  end
+
+  def facility_is_unique
+    organization = Organization.find_by(name: organization_name)
+    facility_group = FacilityGroup.find_by(name: facility_group_name, organization_id: organization.id) if
+        organization.present?
+    facility = Facility.find_by(name: name, facility_group: facility_group.id) if facility_group.present?
+    errors.add(:facility, 'already exists') if
+        organization.present? && facility_group.present? && facility.present?
+
+  end
+
+  CSV::Converters[:strip_whitespace] = ->(value) { value.strip rescue value }
 end

app/policies/facility_policy.rb

@@ -31,6 +31,10 @@ def graphics?
     show?
   end
 
+  def upload?
+    user.owner?
+  end
+
   private
 
   def destroyable?

app/views/admin/facilities/index.html.erb

@@ -1,9 +1,15 @@
 <% if policy(FacilityGroup).new? %>
-  <%= link_to '+ Facility group', new_admin_facility_group_path, class: "btn btn-sm btn-primary float-right" %>
+  <div class="page-header">
+    <h1 class="page-title"">All facilities</h1>
+    <nav class="page-nav">
+      <% if policy(Facility).upload? %>
+        <%= link_to '+ Upload Facilities CSV', upload_admin_facilities_path, class: "btn btn-sm btn-default" %>
+      <% end %>
+      <%= link_to '+ Facility group', new_admin_facility_group_path, class: "btn btn-sm btn-primary" %>
+    </nav>
+  </div>
 <% end %>
 
-<h1 style="margin-bottom: 32px;">All facilities</h1>
-
 <% @organizations.order(:name).each do |organization| %>
   <% if @organizations.size > 1 %><h1><%= organization.name %></h1><% end %>
 

app/views/admin/facilities/upload.html.erb

@@ -0,0 +1,31 @@
+<h1>Upload Facilities</h1>
+<% if @errors.present? %>
+  <div class="alert alert-warning">
+    <p>Please fix the errors below and try again:</p>
+    <ul>
+      <% @errors.each do |error| %>
+        <li> <%= error %> </li>
+      <% end %>
+    </ul>
+    <p>You can also contact [email protected] for assistance</p>
+  </div>
+<% end %>
+<h3>Instructions:</h3>
+<ol>
+  <li>Download and fill in <%= link_to 'this template', '/documents/upload_facilities.csv'%> to create multiple facilities via file upload.</li>
+  <li>Organizations for the facilities must already exist.
+    <% if policy(Organization).new? %>
+      Click here to create an organization:
+      <%= link_to '+ Organization', new_admin_organization_path, class: "btn btn-sm btn-outline-primary" %>
+    <% end %></li>
+  <li>Facility Groups for the facilities must already exist.
+    <% if policy(FacilityGroup).new? %>
+      Click here to create a facility group:
+      <%= link_to '+ Facility group', new_admin_facility_group_path, class: "btn btn-sm btn-outline-primary" %>
+    <% end %></li>
+  <li>Ensure that the Organization (Column A) and Facility Group (Column B) are entered correctly.</li>
+</ol>
+<%= bootstrap_form_tag(url: upload_admin_facilities_url, multipart: true) do |f| %>
+  <%= f.file_field :upload_facilities_file, required: true, accept: '.csv, .xlsx', label:'Upload your filled in facilities file'%>
+  <%= f.primary 'Upload' %>
+<% end %>

config/routes.rb

@@ -163,7 +163,12 @@
 
     resources :organizations
 
-    resources :facilities, only: [:index]
+    resources :facilities, only: [:index] do
+      collection do
+        get 'upload'
+        post 'upload'
+      end
+    end
     resources :facility_groups do
       resources :facilities
     end

public/documents/upload_facilities.csv

@@ -0,0 +1 @@
+organization,facility_group,facility_name,facility_type,street_address (optional),village_or_colony (optional),district,state,country,pin (optional),longitude (optional),latitude (optional)