Merge pull request #3069 from avalonmediasystem/staging

6.4.4 release

Gemfile

@@ -40,6 +40,7 @@ gem 'avalon-about', git: 'https://github.com/avalonmediasystem/avalon-about.git'
 gem 'bootstrap-toggle-rails', git: 'https://github.com/rkallensee/bootstrap-toggle-rails.git', tag: 'v2.2.1.0'
 gem 'bootstrap_form'
 gem 'speedy-af', '~> 0.1.1'
+gem 'recaptcha', require: 'recaptcha/rails'
 
 # Avalon Components
 gem 'avalon-workflow', git: "https://github.com/avalonmediasystem/avalon-workflow.git", tag: 'avalon-r6.2'

Gemfile.lock

@@ -633,6 +633,8 @@ GEM
       rdf (~> 2.1)
     rdf-xsd (2.2.0)
       rdf (~> 2.1)
+    recaptcha (4.12.0)
+      json
     redis (3.3.5)
     redis-actionpack (5.0.2)
       actionpack (>= 4.0, < 6)
@@ -724,7 +726,7 @@ GEM
       mime-types
       nokogiri
       rest-client
-    rubyzip (1.2.1)
+    rubyzip (1.2.2)
     rufus-scheduler (3.4.2)
       et-orbi (~> 1.0)
     safe_yaml (1.0.4)
@@ -920,6 +922,7 @@ DEPENDENCIES
   rb-readline
   rdf (~> 2.2)
   rdf-rdfxml
+  recaptcha
   redis-rails
   resque (~> 1.27.0)
   resque-scheduler (~> 4.3.0)

app/controllers/comments_controller.rb

@@ -29,17 +29,16 @@ def create
     @comment.subject = params[:comment][:subject]
     @comment.comment = params[:comment][:comment]
 
-    if (@comment.valid?)
+    if @comment.valid? and (Settings.recaptcha.blank? or verify_recaptcha(model: @comment))
       begin
         CommentsMailer.contact_email(@comment.to_h).deliver_later
       rescue Errno::ECONNRESET => e
         logger.warn "The mail server does not appear to be responding \n #{e}"
-
-	flash[:notice] = "The message could not be sent in a timely fashion. Contact us at #{Settings.email.support} to report the problem."
-	render action: "index"
+        flash[:notice] = "The message could not be sent in a timely fashion. Contact us at #{Settings.email.support} to report the problem."
+        render action: "index"
       end
     else
-      flash[:error] = "There were problems submitting your comment. Please correct the errors and try again."
+      flash[:error] = "Your comment was missing required information. Please complete all fields and resubmit."
       render action: "index"
     end
   end

app/controllers/media_objects_controller.rb

@@ -255,7 +255,7 @@ def update_media_object
       end
 
       if error_messages.empty?
-        if api_params[:replace_master_files]
+        if api_params[:replace_masterfiles]
           old_ordered_master_files.each do |mf|
             p = MasterFile.find(mf)
             @media_object.master_files.delete(p)
@@ -272,9 +272,13 @@ def update_media_object
         if !@media_object.save
           error_messages += ['Failed to create media object:']+@media_object.errors.full_messages
           @media_object.destroy
-        elsif !!api_params[:publish]
-          @media_object.publish!('REST API')
-          @media_object.workflow.publish
+        else
+          if !!api_params[:publish]
+            @media_object.publish!('REST API')
+            @media_object.workflow.publish
+          else
+            @media_object.publish!('')
+          end
         end
       end
     end
@@ -607,6 +611,6 @@ def master_files_params
   end
 
   def api_params
-    params.permit(:collection_id, :publish, :import_bib_record, :replace_master_files)
+    params.permit(:collection_id, :publish, :import_bib_record, :replace_masterfiles)
   end
 end

app/jobs/bulk_action_jobs.rb

@@ -162,6 +162,8 @@ def perform documents, params
     end
   end
 
+  require 'avalon/intercom'
+
   class IntercomPush < ActiveJob::Base
     def perform documents, user_key, params
       errors = []

app/jobs/cleanup_working_file_job.rb

@@ -15,7 +15,12 @@
 class CleanupWorkingFileJob < ActiveJob::Base
   def perform(masterfile_id)
     masterfile = MasterFile.find(masterfile_id)
-    path = masterfile.working_file_path
-    File.delete(path) if File.exist?(path)
+    masterfile.working_file_path.map do |path|
+      parent_directory = File.dirname(path)
+      File.delete(path) if File.exist?(path)
+      Dir.delete(parent_directory) if Dir.exist?(parent_directory)
+    end
+    masterfile.working_file_path = nil
+    masterfile.save!
   end
 end

app/jobs/ingest_batch_status_email_jobs.rb

@@ -24,7 +24,7 @@ def perform
         # Get the entries for the batch and see if they all complete
         complete = true
         errors = false
-        BatchEntries.where(batch_registries_id: br.id).each do |entry|
+        br.batch_entries.each do |entry|
           complete = false unless entry.complete || entry.error
           errors = true if entry.error
         end

app/models/ability.rb

@@ -219,7 +219,7 @@ def is_editor_of?(collection)
   end
 
   def is_member_of_any_collection?
-    @user.id.present? && Admin::Collection.where("inheritable_edit_access_person_ssim" => @user.user_key).first.present?
+    @user.id.present? && Admin::Collection.exists?("inheritable_edit_access_person_ssim" => @user.user_key)
   end
 
   def full_login?

app/models/avalon/rdf_vocab.rb

@@ -1,11 +1,11 @@
 # Copyright 2011-2018, The Trustees of Indiana University and Northwestern
 #   University.  Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
-# 
+#
 # You may obtain a copy of the License at
-# 
+#
 # http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software distributed
 #   under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 #   CONDITIONS OF ANY KIND, either express or implied. See the License for the
@@ -35,6 +35,7 @@ class Transcoding < RDF::StrictVocabulary("http://avalonmediasystem.org/rdf/voca
     class MasterFile < RDF::StrictVocabulary("http://avalonmediasystem.org/rdf/vocab/master_file#")
       property :posterOffset,        "rdfs:isDefinedBy" => %(avr-master_file:).freeze, type: "rdfs:Class".freeze
       property :thumbnailOffset,     "rdfs:isDefinedBy" => %(avr-master_file:).freeze, type: "rdfs:Class".freeze
+      property :workingFilePath,     "rdfs:isDefinedBy" => %(avr-master_file:).freeze, type: "rdfs:Class".freeze
     end
 
     class Derivative < RDF::StrictVocabulary("http://avalonmediasystem.org/rdf/vocab/derivative#")

app/models/batch_registries.rb

@@ -17,7 +17,7 @@
 class BatchRegistries < ActiveRecord::Base
   before_save :check_user
   validates :file_name, :user_id, :collection, presence: true
-  has_many :batch_entries, -> { order(position: :asc) }
+  has_many :batch_entries, -> { order(position: :asc) }, class_name: BatchEntries
 
   # For FactoryGirl's taps, document more TODO
   def file_name=(fn)

app/models/concerns/media_object_mods.rb

@@ -315,7 +315,7 @@ def terms_of_use
   end
   def terms_of_use=(value)
     delete_all_values(:terms_of_use)
-    descMetadata.add_terms_of_use(value) if value.present?
+    descMetadata.add_terms_of_use(Array(value).first) if value.present?
   end
 
   # has_attributes :table_of_contents, datastream: :descMetadata, at: [:table_of_contents], multiple: true

app/models/master_file.rb

@@ -120,6 +120,9 @@ class MasterFile < ActiveFedora::Base
     index.as :stored_sortable
   end
 
+  # For working file copy when Settings.matterhorn.media_path is set
+  property :working_file_path, predicate: Avalon::RDFVocab::MasterFile.workingFilePath, multiple: true
+
   validates :workflow_name, presence: true, inclusion: { in: proc { WORKFLOWS } }
   validates_each :date_digitized do |record, attr, value|
     unless value.nil?
@@ -167,8 +170,7 @@ def save_parent
   def setContent(file)
     case file
     when Hash #Multiple files for pre-transcoded derivatives
-      saveOriginal( (file.has_key?('quality-high') && File.file?( file['quality-high'] )) ? file['quality-high'] : (file.has_key?('quality-medium') && File.file?( file['quality-medium'] )) ? file['quality-medium'] : file.values[0] )
-      file.each_value {|f| f.close unless f.closed? }
+      saveDerivativesHash(file)
     when ActionDispatch::Http::UploadedFile #Web upload
       saveOriginal(file, file.original_filename)
     when URI, Addressable::URI
@@ -180,7 +182,7 @@ def setContent(file)
         self.file_size = FileLocator::S3File.new(file).object.size
       end
     else #Batch
-      saveOriginal(file)
+      saveOriginal(file, File.basename(file.path))
     end
     reloadTechnicalMetadata!
   end
@@ -249,14 +251,22 @@ def process file=nil
 
     #Build hash for single file skip transcoding
     if !file.is_a?(Hash) && (self.workflow_name == 'avalon-skip-transcoding' || self.workflow_name == 'avalon-skip-transcoding-audio')
-      file = {'quality-high' => FileLocator.new(file_location).attachment}
+      if working_file_path.present?
+        file = {'quality-high' => FileLocator.new(working_file_path.first).attachment}
+      else
+        file = {'quality-high' => FileLocator.new(file_location).attachment}
+      end
     end
 
     input = if file.is_a? Hash
       file_dup = file.dup
       file_dup.each_pair {|quality, f| file_dup[quality] = FileLocator.new(f.to_path).uri.to_s }
     else
-      FileLocator.new(file_location).uri.to_s
+      if working_file_path.present?
+        FileLocator.new(working_file_path.first).uri.to_s
+      else
+        FileLocator.new(file_location).uri.to_s
+      end
     end
 
     ActiveEncodeJob::Create.perform_later(self.id, input, {preset: self.workflow_name})
@@ -501,11 +511,23 @@ def to_solr *args
     end
   end
 
-  def working_file_path
-    path = nil
+  def create_working_file!(full_path)
+    working_path = MasterFile.calculate_working_file_path(full_path)
+    return unless working_path.present?
+
+    self.working_file_path = [working_path]
+    FileUtils.mkdir(File.dirname(working_path))
+    FileUtils.cp(full_path, working_path)
+    working_path
+  end
+
+  def self.calculate_working_file_path(old_path)
     config_path = Settings.matterhorn.media_path
-    path = File.join(config_path, File.basename(self.file_location)) if config_path.present? && File.directory?(config_path)
-    path
+    if config_path.present? && File.directory?(config_path)
+      File.join(config_path, SecureRandom.uuid, File.basename(old_path))
+    else
+      nil
+    end
   end
 
   protected
@@ -647,18 +669,30 @@ def saveOriginal(file, original_name=nil)
         path = File.join(File.dirname(realpath), original_name)
         File.rename(realpath, path)
         realpath = path
-        self.file_location = realpath
       end
 
-      newpath = working_file_path
-      FileUtils.cp(realpath, newpath) unless newpath.blank?
+      create_working_file!(realpath)
     end
     self.file_location = realpath
     self.file_size = file.size.to_s
   ensure
     file.close
   end
 
+  def saveDerivativesHash(derivative_hash)
+    usable_files = derivative_hash.select { |quality, file| File.file?(file) }
+    self.working_file_path = usable_files.values.collect { |file| create_working_file!(File.realpath(file)) }.compact
+
+    %w(quality-high quality-medium quality-low).each do |quality|
+      next unless usable_files.has_key?(quality)
+      self.file_location = File.realpath(usable_files[quality])
+      self.file_size = usable_files[quality].size.to_s
+      break
+    end
+  ensure
+    derivative_hash.values.map { |file| file.close }
+  end
+
   def reloadTechnicalMetadata!
     #Reset mediainfo
     @mediainfo = nil

app/views/batch_registries_mailer/batch_registration_finished_mailer.html.erb

@@ -47,8 +47,8 @@ Unless required by applicable law or agreed to in writing, software distributed
     </tr>
     <% @completed_items.each do |be_completed| %>
       <tr>
-        <% if MediaObject.exists? be_completed.media_object_pid %>
-          <% media_object = MediaObject.find(be_completed.media_object_pid) %>
+        <% media_object = MediaObject.where(id: be_completed.media_object_pid).first %>
+        <% if media_object %>
           <% link_url = media_object.permalink %>
           <% link_url = media_object_url(media_object) if link_url.blank? %>
           <td> <%= be_completed.position + 2 %> </td>

app/views/comments/index.html.erb

@@ -26,9 +26,14 @@ Unless required by applicable law or agreed to in writing, software distributed
       <%= comment.text_field :name, label: 'Name' %>
       <%= comment.hidden_field :nickname %>
       <%= comment.email_field :email, label: 'Email address' %>
-      <%= comment.email_field :email_confirmation, label: 'Confirm email address' %> 
-      <%= comment.select :subject, @subjects, prompt: 'Select one' %>    
+      <%= comment.email_field :email_confirmation, label: 'Confirm email address' %>
+      <%= comment.select :subject, @subjects, prompt: 'Select one' %>
       <%= comment.text_area :comment, rows: 10 %>
+      <% if Settings.recaptcha.present?%>
+        <div class="form-group">
+          <%= recaptcha_tags %>
+        </div>
+      <% end %>
       <%= comment.primary "Submit comments" %>
     </fieldset>
   <% end %>

app/views/layouts/avalon.html.erb

@@ -26,6 +26,7 @@ Unless required by applicable law or agreed to in writing, software distributed
     <%= stylesheet_link_tag "application", media: "all" %>
     <%= yield :page_styles %>
     <%= yield :additional_head_content %>
+    <%= render "modules/google_analytics" %>
   </head>
 
   <body data-mountpoint="<%=root_path%>">

app/views/layouts/embed.html.erb

@@ -30,6 +30,7 @@ Unless required by applicable law or agreed to in writing, software distributed
     <%= stylesheet_link_tag "application", media: "all" %>
     <%= yield :page_styles %>
     <%= yield :additional_head_content %>
+    <%= render "modules/google_analytics" %>
   </head>
 
   <body>

app/views/modules/_google_analytics.html.erb

@@ -0,0 +1,9 @@
+<% if Settings.google_analytics_tracking_id.present? %>
+  <script async src="https://www.googletagmanager.com/gtag/js?id=<%= Settings.google_analytics_tracking_id %>"></script>
+  <script>
+    window.dataLayer = window.dataLayer || [];
+    function gtag(){dataLayer.push(arguments);}
+    gtag('js', new Date());
+    gtag('config', '<%= Settings.google_analytics_tracking_id %>');
+  </script>
+<% end %>

config/application.rb

@@ -8,7 +8,7 @@
 Bundler.require(*Rails.groups)
 
 module Avalon
-  VERSION = '6.4.3'
+  VERSION = '6.4.4'
 
   class Application < Rails::Application
     require 'avalon/configuration'

config/initializers/recaptcha.rb

@@ -0,0 +1,8 @@
+
+# config/initializers/recaptcha.rb
+if Settings.recaptcha.present?
+  Recaptcha.configure do |config|
+    config.site_key = Settings.recaptcha.site_key
+    config.secret_key = Settings.recaptcha.secret_key
+  end
+end

config/settings.yml

@@ -82,3 +82,4 @@ auth:
         :oauth_credentials:
           <%= ENV['LTI_AUTH_KEY'] %>: <%= ENV['LTI_AUTH_SECRET'] %>
             <% end %>
+# google_analytics_tracking_id: "someid"

docker-compose.yml

@@ -25,6 +25,8 @@ services:
       - db
     volumes:
       - fedora:/data
+    environment:
+      - JAVA_OPTIONS=${JAVA_OPTIONS} -Dfcrepo.postgresql.password=${FEDORA_DB_PASSWORD} -Dfcrepo.postgresql.username=fedora -Dfcrepo.postgresql.host=db -Dfcrepo.modeshape.configuration=classpath:/config/jdbc-postgresql/repository.json
     ports:
       - "8984:8080"
   solr:

lib/avalon/batch/entry.rb

@@ -201,6 +201,15 @@ def process!
           files = self.class.gatherFiles(file_spec[:file])
           self.class.attach_datastreams_to_master_file(master_file, file_spec[:file])
           master_file.setContent(files)
+
+          # Overwrite files hash with working file paths to pass to matterhorn
+          if files.is_a?(Hash) && master_file.working_file_path.present?
+            files.each do |quality, file|
+              working_path = master_file.working_file_path.find { |path| File.basename(file) == File.basename(path) }
+              files[quality] = File.new(working_path)
+            end
+          end
+
           master_file.absolute_location = file_spec[:absolute_location] if file_spec[:absolute_location].present?
           master_file.title = file_spec[:label] if file_spec[:label].present?
           master_file.poster_offset = file_spec[:offset] if file_spec[:offset].present?

lib/avalon/intercom.rb

@@ -33,7 +33,7 @@ def collection_valid?(collection)
 
     def push_media_object(media_object, collection_id, include_structure = true)
       return { message: 'Avalon intercom target is not configured.' } unless @avalon.present?
-      return { message: 'You are not autorized to push to this collection.' } unless collection_valid? collection_id
+      return { message: 'You are not authorized to push to this collection.' } unless collection_valid? collection_id
       begin
         resp = RestClient::Request.execute(
           method: :post,