Merge branch 'develop'

Indicia-Team · Apr 16, 2020 · b95b9c8 · b95b9c8
2 parents 6d96908 + 5df997b
commit b95b9c8
Show file tree

Hide file tree

Showing 25 changed files with 206 additions and 105 deletions.
diff --git a/application/config/version.php b/application/config/version.php
@@ -29,14 +29,14 @@
  *
  * @var string
  */
-$config['version'] = '3.2.3';
+$config['version'] = '3.3.0';
 
 /**
  * Version release date.
  *
  * @var string
  */
-$config['release_date'] = '2020-04-06';
+$config['release_date'] = '2020-04-16';
 
 /**
  * Link to the code repository downloads page.

diff --git a/application/controllers/scheduled_tasks.php b/application/controllers/scheduled_tasks.php
@@ -394,7 +394,7 @@ private function sendEmail($notificationIds, $swift, $userId, $emailContent, $cc
         ->update();
       $email_config = Kohana::config('email');
       $userResults = $this->db
-        ->select('people.email_address')
+        ->select('people.email_address, people.first_name, people.surname')
         ->from('people')
         ->join('users', 'users.person_id', 'people.id')
         ->where('users.id', $userId)
@@ -409,18 +409,19 @@ private function sendEmail($notificationIds, $swift, $userId, $emailContent, $cc
           kohana::lang('misc.notification_subject') : kohana::config('email.notification_subject');
         $message = new Swift_Message(
           sprintf($subject, kohana::config('email.server_name')),
-          $emailContent,
+          "<html>$emailContent</html>",
           'text/html'
         );
         $recipients = new Swift_RecipientList();
-        $recipients->addTo($user->email_address);
+        $name = empty($user->first_name) ? "$user->surname" : "$user->first_name $user->surname";
+        $recipients->addTo($user->email_address, $name);
         $cc = explode(',', $cc);
         foreach ($cc as $ccEmail) {
           $recipients->addCc(trim($ccEmail));
         }
         // Send the email.
-        $swift->send($message, $recipients, $email_config['address']);
-        kohana::log('info', 'Email notification sent to ' . $user->email_address);
+        $sent = $swift->send($message, $recipients, $email_config['address']);
+        kohana::log('info', "$sent email notification(s) sent to $user->email_address");
       }
     }
     catch (Exception $e) {
@@ -597,8 +598,11 @@ private function doRecordOwnerNotifications($swift) {
       $this->addArrayToEmailTable($email->occurrence_id, $attrArray, $emailContent);
       $emailContent .= "</table>";
 
-      $message = new Swift_Message(kohana::lang('misc.notification_subject', kohana::config('email.server_name')), $emailContent,
-                                     'text/html');
+      $message = new Swift_Message(
+        kohana::lang('misc.notification_subject', kohana::config('email.server_name')),
+        "<html>$emailContent</html>",
+        'text/html'
+      );
       $recipients = new Swift_RecipientList();
       $recipients->addTo($email->email_address);
       // Send the email.

diff --git a/application/models/sample.php b/application/models/sample.php
@@ -93,14 +93,6 @@ class Sample_Model extends ORM_Tree {
         ['fieldName' => 'sample:external_key'],
       ],
     ],
-    [
-      'description' => 'Sample External Key',
-      'fields' => [
-        ['fieldName' => 'survey_id', 'notInMappings' => TRUE],
-        ['fieldName' => 'sample:sample_method_id'],
-        ['fieldName' => 'sample:external_key'],
-      ],
-    ],
     [
       'description' => 'Grid Ref and Date',
       'fields' => [

diff --git a/modules/cache_builder/config/cache_builder.php b/modules/cache_builder/config/cache_builder.php
@@ -285,7 +285,16 @@
     INTO TEMPORARY ttl_path
     FROM q
     GROUP BY child_pref_ttl_id
-    ORDER BY child_pref_ttl_id;",
+    ORDER BY child_pref_ttl_id;
+
+    SELECT DISTINCT ON (cttl.external_key) cttl.external_key, cttlall.id, tp.path
+    INTO TEMPORARY master_list_paths
+    FROM ttl_path tp
+    JOIN cache_taxa_taxon_lists cttl ON cttl.id=tp.child_pref_ttl_id
+    JOIN cache_taxa_taxon_lists cttlall ON cttlall.external_key=cttl.external_key
+    WHERE cttl.taxon_list_id=COALESCE(#master_list_id#, cttlall.taxon_list_id)
+    AND cttlall.preferred=true
+    ORDER BY cttl.external_key, cttl.allow_data_entry DESC;",
   'Taxon paths' => "
     UPDATE cache_taxon_paths ctp
     SET path=tp.path, external_key=t.external_key
@@ -313,38 +322,39 @@
   'Ranks' => "
     UPDATE cache_taxa_taxon_lists u
     SET family_taxa_taxon_list_id=cttlf.id, family_taxon=cttlf.taxon,
-      order_taxa_taxon_list_id=cttlo.id, order_taxon=cttlo.taxon,
-      kingdom_taxa_taxon_list_id=cttlk.id, kingdom_taxon=cttlk.taxon
-    FROM cache_taxa_taxon_lists cttl
-    -- Ensure only changed taxon concepts are updated
-    JOIN descendants nu ON nu.id=cttl.preferred_taxa_taxon_list_id
-    JOIN cache_taxon_paths ctp ON ctp.external_key=cttl.external_key AND ctp.taxon_list_id=#master_list_id#
-    LEFT JOIN cache_taxa_taxon_lists cttlf ON cttlf.taxon_meaning_id=ANY(ctp.path) and cttlf.taxon_rank='Family' and cttlf.taxon_list_id=#master_list_id# AND cttlf.preferred=true
-    LEFT JOIN cache_taxa_taxon_lists cttlo ON cttlo.taxon_meaning_id=ANY(ctp.path) and cttlo.taxon_rank='Order' and cttlo.taxon_list_id=#master_list_id# AND cttlo.preferred=true
-    LEFT JOIN cache_taxa_taxon_lists cttlk ON cttlk.taxon_meaning_id=ANY(ctp.path) and cttlk.taxon_rank='Kingdom' and cttlk.taxon_list_id=#master_list_id# AND cttlk.preferred=true
-    WHERE cttl.taxon_meaning_id=u.taxon_meaning_id
+        order_taxa_taxon_list_id=cttlo.id, order_taxon=cttlo.taxon,
+        kingdom_taxa_taxon_list_id=cttlk.id, kingdom_taxon=cttlk.taxon
+    FROM master_list_paths mlp
+    JOIN descendants nu ON nu.id=mlp.id
+    LEFT JOIN cache_taxa_taxon_lists cttlf ON cttlf.taxon_meaning_id=ANY(mlp.path) and cttlf.taxon_rank='Family'
+      AND cttlf.taxon_list_id=#master_list_id# AND cttlf.preferred=true AND cttlf.allow_data_entry=true
+    LEFT JOIN cache_taxa_taxon_lists cttlo ON cttlo.taxon_meaning_id=ANY(mlp.path) and cttlo.taxon_rank='Order'
+      AND cttlo.taxon_list_id=#master_list_id# AND cttlo.preferred=true AND cttlo.allow_data_entry=true
+    LEFT JOIN cache_taxa_taxon_lists cttlk ON cttlk.taxon_meaning_id=ANY(mlp.path) and cttlk.taxon_rank='Kingdom'
+      AND cttlk.taxon_list_id=#master_list_id# AND cttlk.preferred=true AND cttlk.allow_data_entry=true
+    WHERE mlp.external_key=u.external_key
     AND (COALESCE(u.family_taxa_taxon_list_id, 0)<>COALESCE(cttlf.id, 0)
-      OR COALESCE(u.family_taxon, '')<>COALESCE(cttlf.taxon, '')
-      OR COALESCE(u.order_taxa_taxon_list_id, 0)<>COALESCE(cttlo.id, 0)
-      OR COALESCE(u.order_taxon, '')<>COALESCE(cttlo.taxon, '')
-      OR COALESCE(u.kingdom_taxa_taxon_list_id, 0)<>COALESCE(cttlk.id, 0)
-      OR COALESCE(u.kingdom_taxon, '')<>COALESCE(cttlk.taxon, '')
+        OR COALESCE(u.family_taxon, '')<>COALESCE(cttlf.taxon, '')
+        OR COALESCE(u.order_taxa_taxon_list_id, 0)<>COALESCE(cttlo.id, 0)
+        OR COALESCE(u.order_taxon, '')<>COALESCE(cttlo.taxon, '')
+        OR COALESCE(u.kingdom_taxa_taxon_list_id, 0)<>COALESCE(cttlk.id, 0)
+        OR COALESCE(u.kingdom_taxon, '')<>COALESCE(cttlk.taxon, '')
     );
 
     UPDATE cache_occurrences_functional u
-    SET family_taxa_taxon_list_id=cttlf.id,
-      taxon_path=ctp.path
+    SET family_taxa_taxon_list_id=cttl.family_taxa_taxon_list_id,
+      taxon_path=mlp.path
     FROM cache_taxa_taxon_lists cttl
     -- Ensure only changed taxon concepts are updated
     JOIN descendants nu ON nu.id=cttl.preferred_taxa_taxon_list_id
-    JOIN cache_taxon_paths ctp ON ctp.external_key=cttl.external_key AND ctp.taxon_list_id=COALESCE(#master_list_id#, cttl.taxon_list_id)
-    LEFT JOIN cache_taxa_taxon_lists cttlf ON ctp.path @> ARRAY[cttlf.taxon_meaning_id] and cttlf.taxon_rank='Family' and cttlf.taxon_list_id=#master_list_id# AND cttlf.preferred=true
-    WHERE cttl.taxon_meaning_id=u.taxon_meaning_id
-    AND (COALESCE(u.family_taxa_taxon_list_id, 0)<>COALESCE(cttlf.id, 0)
-    OR COALESCE(u.taxon_path, ARRAY[]::integer[])<>COALESCE(ctp.path, ARRAY[]::integer[]));",
+    JOIN master_list_paths mlp ON mlp.external_key=cttl.external_key
+    WHERE cttl.id=u.taxa_taxon_list_id
+    AND (COALESCE(u.family_taxa_taxon_list_id, 0)<>COALESCE(cttl.family_taxa_taxon_list_id, 0)
+    OR COALESCE(u.taxon_path, ARRAY[]::integer[])<>COALESCE(mlp.path, ARRAY[]::integer[]));",
   "teardown" => "
     DROP TABLE descendants;
-    DROP TABLE ttl_path;",
+    DROP TABLE ttl_path;
+    DROP TABLE master_list_paths;",
 );
 
 // --------------------------------------------------------------------------------------------------------------------------

diff --git a/modules/cache_builder/db/version_0_1_2/201504071219_cache_taxon_searchterms_external_key.sql b/modules/cache_builder/db/version_0_1_2/201504071219_cache_taxon_searchterms_external_key.sql
@@ -1,13 +1,4 @@
--- #slow script#
--- Took 27 minutes on testwarehouse.
-
-ALTER TABLE cache_taxon_searchterms
+ALTER TABLE cache_taxon_searchterms
    ADD COLUMN external_key character varying;
 COMMENT ON COLUMN cache_taxon_searchterms.external_key
-  IS 'External identifier for the taxon.';
-
-UPDATE cache_taxon_searchterms cts
-SET external_key = cttl.external_key
-FROM cache_taxa_taxon_lists cttl
-WHERE cttl.id=cts.taxa_taxon_list_id
-AND cttl.external_key IS NOT NULL;
+  IS 'External identifier for the taxon.';
diff --git a/...cache_builder/db/version_0_1_2/201504071220_cache_taxon_searchterms_external_key_data.sql b/...cache_builder/db/version_0_1_2/201504071220_cache_taxon_searchterms_external_key_data.sql
@@ -0,0 +1,7 @@
+-- #slow script#
+
+UPDATE cache_taxon_searchterms cts
+SET external_key = cttl.external_key
+FROM cache_taxa_taxon_lists cttl
+WHERE cttl.id=cts.taxa_taxon_list_id
+AND cttl.external_key IS NOT NULL;
diff --git a/modules/cache_builder/db/version_0_1_2/201504211044_cache_occurrences_record_substatus.sql b/modules/cache_builder/db/version_0_1_2/201504211044_cache_occurrences_record_substatus.sql
@@ -1,4 +1,3 @@
--- #slow script#
 -- move into a cache_builder script
 ALTER TABLE cache_occurrences
    ADD COLUMN record_substatus smallint,

diff --git a/modules/cache_builder/db/version_0_1_2/201512131953_occurrences_family_filter.sql b/modules/cache_builder/db/version_0_1_2/201512131953_occurrences_family_filter.sql
@@ -1,9 +1,3 @@
--- #slow script#
 -- Adds family_taxa_taxon_list_id to cache_occurrences since it makes a big improvement to filtering
 ALTER TABLE cache_occurrences
-  ADD family_taxa_taxon_list_id integer NULL;
-
-UPDATE cache_occurrences o
-SET family_taxa_taxon_list_id=cttl.family_taxa_taxon_list_id
-FROM cache_taxa_taxon_lists cttl
-WHERE cttl.id=o.taxa_taxon_list_id;
+  ADD family_taxa_taxon_list_id integer NULL;
diff --git a/modules/cache_builder/db/version_0_1_2/201512131954_occurrences_family_filter_data.sql b/modules/cache_builder/db/version_0_1_2/201512131954_occurrences_family_filter_data.sql
@@ -0,0 +1,6 @@
+-- #slow script#
+
+UPDATE cache_occurrences o
+SET family_taxa_taxon_list_id=cttl.family_taxa_taxon_list_id
+FROM cache_taxa_taxon_lists cttl
+WHERE cttl.id=o.taxa_taxon_list_id;
diff --git a/modules/cache_builder/db/version_1_0_0/201512300956_cache_occurrences_view.sql b/modules/cache_builder/db/version_1_0_0/201512300956_cache_occurrences_view.sql
@@ -1,4 +1,3 @@
--- #slow script#
 DROP TABLE cache_occurrences;
 
 -- Create a view to ease the migration path to the new cache occurrences structure.

diff --git a/modules/cache_builder/db/version_1_28_0/201703171025_confidential.sql b/modules/cache_builder/db/version_1_28_0/201703171025_confidential.sql
@@ -1,10 +1,2 @@
--- #slow script#
-
 ALTER TABLE cache_occurrences_functional
-  ADD COLUMN confidential BOOLEAN DEFAULT FALSE;
-
-UPDATE cache_occurrences_functional co
-  SET confidential=true
-FROM occurrences o
-WHERE o.id=co.id
-AND o.confidential=true;
+  ADD COLUMN confidential BOOLEAN DEFAULT FALSE;
diff --git a/modules/cache_builder/db/version_1_28_0/201703171026_confidential_data.sql b/modules/cache_builder/db/version_1_28_0/201703171026_confidential_data.sql
@@ -0,0 +1,7 @@
+-- #slow script#
+
+UPDATE cache_occurrences_functional co
+  SET confidential=true
+FROM occurrences o
+WHERE o.id=co.id
+AND o.confidential=true;
diff --git a/...201401010909_searchterm_id_difficulty.sql → ...201401010909_searchterm_id_difficulty.sql b/...201401010909_searchterm_id_difficulty.sql → ...201401010909_searchterm_id_difficulty.sql
diff --git a/...121607_cache_occurrences_id_diff_data.sql → ...121607_cache_occurrences_id_diff_data.sql b/...121607_cache_occurrences_id_diff_data.sql → ...121607_cache_occurrences_id_diff_data.sql
diff --git a/modules/indicia_setup/db/version_0_9_1/201504210945_occurrence_record_status_updates.sql b/modules/indicia_setup/db/version_0_9_1/201504210945_occurrence_record_status_updates.sql
@@ -1,8 +1,8 @@
--- #slow script#
 ALTER TABLE occurrences
-   ADD COLUMN record_substatus smallint CONSTRAINT occurrences_record_substatus_check CHECK (record_substatus BETWEEN 1 AND 5),  
+   ADD COLUMN record_substatus smallint CONSTRAINT occurrences_record_substatus_check CHECK (record_substatus BETWEEN 1 AND 5),
    ADD COLUMN record_decision_source character CONSTRAINT record_decision_source_check CHECK (record_decision_source IN ('H', 'M'));
 
+COMMENT ON COLUMN occurrences.record_status IS 'Status of this record. I - in progress, C - completed, V - verified, R - rejected, D - dubious/queried (deprecated), T - test.';
 COMMENT ON COLUMN occurrences.record_substatus IS 'Provides additional detail on the record status. Values are: 1=accepted as correct, 2=accepted as considered correct, 3=plausible, 4=not accepted as unable to verify, 5=not accepted, incorrect. Null for unchecked records.';
 COMMENT ON COLUMN occurrences.record_decision_source IS 'Defines if the record status decision was by a human (H) or machine (M).';
 
@@ -11,26 +11,8 @@ ALTER TABLE occurrence_comments
    -- Add columns to log occurrence status changes with the comments
    ADD COLUMN record_status character(1) CONSTRAINT occurrence_comments_record_status_check CHECK (record_status = ANY (ARRAY['I'::bpchar, 'C'::bpchar, 'V'::bpchar, 'R'::bpchar, 'T'::bpchar, 'D'::bpchar])),
    ADD COLUMN record_substatus smallint CONSTRAINT occurrence_comments_record_substatus_check CHECK (record_substatus BETWEEN 1 AND 5);
-UPDATE occurrence_comments SET query=FALSE;
 ALTER TABLE occurrence_comments ALTER query SET default false;
 
 COMMENT ON COLUMN occurrence_comments.query IS 'Set to true if this comment asks a question that needs a response.';
 COMMENT ON COLUMN occurrence_comments.record_status IS 'If this comment relates to the changing of the status of a record, then determines the status it was changed to. Provides and audit trail of verification changes.';
-COMMENT ON COLUMN occurrence_comments.record_substatus IS 'As record_status but provides an audit trail of the occurrences.record_substatus field';
-
-UPDATE occurrence_comments oc 
-SET query=true 
-FROM occurrences o 
-WHERE (oc.comment LIKE 'I emailed this record%' OR oc.comment LIKE 'Query%')
-AND oc.occurrence_id=o.id
-AND o.record_status IN ('S', 'C', 'D');
-
--- Sent status is no longer valid. We track queries via comments instead.
-UPDATE occurrences SET record_status='C' where record_status='S';
-
-ALTER TABLE occurrences DROP CONSTRAINT occurrences_record_status_check;
-
-ALTER TABLE occurrences
-  ADD CONSTRAINT occurrences_record_status_check CHECK (record_status = ANY (ARRAY['I'::bpchar, 'C'::bpchar, 'V'::bpchar, 'R'::bpchar, 'T'::bpchar, 'D'::bpchar]));
-
-COMMENT ON COLUMN occurrences.record_status IS 'Status of this record. I - in progress, C - completed, V - verified, R - rejected, D - dubious/queried (deprecated), T - test.';
+COMMENT ON COLUMN occurrence_comments.record_substatus IS 'As record_status but provides an audit trail of the occurrences.record_substatus field';
diff --git a/...les/indicia_setup/db/version_0_9_1/201504210946_occurrence_record_status_updates_data.sql b/...les/indicia_setup/db/version_0_9_1/201504210946_occurrence_record_status_updates_data.sql
@@ -0,0 +1,17 @@
+-- #slow script#
+UPDATE occurrence_comments SET query=FALSE;
+
+UPDATE occurrence_comments oc
+SET query=true
+FROM occurrences o
+WHERE (oc.comment LIKE 'I emailed this record%' OR oc.comment LIKE 'Query%')
+AND oc.occurrence_id=o.id
+AND o.record_status IN ('S', 'C', 'D');
+
+-- Sent status is no longer valid. We track queries via comments instead.
+UPDATE occurrences SET record_status='C' where record_status='S';
+
+ALTER TABLE occurrences DROP CONSTRAINT occurrences_record_status_check;
+
+ALTER TABLE occurrences
+  ADD CONSTRAINT occurrences_record_status_check CHECK (record_status = ANY (ARRAY['I'::bpchar, 'C'::bpchar, 'V'::bpchar, 'R'::bpchar, 'T'::bpchar, 'D'::bpchar]));
diff --git a/modules/indicia_setup/db/version_3_3_0/202001210925_sample_delete.sql b/modules/indicia_setup/db/version_3_3_0/202001210925_sample_delete.sql
@@ -0,0 +1,27 @@
+CREATE OR REPLACE FUNCTION cascade_sample_delete() RETURNS TRIGGER AS $$
+  BEGIN
+    IF (OLD.deleted = false AND NEW.deleted = true) THEN
+      UPDATE occurrences
+      SET deleted = true, updated_on=now(), updated_by_id=new.updated_by_id
+      WHERE sample_id  = OLD.id;
+
+      UPDATE sample_attribute_values
+      SET deleted = true, updated_on=now(), updated_by_id=new.updated_by_id
+      WHERE sample_id  = OLD.id;
+
+      UPDATE sample_comments
+      SET deleted = true, updated_on=now(), updated_by_id=new.updated_by_id
+      WHERE sample_id  = OLD.id;
+
+      UPDATE sample_media
+      SET deleted = true, updated_on=now(), updated_by_id=new.updated_by_id
+      WHERE sample_id  = OLD.id;
+
+      UPDATE samples
+      SET deleted = true, updated_on=now(), updated_by_id=new.updated_by_id
+      WHERE parent_id = OLD.id;
+
+    END IF;
+  RETURN OLD;
+END;
+$$ LANGUAGE 'plpgsql';
diff --git a/modules/indicia_setup/db/version_3_3_0/20200224_es_sensitivity_index.sql b/modules/indicia_setup/db/version_3_3_0/20200224_es_sensitivity_index.sql
@@ -0,0 +1,3 @@
+-- Improves performance when extracting sensitive data to move to Elasticsearch.
+CREATE INDEX IF NOT EXISTS ix_cache_occ_functional_id_tracking_sens
+ON cache_occurrences_functional(id, tracking) WHERE sensitive=true;
diff --git a/modules/indicia_svc_data/controllers/data_service_base.php b/modules/indicia_svc_data/controllers/data_service_base.php
@@ -72,6 +72,9 @@ protected function handle_request() {
       if ($mode !== 'dwca') {
         throw $e;
       }
+      elseif (!$this->failedRequestDetail) {
+        $this->failedRequestDetail = $e->getMessage();
+      }
     }
     switch ($mode) {
       case 'json':