Merge branch 'develop'

CHANGELOG.md

@@ -1,3 +1,20 @@
+# Version 4.10.0 
+*2020-12-17*
+
+* New uksi_operations module allows update logs for UKSI to be directly imported, with support for
+  new taxon, amend metadata, promote name, merge taxa, rename taxon operations.
+* Addition of organism_key to taxon table for improved links with UKSI.
+* Generation of taxon path data now more reliable (report filtering by higher taxa).
+* New work queue task for efficient updates of taxonomy related fields in occurrence cache tables.
+* Server status message on home page fixed immediately after install.
+* Minor tweaks to improve the install process.
+* Support for dynamic addition of termlist terms when inserting attribute values (tag style, e.g.
+  using the data_entry_helper::sublist control).
+* Fix to check constraint for unique email addresses. Excludes deleted records.
+* Fixes a problem where importing to update existing records caused duplicates.
+* Elasticsearch downloads now support automatic inclusion of custom attributes for a survey.
+* Fix for importing anonymous records into Recorder 6 using Indicia2Recorder.
+
 # Version 4.5.0
 *2020-09-16*
 

application/config/version.php

@@ -29,14 +29,14 @@
  *
  * @var string
  */
-$config['version'] = '4.9.1';
+$config['version'] = '4.10.0';
 
 /**
  * Version release date.
  *
  * @var string
  */
-$config['release_date'] = '2020-12-14';
+$config['release_date'] = '2020-12-17';
 
 /**
  * Link to the code repository downloads page.

application/helpers/serverStatus.php

@@ -86,6 +86,7 @@ private static function checkScheduledTasksHasBeenSetup($db, array &$messages) {
       ->select('count(*)')
       ->from('system')
       ->where('last_scheduled_task_check is not null')
+      ->where("name<>'indicia'")
       ->get()->current();
     if ($query->count === '0') {
       $description = <<<DESC

application/libraries/MY_ORM.php

@@ -1676,7 +1676,9 @@ protected function createAttributes($isInsert) {
                 && !empty($this->submission['fields']["$field:upper"]['value'])) {
               $value .= ' - ' . $this->submission['fields']["$field:upper"]['value'];
             }
-            $attr = $this->createAttributeRecord($attrId, $valueId, $value, $attrDef);
+            $allowTermCreationLang = empty($this->submission['fields']["$this->attrs_field_prefix:$attrId:allowTermCreationLang"])
+              ? NULL : $this->submission['fields']["$this->attrs_field_prefix:$attrId:allowTermCreationLang"]['value'];
+            $attr = $this->createAttributeRecord($attrId, $valueId, $value, $attrDef, $allowTermCreationLang);
             if ($attr === FALSE) {
               // Failed to create attribute so drop out.
               return FALSE;
@@ -1750,7 +1752,7 @@ protected function createAttributesFromMetafields() {
    * @return mixed
    *   FALSE on fail, else array of attribute value IDs objects.
    */
-  protected function createAttributeRecord($attrId, $valueId, $value, $attrDef) {
+  protected function createAttributeRecord($attrId, $valueId, $value, $attrDef, $allowTermCreationLang = NULL) {
     // There are particular circumstances when $value is actually an array: when a attribute is multi value,
     // AND has yet to be created, AND is passed in as multiple ***Attr:<n>[] POST variables. This should only happen when
     // the attribute has yet to be created, as after this point the $valueID is filled in and that specific attribute POST variable
@@ -1761,7 +1763,7 @@ protected function createAttributeRecord($attrId, $valueId, $value, $attrDef) {
       if (is_null($valueId)) {
         $r = [];
         foreach($value as $singlevalue) { // recurse over array.
-          $attrIds = $this->createAttributeRecord($attrId, $valueId, $singlevalue, $attrDef);
+          $attrIds = $this->createAttributeRecord($attrId, $valueId, $singlevalue, $attrDef, $allowTermCreationLang);
           if ($attrIds === FALSE) {
             return FALSE;
           }
@@ -1868,22 +1870,37 @@ protected function createAttributeRecord($attrId, $valueId, $value, $attrDef) {
       case 'L':
         // Lookup list
         $vf = 'int_value';
-        if (!empty($value) && $fk) {
-          // value must be looked up
+        if (!empty($value)) {
+          $creatingTerm = $allowTermCreationLang && substr($value, 0, 11) === 'createTerm:';
+          if ($creatingTerm) {
+            // Chop off prefix.
+            $value = substr($value, 11);
+          }
+          // Find existing value.
           $r = $this->fkLookup(array(
             'fkTable' => 'lookup_term',
             'fkSearchField' => 'term',
             'fkSearchValue' => $value,
             'fkSearchFilterField' => 'termlist_id',
             'fkSearchFilterValue' => $attrDef->termlist_id,
           ));
-          if ($r) {
+          if (($fk || $creatingTerm) && $r) {
+            // Term lookup succeeded and we are submitting fk_field, or a
+            // normal field that allows term creation. In the latter case
+            // we use the lookup to avoid duplication.
             $value = $r;
-          } else {
+          }
+          elseif ($fk) {
             $this->errors[$fieldId] = "Invalid value $value for attribute ".$attrDef->caption;
             kohana::log('debug', "Could not accept value $value into field $vf  for attribute $fieldId.");
             return FALSE;
           }
+          elseif ($creatingTerm) {
+            $escapedTerm = pg_escape_string($value);
+            $value = $this->db
+              ->query("select insert_term('$escapedTerm', '$allowTermCreationLang', null, $attrDef->termlist_id, null);")
+              ->insert_id();
+          }
         }
         break;
       default:

application/models/taxon.php

@@ -47,6 +47,7 @@ public function validate(Validation $array, $save = FALSE) {
       'freshwater_flag',
       'terrestrial_flag',
       'non_native_flag',
+      'organism_key',
     );
     return parent::validate($array, $save);
   }

application/views/attribute_by_survey/attribute_by_survey_edit.php

@@ -399,7 +399,7 @@
       else {
         echo <<<HTML
 <div class="alert alert-warning">
-Set the taxon_list_id configuration in application/config/indicia.php to enable linking attributes to taxa.
+Set the master_list_id configuration in application/config/indicia.php to enable linking attributes to taxa.
 </div>
 HTML;
       }

application/views/attribute_by_taxon_list/attribute_by_taxon_list_edit.php

@@ -348,7 +348,7 @@
     else {
       echo <<<HTML
 <div class="alert alert-warning">
-  Set the taxon_list_id configuration in application/config/indicia.php to enable linking attributes to taxa.
+  Set the master_list_id configuration in application/config/indicia.php to enable linking attributes to taxa.
 </div>
 HTML;
     }

application/views/taxa_taxon_list/taxa_taxon_list_edit.php

@@ -116,7 +116,7 @@
       'helpText' => 'Description which applies only to this taxon within the context of this list.',
     ]);
     $helpText = <<<TXT
-Unique identifier for this taxon concept as defined by an external source. For example in the UK this field is
+Unique identifier for the accepted name for this taxon as defined by an external source. For example in the UK this field is
 typically used to store an NBN Taxon Version Key for the accepted name.
 TXT;
     echo data_entry_helper::text_input([
@@ -137,6 +137,16 @@
       'default' => html::initial_value($values, 'taxon:search_code'),
       'helpText' => $helpText,
     ]);
+    $helpText = <<<TXT
+Unique identifier for this taxon concept as defined by an external source. When linking to UKSI, this
+field is used to store the Organism Key.
+TXT;
+    echo data_entry_helper::text_input([
+      'fieldname' => 'taxon:organism_key',
+      'label' => 'Organism unique identifier (organism key)',
+      'default' => html::initial_value($values, 'taxon:organism_key'),
+      'helpText' => $helpText,
+    ]);
     echo data_entry_helper::text_input([
       'fieldname' => 'taxon_meaning:id',
       'label' => 'Taxon meaning ID',

modules/cache_builder/config/cache_builder.php

@@ -260,6 +260,7 @@
 
 $config['taxa_taxon_lists']['extra_multi_record_updates'] = array(
   'setup' => "
+    -- Find children of updated taxa to ensure they are also changed.
     WITH RECURSIVE q AS (
       SELECT ttl.id
       FROM taxa_taxon_lists ttl
@@ -274,26 +275,36 @@
     INTO TEMPORARY descendants FROM q;
 
     WITH RECURSIVE q AS (
-      SELECT distinct ttlpref.id AS child_pref_ttl_id, ttlpref.parent_id,
-      ttlpref.taxon_meaning_id AS rank_taxon_meaning_id, ttlpref.taxon_list_id, 0 as distance
+      SELECT distinct ttlpref.id AS child_pref_ttl_id, ttlpref.allow_data_entry as child_pref_allow_data_entry,
+      ttlpref.taxon_meaning_id as child_pref_taxon_meaning_id, ttlpref.taxon_list_id as child_pref_taxon_list_id,
+      ttlpref.parent_id, ttlpref.taxon_meaning_id AS rank_taxon_meaning_id, ttlpref.taxon_list_id, 0 as distance
       FROM taxa_taxon_lists ttlpref
       JOIN taxa t ON t.id=ttlpref.taxon_id and t.deleted=false
       JOIN descendants d ON d.id=ttlpref.id
       WHERE ttlpref.preferred=true
       AND ttlpref.deleted=false
       UNION ALL
-      SELECT q.child_pref_ttl_id, ttl.parent_id,
-          ttl.taxon_meaning_id AS rank_taxon_meaning_id, ttl.taxon_list_id, q.distance+1
+      SELECT q.child_pref_ttl_id, q.child_pref_allow_data_entry, q.child_pref_taxon_meaning_id, q.child_pref_taxon_list_id,
+          ttl.parent_id, ttl.taxon_meaning_id AS rank_taxon_meaning_id, ttl.taxon_list_id, q.distance+1
       FROM q
       JOIN taxa_taxon_lists ttl ON ttl.id=q.parent_id and ttl.deleted=false and ttl.taxon_list_id=q.taxon_list_id
       JOIN taxa t ON t.id=ttl.taxon_id and t.deleted=false
     )
-    SELECT child_pref_ttl_id, array_agg(rank_taxon_meaning_id order by distance desc) as path
+    SELECT child_pref_ttl_id, child_pref_allow_data_entry, child_pref_taxon_meaning_id, child_pref_taxon_list_id,
+	    array_agg(rank_taxon_meaning_id order by distance desc) as path
     INTO TEMPORARY ttl_path
     FROM q
-    GROUP BY child_pref_ttl_id
+    GROUP BY child_pref_ttl_id, child_pref_allow_data_entry, child_pref_taxon_meaning_id, child_pref_taxon_list_id
     ORDER BY child_pref_ttl_id;
 
+    -- Remove any for redundant taxa where path covered by a non-redundant taxa.
+    DELETE FROM ttl_path t1
+    USING ttl_path t2
+    WHERE t1.child_pref_allow_data_entry=false
+    AND t2.child_pref_allow_data_entry=true
+    AND t2.child_pref_taxon_meaning_id=t1.child_pref_taxon_meaning_id
+    AND t2.child_pref_taxon_list_id=t1.child_pref_taxon_list_id;
+
     SELECT DISTINCT ON (cttl.external_key) cttl.external_key, cttlall.id, tp.path
     INTO TEMPORARY master_list_paths
     FROM ttl_path tp

modules/cache_builder/helpers/task_cache_builder_taxonomy_occurrence.php

@@ -0,0 +1,86 @@
+<?php
+
+/**
+ * @file
+ * Queue worker to update cache_occurrences_functional.taxon_path.
+ *
+ * Indicia, the OPAL Online Recording Toolkit.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * any later version.
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/gpl.html.
+ *
+ * @author Indicia Team
+ * @license http://www.gnu.org/licenses/gpl.html GPL
+ * @link https://github.com/indicia-team/warehouse
+ */
+
+ defined('SYSPATH') or die('No direct script access.');
+
+/**
+ * Queue worker to update cache_occurrences_functional taxonomy related fields.
+ *
+ * E.g. for a rename taxon UKSI operation, occurrences get a new external key, 
+ * so need to update cache_occurrences_functional.
+ */
+class task_cache_builder_taxonomy_occurrence {
+
+  /**
+   * Not a fast operation.
+   */
+  const BATCH_SIZE = 10;
+
+  /**
+   * Perform the processing for a task batch found in the queue.
+   * 
+   * Note that this task updates occurrences, but the stored record in the 
+   * work_queue task should be a taxa_taxon_list. Otherwise huge numbers of
+   * work queue tasks are required.
+   *
+   * @param object $db
+   *   Database connection object.
+   * @param object $taskType
+   *   Object read from the database for the task batch. Contains the task
+   *   name, entity, priority, created_on of the first record in the batch
+   *   count (total number of queued tasks of this type).
+   * @param string $procId
+   *   Unique identifier of this work queue processing run. Allows filtering
+   *   against the work_queue table's claimed_by field to determine which
+   *   tasks to perform.
+   */
+  public static function process($db, $taskType, $procId) {
+    $masterListId = warehouse::getMasterTaxonListId();
+    $sql = <<<SQL
+UPDATE cache_occurrences_functional o
+SET taxon_path=ctp.path,
+  preferred_taxa_taxon_list_id=cttl.preferred_taxa_taxon_list_id,
+  taxa_taxon_list_external_key=cttl.external_key,
+  taxon_meaning_id=cttl.taxon_meaning_id,
+  family_taxa_taxon_list_id=cttl.family_taxa_taxon_list_id,
+  taxon_group_id=cttl.taxon_group_id,
+  taxon_rank_sort_order=cttl.taxon_rank_sort_order,
+  marine_flag=cttl.marine_flag,
+  freshwater_flag=cttl.freshwater_flag,
+  terrestrial_flag=cttl.terrestrial_flag,
+  non_native_flag=cttl.non_native_flag
+FROM work_queue q, cache_taxa_taxon_lists cttl
+LEFT JOIN cache_taxon_paths ctp
+  ON ctp.taxon_meaning_id=cttl.taxon_meaning_id
+  AND ctp.taxon_list_id=$masterListId
+WHERE cttl.id=o.taxa_taxon_list_id
+AND cttl.taxon_list_id=$masterListId
+AND o.taxa_taxon_list_id=q.record_id
+AND q.entity='taxa_taxon_list'
+AND q.task='task_cache_builder_taxonomy_occurrence'
+AND q.claimed_by='$procId';
+SQL;
+    $db->query($sql);
+  }
+}

modules/indicia_setup/db/version_4_10_0/202011091217_base36_fn.sql

@@ -0,0 +1,24 @@
+CREATE OR REPLACE FUNCTION base36_encode(IN digits bigint, IN min_width int = 0) RETURNS varchar AS $$
+DECLARE
+    chars char[];
+    ret varchar;
+    val bigint;
+BEGIN
+    chars := ARRAY['0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z'];
+    val := digits;
+    ret := '';
+    IF val < 0 THEN
+        val := val * -1;
+    END IF;
+    WHILE val != 0 LOOP
+        ret := chars[(val % 36)+1] || ret;
+        val := val / 36;
+    END LOOP;
+
+    IF min_width > 0 AND char_length(ret) < min_width THEN
+        ret := lpad(ret, min_width, '0');
+    END IF;
+
+    RETURN ret;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;

modules/indicia_setup/db/version_4_10_0/202011101004_reduce_precision.sql

@@ -0,0 +1,51 @@
+CREATE OR REPLACE FUNCTION reduce_precision(
+    geom_in geometry,
+    confidential boolean,
+    reduce_to_precision integer)
+  RETURNS geometry AS
+$BODY$
+DECLARE geom geometry;
+DECLARE geomltln geometry;
+DECLARE r geometry;
+DECLARE precisionM integer;
+DECLARE x float;
+DECLARE y float;
+DECLARE srid integer;
+DECLARE sref_metadata record;
+BEGIN
+  IF confidential = true OR COALESCE(reduce_to_precision, 0)>1 THEN
+    precisionM = CASE
+      WHEN COALESCE(reduce_to_precision, 0)>1 THEN reduce_to_precision
+      ELSE 1000
+    END;
+    srid = get_output_srid(geom_in);
+    IF srid<>900913 THEN
+      geom = st_transform(geom_in, srid);
+    ELSE
+      geom = geom_in;
+    END IF;
+    -- If lower precision than requested, then can return as it is.
+    -- If same precision allowing for rounding errors, then re-calculate to ensure
+    -- all square geoms are consistent for aggregation purposes.
+    IF floor(sqrt(st_area(geom))) > precisionM THEN
+      r = geom_in;
+    ELSE
+      geom = st_centroid(geom);
+      -- need to reduce this to a square on the grid
+      x = floor(st_xmin(geom)::NUMERIC / precisionM) * precisionM;
+      y = floor(st_ymin(geom)::NUMERIC / precisionM) * precisionM;
+      r = st_geomfromtext('polygon((' || x::varchar || ' ' || y::varchar || ',' || (x + precisionM)::varchar || ' ' || y::varchar || ','
+          || (x + precisionM)::varchar || ' ' || (y + precisionM)::varchar || ',' || x::varchar || ' ' || (y + precisionM)::varchar || ','
+          || x::varchar || ' ' || y::varchar || '))', srid);
+      IF srid<>900913 THEN
+        r = st_transform(r, 900913);
+      END IF;
+    END IF;
+  ELSE
+    r = geom_in;
+  END IF;
+RETURN r;
+END;
+$BODY$
+  LANGUAGE plpgsql IMMUTABLE
+  COST 100;

modules/indicia_setup/db/version_4_10_0/202012041304_unique_email_idx.sql

@@ -0,0 +1,3 @@
+DROP INDEX ix_unique_email_address;
+
+CREATE UNIQUE INDEX ix_unique_email_address ON people (email_address)  WHERE deleted='f' AND email_address !='' AND email_address is not null;

modules/indicia_setup/db/version_4_10_0/202012070927_organism_key.sql

@@ -0,0 +1,5 @@
+ALTER TABLE taxa
+  ADD COLUMN organism_key varchar;
+
+COMMENT ON COLUMN taxa.organism_key
+    IS 'Identifier for the organism concept, e.g. when linking to UKSI.';