Merge branch 'develop'

CHANGELOG.md

@@ -1,3 +1,25 @@
+# Version 2.7.0
+*2019-02-13
+
+* Saving a record slightly faster, because ap square updates are done in a single update
+  statement rather than one per square size.
+* Saving records also slightly faster due to reduced number of update statements that are
+  run on the cache tables.
+* New tracking field to store an autogenerated sequential system unique update ID in
+  cache_occurrences_functional and cache_samples_functional. This makes it easier to
+  track any changes to the cache tables, e.g. for feeding changes through to other
+  systems such as Elasticsearch. It also makes change tracking of system generated
+  changes (such as spatial indexing) seperate to user instigated record changes so that
+  notifications are not unnecessarily generated for the former.
+* New standard reporting parameters `tracking_from` and `tracking_to` for filtering on
+  change tracking IDs.
+* REST API uses tracking data rather than updated_on field when using autofeed mode to
+  autogenerate a feed of updates.
+* Spatial indexing no longer changes cache table updated_on fields therefore does not
+  fire notifications.
+* Fixes a problem in the update kit with a missing class file
+  (https://github.com/Indicia-Team/warehouse/issues/315)
+
 # Version 2.6.0
 *2019-02-07*
 

application/config/version.php

@@ -29,7 +29,7 @@
  *
  * @var string
  */
-$config['version'] = '2.6.2';
+$config['version'] = '2.7.0';
 
 /**
  * Version release date.

application/helpers/postgreSQL.php

@@ -177,57 +177,81 @@ public static function insertMapSquaresForSamples($ids, $size, $db = NULL) {
    * submitted directly (i.e. not as part of a sample), to make sure that any
    * changed occurrences are linked to their map square entries properly.
    */
-  public static function insertMapSquaresForOccurrences($ids, $size, $db = NULL) {
-    self::insertMapSquares($ids, 'o', $size, $db);
+  public static function insertMapSquaresForOccurrences($ids, $db) {
+    self::insertMapSquares($ids, 'o', $db);
   }
 
   /**
    * Generic shared code for the insertMapSquaresFor... methods.
    */
-  private static function insertMapSquares($ids, $alias, $size, $db = NULL) {
+  private static function insertMapSquares($ids, $alias, $db) {
     if (count($ids) > 0) {
       static $srid;
       if (!isset($srid)) {
         $srid = kohana::config('sref_notations.internal_srid');
       }
-      if (!$db)
-        $db = new Database();
       $idlist = implode(',', $ids);
-      // Seems much faster to break this into small queries than one big left join.
-      $smpInfo = $db->query(
-      "SELECT DISTINCT s.id, o.website_id, s.survey_id, st_astext(coalesce(s.geom, l.centroid_geom)) as geom, o.confidential,
-          GREATEST(round(sqrt(st_area(st_transform(s.geom, sref_system_to_srid(entered_sref_system)))))::integer, o.sensitivity_precision, s.privacy_precision, $size) as size,
-          coalesce(s.entered_sref_system, l.centroid_sref_system) as entered_sref_system,
-          round(st_x(st_centroid(reduce_precision(
-            coalesce(s.geom, l.centroid_geom), o.confidential,
-            GREATEST(round(sqrt(st_area(st_transform(s.geom, sref_system_to_srid(entered_sref_system)))))::integer, o.sensitivity_precision, s.privacy_precision, $size),
-            s.entered_sref_system)
-          ))) as x,
-          round(st_y(st_centroid(reduce_precision(
-            coalesce(s.geom, l.centroid_geom), o.confidential,
-            GREATEST(round(sqrt(st_area(st_transform(s.geom, sref_system_to_srid(entered_sref_system)))))::integer, o.sensitivity_precision, s.privacy_precision, $size), s.entered_sref_system)
-          ))) as y
-        FROM samples s
-        JOIN occurrences o ON o.sample_id=s.id
-        LEFT JOIN locations l on l.id=s.location_id AND l.deleted=false
-        WHERE $alias.id IN ($idlist)")->result_array(TRUE);
-      $km = $size / 1000;
-      foreach ($smpInfo as $s) {
-        $existing = $db->query("SELECT id FROM map_squares WHERE x={$s->x} AND y={$s->y} AND size={$s->size}")
-          ->result_array(FALSE);
-        if (count($existing)===0) {
-          $qry = $db->query("INSERT INTO map_squares (geom, x, y, size)
-            VALUES (reduce_precision(st_geomfromtext('{$s->geom}', $srid), '{$s->confidential}', {$s->size}, '{$s->entered_sref_system}'), {$s->x}, {$s->y}, {$s->size})");
-          $msqId = $qry->insert_id();
+      $sizes = [1000, 2000, 10000];
+      $fieldsForEachSquare = '';
+      foreach ($sizes as $idx => $size) {
+        $fieldsForEachSquare .= <<<SQL
+  GREATEST(round(sqrt(st_area(st_transform(s.geom, sref_system_to_srid(entered_sref_system)))))::integer, o.sensitivity_precision, s.privacy_precision, $size) as size$size,
+  round(st_x(st_centroid(reduce_precision(
+    coalesce(s.geom, l.centroid_geom), o.confidential,
+    GREATEST(round(sqrt(st_area(st_transform(s.geom, sref_system_to_srid(entered_sref_system)))))::integer, o.sensitivity_precision, s.privacy_precision, $size),
+    s.entered_sref_system)
+  ))) as x$size,
+  round(st_y(st_centroid(reduce_precision(
+    coalesce(s.geom, l.centroid_geom), o.confidential,
+    GREATEST(round(sqrt(st_area(st_transform(s.geom, sref_system_to_srid(entered_sref_system)))))::integer, o.sensitivity_precision, s.privacy_precision, $size), s.entered_sref_system)
+  ))) as y$size
+SQL;
+        if ($idx < 2) {
+          $fieldsForEachSquare .= ",\n";
         }
-        else {
-          $msqId = $existing[0]['id'];
+      }
+      $query = <<<SQL
+SELECT DISTINCT s.id,
+  st_astext(coalesce(s.geom, l.centroid_geom)) as geom,
+  o.confidential,
+  coalesce(s.entered_sref_system, l.centroid_sref_system) as entered_sref_system,
+  $fieldsForEachSquare
+FROM samples s
+JOIN occurrences o ON o.sample_id=s.id
+LEFT JOIN locations l on l.id=s.location_id AND l.deleted=false
+WHERE $alias.id IN ($idlist)
+SQL;
+      $smpInfo = $db->query($query)->result_array(TRUE);
+      foreach ($smpInfo as $s) {
+        $updateFieldSQL = [];
+        $updateFilterSQL = [];
+        foreach ($sizes as $size) {
+          $km = $size / 1000;
+          $squareDetail = [
+            'x' => $s->{"x$size"},
+            'y' => $s->{"y$size"},
+            'size' => $s->{"size$size"},
+          ];
+          $existing = $db
+            ->query("SELECT id FROM map_squares WHERE x=$squareDetail[x] AND y=$squareDetail[y] AND size=$squareDetail[size]")
+            ->result_array(FALSE);
+          if (count($existing) === 0) {
+            $qry = $db->query("INSERT INTO map_squares (geom, x, y, size)
+              VALUES (reduce_precision(st_geomfromtext('{$s->geom}', $srid), '{$s->confidential}', $squareDetail[size], '{$s->entered_sref_system}'), $squareDetail[x], $squareDetail[y], $squareDetail[size])");
+            $msqId = $qry->insert_id();
+          }
+          else {
+            $msqId = $existing[0]['id'];
+          }
+          $updateFieldSQL[] = "map_sq_{$km}km_id=$msqId";
+          $updateFilterSQL[] = "map_sq_{$km}km_id IS NULL OR map_sq_{$km}km_id<>$msqId";
         }
-        $db->query("UPDATE cache_occurrences_functional SET map_sq_{$km}km_id=$msqId " .
-          "WHERE website_id={$s->website_id} AND survey_id={$s->survey_id} AND sample_id={$s->id} " .
-          "AND (map_sq_{$km}km_id IS NULL OR map_sq_{$km}km_id<>$msqId)");
-        $db->query("UPDATE cache_samples_functional SET map_sq_{$km}km_id=$msqId " .
-          "WHERE id={$s->id} AND (map_sq_{$km}km_id IS NULL OR map_sq_{$km}km_id<>$msqId)");
+        $db->query("UPDATE cache_occurrences_functional SET " . implode(', ', $updateFieldSQL) .
+          "WHERE sample_id={$s->id} " .
+          'AND (' . implode(' OR ', $updateFilterSQL) . ')');
+        $db->query("UPDATE cache_samples_functional SET " . implode(', ', $updateFieldSQL) .
+          "WHERE id={$s->id} " .
+          'AND (' . implode(' OR ', $updateFilterSQL) . ')');
       }
     }
   }

application/helpers/report_standard_params_occurrences.php

@@ -400,6 +400,36 @@ public static function getParameters() {
           ],
         ],
       ],
+      'tracking_from' => [
+        'datatype' => 'integer',
+        'display' => 'First squential update ID to include',
+        'description' => 'All record inserts and updates are given a sequential tracking ID. Filter by this to limit ' .
+          'the range of records returned to a contiguous batch of updates. Tracking is updated when the record is ' .
+          'affected in any way, not just when it is edited. E.g. an update to spatial indexing will update the tracking.',
+        'wheres' => [
+          [
+            'value' => '',
+            'operator' => '',
+            'sql' =>
+            "o.tracking >= #tracking_from#",
+          ],
+        ],
+      ],
+      'tracking_to' => [
+        'datatype' => 'integer',
+        'display' => 'Last squential update ID to include',
+        'description' => 'All record inserts and updates are given a sequential tracking ID. Filter by this to limit ' .
+          'the range of records returned to a contiguous batch of updates. Tracking is updated when the record is ' .
+          'affected in any way, not just when it is edited. E.g. an update to spatial indexing will update the tracking.',
+        'wheres' => [
+          [
+            'value' => '',
+            'operator' => '',
+            'sql' =>
+            "o.tracking <= #tracking_from#",
+          ],
+        ],
+      ],
       'quality' => [
         'datatype' => 'lookup',
         'display' => 'Quality',

application/helpers/report_standard_params_samples.php

@@ -228,6 +228,36 @@ public static function getParameters() {
           array('value'=>'', 'operator'=>'', 'sql'=>"s.verified_on>now()-'#verified_date_age#'::interval")
         )
       ),
+      'tracking_from' => [
+        'datatype' => 'integer',
+        'display' => 'First squential update ID to include',
+        'description' => 'All record inserts and updates are given a sequential tracking ID. Filter by this to limit ' .
+          'the range of records returned to a contiguous batch of updates. Tracking is updated when the record is ' .
+          'affected in any way, not just when it is edited. E.g. an update to spatial indexing will update the tracking.',
+        'wheres' => [
+          [
+            'value' => '',
+            'operator' => '',
+            'sql' =>
+            "s.tracking >= #tracking_from#",
+          ],
+        ],
+      ],
+      'tracking_to' => [
+        'datatype' => 'integer',
+        'display' => 'Last squential update ID to include',
+        'description' => 'All record inserts and updates are given a sequential tracking ID. Filter by this to limit ' .
+          'the range of records returned to a contiguous batch of updates. Tracking is updated when the record is ' .
+          'affected in any way, not just when it is edited. E.g. an update to spatial indexing will update the tracking.',
+        'wheres' => [
+          [
+            'value' => '',
+            'operator' => '',
+            'sql' =>
+            "s.tracking <= #tracking_from#",
+          ],
+        ],
+      ],
       'quality' => array('datatype'=>'lookup', 'display'=>'Quality',
         'description'=>'Minimum quality of records to include',
         'lookup_values'=>'V:Accepted records only,P:Not reviewed,!D:Exclude queried or not accepted records,' .

application/libraries/MY_ORM.php

@@ -751,16 +751,12 @@ private function postProcess() {
       }
       if (!empty($samples)) {
         // @todo Map squares could be added to work queue.
-        postgreSQL::insertMapSquaresForSamples($samples, 1000, $this->db);
-        postgreSQL::insertMapSquaresForSamples($samples, 2000, $this->db);
-        postgreSQL::insertMapSquaresForSamples($samples, 10000, $this->db);
+        postgreSQL::insertMapSquaresForSamples($samples, $this->db);
       }
       elseif (!empty($occurrences)) {
         // No need to do occurrence map square update if inserting a sample, as
         // the above code does the occurrences in bulk.
-        postgreSQL::insertMapSquaresForOccurrences($occurrences, 1000, $this->db);
-        postgreSQL::insertMapSquaresForOccurrences($occurrences, 2000, $this->db);
-        postgreSQL::insertMapSquaresForOccurrences($occurrences, 10000, $this->db);
+        postgreSQL::insertMapSquaresForOccurrences($occurrences, $this->db);
       }
     }
     if (!empty(self::$changedRecords['insert']['occurrence_association']) ||

application/models/occurrence_comment.php

@@ -66,17 +66,16 @@ public function caption()
   /**
    * Implement an instant update of the cache occurrences queried field, so the verification UI
    * can report on in as changes are made.
+   *
    * @param $isInsert
    * @return bool
    */
   public function postSubmit($isInsert) {
     if ($isInsert && $this->auto_generated!=='t' and $this->query==='t') {
-      $this->db->query("update cache_occurrences_functional set query='Q' where id={$this->occurrence_id}");
+      $this->db->query("update cache_occurrences_functional set query='Q' where id={$this->occurrence_id} and query<>'Q'");
     }
-    // answers don't need to be instant, just queries
-    return true;
+    // Answers don't need to be instant, just queries.
+    return TRUE;
   }
 
-
-
 }

modules/cache_builder/config/cache_builder.php

@@ -765,7 +765,8 @@
     when sc2.id is null and s.updated_on<=sc1.created_on then 'Q'
     else 'A'
   end,
-  parent_sample_id=s.parent_id
+  parent_sample_id=s.parent_id,
+  media_count=(SELECT COUNT(sm.*) FROM sample_media sm WHERE sm.sample_id=s.id AND sm.deleted=false)
 FROM samples s
 #join_needs_update#
 LEFT JOIN samples sp ON sp.id=s.parent_id AND  sp.deleted=false
@@ -779,22 +780,14 @@
 WHERE s.id=s_update.id
 ";
 
-$config['samples']['update']['functional_media'] = "
-UPDATE cache_samples_functional u
-SET media_count=(SELECT COUNT(sm.*)
-FROM sample_media sm WHERE sm.sample_id=u.id AND sm.deleted=false)
-FROM samples s
-#join_needs_update#
-WHERE s.id=u.id
-";
-
 $config['samples']['update']['functional_sensitive'] = "
-UPDATE cache_samples_functional
+UPDATE cache_samples_functional u
 SET location_id=null, location_name=null
 FROM samples s
 #join_needs_update#
 JOIN occurrences o ON o.sample_id=s.id AND o.deleted=false AND o.sensitivity_precision IS NOT NULL
-WHERE s.id=cache_samples_functional.id
+WHERE s.id=u.id
+AND (u.location_id IS NOT NULL OR u.location_name IS NOT NULL)
 ";
 
 $config['samples']['update']['nonfunctional'] = "
@@ -938,7 +931,7 @@
 INSERT INTO cache_samples_functional(
             id, website_id, survey_id, input_form, location_id, location_name,
             public_geom, date_start, date_end, date_type, created_on, updated_on, verified_on, created_by_id,
-            group_id, record_status, query, parent_sample_id)
+            group_id, record_status, query, parent_sample_id, media_count)
 SELECT distinct on (s.id) s.id, su.website_id, s.survey_id, COALESCE(sp.input_form, s.input_form), s.location_id,
   CASE WHEN s.privacy_precision IS NOT NULL THEN NULL ELSE COALESCE(l.name, s.location_name, lp.name, sp.location_name) END,
   reduce_precision(coalesce(s.geom, l.centroid_geom), false, s.privacy_precision,
@@ -950,7 +943,8 @@
     when sc2.id is null and s.updated_on<=sc1.created_on then 'Q'
     else 'A'
   end,
-  s.parent_id
+  s.parent_id,
+  (SELECT COUNT(sm.*) FROM sample_media sm WHERE sm.sample_id=s.id AND sm.deleted=false)
 FROM samples s
 #join_needs_update#
 LEFT JOIN cache_samples_functional cs on cs.id=s.id
@@ -966,8 +960,6 @@
 AND cs.id IS NULL
 ";
 
-$config['samples']['insert']['functional_media'] = $config['samples']['update']['functional_media'];
-
 $config['samples']['insert']['functional_sensitive'] = "
 UPDATE cache_samples_functional
 SET location_id=null, location_name=null
@@ -1395,7 +1387,8 @@
   external_key=o.external_key,
   taxon_path=ctp.path,
   parent_sample_id=s.parent_id,
-  verification_checks_enabled=w.verification_checks_enabled
+  verification_checks_enabled=w.verification_checks_enabled,
+  media_count=(SELECT COUNT(om.*) FROM occurrence_media om WHERE om.occurrence_id=o.id AND om.deleted=false)
 FROM occurrences o
 #join_needs_update#
 left join cache_occurrences_functional co on co.id=o.id
@@ -1422,15 +1415,6 @@
 WHERE cache_occurrences_functional.id=o.id
 ";
 
-$config['occurrences']['update']['functional_media'] = "
-UPDATE cache_occurrences_functional u
-SET media_count=(SELECT COUNT(om.*)
-FROM occurrence_media om WHERE om.occurrence_id=o.id AND om.deleted=false)
-FROM occurrences o
-#join_needs_update#
-WHERE o.id=u.id
-";
-
 $config['occurrences']['update']['functional_sensitive'] = "
 UPDATE cache_samples_functional cs
 SET location_id=null, location_name=null
@@ -1623,7 +1607,8 @@
             taxon_group_id, taxon_rank_sort_order, record_status, record_substatus,
             certainty, query, sensitive, release_status, marine_flag, data_cleaner_result,
             training, zero_abundance, licence_id, import_guid, confidential, external_key,
-            taxon_path, blocked_sharing_tasks, parent_sample_id, verification_checks_enabled)
+            taxon_path, blocked_sharing_tasks, parent_sample_id, verification_checks_enabled,
+            media_count)
 SELECT distinct on (o.id) o.id, o.sample_id, o.website_id, s.survey_id, COALESCE(sp.input_form, s.input_form), s.location_id,
     case when o.confidential=true or o.sensitivity_precision is not null or s.privacy_precision is not null
         then null else coalesce(l.name, s.location_name, lp.name, sp.location_name) end,
@@ -1663,7 +1648,8 @@
       ], NULL)
     END,
     s.parent_id,
-    w.verification_checks_enabled
+    w.verification_checks_enabled,
+    (SELECT COUNT(om.*) FROM occurrence_media om WHERE om.occurrence_id=o.id AND om.deleted=false)
 FROM occurrences o
 #join_needs_update#
 LEFT JOIN cache_occurrences_functional co on co.id=o.id
@@ -1691,8 +1677,6 @@
 AND co.id IS NULL
 ";
 
-$config['occurrences']['insert']['functional_media'] = $config['occurrences']['update']['functional_media'];
-
 $config['occurrences']['insert']['functional_sensitive'] = "
 UPDATE cache_samples_functional cs
 SET location_id=null, location_name=null